From phk at FreeBSD.org  Mon Mar  3 09:04:17 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 03 Mar 2014 10:04:17 +0100
Subject: [master] 51ab3f2 Collapse two fetch states and make the X-Varnish
 heading for the backend available in vcl_backend_fetch{}
Message-ID: <E1WKOnJ-0007AL-HL@project.varnish-software.com>


commit 51ab3f207c2b6a3c3ac1c3411be232730801db39
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 3 09:03:51 2014 +0000

    Collapse two fetch states and make the X-Varnish heading for the backend
    available in vcl_backend_fetch{}

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0c621df..c52ba05 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -265,6 +265,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 {
+	int i, do_ims;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -277,12 +278,12 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
 	HTTP_Copy(bo->bereq, bo->bereq0);
 
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
+
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
 	bo->uncacheable = bo->do_pass;
-
-	http_PrintfHeader(bo->bereq,
-	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
 	if (wrk->handling == VCL_RET_ABANDON) {
 		if (bo->req != NULL)
 			vbf_release_req(bo);
@@ -291,22 +292,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_DONE);
 	}
 	assert (wrk->handling == VCL_RET_FETCH);
-	return (F_STP_FETCHHDR);
-}
-
-
-/*--------------------------------------------------------------------
- */
-
-static enum fetch_step
-vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
-{
-	int i, do_ims;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	xxxassert (wrk->handling == VCL_RET_FETCH);
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 9ef7f6c..47da3fc 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -52,7 +52,6 @@ REQ_STEP(error,		ERROR,		(wrk, req))
 FETCH_STEP(mkbereq,	MKBEREQ,	(wrk, bo))
 FETCH_STEP(retry,	RETRY,		(wrk, bo))
 FETCH_STEP(startfetch,	STARTFETCH,	(wrk, bo))
-FETCH_STEP(fetchhdr,	FETCHHDR,	(wrk, bo))
 FETCH_STEP(condfetch,	CONDFETCH,	(wrk, bo))
 FETCH_STEP(fetch,	FETCH,		(wrk, bo))
 FETCH_STEP(error,	ERROR,		(wrk, bo))



From phk at FreeBSD.org  Mon Mar  3 09:31:21 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 03 Mar 2014 10:31:21 +0100
Subject: [master] 2edbe43 Always release the req early if non-pass fetch,
 don't release it if it is pass, we may beed the body for retries.
Message-ID: <E1WKPDV-0002Bs-GO@project.varnish-software.com>


commit 2edbe43feda671dd432152e82e4cc376e1b79cb4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 3 09:30:47 2014 +0000

    Always release the req early if non-pass fetch, don't release it if
    it is pass, we may beed the body for retries.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c52ba05..8672375 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -35,26 +35,10 @@
 #include <stdlib.h>
 
 #include "cache.h"
-
 #include "hash/hash_slinger.h"
-
 #include "vcl.h"
 #include "vtim.h"
 
-
-/*--------------------------------------------------------------------
- */
-
-static void
-vbf_release_req(struct busyobj *bo)
-{
-	assert(bo->state == BOS_INVALID);
-	AN(bo->req);
-	bo->req = NULL;
-	http_CopyHome(bo->bereq);
-	VBO_setstate(bo, BOS_REQ_DONE);
-}
-
 /*--------------------------------------------------------------------
  * Allocate an object, with fall-back to Transient.
  * XXX: This somewhat overlaps the stuff in stevedore.c
@@ -187,7 +171,7 @@ vbf_beresp2obj(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
- * Copy req->bereq
+ * Copy req->bereq and release req if not pass fetch
  */
 
 static enum fetch_step
@@ -199,6 +183,7 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
 
+	assert(bo->state == BOS_INVALID);
 	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
@@ -207,18 +192,16 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod);
 	http_FilterReq(bo->bereq0, bo->req->http,
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
+
 	if (!bo->do_pass) {
-		// XXX: Forcing GET should happen in vcl_miss{} ?
 		http_ForceGet(bo->bereq0);
-		if (cache_param->http_gzip_support) {
-			/*
-			 * We always ask the backend for gzip, even if the
-			 * client doesn't grok it.  We will uncompress for
-			 * the minority of clients which don't.
-			 */
+		if (cache_param->http_gzip_support)
 			http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
-		}
+		AN(bo->req);
+		bo->req = NULL;
+		http_CopyHome(bo->bereq0);
 	}
+
 	if (bo->ims_obj != NULL) {
 		if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
 			http_PrintfHeader(bo->bereq0,
@@ -231,6 +214,7 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 		}
 	}
 
+	VBO_setstate(bo, BOS_REQ_DONE);
 	return (F_STP_STARTFETCH);
 }
 
@@ -275,6 +259,11 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
 
+	if (bo->do_pass)
+		AN(bo->req);
+	else
+		AZ(bo->req);
+
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
 	HTTP_Copy(bo->bereq, bo->bereq0);
 
@@ -285,8 +274,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	bo->uncacheable = bo->do_pass;
 	if (wrk->handling == VCL_RET_ABANDON) {
-		if (bo->req != NULL)
-			vbf_release_req(bo);
 		HSH_Fail(bo->fetch_objcore);
 		VBO_setstate(bo, BOS_FAILED);
 		return (F_STP_DONE);
@@ -295,9 +282,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 
-	if (!bo->do_pass && bo->req != NULL)
-		vbf_release_req(bo); /* XXX: retry ?? */
-
 	assert(bo->state <= BOS_REQ_DONE);
 
 	i = V1F_fetch_hdr(wrk, bo, bo->req);
@@ -316,11 +300,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_ERROR);
 	}
 
-	if (bo->do_pass && bo->req != NULL)
-		vbf_release_req(bo); /* XXX : retry ?? */
-
-	AZ(bo->req);
-
 	AN(bo->vbc);
 	http_VSL_log(bo->beresp);
 
@@ -690,9 +669,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	xxxassert(wrk->handling == VCL_RET_DELIVER);
 
-	if (bo->req != NULL)
-		vbf_release_req(bo);
-
 	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
 	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
 



From martin at varnish-software.com  Mon Mar  3 11:52:09 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 03 Mar 2014 12:52:09 +0100
Subject: [master] 2f05fe4 Serve ESI-includes from a different backend.
Message-ID: <E1WKRPl-0001Vj-SU@project.varnish-software.com>


commit 2f05fe44f072373bd11da06b01586110412fc15c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 3 12:51:22 2014 +0100

    Serve ESI-includes from a different backend.
    
    Serve ESI-includes from a different backend so that we don't race
    against when the backend connection is ready for reuse.
    
    Fixes: #1440

diff --git a/bin/varnishtest/tests/e00019.vtc b/bin/varnishtest/tests/e00019.vtc
index 1aeaa06..1010eab 100644
--- a/bin/varnishtest/tests/e00019.vtc
+++ b/bin/varnishtest/tests/e00019.vtc
@@ -22,13 +22,25 @@ server s1 {
 	chunked {<esi:comment/>}
 
 	chunkedlen 0
+} -start
 
+# The included object gets served from a different backend.
+# This is to avoid a race between when a backend connection
+# gets put up for reuse because of background fetches in
+# Varnish 4
+server s2 {
 	rxreq
 	expect req.url == "bar/foo"
 	txresp -body {<INCL>}
 } -start
 
 varnish v1 -vcl+backend {
+	sub vcl_backend_fetch {
+		if (bereq.url != "bar") {
+			set bereq.backend = s2;
+		}
+	}
+
 	sub vcl_backend_response {
 		if (bereq.url == "bar") {
 			set beresp.do_esi = true;



From perbu at varnish-software.com  Mon Mar  3 12:33:38 2014
From: perbu at varnish-software.com (Per Buer)
Date: Mon, 03 Mar 2014 13:33:38 +0100
Subject: [master] f10e0a5 Regenerated from inline help. This should be
 automatic at some point. Also, the inline help might be a bit more verbose.
Message-ID: <E1WKS3u-0002Iw-QP@project.varnish-software.com>


commit f10e0a58125c8a7251e868bf4a37b2b1ca2fbd50
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Mar 3 10:43:18 2014 +0100

    Regenerated from inline help. This should be automatic at some point.
    Also, the inline help might be a bit more verbose.

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index f967465..2e17c2e 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -71,112 +71,74 @@ be entered with the \\xnn syntax.
 Commands
 --------
 
-backend.list
-      Lists the defined backends including health state.
-
-backend.set_health matcher state
-      Sets the health state on a specific backend, overriding the state
-      determined by a probe.  This is useful if you want to take a
-      certain backend out of circulation.
-
-      *state* can be 'auto', 'sick' or 'healthy'.
-
-ban   *field operator argument* [&& field operator argument [...]]
-      Immediately invalidate all documents matching the ban
-      expression.  See *Ban Expressions* for more documentation and
-      examples.
-
-ban.list
-      All requests for objects from the cache are matched against
-      items on the ban list.  If an object in the cache is older than
-      a matching ban list item, it is considered "banned", and will be
-      fetched from the backend instead.
-
-      When a ban expression is older than all the objects in the
-      cache, it is removed from the list.
+help [command]
+  Show command/protocol help
 
-      ban.list displays the ban list. The output looks something like
-      this::
+ping [timestamp]
+  Keep connection alive
 
-        0x7fea4fcb0580 1303835108.618863   131G   req.url ~ /some/url
+auth response
+  Authenticate.
 
-      The first field is the address of the ban.
+quit
+  Close connection
 
-      The second is the time of entry into the list, given
-      as a high precision timestamp.
+banner
+  Print welcome banner.
 
-      The third field describes many objects point to this ban. When
-      an object is compared to a ban the object is marked with a
-      reference to the newest ban it was tested against. This isn't
-      really useful unless you're debugging.
+status
+  Check status of Varnish cache process.
 
-      A "G" marks that the ban is "Gone". Meaning it has been marked
-      as a duplicate or it is no longer valid. It stays in the list
-      for effiency reasons.
+start
+  Start the Varnish cache process.
 
-      Then follows the actual ban it self.
+stop
+  Stop the Varnish cache process
 
-help [command]
-      Display a list of available commands.
-      If the command is specified, display help for this command.
+vcl.load <configname> <filename>
+  Compile and load the VCL file under the name provided.
 
-param.set param value
-      Set the parameter specified by param to the specified value.
-      See Run-Time Parameters for a list of parameters.
+vcl.inline <configname> <quoted_VCLstring>
+  Compile and load the VCL data under the name provided.
 
-param.show [-l] [param]
-      Display a list if run-time parameters and their values.
+vcl.use <configname>
+  Switch to the named configuration immediately.
 
-      If the -l option is specified, the list includes a brief
-      explanation of each parameter.
+vcl.discard <configname>
+  Unload the named configuration (when possible).
 
-      If a param is specified, display only the value and explanation
-      for this parameter.
+vcl.list
+  List all loaded configuration.
 
-ping  [timestamp]
-      Ping the Varnish cache process, keeping the connection alive.
+vcl.show <configname>
+  Display the source code for the specified configuration.
 
-quit
-      Close the connection to the Varnish admin port.
+param.show [-l] [<param>]
+  Show parameters and their values.
 
-start
-      Start the Varnish cache process if it is not already running.
+param.set <param> <value>
+  Set parameter value.
 
-status
-      Check the status of the Varnish cache process.
+panic.show
+  Return the last panic, if any.
 
-stop
-      Stop the Varnish cache process.
+panic.clear
+  Clear the last panic, if any.
 
 storage.list
-      Lists the defined storage backends.
-
-vcl.discard configname
-      Discard the configuration specified by configname.  This will
-      have no effect if the specified configuration has a non-zero
-      reference count.
+  List storage devices
 
-vcl.inline configname vcl
-      Create a new configuration named configname with the VCL code
-      specified by vcl, which must be a quoted string.
-
-vcl.list
-      List available configurations and their respective reference
-      counts.  The active configuration is indicated with an asterisk
-      ("*").
-
-vcl.load configname filename
-      Create a new configuration named configname with the contents of
-      the specified file.
+backend.list
+  List all backends
 
-vcl.show configname
-      Display the source code for the specified configuration.
+backend.set_health matcher state
+  Set health status on a backend
 
-vcl.use configname
-      Start using the configuration specified by configname for all
-      new requests.  Existing requests will continue using whichever
-      configuration was in use when they arrived.
+ban <field> <operator> <arg> [&& <field> <oper> <arg>]...
+  All objects where the all the conditions match will be marked obsolete.
 
+ban.list
+  List the active bans.
 
 
 Ban Expressions



From perbu at varnish-software.com  Mon Mar  3 12:33:38 2014
From: perbu at varnish-software.com (Per Buer)
Date: Mon, 03 Mar 2014 13:33:38 +0100
Subject: [master] 805fa87 First stab at a mostly syntax-oriented VCL manpage.
 Feedback is welcome.
Message-ID: <E1WKS3u-0002J3-S1@project.varnish-software.com>


commit 805fa87a555972f724a3672439f72220c17f5429
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Mar 3 13:28:49 2014 +0100

    First stab at a mostly syntax-oriented VCL manpage.
    Feedback is welcome.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 57072d0..96a4aab 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -19,273 +19,232 @@ When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
 is then dynamically linked into the server process.
 
-For an overview of the VCL syntax please see the users guide.
+This document focuses on the syntax of the VCL language. Full a full
+description of syntax and semantics, with ample examples, please see
+the users guide at https://www.varnish-cache.org/doc/
+
+VCL consists of the following elements:
+ * Operators
+ * Conditionals
+ * Strings, booleans, time, duration, ints
+ * Regular expressions
+
+In addition VCL has the following contructs:
+ * Include
+ * Backend definitions
+ * Probes
+ * Access control lists - ACLs
+ * Import statement
+ * Functions
+ * Subroutines
+
+Note that are no loops or iterators of any kind in VCL.
+
+Operators
+---------
 
+The following operators are available in VCL:
 
-Backend declarations
---------------------
+  =
+    Assignment operator.
 
-A backend declaration creates and initializes a named backend object::
+  ==
+    Comparison.
 
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-    }
+  ~
+    Match. Can either be used with regular expressions or ACLs.
 
-The backend object can later be used to select a backend at request time::
+  !
+    Negation.
 
-    if (req.http.host ~ "(?i)^(www.)?example.com$") {
-        set req.backend_hint = www;
-    }
+  &&
+    Logical and
 
-To avoid overloading backend servers, .max_connections can be set to
-limit the maximum number of concurrent backend connections.
+  ||
+    Logical or
 
-The timeout parameters can be overridden in the backend declaration.
-The timeout parameters are .connect_timeout for the time to wait for a
-backend connection, .first_byte_timeout for the time to wait for the
-first byte from the backend and .between_bytes_timeout for time to
-wait between each received byte.
 
-These can be set in the declaration like this::
+Conditionals
+------------
 
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-        .connect_timeout = 1s;
-        .first_byte_timeout = 5s;
-        .between_bytes_timeout = 2s;
-    }
+VCL has *if* statments.
 
-.. _ref-vcl-director:
 
-Directors
----------
+Strings, booleans, time, duration and ints
+------------------------------------------
 
-A director is a logical group of backend servers clustered together
-for redundancy. The basic role of the director is to let Varnish
-choose a backend server amongst several so if one is down another can
-be used.
+These are the data types in Varnish. You can *set* or *unset* these.
 
-There are several types of directors. The different director types
-use different algorithms to choose which backend to use.
+Example::
 
-Configuring a director may look like this::
+  set req.http.user-agent = "unknown";
 
-    director b2 random {
-        .retries = 5;
-        {
-            # We can refer to named backends
-            .backend = b1;
-            .weight = 7;
-        }
-        {
-            # Or define them inline
-            .backend  = {
-                .host = "fs2";
-            }
-            .weight = 3;
-        }
-    }
 
-The family of random directors
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Strings
+~~~~~~~
 
-There are three directors that share the same logic, called the random
-director, client director and hash director. They each distribute traffic
-among the backends assigned to it using a random distribution seeded with
-either the client identity, a random number or the cache hash (typically
-url). Beyond the initial seed, they act the same.
+Basic strings are enclosed in " ... ", and may not contain
+newlines. Long strings are enclosed in {" ... "}. They may contain any
+character including ", newline and other control characters except for
+the NUL (0x00) character
 
-Each backend requires a .weight option which sets the amount of traffic
-each backend will get compared to the others. Equal weight means equal
-traffic. A backend with lower weight than an other will get proportionally
-less traffic.
+Booleans
+~~~~~~~~
 
-The director has an optional .retries option which defaults to the number
-of backends the director has. The director will attempt .retries times to
-find a healthy backend if the first attempt fails. Each attempt re-uses the
-previous seed in an iterative manner. For the random director this detail
-is of no importance as it will give different results each time. For the
-hash and client director, this means the same URL or the same client will
-fail to the same server consistently.
+Booleans can be either true or false.
 
-The random director
-...................
+Time
+----
 
-This uses a random number to seed the backend selection.
+VCL has time. The function *now* returns a time. A duration can be
+added to a time to make another time. In string context they return a
+formatted string.
 
-The client director
-...................
+Durations
+---------
 
-The client director picks a backend based on the clients
-*identity*. You can set the VCL variable *client.identity* to identify
-the client by picking up the value of a session cookie or similar.
+Durations are defined by a number and a designation. The number can be a real so 1.5w is allowed.
 
-The hash director
-.................
+  s
+    seconds
 
-The hash director will pick a backend based on the URL hash
-value.
+  m
+    minutes
 
-This is useful is you are using Varnish to load balance in front of
-other Varnish caches or other web accelerators as objects won't be
-duplicated across caches.
+  h
+    hours
 
-It will use the value of req.hash, just as the normal cache lookup methods.
+  d
+    days
 
+  w
+    weeks
 
-The round-robin director
-~~~~~~~~~~~~~~~~~~~~~~~~
+  y
+    years.
 
-The round-robin director does not take any options.
 
-It will use the first backend for the first request, the second backend for
-the second request and so on, and start from the top again when it gets to
-the end.
+Ints
+----
 
-If a backend is unhealthy or Varnish fails to connect, it will be skipped.
-The round-robin director will try all the backends once before giving up.
+Certain fields are integers, used as expected. In string context they
+return a string.
 
-The DNS director
-~~~~~~~~~~~~~~~~
 
-The DNS director can use backends in two different ways. Either like the
-random or round-robin director or using .list::
+Regular Expressions
+-------------------
 
-  director directorname dns {
-          .list = {
-                  .host_header = "www.example.com";
-                  .port = "80";
-                  .connect_timeout = 0.4s;
-                  "192.168.15.0"/24;
-                  "192.168.16.128"/25;
-          }
-          .ttl = 5m;
-          .suffix = "internal.example.net";
-  }
+Varnish uses PCRE - Perl-compatible regular expressions. For a
+complete description of PCRE please see the pcre(3) man page.
 
-This will specify 384 backends, all using port 80 and a connection timeout
-of 0.4s. Options must come before the list of IPs in the .list statement.
-The .list-method does not support IPv6. It is not a white-list, it is an
-actual list of backends that will be created internally in Varnish - the
-larger subnet the more overhead.
+To send flags to the PCRE engine, such as to turn on *case insensitivity* 
+add the flag within parens following a question mark,
+like this::
 
-The .ttl defines the cache duration of the DNS lookups.
+    # If host is NOT example dot com..
+    if (req.http.host !~ "(?i)example.com$") {
+        ...
+    }
 
-The above example will append "internal.example.net" to the incoming Host
-header supplied by the client, before looking it up. All settings are
-optional.
 
-Health checks are not thoroughly supported.
+Include statement
+-----------------
 
-DNS round robin balancing is supported. If a hostname resolves to multiple
-backends, the director will divide the traffic between all of them in a
-round-robin manner.
+To include a VCL file in another file use the include keyword::
 
-The fallback director
-~~~~~~~~~~~~~~~~~~~~~
+  include "foo.vcl";
 
-The fallback director will pick the first backend that is healthy. It
-considers them in the order in which they are listed in its definition.
 
-The fallback director does not take any options.
+Backend definition
+------------------
 
-An example of a fallback director::
+A backend declaration creates and initializes a named backend
+object. A declaration start with the keyword *backend* followed by the
+name of the backend. The actual declaration is in curly brackets, in a
+key/value fashion.::
 
-    director b3 fallback {
-        { .backend = www1; }
-        { .backend = www2; } # will only be used if www1 is unhealthy.
-        { .backend = www3; } # will only be used if both www1 and www2
-                             # are unhealthy.
+    backend name {
+        .attribute = "value";
     }
 
-Backend probes
---------------
-
-Backends can be probed to see whether they should be considered
-healthy or not.  The return status can also be checked by using the
-healthy function from the std vmod.
-
-Probes take the following parameters:
-
-.url
-  Specify a URL to request from the backend.
-  Defaults to "/".
-.request
-  Specify a full HTTP request using multiple strings. .request will
-  have \\r\\n automatically inserted after every string.
-  If specified, .request will take precedence over .url.
-.window
-  How many of the latest polls we examine to determine backend health.
-  Defaults to 8.
-.threshold
-  How many of the polls in .window must have succeeded for us to
-  consider the backend healthy. If this is set to more than or equal
-  to the threshold, the backend starts as healthy. Defaults to the
-  value of threshold - 1. In this case, the backend starts as sick and
-  requires one poll to pass to become healthy.
-  Defaults to threshold - 1.
-.initial
-  How many of the polls in .window are considered good when Varnish
-  starts. Defaults to the value of threshold - 1. In this case, the
-  backend starts as sick and requires one single poll to be considered
-  healthy.
-.expected_response
-  The expected backend HTTP response code.
-  Defaults to 200.
-.interval
-  Defines how often the probe should check the backend.
-  Default is every 5 seconds.
-.timeout
-  How fast each probe times out.
-  Default is 2 seconds.
-
-A backend with a probe can be defined like this, together with the
-backend or director::
-
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-        .probe = {
-            .url = "/test.jpg";
-            .timeout = 0.3s;
-            .window = 8;
-            .threshold = 3;
-            .initial = 3;
-        }
-    }
+The only mandatory attribute is host. The attributes will inherit
+their defaults from the global parameters. The following attributes
+are availble:
 
-Or it can be defined separately and then referenced::
+  host (mandatory)
+    The host to be used. IP address or a hostname that resolves to a
+    single IP address.
 
-    probe healthcheck {
-        .url = "/status.cgi";
-        .interval = 60s;
-        .timeout = 0.3s;
-        .window = 8;
-        .threshold = 3;
-        .initial = 3;
-        .expected_response = 200;
-    }
+  port
+    The port on the backend that Varnish should connect to.
 
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-        .probe = healthcheck;
-    }
+  host_header
+    A host header to add.
+
+  connect_timeout
+    Timeout for connections.
+
+  first_byte_timeout
+    Timeout for first byte.
+
+  between_bytes_timeout
+    Timeout between bytes.
 
-If you have many backends this can simplify the config a lot.
+  probe
+    Attach a probe to the backend. See Probes.
 
-It is also possible to specify the raw HTTP request::
+  max_connections
+    Maximum number of open connections towards this backend. If
+    Varnish reaches the maximum Varnish it will start failing
+    connections.
 
-    probe rawprobe {
-        # NB: \r\n automatically inserted after each string!
-        .request =
-          "GET / HTTP/1.1"
-          "Host: www.foo.bar"
-          "Connection: close";
+Probes
+------
+
+Probes will query the backend for status on a regular basis and mark
+the backend as down it they fail. A probe is defined as this:::
+
+    probe name {
+         .attribute = "value";
     }
 
+There are no mandatory options. These are the options you can set:
+
+  url
+    The URL to query. Defaults to "/".
+
+  request
+    Specify a full HTTP request using multiple strings. .request will
+    have \r\n automatically inserted after every string. If specified,
+    .request will take precedence over .url.
+
+  expected_response
+    The expected HTTP response code. Defaults to 200.
+            
+  timeout
+    The timeout for the probe. Default it 2s.
+
+  interval
+    How often the probe is run. Default is 5s.
+
+  initial
+    How many of the polls in .window are considered good when Varnish
+    starts. Defaults to the value of threshold - 1. In this case, the
+    backend starts as sick and requires one single poll to be
+    conqsidered healthy.
+            
+  window
+    How many of the latest polls we examine to determine backend health. Defaults to 8.
+            
+  threshold
+    How many of the polls in .window must have succeeded for us to
+    consider the backend healthy. If this is set to more than or equal
+    to the threshold, the backend starts as healthy. Defaults to the
+    value of threshold - 1. In this case, the backend starts as sick
+    and requires one poll to pass to become healthy. Defaults to
+    threshold - 1.
+
+
 ACLs
 ----
 
@@ -310,45 +269,9 @@ To match an IP address against an ACL, simply use the match operator::
         return (pipe);
     }
 
-Regular Expressions
--------------------
-
-Varnish uses PCRE - Perl-compatible regular expressions. For a
-complete description of PCRE please see the pcre(3) man page.
-
-To send flags to the PCRE engine, such as to turn on *case
-insensitivity* add the flag within parens following a question mark,
-like this::
-
-    # If host is NOT example dot com..
-    if (req.http.host !~ "(?i)example.com$") {
-        ...
-    }
-
-Functions
----------
-
-The following built-in functions are available:
-
-ban(expression)
-  Bans all objects in cache that match the expression.
-
-hash_data(str)
-  Adds a string to the hash input. In the built-in VCL hash_data()
-  is called on the host and URL of the *request*.
-
-regsub(str, regex, sub)
-  Returns a copy of str with the first occurrence of the regular
-  expression regex replaced with sub. Within sub, \\0 (which can
-  also be spelled \\&) is replaced with the entire matched string,
-  and \\n is replaced with the contents of subgroup n in the
-  matched string.
-
-regsuball(str, regex, sub)
-  As regsub() but this replaces all occurrences.
 
 Subroutines
-~~~~~~~~~~~
+-----------
 
 A subroutine is used to group code for legibility or reusability::
 
@@ -358,622 +281,71 @@ A subroutine is used to group code for legibility or reusability::
         }
     }
 
-Subroutines in VCL do not take arguments, nor do they return values.
+Subroutines in VCL do not take arguments, nor do they return
+values. The built in subroutines all have names beginning with vcl_,
+which is reserved.
 
 To call a subroutine, use the call keyword followed by the subroutine's name::
 
     call pipe_if_local;
 
-There are a number of special subroutines which hook into the Varnish
-workflow.  These subroutines may inspect and manipulate HTTP headers
-and various other aspects of each request, and to a certain extent
-decide how the request should be handled.  Each subroutine terminates
-by calling one of a small number of keywords which indicates the
-desired outcome.
-
-vcl_init
-  Called when VCL is loaded, before any requests pass through it.
-  Typically used to initialize VMODs.
-
-  return() values:
-
-  ok
-    Normal return, VCL continues loading.
-
-vcl_recv
-  Called at the beginning of a request, after the complete request has
-  been received and parsed.  Its purpose is to decide whether or not
-  to serve the request, how to do it, and, if applicable, which backend
-  to use.
-
-  The vcl_recv subroutine may terminate with calling return() on one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  hash
-    Look up the requested object in the cache.  Control will
-    eventually pass to vcl_hit or vcl_miss, depending on whether the
-    object is in the cache.  The ``bereq.method`` value will be set
-    to ``GET`` regardless of the value of ``req.method``.
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  pipe
-    Switch to pipe mode.  Control will eventually pass to vcl_pipe.
-
-  purge
-    ?
-
-vcl_hash
-  You may call hash_data() on the data you would like to add to the hash.
-
-  The vcl_hash subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  lookup
-    Proceed.
-
-vcl_pass
-  Called upon entering pass mode.  In this mode, the request is passed
-  on to the backend, and the backend's response is passed on to the
-  client, but is not entered into the cache.  Subsequent requests
-  submitted over the same client connection are handled normally.
-
-  The vcl_pass subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  fetch
-    Proceed with pass mode.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_pipe
-  Called upon entering pipe mode.  In this mode, the request is passed
-  on to the backend, and any further data from either client or
-  backend is passed on unaltered until either end closes the
-  connection.
-
-  The vcl_pipe subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  pipe
-    Proceed with pipe mode.
-
-vcl_purge
-  ?
-
-  The vcl_purge subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-..  fetch
-..    Not implemented.
-
-vcl_hit
-  Called after a cache lookup if the requested document was found in the cache.
-
-  The vcl_hit subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  deliver
-    Deliver the cached object to the client.  Control will eventually
-    pass to vcl_deliver.
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
+Return statements
+~~~~~~~~~~~~~~~~~
 
-  fetch
-    ?
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_miss
-  Called after a cache lookup if the requested document was not found
-  in the cache.  Its purpose is to decide whether or not to attempt to
-  retrieve the document from the backend, and which backend to use.
-
-  The vcl_miss subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_backend_fetch.
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  restart
-    ?
-
-vcl_backend_fetch
-  ?
-
-  The vcl_backend_fetch subroutine may terminate with calling return()
-  with one of the following keywords:
-
-  abandon
-    ?
-
-  fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_backend_response.
-
-vcl_backend_response
-  Called after a document has been successfully retrieved from the backend.
-
-  The vcl_backend_response subroutine may terminate with calling return()
-  with one of the following keywords:
-
-  abandon
-    ?
-
-  deliver
-    Possibly insert the object into the cache, then deliver it to the
-    client.  Control will eventually pass to vcl_deliver.
-
-  retry
-    ?
-
-vcl_deliver
-  Called before a cached object is delivered to the client.
-
-  The vcl_deliver subroutine may terminate with one of the following
-  keywords:
-
-  deliver
-    Deliver the object to the client.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_error
-  Called when we hit an error, either explicitly or implicitly due to
-  backend or internal errors.
-
-  The vcl_error subroutine may terminate by calling return with one of
-  the following keywords:
-
-  deliver
-    Deliver the error object to the client.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_fini
-  Called when VCL is discarded only after all requests have exited the VCL.
-  Typically used to clean up VMODs.
-
-  return() values:
-
-  ok
-    Normal return, VCL will be discarded.
-
-If one of these subroutines is left undefined or terminates without
-reaching a handling decision, control will be handed over to the
-builtin default.  See the EXAMPLES section for a listing of the
-default code.
+The subroutine executions ends when a return(*action*) statement is
+made. The *action* specifies how execution should proceed. The context
+defines which actions are availble. See the user guide for information
+on what actions are available where.
 
 Multiple subroutines
 ~~~~~~~~~~~~~~~~~~~~
+
 If multiple subroutines with the name of one of the builtin
 ones are defined, they are concatenated in the order in which they
 appear in the source.
-The default versions distributed with Varnish will be implicitly
-concatenated as a last resort at the end.
 
-Example::
+The default versions distributed with Varnish will be implicitly
+concatenated.
 
-    # in file "main.vcl"
-    include "backends.vcl";
-    include "ban.vcl";
-
-    # in file "backends.vcl"
-    sub vcl_recv {
-        if (req.http.host ~ "(?i)example.com") {
-            set req.backend_hint = foo;
-	} elsif (req.http.host ~ "(?i)example.org") {
-	    set req.backend_hint = bar;
-	}
-    }
 
-    # in file "ban.vcl"
-    sub vcl_recv {
-        if (client.ip ~ admin_network) {
-	    if (req.http.Cache-Control ~ "no-cache") {
-                ban("req.url ~ " + req.url);
-            }
-        }
-    }
 
 Variables
-~~~~~~~~~
-
-Although subroutines take no arguments, the necessary information is
-made available to the handler subroutines through global variables.
-
-The following variables are always available:
-
-now
-  The current time, in seconds since the epoch. When used in string context
-  it returns a formatted string.
-
-The following variables are available in backend declarations:
-
-.host
-  Host name or IP address of a backend.
-
-.port
-  Service name or port number of a backend.
-
-The following variables are available while processing a request:
-
-client.ip
-  The client's IP address.
-
-client.identity
-  Identification of the client, used to load balance in the client director.
-
-server.ip
-  The IP address of the socket on which the client connection was received.
-
-server.hostname
-  The host name of the server.
-
-server.identity
-  The identity of the server, as set by the -i parameter.
-  If the -i parameter is not passed to varnishd, server.identity will be
-  set to the name of the instance, as specified by the -n parameter.
-
-req.method
-  The request type (e.g. "GET", "HEAD").
-
-req.url
-  The requested URL.
-
-req.proto
-  The HTTP protocol version used by the client.
-
-req.http.header
-  The corresponding HTTP header.
-
-req.backend_hint
-  Set bereq.backend to this if we attempt to fetch. This variable is
-  a convenience so the overall policy can be set up once and for all
-  in vcl_recv.
-
-req.can_gzip
-  Does the client accept the gzip transfer encoding.
-
-req.esi
-  Boolean. Set to false to disable ESI processing regardless of any
-  value in beresp.do_esi. Defaults to true. This variable is subject
-  to change in future versions, you should avoid using it.
-
-req.esi_level
-  A count of how many levels of ESI requests we're currently at.
-
-req.hash_always_miss
-  Force a cache miss for this request. If set to true Varnish will disregard
-  any existing objects and always (re)fetch from the backend.
-
-req.hash_ignore_busy
-  Ignore any busy object during cache lookup. You would want to do
-  this if you have two server looking up content from each other to
-  avoid potential deadlocks.
-
-req.restarts
-  A count of how many times this request has been restarted.
-
-req.ttl
-  ?
-
-req.xid
-  Unique ID of this request.
-
-The following variables are available while preparing a backend
-request (either for a cache miss or for pass or pipe mode):
-
-bereq.method
-  The request type (e.g. "GET", "HEAD").
-
-bereq.url
-  The requested URL.
-
-bereq.proto
-  The HTTP protocol version used to talk to the server.
-
-bereq.http.header
-  The corresponding HTTP header.
-
-bereq.backend
-  ?
-
-bereq.between_bytes_timeout
-  The time in seconds to wait between each received byte from the
-  backend. Not available in pipe mode.
-
-bereq.connect_timeout
-  The time in seconds to wait for a backend connection.
-
-bereq.first_byte_timeout
-  The time in seconds to wait for the first byte from the backend. Not
-  available in pipe mode.
-
-bereq.retries
-  ?
-
-bereq.uncacheable
-  ?
-
-The following variables are available after the requested object has
-been retrieved from the backend, before it is entered into the cache. In
-other words, they are available in vcl_backend_response:
-
-beresp.proto
-  The HTTP protocol version used the backend replied with.
-
-beresp.status
-  The HTTP status code returned by the server.
-
-beresp.reason
-  The HTTP status message returned by the server.
-
-beresp.http.header
-  The corresponding HTTP header.
-
-beresp.backend.ip
-  IP of the backend this response was fetched from.
-
-beresp.backend.name
-  Name of the backend this response was fetched from.
-
-beresp.do_esi
-  Boolean. ESI-process the object after fetching it. Defaults to
-  false. Set it to true to parse the object for ESI directives. Will
-  only be honored if req.esi is true.
-
-beresp.do_gunzip
-  Boolean. Unzip the object before storing it in the cache. Defaults to
-  false.
-
-beresp.do_gzip
-  Boolean. Gzip the object before storing it. Defaults to false. When
-  http_gzip_support is on Varnish will request already compressed
-  content from the backend and as such compression in Varnish is not needed.
-
-beresp.do_stream
-  Deliver the object to the client directly without fetching the whole
-  object into varnish. If this request is pass'ed it will not be
-  stored in memory. As of Varnish Cache 3.0 the object will marked as busy
-  as it is delivered so only client can access the object.
-
-beresp.grace
-  Set to a period to enable grace.
-
-beresp.keep
-  ?
-
-beresp.storage_hint
-  Hint to Varnish that you want to save this object to a particular storage
-  backend.
-
-beresp.ttl
-  The object's remaining time to live, in seconds. beresp.ttl is writable.
-
-beresp.uncacheable
-  ?
-
-After the object is entered into the cache, the following (mostly
-read-only) variables are available when the object has been located in
-cache, typically in vcl_hit, or when constructing a synthetic reply in
-vcl_error:
-
-obj.proto
-  The HTTP protocol version used when the object was retrieved.
-
-obj.status
-  The HTTP status code returned by the server.
-
-obj.reason
-  The HTTP status message returned by the server.
-
-obj.http.header
-  The corresponding HTTP header.
-
-obj.grace
-  The object's grace period in seconds. obj.grace is writable.
-
-obj.hits
-  The approximate number of times the object has been delivered. A value
-  of 0 indicates a cache miss. This variable is also available in
-  vcl_deliver.
-
-obj.keep
-  ?
-
-obj.last_use
-  The approximate time elapsed since the object was last requests, in
-  seconds. This variable is also available in vcl_deliver.
-
-obj.ttl
-  The object's remaining time to live, in seconds. obj.ttl is writable.
-
-obj.uncacheable
-  ?
-
-The following variables are available while preparing a response to the client:
-
-resp.proto
-  The HTTP protocol version to use for the response.
-
-resp.status
-  The HTTP status code that will be returned.
+---------
 
-resp.reason
-  The HTTP status message that will be returned.
+In VCL you have access to certain variable objects. These contain
+requests and responses currently beeing worked on. What variables are
+availble depends on context.
 
-resp.http.header
-  The corresponding HTTP header.
+.. include:: vcl_var.rst
 
-Values may be assigned to variables using the set keyword::
 
-    sub vcl_recv {
-        # Normalize the Host: header
-        if (req.http.host ~ "(?i)^(www.)?example.com$") {
-            set req.http.host = "www.example.com";
-        }
-    }
+Functions
+---------
 
-HTTP headers can be removed entirely using the unset keyword::
+The following built-in functions are available:
 
-    sub vcl_backend_response {
-        # Don't cache cookies
-        unset beresp.http.Set-Cookie;
-    }
+ban(expression)
+  Bans all objects in cache that match the expression.
 
-Grace
------
+hash_data(str)
+  Adds a string to the hash input. In the built-in VCL hash_data()
+  is called on the host and URL of the *request*.
 
-If the backend takes a long time to generate an object there is a risk
-of a thread pile up.  In order to prevent this you can enable *grace*.
-This allows Varnish to serve an expired version of the object while a
-fresh object is being generated by the backend.
+regsub(str, regex, sub)
+  Returns a copy of str with the first occurrence of the regular
+  expression regex replaced with sub. Within sub, \\0 (which can
+  also be spelled \\&) is replaced with the entire matched string,
+  and \\n is replaced with the contents of subgroup n in the
+  matched string.
 
-The following vcl code will make Varnish serve expired objects.  All
-object will be kept up to two minutes past their expiration time or a
-fresh object is generated.
-::
+regsuball(str, regex, sub)
+  As regsub() but this replaces all occurrences.
 
-    sub vcl_backend_response {
-        set beresp.grace = 2m;
-    }
 
 EXAMPLES
 ========
 
-The following code is the equivalent of the default configuration with
-the backend address set to "backend.example.com" and no backend port
-specified::
-
-    backend default {
-        .host = "backend.example.com";
-        .port = "http";
-    }
-
-.. include:: ../../../bin/varnishd/builtin.vcl
-  :literal:
-
-The following example shows how to support multiple sites running on
-separate backends in the same Varnish instance, by selecting backends
-based on the request URL::
-
-    backend www {
-        .host = "www.example.com";
-        .port = "80";
-    }
-
-    backend images {
-        .host = "images.example.com";
-        .port = "80";
-    }
-
-    sub vcl_recv {
-        if (req.http.host ~ "(?i)^(www.)?example.com$") {
-            set req.http.host = "www.example.com";
-            set req.backend_hint = www;
-        } elsif (req.http.host ~ "(?i)^images.example.com$") {
-            set req.backend_hint = images;
-        } else {
-            return (error(404, "Unknown virtual host"));
-        }
-    }
-
-The following snippet demonstrates how to force a minimum TTL for
-all documents.  Note that this is not the same as setting the
-default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL::
-
-    import std; # needed for std.log
-
-    sub vcl_backend_response {
-        if (beresp.ttl < 120s) {
-            std.log("Adjusting TTL");
-            set beresp.ttl = 120s;
-        }
-    }
-
-The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present::
-
-    sub vcl_recv {
-        if (req.method == "GET" && req.http.cookie) {
-            return (hash);
-        }
-    }
-
-    sub vcl_backend_response {
-        if (beresp.http.Set-Cookie) {
-            return (deliver);
-        }
-    }
-
-The following code implements the HTTP PURGE method as used by Squid
-for object invalidation::
-
-    acl purge {
-        "localhost";
-        "192.0.2.1"/24;
-    }
-
-    sub vcl_recv {
-        if (req.method == "PURGE") {
-            if (!client.ip ~ purge) {
-                return (error(405, "Not allowed."));
-            }
-            return (hash);
-        }
-    }
-
-    sub vcl_hit {
-        if (req.method == "PURGE") {
-            purge;
-            return (error(200, "Purged."));
-        }
-    }
-
-    sub vcl_miss {
-        if (req.method == "PURGE") {
-            purge;
-            return (error(200, "Purged."));
-        }
-    }
+For examples, please see the guide guide.
 
 SEE ALSO
 ========
@@ -986,10 +358,9 @@ HISTORY
 =======
 
 VCL was developed by Poul-Henning Kamp in cooperation with Verdens
-Gang AS, Redpill Linpro and Varnish Software.  This manual page was
-written by Dag-Erling Sm?rgrav and later edited by Poul-Henning Kamp,
-Kristian Lyngst?l and Per Buer.
-
+Gang AS, Redpill Linpro and Varnish Software.  This manual page is
+written by Per Buer, Poul-Henning Kamp, Martin Blix Grydeland,
+Kristian Lyngst?l and possibly others.
 
 COPYRIGHT
 =========



From perbu at varnish-software.com  Mon Mar  3 12:50:46 2014
From: perbu at varnish-software.com (Per Buer)
Date: Mon, 03 Mar 2014 13:50:46 +0100
Subject: [master] 0038eeb Forgot about microseconds and reals
Message-ID: <E1WKSKU-0006YZ-1W@project.varnish-software.com>


commit 0038eebb241d2d2e938a2fe2fc6b474c5eea7ab8
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Mar 3 13:50:43 2014 +0100

    Forgot about microseconds and reals

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 96a4aab..5a7fe6b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -105,6 +105,9 @@ Durations
 
 Durations are defined by a number and a designation. The number can be a real so 1.5w is allowed.
 
+  ms
+    milliseconds
+
   s
     seconds
 
@@ -130,6 +133,12 @@ Ints
 Certain fields are integers, used as expected. In string context they
 return a string.
 
+Reals
+-----
+
+VCL understands real numbers. As with integers, when used in a string
+context they will return a string.
+
 
 Regular Expressions
 -------------------



From fgsch at lodoss.net  Mon Mar  3 16:03:17 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 03 Mar 2014 17:03:17 +0100
Subject: [master] f479083 Better error on missing import
Message-ID: <E1WKVKn-00079a-Fw@project.varnish-software.com>


commit f47908316d9ff23fdc2a438f68e92d27bd194853
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Mar 3 16:01:56 2014 +0000

    Better error on missing import
    
    Fixes: #1436

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 69e99e5..b23d21e 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -98,3 +98,9 @@ varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_hash { if (2 == 3) { } }
 }
+
+varnish v1 -errvcl {Object not found: 'foo.bar'} {
+	sub vcl_init {
+		new bar = foo.bar();
+	}
+}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index b0f7d98..35f88f8 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -190,6 +190,13 @@ parse_new(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	sy2 = VCC_FindSymbol(tl, tl->t, SYM_OBJECT);
+	if (sy2 == NULL) {
+		VSB_printf(tl->sb, "Object not found: ");
+		vcc_ErrToken(tl, tl->t);
+		VSB_printf(tl->sb, " at ");
+		vcc_ErrWhere(tl, tl->t);
+		return;
+	}
 	XXXAN(sy2);
 
 	/*lint -save -e448 */



From phk at FreeBSD.org  Tue Mar  4 10:23:05 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 04 Mar 2014 11:23:05 +0100
Subject: [master] 34e2bba Also make IMS fetches possible in the interval
 ttl+grace and keep.
Message-ID: <E1WKmV7-0004RK-2h@project.varnish-software.com>


commit 34e2bba2c8c51fc1bc1f6b06440860011a16cbb5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 4 10:21:51 2014 +0000

    Also make IMS fetches possible in the interval ttl+grace and keep.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ba3ef0d..8696dad 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -388,7 +388,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	}
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AZ (oc->flags & OC_F_BUSY);
+	AZ(oc->flags & OC_F_BUSY);
 	AZ(req->objcore);
 
 	o = oc_getobj(&wrk->stats, oc);
@@ -427,11 +427,11 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_DELIVER;
 		return (REQ_FSM_MORE);
 	case VCL_RET_FETCH:
-		(void)HSH_DerefObj(&wrk->stats, &req->obj);
 		req->objcore = boc;
 		if (req->objcore != NULL)
 			req->req_step = R_STP_MISS;
 		else {
+			(void)HSH_DerefObj(&wrk->stats, &req->obj);
 			/*
 			 * We don't have a busy object, so treat this
 			 * like a pass
@@ -486,18 +486,21 @@ DOT
 static enum req_fsm_nxt
 cnt_miss(struct worker *wrk, struct req *req)
 {
+	struct object *o;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	AZ(req->obj);
+
+	o = req->obj;
+	req->obj = NULL;
 
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
 	switch (wrk->handling) {
 	case VCL_RET_FETCH:
 		wrk->stats.cache_miss++;
-		VBF_Fetch(wrk, req, req->objcore, NULL, VBF_NORMAL);
+		VBF_Fetch(wrk, req, req->objcore, o, VBF_NORMAL);
 		req->req_step = R_STP_FETCH;
 		return (REQ_FSM_MORE);
 	case VCL_RET_ERROR:
@@ -513,6 +516,8 @@ cnt_miss(struct worker *wrk, struct req *req)
 		WRONG("Illegal return from vcl_miss{}");
 	}
 	free(req->vary_b);
+	if (o != NULL)
+		(void)HSH_DerefObj(&wrk->stats, &o);
 	AZ(HSH_DerefObjCore(&wrk->stats, &req->objcore));
 	return (REQ_FSM_MORE);
 }



From phk at FreeBSD.org  Tue Mar  4 10:23:05 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 04 Mar 2014 11:23:05 +0100
Subject: [master] c48e48a Fix IMS fetching of ESI objects.
Message-ID: <E1WKmV7-0004RQ-4m@project.varnish-software.com>


commit c48e48ad2cbf2bb4b8e957baeeed2784123dffd1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 4 10:22:43 2014 +0000

    Fix IMS fetching of ESI objects.
    
    Fixes #1442

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 8672375..2d5f5ae 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -584,7 +584,14 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	obj->gzip_last = bo->ims_obj->gzip_last;
 	obj->gzip_stop = bo->ims_obj->gzip_stop;
 
-	XXXAZ(bo->ims_obj->esidata);
+	if (bo->ims_obj->esidata != NULL) {
+		obj->esidata = STV_alloc(bo, bo->ims_obj->esidata->len);
+		XXXAN(obj->esidata);
+		xxxassert(obj->esidata->space >= bo->ims_obj->esidata->len);
+		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr,
+		    bo->ims_obj->esidata->len);
+		obj->esidata->len = bo->ims_obj->esidata->len;
+	}
 
 	if (bo->ims_obj->vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
@@ -601,10 +608,10 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 
 	AZ(WS_Overflowed(bo->ws_o));
-	VBO_setstate(bo, BOS_STREAM);
+	if (bo->do_stream)
+		VBO_setstate(bo, BOS_STREAM);
 	HSH_Unbusy(&wrk->stats, obj->objcore);
 
-
 	st = NULL;
 	al = 0;
 
diff --git a/bin/varnishtest/tests/r01442.vtc b/bin/varnishtest/tests/r01442.vtc
new file mode 100644
index 0000000..939d5c9
--- /dev/null
+++ b/bin/varnishtest/tests/r01442.vtc
@@ -0,0 +1,31 @@
+varnishtest "ESI + IMS"
+
+server s1 {
+	rxreq
+	txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+		-body {<html><esi:remove>bla bla bla</esi:remove></html>}
+	rxreq
+	expect req.http.if-modified-since == "Thu, 26 Jun 2008 12:00:01 GMT"
+	txresp -status "304" -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" -nolen
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+		set beresp.grace = 0s;
+		set beresp.keep = 60s;
+		set beresp.ttl = 1s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+	delay 3
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+} -run
+



From lkarsten at varnish-software.com  Tue Mar  4 13:45:32 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 04 Mar 2014 14:45:32 +0100
Subject: [master] 70b1004 Run-through of the first chapter in the docs
Message-ID: <E1WKpf2-0007gU-G0@project.varnish-software.com>


commit 70b1004ce4adbc52f5c872972cba9738862c9114
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Mar 4 14:43:37 2014 +0100

    Run-through of the first chapter in the docs
    
    I'm happy with the first chapter for 4.0 release.

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index e83d1dc..dde63a6 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -15,12 +15,24 @@
 Welcome to the Varnish documentation!
 =====================================
 
+Introduction
+------------
+
 Varnish is a state of the art web accelerator. It has its mission in front of a
-web server and cache content. It makes your web site go faster.
+web server and cache content.
+
+It makes your web site go faster.
 
 We suggest you start by reading the installation guide
 :ref:`install-index`. Once you have Varnish up and running go through
-our tutorial - :ref:`tutorial-index`.
+our tutorial - :ref:`tutorial-index`, and finally the :ref:`users_guide_index`.
+
+If you need to look up how a specific Varnish tool works, the
+:ref:`reference-index` can help you out. Changes from previous versions are in
+the :ref:`whats-new-index` chapter. In closing we have :ref:`phk`, a collection
+of blog posts related to Varnish and HTTP.
+
+
 
 .. toctree::
    :maxdepth: 1
diff --git a/doc/sphinx/installation/bugs.rst b/doc/sphinx/installation/bugs.rst
index 16f0c51..98c0e19 100644
--- a/doc/sphinx/installation/bugs.rst
+++ b/doc/sphinx/installation/bugs.rst
@@ -28,11 +28,11 @@ Varnish crashes
 Plain and simple: **boom**
 
 Varnish is split over two processes, the manager and the child.  The child
-does all the work, and the manager hangs around to resurect it if it
+does all the work, and the manager hangs around to resurrect it if it
 crashes.
 
 Therefore, the first thing to do if you see a Varnish crash, is to examine
-your syslogs, to see if it has happened before.  (One site is rumoured
+your syslogs to see if it has happened before.  (One site is rumoured
 to have had Varnish restarting every 10 minutes and *still* provide better
 service than their CMS system.)
 
@@ -138,11 +138,11 @@ transactions recorded with ``varnishlog`` and your explanation of
 what is wrong about what Varnish does.
 
 Be aware, that often Varnish does exactly what you asked it to, rather
-than what you intended it to do, so it sounds like a bug that would
+than what you intended it to do. If it sounds like a bug that would
 have tripped up everybody else, take a moment to read though your
 VCL and see if it really does what you think.
 
-You can also try setting the ``vcl_trace`` parameter, that will
-generate log records with like and char number for each statement
-executed in your VCL program.
+You can also try setting the ``vcl_trace`` parameter, that will generate log
+records with like and character number for each statement executed in your VCL
+program.
 
diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 4fb8362..aaea9ec 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -7,7 +7,7 @@ we try to help as much as time permits and have tried to streamline
 this process as much as possible.
 
 But before you grab hold of us, spend a moment composing your thoughts and
-formulate your question, there is nothing as pointless as simply telling
+formulate your question. There is nothing as pointless as simply telling
 us "Varnish does not work for me" with no further information to give
 any clue to why.
 
@@ -18,15 +18,15 @@ much faster that way.
 IRC Channel
 ===========
 
-The most immediate way to get hold of us, is to join our IRC channel:
+The most immediate way to get hold of us is to join our IRC channel:
 
 	``#varnish on server irc.linpro.no``
 
-The main timezone of the channel is Europe+America.
+The main timezone of the channel is Europe work hours.
 
 If you can explain your problem in a few clear sentences, without too
 much copy&paste, IRC is a good way to try to get help. If you do need
-to paste log files, VCL and so on, please use a pastebin_.
+to paste log files, VCL and so on, please use a pastebin_ service.
 
 If the channel is all quiet, try again some time later, we do have lives,
 families and jobs to deal with also.
@@ -39,7 +39,7 @@ it mostly on topic, and don't paste random links unless they are
 Mailing Lists
 =============
 
-Getting on or off our mailing lists happens through MailMan_.
+Getting on or off our mailing lists happens through Mailman_.
 
 If you are going to use Varnish, subscribing to our ``varnish-announce``
 mailing list is probably a very good idea. The typical pattern is that
@@ -66,6 +66,10 @@ usually quite focused on source-code and such.  Everybody on
 the -dev list is also on -misc, so cross-posting only serves to annoy
 those people.
 
+.. XXX: we should introduce the wiki (if we care about it) before
+.. we start referring to it (below). Make a wiki chapter?
+
+
 Trouble Tickets
 ===============
 
@@ -90,6 +94,6 @@ an email to phk at FreeBSD.org.
 	sales at varnish-software.com
 
 
-.. _Mailman: http://lists.varnish-cache.org/mailman/listinfo
+.. _mailman: http://lists.varnish-cache.org/mailman/listinfo
 .. _pastebin: http://gist.github.com/
 .. _"Shopping-List" wiki page: http://varnish-cache.org/wiki/PostTwoShoppingList
diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index 60d43e0..0b0c665 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -5,10 +5,13 @@ Varnish Installation
 %%%%%%%%%%%%%%%%%%%%
 
 This document explains how to get Varnish onto your system, where
-to get help, how report bugs and so on. In other words, it is a manual
-about pretty much everything else than actually using Varnish to
+to get help, how report bugs and so on.
+
+In other words, it is a manual about pretty much everything else than actually using Varnish to
 move traffic.
 
+.. XXX: rewrite the last paragraph.
+
 .. toctree::
 
 	prerequisites.rst
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 45c5e4e..a32eb68 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -4,7 +4,7 @@ Installing Varnish
 ==================
 
 With open source software, you can choose to install binary packages
-or compile stuff from source-code. To install a package or compile
+or compile it yourself from source code. To install a package or compile
 from source is a matter of personal taste. If you don't know which
 method to choose, read the whole document and choose the method you
 are most comfortable with.
@@ -29,7 +29,7 @@ CentOS/RedHat
 -------------
 
 We try to keep the latest version available as prebuilt RPMs (el5 and el6)
-on `repo.varnish-cache.org <http://repo.varnish-cache.org/>`_.  See the
+on `repo.varnish-cache.org <http://repo.varnish-cache.org/>`_.  See the online
 `RedHat installation instructions
 <http://www.varnish-cache.org/installation/redhat>`_ for more information.
 
@@ -46,7 +46,7 @@ Debian/Ubuntu
 Varnish is distributed with both Debian and Ubuntu. In order to get
 Varnish up and running type `sudo apt-get install varnish`. Please
 note that this might not be the latest version of Varnish.  If you
-need a later version of Varnish, please follow the installation
+need a later version of Varnish, please follow the online installation
 instructions for `Debian
 <http://www.varnish-cache.org/installation/debian>`_ or `Ubuntu
 <http://www.varnish-cache.org/installation/ubuntu>`_.
@@ -92,10 +92,10 @@ If you're building from git, you also need the following:
 * python-docutils
 * python-sphinx (optional, if you want to build the documentation)
 
-Build dependencies on Red Hat / CentOS
+Build dependencies on RedHat / CentOS
 --------------------------------------
 
-To build Varnish on a Red Hat or CentOS system you need the following
+To build Varnish on a RedHat or CentOS system you need the following
 packages installed:
 
 * automake
@@ -115,8 +115,8 @@ If you're building from git, you also need the following:
 Compiling Varnish
 -----------------
 
-Next, configuration: The configuration will need the dependencies
-above satisfied. Once that is taken care of::
+The configuration will need the dependencies above satisfied. Once that is
+taken care of::
 
 	cd varnish-cache
 	sh autogen.sh
@@ -124,7 +124,7 @@ above satisfied. Once that is taken care of::
 	make
 
 The ``configure`` script takes some arguments, but more likely than
-not, you can forget about that for now, almost everything in Varnish
+not you can forget about that for now, almost everything in Varnish
 are run time parameters.
 
 Before you install, you may want to run the test suite, make a cup of
@@ -143,10 +143,11 @@ Installing
 
 And finally, the true test of a brave heart::
 
-	make install
+	sudo make install
 
 Varnish will now be installed in /usr/local. The varnishd binary is in
 /usr/local/sbin/varnishd and its default configuration will be
 /usr/local/etc/varnish/default.vcl.
 
 You can now proceed to the :ref:`tutorial-index`.
+
diff --git a/doc/sphinx/installation/platformnotes.rst b/doc/sphinx/installation/platformnotes.rst
index bacc57e..a7d3fb4 100644
--- a/doc/sphinx/installation/platformnotes.rst
+++ b/doc/sphinx/installation/platformnotes.rst
@@ -2,6 +2,10 @@
 Platform specific notes
 ------------------------
 
+On some platforms it is necessary to adjust the operating system before running
+Varnish on it. The systems and steps known to us are described in this section.
+
+
 Transparent hugepages on Redhat Enterprise Linux 6
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -12,7 +16,8 @@ It is recommended to disable transparent hugepages on affected systems::
 
     $ echo "never" > /sys/kernel/mm/redhat_transparent_hugepage/enabled
 
-On Debian/Ubuntu systems running 3.2 kernels the default value is "madvise" and does not need to changed.
+On Debian/Ubuntu systems running 3.2 kernels the default value is "madvise" and
+does not need to be changed.
 
 
 OpenVZ
@@ -38,12 +43,12 @@ in the Varnish startup script.
 TCP keep-alive configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-On some systems, Varnish is not able to set the TCP keep-alive values
-per socket, and therefor the tcp_keepalive_* Varnish runtime
-parameters are not available. On these platforms it can be benefitial
-to tune the system wide values for these in order to more reliably
-detect remote close for sessions spending long time on
-waitinglists. This will help free up resources faster.
+On some Solaris, FreeBSD and OS X systems, Varnish is not able to set the TCP
+keep-alive values per socket, and therefor the tcp_keepalive_* Varnish runtime
+parameters are not available. On these platforms it can be benefitial to tune
+the system wide values for these in order to more reliably detect remote close
+for sessions spending long time on waitinglists. This will help free up
+resources faster.
 
 Systems to not support TCP keep-alive values per socket include:
 



From phk at FreeBSD.org  Wed Mar  5 08:24:14 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 05 Mar 2014 09:24:14 +0100
Subject: [master] b9e6d62 Add code to do the right "backwards" merge of IMS
 responses.
Message-ID: <E1WL77e-00084O-LG@project.varnish-software.com>


commit b9e6d624e0f07d1de469dcc7f9b1df385482c7c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 5 08:23:51 2014 +0000

    Add code to do the right "backwards" merge of IMS responses.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7c6079e..f18c69a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -192,6 +192,7 @@ struct http {
 	txt			*hd;
 	unsigned char		*hdf;
 #define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
+#define HDF_MARKER		(1 << 1)	/* Marker bit */
 	uint16_t		shd;		/* Size of hd space */
 	uint16_t		nhd;		/* Next free hd */
 	uint16_t		status;
@@ -997,6 +998,7 @@ void http_CopyHome(const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
+void http_Merge(const struct http *fm, struct http *to);
 
 /* cache_http1_proto.c */
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2d5f5ae..20b60fe 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -339,8 +339,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->ims_obj != NULL && bo->beresp->status == 304) {
 		bo->beresp->status = 200;
-		http_PrintfHeader(bo->beresp, "Content-Length: %jd",
-		    (intmax_t)bo->ims_obj->len);
+		http_Merge(bo->ims_obj->http, bo->beresp);
 		do_ims = 1;
 	} else
 		do_ims = 0;
@@ -567,7 +566,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		l += vl;
 	} else
 		vl = 0;
-	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
+	l += http_EstimateWS(bo->beresp, 0, &nhttp);
 
 	obj = vbf_allocobj(bo, l, nhttp);
 	if (obj == NULL) {
@@ -603,7 +602,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 	obj->http->logtag = SLT_ObjMethod;
 	/* XXX: we should have our own HTTP_A_CONDFETCH */
-	http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
+	http_FilterResp(bo->beresp, obj->http, HTTPH_A_INS);
 	http_CopyHome(obj->http);
 
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ec28396..2a9b8de 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -254,7 +254,6 @@ http_CollectHdr(struct http *hp, const char *hdr)
 	WS_ReleaseP(hp->ws, b + 1);
 }
 
-
 /*--------------------------------------------------------------------*/
 
 static unsigned
@@ -300,7 +299,6 @@ http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
 	return (1);
 }
 
-
 /*--------------------------------------------------------------------
  * Find a given data element in a header according to RFC2616's #rule
  * (section 2.1, p15)
@@ -639,6 +637,30 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 }
 
 /*--------------------------------------------------------------------
+ * Merge two HTTP headers the "wrong" way.
+ */
+
+void
+http_Merge(const struct http *fm, struct http *to)
+{
+	unsigned u, v;
+	const char *p;
+
+	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
+		fm->hdf[u] |= HDF_MARKER;
+	for (v = HTTP_HDR_FIRST; v < to->nhd; v++) {
+		p = strchr(to->hd[v].b, ':');
+		AN(p);
+		u = http_findhdr(fm, p - to->hd[v].b, to->hd[v].b);
+		if (u)
+			fm->hdf[u] &= ~HDF_MARKER;
+	}
+	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
+		if (fm->hdf[u] & HDF_MARKER)
+			http_SetHeader(to, fm->hd[u].b);
+}
+
+/*--------------------------------------------------------------------
  * This function copies any header fields which reference foreign
  * storage into our own WS.
  */
diff --git a/bin/varnishtest/tests/c00060.vtc b/bin/varnishtest/tests/c00060.vtc
new file mode 100644
index 0000000..ece157e
--- /dev/null
+++ b/bin/varnishtest/tests/c00060.vtc
@@ -0,0 +1,45 @@
+varnishtest "IMS header merging"
+
+server s1 {
+	rxreq
+	txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+		-hdr "Foobar: foo" \
+		-hdr "Snafu: 1" \
+		-bodylen 13
+	rxreq
+	expect req.http.if-modified-since == "Thu, 26 Jun 2008 12:00:01 GMT"
+	txresp -status "304" \
+		-hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+		-hdr "Snafu: 2" \
+		-nolen
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.grace = 0s;
+		set beresp.keep = 60s;
+		set beresp.ttl = 1s;
+		if (beresp.http.foobar == "foo") {
+			set beresp.http.foobar = "foo0";
+		}
+		if (beresp.http.snafu == "2") {
+			set beresp.http.snafu = "2a";
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+	expect resp.http.foobar == foo0
+	expect resp.http.snafu == 1
+	delay 3
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+	expect resp.http.foobar == foo0
+	expect resp.http.snafu == 2a
+} -run
+



From perbu at varnish-software.com  Wed Mar  5 13:03:07 2014
From: perbu at varnish-software.com (Per Buer)
Date: Wed, 05 Mar 2014 14:03:07 +0100
Subject: [master] 774d865 Add link to the man page
Message-ID: <E1WLBTX-0007XZ-VS@project.varnish-software.com>


commit 774d86583d2a48d7bf34e63e49d29cf167c25f85
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 4 13:54:05 2014 +0100

    Add link to the man page

diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 15b6da3..889ea97 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -9,6 +9,9 @@ preferences.
 
 Note that VCL doesn't contain any loops or jump statements.
 
+This document gives an outline of the most important parts of the
+syntax. For a full documentation of VCL syntax please see
+:ref:`reference-vcl` in the reference.
 
 Strings
 ~~~~~~~



From perbu at varnish-software.com  Wed Mar  5 13:03:08 2014
From: perbu at varnish-software.com (Per Buer)
Date: Wed, 05 Mar 2014 14:03:08 +0100
Subject: [master] 10fb858 First stab at director documentation
Message-ID: <E1WLBTY-0007Xf-0u@project.varnish-software.com>


commit 10fb858033ad4709131d5eda7a4f31eb3c7768de
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 5 14:01:13 2014 +0100

    First stab at director documentation

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 609fc77..4905858 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -27,18 +27,125 @@
 
 $Module directors 3 Backend traffic directors
 
+DESCRIPTION
+===========
+
+Vmod_direcors enables load balancing in Varnish. The module are serves
+as an example on how one could extend the load balancing capabilites
+of Varnish.
+
+To enable load balancing you must import this vmod (directors) in your
+VCL:::
+
+  import directors;
+
+Then you define your backends. Once you have the backends declared you
+can add them to a director. This happens in executed VCL code. If you
+want to emulate the previous behaviour of Varnish 3.0 you can just
+initialize the directors in vcl_init, like this:::
+
+    sub vcl_init {
+        new bar = directors.round_robin();
+        bar.add_backend(server1);
+        bar.add_backend(server2);
+    }
+
+As you can see there is nothing keeping you from manipulting the
+directors elsewhere in VCL. So, you could have VCL code that would
+add more backends to a director when a certin URL is called.
+
 $Object round_robin()
+
+Description
+       Create a round robin director. This director will pick backends
+       in a round robin fashion.
+Example
+       new bar = directors.round_robin();
+
 $Method VOID .add_backend(BACKEND)
+
+Description
+       Adds a backend to the director.
+Example
+       rrdir.add_backend(backend1);
+
 $Method BACKEND .backend()
 
+Description
+       Picks a backend from the director.
+Example
+       req.backend = rrdir.backend();
+
 $Object fallback()
+
+Description
+        Creates a fallback director. This director will asdasoidwqjwqdasdas
+Example
+        new foo = directors.fallback();
+
 $Method VOID .add_backend(BACKEND)
+
+Description
+       Adds a backend to the director.
+Example
+       bar.add_backend(backend1);
+
 $Method BACKEND .backend()
 
+Description
+       Picks a backend from the director.
+Example
+       req.backend = rrdir.backend();
+
 $Object random()
+
+Description
+       Adds a random director. This director chooses backend based on
+       a random number. As you add backends to the director each
+       backends gets a weight, which is used to when requests are
+       being distributed. So, a backend with a weight of 1 would get
+       more or less 1% of the traffic of a backend in the same
+       director with a weight of 100.
+Example
+       new rand_dir = director.random();
+
 $Method VOID .add_backend(BACKEND, REAL)
+
+Description
+       Adds a backend to the director with weight.
+Example
+       bar.add_backend(backend1, 3.14);
+
 $Method BACKEND .backend()
 
+Description
+       Picks a backend from the director.
+Example
+       req.backend = rrdir.backend();
+
 $Object hash()
+
+Description
+       Creates a hash director. The hash director chooses the backend
+       bashed on hashing an arbitrary string. If you provide it with a
+       session cookie, you'll have the client connecting to the same
+       backend every time.
+Example
+       new hdir = director.hash();
+
 $Method VOID .add_backend(BACKEND, REAL)
+
+Description
+       Adds a backend to the director.
+Example
+       hdir.add_backend(backend1);
+
 $Method BACKEND .backend(STRING_LIST)
+
+Description
+       Picks a backend from the director. Use the string or list of
+       strings provided to pick the backend.
+Example
+       # pick a backend based on the cookie header from the client
+       req.backend = hdir.backend(req.http.cookie);
+



From perbu at varnish-software.com  Wed Mar  5 14:02:02 2014
From: perbu at varnish-software.com (Per Buer)
Date: Wed, 05 Mar 2014 15:02:02 +0100
Subject: [master] eb0513a remove the vcl vars from the index. it is included
 in the vcl man page
Message-ID: <E1WLCOY-0000JJ-Eq@project.varnish-software.com>


commit eb0513acf8b25ee21671cab0d754c04785219caa
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 5 14:16:52 2014 +0100

    remove the vcl vars from the index.
    it is included in the vcl man page

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index e8377f0..1537fb4 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -24,7 +24,6 @@ The Varnish Reference Manual
 	vmod_std.rst
 	vsl.rst
 	vsl-query.rst
-	vcl_var.rst
 
 .. todo::
         The programs:



From perbu at varnish-software.com  Wed Mar  5 14:02:02 2014
From: perbu at varnish-software.com (Per Buer)
Date: Wed, 05 Mar 2014 15:02:02 +0100
Subject: [master] 825ecef req.backend is gone now. req.backend_hint is it.
Message-ID: <E1WLCOY-0000JR-Ge@project.varnish-software.com>


commit 825ecef42dabfc43076624dc0e5de97bf6f67d35
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 5 14:19:07 2014 +0100

    req.backend is gone now. req.backend_hint is it.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 4905858..67ad069 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -74,7 +74,7 @@ $Method BACKEND .backend()
 Description
        Picks a backend from the director.
 Example
-       req.backend = rrdir.backend();
+       req.backend_hint = rrdir.backend();
 
 $Object fallback()
 
@@ -95,7 +95,7 @@ $Method BACKEND .backend()
 Description
        Picks a backend from the director.
 Example
-       req.backend = rrdir.backend();
+       req.backend_hint = rrdir.backend();
 
 $Object random()
 
@@ -121,7 +121,7 @@ $Method BACKEND .backend()
 Description
        Picks a backend from the director.
 Example
-       req.backend = rrdir.backend();
+       req.backend_hint = rrdir.backend();
 
 $Object hash()
 
@@ -147,5 +147,5 @@ Description
        strings provided to pick the backend.
 Example
        # pick a backend based on the cookie header from the client
-       req.backend = hdir.backend(req.http.cookie);
+       req.backend_hint = hdir.backend(req.http.cookie);
 



From phk at FreeBSD.org  Wed Mar  5 14:55:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 05 Mar 2014 15:55:25 +0100
Subject: [master] 990197a Reuse the same beresp->obj function for both cond
 and normal fetches.
Message-ID: <E1WLDED-0007kq-U6@project.varnish-software.com>


commit 990197ad3fbd8b4bb30d9f9133c57ef35c9de27e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 5 14:54:55 2014 +0000

    Reuse the same beresp->obj function for both cond and normal fetches.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 20b60fe..27a2df8 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -140,9 +140,6 @@ vbf_beresp2obj(struct busyobj *bo)
 	AZ(bo->fetch_obj);
 	bo->fetch_obj = obj;
 
-	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
-		obj->gziped = 1;
-
 	if (vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
 		    VSB_data(vary), varyl);
@@ -464,6 +461,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	obj = bo->fetch_obj;
 
+	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
+		obj->gziped = 1;
+
 	/*
 	 * Ready to fetch the body
 	 */
@@ -548,40 +548,18 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 {
-	unsigned l;
-	uint16_t nhttp;
 	struct object *obj;
 	struct objiter *oi;
 	void *sp;
-	ssize_t sl, al, tl, vl;
+	ssize_t sl, al, tl;
 	struct storage *st;
 	enum objiter_status ois;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	l = 0;
-	if (bo->ims_obj->vary != NULL) {
-		vl = VRY_Validate(bo->ims_obj->vary);
-		l += vl;
-	} else
-		vl = 0;
-	l += http_EstimateWS(bo->beresp, 0, &nhttp);
-
-	obj = vbf_allocobj(bo, l, nhttp);
-	if (obj == NULL) {
-		(void)VFP_Error(bo, "Could not get storage");
-		VDI_CloseFd(&bo->vbc);
-		return (F_STP_DONE);
-	}
-
-	AZ(bo->fetch_obj);
-	bo->fetch_obj = obj;
-
-	obj->gziped = bo->ims_obj->gziped;
-	obj->gzip_start = bo->ims_obj->gzip_start;
-	obj->gzip_last = bo->ims_obj->gzip_last;
-	obj->gzip_stop = bo->ims_obj->gzip_stop;
+	AZ(vbf_beresp2obj(bo));
+	obj = bo->fetch_obj;
 
 	if (bo->ims_obj->esidata != NULL) {
 		obj->esidata = STV_alloc(bo, bo->ims_obj->esidata->len);
@@ -592,19 +570,10 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		obj->esidata->len = bo->ims_obj->esidata->len;
 	}
 
-	if (bo->ims_obj->vary != NULL) {
-		obj->vary = (void *)WS_Copy(obj->http->ws,
-		    bo->ims_obj->vary, vl);
-		assert(vl == VRY_Validate(obj->vary));
-	}
-
-	obj->vxid = bo->vsl->wid;
-
-	obj->http->logtag = SLT_ObjMethod;
-	/* XXX: we should have our own HTTP_A_CONDFETCH */
-	http_FilterResp(bo->beresp, obj->http, HTTPH_A_INS);
-	http_CopyHome(obj->http);
-
+	obj->gziped = bo->ims_obj->gziped;
+	obj->gzip_start = bo->ims_obj->gzip_start;
+	obj->gzip_last = bo->ims_obj->gzip_last;
+	obj->gzip_stop = bo->ims_obj->gzip_stop;
 
 	AZ(WS_Overflowed(bo->ws_o));
 	if (bo->do_stream)



From fgsch at lodoss.net  Wed Mar  5 16:59:45 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 05 Mar 2014 17:59:45 +0100
Subject: [3.0] ac7ced6 Allow backends to start with if, include or else
Message-ID: <E1WLFAX-0005r5-KI@project.varnish-software.com>


commit ac7ced62b57e050a678a179d0701afdb83238366
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 5 15:58:32 2014 +0000

    Allow backends to start with if, include or else
    
    Cherry pick changes from master to allow it.
    
    Fixes: #1439

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 691cc56..b3bbbf2 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -97,3 +97,15 @@ varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_hash { if (2 == 3) { } }
 }
+
+varnish v1 -vcl {
+	backend if0 { .host = "127.0.0.1"; }
+}
+
+varnish v1 -vcl {
+	backend include0 { .host = "127.0.0.1"; }
+}
+
+varnish v1 -vcl {
+	backend else0 { .host = "127.0.0.1"; }
+}
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index fde8bb2..84ad6d2 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -62,11 +62,6 @@ tokens = {
 	"T_MUL":	"*=",
 	"T_DIV":	"/=",
 	"T_NOMATCH":	"!~",
-	"T_INCLUDE":	"include",
-	"T_IF":		"if",
-	"T_ELSEIF":	"elseif",
-	"T_ELSIF":	"elsif",
-	"T_ELSE":	"else",
 
 	# Single char tokens, for convenience on one line
 	None:		"{}()*+-/%><=;!&.|~,",
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 4bc29c7..1bd88d6 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -439,7 +439,7 @@ vcc_resolve_includes(struct vcc *tl)
 	struct source *sp;
 
 	VTAILQ_FOREACH(t, &tl->tokens, list) {
-		if (t->tok != T_INCLUDE)
+		if (t->tok != ID || !vcc_IdIs(t, "include"))
 			continue;
 
 		t1 = VTAILQ_NEXT(t, list);
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index 4a242e5..ea86b47 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -88,37 +88,47 @@ static void
 vcc_IfStmt(struct vcc *tl)
 {
 
-	SkipToken(tl, T_IF);
+	SkipToken(tl, ID);
 	Fb(tl, 1, "if ");
 	vcc_Conditional(tl);
 	ERRCHK(tl);
 	L(tl, vcc_Compound(tl));
 	ERRCHK(tl);
-	while (1) {
-		switch (tl->t->tok) {
-		case T_ELSE:
+	while (tl->t->tok == ID) {
+		if (vcc_IdIs(tl->t, "else")) {
 			vcc_NextToken(tl);
-			if (tl->t->tok != T_IF) {
+			if (tl->t->tok == '{') {
 				Fb(tl, 1, "else\n");
 				L(tl, vcc_Compound(tl));
 				ERRCHK(tl);
 				return;
 			}
-			/* FALLTHROUGH */
-		case T_ELSEIF:
-		case T_ELSIF:
+			if (tl->t->tok != ID || !vcc_IdIs(tl->t, "if")) {
+				VSB_printf(tl->sb,
+				    "'else' must be followed by 'if' or '{'\n");
+				vcc_ErrWhere(tl, tl->t);
+				return;
+			}
 			Fb(tl, 1, "else if ");
 			vcc_NextToken(tl);
 			vcc_Conditional(tl);
 			ERRCHK(tl);
 			L(tl, vcc_Compound(tl));
 			ERRCHK(tl);
+		} else if (vcc_IdIs(tl->t, "elseif") ||
+		    vcc_IdIs(tl->t, "elsif") ||
+		    vcc_IdIs(tl->t, "elif")) {
+			Fb(tl, 1, "else if ");
+			vcc_NextToken(tl);
+			vcc_Conditional(tl);
+			ERRCHK(tl);
+			L(tl, vcc_Compound(tl));
+			ERRCHK(tl);
+		} else {
 			break;
-		default:
-			C(tl, ";");
-			return;
 		}
 	}
+	C(tl, ";");
 }
 
 /*--------------------------------------------------------------------
@@ -148,9 +158,6 @@ vcc_Compound(struct vcc *tl)
 		case '{':
 			vcc_Compound(tl);
 			break;
-		case T_IF:
-			vcc_IfStmt(tl);
-			break;
 		case '}':
 			vcc_NextToken(tl);
 			tl->indent -= INDENT;
@@ -168,11 +175,16 @@ vcc_Compound(struct vcc *tl)
 			tl->err = 1;
 			return;
 		case ID:
-			i = vcc_ParseAction(tl);
-			ERRCHK(tl);
-			if (i) {
-				SkipToken(tl, ';');
+			if (vcc_IdIs(tl->t, "if")) {
+				vcc_IfStmt(tl);
 				break;
+			} else {
+				i = vcc_ParseAction(tl);
+				ERRCHK(tl);
+				if (i) {
+					SkipToken(tl, ';');
+					break;
+				}
 			}
 			/* FALLTHROUGH */
 		default:



From perbu at varnish-software.com  Thu Mar  6 13:10:17 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 06 Mar 2014 14:10:17 +0100
Subject: [master] 35baa30 Make ESI a separate chapter
Message-ID: <E1WLY41-0003qO-6G@project.varnish-software.com>


commit 35baa30ebf557a87fe9307f60392ce1b28e81bca
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:07:48 2014 +0100

    Make ESI a separate chapter

diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 58b31f2..9035224 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -1,7 +1,7 @@
 .. _users-guide-esi:
 
-Edge Side Includes
-------------------
+Content composition with Edge Side Includes
+-------------------------------------------
 
 Varnish can cache create web pages by putting different pages
 together. These *fragments* can have individual cache policies. If you



From perbu at varnish-software.com  Thu Mar  6 13:10:17 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 06 Mar 2014 14:10:17 +0100
Subject: [master] 03fd530 Move the content from cookies and vary into
 increasing perf
Message-ID: <E1WLY41-0003qU-7p@project.varnish-software.com>


commit 03fd530f057bc98e4421a9f1ac77e5e14e8028a9
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:08:09 2014 +0100

    Move the content from cookies and vary into increasing perf

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 431c0fc..3e1adc5 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -114,6 +114,104 @@ your expectations.
 
 Let's take a look at the important headers you should be aware of:
 
+.. _users-guide-cookies:
+
+Cookies
+-------
+
+Varnish will, in the default configuration, not cache a object coming
+from the backend with a Set-Cookie header present. Also, if the client
+sends a Cookie header, Varnish will bypass the cache and go directly to
+the backend.
+
+This can be overly conservative. A lot of sites use Google Analytics
+(GA) to analyze their traffic. GA sets a cookie to track you. This
+cookie is used by the client side javascript and is therefore of no
+interest to the server. 
+
+Cookies from the client
+~~~~~~~~~~~~~~~~~~~~~~~
+
+For a lot of web application it makes sense to completely disregard the
+cookies unless you are accessing a special part of the web site. This
+VCL snippet in vcl_recv will disregard cookies unless you are
+accessing /admin/::
+
+  if ( !( req.url ~ ^/admin/) ) {
+    unset req.http.Cookie;
+  }
+
+Quite simple. If, however, you need to do something more complicated,
+like removing one out of several cookies, things get
+difficult. Unfortunately Varnish doesn't have good tools for
+manipulating the Cookies. We have to use regular expressions to do the
+work. If you are familiar with regular expressions you'll understand
+whats going on. If you don't I suggest you either pick up a book on
+the subject, read through the *pcrepattern* man page or read through
+one of many online guides.
+
+Let me show you what Varnish Software uses. We use some cookies for
+Google Analytics tracking and similar tools. The cookies are all set
+and used by Javascript. Varnish and Drupal doesn't need to see those
+cookies and since Varnish will cease caching of pages when the client
+sends cookies we will discard these unnecessary cookies in VCL. 
+
+In the following VCL we discard all cookies that start with a
+underscore::
+
+  // Remove has_js and Google Analytics __* cookies.
+  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
+  // Remove a ";" prefix, if present.
+  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
+
+Let me show you an example where we remove everything except the
+cookies named COOKIE1 and COOKIE2 and you can marvel at it::
+
+  sub vcl_recv {
+    if (req.http.Cookie) {
+      set req.http.Cookie = ";" + req.http.Cookie;
+      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
+      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
+      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
+      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
+
+      if (req.http.Cookie == "") {
+          remove req.http.Cookie;
+      }
+    }
+  }
+
+A somewhat simpler example that can accomplish almost the same can be
+found below. Instead of filtering out the other cookies it picks out
+the one cookie that is needed, copies it to another header and then
+copies it back, deleting the original cookie header.::
+
+  sub vcl_recv {
+         # save the original cookie header so we can mangle it
+        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
+        # using a capturing sub pattern, extract the continuous string of 
+        # alphanumerics that immediately follows "PHPSESSID="
+        set req.http.X-Varnish-PHP_SID = 
+           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
+        set req.http.Cookie = req.X-Varnish-PHP_SID;
+        remove req.X-Varnish-PHP_SID;
+   }   
+
+There are other scary examples of what can be done in VCL in the
+Varnish Cache Wiki.
+
+
+Cookies coming from the backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your backend server sets a cookie using the Set-Cookie header
+Varnish will not cache the page in the default configuration.  A
+hit-for-pass object (see :ref:`user-guide-vcl_actions`) is created.
+So, if the backend server acts silly and sets unwanted cookies just unset
+the Set-Cookie header and all should be fine. 
+
+
+
 Cache-Control
 ~~~~~~~~~~~~~
 
@@ -145,7 +243,7 @@ Pragma
 An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
 header. You could easily add support for this header in VCL.
 
-In vcl_fetch::
+In vcl_backend_response::
 
   if (beresp.http.Pragma ~ "nocache") {
         set beresp.uncacheable = true;
@@ -168,7 +266,7 @@ somewhat cumbersome backend.
 You need VCL to identify the objects you want and then you set the
 beresp.ttl to whatever you want::
 
-  sub vcl_fetch {
+  sub vcl_backend_response {
       if (req.url ~ "^/legacy_broken_cms/") {
           set beresp.ttl = 5d;
       }
@@ -203,3 +301,80 @@ setting up redirects or by using the following VCL::
   }
 
 
+.. _users-guide-vary:
+
+HTTP Vary
+---------
+
+*HTTP Vary is not a trivial concept. It is by far the most
+misunderstood HTTP header.*
+
+A lot of the response headers tell the client something about the HTTP
+object being delivered. Clients can request different variants a an
+HTTP object, based on their preference. Their preferences might cover
+stuff like encoding or language. When a client prefers UK English this
+is indicated through "Accept-Language: en-uk". Caches need to keep
+these different variants apart and this is done through the HTTP
+response header "Vary".
+
+When a backend server issues a "Vary: Accept-Language" it tells
+Varnish that its needs to cache a separate version for every different
+Accept-Language that is coming from the clients.
+
+If two clients say they accept the languages "en-us, en-uk" and "da,
+de" respectively, Varnish will cache and serve two different versions
+of the page if the backend indicated that Varnish needs to vary on the
+Accept-Language header.
+
+Please note that the headers that Vary refer to need to match
+*exactly* for there to be a match. So Varnish will keep two copies of
+a page if one of them was created for "en-us, en-uk" and the other for
+"en-us,en-uk". Just the lack of space will force Varnish to cache
+another version.
+
+To achieve a high hitrate whilst using Vary is there therefor crucial
+to normalize the headers the backends varies on. Remember, just a
+difference in case can force different cache entries.
+
+The following VCL code will normalize the Accept-Language headers, to
+one of either "en","de" or "fr"::
+
+    if (req.http.Accept-Language) {
+        if (req.http.Accept-Language ~ "en") {
+            set req.http.Accept-Language = "en";
+        } elsif (req.http.Accept-Language ~ "de") {
+            set req.http.Accept-Language = "de";
+        } elsif (req.http.Accept-Language ~ "fr") {
+            set req.http.Accept-Language = "fr";
+        } else {
+            # unknown language. Remove the accept-language header and 
+	    # use the backend default.
+            remove req.http.Accept-Language
+        }
+    }
+
+The code sets the Accept-Encoding header from the client to either
+gzip, deflate with a preference for gzip.
+
+Vary parse errors
+~~~~~~~~~~~~~~~~~
+
+Varnish will return a 503 internal server error page when it fails to
+parse the Vary server header, or if any of the client headers listed
+in the Vary header exceeds the limit of 65k characters. An SLT_Error
+log entry is added in these cases.
+
+Pitfall - Vary: User-Agent
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some applications or application servers send *Vary: User-Agent* along
+with their content. This instructs Varnish to cache a separate copy
+for every variation of User-Agent there is. There are plenty. Even a
+single patchlevel of the same browser will generate at least 10
+different User-Agent headers based just on what operating system they
+are running. 
+
+So if you *really* need to Vary based on User-Agent be sure to
+normalize the header or your hit rate will suffer badly. Use the above
+code as a template.
+



From perbu at varnish-software.com  Thu Mar  6 13:10:17 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 06 Mar 2014 14:10:17 +0100
Subject: [master] 4f487ab Update index, remove orphans, add esi
Message-ID: <E1WLY41-0003qj-9I@project.varnish-software.com>


commit 4f487ab8e568c2ab80c739cc4ad3fa8a11cdd758
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:08:24 2014 +0100

    Update index, remove orphans, add esi

diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 1000a6f..7cd0c25 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -40,6 +40,7 @@ from transaction level to aggregate statistics.
    vcl
    report
    performance
+   esi   
    troubleshooting
    orphans
 



From perbu at varnish-software.com  Thu Mar  6 13:10:17 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 06 Mar 2014 14:10:17 +0100
Subject: [master] dcf8e96 rebreak lines
Message-ID: <E1WLY41-0003qp-AI@project.varnish-software.com>


commit dcf8e960f9e3dc35e6434d9968bfe0566cf020db
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:09:02 2014 +0100

    rebreak lines

diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index 0b0c665..7952958 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -7,8 +7,8 @@ Varnish Installation
 This document explains how to get Varnish onto your system, where
 to get help, how report bugs and so on.
 
-In other words, it is a manual about pretty much everything else than actually using Varnish to
-move traffic.
+In other words, it is a manual about pretty much everything else than
+actually using Varnish to move traffic.
 
 .. XXX: rewrite the last paragraph.
 



From perbu at varnish-software.com  Thu Mar  6 13:10:17 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 06 Mar 2014 14:10:17 +0100
Subject: [master] e8384a5 remove orphans - content absorbed elsewhere
Message-ID: <E1WLY41-0003qz-BS@project.varnish-software.com>


commit e8384a53979b44526573a868a444a32fd84bc2ec
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:10:05 2014 +0100

    remove orphans - content absorbed elsewhere

diff --git a/doc/sphinx/users-guide/cookies.rst b/doc/sphinx/users-guide/cookies.rst
deleted file mode 100644
index c57478b..0000000
--- a/doc/sphinx/users-guide/cookies.rst
+++ /dev/null
@@ -1,96 +0,0 @@
-.. _users-guide-cookies:
-
-Cookies
--------
-
-Varnish will, in the default configuration, not cache a object coming
-from the backend with a Set-Cookie header present. Also, if the client
-sends a Cookie header, Varnish will bypass the cache and go directly to
-the backend.
-
-This can be overly conservative. A lot of sites use Google Analytics
-(GA) to analyze their traffic. GA sets a cookie to track you. This
-cookie is used by the client side javascript and is therefore of no
-interest to the server. 
-
-Cookies from the client
-~~~~~~~~~~~~~~~~~~~~~~~
-
-For a lot of web application it makes sense to completely disregard the
-cookies unless you are accessing a special part of the web site. This
-VCL snippet in vcl_recv will disregard cookies unless you are
-accessing /admin/::
-
-  if ( !( req.url ~ ^/admin/) ) {
-    unset req.http.Cookie;
-  }
-
-Quite simple. If, however, you need to do something more complicated,
-like removing one out of several cookies, things get
-difficult. Unfortunately Varnish doesn't have good tools for
-manipulating the Cookies. We have to use regular expressions to do the
-work. If you are familiar with regular expressions you'll understand
-whats going on. If you don't I suggest you either pick up a book on
-the subject, read through the *pcrepattern* man page or read through
-one of many online guides.
-
-Let me show you what Varnish Software uses. We use some cookies for
-Google Analytics tracking and similar tools. The cookies are all set
-and used by Javascript. Varnish and Drupal doesn't need to see those
-cookies and since Varnish will cease caching of pages when the client
-sends cookies we will discard these unnecessary cookies in VCL. 
-
-In the following VCL we discard all cookies that start with a
-underscore::
-
-  // Remove has_js and Google Analytics __* cookies.
-  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
-  // Remove a ";" prefix, if present.
-  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
-
-Let me show you an example where we remove everything except the
-cookies named COOKIE1 and COOKIE2 and you can marvel at it::
-
-  sub vcl_recv {
-    if (req.http.Cookie) {
-      set req.http.Cookie = ";" + req.http.Cookie;
-      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
-      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
-
-      if (req.http.Cookie == "") {
-          remove req.http.Cookie;
-      }
-    }
-  }
-
-A somewhat simpler example that can accomplish almost the same can be
-found below. Instead of filtering out the other cookies it picks out
-the one cookie that is needed, copies it to another header and then
-copies it back, deleting the original cookie header.::
-
-  sub vcl_recv {
-         # save the original cookie header so we can mangle it
-        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
-        # using a capturing sub pattern, extract the continuous string of 
-        # alphanumerics that immediately follows "PHPSESSID="
-        set req.http.X-Varnish-PHP_SID = 
-           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
-        set req.http.Cookie = req.X-Varnish-PHP_SID;
-        remove req.X-Varnish-PHP_SID;
-   }   
-
-There are other scary examples of what can be done in VCL in the
-Varnish Cache Wiki.
-
-
-Cookies coming from the backend
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If your backend server sets a cookie using the Set-Cookie header
-Varnish will not cache the page in the default configuration.  A
-hit-for-pass object (see :ref:`user-guide-vcl_actions`) is created.
-So, if the backend server acts silly and sets unwanted cookies just unset
-the Set-Cookie header and all should be fine. 
-
diff --git a/doc/sphinx/users-guide/orphans.rst b/doc/sphinx/users-guide/orphans.rst
deleted file mode 100644
index 76fbf5d..0000000
--- a/doc/sphinx/users-guide/orphans.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Orphans
-=======
-
-XXX: These are chapters which need to find a new home in the other sections.
-
-.. toctree::
-   :maxdepth: 2
-
-   esi
-   vary
-   cookies
diff --git a/doc/sphinx/users-guide/vary.rst b/doc/sphinx/users-guide/vary.rst
deleted file mode 100644
index 00c946c..0000000
--- a/doc/sphinx/users-guide/vary.rst
+++ /dev/null
@@ -1,77 +0,0 @@
-.. _users-guide-vary:
-
-HTTP Vary
----------
-
-*HTTP Vary is not a trivial concept. It is by far the most
-misunderstood HTTP header.*
-
-A lot of the response headers tell the client something about the HTTP
-object being delivered. Clients can request different variants a an
-HTTP object, based on their preference. Their preferences might cover
-stuff like encoding or language. When a client prefers UK English this
-is indicated through "Accept-Language: en-uk". Caches need to keep
-these different variants apart and this is done through the HTTP
-response header "Vary".
-
-When a backend server issues a "Vary: Accept-Language" it tells
-Varnish that its needs to cache a separate version for every different
-Accept-Language that is coming from the clients.
-
-If two clients say they accept the languages "en-us, en-uk" and "da,
-de" respectively, Varnish will cache and serve two different versions
-of the page if the backend indicated that Varnish needs to vary on the
-Accept-Language header.
-
-Please note that the headers that Vary refer to need to match
-*exactly* for there to be a match. So Varnish will keep two copies of
-a page if one of them was created for "en-us, en-uk" and the other for
-"en-us,en-uk". Just the lack of space will force Varnish to cache
-another version.
-
-To achieve a high hitrate whilst using Vary is there therefor crucial
-to normalize the headers the backends varies on. Remember, just a
-difference in case can force different cache entries.
-
-The following VCL code will normalize the Accept-Language headers, to
-one of either "en","de" or "fr"::
-
-    if (req.http.Accept-Language) {
-        if (req.http.Accept-Language ~ "en") {
-            set req.http.Accept-Language = "en";
-        } elsif (req.http.Accept-Language ~ "de") {
-            set req.http.Accept-Language = "de";
-        } elsif (req.http.Accept-Language ~ "fr") {
-            set req.http.Accept-Language = "fr";
-        } else {
-            # unknown language. Remove the accept-language header and 
-	    # use the backend default.
-            remove req.http.Accept-Language
-        }
-    }
-
-The code sets the Accept-Encoding header from the client to either
-gzip, deflate with a preference for gzip.
-
-Vary parse errors
-~~~~~~~~~~~~~~~~~
-
-Varnish will return a 503 internal server error page when it fails to
-parse the Vary server header, or if any of the client headers listed
-in the Vary header exceeds the limit of 65k characters. An SLT_Error
-log entry is added in these cases.
-
-Pitfall - Vary: User-Agent
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Some applications or application servers send *Vary: User-Agent* along
-with their content. This instructs Varnish to cache a separate copy
-for every variation of User-Agent there is. There are plenty. Even a
-single patchlevel of the same browser will generate at least 10
-different User-Agent headers based just on what operating system they
-are running. 
-
-So if you *really* need to Vary based on User-Agent be sure to
-normalize the header or your hit rate will suffer badly. Use the above
-code as a template.
-



From perbu at varnish-software.com  Thu Mar  6 13:30:13 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 06 Mar 2014 14:30:13 +0100
Subject: [master] 46b5fac Duplicate.
Message-ID: <E1WLYNJ-0001hZ-PT@project.varnish-software.com>


commit 46b5fac332390aeb410360188757f661d50f4157
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:29:53 2014 +0100

    Duplicate.

diff --git a/doc/sphinx/users-guide/websockets.rst b/doc/sphinx/users-guide/websockets.rst
deleted file mode 100644
index 7217b88..0000000
--- a/doc/sphinx/users-guide/websockets.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-
-Implementing websocket support
-------------------------------
-
-Websockets is a technology for creating a bidirectional stream-based channel over HTTP.
-
-To run websockets through Varnish you need to pipe it, and copy the Upgrade header. Use the following
-VCL config to do so::
-
-    sub vcl_pipe {
-         if (req.http.upgrade) {
-             set bereq.http.upgrade = req.http.upgrade;
-         }
-    }
-    sub vcl_recv {
-         if (req.http.Upgrade ~ "(?i)websocket") {
-             return (pipe);
-         }
-    }
-



From fgsch at lodoss.net  Thu Mar  6 15:43:58 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 06 Mar 2014 16:43:58 +0100
Subject: [master] 839cdf3 Correct spelling
Message-ID: <E1WLaSk-0007Lv-HR@project.varnish-software.com>


commit 839cdf399a581d5cd05c46eb68f367dcecb3a104
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 5 21:32:04 2014 +0000

    Correct spelling

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index bf4bcd0..0502c36 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -847,7 +847,7 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 	    vslc_vtx_next(&vtx->c.cursor) == 1) {
 		ptr = vtx->c.cursor.rec.ptr;
 		if (VSL_ID(ptr) != vtx->key.vxid) {
-			(void)vtx_diag_tag(vtx, ptr, "vxid missmatch");
+			(void)vtx_diag_tag(vtx, ptr, "vxid mismatch");
 			continue;
 		}
 



From fgsch at lodoss.net  Thu Mar  6 15:43:58 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 06 Mar 2014 16:43:58 +0100
Subject: [master] 41db0bc Don't expose certain variables in vcl_pipe
Message-ID: <E1WLaSk-0007M1-J0@project.varnish-software.com>


commit 41db0bc0b31b46cea33e6d93f598fa210eeb343d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 6 15:36:54 2014 +0000

    Don't expose certain variables in vcl_pipe
    
    Accessing bereq.first_byte_timeout and bereq.between_bytes_timeout
    in vcl_pipe makes no sense so disallow it.
    
    Fixes: #1435

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index b23d21e..4314dce 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -104,3 +104,17 @@ varnish v1 -errvcl {Object not found: 'foo.bar'} {
 		new bar = foo.bar();
 	}
 }
+
+varnish v1 -errvcl {'bereq.first_byte_timeout': cannot be set} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_pipe {
+		set bereq.first_byte_timeout = 10s;
+	}
+}
+
+varnish v1 -errvcl {'bereq.between_bytes_timeout': cannot be set} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_pipe {
+		set bereq.between_bytes_timeout = 10s;
+	}
+}
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index f313bd5..8c9461a 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -351,16 +351,16 @@ The client's IP address.
 	),
 	('bereq.first_byte_timeout',
 		'DURATION',
-		( 'pipe', 'backend', ),
-		( 'pipe', 'backend', ), """
+		( 'backend', ),
+		( 'backend', ), """
 		The time in seconds to wait for the first byte from
 		the backend.  Not available in pipe mode.
 		"""
 	),
 	('bereq.between_bytes_timeout',
 		'DURATION',
-		( 'pipe', 'backend', ),
-		( 'pipe', 'backend', ), """
+		( 'backend', ),
+		( 'backend', ), """
 		The time in seconds to wait between each received byte from the
 		backend.  Not available in pipe mode.
 		"""



From lkarsten at varnish-software.com  Fri Mar  7 12:31:02 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 07 Mar 2014 13:31:02 +0100
Subject: [3.0] 0152502 Document that vcl_pass/vcl_miss can't restart
Message-ID: <E1WLtva-0002iW-DQ@project.varnish-software.com>


commit 0152502b34b24ae3c3ad9ab774c4116744215b67
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 7 11:26:42 2014 +0100

    Document that vcl_pass/vcl_miss can't restart
    
    Restart from vcl_pass and vcl_miss does not work, but was
    documented that it did.
    
    Mentioned in: #1443

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index fdf7cee..d0f3bf2 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1186,8 +1186,6 @@ DOT		label="vcl_miss()|req.\nbereq."
 DOT	]
 DOT	miss -> vcl_miss [style=bold,color=blue]
 DOT }
-DOT vcl_miss -> rst_miss [label="restart",color=purple]
-DOT rst_miss [label="RESTART",shape=plaintext]
 DOT vcl_miss -> err_miss [label="error"]
 DOT err_miss [label="ERROR",shape=plaintext]
 DOT vcl_miss -> fetch [label="fetch",style=bold,color=blue]
@@ -1272,8 +1270,6 @@ DOT	pass2 -> vcl_pass [style=bold, color=red]
 DOT	vcl_pass -> pass_do [label="pass"] [style=bold, color=red]
 DOT }
 DOT pass_do -> fetch [style=bold, color=red]
-DOT vcl_pass -> rst_pass [label="restart",color=purple]
-DOT rst_pass [label="RESTART",shape=plaintext]
 DOT vcl_pass -> err_pass [label="error"]
 DOT err_pass [label="ERROR",shape=plaintext]
  */
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index fd4f588..76aae84 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -522,11 +522,6 @@ vcl_pass
   pass
     Proceed with pass mode.
 
-  restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
-    error.
-
 vcl_hash
   You may call hash_data() on the data you would like to add to the hash.
   



From phk at FreeBSD.org  Mon Mar 10 07:31:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 10 Mar 2014 08:31:00 +0100
Subject: [master] 818f00c Make vmod_priv_fini() a real function and name it
 VRT_priv_fini() for consistency.
Message-ID: <E1WMufs-00031G-3P@project.varnish-software.com>


commit 818f00c637486c17168412fc7c188d9051a31cf6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 07:30:35 2014 +0000

    Make vmod_priv_fini() a real function and name it VRT_priv_fini()
    for consistency.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index ce536e5..f78f140 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -498,6 +498,16 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+void
+VRT_priv_fini(const struct vmod_priv *p)
+{
+	if (p->priv != (void*)0 && p->free != (void*)0)
+		p->free(p->priv);
+}
+
+/*--------------------------------------------------------------------
  * Simple stuff
  */
 
diff --git a/include/vrt.h b/include/vrt.h
index 9b3664a..dc84852 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -246,13 +246,7 @@ struct vmod_priv {
 
 typedef int vmod_init_f(struct vmod_priv *,  const struct VCL_conf *);
 
-static inline void
-vmod_priv_fini(const struct vmod_priv *p)
-{
-
-	if (p->priv != (void*)0 && p->free != (void*)0)
-		p->free(p->priv);
-}
+void VRT_priv_fini(const struct vmod_priv *p);
 
 /* Stevedore related functions */
 int VRT_Stv(const char *nm);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 08f91df..c143cf1 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -561,7 +561,7 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 			bprintf(buf, "vmod_priv_%u", tl->unique++);
 			ifp = New_IniFin(tl);
 			Fh(tl, 0, "static struct vmod_priv %s;\n", buf);
-			VSB_printf(ifp->fin, "\tvmod_priv_fini(&%s);", buf);
+			VSB_printf(ifp->fin, "\tVRT_priv_fini(&%s);", buf);
 			e2 = vcc_mk_expr(VOID, "&%s", buf);
 			p += strlen(p) + 1;
 		} else if (fmt == ENUM) {
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index ac88b59..044cd24 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -119,7 +119,7 @@ vcc_ParseImport(struct vcc *tl)
 	VSB_printf(ifp->ini, "\t\treturn(1);");
 
 	/* XXX: zero the function pointer structure ?*/
-	VSB_printf(ifp->fin, "\tvmod_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
 	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
 
 	ifp = NULL;
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 7109644..146f46b 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -126,6 +126,6 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 		p += 2;
 		q += 1;
 	}
-	vmod_priv_fini(b);
+	VRT_priv_fini(b);
 	return (s);
 }



From phk at FreeBSD.org  Mon Mar 10 09:23:10 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 10 Mar 2014 10:23:10 +0100
Subject: [master] 42baff6 Don't merge Content-Encoding from IMS object before
 VCL, in order to not confuse VCL by "backend" suddenly changing gzip status,
 but do unconditionally smash it in there afterwards, no matter what VCL does,
 so it will be correct relative to the existing body.
Message-ID: <E1WMwQQ-0004qV-UW@project.varnish-software.com>


commit 42baff6df2736e86fb7c73a90686b83756575dd1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 09:21:50 2014 +0000

    Don't merge Content-Encoding from IMS object before VCL, in order
    to not confuse VCL by "backend" suddenly changing gzip status, but do
    unconditionally smash it in there afterwards, no matter what VCL
    does, so it will be correct relative to the existing body.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f18c69a..5b4028d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -594,8 +594,9 @@ struct object {
 
 	uint8_t			*vary;
 
-	/* XXX: make bitmap */
-	uint8_t			gziped;
+	unsigned		gziped:1;
+	unsigned		changed_gzip:1;
+
 	/* Bit positions in the gzip stream */
 	ssize_t			gzip_start;
 	ssize_t			gzip_last;
@@ -998,7 +999,7 @@ void http_CopyHome(const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
-void http_Merge(const struct http *fm, struct http *to);
+void http_Merge(const struct http *fm, struct http *to, int not_ce);
 
 /* cache_http1_proto.c */
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 27a2df8..b42674a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -336,7 +336,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->ims_obj != NULL && bo->beresp->status == 304) {
 		bo->beresp->status = 200;
-		http_Merge(bo->ims_obj->http, bo->beresp);
+		http_Merge(bo->ims_obj->http, bo->beresp,
+		    bo->ims_obj->changed_gzip);
 		do_ims = 1;
 	} else
 		do_ims = 0;
@@ -464,6 +465,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
 		obj->gziped = 1;
 
+	if (bo->do_gzip || bo->do_gunzip)
+		obj->changed_gzip = 1;
+
 	/*
 	 * Ready to fetch the body
 	 */
@@ -554,10 +558,22 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	ssize_t sl, al, tl;
 	struct storage *st;
 	enum objiter_status ois;
+	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	if (bo->ims_obj->changed_gzip) {
+		/*
+		 * If we modified the gzip status of the IMS object, that
+		 * must control the C-E header, if any.
+		 */
+		http_Unset(bo->beresp, H_Content_Encoding);
+		if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p))
+			http_PrintfHeader(bo->beresp,
+			    "Content-Encoding: %s", p);
+	}
+
 	AZ(vbf_beresp2obj(bo));
 	obj = bo->fetch_obj;
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 2a9b8de..6898987 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -641,13 +641,19 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
  */
 
 void
-http_Merge(const struct http *fm, struct http *to)
+http_Merge(const struct http *fm, struct http *to, int not_ce)
 {
 	unsigned u, v;
 	const char *p;
 
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
+	if (not_ce) {
+		u = http_findhdr(fm,
+		    H_Content_Encoding[0] - 1, H_Content_Encoding + 1);
+		if (u > 0)
+			fm->hdf[u] &= ~HDF_MARKER;
+	}
 	for (v = HTTP_HDR_FIRST; v < to->nhd; v++) {
 		p = strchr(to->hd[v].b, ':');
 		AN(p);
diff --git a/bin/varnishtest/tests/g00006.vtc b/bin/varnishtest/tests/g00006.vtc
new file mode 100644
index 0000000..740d4bd
--- /dev/null
+++ b/bin/varnishtest/tests/g00006.vtc
@@ -0,0 +1,72 @@
+varnishtest "IMS'ing g[un]zip'ed objects"
+
+server s1 {
+	rxreq
+	expect req.url == /1
+	txresp -hdr  "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -bodylen 20
+
+	rxreq
+	expect req.url == /1
+	expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT"
+	txresp -status 304 -nolen
+
+	rxreq
+	expect req.url == /2
+	txresp -hdr  "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -gzipbody "012345678901234567"
+
+	rxreq
+	expect req.url == /2
+	expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT"
+	txresp -status 304 -hdr "Content-Encoding: gzip,rot13" -nolen
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.http.foobar = beresp.http.content-encoding;
+		if (bereq.url == "/1") {
+			set beresp.do_gzip = true;
+		} else {
+			set beresp.do_gunzip = true;
+		}
+		set beresp.ttl = 1s;
+		set beresp.grace = 0s;
+		set beresp.keep = 60s;
+	}
+} -start
+
+client c1 {
+	txreq -url /1 -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	expect resp.http.foobar == ""
+	gunzip
+	expect resp.bodylen == 20
+
+	delay 1
+
+	txreq -url /1 -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	expect resp.http.foobar == ""
+	gunzip
+	expect resp.bodylen == 20
+
+	delay .2
+
+	txreq -url /2
+	rxresp
+	expect resp.http.content-encoding == "<undef>"
+	expect resp.http.foobar == "gzip"
+	expect resp.bodylen == 18
+
+	delay 1
+
+	txreq -url /2
+	rxresp
+	expect resp.http.content-encoding == "<undef>"
+	# Here we see the C-E of the IMS OBJ
+	expect resp.http.foobar == "gzip,rot13"
+	expect resp.bodylen == 18
+
+} -run



From phk at FreeBSD.org  Mon Mar 10 14:37:14 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 10 Mar 2014 15:37:14 +0100
Subject: [master] 3edd1f7 Make synth {} work in vcl_backend_response{}
Message-ID: <E1WN1KM-00072I-35@project.varnish-software.com>


commit 3edd1f7b0d83f9baaa20c01109e8c0e4bd43c20b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 14:36:45 2014 +0000

    Make synth {} work in vcl_backend_response{}

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index a6a126d..a0eb015 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -182,6 +182,26 @@ sub vcl_backend_response {
 }
 
 sub vcl_backend_error {
+    set beresp.http.Content-Type = "text/html; charset=utf-8";
+    set beresp.http.Retry-After = "5";
+    synthetic {"
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html>
+  <head>
+    <title>"} + beresp.status + " " + beresp.reason + {"</title>
+  </head>
+  <body>
+    <h1>Error "} + beresp.status + " " + beresp.reason + {"</h1>
+    <p>"} + beresp.reason + {"</p>
+    <h3>Guru Meditation:</h3>
+    <p>XID: "} + bereq.xid + {"</p>
+    <hr>
+    <p>Varnish cache server</p>
+  </body>
+</html>
+"};
     return (deliver);
 }
 
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5b4028d..a954316 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -579,6 +579,8 @@ struct busyobj {
 
 	/* Workspace for object only needed during fetch */
 	struct ws		ws_o[1];
+
+	struct vsb		*synth_body;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b42674a..9368385 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -640,28 +640,33 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 {
+	struct storage *st;
+	ssize_t l;
+
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
+	AZ(bo->synth_body);
+	bo->synth_body = VSB_new_auto();
+	AN(bo->synth_body);
+
 	// XXX: reset all beresp flags ?
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
-	http_SetHeader(bo->beresp, "Content-Length: 0");
-	http_SetHeader(bo->beresp, "Connection: close");
 
+	bo->exp.t_origin = VTIM_real();
 	bo->exp.ttl = 0;
 	bo->exp.grace = 0;
 	bo->exp.keep = 0;
 
 	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
-	xxxassert(wrk->handling == VCL_RET_DELIVER);
+	AZ(VSB_finish(bo->synth_body));
 
-	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
-	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
+	xxxassert(wrk->handling == VCL_RET_DELIVER);
 
 	if (vbf_beresp2obj(bo)) {
 		VBO_setstate(bo, BOS_FAILED);
@@ -669,9 +674,29 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_DONE);
 	}
 
+	l = VSB_len(bo->synth_body);
+	if (l > 0) {
+		st = VFP_GetStorage(bo, l);
+		if (st != NULL) {
+			if (st->space < l) {
+				VSLb(bo->vsl, SLT_Error,
+				    "No space for %zd bytes of synth body", l);
+			} else {
+				memcpy(st->ptr, VSB_data(bo->synth_body), l);
+				st->len = l;
+				VBO_extend(bo, l);
+			}
+		}
+	}
+	VSB_delete(bo->synth_body);
+	bo->synth_body = NULL;
+
 	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
+
 	VBO_setstate(bo, BOS_FINISHED);
+
 	HSH_Complete(bo->fetch_obj->objcore);
+
 	return (F_STP_DONE);
 }
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index f78f140..95a37b0 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -377,9 +377,14 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 
 	(void)flags;
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
-	vsb = SMS_Makesynth(ctx->req->obj);
+	if (ctx->method == VCL_MET_BACKEND_ERROR) {
+		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+		vsb = ctx->bo->synth_body;
+	} else {
+		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+		CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
+		vsb = SMS_Makesynth(ctx->req->obj);
+	}
 	AN(vsb);
 
 	va_start(ap, str);
@@ -391,8 +396,10 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	SMS_Finish(ctx->req->obj);
-	http_Unset(ctx->req->obj->http, H_Content_Length);
+	if (ctx->method != VCL_MET_BACKEND_ERROR) {
+		SMS_Finish(ctx->req->obj);
+		http_Unset(ctx->req->obj->http, H_Content_Length);
+	}
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 2c79c0e..b6e2835 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -466,7 +466,7 @@ VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0, ctx->bo->exp.t_origin,)
 VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0, ctx->bo->exp.t_origin,)
 
 /*--------------------------------------------------------------------
- * req.xid
+ * [be]req.xid
  */
 
 const char *
@@ -480,6 +480,17 @@ VRT_r_req_xid(const struct vrt_ctx *ctx)
 	    ctx->req->vsl->wid & VSL_IDENTMASK));
 }
 
+const char *
+VRT_r_bereq_xid(const struct vrt_ctx *ctx)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+
+	return (WS_Printf(ctx->bo->bereq->ws, "%u",
+	    ctx->bo->vsl->wid & VSL_IDENTMASK));
+}
+
 /*--------------------------------------------------------------------*/
 
 #define REQ_BOOL(hash_var)					\
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 8c9461a..c3f26ff 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -296,6 +296,13 @@ The client's IP address.
 		always (re)fetch from the backend.
 		"""
 	),
+	('bereq.xid',
+		'STRING',
+		( 'backend',),
+		( ), """
+		Unique ID of this request.
+		"""
+	),
 	('bereq.retries',
 		'INT',
 		( 'backend',),
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 35f88f8..0026ad4 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -421,7 +421,8 @@ static struct action_table {
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },
-	{ "synthetic",		parse_synthetic, VCL_MET_ERROR },
+	{ "synthetic",		parse_synthetic,
+		VCL_MET_ERROR | VCL_MET_BACKEND_ERROR },
 	{ "unset",		parse_unset },
 	{ NULL,			NULL }
 };



From phk at FreeBSD.org  Mon Mar 10 20:00:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 10 Mar 2014 21:00:55 +0100
Subject: [master] 5cf517e Change the way we build synth content for
 vcl_error{}.
Message-ID: <E1WN6Nb-00012O-KG@project.varnish-software.com>


commit 5cf517e858b212a37a106a8ae78ffabe13a0dd5b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 19:59:24 2014 +0000

    Change the way we build synth content for vcl_error{}.
    
    This eliminates the need for the SMS stevedore.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 8cd8cf5..d9a8e62 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -78,7 +78,6 @@ varnishd_SOURCES = \
 	storage/storage_persistent_mgt.c \
 	storage/storage_persistent_silo.c \
 	storage/storage_persistent_subr.c \
-	storage/storage_synth.c \
 	storage/storage_umem.c \
 	waiter/mgt_waiter.c \
 	waiter/cache_waiter.c \
diff --git a/bin/varnishd/Makefile.phk b/bin/varnishd/Makefile.phk
index 1cbd5c2..f8bf48a 100644
--- a/bin/varnishd/Makefile.phk
+++ b/bin/varnishd/Makefile.phk
@@ -73,7 +73,6 @@ PROG_SRC += storage/storage_persistent.c
 PROG_SRC += storage/storage_persistent_mgt.c
 PROG_SRC += storage/storage_persistent_silo.c
 PROG_SRC += storage/storage_persistent_subr.c
-PROG_SRC += storage/storage_synth.c
 PROG_SRC += storage/storage_umem.c
 
 PROG_SRC += waiter/cache_waiter.c
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a954316..e0f45c9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -722,6 +722,9 @@ struct req {
 
 	/* Temporary accounting */
 	struct acct		acct_req;
+
+	/* Synth content in vcl_error */
+	struct vsb		*synth_body;
 };
 
 /*--------------------------------------------------------------------
@@ -1226,11 +1229,6 @@ int STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len);
 void STV_BanExport(const uint8_t *bans, unsigned len);
 struct storage *STV_alloc_transient(size_t size);
 
-/* storage_synth.c */
-struct vsb *SMS_Makesynth(struct object *obj);
-void SMS_Finish(struct object *obj);
-void SMS_Init(void);
-
 /* storage_persistent.c */
 void SMP_Init(void);
 void SMP_Ready(void);
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 8e116c9..a5674c0 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -226,7 +226,6 @@ child_main(void)
 
 	VCA_Init();
 
-	SMS_Init();
 	SMP_Init();
 	STV_open();
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8696dad..f5ae189 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -190,6 +190,8 @@ cnt_error(struct worker *wrk, struct req *req)
 	struct http *h;
 	struct busyobj *bo;
 	char date[40];
+	ssize_t l;
+	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -205,7 +207,6 @@ cnt_error(struct worker *wrk, struct req *req)
 	    TRANSIENT_STORAGE, cache_param->http_resp_size,
 	    (uint16_t)cache_param->http_max_hdr);
 	req->obj = bo->fetch_obj;
-	bo->stats = NULL;
 	if (req->obj == NULL) {
 		req->doclose = SC_OVERLOAD;
 		req->director_hint = NULL;
@@ -213,6 +214,7 @@ cnt_error(struct worker *wrk, struct req *req)
 		bo->fetch_objcore = NULL;
 		http_Teardown(bo->beresp);
 		http_Teardown(bo->bereq);
+		bo->stats = NULL;
 		VBO_DerefBusyObj(wrk, &bo);
 		return (REQ_FSM_DONE);
 	}
@@ -239,13 +241,25 @@ cnt_error(struct worker *wrk, struct req *req)
 		http_PutResponse(h, req->err_reason);
 	else
 		http_PutResponse(h, http_StatusMessage(req->err_code));
+
+	AZ(req->synth_body);
+	req->synth_body = VSB_new_auto();
+	AN(req->synth_body);
+
 	VCL_error_method(req->vcl, wrk, req, NULL, req->http->ws);
 
+	http_Unset(req->obj->http, H_Content_Length);
+
+	AZ(VSB_finish(req->synth_body));
+
 	/* Stop the insanity before it turns "Hotel California" on us */
 	if (req->restarts >= cache_param->max_restarts)
 		wrk->handling = VCL_RET_DELIVER;
 
 	if (wrk->handling == VCL_RET_RESTART) {
+		VSB_delete(req->synth_body);
+		req->synth_body = NULL;
+		bo->stats = NULL;
 		VBO_DerefBusyObj(wrk, &bo);
 		HSH_Drop(wrk, &req->obj);
 		req->req_step = R_STP_RESTART;
@@ -258,10 +272,28 @@ cnt_error(struct worker *wrk, struct req *req)
 	req->wantbody = 1;
 
 	assert(wrk->handling == VCL_RET_DELIVER);
+
+	l = VSB_len(req->synth_body);
+	if (l > 0) {
+		st = STV_alloc(bo, l);
+		if (st != NULL) {
+			VTAILQ_INSERT_TAIL(&req->obj->store, st, list);
+			if (st->space >= l) {
+				memcpy(st->ptr, VSB_data(req->synth_body), l);
+				st->len = l;
+				req->obj->len = l;
+			}
+		}
+	}
+
+	VSB_delete(req->synth_body);
+	req->synth_body = NULL;
+
 	req->err_code = 0;
 	req->err_reason = NULL;
 	http_Teardown(bo->bereq);
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	bo->stats = NULL;
 	VBO_DerefBusyObj(wrk, &bo);
 	req->req_step = R_STP_DELIVER;
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 95a37b0..c423eb2 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -369,21 +369,19 @@ VRT_Rollback(const struct vrt_ctx *ctx)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
+VRT_synth_page(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
 	const char *p;
 	struct vsb *vsb;
 
-	(void)flags;
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (ctx->method == VCL_MET_BACKEND_ERROR) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 		vsb = ctx->bo->synth_body;
 	} else {
 		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-		CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
-		vsb = SMS_Makesynth(ctx->req->obj);
+		vsb = ctx->req->synth_body;
 	}
 	AN(vsb);
 
@@ -396,10 +394,6 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	if (ctx->method != VCL_MET_BACKEND_ERROR) {
-		SMS_Finish(ctx->req->obj);
-		http_Unset(ctx->req->obj->http, H_Content_Length);
-	}
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage/storage_synth.c b/bin/varnishd/storage/storage_synth.c
deleted file mode 100644
index ba3fa24..0000000
--- a/bin/varnishd/storage/storage_synth.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Storage method for synthetic content, based on vsb.
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache/cache.h"
-#include "storage/storage.h"
-
-
-static struct lock		sms_mtx;
-
-static void
-sms_free(struct storage *sto)
-{
-
-	CHECK_OBJ_NOTNULL(sto, STORAGE_MAGIC);
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nobj--;
-	VSC_C_main->sms_nbytes -= sto->len;
-	VSC_C_main->sms_bfree += sto->len;
-	Lck_Unlock(&sms_mtx);
-	VSB_delete(sto->priv);
-	free(sto);
-}
-
-void
-SMS_Init(void)
-{
-
-	Lck_New(&sms_mtx, lck_sms);
-}
-
-static struct stevedore sms_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"synth",
-	.free	=	sms_free,
-};
-
-struct vsb *
-SMS_Makesynth(struct object *obj)
-{
-	struct storage *sto;
-	struct vsb *vsb;
-
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	STV_Freestore(obj);
-	obj->len = 0;
-
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nreq++;
-	VSC_C_main->sms_nobj++;
-	Lck_Unlock(&sms_mtx);
-
-	sto = calloc(sizeof *sto, 1);
-	XXXAN(sto);
-	vsb = VSB_new_auto();
-	XXXAN(vsb);
-	sto->priv = vsb;
-	sto->len = 0;
-	sto->space = 0;
-	sto->stevedore = &sms_stevedore;
-	sto->magic = STORAGE_MAGIC;
-
-	VTAILQ_INSERT_TAIL(&obj->store, sto, list);
-	return (vsb);
-}
-
-void
-SMS_Finish(struct object *obj)
-{
-	struct storage *sto;
-	struct vsb *vsb;
-
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	sto = VTAILQ_FIRST(&obj->store);
-	assert(sto->stevedore == &sms_stevedore);
-	vsb = sto->priv;
-	AZ(VSB_finish(vsb));
-
-	sto->ptr = (void*)VSB_data(vsb);
-	sto->len = VSB_len(vsb);
-	sto->space = VSB_len(vsb);
-	obj->len = sto->len;
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nbytes += sto->len;
-	VSC_C_main->sms_balloc += sto->len;
-	Lck_Unlock(&sms_mtx);
-}
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index 633e6e1..0912e64 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -7,12 +7,18 @@ varnish v1 -vcl {
 	sub vcl_recv {
 		return (error(888));
 	}
+
+	sub vcl_error {
+		synthetic "Custom vcl_error's synth output";
+		return (deliver);
+	}
 } -start
 
 client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 888
+	expect resp.bodylen == 31
 	expect resp.http.connection == close
 } -run
 
diff --git a/include/vrt.h b/include/vrt.h
index dc84852..8e5acba 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -222,7 +222,7 @@ void VRT_memmove(void *dst, const void *src, unsigned len);
 void VRT_Rollback(const struct vrt_ctx *);
 
 /* Synthetic pages */
-void VRT_synth_page(const struct vrt_ctx *, unsigned flags, const char *, ...);
+void VRT_synth_page(const struct vrt_ctx *, const char *, ...);
 
 /* Backend related */
 void VRT_init_dir(struct cli *, struct director **, int idx, const void *priv);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 0026ad4..ff5f0f4 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -396,7 +396,7 @@ parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_synth_page(ctx, 0, ");
+	Fb(tl, 1, "VRT_synth_page(ctx, ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");



From phk at FreeBSD.org  Mon Mar 10 20:27:59 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 10 Mar 2014 21:27:59 +0100
Subject: [master] 2eabf10 Require the "synthetic" argument to be function-like:
Message-ID: <E1WN6nn-0004sf-Ky@project.varnish-software.com>


commit 2eabf10ffa3b7d8c75bf01336f671e872a2832ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 20:27:12 2014 +0000

    Require the "synthetic" argument to be function-like:
    
    	synthetic ( STRING_LIST ) ;

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index a0eb015..96d74ea 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -137,7 +137,7 @@ sub vcl_deliver {
 sub vcl_error {
     set obj.http.Content-Type = "text/html; charset=utf-8";
     set obj.http.Retry-After = "5";
-    synthetic {"
+    synthetic( {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
@@ -154,7 +154,7 @@ sub vcl_error {
     <p>Varnish cache server</p>
   </body>
 </html>
-"};
+"} );
     return (deliver);
 }
 
@@ -184,7 +184,7 @@ sub vcl_backend_response {
 sub vcl_backend_error {
     set beresp.http.Content-Type = "text/html; charset=utf-8";
     set beresp.http.Retry-After = "5";
-    synthetic {"
+    synthetic( {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
@@ -201,7 +201,7 @@ sub vcl_backend_error {
     <p>Varnish cache server</p>
   </body>
 </html>
-"};
+"} );
     return (deliver);
 }
 
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index 0912e64..faac1e8 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_error {
-		synthetic "Custom vcl_error's synth output";
+		synthetic("Custom vcl_error's synth output");
 		return (deliver);
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00936.vtc b/bin/varnishtest/tests/r00936.vtc
index 99e1d65..7e6ad64 100644
--- a/bin/varnishtest/tests/r00936.vtc
+++ b/bin/varnishtest/tests/r00936.vtc
@@ -9,7 +9,8 @@ varnish v1 -errvcl {'synthetic': not a valid action in method 'vcl_recv'} {
 	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_recv {
-		synthetic "HELLOO"; return (error(503));
+		synthetic("HELLOO");
+		return (error(503));
 	}
 }
 
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
index 8560ce3..fc1ebd2 100644
--- a/bin/varnishtest/tests/r01287.vtc
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -8,7 +8,7 @@ varnish v1 -vcl+backend {
 		return (error(200, "OK"));
 	}
 	sub vcl_error {
-		synthetic obj.http.blank;
+		synthetic(obj.http.blank);
 		return (deliver);
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 720c816..9e97762 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -100,7 +100,7 @@ varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
 
 varnish v1 -errvcl {Unknown token '<<' when looking for STRING_LIST} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_error { synthetic << "foo"; }
+	sub vcl_error { synthetic( << "foo"; }
 }
 
 varnish v1 -errvcl {Syntax has changed, use:} {
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index ff5f0f4..4e3f12c 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -396,10 +396,18 @@ parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
 
+	ExpectErr(tl, '(');
+	ERRCHK(tl);
+	vcc_NextToken(tl);
+
 	Fb(tl, 1, "VRT_synth_page(ctx, ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");
+
+	ExpectErr(tl, ')');
+	vcc_NextToken(tl);
+	ERRCHK(tl);
 }
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Tue Mar 11 09:08:14 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 11 Mar 2014 10:08:14 +0100
Subject: [master] 33d80ab Implement "return(retry)" in vcl_backend_error{}
Message-ID: <E1WNIfW-0007Xg-JT@project.varnish-software.com>


commit 33d80aba55165ea35af1a182b2b6a45303075be0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 09:07:53 2014 +0000

    Implement "return(retry)" in vcl_backend_error{}

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9368385..c3cefac 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -666,7 +666,20 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	AZ(VSB_finish(bo->synth_body));
 
-	xxxassert(wrk->handling == VCL_RET_DELIVER);
+	if (wrk->handling == VCL_RET_RETRY) {
+		VSB_delete(bo->synth_body);
+		bo->synth_body = NULL;
+	    	if (bo->retries++ < cache_param->max_retries) {
+			return (F_STP_RETRY);
+		}
+		bo->synth_body = NULL;
+		HSH_Fail(bo->fetch_objcore);
+		VBO_setstate(bo, BOS_FAILED);
+		HSH_Complete(bo->fetch_objcore);
+		return (F_STP_DONE);
+	}
+
+	assert(wrk->handling == VCL_RET_DELIVER);
 
 	if (vbf_beresp2obj(bo)) {
 		VBO_setstate(bo, BOS_FAILED);
diff --git a/bin/varnishtest/tests/c00061.vtc b/bin/varnishtest/tests/c00061.vtc
new file mode 100644
index 0000000..ecb01a4
--- /dev/null
+++ b/bin/varnishtest/tests/c00061.vtc
@@ -0,0 +1,24 @@
+varnishtest "retry in vcl_backend_error"
+
+varnish v1 -vcl {
+
+	backend b1 { .host = "${bad_ip}"; }
+
+	sub vcl_backend_error {
+		return (retry);
+	}
+
+	sub vcl_error {
+		set obj.status = 504;
+	}
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set connect_timeout 1"
+varnish v1 -cliok "param.set max_retries 2"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 504
+} -run



From fgsch at lodoss.net  Tue Mar 11 09:51:28 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 11 Mar 2014 10:51:28 +0100
Subject: [master] 6b2faff Remove user-guides/cookies.rst from here as well
Message-ID: <E1WNJLM-00053e-2r@project.varnish-software.com>


commit 6b2faff67e21d6ff4cf2a8e75a243bbf8d537d6d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 09:50:47 2014 +0000

    Remove user-guides/cookies.rst from here as well

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index ac09cc0..3a6bb2b 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -138,7 +138,6 @@ EXTRA_DIST = \
 	tutorial/now_what.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
-	users-guide/cookies.rst \
 	users-guide/devicedetection.rst \
 	users-guide/esi.rst \
 	users-guide/increasing-your-hitrate.rst \
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index fea357f..d89f4e5 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -147,7 +147,6 @@ EXTRA_DIST = \
 	tutorial/now_what.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
-	users-guide/cookies.rst \
 	users-guide/devicedetection.rst \
 	users-guide/esi.rst \
 	users-guide/increasing-your-hitrate.rst \



From fgsch at lodoss.net  Tue Mar 11 10:20:33 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 11 Mar 2014 11:20:33 +0100
Subject: [master] 03ff391 Remove more nonexistent rst files to please make
 dist and jenkins
Message-ID: <E1WNJnV-0001HI-Fl@project.varnish-software.com>


commit 03ff3917bff138419484fa27e6c53cf274824e60
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 10:19:26 2014 +0000

    Remove more nonexistent rst files to please make dist and jenkins

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 3a6bb2b..f36f569 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -153,7 +153,6 @@ EXTRA_DIST = \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
-	users-guide/vary.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
 	users-guide/vcl-examples.rst \
@@ -162,8 +161,7 @@ EXTRA_DIST = \
 	users-guide/vcl.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst \
-	users-guide/websockets.rst
+	users-guide/vcl-variables.rst
 
 dist-hook:
 	$(MAKE) html
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index d89f4e5..611ff18 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -163,7 +163,6 @@ EXTRA_DIST = \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
-	users-guide/vary.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
 	users-guide/vcl-examples.rst \
@@ -172,9 +171,7 @@ EXTRA_DIST = \
 	users-guide/vcl.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst \
-	users-guide/websockets.rst
-
+	users-guide/vcl-variables.rst
 
 dist-hook:
 	$(MAKE) html



From phk at FreeBSD.org  Tue Mar 11 10:50:08 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 11 Mar 2014 11:50:08 +0100
Subject: [master] 94e99cb Add a V1D_Deliver_Synth() for delivering synth
 vcl_error{} responses.
Message-ID: <E1WNKG8-0006XN-Kh@project.varnish-software.com>


commit 94e99cb511c5d50108636197cb0001d5b9b32c92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 10:04:01 2014 +0000

    Add a V1D_Deliver_Synth() for delivering synth vcl_error{} responses.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e0f45c9..94fdf32 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -855,6 +855,8 @@ int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
 
 /* cache_http1_deliver.c */
 void V1D_Deliver(struct req *);
+void V1D_Deliver_Synth(struct req *req);
+
 
 static inline int
 VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len)
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index c9f373a..5a3da11 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -271,7 +271,7 @@ V1D_Deliver(struct req *req)
 	    req->wantbody &&
 	    !(req->res_mode & (RES_ESI|RES_ESI_CHILD)) &&
 	    cache_param->http_range_support &&
-	    http_GetStatus(req->obj->http) == 200) {
+	    http_GetStatus(req->resp) == 200) {
 		http_SetHeader(req->resp, "Accept-Ranges: bytes");
 		if (http_GetHdr(req->http, H_Range, &r))
 			v1d_dorange(req, r);
@@ -318,3 +318,101 @@ V1D_Deliver(struct req *req)
 	if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0)
 		SES_Close(req->sp, SC_REM_CLOSE);
 }
+
+void
+V1D_Deliver_Synth(struct req *req)
+{
+	char *r;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(req->obj);
+	AN(req->synth_body);
+
+	req->res_mode = 0;
+	if (req->resp->status == 304) {
+		req->res_mode &= ~RES_LEN;
+		http_Unset(req->resp, H_Content_Length);
+		req->wantbody = 0;
+	} else {
+		req->res_mode |= RES_LEN;
+		http_Unset(req->resp, H_Content_Length);
+		http_PrintfHeader(req->resp, "Content-Length: %zd",
+		    VSB_len(req->synth_body));
+	}
+
+	if (req->esi_level > 0) {
+		/* Included ESI object, always CHUNKED or EOF */
+		req->res_mode &= ~RES_LEN;
+		req->res_mode |= RES_ESI_CHILD;
+	}
+
+	if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
+		/* We havn't chosen yet, do so */
+		if (!req->wantbody) {
+			/* Nothing */
+		} else if (req->http->protover >= 11) {
+			req->res_mode |= RES_CHUNKED;
+		} else {
+			req->res_mode |= RES_EOF;
+			req->doclose = SC_TX_EOF;
+		}
+	}
+	VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
+
+	if (!(req->res_mode & RES_LEN))
+		http_Unset(req->resp, H_Content_Length);
+
+	if (req->res_mode & RES_GUNZIP)
+		http_Unset(req->resp, H_Content_Encoding);
+
+	if (req->res_mode & RES_CHUNKED)
+		http_SetHeader(req->resp, "Transfer-Encoding: chunked");
+
+	http_SetHeader(req->resp,
+	    req->doclose ? "Connection: close" : "Connection: keep-alive");
+
+	req->vdps[0] = v1d_bytes;
+	req->vdp_nxt = 0;
+
+	if (
+	    req->wantbody &&
+	    !(req->res_mode & RES_ESI_CHILD) &&
+	    cache_param->http_range_support &&
+	    http_GetStatus(req->resp) == 200) {
+		http_SetHeader(req->resp, "Accept-Ranges: bytes");
+		if (http_GetHdr(req->http, H_Range, &r))
+			v1d_dorange(req, r);
+	}
+
+	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp);
+
+	/*
+	 * Send HTTP protocol header, unless interior ESI object
+	 */
+	if (!(req->res_mode & RES_ESI_CHILD))
+		req->acct_req.hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+
+	if (req->res_mode & RES_CHUNKED)
+		WRW_Chunked(req->wrk);
+
+	if (!req->wantbody) {
+		/* This was a HEAD or conditional request */
+#if 0
+	XXX: Missing pretend GZIP for esi-children
+	} else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) {
+		while (req->obj->objcore->busyobj)
+			(void)usleep(10000);
+		ESI_DeliverChild(req);
+#endif
+	} else {
+		(void)VDP_bytes(req, VDP_FLUSH, VSB_data(req->synth_body),
+		    VSB_len(req->synth_body));
+		(void)VDP_bytes(req, VDP_FINISH,  NULL, 0);
+	}
+
+	if (req->res_mode & RES_CHUNKED && !(req->res_mode & RES_ESI_CHILD))
+		WRW_EndChunk(req->wrk);
+
+	if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0)
+		SES_Close(req->sp, SC_REM_CLOSE);
+}



From phk at FreeBSD.org  Tue Mar 11 10:50:08 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 11 Mar 2014 11:50:08 +0100
Subject: [master] a0e68b3 Change vcl_error{} to do a direct response without
 going the detour over a bogo-object.
Message-ID: <E1WNKG8-0006XY-M5@project.varnish-software.com>


commit a0e68b31df6eeba8634e1a436cccb176e889a33f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 10:48:53 2014 +0000

    Change vcl_error{} to do a direct response without going the detour
    over a bogo-object.
    
    This means that vcl_error{} looses access to obj.* variables, but
    gains access to resp.* vaiables instead.
    
    A couple of now obsolute test-cases retired.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 96d74ea..da30346 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -135,19 +135,19 @@ sub vcl_deliver {
  * We can come here "invisibly" with the following errors: 413, 417 & 503
  */
 sub vcl_error {
-    set obj.http.Content-Type = "text/html; charset=utf-8";
-    set obj.http.Retry-After = "5";
+    set resp.http.Content-Type = "text/html; charset=utf-8";
+    set resp.http.Retry-After = "5";
     synthetic( {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
   <head>
-    <title>"} + obj.status + " " + obj.reason + {"</title>
+    <title>"} + resp.status + " " + resp.reason + {"</title>
   </head>
   <body>
-    <h1>Error "} + obj.status + " " + obj.reason + {"</h1>
-    <p>"} + obj.reason + {"</p>
+    <h1>Error "} + resp.status + " " + resp.reason + {"</h1>
+    <p>"} + resp.reason + {"</p>
     <h3>Guru Meditation:</h3>
     <p>XID: "} + req.xid + {"</p>
     <hr>
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c3cefac..3aabd8d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -669,9 +669,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		VSB_delete(bo->synth_body);
 		bo->synth_body = NULL;
-	    	if (bo->retries++ < cache_param->max_retries) {
+		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
-		}
 		bo->synth_body = NULL;
 		HSH_Fail(bo->fetch_objcore);
 		VBO_setstate(bo, BOS_FAILED);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f5ae189..467ea8d 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -187,56 +187,29 @@ DOT }
 static enum req_fsm_nxt
 cnt_error(struct worker *wrk, struct req *req)
 {
-	struct http *h;
-	struct busyobj *bo;
 	char date[40];
-	ssize_t l;
-	struct storage *st;
+	struct http *h;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AZ(req->objcore);
-	AZ(req->obj);
 
 	req->acct_req.error++;
-	bo = VBO_GetBusyObj(wrk, req);
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
-	bo->fetch_objcore = HSH_Private(wrk);
-	bo->fetch_obj = STV_NewObject(bo,
-	    TRANSIENT_STORAGE, cache_param->http_resp_size,
-	    (uint16_t)cache_param->http_max_hdr);
-	req->obj = bo->fetch_obj;
-	if (req->obj == NULL) {
-		req->doclose = SC_OVERLOAD;
-		req->director_hint = NULL;
-		AZ(HSH_DerefObjCore(&wrk->stats, &bo->fetch_objcore));
-		bo->fetch_objcore = NULL;
-		http_Teardown(bo->beresp);
-		http_Teardown(bo->bereq);
-		bo->stats = NULL;
-		VBO_DerefBusyObj(wrk, &bo);
-		return (REQ_FSM_DONE);
-	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	AZ(req->objcore);
-	req->obj->vxid = bo->vsl->wid;
-	req->obj->exp.t_origin = req->t_req;
 
-	h = req->obj->http;
+	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
+	h = req->resp;
+	req->t_resp = VTIM_real();
 
 	if (req->err_code < 100 || req->err_code > 999)
 		req->err_code = 501;
 
+	http_ClrHeader(h);
 	http_PutProtocol(h, "HTTP/1.1");
 	http_PutStatus(h, req->err_code);
-	VTIM_format(W_TIM_real(wrk), date);
+	VTIM_format(req->t_resp, date);
 	http_PrintfHeader(h, "Date: %s", date);
 	http_SetHeader(h, "Server: Varnish");
-
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	HSH_Ref(req->obj->objcore);
-
+	http_PrintfHeader(req->resp,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 	if (req->err_reason != NULL)
 		http_PutResponse(h, req->err_reason);
 	else
@@ -248,56 +221,27 @@ cnt_error(struct worker *wrk, struct req *req)
 
 	VCL_error_method(req->vcl, wrk, req, NULL, req->http->ws);
 
-	http_Unset(req->obj->http, H_Content_Length);
+	http_Unset(h, H_Content_Length);
 
 	AZ(VSB_finish(req->synth_body));
 
-	/* Stop the insanity before it turns "Hotel California" on us */
-	if (req->restarts >= cache_param->max_restarts)
-		wrk->handling = VCL_RET_DELIVER;
-
 	if (wrk->handling == VCL_RET_RESTART) {
+		http_ClrHeader(h);
 		VSB_delete(req->synth_body);
 		req->synth_body = NULL;
-		bo->stats = NULL;
-		VBO_DerefBusyObj(wrk, &bo);
-		HSH_Drop(wrk, &req->obj);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-
-	/* We always close when we take this path */
-	req->doclose = SC_TX_ERROR;
-	req->wantbody = 1;
-
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	l = VSB_len(req->synth_body);
-	if (l > 0) {
-		st = STV_alloc(bo, l);
-		if (st != NULL) {
-			VTAILQ_INSERT_TAIL(&req->obj->store, st, list);
-			if (st->space >= l) {
-				memcpy(st->ptr, VSB_data(req->synth_body), l);
-				st->len = l;
-				req->obj->len = l;
-			}
-		}
-	}
+	V1D_Deliver_Synth(req);
 
 	VSB_delete(req->synth_body);
 	req->synth_body = NULL;
 
 	req->err_code = 0;
 	req->err_reason = NULL;
-	http_Teardown(bo->bereq);
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	bo->stats = NULL;
-	VBO_DerefBusyObj(wrk, &bo);
-	req->req_step = R_STP_DELIVER;
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	return (REQ_FSM_MORE);
+	return (REQ_FSM_DONE);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b6e2835..8a999c9 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -62,7 +62,7 @@ vrt_do_string(const struct http *hp, int fld,
 	va_end(ap);
 }
 
-#define VRT_DO_HDR(obj, hdr, fld)					\
+#define VRT_HDR_L(obj, hdr, fld)					\
 void									\
 VRT_l_##obj##_##hdr(const struct vrt_ctx *ctx, const char *p, ...)	\
 {									\
@@ -72,8 +72,9 @@ VRT_l_##obj##_##hdr(const struct vrt_ctx *ctx, const char *p, ...)	\
 	va_start(ap, p);						\
 	vrt_do_string(ctx->http_##obj, fld, #obj "." #hdr, p, ap);	\
 	va_end(ap);							\
-}									\
-									\
+}
+
+#define VRT_HDR_R(obj, hdr, fld)					\
 const char *								\
 VRT_r_##obj##_##hdr(const struct vrt_ctx *ctx)				\
 {									\
@@ -82,7 +83,11 @@ VRT_r_##obj##_##hdr(const struct vrt_ctx *ctx)				\
 	return (ctx->http_##obj->hd[fld].b);				\
 }
 
-#define VRT_DO_STATUS(obj)						\
+#define VRT_HDR_LR(obj, hdr, fld)					\
+	VRT_HDR_L(obj, hdr, fld)					\
+	VRT_HDR_R(obj, hdr, fld)
+
+#define VRT_STATUS_L(obj)						\
 void									\
 VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 {									\
@@ -91,8 +96,9 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 	CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC);			\
 	assert(num >= 100 && num <= 999);				\
 	ctx->http_##obj->status = (uint16_t)num;			\
-}									\
-									\
+}
+
+#define VRT_STATUS_R(obj)						\
 long									\
 VRT_r_##obj##_status(const struct vrt_ctx *ctx)				\
 {									\
@@ -102,22 +108,26 @@ VRT_r_##obj##_status(const struct vrt_ctx *ctx)				\
 	return(ctx->http_##obj->status);				\
 }
 
-VRT_DO_HDR(req,    method,	HTTP_HDR_METHOD)
-VRT_DO_HDR(req,    url,		HTTP_HDR_URL)
-VRT_DO_HDR(req,    proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,    proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,    reason,	HTTP_HDR_RESPONSE)
-VRT_DO_STATUS(obj)
-VRT_DO_HDR(resp,   proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(resp,   reason,	HTTP_HDR_RESPONSE)
-VRT_DO_STATUS(resp)
-
-VRT_DO_HDR(bereq,  method,	HTTP_HDR_METHOD)
-VRT_DO_HDR(bereq,  url,		HTTP_HDR_URL)
-VRT_DO_HDR(bereq,  proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp, proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp, reason,	HTTP_HDR_RESPONSE)
-VRT_DO_STATUS(beresp)
+VRT_HDR_LR(req,    method,	HTTP_HDR_METHOD)
+VRT_HDR_LR(req,    url,		HTTP_HDR_URL)
+VRT_HDR_LR(req,    proto,	HTTP_HDR_PROTO)
+
+VRT_HDR_R(obj,    proto,	HTTP_HDR_PROTO)
+VRT_HDR_R(obj,    reason,	HTTP_HDR_RESPONSE)
+VRT_STATUS_R(obj)
+
+VRT_HDR_LR(resp,   proto,	HTTP_HDR_PROTO)
+VRT_HDR_LR(resp,   reason,	HTTP_HDR_RESPONSE)
+VRT_STATUS_L(resp)
+VRT_STATUS_R(resp)
+
+VRT_HDR_LR(bereq,  method,	HTTP_HDR_METHOD)
+VRT_HDR_LR(bereq,  url,		HTTP_HDR_URL)
+VRT_HDR_LR(bereq,  proto,	HTTP_HDR_PROTO)
+VRT_HDR_LR(beresp, proto,	HTTP_HDR_PROTO)
+VRT_HDR_LR(beresp, reason,	HTTP_HDR_RESPONSE)
+VRT_STATUS_L(beresp)
+VRT_STATUS_R(beresp)
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index faac1e8..dbe673b 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -1,4 +1,4 @@
-varnishtest "Check that we close one error"
+varnishtest "Check that we do not close one error"
 
 varnish v1 -vcl {
 	backend foo {
@@ -19,7 +19,7 @@ client c1 {
 	rxresp
 	expect resp.status == 888
 	expect resp.bodylen == 31
-	expect resp.http.connection == close
+	expect resp.http.connection != close
 } -run
 
 varnish v1 -expect s_error == 1
diff --git a/bin/varnishtest/tests/b00019.vtc b/bin/varnishtest/tests/b00019.vtc
index 597e8b9..59ceb61 100644
--- a/bin/varnishtest/tests/b00019.vtc
+++ b/bin/varnishtest/tests/b00019.vtc
@@ -27,14 +27,14 @@ varnish v1 -vcl+backend {
 
 	sub vcl_error {
 		if (req.restarts == 2) {
-			set obj.status = 200;
-			set obj.reason = "restart=2";
+			set resp.status = 200;
+			set resp.reason = "restart=2";
 		} elsif (req.restarts > 2) {
-			set obj.status = 501;
-			set obj.reason = "restart>2";
+			set resp.status = 501;
+			set resp.reason = "restart>2";
 		} elsif (req.restarts < 2) {
-			set obj.status = 500;
-			set obj.reason = "restart<2";
+			set resp.status = 500;
+			set resp.reason = "restart<2";
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc
index 8b945e1..a7b6c57 100644
--- a/bin/varnishtest/tests/b00030.vtc
+++ b/bin/varnishtest/tests/b00030.vtc
@@ -13,7 +13,7 @@ varnish v1 -vcl+backend {
 		return (error(200,req.ttl));
 	}
 	sub vcl_error {
-		set obj.http.x-timestamp = now;
+		set resp.http.x-timestamp = now;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00024.vtc b/bin/varnishtest/tests/c00024.vtc
index 2a1e542..6167914 100644
--- a/bin/varnishtest/tests/c00024.vtc
+++ b/bin/varnishtest/tests/c00024.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl+backend {
 		if (req.restarts < 1) {
 			return (restart);
 		} else {
-			set obj.status = 201;
+			set resp.status = 201;
 		}
 	 }
  } -start
diff --git a/bin/varnishtest/tests/c00061.vtc b/bin/varnishtest/tests/c00061.vtc
index ecb01a4..c99f324 100644
--- a/bin/varnishtest/tests/c00061.vtc
+++ b/bin/varnishtest/tests/c00061.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_error {
-		set obj.status = 504;
+		set resp.status = 504;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00310.vtc b/bin/varnishtest/tests/r00310.vtc
index 38c8a32..9d0c431 100644
--- a/bin/varnishtest/tests/r00310.vtc
+++ b/bin/varnishtest/tests/r00310.vtc
@@ -1,6 +1,6 @@
 varnishtest "Test obj.http.x-cache in vcl_hit"
 
-varnish v1 -errvcl {'obj.http.x-cache': cannot be set in method 'vcl_hit'.} {
+varnish v1 -errvcl {Variable 'obj.http.x-cache' is read only.} {
 	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_hit {
diff --git a/bin/varnishtest/tests/r00769.vtc b/bin/varnishtest/tests/r00769.vtc
index 838ef17..451029c 100644
--- a/bin/varnishtest/tests/r00769.vtc
+++ b/bin/varnishtest/tests/r00769.vtc
@@ -17,9 +17,9 @@ varnish v1 -vcl+backend {
 		return (deliver);
 	}
 	sub vcl_error {
-		if (obj.status == 700) {
-			set obj.status=404;
-			set obj.http.X-status = obj.status;
+		if (resp.status == 700) {
+			set resp.status=404;
+			set resp.http.X-status = resp.status;
 			return (deliver);
 		}
 	}
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index ccd01b5..d9cbdba 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -28,7 +28,7 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_error {
-		if (obj.status == 988) { return (restart); }
+		if (resp.status == 988) { return (restart); }
 	}
 
 	sub vcl_miss {
diff --git a/bin/varnishtest/tests/r01031.vtc b/bin/varnishtest/tests/r01031.vtc
deleted file mode 100644
index 27bbab2..0000000
--- a/bin/varnishtest/tests/r01031.vtc
+++ /dev/null
@@ -1,30 +0,0 @@
-varnishtest "Test overflowing the response through sp->err_reason"
-
-varnish v1 -vcl {
-	backend blatti {
-		.host = "127.0.0.1";
-	}
-
-	sub vcl_recv {
-		return (error(200,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
-	}
-	sub vcl_error {
-		return(deliver);
-	}
-} -start
-
-client c1 {
-	txreq -req GET
-	rxresp
-	expect resp.status == 200
-	expect resp.msg == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-} -run
-
-varnish v1 -cliok "param.set http_resp_size 256"
-
-client c2 {
-	txreq -req GET
-	rxresp
-	expect resp.status == 200
-	expect resp.msg == "Lost Response"
-} -run
diff --git a/bin/varnishtest/tests/r01283.vtc b/bin/varnishtest/tests/r01283.vtc
deleted file mode 100644
index 9b843ce..0000000
--- a/bin/varnishtest/tests/r01283.vtc
+++ /dev/null
@@ -1,48 +0,0 @@
-varnishtest "#1283 - Test failure to allocate object for error (transient full)"
-
-server s1 {
-	rxreq
-	txresp -bodylen 1048198
-} -start
-
-varnish v1 \
-	-arg "-p nuke_limit=0" \
-	-arg "-sTransient=malloc,1m" \
-	-vcl+backend {
-	sub vcl_recv {
-		if (req.http.x-do-error) {
-			return (error(500));
-		}
-	}
-	sub vcl_backend_response {
-		set beresp.do_stream = false;
-		set beresp.storage_hint = "Transient";
-	}
-} -start
-
-client c1 {
-	# Fill transient
-	txreq
-	rxresp
-	expect resp.status == 200
-} -run
-
-varnish v1 -expect SMA.Transient.g_bytes > 1048000
-varnish v1 -expect SMA.Transient.g_space < 200
-varnish v1 -expect SMA.Transient.c_fail == 0
-
-client c1 {
-	# Trigger vcl_error. Don't wait for reply as Varnish will not send one
-	# due to Transient full
-	txreq -hdr "X-Do-Error: true"
-	delay 1
-} -run
-
-varnish v1 -expect SMA.Transient.c_fail == 1
-
-client c1 {
-	# Check that Varnish is still alive
-	txreq
-	rxresp
-	expect resp.status == 200
-} -run
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 123ddb2..f89b579 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -42,8 +42,8 @@ client c1 {
 	delay 1
 } -run
 
-# Three failures, one for obj2, one for vcl_backend_error{} and for vcl_error{}
-varnish v1 -expect SMA.Transient.c_fail == 3
+# Three failures, one for obj2, one for vcl_backend_error{}
+varnish v1 -expect SMA.Transient.c_fail == 2
 
 client c1 {
 	# Check that Varnish is still alive
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
index fc1ebd2..7e329ac 100644
--- a/bin/varnishtest/tests/r01287.vtc
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -8,7 +8,7 @@ varnish v1 -vcl+backend {
 		return (error(200, "OK"));
 	}
 	sub vcl_error {
-		synthetic(obj.http.blank);
+		synthetic(resp.http.blank);
 		return (deliver);
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc
index 162c66a..3a3d842 100644
--- a/bin/varnishtest/tests/v00011.vtc
+++ b/bin/varnishtest/tests/v00011.vtc
@@ -52,6 +52,5 @@ logexpect l1 -wait
 client c1 {
 	txreq
 	rxresp
-	expect resp.http.X-Varnish == "1007"
-
+	expect resp.http.X-Varnish == "1006"
 } -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c3f26ff..48b0175 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -490,8 +490,8 @@ The client's IP address.
 	),
 	('obj.proto',
 		'STRING',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		The HTTP protocol version used when the object was retrieved.
 		"""
 	),
@@ -520,30 +520,30 @@ The client's IP address.
 	),
 	('obj.http.',
 		'HEADER',
-		( 'hit', 'error',),
-		( 'error',), """
+		( 'hit', ),
+		( ), """
 		The corresponding HTTP header.
 		"""
 	),
 	('obj.ttl',
 		'DURATION',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		The object's remaining time to live, in seconds.
 		obj.ttl is writable.
 		"""
 	),
 	('obj.grace',
 		'DURATION',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		The object's grace period in seconds. obj.grace is writable.
 		"""
 	),
 	('obj.keep',
 		'DURATION',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		"""
 	),
 	('obj.last_use',
@@ -557,35 +557,35 @@ The client's IP address.
 	),
 	('obj.uncacheable',
 		'BOOL',
-		( 'hit', 'deliver', 'error',),
+		( 'hit', ),
 		( ), """
 		"""
 	),
 	('resp.proto',
 		'STRING',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The HTTP protocol version to use for the response.
 		"""
 	),
 	('resp.status',
 		'INT',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The HTTP status code that will be returned.
 		"""
 	),
 	('resp.reason',
 		'STRING',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The HTTP status message that will be returned.
 		"""
 	),
 	('resp.http.',
 		'HEADER',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The corresponding HTTP header.
 		"""
 	),



From fgsch at lodoss.net  Tue Mar 11 13:50:13 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 11 Mar 2014 14:50:13 +0100
Subject: [master] 7bef976 Use `Symbol' instead of `Object'
Message-ID: <E1WNN4P-00005Q-3r@project.varnish-software.com>


commit 7bef97657858f646b0723bcd8a5f01a9d1496f94
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 11:59:38 2014 +0000

    Use `Symbol' instead of `Object'
    
    This matches the rest of the code.

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 4314dce..b42106d 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -99,7 +99,7 @@ varnish v1 -vcl {
 	sub vcl_hash { if (2 == 3) { } }
 }
 
-varnish v1 -errvcl {Object not found: 'foo.bar'} {
+varnish v1 -errvcl {Symbol not found: 'foo.bar'} {
 	sub vcl_init {
 		new bar = foo.bar();
 	}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 4e3f12c..5ed9146 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -191,7 +191,7 @@ parse_new(struct vcc *tl)
 	ExpectErr(tl, ID);
 	sy2 = VCC_FindSymbol(tl, tl->t, SYM_OBJECT);
 	if (sy2 == NULL) {
-		VSB_printf(tl->sb, "Object not found: ");
+		VSB_printf(tl->sb, "Symbol not found: ");
 		vcc_ErrToken(tl, tl->t);
 		VSB_printf(tl->sb, " at ");
 		vcc_ErrWhere(tl, tl->t);



From fgsch at lodoss.net  Tue Mar 11 13:50:13 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 11 Mar 2014 14:50:13 +0100
Subject: [master] 9e298b4 Sync with reality
Message-ID: <E1WNN4P-00005W-57@project.varnish-software.com>


commit 9e298b48ffaf7c4b7f58ac4c56ac9407a2d27405
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 13:46:08 2014 +0000

    Sync with reality
    
    Only varnishreplay is left now.

diff --git a/bin/Makefile.am b/bin/Makefile.am
index a6defb4..1f33d4e 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -2,8 +2,8 @@
 
 # Disabling building of the tools while api is in flux
 
-#SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishadm varnishd varnishlog varnishtest varnishncsa
+#SUBDIRS = varnishreplay
+SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishtest
 
 if VARNISH_CURSES
 SUBDIRS += varnishhist



From fgsch at lodoss.net  Tue Mar 11 13:50:13 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 11 Mar 2014 14:50:13 +0100
Subject: [master] 8e7d2f1 Update list of rst files
Message-ID: <E1WNN4P-00005f-66@project.varnish-software.com>


commit 8e7d2f19cda7e988c611eb4696fd405e37a0b095
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 13:47:03 2014 +0000

    Update list of rst files

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index f36f569..64e6020 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -96,46 +96,48 @@ EXTRA_DIST = \
 	installation/help.rst \
 	installation/index.rst \
 	installation/install.rst \
-	installation/prerequisites.rst \
 	installation/platformnotes.rst \
+	installation/prerequisites.rst \
 	phk/autocrap.rst \
 	phk/backends.rst \
 	phk/barriers.rst \
 	phk/gzip.rst \
+	phk/http20.rst \
 	phk/index.rst \
 	phk/ipv6suckage.rst \
 	phk/platforms.rst \
+	phk/spdy.rst \
 	phk/sphinx.rst \
 	phk/ssl.rst \
+	phk/thetoolsweworkwith.rst \
 	phk/thoughts.rst \
 	phk/three-zero.rst \
+	phk/varnish_does_not_hash.rst \
 	phk/vcl_expr.rst \
 	phk/wanton_destruction.rst \
 	reference/index.rst \
-	reference/varnishadm.rst \
 	reference/varnish-cli.rst \
+	reference/varnishadm.rst \
 	reference/varnishd.rst \
 	reference/varnishhist.rst \
 	reference/varnishlog.rst \
 	reference/varnishncsa.rst \
-	reference/varnishreplay.rst \
-	reference/varnishsizes.rst \
 	reference/varnishstat.rst \
 	reference/varnishtest.rst \
 	reference/varnishtop.rst \
-	reference/varnishhist.rst \
 	reference/vcl.rst \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
-	reference/vsm.rst \
 	reference/vsl-query.rst \
 	reference/vsl.rst \
+	reference/vsm.rst \
+	tutorial/backend_servers.rst \
 	tutorial/index.rst \
 	tutorial/introduction.rst \
-	tutorial/starting_varnish.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/backend_servers.rst \
 	tutorial/now_what.rst \
+	tutorial/peculiarities.rst \
+	tutorial/putting_varnish_on_port_80.rst \
+	tutorial/starting_varnish.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
 	users-guide/devicedetection.rst \
@@ -149,19 +151,27 @@ EXTRA_DIST = \
 	users-guide/performance.rst \
 	users-guide/purging.rst \
 	users-guide/report.rst \
+	users-guide/run_cli.rst \
+	users-guide/run_security.rst \
 	users-guide/running.rst \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
+	users-guide/vcl-built-in-subs.rst \
+	users-guide/vcl-example-acls.rst \
+	users-guide/vcl-example-manipulating-headers.rst \
+	users-guide/vcl-example-manipulating-responses.rst \
+	users-guide/vcl-example-websockets.rst \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl.rst \
+	users-guide/vcl-intro.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst
+	users-guide/vcl-variables.rst \
+	users-guide/vcl.rst
 
 dist-hook:
 	$(MAKE) html
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index 611ff18..fbfc35f 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -105,46 +105,48 @@ EXTRA_DIST = \
 	installation/help.rst \
 	installation/index.rst \
 	installation/install.rst \
-	installation/prerequisites.rst \
 	installation/platformnotes.rst \
+	installation/prerequisites.rst \
 	phk/autocrap.rst \
 	phk/backends.rst \
 	phk/barriers.rst \
 	phk/gzip.rst \
+	phk/http20.rst \
 	phk/index.rst \
 	phk/ipv6suckage.rst \
 	phk/platforms.rst \
+	phk/spdy.rst \
 	phk/sphinx.rst \
 	phk/ssl.rst \
+	phk/thetoolsweworkwith.rst \
 	phk/thoughts.rst \
 	phk/three-zero.rst \
+	phk/varnish_does_not_hash.rst \
 	phk/vcl_expr.rst \
 	phk/wanton_destruction.rst \
 	reference/index.rst \
-	reference/params.rst \
-	reference/varnishadm.rst \
 	reference/varnish-cli.rst \
+	reference/varnishadm.rst \
 	reference/varnishd.rst \
 	reference/varnishhist.rst \
 	reference/varnishlog.rst \
 	reference/varnishncsa.rst \
-	reference/varnishreplay.rst \
-	reference/varnishsizes.rst \
 	reference/varnishstat.rst \
 	reference/varnishtest.rst \
 	reference/varnishtop.rst \
 	reference/vcl.rst \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
-	reference/vsm.rst \
 	reference/vsl-query.rst \
 	reference/vsl.rst \
+	reference/vsm.rst \
+	tutorial/backend_servers.rst \
 	tutorial/index.rst \
 	tutorial/introduction.rst \
-	tutorial/starting_varnish.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/backend_servers.rst \
 	tutorial/now_what.rst \
+	tutorial/peculiarities.rst \
+	tutorial/putting_varnish_on_port_80.rst \
+	tutorial/starting_varnish.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
 	users-guide/devicedetection.rst \
@@ -152,26 +154,33 @@ EXTRA_DIST = \
 	users-guide/increasing-your-hitrate.rst \
 	users-guide/index.rst \
 	users-guide/intro.rst \
-	users-guide/operation-cli.rst \
 	users-guide/operation-logging.rst \
 	users-guide/operation-statistics.rst \
 	users-guide/params.rst \
 	users-guide/performance.rst \
 	users-guide/purging.rst \
 	users-guide/report.rst \
+	users-guide/run_cli.rst \
+	users-guide/run_security.rst \
 	users-guide/running.rst \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
+	users-guide/vcl-built-in-subs.rst \
+	users-guide/vcl-example-acls.rst \
+	users-guide/vcl-example-manipulating-headers.rst \
+	users-guide/vcl-example-manipulating-responses.rst \
+	users-guide/vcl-example-websockets.rst \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl.rst \
+	users-guide/vcl-intro.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst
+	users-guide/vcl-variables.rst \
+	users-guide/vcl.rst
 
 dist-hook:
 	$(MAKE) html



From phk at FreeBSD.org  Tue Mar 11 14:22:58 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 11 Mar 2014 15:22:58 +0100
Subject: [master] e6a2d41 Correctly account the length of vcl_backend_error{}
 synthetic body.
Message-ID: <E1WNNa6-0000aU-Ty@project.varnish-software.com>


commit e6a2d417bd256f7a6970174036bafb0419030d76
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 14:22:33 2014 +0000

    Correctly account the length of vcl_backend_error{} synthetic body.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3aabd8d..297f6f2 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -695,7 +695,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 				    "No space for %zd bytes of synth body", l);
 			} else {
 				memcpy(st->ptr, VSB_data(bo->synth_body), l);
-				st->len = l;
 				VBO_extend(bo, l);
 			}
 		}



From fgsch at lodoss.net  Tue Mar 11 14:54:09 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 11 Mar 2014 15:54:09 +0100
Subject: [master] 1ea3585 Unhook varnishreplay for the time being
Message-ID: <E1WNO4H-0004Eg-7U@project.varnish-software.com>


commit 1ea35850b69ae1fda724e5787f383017b986a548
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 14:49:27 2014 +0000

    Unhook varnishreplay for the time being
    
    This will be revisited post 4.0. Reorder while I'm here.

diff --git a/man/Makefile.am b/man/Makefile.am
index 9343063..581a9de 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -14,13 +14,12 @@ dist_man_MANS = \
 	vsl-query.7 \
 	varnishadm.1 \
 	varnishd.1 \
+	varnishhist.1 \
 	varnishlog.1 \
 	varnishncsa.1 \
-	varnishreplay.1 \
 	varnishstat.1 \
 	varnishtest.1 \
-	varnishtop.1 \
-	varnishhist.1
+	varnishtop.1
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 



From phk at FreeBSD.org  Wed Mar 12 10:14:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Mar 2014 11:14:49 +0100
Subject: [master] 12b2362 These two test-cases cannot work if streaming is
 allowed.
Message-ID: <E1WNgBV-0005lV-6I@project.varnish-software.com>


commit 12b236205d184be5f2cb5acbd0e9d45ed604a58c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 10:10:13 2014 +0000

    These two test-cases cannot work if streaming is allowed.

diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc
index 7bc36ae..b9578aa 100644
--- a/bin/varnishtest/tests/c00013.vtc
+++ b/bin/varnishtest/tests/c00013.vtc
@@ -4,13 +4,22 @@ server s1 {
 	rxreq
 	expect req.url == "/foo"
 	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay .2
 	sema r1 sync 2
+	delay .2
 	send "line1\n"
+	delay .2
 	sema r1 sync 2
 	send "line2\n"
 } -start
 
-varnish v1 -vcl+backend { } -start
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
 
 client c1 {
 	txreq -url "/foo" -hdr "client: c1"
@@ -24,7 +33,7 @@ sema r1 sync 2
 
 client c2 {
 	txreq -url "/foo" -hdr "client: c2"
-	delay .1
+	delay .2
 	sema r1 sync 2
 	rxresp
 	expect resp.status == 200
diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc
index a8b2d03..fffbbd7 100644
--- a/bin/varnishtest/tests/c00014.vtc
+++ b/bin/varnishtest/tests/c00014.vtc
@@ -15,6 +15,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
+		set beresp.do_stream = false;
 		set beresp.uncacheable = true;
 	}
 } -start



From phk at FreeBSD.org  Wed Mar 12 10:14:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Mar 2014 11:14:49 +0100
Subject: [master] 56bf8ab Add a debugging vsl to indicate when we enable
 streaming.
Message-ID: <E1WNgBV-0005lb-7P@project.varnish-software.com>


commit 56bf8ab581345565bc81fea2b2d5ded71987081e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 10:14:01 2014 +0000

    Add a debugging vsl to indicate when we enable streaming.
    
    This should possibly be an official VSL record ?

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 297f6f2..1172a61 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -480,6 +480,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->do_stream) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
+		VSLb(bo->vsl, SLT_Debug, "Allow streaming");
 		VBO_setstate(bo, BOS_STREAM);
 	}
 



From phk at FreeBSD.org  Wed Mar 12 10:14:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Mar 2014 11:14:49 +0100
Subject: [master] 3781556 Allow streaming for secondary requests.
Message-ID: <E1WNgBV-0005lk-8J@project.varnish-software.com>


commit 3781556d8294e17bc903657d2bdd2690a3e204f0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 10:14:33 2014 +0000

    Allow streaming for secondary requests.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9d5f9d1..882c84b 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -404,7 +404,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		if (oc->flags & (OC_F_FAILED | OC_F_DYING))
 			continue;
 
-		if (oc->flags & OC_F_BUSY || oc->busyobj != NULL) {
+		if (oc->flags & OC_F_BUSY) {
 			CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC);
 			if (req->hash_ignore_busy)
 				continue;



From phk at FreeBSD.org  Wed Mar 12 12:44:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Mar 2014 13:44:54 +0100
Subject: [master] 86b11dd Have SLT_Fetch_Body report if we're streaming or not.
Message-ID: <E1WNiWk-0005Vs-1u@project.varnish-software.com>


commit 86b11dd76e3e984d1574dce0523e4ef3b970858f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 12:44:39 2014 +0000

    Have SLT_Fetch_Body report if we're streaming or not.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1172a61..74724e0 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -480,10 +480,13 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->do_stream) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
-		VSLb(bo->vsl, SLT_Debug, "Allow streaming");
 		VBO_setstate(bo, BOS_STREAM);
 	}
 
+	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
+	    bo->htc.body_status, body_status_2str(bo->htc.body_status),
+	    bo->do_stream ? "stream" : "-");
+
 	if (bo->htc.body_status != BS_NONE) {
 		assert(bo->htc.body_status  != BS_ERROR);
 		VFP_Fetch_Body(bo, est);
@@ -491,9 +494,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	bo->t_body = VTIM_mono();
 
-	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
-	    bo->htc.body_status, body_status_2str(bo->htc.body_status));
-
 	if (bo->failed) {
 		wrk->stats.fetch_failed++;
 	} else {
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 5dee431..9217582 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -255,13 +255,13 @@ SLTM(TTL, 0, "TTL set on object",
 )
 
 SLTM(Fetch_Body, 0, "Body fetched from backend",
-	"Finished fetching body from backend.\n\n"
+	"Ready to fetch body from backend.\n\n"
 	"The format is::\n\n"
-	"\t%d(%s) cls %d\n"
-	"\t|  |       |\n"
-	"\t|  |       +- 1 if the backend connection was closed\n"
-	"\t|  +--------- Text description of body status\n"
-	"\t+------------ Body status\n"
+	"\t%d (%s) %s\n"
+	"\t|   |    |\n"
+	"\t|   |    +---- 'stream' or '-'\n"
+	"\t|   +--------- Text description of body fetch mode\n"
+	"\t+------------- Body fetch mode\n"
 	"\n"
 )
 



From lkarsten at varnish-software.com  Wed Mar 12 13:05:42 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 12 Mar 2014 14:05:42 +0100
Subject: [3.0] f6f75ea Fix varnishd segfault in jemalloc's arena_dalloc()
Message-ID: <E1WNiqs-0007ll-BM@project.varnish-software.com>


commit f6f75eac911a433f86ce28df9f2a8f489eed3503
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Mar 12 13:26:10 2014 +0100

    Fix varnishd segfault in jemalloc's arena_dalloc()
    
    Merging tiny patch from:
    
        http://lists.freebsd.org/pipermail/freebsd-current/2010-January/015138.html
    
    Fixes: #1448

diff --git a/lib/libjemalloc/malloc.c b/lib/libjemalloc/malloc.c
index 56d8a98..d007523 100644
--- a/lib/libjemalloc/malloc.c
+++ b/lib/libjemalloc/malloc.c
@@ -3874,6 +3874,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 					arena_dalloc_small(arena, chunk, ptr,
 					    mapelm);
 					malloc_spin_unlock(&arena->lock);
+					return;
 				}
 				mag_rack = rack;
 			}



From lkarsten at varnish-software.com  Wed Mar 12 14:00:53 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 12 Mar 2014 15:00:53 +0100
Subject: [3.0] efe5068 Document req.grace=0s strangeness
Message-ID: <E1WNjiH-00049N-AJ@project.varnish-software.com>


commit efe50681b6a5a1e113587acafbc14394386c34eb
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Mar 12 14:56:19 2014 +0100

    Document req.grace=0s strangeness
    
    Document that our use of 0s as a magic constant had the side
    effect of enabling grace if you tried to disable it in VCL, and
    how to get it to work.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 76aae84..ffd7c0c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -760,6 +760,11 @@ req.esi_level
 req.grace
   Set to a period to enable grace.
 
+  Known limitation in 3.0: Disabling grace by setting req.grace or beresp.grace
+  to 0s does not have the desired effect, but will rather set the grace time to
+  the value of default_grace. To disable grace for a request, either set
+  parameter default_grace = 0s or set req.grace = 0.000001s in VCL.
+
 req.xid
   Unique ID of this request.
 



From fgsch at lodoss.net  Wed Mar 12 14:50:35 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 12 Mar 2014 15:50:35 +0100
Subject: [master] 3ae1b7a Spelling
Message-ID: <E1WNkUN-0004zi-Tm@project.varnish-software.com>


commit 3ae1b7a718c25e3e1ad0fea22a5f7f6ff8508cb5
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 12 09:23:16 2014 +0000

    Spelling

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 67ad069..6970ae0 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -30,7 +30,7 @@ $Module directors 3 Backend traffic directors
 DESCRIPTION
 ===========
 
-Vmod_direcors enables load balancing in Varnish. The module are serves
+Vmod_direcors enables load balancing in Varnish. The module serves
 as an example on how one could extend the load balancing capabilites
 of Varnish.
 
@@ -50,7 +50,7 @@ initialize the directors in vcl_init, like this:::
         bar.add_backend(server2);
     }
 
-As you can see there is nothing keeping you from manipulting the
+As you can see there is nothing keeping you from manipulating the
 directors elsewhere in VCL. So, you could have VCL code that would
 add more backends to a director when a certin URL is called.
 
@@ -127,7 +127,7 @@ $Object hash()
 
 Description
        Creates a hash director. The hash director chooses the backend
-       bashed on hashing an arbitrary string. If you provide it with a
+       based on hashing an arbitrary string. If you provide it with a
        session cookie, you'll have the client connecting to the same
        backend every time.
 Example



From phk at FreeBSD.org  Thu Mar 13 08:55:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 09:55:31 +0100
Subject: [master] f2a0ace Correctly bail if there is insuficient storage for
 ESI data clone on IMS fetch.
Message-ID: <E1WO1QJ-00033t-Pd@project.varnish-software.com>


commit f2a0acec1566a38bdd41101773bf053cb6d103f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 08:55:14 2014 +0000

    Correctly bail if there is insuficient storage for ESI data clone
    on IMS fetch.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 74724e0..cac3acb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -579,12 +579,18 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	obj = bo->fetch_obj;
 
 	if (bo->ims_obj->esidata != NULL) {
-		obj->esidata = STV_alloc(bo, bo->ims_obj->esidata->len);
-		XXXAN(obj->esidata);
-		xxxassert(obj->esidata->space >= bo->ims_obj->esidata->len);
-		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr,
-		    bo->ims_obj->esidata->len);
-		obj->esidata->len = bo->ims_obj->esidata->len;
+		sl = bo->ims_obj->esidata->len;
+		obj->esidata = STV_alloc(bo, sl);
+		if (obj->esidata == NULL || obj->esidata->space < sl) {
+			VSLb(bo->vsl, SLT_Error,
+			    "No space for %zd bytes of ESI data", sl);
+			HSH_Fail(bo->fetch_objcore);
+			VBO_setstate(bo, BOS_FAILED);
+			HSH_Complete(obj->objcore);
+			return (F_STP_DONE);
+		}
+		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
+		obj->esidata->len = sl;
 	}
 
 	obj->gziped = bo->ims_obj->gziped;



From phk at FreeBSD.org  Thu Mar 13 09:24:21 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] d9cc0a7 Fix a improbable memory leak.
Message-ID: <E1WO1sD-0006ym-L3@project.varnish-software.com>


commit d9cc0a700b9777b2a8ea192df1063c33e59caef5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 23 22:23:26 2014 +0000

    Fix a improbable memory leak.

diff --git a/bin/varnishd/mgt/mgt_param_tcp.c b/bin/varnishd/mgt/mgt_param_tcp.c
index 5ef6b94..8f107e3 100644
--- a/bin/varnishd/mgt/mgt_param_tcp.c
+++ b/bin/varnishd/mgt/mgt_param_tcp.c
@@ -106,13 +106,12 @@ tcp_keep_probes(void)
 		return;		// XXX: log
 	assert (i > 0);
 	s = VSS_listen(ta[0], 10);
-	if (s < 0)
-		return;		// XXX: log
-
-	tcp_probe(s, TCP_KEEPIDLE, "tcp_keepalive_time",	600);
-	tcp_probe(s, TCP_KEEPCNT, "tcp_keepalive_probes",	5);
-	tcp_probe(s, TCP_KEEPINTVL, "tcp_keepalive_intvl",	5);
-	AZ(close(s));
+	if (s >= 0) {
+		tcp_probe(s, TCP_KEEPIDLE, "tcp_keepalive_time",	600);
+		tcp_probe(s, TCP_KEEPCNT, "tcp_keepalive_probes",	5);
+		tcp_probe(s, TCP_KEEPINTVL, "tcp_keepalive_intvl",	5);
+		AZ(close(s));
+	}
 	free(ta);
 }
 #endif



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] 3a308c3 Rename default.vcl to builtin.vcl.
Message-ID: <E1WO1sD-0006ys-M5@project.varnish-software.com>


commit 3a308c36d471f38e338502253e198bac98128774
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jan 27 11:20:23 2014 +0100

    Rename default.vcl to builtin.vcl.
    
    The default.vcl name was thought up in 2007 and have been a point of confusion
    since.
    
    New users take the old default VCL, uncomment what they think makes sense, and
    always end with return(lookup) because that seems to be needed. But it isn't,
    and the defaults in the built-in VCL are much better at handling all things
    they haven't learned about yet.

diff --git a/.gitignore b/.gitignore
index f545294..78cdba5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,8 +44,8 @@ cscope.*out
 /varnishapi-uninstalled.pc
 
 # Default vcl made from bin/varnishd/default.vcl
-/bin/varnishd/default_vcl.h
-/etc/default.vcl
+/bin/varnishd/builtin_vcl.h
+/etc/builtin.vcl
 
 # Various auto-generated code snippets
 /include/vcl.h
diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 9d281f1..e0a8461 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -93,7 +93,7 @@ noinst_HEADERS = \
 	cache/cache_backend.h \
 	cache/cache_esi.h \
 	common/heritage.h \
-	default_vcl.h \
+	builtin_vcl.h \
 	hash/hash_slinger.h \
 	mgt/mgt.h \
 	mgt/mgt_cli.h \
@@ -126,26 +126,26 @@ varnishd_LDADD = \
 	@PCRE_LIBS@ \
 	${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} ${LIBUMEM}
 
-EXTRA_DIST = default.vcl
-DISTCLEANFILES = default_vcl.h
+EXTRA_DIST = builtin.vcl
+DISTCLEANFILES = builtin_vcl.h
 
 #
-# Turn the default.vcl file into a C-string we can include in the program.
+# Turn the builtin.vcl file into a C-string we can include in the program.
 #
-default_vcl.h:	default.vcl
+builtin_vcl.h:	builtin.vcl
 	echo '/*' > $@
 	echo ' * NB:  This file is machine generated, DO NOT EDIT!' >> $@
 	echo ' *' >> $@
-	echo ' * Edit default.vcl instead and run make' >> $@
+	echo ' * Edit builtin.vcl instead and run make' >> $@
 	echo ' *' >> $@
 	echo ' */' >> $@
 	echo '' >> $@
 	sed -e 's/"/\\"/g' \
 	    -e 's/$$/\\n"/' \
-	    -e 's/^/ "/' $(srcdir)/default.vcl >> $@
+	    -e 's/^/ "/' $(srcdir)/builtin.vcl >> $@
 
 # Explicitly record dependency
-mgt/mgt_vcc.c:	default_vcl.h
+mgt/mgt_vcc.c:	builtin_vcl.h
 
 varnishd.1: $(top_srcdir)/doc/sphinx/reference/varnishd.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishd/Makefile.phk b/bin/varnishd/Makefile.phk
index d0ee858..1cbd5c2 100644
--- a/bin/varnishd/Makefile.phk
+++ b/bin/varnishd/Makefile.phk
@@ -95,7 +95,7 @@ LD_ADD	+= -lm
 
 RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishd.rst
 
-MADE_FILES += default_vcl.h
+MADE_FILES += builtin_vcl.h
 
 TODO_INSTALL += install_etc
 
@@ -103,20 +103,20 @@ TOPDIR  = $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
 
 #
-# Turn the default.vcl file into a C-string we can include in the program.
+# Turn the builtin.vcl file into a C-string we can include in the program.
 #
-default_vcl.h:	default.vcl
+builtint_vcl.h:	builtin.vcl
 	echo '/*' > $@
 	echo ' * NB:  This file is machine generated, DO NOT EDIT!' >> $@
 	echo ' *' >> $@
-	echo ' * Edit default.vcl instead and run make' >> $@
+	echo ' * Edit builtin.vcl instead and run make' >> $@
 	echo ' *' >> $@
 	echo ' */' >> $@
 	echo '' >> $@
 	sed -e 's/"/\\"/g' \
 	    -e 's/$$/\\n"/' \
-	    -e 's/^/ "/' default.vcl >> $@
+	    -e 's/^/ "/' builtin.vcl >> $@
 
 .PHONY: install_etc
-install_etc: default_vcl.h
-	@cp default_vcl.h $(INSTALL_BASE)/etc/
+install_etc: builtin_vcl.h
+	@cp builtin_vcl.h $(INSTALL_BASE)/etc/
diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
new file mode 100644
index 0000000..3a723e3
--- /dev/null
+++ b/bin/varnishd/builtin.vcl
@@ -0,0 +1,185 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+
+ *
+ * The built-in (previously called default) VCL code.
+ *
+ * NB! You do NOT need to copy & paste all of these functions into your
+ * own vcl code, if you do not provide a definition of one of these
+ * functions, the compiler will automatically fall back to the default
+ * code from this file.
+ *
+ * This code will be prefixed with a backend declaration built from the
+ * -b argument.
+ */
+
+vcl 4.0;
+
+sub vcl_recv {
+    if (req.restarts == 0) {
+	if (req.http.x-forwarded-for) {
+	    set req.http.X-Forwarded-For =
+		req.http.X-Forwarded-For + ", " + client.ip;
+	} else {
+	    set req.http.X-Forwarded-For = client.ip;
+	}
+    }
+    if (req.method != "GET" &&
+      req.method != "HEAD" &&
+      req.method != "PUT" &&
+      req.method != "POST" &&
+      req.method != "TRACE" &&
+      req.method != "OPTIONS" &&
+      req.method != "DELETE") {
+        /* Non-RFC2616 or CONNECT which is weird. */
+        return (pipe);
+    }
+    if (req.method != "GET" && req.method != "HEAD") {
+        /* We only deal with GET and HEAD by default */
+        return (pass);
+    }
+    if (req.http.Authorization || req.http.Cookie) {
+        /* Not cacheable by default */
+        return (pass);
+    }
+    return (hash);
+}
+
+sub vcl_pipe {
+    # Note that only the first request to the backend will have
+    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
+    # have it set for all requests, make sure to have:
+    # set bereq.http.connection = "close";
+    # here.  It is not set by default as it might break some broken web
+    # applications, like IIS with NTLM authentication.
+    return (pipe);
+}
+
+sub vcl_pass {
+    return (fetch);
+}
+
+sub vcl_hash {
+    hash_data(req.url);
+    if (req.http.host) {
+        hash_data(req.http.host);
+    } else {
+        hash_data(server.ip);
+    }
+    return (lookup);
+}
+
+sub vcl_purge {
+    return (error(200, "Purged"));
+}
+
+sub vcl_hit {
+    if (obj.ttl >= 0s) {
+	// A pure unadultered hit, deliver it
+	return (deliver);
+    }
+    if (obj.ttl + obj.grace > 0s) {
+	// Object is in grace, delive it
+	// Automatically triggers a background fetch
+	return (deliver);
+    }
+    // fetch & deliver once we get the result
+    return (fetch);
+}
+
+sub vcl_miss {
+    return (fetch);
+}
+
+
+sub vcl_backend_fetch {
+    return (fetch);
+}
+
+sub vcl_backend_response {
+    if (beresp.ttl <= 0s ||
+        beresp.http.Set-Cookie ||
+        beresp.http.Surrogate-control ~ "no-store" ||
+        (!beresp.http.Surrogate-Control &&
+          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
+        beresp.http.Vary == "*") {
+		/*
+		 * Mark as "Hit-For-Pass" for the next 2 minutes
+		 */
+		set beresp.ttl = 120 s;
+		set beresp.uncacheable = true;
+    }
+    return (deliver);
+}
+
+sub vcl_deliver {
+    /*
+     * These two write to the stored object causing extra page faults
+     * Enable them only if you need them.
+     *
+     * set obj.hits = obj.hits + 1;
+     * set obj.last_use = now;
+     */
+    return (deliver);
+}
+
+/*
+ * We can come here "invisibly" with the following errors:  413, 417 & 503
+ */
+sub vcl_error {
+    set obj.http.Content-Type = "text/html; charset=utf-8";
+    set obj.http.Retry-After = "5";
+    synthetic {"
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html>
+  <head>
+    <title>"} + obj.status + " " + obj.response + {"</title>
+  </head>
+  <body>
+    <h1>Error "} + obj.status + " " + obj.response + {"</h1>
+    <p>"} + obj.response + {"</p>
+    <h3>Guru Meditation:</h3>
+    <p>XID: "} + req.xid + {"</p>
+    <hr>
+    <p>Varnish cache server</p>
+  </body>
+</html>
+"};
+    return (deliver);
+}
+
+sub vcl_init {
+	return (ok);
+}
+
+sub vcl_fini {
+	return (ok);
+}
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
deleted file mode 100644
index 8e8fa26..0000000
--- a/bin/varnishd/default.vcl
+++ /dev/null
@@ -1,185 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2014 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
-
- *
- * The default VCL code.
- *
- * NB! You do NOT need to copy & paste all of these functions into your
- * own vcl code, if you do not provide a definition of one of these
- * functions, the compiler will automatically fall back to the default
- * code from this file.
- *
- * This code will be prefixed with a backend declaration built from the
- * -b argument.
- */
-
-vcl 4.0;
-
-sub vcl_recv {
-    if (req.restarts == 0) {
-	if (req.http.x-forwarded-for) {
-	    set req.http.X-Forwarded-For =
-		req.http.X-Forwarded-For + ", " + client.ip;
-	} else {
-	    set req.http.X-Forwarded-For = client.ip;
-	}
-    }
-    if (req.method != "GET" &&
-      req.method != "HEAD" &&
-      req.method != "PUT" &&
-      req.method != "POST" &&
-      req.method != "TRACE" &&
-      req.method != "OPTIONS" &&
-      req.method != "DELETE") {
-        /* Non-RFC2616 or CONNECT which is weird. */
-        return (pipe);
-    }
-    if (req.method != "GET" && req.method != "HEAD") {
-        /* We only deal with GET and HEAD by default */
-        return (pass);
-    }
-    if (req.http.Authorization || req.http.Cookie) {
-        /* Not cacheable by default */
-        return (pass);
-    }
-    return (hash);
-}
-
-sub vcl_pipe {
-    # Note that only the first request to the backend will have
-    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
-    # have it set for all requests, make sure to have:
-    # set bereq.http.connection = "close";
-    # here.  It is not set by default as it might break some broken web
-    # applications, like IIS with NTLM authentication.
-    return (pipe);
-}
-
-sub vcl_pass {
-    return (fetch);
-}
-
-sub vcl_hash {
-    hash_data(req.url);
-    if (req.http.host) {
-        hash_data(req.http.host);
-    } else {
-        hash_data(server.ip);
-    }
-    return (lookup);
-}
-
-sub vcl_purge {
-    return (error(200, "Purged"));
-}
-
-sub vcl_hit {
-    if (obj.ttl >= 0s) {
-	// A pure unadultered hit, deliver it
-	return (deliver);
-    }
-    if (obj.ttl + obj.grace > 0s) {
-	// Object is in grace, delive it
-	// Automatically triggers a background fetch
-	return (deliver);
-    }
-    // fetch & deliver once we get the result
-    return (fetch);
-}
-
-sub vcl_miss {
-    return (fetch);
-}
-
-
-sub vcl_backend_fetch {
-    return (fetch);
-}
-
-sub vcl_backend_response {
-    if (beresp.ttl <= 0s ||
-        beresp.http.Set-Cookie ||
-        beresp.http.Surrogate-control ~ "no-store" ||
-        (!beresp.http.Surrogate-Control &&
-          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
-        beresp.http.Vary == "*") {
-		/*
-		 * Mark as "Hit-For-Pass" for the next 2 minutes
-		 */
-		set beresp.ttl = 120 s;
-		set beresp.uncacheable = true;
-    }
-    return (deliver);
-}
-
-sub vcl_deliver {
-    /*
-     * These two write to the stored object causing extra page faults
-     * Enable them only if you need them.
-     *
-     * set obj.hits = obj.hits + 1;
-     * set obj.last_use = now;
-     */
-    return (deliver);
-}
-
-/*
- * We can come here "invisibly" with the following errors:  413, 417 & 503
- */
-sub vcl_error {
-    set obj.http.Content-Type = "text/html; charset=utf-8";
-    set obj.http.Retry-After = "5";
-    synthetic {"
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html>
-  <head>
-    <title>"} + obj.status + " " + obj.response + {"</title>
-  </head>
-  <body>
-    <h1>Error "} + obj.status + " " + obj.response + {"</h1>
-    <p>"} + obj.response + {"</p>
-    <h3>Guru Meditation:</h3>
-    <p>XID: "} + req.xid + {"</p>
-    <hr>
-    <p>Varnish cache server</p>
-  </body>
-</html>
-"};
-    return (deliver);
-}
-
-sub vcl_init {
-	return (ok);
-}
-
-sub vcl_fini {
-	return (ok);
-}
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index d3e81de..09e4eb4 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -71,8 +71,8 @@ static struct vcc *vcc;
 
 /*--------------------------------------------------------------------*/
 
-static const char * const default_vcl =
-#include "default_vcl.h"
+static const char * const builtin_vcl =
+#include "builtin_vcl.h"
     ""	;
 
 /*--------------------------------------------------------------------
@@ -488,7 +488,7 @@ mgt_vcc_init(void)
 
 	vcc = VCC_New();
 	AN(vcc);
-	VCC_Default_VCL(vcc, default_vcl);
+	VCC_Default_VCL(vcc, builtin_vcl);
 	AZ(atexit(mgt_vcc_atexit));
 }
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 8a3ee2c..d769e81 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -922,7 +922,7 @@ specified:
    .port = "http";
   }
 
-.. include:: ../../../bin/varnishd/default.vcl
+.. include:: ../../../bin/varnishd/builtin.vcl
   :literal:
 
 The following example shows how to support multiple sites running on
diff --git a/etc/Makefile.am b/etc/Makefile.am
index 2fb4d38..af964e9 100644
--- a/etc/Makefile.am
+++ b/etc/Makefile.am
@@ -1,23 +1,11 @@
 #
 
-DISTCLEANFILES = default.vcl
+DISTCLEANFILES = builtin.vcl
 
-dist_varnishconf_DATA = default.vcl
+dist_varnishconf_DATA = builtin.vcl
 
-default.vcl:	$(top_srcdir)/bin/varnishd/default.vcl
-	( printf "vcl 4.0;\nThis is a basic VCL configuration file for varnish.  See the vcl(7)\n\
-man page for details on VCL syntax and semantics.\n\
-\n\
-Default backend definition.  Set this to point to your content\n\
-server.\n\
-\n\
-backend default {\n\
-    .host = \"127.0.0.1\";\n\
-    .port = \"8080\";\n\
-}\n\
-\n\
-Below is a commented-out copy of the default VCL logic.  If you\n\
-redefine any of these subroutines, the built-in logic will be\n\
-appended to your code.\n" ; \
-	sed -n '/vcl_recv/,$$p' $(top_srcdir)/bin/varnishd/default.vcl ) | \
-	sed 's/^\(.*\)$$/# \1/' > default.vcl
+builtin.vcl:	$(top_srcdir)/bin/varnishd/builtin.vcl
+	( printf "This is the VCL configuration Varnish will automatically append to your VCL\nfile during compilation/loading. See the vcl(7) man page for details on syntax\nand semantics.\n\
+New users is recommended to use the example.vcl file as a starting point.\n\n";\
+	sed -n '/vcl_recv/,$$p' $(top_srcdir)/bin/varnishd/builtin.vcl ) | \
+	sed 's/^\(.*\)$$/# \1/' > builtin.vcl
diff --git a/man/Makefile.am b/man/Makefile.am
index 56b0a52..1ed3a5d 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -12,7 +12,7 @@ dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7 vsl.7 vsl-query.7
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
-	$(top_srcdir)/bin/varnishd/default.vcl
+	$(top_srcdir)/bin/varnishd/builtin.vcl
 if HAVE_RST2MAN
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
 else



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] e631d31 Add a skeleton VCL file to help new users along.
Message-ID: <E1WO1sD-0006z2-Mz@project.varnish-software.com>


commit e631d319167d1f8a576bebc91e3b2b4f999ef15c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jan 27 11:32:13 2014 +0100

    Add a skeleton VCL file to help new users along.

diff --git a/etc/example.vcl b/etc/example.vcl
new file mode 100644
index 0000000..8122340
--- /dev/null
+++ b/etc/example.vcl
@@ -0,0 +1,36 @@
+#
+# This is an example VCL file for Varnish.
+# It does not do anything by default, delegating control to the builtin vcl.
+# The builtin VCL is called when there is no explicit explicit return
+# statement.
+
+# See the VCL tutorial at https://www.varnish-cache.org/docs/trunk/tutorial/
+# See http://varnish-cache.org/trac/wiki/VCLExamples for more examples.
+
+# Marker to tell the VCL compiler that this VCL has been adapted to the new 4.0
+# format.
+vcl 4.0;
+
+# Default backend definition. Set this to point to your content server.
+backend default {
+    .host = "127.0.0.1";
+    .port = "8080";
+}
+
+sub vcl_recv {
+    # Happens before we check if we have this in cache already.
+    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_recv
+}
+
+sub vcl_fetch {
+    # Happens after we have read the response headers from the backend.
+    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
+#
+}
+
+sub vcl_deliver {
+    # Happens when we have all the pieces we need, and are about to send the
+    # response to the client.
+    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
+}
+



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] c93b3ee Fix incorrect VCL in example.vcl.
Message-ID: <E1WO1sD-0006zB-Nu@project.varnish-software.com>


commit c93b3eebf9a05d9afdb5aa86c56627ee6a84d7c3
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jan 27 12:51:02 2014 +0100

    Fix incorrect VCL in example.vcl.
    
    Old habits die hard.
    
    Thanks to Federico Schwindt for pointing this out.

diff --git a/etc/example.vcl b/etc/example.vcl
index 8122340..c9b9aad 100644
--- a/etc/example.vcl
+++ b/etc/example.vcl
@@ -22,7 +22,7 @@ sub vcl_recv {
     # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_recv
 }
 
-sub vcl_fetch {
+sub vcl_backend_response {
     # Happens after we have read the response headers from the backend.
     # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
 #



From tfheen at varnish-software.com  Thu Mar 13 09:24:21 2014
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] 3756761 Move builtin.vcl to doc, ship example.vcl too
Message-ID: <E1WO1sD-0006zK-Oo@project.varnish-software.com>


commit 37567613edcfd22e499a98fd6bf6e40067bc9f48
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 27 13:30:50 2014 +0100

    Move builtin.vcl to doc, ship example.vcl too

diff --git a/etc/Makefile.am b/etc/Makefile.am
index af964e9..d02d710 100644
--- a/etc/Makefile.am
+++ b/etc/Makefile.am
@@ -2,7 +2,8 @@
 
 DISTCLEANFILES = builtin.vcl
 
-dist_varnishconf_DATA = builtin.vcl
+dist_doc_DATA = builtin.vcl \
+		example.vcl
 
 builtin.vcl:	$(top_srcdir)/bin/varnishd/builtin.vcl
 	( printf "This is the VCL configuration Varnish will automatically append to your VCL\nfile during compilation/loading. See the vcl(7) man page for details on syntax\nand semantics.\n\



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] e90fce3 Be consistent with capitalisation and whitepace.
Message-ID: <E1WO1sD-0006zT-Pt@project.varnish-software.com>


commit e90fce30053424791fcb87af41c05b579be699fb
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jan 27 14:07:50 2014 +0100

    Be consistent with capitalisation and whitepace.

diff --git a/doc/sphinx/installation/bugs.rst b/doc/sphinx/installation/bugs.rst
index dde28bf..b1cb06a 100644
--- a/doc/sphinx/installation/bugs.rst
+++ b/doc/sphinx/installation/bugs.rst
@@ -31,9 +31,9 @@ Varnish is split over two processes, the manager and the child.  The child
 does all the work, and the manager hangs around to resurect it, if it
 crashes.
 
-Therefore, the first thing to do if you see a varnish crash, is to examine
+Therefore, the first thing to do if you see a Varnish crash, is to examine
 your syslogs, to see if it has happened before.  (One site is rumoured
-to have had varnish restarting every 10 minutes and *still* provide better
+to have had Varnish restarting every 10 minutes and *still* provide better
 service than their CMS system.)
 
 When it crashes, if at all possible, Varnish will spew out a crash dump
@@ -116,7 +116,7 @@ able to figure that out.
 
 If one or more threads are spinning, use ``strace`` or ``ktrace`` or ``truss``
 (or whatever else your OS provides) to get a trace of which system calls
-the varnish process issues.  Be aware that this may generate a lot
+the Varnish process issues.  Be aware that this may generate a lot
 of very repetitive data, usually one second worth is more than enough.
 
 Also, run ``varnishlog`` for a second, and collect the output
diff --git a/doc/sphinx/installation/platformnotes.rst b/doc/sphinx/installation/platformnotes.rst
index 048442c..bacc57e 100644
--- a/doc/sphinx/installation/platformnotes.rst
+++ b/doc/sphinx/installation/platformnotes.rst
@@ -26,7 +26,7 @@ performance.
 If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
 the maximum stack size before starting Varnish.
 
-The default allocates to much memory per thread, which will make varnish fail
+The default allocates to much memory per thread, which will make Varnish fail
 as soon as the number of threads (traffic) increases.
 
 Reduce the maximum stack size by running::
diff --git a/doc/sphinx/installation/prerequisites.rst b/doc/sphinx/installation/prerequisites.rst
index 3508c1e..ad92ee0 100644
--- a/doc/sphinx/installation/prerequisites.rst
+++ b/doc/sphinx/installation/prerequisites.rst
@@ -8,12 +8,12 @@ In order for you to install Varnish you must have the following:
     - Linux
     - FreeBSD
     - Solaris (x86 only)
-  * root access to said system
+  * root access to said system.
 
 
 Varnish can be installed on other UNIX systems as well, but it is not
-tested particularly well on these plattforms. Varnish is, from time to
-time, said to work on: 
+tested particularly well on these platforms. Varnish is, from time to
+time, said to work on:
 
   * 32 bit versions of the before-mentioned systems.
   * OS X
diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 834a776..d820d69 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -1,6 +1,6 @@
-=======
-varnish
-=======
+===========
+Varnish CLI
+===========
 
 ------------------------------
 Varnish Command Line Interface
@@ -142,7 +142,7 @@ ping  [timestamp]
       Ping the Varnish cache process, keeping the connection alive.
 
 quit
-      Close the connection to the varnish admin port.
+      Close the connection to the Varnish admin port.
 
 start
       Start the Varnish cache process if it is not already running.
@@ -214,7 +214,7 @@ code and length field always is exactly 13 characters long, including
 the NL character.
 
 For your reference the sourcefile lib/libvarnish/cli_common.h contains
-the functions varnish code uses to read and write CLI response.
+the functions Varnish code uses to read and write CLI response.
 
 .. _ref_psk_auth:
 
@@ -318,9 +318,9 @@ SEE ALSO
 HISTORY
 =======
 
-The varnish manual page was written by Per Buer in 2011. Some of the
+The Varnish manual page was written by Per Buer in 2011. Some of the
 text was taken from the Varnish Cache wiki, the varnishd(7) man page
-or the varnish source code.
+or the Varnish source code.
 
 COPYRIGHT
 =========
@@ -328,4 +328,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2011 Varnish Software AS
+* Copyright (c) 2011-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index 3521474..95113fa 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -3,7 +3,7 @@ varnishadm
 ==========
 
 ----------------------------------
-Control a running varnish instance
+Control a running Varnish instance
 ----------------------------------
 
 :Author: Cecilie Fritzvold
@@ -90,4 +90,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2011 Varnish Software AS
+* Copyright (c) 2007-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 201c99a..a430fc2 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -70,7 +70,7 @@ OPTIONS
             Specifies the hash algorithm.  See Hash Algorithms for a list of supported algorithms.
 
 -i identity
-            Specify the identity of the varnish server.  This can be accessed using server.identity
+            Specify the identity of the Varnish server.  This can be accessed using server.identity
             from VCL
 
 -l shmlogsize
diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst
index ffc8289..017bb55 100644
--- a/doc/sphinx/reference/varnishhist.rst
+++ b/doc/sphinx/reference/varnishhist.rst
@@ -90,4 +90,4 @@ This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2011 Varnish Software AS
+* Copyright (c) 2006-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/varnishreplay.rst b/doc/sphinx/reference/varnishreplay.rst
index da3df36..b8b89e9 100644
--- a/doc/sphinx/reference/varnishreplay.rst
+++ b/doc/sphinx/reference/varnishreplay.rst
@@ -20,14 +20,14 @@ varnishreplay [-D] -a address:port -r file
 DESCRIPTION
 ===========
 
-The varnishreplay utility parses varnish logs and attempts to
-reproduce the traffic. It is typcally used to *warm* up caches or
+The varnishreplay utility parses Varnish logs and attempts to
+reproduce the traffic. It is typically used to *warm* up caches or
 various forms of testing.
 
 The following options are available:
 
--a backend           Send the traffic over tcp to this server, specified by an 
-   		     address and a port.  This option is 
+-a backend           Send the traffic over tcp to this server, specified by an
+   		     address and a port.  This option is
    		     mandatory. Only IPV4 is supported at this time.
 
 -D                   Turn on debugging mode.
@@ -54,4 +54,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2010 Varnish Software AS
+* Copyright (c) 2007-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index c27af28..40951e1 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -125,4 +125,4 @@ This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2011 Varnish Software AS
+* Copyright (c) 2006-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index a0df5e2..8a3124c 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -109,7 +109,7 @@ An example::
         } -run
 
 When run, the above script will simulate a server (s1) that expects two
-different requests. It will start a varnish server (v1) and add the backend
+different requests. It will start a Varnish server (v1) and add the backend
 definition to the VCL specified (-vcl+backend). Finally it starts the
 c1-client, which is a single client sending two requests.
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index d769e81..9aea743 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -467,7 +467,7 @@ vcl_pass
 
   restart
     Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
     error.
 
 vcl_hash
@@ -497,7 +497,7 @@ vcl_hit
 
   restart
     Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
     error.
 
 vcl_miss
@@ -546,7 +546,7 @@ vcl_fetch
 
   restart
     Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
     error.
 
 vcl_deliver
@@ -560,7 +560,7 @@ vcl_deliver
 
   restart
     Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
     error.
 
 vcl_error
@@ -575,7 +575,7 @@ vcl_error
 
   restart
     Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
     error.
 
 vcl_fini
@@ -755,7 +755,7 @@ other words, they are available in vcl_fetch:
 
 beresp.do_stream 
   Deliver the object to the client directly without fetching the whole
-  object into varnish. If this request is pass'ed it will not be
+  object into Varnish. If this request is pass'ed it will not be
   stored in memory. As of Varnish Cache 3.0 the object will marked as busy
   as it is delivered so only client can access the object.
 
@@ -883,7 +883,7 @@ Grace and saint mode
 
 If the backend takes a long time to generate an object there is a risk
 of a thread pile up.  In order to prevent this you can enable *grace*.
-This allows varnish to serve an expired version of the object while a
+This allows Varnish to serve an expired version of the object while a
 fresh object is being generated by the backend.
 
 The following vcl code will make Varnish serve expired objects.  All
@@ -1035,4 +1035,4 @@ This document is licensed under the same license as Varnish
 itself. See LICENSE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2011 Varnish Software AS
+* Copyright (c) 2006-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 88a511d..95987e1 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -25,9 +25,9 @@ function shown above.  The full contents of the "std" module is
 documented in vmod_std(7).
 
 This part of the manual is about how you go about writing your own
-VMOD, how the language interface between C and VCC works, where you 
+VMOD, how the language interface between C and VCC works, where you
 can find contributed VMODs etc. This explanation will use the "std"
-VMOD as example, having a varnish source tree handy may be a good
+VMOD as example, having a Varnish source tree handy may be a good
 idea.
 
 VMOD Directory
@@ -35,7 +35,8 @@ VMOD Directory
 
 The VMOD directory is an up-to-date compilation of maintained
 extensions written for Varnish Cache:
-https://www.varnish-cache.org/vmods
+
+    https://www.varnish-cache.org/vmods
 
 The vmod.vcc file
 =================
@@ -47,11 +48,11 @@ data structures that does all the hard work.
 
 The std VMODs vmod.vcc file looks somewhat like this::
 
-	Module std
-	Init init_function
-	Function STRING toupper(PRIV_CALL, STRING_LIST)
-	Function STRING tolower(PRIV_VCL, STRING_LIST)
-	Function VOID set_ip_tos(INT)
+	$Module std
+	$Init init_function
+	$Function STRING toupper(PRIV_CALL, STRING_LIST)
+	$Function STRING tolower(PRIV_VCL, STRING_LIST)
+	$Function VOID set_ip_tos(INT)
 
 The first line gives the name of the module, nothing special there.
 
diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index d80cb5b..36a2678 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -153,8 +153,8 @@ Description
 	fails, *fallback* will be returned.
 Example
 	if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { ... }
-	
-	
+
+
 SEE ALSO
 ========
 
@@ -174,4 +174,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2011 Varnish Software
+* Copyright (c) 2011-2014 Varnish Software
diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
index ebc05ad..69d1c28 100644
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -7,7 +7,7 @@ Varnish has a concept of "backend" or "origin" servers. A backend
 server is the server providing the content Varnish will accelerate.
 
 Our first task is to tell Varnish where it can find its content. Start
-your favorite text editor and open the varnish default configuration
+your favorite text editor and open the Varnish default configuration
 file. If you installed from source this is
 /usr/local/etc/varnish/default.vcl, if you installed from a package it
 is probably /etc/varnish/default.vcl.
@@ -21,7 +21,7 @@ the configuration that looks like this:::
   }
 
 This means we set up a backend in Varnish that fetches content from
-the host www.varnish-cache.org on port 80. 
+the host www.varnish-cache.org on port 80.
 
 Since you probably don't want to be mirroring varnish-cache.org we
 need to get Varnish to fetch content from your own origin
diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst
index ea44323..65dec6a 100644
--- a/doc/sphinx/users-guide/command-line.rst
+++ b/doc/sphinx/users-guide/command-line.rst
@@ -24,7 +24,7 @@ You will most likely want to set this to ":80" which is the Well
 Known Port for HTTP.
 
 You can specify multiple addresses separated by a comma, and you
-can use numeric or host/service names if you like, varnish will try
+can use numeric or host/service names if you like, Varnish will try
 to open and service as many of them as possible, but if none of them
 can be opened, varnishd will not start.
 
@@ -38,7 +38,7 @@ Here are some examples::
 
 If your webserver runs on the same computer, you will have to move
 it to another port number first.
- 
+
 -f *VCL-file* or -b *backend*
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index 3d2fb0d..e497e00 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -33,7 +33,7 @@ Interface, which can be used manually, from scripts or programs.
 
 The CLI offers almost full control of what Varnish actually does
 to your HTTP traffic, and we have gone to great lengths to ensure
-that you should not need to restart the varnish processes, unless
+that you should not need to restart the Varnish processes, unless
 you need to change something very fundamental.
 
 The CLI can be safely accessed remotely, using a simple and flexible
diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
index f15425d..856cbfa 100644
--- a/doc/sphinx/users-guide/operation-logging.rst
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -18,7 +18,7 @@ is logging. Varnishlog gives you the raw logs, everything that is
 written to the logs. There are other clients as well, we'll show you
 these later.
 
-In the terminal window you started varnish now type *varnishlog* and
+In the terminal window you started Varnish now type *varnishlog* and
 press enter.
 
 You'll see lines like these scrolling slowly by.::
@@ -27,7 +27,7 @@ You'll see lines like these scrolling slowly by.::
     0 CLI          - Wr 200 PONG 1273698726 1.0
 
 These is the Varnish master process checking up on the caching process
-to see that everything is OK. 
+to see that everything is OK.
 
 Now go to the browser and reload the page displaying your web
 app. You'll see lines like these.::
@@ -55,10 +55,10 @@ Now, you can filter quite a bit with varnishlog. The basic option you
 want to know are:
 
 -b
- Only show log lines from traffic going between Varnish and the backend 
+ Only show log lines from traffic going between Varnish and the backend
  servers. This will be useful when we want to optimize cache hit rates.
 
--c 
+-c
  Same as -b but for client side traffic.
 
 -m tag:regex
diff --git a/doc/sphinx/users-guide/operation-statistics.rst b/doc/sphinx/users-guide/operation-statistics.rst
index ad57037..8736c5e 100644
--- a/doc/sphinx/users-guide/operation-statistics.rst
+++ b/doc/sphinx/users-guide/operation-statistics.rst
@@ -4,7 +4,7 @@
 Statistics
 ----------
 
-Now that your varnish is up and running let's have a look at how it is
+Now that your Varnish is up and running let's have a look at how it is
 doing. There are several tools that can help.
 
 varnishtop
@@ -48,10 +48,10 @@ varnishstat
 Varnish has lots of counters. We count misses, hits, information about
 the storage, threads created, deleted objects. Just about
 everything. varnishstat will dump these counters. This is useful when
-tuning varnish. 
+tuning Varnish.
 
 There are programs that can poll varnishstat regularly and make nice
 graphs of these counters. One such program is Munin. Munin can be
 found at http://munin-monitoring.org/ . There is a plugin for munin in
-the varnish source code.
+the Varnish source code.
 
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 3c652ea..5604d96 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -11,7 +11,7 @@ bad for business.
 
 The solution is to notify Varnish when there is fresh content
 available. This can be done through three mechanisms. HTTP purging,
-banning and forced cache misses. First, let me explain the HTTP purges. 
+banning and forced cache misses. First, let me explain the HTTP purges.
 
 
 HTTP Purges
@@ -31,7 +31,7 @@ following VCL in place::
 	  "localhost";
 	  "192.168.55.0"/24;
   }
-  
+
   sub vcl_recv {
       	  # allow PURGE from localhost and 192.168.55...
 
@@ -42,14 +42,14 @@ following VCL in place::
 		  return (lookup);
 	  }
   }
-  
+
   sub vcl_hit {
 	  if (req.method == "PURGE") {
 	          purge;
 		  error 200 "Purged.";
 	  }
   }
-  
+
   sub vcl_miss {
 	  if (req.method == "PURGE") {
 	          purge;
@@ -93,7 +93,7 @@ the following command::
 Quite powerful, really.
 
 Bans are checked when we hit an object in the cache, but before we
-deliver it. *An object is only checked against newer bans*. 
+deliver it. *An object is only checked against newer bans*.
 
 Bans that only match against obj.* are also processed by a background
 worker threads called the *ban lurker*. The ban lurker will walk the
@@ -168,7 +168,7 @@ Forcing a cache miss
 
 The final way to invalidate an object is a method that allows you to
 refresh an object by forcing a hash miss for a single request. If you set
-req.hash_always_miss to true, varnish will miss the current object in the
+req.hash_always_miss to true, Varnish will miss the current object in the
 cache, thus forcing a fetch from the backend. This can in turn add the
 freshly fetched object to the cache, thus overriding the current one. The
 old object will stay in the cache until ttl expires or it is evicted by
diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index 7eda18e..a2557a2 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -79,9 +79,9 @@ only allow specific CLI commands.
 It is also possible to configure varnishd for "reverse mode", using
 the '-M' argument.  In that case varnishd will attempt to open a
 TCP connection to the specified address, and initiate a CLI connection
-to your central varnish management facility.
+to your central Varnish management facility.
 
-The connection is also in this case without secrecy, but 
+The connection is also in this case without secrecy, but
 the remote end must still satisfy -S/PSK authentication.
 
 Finally, if you run varnishd with the '-d' option, you get a CLI
@@ -217,4 +217,4 @@ If you have "administrative" HTTP requests, for instance PURGE
 requests, we strongly recommend that you restrict them to trusted
 IP numbers/nets using VCL's Access Control Lists.
 
-(XXX: missing ref to ACL)
+.. (XXX: missing ref to ACL)
diff --git a/doc/sphinx/users-guide/sizing-your-cache.rst b/doc/sphinx/users-guide/sizing-your-cache.rst
index 7d48200..8f2dfba 100644
--- a/doc/sphinx/users-guide/sizing-your-cache.rst
+++ b/doc/sphinx/users-guide/sizing-your-cache.rst
@@ -19,7 +19,7 @@ task. A few things to consider:
 
 Be aware that every object that is stored also carries overhead that
 is kept outside the actually storage area. So, even if you specify -s
-malloc,16G varnish might actually use **double** that. Varnish has a
+malloc,16G Varnish might actually use **double** that. Varnish has a
 overhead of about 1k per object. So, if you have lots of small objects
 in your cache the overhead might be significant.
 
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index b6f6558..fa01890 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -5,7 +5,7 @@ Troubleshooting Varnish
 
 Sometimes Varnish misbehaves. In order for you to understand whats
 going on there are a couple of places you can check. varnishlog,
-/var/log/syslog, /var/log/messages are all places where varnish might
+/var/log/syslog, /var/log/messages are all places where Varnish might
 leave clues of whats going on. This section will guide you through
 basic troubleshooting in Varnish.
 
@@ -19,7 +19,7 @@ permissions on /dev/null to other processes blocking the ports.
 
 Starting Varnish in debug mode to see what is going on.
 
-Try to start varnish by::
+Try to start Varnish by::
 
     # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1: 2000  -a 0.0.0.0:8080 -d
 
@@ -31,7 +31,7 @@ listening on its port.::
     storage_malloc: max size 1024 MB.
     Using old SHMFILE
     Platform: Linux,2.6.32-21-generic,i686,-smalloc,-hcritbit
-    200 193     
+    200 193
     -----------------------------
     Varnish Cache CLI.
     -----------------------------
@@ -45,7 +45,7 @@ instruct the master process to start the cache by issuing "start".::
 
 	 start
 	 bind(): Address already in use
-	 300 22      
+	 300 22
 	 Could not open sockets
 
 And here we have our problem. Something else is bound to the HTTP port
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 7dc8c95..dc9a3cb 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -29,7 +29,7 @@ connect to port 8080 on localhost (127.0.0.1).
 
 Varnish can have several backends defined and can you can even join
 several backends together into clusters of backends for load balancing
-purposes. 
+purposes.
 
 
 Multiple backends
@@ -72,7 +72,7 @@ It's quite simple, really. Lets stop and think about this for a
 moment. As you can see you can define how you choose backends based on
 really arbitrary data. You want to send mobile devices to a different
 backend? No problem. if (req.User-agent ~ /mobile/) .. should do the
-trick. 
+trick.
 
 
 Backends and virtual hosts in Varnish
@@ -104,7 +104,7 @@ more tight, maybe relying on the == operator in stead, like this:::
     if (req.http.host == "foo.com" or req.http.host == "www.foo.com") {
       set req.backend = foo;
     }
-  } 
+  }
 
 
 .. _users-guide-advanced_backend_servers-directors:
@@ -181,7 +181,7 @@ Whats new here is the probe. Varnish will check the health of each
 backend with a probe. The options are
 
 url
- What URL should varnish request.
+ What URL should Varnish request.
 
 interval
  How often should we poll
@@ -193,10 +193,10 @@ window
  Varnish will maintain a *sliding window* of the results. Here the
  window has five checks.
 
-threshold 
+threshold
  How many of the .window last polls must be good for the backend to be declared healthy.
 
-initial 
+initial
  How many of the of the probes a good when Varnish starts - defaults
  to the same amount as the threshold.
 
@@ -206,11 +206,11 @@ Now we define the director.::
         {
                 .backend = server1;
         }
-        # server2 
+        # server2
         {
                 .backend = server2;
         }
-	
+
         }
 
 You use this director just as you would use any other director or
diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index d1c31e6..0ab6ced 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -23,20 +23,20 @@ vcl_recv
  been received and parsed.  Its purpose is to decide whether or not
  to serve the request, how to do it, and, if applicable, which backend
  to use.
-  
+
  The vcl_recv subroutine may terminate with calling ``return()`` on one of
  the following keywords:
 
   error code [reason]
     Return the specified error code to the client and abandon the request.
 
-  pass    
+  pass
     Switch to pass mode.  Control will eventually pass to vcl_pass.
 
-  pipe    
+  pipe
     Switch to pipe mode.  Control will eventually pass to vcl_pipe.
 
-  lookup  
+  lookup
     Look up the requested object in the cache.  Control will
     eventually pass to vcl_hit or vcl_miss, depending on whether the
     object is in the cache.  The ``bereq.method`` value will be set
@@ -50,7 +50,7 @@ vcl_pipe
  on to the backend, and any further data from either client or
  backend is passed on unaltered until either end closes the
  connection.
-  
+
  The vcl_pipe subroutine may terminate with calling return() with one of
  the following keywords:
 
@@ -67,10 +67,10 @@ vcl_pass
  on to the backend, and the backend's response is passed on to the
  client, but is not entered into the cache.  Subsequent requests
  submitted over the same client connection are handled normally.
-  
+
  The vcl_pass subroutine may terminate with calling return() with one
  of the following keywords:
-  
+
   error code [reason]
     Return the specified error code to the client and abandon the request.
 
@@ -78,8 +78,8 @@ vcl_pass
     Proceed with pass mode.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_miss
@@ -88,7 +88,7 @@ vcl_miss
 Called after a cache lookup if the requested document was not found in
 the cache.  Its purpose is to decide whether or not to attempt to
 retrieve the document from the backend, and which backend to use.
-  
+
 The vcl_miss subroutine may terminate with calling return() with one
 of the following keywords:
 
@@ -106,7 +106,7 @@ vcl_fetch
 ~~~~~~~~~
 
 Called after a document has been successfully retrieved from the backend.
-  
+
 The vcl_fetch subroutine may terminate with calling return() with one
 of the following keywords:
 
@@ -117,7 +117,7 @@ of the following keywords:
   error code [reason]
     Return the specified error code to the client and abandon the request.
 
-  hit_for_pass 
+  hit_for_pass
     Pass in fetch. Passes the object without caching it. This will
     create a so-called hit_for_pass object which has the side effect
     that the decision not to cache will be cached. This is to allow
@@ -131,15 +131,15 @@ of the following keywords:
     object.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_deliver
 ~~~~~~~~~~~
 
 Called before a cached object is delivered to the client.
-  
+
 The vcl_deliver subroutine may terminate with one of the following
 keywords:
 
@@ -147,25 +147,25 @@ keywords:
     Deliver the object to the client.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_error
 ~~~~~~~~~
 
-Called when we hit an error, either explicitly or implicitly due to 
+Called when we hit an error, either explicitly or implicitly due to
 backend or internal errors.
 
 The vcl_error subroutine may terminate by calling return with one of
 the following keywords:
- 
+
   deliver
     Deliver the error object to the client.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_fini
diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index f1fcd99..97e3a0d 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -11,7 +11,7 @@ Much of the VCL syntax has changed in Varnish 4. We've tried to compile a list o
 
 Version statement
 ~~~~~~~~~~~~~~~~~
-To make sure that people have upgraded their VCL to the current version, varnish now requires the first line of VCL to indicate the VCL version number::
+To make sure that people have upgraded their VCL to the current version, Varnish now requires the first line of VCL to indicate the VCL version number::
 
 	vcl 4.0;
 
@@ -30,7 +30,7 @@ Since the client director was already a special case of the hash director, it ha
         	h.add_backend(b1, 1);
         	h.add_backend(b2, 1);
 	}
-	
+
 	sub vcl_recv {
 		set req.backend = h.backend(client.ip);
 	}



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] 90a4471 Merge branch 'master' of
 git.varnish-cache.org:varnish-cache
Message-ID: <E1WO1sD-0006zd-Qo@project.varnish-software.com>


commit 90a4471b9c75ea82a46e8fdc357bea18e0bece36
Merge: e90fce3 3756761
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jan 27 14:09:23 2014 +0100

    Merge branch 'master' of git.varnish-cache.org:varnish-cache



From phk at FreeBSD.org  Thu Mar 13 09:24:21 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] 8815683 Minor polish
Message-ID: <E1WO1sD-0006zm-Rk@project.varnish-software.com>


commit 8815683613a7a1b3159bcf608a8336dda667d734
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 27 10:40:01 2014 +0000

    Minor polish

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 426faa7..477aa6a 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2014 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -1170,27 +1170,20 @@ ban_lurker(struct worker *wrk, void *priv)
 {
 	struct vsl_log vsl;
 	volatile double d;
-	int i;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AZ(priv);
 
 	VSL_Setup(&vsl, NULL, 0);
 
-	(void)priv;
 	while (!ban_shutdown) {
-		i = 0;
 		d = cache_param->ban_lurker_sleep;
-		if (d > 0.0)
-			i = ban_lurker_work(wrk, &vsl);
+		if (d <= 0.0 || !ban_lurker_work(wrk, &vsl))
+			d = 0.609;	// Random, non-magic
 		ban_cleantail();
-		if (ban_shutdown)
-			break;
-		if (i)
-			VTIM_sleep(d);
-		else
-			VTIM_sleep(0.609);	// Random, non-magic
+		VTIM_sleep(d);
 	}
-
 	pthread_exit(0);
-
 	NEEDLESS_RETURN(NULL);
 }
 



From phk at FreeBSD.org  Thu Mar 13 09:24:21 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] cfe912d Close a hypothetical loophole,
 that could let BAN'ed objects not get EXP'ed properly
Message-ID: <E1WO1sD-0006zv-Sh@project.varnish-software.com>


commit cfe912d93cc81d798142ff00231092e6c300d56e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 27 14:36:34 2014 +0000

    Close a hypothetical loophole, that could let BAN'ed objects not
    get EXP'ed properly

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 0126357..f1b65f3 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -245,7 +245,7 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.9f e=%.9f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
-	if (oc->timer_when == when)
+	if (when > o->exp.t_origin && when > oc->timer_when)
 		return;
 
 	lru = oc_getlru(oc);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index e9904b2..5d89c77 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -581,11 +581,6 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 			 */
 			continue;
 		}
-		xxxassert(spc > sizeof *ocp);
-
-		(void)oc_getobj(&wrk->stats, oc);
-		    /* XXX: still needed ? */
-
 		xxxassert(spc >= sizeof *ocp);
 		oc->refcnt++;
 		spc -= sizeof *ocp;



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] f2ea95a Run make check properly in parallel
Message-ID: <E1WO1sD-000704-Tl@project.varnish-software.com>


commit f2ea95a32cc351a58f82b03a006b43f9a1843769
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jan 28 14:01:24 2014 +0100

    Run make check properly in parallel

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index eaf92f3..a740b78 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -153,7 +153,7 @@ rm -rf doc/sphinx/\=build
 	%endif
 %endif
 
-make check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcc/.libs:../../lib/libvgz/.libs" TESTS_PARALLELISM=1 VERBOSE=1
+make check %{?_smp_mflags} LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcc/.libs:../../lib/libvgz/.libs" VERBOSE=1
 
 %install
 rm -rf %{buildroot}



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] fd870fb Minor whitespace fix
Message-ID: <E1WO1sD-00070D-Uh@project.varnish-software.com>


commit fd870fba0ecdcadcc63d6b7906c421f58aaebed8
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jan 28 14:22:05 2014 +0100

    Minor whitespace fix

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index a740b78..edb6f02 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -98,7 +98,7 @@ cp bin/varnishd/default.vcl examples
 
 export CFLAGS="$CFLAGS -Wp,-D_FORTIFY_SOURCE=0"
 
-# Remove "--disable static" if you want to build static libraries 
+# Remove "--disable static" if you want to build static libraries
 # jemalloc is not compatible with Red Hat's ppc64 RHEL kernel :-(
 %ifarch ppc64 ppc
 	%configure --disable-static --localstatedir=/var/lib --without-jemalloc  --without-rst2man --without-rst2html



From lkarsten at varnish-software.com  Thu Mar 13 09:24:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:21 +0100
Subject: [4.0] 2bb6d93 Package builtin.vcl and example.vcl correctly.
Message-ID: <E1WO1sD-00070M-Vd@project.varnish-software.com>


commit 2bb6d93a2e00a4677683f0b52fa99390b97a0a63
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jan 28 14:22:58 2014 +0100

    Package builtin.vcl and example.vcl correctly.
    
    New world order says that builtin.vcl and example.vcl are
    documentation example, and that example.vcl should be the file
    that the init script try to use when starting varnishd.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index edb6f02..0b53613 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -87,7 +87,7 @@ Documentation files for %name
 #%setup -q -n varnish-trunk
 
 mkdir examples
-cp bin/varnishd/default.vcl examples
+cp etc/builtin.vcl etc/example.vcl examples
 
 %build
 # No pkgconfig/libpcre.pc in rhel4
@@ -113,17 +113,6 @@ export CFLAGS="$CFLAGS -Wp,-D_FORTIFY_SOURCE=0"
 
 make %{?_smp_mflags} V=1
 
-head -6 etc/default.vcl > redhat/default.vcl
-
-cat << EOF >> redhat/default.vcl
-backend default {
-  .host = "127.0.0.1";
-  .port = "80";
-}
-EOF
-
-tail -n +11 etc/default.vcl >> redhat/default.vcl
-
 %if 0%{?fedora}%{?rhel} != 0 && 0%{?rhel} <= 4 && 0%{?fedora} <= 8
 	# Old style daemon function
 	sed -i 's,--pidfile \$pidfile,,g;
@@ -169,7 +158,7 @@ mkdir -p %{buildroot}/var/lib/varnish
 mkdir -p %{buildroot}/var/log/varnish
 mkdir -p %{buildroot}/var/run/varnish
 mkdir -p %{buildroot}%{_sysconfdir}/ld.so.conf.d/
-install -D -m 0644 redhat/default.vcl %{buildroot}%{_sysconfdir}/varnish/default.vcl
+install -D -m 0644 etc/example.vcl %{buildroot}%{_sysconfdir}/varnish/default.vcl
 install -D -m 0644 redhat/varnish.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/varnish
 
 # systemd support



From lkarsten at varnish-software.com  Thu Mar 13 09:24:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] a93acd2 Package all new example VCL files
Message-ID: <E1WO1sE-00070V-0R@project.varnish-software.com>


commit a93acd2fb33bb40b1e9c37f2b632e0ebc55e3663
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jan 28 15:19:32 2014 +0100

    Package all new example VCL files

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0b53613..c441033 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -87,7 +87,6 @@ Documentation files for %name
 #%setup -q -n varnish-trunk
 
 mkdir examples
-cp etc/builtin.vcl etc/example.vcl examples
 
 %build
 # No pkgconfig/libpcre.pc in rhel4
@@ -193,7 +192,8 @@ rm -rf %{buildroot}
 %{_mandir}/man3/*.3*
 %{_mandir}/man7/*.7*
 %doc LICENSE README redhat/README.redhat ChangeLog
-%doc examples
+%doc examples/*
+%{_docdir}/varnish/
 %dir %{_sysconfdir}/varnish/
 %config(noreplace) %{_sysconfdir}/varnish/default.vcl
 %config(noreplace) %{_sysconfdir}/logrotate.d/varnish



From lkarsten at varnish-software.com  Thu Mar 13 09:24:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 4f4e0bd Hopefully last nitpicking packaging commit for now
Message-ID: <E1WO1sE-00070e-1L@project.varnish-software.com>


commit 4f4e0bda9486aab9ab5264f0116353f04d78e954
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jan 28 15:39:11 2014 +0100

    Hopefully last nitpicking packaging commit for now

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index c441033..451d131 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -86,8 +86,6 @@ Documentation files for %name
 %setup -n varnish-%{version}%{?vd_rc}
 #%setup -q -n varnish-trunk
 
-mkdir examples
-
 %build
 # No pkgconfig/libpcre.pc in rhel4
 %if 0%{?rhel} == 4
@@ -192,7 +190,6 @@ rm -rf %{buildroot}
 %{_mandir}/man3/*.3*
 %{_mandir}/man7/*.7*
 %doc LICENSE README redhat/README.redhat ChangeLog
-%doc examples/*
 %{_docdir}/varnish/
 %dir %{_sysconfdir}/varnish/
 %config(noreplace) %{_sysconfdir}/varnish/default.vcl



From phk at FreeBSD.org  Thu Mar 13 09:24:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 6df57da Things you can do while hanging on the telephone: Make
 sure lines wrap properly and remove trailing blank lines.
Message-ID: <E1WO1sE-00070n-2g@project.varnish-software.com>


commit 6df57da0060f74902f8c8613bb18f664eaa6cc6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 28 19:10:45 2014 +0000

    Things you can do while hanging on the telephone:  Make sure lines
    wrap properly and remove trailing blank lines.

diff --git a/Makefile.inc.phk b/Makefile.inc.phk
index 5c25d59..216dcad 100644
--- a/Makefile.inc.phk
+++ b/Makefile.inc.phk
@@ -7,7 +7,8 @@ WARNS	?=	1
 .PHONY: depend all clean install test
 depend all clean install test:
 	@$(MAKE) --no-print-directory -f Makefile.phk TGT=$@ real-$@ \
-	    $(shell env CC=$(CC) WARNS=$(WARNS) sh ${TOPDIR}/config.phk $(TOPDIR) $(CURDIR) )
+	    $(shell env CC=$(CC) WARNS=$(WARNS) \
+	    sh ${TOPDIR}/config.phk $(TOPDIR) $(CURDIR) )
 
 CFLAGS += $(CF_CFLAGS) $(CF_CWFLAGS)
 CFLAGS += -I$(CURDIR)
@@ -241,4 +242,3 @@ real-test: $(TODO_TEST)
 	@true
 
 #######################################################################
-
diff --git a/Makefile.phk b/Makefile.phk
index 237992c..01d3e8c 100644
--- a/Makefile.phk
+++ b/Makefile.phk
@@ -12,4 +12,3 @@ top_install:
 	@for d in man doc etc bin lib libexec ; do \
 		mkdir -p $(INSTALL_BASE)/$$d ; \
 	done
-	
diff --git a/autogen.des b/autogen.des
index d546bc7..a385c71 100755
--- a/autogen.des
+++ b/autogen.des
@@ -18,6 +18,7 @@ rm -f configure
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 export CONFIG_SHELL=/bin/sh
 
+env MAKE=gmake \
 ./configure \
     --enable-developer-warnings \
     --enable-debugging-symbols \
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 24c6eab..5e0dcd2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -257,7 +257,8 @@ enum vfp_status {
 	VFP_OK = 0,
 	VFP_END = 1,
 };
-typedef enum vfp_status vfp_pull_f(struct busyobj *bo, void *p, ssize_t *len, intptr_t *priv);
+typedef enum vfp_status
+    vfp_pull_f(struct busyobj *bo, void *p, ssize_t *len, intptr_t *priv);
 
 extern vfp_pull_f vfp_gunzip_pull;
 extern vfp_pull_f vfp_gzip_pull;
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index e5f3198..5a43c2e 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -455,9 +455,11 @@ cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
 
 static struct cli_proto backend_cmds[] = {
 	{ "backend.list", "backend.list",
-	    "\tList all backends\n", 0, 1, "", cli_backend_list },
+	    "\tList all backends\n",
+	    0, 1, "", cli_backend_list },
 	{ "backend.set_health", "backend.set_health matcher state",
-	    "\tSet health status on a backend\n", 2, 2, "", cli_backend_set_health },
+	    "\tSet health status on a backend\n",
+	    2, 2, "", cli_backend_set_health },
 	{ NULL }
 };
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ebbff7e..9bbc6c1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -551,7 +551,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->state != BOS_FAILED)
 		VBO_setstate(bo, BOS_FINISHED);
 
-VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
+VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d",
+    bo->refcount, bo->fetch_obj->objcore->refcnt);
 	return (F_STP_DONE);
 }
 
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 89e858b..e25ced9 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -389,4 +389,3 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	}
 	return (0);
 }
-
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 12a4ba6..8d61a64 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -539,4 +539,3 @@ HTTP1_Write(const struct worker *w, const struct http *hp, int resp)
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
 }
-
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 49c6712..3cfb36f 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -112,8 +112,10 @@ pan_ws(const struct ws *ws, int indent)
 	if (VALID_OBJ(ws, WS_MAGIC)) {
 		if (WS_Overflowed(ws))
 			VSB_printf(pan_vsp, " OVERFLOW");
-		VSB_printf(pan_vsp, "\n%*sid = \"%s\",\n", indent + 2, "", ws->id);
-		VSB_printf(pan_vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
+		VSB_printf(pan_vsp,
+		    "\n%*sid = \"%s\",\n", indent + 2, "", ws->id);
+		VSB_printf(pan_vsp,
+		    "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
 		if (ws->f > ws->s)
 			VSB_printf(pan_vsp, ",+%ld", (long) (ws->f - ws->s));
 		else
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index a14e6b2..f38c84d 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -65,7 +65,6 @@ enum lookup_e {
 void HSH_Cleanup(struct worker *w);
 enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **,
     int wait_for_busy, int always_insert);
-// struct objcore *HSH_Lookup(struct req *, int wait_for_busy, int always_insert);
 void HSH_Ref(struct objcore *o);
 void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 621129d..b5aec87 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -57,31 +57,38 @@ static const int tab0 = 3;
 /*--------------------------------------------------------------------*/
 
 static const char OBJ_STICKY_TEXT[] =
-	"\n\nNB: This parameter is evaluated only when objects are created."
+	"\n\n"
+	"NB: This parameter is evaluated only when objects are created."
 	"To change it for all objects, restart or ban everything.";
 
 static const char DELAYED_EFFECT_TEXT[] =
-	"\n\nNB: This parameter may take quite some time to take (full) effect.";
+	"\n\n"
+	"NB: This parameter may take quite some time to take (full) effect.";
 
 static const char MUST_RESTART_TEXT[] =
-	"\n\nNB: This parameter will not take any effect until the "
+	"\n\n"
+	"NB: This parameter will not take any effect until the "
 	"child process has been restarted.";
 
 static const char MUST_RELOAD_TEXT[] =
-	"\n\nNB: This parameter will not take any effect until the "
+	"\n\n"
+	"NB: This parameter will not take any effect until the "
 	"VCL programs have been reloaded.";
 
 static const char EXPERIMENTAL_TEXT[] =
-	"\n\nNB: We do not know yet if it is a good idea to change "
+	"\n\n"
+	"NB: We do not know yet if it is a good idea to change "
 	"this parameter, or if the default value is even sensible.  "
 	"Caution is advised, and feedback is most welcome.";
 
 static const char WIZARD_TEXT[] =
-	"\n\nNB: Do not change this parameter, unless a developer tell "
+	"\n\n"
+	"NB: Do not change this parameter, unless a developer tell "
 	"you to do so.";
 
 static const char PROTECTED_TEXT[] =
-	"\n\nNB: This parameter is protected and can not be changed.";
+	"\n\n"
+	"NB: This parameter is protected and can not be changed.";
 
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index c3dc8a5..255b99c 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -388,7 +388,10 @@ mgt_sandbox_solaris_waive(enum sandbox_e who)
 		return;
 	}
 
-	/* simple scheme: (inheritable subset-of effective) subset-of permitted */
+	/*
+	 * simple scheme:
+	 *     (inheritable subset-of effective) subset-of permitted
+	 */
 
 	priv_emptyset(inheritable);
 	mgt_sandbox_solaris_add_inheritable(inheritable, who);
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
index 92f531b..081b93a 100644
--- a/bin/varnishd/storage/stevedore_utils.c
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -205,7 +205,7 @@ STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx)
 
 		if (l < 1024*1024)
 			ARGV_ERR("(%s) size \"%s\": too small, "
-				 "did you forget to specify M or G?\n", ctx, size);
+			    "did you forget to specify M or G?\n", ctx, size);
 	}
 
 	/*
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index d00ec7c..99c54fa 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -630,7 +630,8 @@ varnish_cli(struct varnish *v, const char *cli, unsigned exp)
  */
 
 static void
-varnish_vcl(struct varnish *v, const char *vcl, enum VCLI_status_e expect, char **resp)
+varnish_vcl(struct varnish *v, const char *vcl, enum VCLI_status_e expect,
+    char **resp)
 {
 	struct vsb *vsb;
 	enum VCLI_status_e u;
diff --git a/config.phk b/config.phk
index aa6856f..16c3571 100644
--- a/config.phk
+++ b/config.phk
@@ -210,7 +210,8 @@ ask_compiler CF_CFLAGS '
 		"-D_GNU_SOURCE=1"
 	#endif
 
-	"-DVCC_CC=~\~exec $(CC) -D_THREAD_SAFE -std=gnu99 -g -O2 -Wall -Werror -pthread -fpic -shared -Wl,-x -o %o %s\~~"
+	"-DVCC_CC=~\~exec $(CC) -D_THREAD_SAFE -std=gnu99 -g " \
+	"-O2 -Wall -Werror -pthread -fpic -shared -Wl,-x -o %o %s\~~"
 
 	"-DVCC_WARNS=~\~$(CF_CWFLAGS)\~~"
 
@@ -295,4 +296,3 @@ exit 0
 # _FILE_OFFSET_BITS -
 # 
 # #echo 'CF_CFLAGS="-Wall"'
-
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 0f4829b..5dee431 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -42,7 +42,8 @@
  *	Long Description (in RST "definition list" format)
  */
 
-#define NODEF_NOTICE "Note: This log record is masked by default and must be explicitly enabled\n\n"
+#define NODEF_NOTICE \
+    "NB: This log record is masked by default.\n\n"
 
 SLTM(Debug, SLT_F_BINARY, "Debug messages",
 	"Debug messages can normally be ignored, but are sometimes"
@@ -206,8 +207,8 @@ SLTM(FetchError, 0, "Error while fetching object",
 #undef SLTH
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Beresp##tag, (resp ? 0 : SLT_F_UNUSED), "Backend response " sdesc, \
-	    ldesc)
+	SLTM(Beresp##tag, (resp ? 0 : SLT_F_UNUSED), "Backend response " \
+	    sdesc, ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index d0904eb..88be5d9 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -59,8 +59,10 @@ int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
 	 *	 1 Handled.
 	 */
 
-struct VSC_C_mgt *VSC_Mgt(const struct VSM_data *vd, struct VSM_fantom *fantom);
-struct VSC_C_main *VSC_Main(const struct VSM_data *vd, struct VSM_fantom *fantom);
+struct VSC_C_mgt *VSC_Mgt(const struct VSM_data *vd,
+    struct VSM_fantom *fantom);
+struct VSC_C_main *VSC_Main(const struct VSM_data *vd,
+    struct VSM_fantom *fantom);
 	/*
 	 * Looks up and returns the management stats and the child main
 	 * stats structure. If fantom is non-NULL, it can later be used
@@ -80,8 +82,8 @@ struct VSC_C_main *VSC_Main(const struct VSM_data *vd, struct VSM_fantom *fantom
 	 *  non-NULL: Success
 	 */
 
-void *VSC_Get(const struct VSM_data *vd, struct VSM_fantom *fantom, const char *type,
-    const char *ident);
+void *VSC_Get(const struct VSM_data *vd, struct VSM_fantom *fantom,
+    const char *type, const char *ident);
 	/*
 	 * Looks up the given VSC type and identifier. If fantom is
 	 * non-NULL, it can if successful later be used with
diff --git a/include/vrnd.h b/include/vrnd.h
index ac46c0d..7cfab1d 100644
--- a/include/vrnd.h
+++ b/include/vrnd.h
@@ -29,4 +29,3 @@
  */
 
 void VRND_Seed(void);		/* Seed random(3) properly */
-
diff --git a/include/vrt.h b/include/vrt.h
index 3d465c6..9b3664a 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -209,7 +209,8 @@ void VRT_error(const struct vrt_ctx *, unsigned, const char *);
 int VRT_switch_config(const char *);
 
 char *VRT_GetHdr(const struct vrt_ctx *, const struct gethdr_s *);
-void VRT_SetHdr(const struct vrt_ctx *, const struct gethdr_s *, const char *, ...);
+void VRT_SetHdr(const struct vrt_ctx *, const struct gethdr_s *,
+    const char *, ...);
 void VRT_handling(const struct vrt_ctx *, unsigned hand);
 
 void VRT_hashdata(const struct vrt_ctx *, const char *str, ...);
diff --git a/lib/libvarnish/Makefile.phk b/lib/libvarnish/Makefile.phk
index 677e687..2a0c37a 100644
--- a/lib/libvarnish/Makefile.phk
+++ b/lib/libvarnish/Makefile.phk
@@ -27,4 +27,3 @@ LIB_SRC += vtim.c
 
 TOPDIR= $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
-
diff --git a/lib/libvarnishapi/Makefile.phk b/lib/libvarnishapi/Makefile.phk
index 8e86df4..2c3e907 100644
--- a/lib/libvarnishapi/Makefile.phk
+++ b/lib/libvarnishapi/Makefile.phk
@@ -33,5 +33,3 @@ include $(TOPDIR)/Makefile.inc.phk
 
 $(MADE_FILES):  generate.py
 	python generate.py
-
-
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index b9455db..363085a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -163,7 +163,8 @@ VSL_ResetError(struct VSL_data *vsl)
 }
 
 static int
-vsl_match_IX(struct VSL_data *vsl, const vslf_list *list, const struct VSL_cursor *c)
+vsl_match_IX(struct VSL_data *vsl, const vslf_list *list,
+    const struct VSL_cursor *c)
 {
 	enum VSL_tag_e tag;
 	const char *cdata;
diff --git a/lib/libvarnishcompat/execinfo.c b/lib/libvarnishcompat/execinfo.c
index 7babdd4..a9b4471 100644
--- a/lib/libvarnishcompat/execinfo.c
+++ b/lib/libvarnishcompat/execinfo.c
@@ -140,7 +140,7 @@ getreturnaddr(int level)
 {
 
 	switch(level) {
-#define DO_P2_TIMES_0(x)	case (x): return __builtin_return_address((x) + 1)
+#define DO_P2_TIMES_0(x)  case (x): return __builtin_return_address((x) + 1)
 	DO_P2_TIMES_7(0);
 #undef DO_P2_TIMES_0
 	default: return NULL;
@@ -152,7 +152,7 @@ getframeaddr(int level)
 {
 
 	switch(level) {
-#define DO_P2_TIMES_0(x)	case (x): return __builtin_frame_address((x) + 1)
+#define DO_P2_TIMES_0(x)  case (x): return __builtin_frame_address((x) + 1)
 	DO_P2_TIMES_7(0);
 #undef DO_P2_TIMES_0
 	default: return NULL;
diff --git a/lib/libvarnishtools/Makefile.phk b/lib/libvarnishtools/Makefile.phk
index 366a04d..0781a45 100644
--- a/lib/libvarnishtools/Makefile.phk
+++ b/lib/libvarnishtools/Makefile.phk
@@ -4,4 +4,3 @@ LIB_SRC += vut.c
 
 TOPDIR= $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
-
diff --git a/lib/libvcc/Makefile.phk b/lib/libvcc/Makefile.phk
index 131f850..333bad4 100644
--- a/lib/libvcc/Makefile.phk
+++ b/lib/libvcc/Makefile.phk
@@ -23,4 +23,3 @@ include $(TOPDIR)/Makefile.inc.phk
 
 $(MADE_FILES):	generate.py
 	$(PYTHON) generate.py
-
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index dbffe99..a36c2d4 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -78,19 +78,58 @@ tokens = {
 # Our methods and actions
 
 returns =(
-	('recv',		"C", ('error', 'pass', 'pipe', 'hash', 'purge',)),
-	('pipe',		"C", ('error', 'pipe',)),
-	('pass',		"C", ('error', 'restart', 'fetch',)),
-	('hash',		"C", ('lookup',)),
-	('purge',		"C", ('error', 'fetch',)),
-	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
-	('hit',			"C", ('error', 'restart', 'pass', 'fetch', 'deliver',)),
-	('backend_fetch',	"B", ('fetch', 'abandon')),
-	('backend_response',	"B", ('deliver', 'retry', 'abandon')),
-	('deliver',		"C", ('restart', 'deliver',)),
-	('error',		"C", ('restart', 'deliver',)),
-	('init',		"", ('ok',)),
-	('fini',		"", ('ok',)),
+	('recv',
+		"C",
+		('error', 'pass', 'pipe', 'hash', 'purge',)
+	),
+	('pipe',
+		"C",
+		('error', 'pipe',)
+	),
+	('pass',
+		"C",
+		('error', 'restart', 'fetch',)
+	),
+	('hash',
+		"C",
+		('lookup',)
+	),
+	('purge',
+		"C",
+		('error', 'fetch',)
+	),
+	('miss',
+		"C",
+		('error', 'restart', 'pass', 'fetch',)
+	),
+	('hit',
+		"C",
+		('error', 'restart', 'pass', 'fetch', 'deliver',)
+	),
+	('backend_fetch',
+		"B",
+		('fetch', 'abandon')
+	),
+	('backend_response',
+		"B",
+		('deliver', 'retry', 'abandon')
+	),
+	('deliver',
+		"C",
+		('restart', 'deliver',)
+	),
+	('error',
+		"C",
+		('restart', 'deliver',)
+	),
+	('init',
+		"",
+		('ok',)
+	),
+	('fini',
+		"",
+		('ok',)
+	),
 )
 
 #######################################################################
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 6f473e9..1ab670a 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -352,7 +352,8 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 	struct token *t;
 
 	Fh(tl, 0, "\nstatic int\n");
-	Fh(tl, 0, "match_acl_%s_%s(const struct vrt_ctx *ctx, const VCL_IP p)\n",
+	Fh(tl, 0,
+	    "match_acl_%s_%s(const struct vrt_ctx *ctx, const VCL_IP p)\n",
 	    anon ? "anon" : "named", acln);
 	Fh(tl, 0, "{\n");
 	Fh(tl, 0, "\tconst unsigned char *a;\n");
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 5b8672a..959f203 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -131,8 +131,6 @@ class vmod(object):
 			fo.write("int " + self.init)
 			fo.write(
 			    "(struct vmod_priv *, const struct VCL_conf *);\n")
-		#fo.write("\n")
-		#fo.write("extern const void * const Vmod_" + self.nam + "_Id;\n")
 
 	def c_typedefs_(self):
 		l = list()
diff --git a/lib/libvmod_debug/Makefile.phk b/lib/libvmod_debug/Makefile.phk
index 98e52e7..a692640 100644
--- a/lib/libvmod_debug/Makefile.phk
+++ b/lib/libvmod_debug/Makefile.phk
@@ -4,4 +4,3 @@ VMOD_SRC += vmod_debug_obj.c
 
 TOPDIR= $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
-
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 8d2919e..4f74a24 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -78,4 +78,3 @@ Foo indeed.
 $Method TIME .date()
 
 You never know when you need a date.
-
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 46811d6..7109644 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -129,4 +129,3 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 	vmod_priv_fini(b);
 	return (s);
 }
-
diff --git a/lib/libvmod_directors/Makefile.phk b/lib/libvmod_directors/Makefile.phk
index 424a275..bf0a921 100644
--- a/lib/libvmod_directors/Makefile.phk
+++ b/lib/libvmod_directors/Makefile.phk
@@ -7,4 +7,3 @@ VMOD_SRC += vdir.c
 
 TOPDIR= $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
-
diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c
index dacd648..c8a8c84 100644
--- a/lib/libvmod_directors/vdir.c
+++ b/lib/libvmod_directors/vdir.c
@@ -204,4 +204,3 @@ vdir_pick_be(struct vdir *vd, double w, unsigned nloops)
 	vdir_unlock(vd);
 	return (be);
 }
-
diff --git a/lib/libvmod_std/Makefile.phk b/lib/libvmod_std/Makefile.phk
index ca093a0..1b3e34b 100644
--- a/lib/libvmod_std/Makefile.phk
+++ b/lib/libvmod_std/Makefile.phk
@@ -5,4 +5,3 @@ VMOD_SRC += vmod_std_fileread.c
 
 TOPDIR= $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
-
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index 5ddc6bf..4fce9a2 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -83,7 +83,8 @@ free_frfile(void *ptr)
 }
 
 VCL_STRING __match_proto__(td_std_fileread)
-vmod_fileread(const struct vrt_ctx *ctx, struct vmod_priv *priv, VCL_STRING file_name)
+vmod_fileread(const struct vrt_ctx *ctx, struct vmod_priv *priv,
+    VCL_STRING file_name)
 {
 	struct frfile *frf = NULL;
 	char *s;



From phk at FreeBSD.org  Thu Mar 13 09:24:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 4fe2ad4 Add a WS_Printf() function and use it.
Message-ID: <E1WO1sE-00070x-3j@project.varnish-software.com>


commit 4fe2ad4a339bcb89980f4859e97b05006d13b4cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 28 20:11:20 2014 +0000

    Add a WS_Printf() function and use it.
    
    Other minor polish around workspaces.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5e0dcd2..6e522bb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1197,6 +1197,7 @@ char *WS_Alloc(struct ws *ws, unsigned bytes);
 void *WS_Copy(struct ws *ws, const void *str, int len);
 char *WS_Snapshot(struct ws *ws);
 int WS_Overflowed(const struct ws *ws);
+void *WS_Printf(struct ws *ws, const char *fmt, ...) __printflike(2, 3);
 
 /* rfc2616.c */
 void RFC2616_Ttl(struct busyobj *);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 6c3abbc..ec28396 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -718,20 +718,17 @@ static void
 http_PutField(const struct http *to, int field, const char *string)
 {
 	char *p;
-	unsigned l;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	l = strlen(string);
-	p = WS_Alloc(to->ws, l + 1);
+	p = WS_Copy(to->ws, string, -1);
 	if (p == NULL) {
 		VSLb(to->vsl, SLT_LostHeader, "%s", string);
 		to->hd[field].b = NULL;
 		to->hd[field].e = NULL;
 		to->hdf[field] = 0;
 	} else {
-		memcpy(p, string, l + 1L);
 		to->hd[field].b = p;
-		to->hd[field].e = p + l;
+		to->hd[field].e = strchr(p, '\0');
 		to->hdf[field] = 0;
 		http_VSLH(to, field);
 	}
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 8d61a64..a896546 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -517,6 +517,7 @@ HTTP1_Write(const struct worker *w, const struct http *hp, int resp)
 		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(hp->ws, 4);
 		AN(hp->hd[HTTP_HDR_STATUS].b);
 
+		assert(hp->status >= 100 && hp->status <= 999);
 		sprintf(hp->hd[HTTP_HDR_STATUS].b, "%3d", hp->status);
 		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 3cfb36f..81b794d 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -109,7 +109,10 @@ pan_ws(const struct ws *ws, int indent)
 {
 
 	VSB_printf(pan_vsp, "%*sws = %p {", indent, "", ws);
-	if (VALID_OBJ(ws, WS_MAGIC)) {
+	if (!VALID_OBJ(ws, WS_MAGIC)) {
+		if (ws != NULL)
+			VSB_printf(pan_vsp, " BAD_MAGIC(0x%08x) ", ws->magic);
+	} else {
 		if (WS_Overflowed(ws))
 			VSB_printf(pan_vsp, " OVERFLOW");
 		VSB_printf(pan_vsp,
@@ -128,8 +131,6 @@ pan_ws(const struct ws *ws, int indent)
 			VSB_printf(pan_vsp, ",+%ld", (long) (ws->e - ws->s));
 		else
 			VSB_printf(pan_vsp, ",%p", ws->e);
-	} else {
-		VSB_printf(pan_vsp, " BAD_MAGIC(0x%08x) ", ws->magic);
 	}
 	VSB_printf(pan_vsp, "},\n");
 	VSB_printf(pan_vsp, "%*s},\n", indent, "" );
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 7138da5..3d45e4c 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -313,27 +313,17 @@ VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip)
 char *
 VRT_INT_string(const struct vrt_ctx *ctx, long num)
 {
-	char *p;
-	int size;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	size = snprintf(NULL, 0, "%ld", num) + 1;
-	AN(p = WS_Alloc(ctx->ws, size));
-	assert(snprintf(p, size, "%ld", num) < size);
-	return (p);
+	return (WS_Printf(ctx->ws, "%ld", num));
 }
 
 char *
 VRT_REAL_string(const struct vrt_ctx *ctx, double num)
 {
-	char *p;
-	int size;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	size = snprintf(NULL, 0, "%.3f", num) + 1;
-	AN(p = WS_Alloc(ctx->ws, size));
-	assert(snprintf(p, size, "%.3f", num) < size);
-	return (p);
+	return (WS_Printf(ctx->ws, "%.3f", num));
 }
 
 char *
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index fb2d481..dad35f3 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -471,16 +471,12 @@ VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0, ctx->bo->exp.t_origin,)
 const char *
 VRT_r_req_xid(const struct vrt_ctx *ctx)
 {
-	char *p;
-	int size;
+
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 
-	// XXX ?
-	size = snprintf(NULL, 0, "%u", ctx->req->vsl->wid & VSL_IDENTMASK) + 1;
-	AN(p = WS_Alloc(ctx->req->http->ws, size));
-	assert(snprintf(p, size, "%u", ctx->req->vsl->wid & VSL_IDENTMASK) < size);
-	return (p);
+	return (WS_Printf(ctx->req->http->ws, "%u",
+	    ctx->req->vsl->wid & VSL_IDENTMASK));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 0ee8124..5ed9040 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -30,6 +30,9 @@
 
 #include "config.h"
 
+#include <stdio.h>
+#include <stdarg.h>
+
 #include "cache.h"
 
 void
@@ -159,6 +162,28 @@ WS_Copy(struct ws *ws, const void *str, int len)
 	return (r);
 }
 
+void *
+WS_Printf(struct ws *ws, const char *fmt, ...)
+{
+	unsigned u, v;
+	va_list ap;
+	char *p;
+
+	WS_Assert(ws);
+	assert(ws->r == NULL);
+	u = WS_Reserve(ws, 0);
+	p = ws->f;
+	va_start(ap, fmt);
+	v = vsnprintf(p, u, fmt, ap);
+	if (v > u) {
+		WS_Release(ws, 0);
+		p = NULL;
+	} else {
+		WS_Release(ws, v);
+	}
+	return (p);
+}
+
 char *
 WS_Snapshot(struct ws *ws)
 {



From lkarsten at varnish-software.com  Thu Mar 13 09:24:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 2011a6b Use gmake as a fallback if make doesn't exist
Message-ID: <E1WO1sE-000717-4d@project.varnish-software.com>


commit 2011a6b2d87fae2042c6b5183ef41bb41a5b1bd1
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Jan 29 10:14:14 2014 +0100

    Use gmake as a fallback if make doesn't exist
    
    gmake doesn't exist on Linux servers.

diff --git a/autogen.des b/autogen.des
index a385c71..07afad4 100755
--- a/autogen.des
+++ b/autogen.des
@@ -18,7 +18,10 @@ rm -f configure
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 export CONFIG_SHELL=/bin/sh
 
-env MAKE=gmake \
+if [ ! -x "/usr/bin/xmake" -a "x${MAKE}" == "x"]; then
+    export MAKE=gmake
+fi
+
 ./configure \
     --enable-developer-warnings \
     --enable-debugging-symbols \



From phk at FreeBSD.org  Thu Mar 13 09:24:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] bf4dbd5 Don't leak the workaround I need to run distcheck on
 FreeBSD
Message-ID: <E1WO1sE-00071G-5e@project.varnish-software.com>


commit bf4dbd5fc926cfbc2f83addafcc4fa07bf53df82
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 29 09:31:10 2014 +0000

    Don't leak the workaround I need to run distcheck on FreeBSD

diff --git a/autogen.des b/autogen.des
index a385c71..8c2a3b2 100755
--- a/autogen.des
+++ b/autogen.des
@@ -18,7 +18,7 @@ rm -f configure
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 export CONFIG_SHELL=/bin/sh
 
-env MAKE=gmake \
+# env MAKE=gmake \
 ./configure \
     --enable-developer-warnings \
     --enable-debugging-symbols \



From phk at FreeBSD.org  Thu Mar 13 09:24:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 8152f09 Merge branch 'master' of
 git.varnish-cache.org:varnish-cache
Message-ID: <E1WO1sE-00071P-6a@project.varnish-software.com>


commit 8152f09f0e4f9a74fbbd5931d027ad93c175fac3
Merge: bf4dbd5 2011a6b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 29 09:49:11 2014 +0000

    Merge branch 'master' of git.varnish-cache.org:varnish-cache
    
    Conflicts:
    	autogen.des



From phk at FreeBSD.org  Thu Mar 13 09:24:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 08e2b3a Document why this hack is needed.
Message-ID: <E1WO1sE-00071Y-7c@project.varnish-software.com>


commit 08e2b3a80049c6a16bef95b0615639e556a709fe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 29 09:50:27 2014 +0000

    Document why this hack is needed.

diff --git a/autogen.des b/autogen.des
index 8c2a3b2..0cbbbc0 100755
--- a/autogen.des
+++ b/autogen.des
@@ -18,6 +18,8 @@ rm -f configure
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 export CONFIG_SHELL=/bin/sh
 
+# NB: Workaround for make distcheck not working with
+# NB: FreeBSD's make on -current
 # env MAKE=gmake \
 ./configure \
     --enable-developer-warnings \



From phk at FreeBSD.org  Thu Mar 13 09:24:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 4067f50 Don't put objcores on the ban list until they go
 non-BUSY.
Message-ID: <E1WO1sE-00071h-8g@project.varnish-software.com>


commit 4067f504827b349ee61d481b5218b1e1b09afed1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 29 11:18:33 2014 +0000

    Don't put objcores on the ban list until they go non-BUSY.
    
    Fixes	#1419

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 477aa6a..1c6b0c8 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1030,6 +1030,7 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
 				 * We got the lock, and the oc is not being
 				 * dismantled under our feet, run with it...
 				 */
+				AZ(oc->flags & OC_F_BUSY);
 				oc->refcnt += 1;
 				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
 				VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 5d89c77..eb69164 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -667,10 +667,6 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
 
 	AN(oc->flags & OC_F_BUSY);
-	if (oc->flags & OC_F_PRIVATE)
-		AZ(oc->ban);
-	else
-		AN(oc->ban);
 	assert(oh->refcnt > 0);
 
 	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
@@ -683,6 +679,8 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	if (oh->waitinglist != NULL)
 		hsh_rush(ds, oh);
 	Lck_Unlock(&oh->mtx);
+	if (!(oc->flags & OC_F_PRIVATE))
+		BAN_NewObjCore(oc);
 }
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index e1cc3c8..c2788a9 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -291,8 +291,6 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 	bo->stats->n_object++;
 
 	o->objcore = bo->fetch_objcore;
-	if (!(o->objcore->flags & OC_F_PRIVATE))
-		BAN_NewObjCore(o->objcore);
 
 	o->objcore->methods = &default_oc_methods;
 	o->objcore->priv = o;
diff --git a/bin/varnishtest/tests/r01419.vtc b/bin/varnishtest/tests/r01419.vtc
new file mode 100644
index 0000000..5165c63
--- /dev/null
+++ b/bin/varnishtest/tests/r01419.vtc
@@ -0,0 +1,43 @@
+varnishtest "Make sure banlurker skips busy objects"
+
+server s1 {
+	rxreq
+	send "HTTP/1.0 200 Ok\r\n"
+	sema r1 sync 2
+	send "Foobar: blaf\r\n"
+	send "Content-Length: 10\r\n"
+	send "\r\n\r\n"
+	send "abcde"
+	sema r2 sync 2
+	send "abcdefghij"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v1 -cliok {param.set debug +lurker}
+varnish v1 -cliok {param.set ban_lurker_age 1}
+varnish v1 -cliok {ban.list}
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.foobar == blaf
+} -start
+
+sema r1 sync 2
+varnish v1 -cliok {ban.list}
+
+varnish v1 -cliok {ban obj.http.goo == bar}
+varnish v1 -cliok {ban.list}
+delay 2
+varnish v1 -cliok {ban.list}
+
+
+sema r2 sync 2
+
+client c1 -wait



From lkarsten at varnish-software.com  Thu Mar 13 09:24:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] ee55121 Replace non-breaking spaces to make pdflatex happy
Message-ID: <E1WO1sE-00071q-9d@project.varnish-software.com>


commit ee55121ab437f45df345290787d6b7f8bb70b409
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Jan 29 13:17:20 2014 +0100

    Replace non-breaking spaces to make pdflatex happy

diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index fc612e7..e78ceda 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -21,8 +21,8 @@ that content is to be served to.
 
 Setting this header can be as simple as::
 
-   sub vcl_recv {?
-       if (req.http.User-Agent ~ "(?i)iphone"?{
+   sub vcl_recv {
+       if (req.http.User-Agent ~ "(?i)iphone" {
            set req.http.X-UA-Device = "mobile-iphone";
        }
    }



From lkarsten at varnish-software.com  Thu Mar 13 09:24:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 32b6b10 Rewrite odd UTF8 that makes pdflatex sad.
Message-ID: <E1WO1sE-00071z-Ao@project.varnish-software.com>


commit 32b6b10f6dfe54795826e7c5f3521121550965c7
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Jan 29 14:06:05 2014 +0100

    Rewrite odd UTF8 that makes pdflatex sad.
    
    Mostly non-breaking space and angle brackets. Stick with what has worked
    for 51 years now, proper 7bit US-ASCII! ;-)

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index d820d69..33e9a67 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -127,7 +127,7 @@ help [command]
 
 param.set param value
       Set the parameter specified by param to the specified value.
-      See Run-Time Parameters for a list of parame? ters.
+      See Run-Time Parameters for a list of parameters.
 
 param.show [-l] [param]
       Display a list if run-time parameters and their values.
@@ -179,7 +179,7 @@ vcl.show configname
 
 vcl.use configname
       Start using the configuration specified by configname for all
-      new requests.  Existing requests will con? tinue using whichever
+      new requests.  Existing requests will continue using whichever
       configuration was in use when they arrived.
 
 
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index a430fc2..48fd44c 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -19,11 +19,11 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config] 
+varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config]
 	 [-g group] [-h type[,options]] [-i identity]
-	 [-l shmlogsize] [-n name] [-P file] [-p param=value] 
+	 [-l shmlogsize] [-n name] [-P file] [-p param=value]
 	 [-s type[,options]] [-T address[:port]] [-t ttl]
-	 [-r param[,param...]] [-u user] [-V] 
+	 [-r param[,param...]] [-u user] [-V]
 
 DESCRIPTION
 ===========
@@ -41,14 +41,14 @@ OPTIONS
             name (?localhost?), an IPv4 dotted-quad (?127.0.0.1?), or an IPv6 address enclosed in
             square brackets (?[::1]?).  If address is not specified, varnishd will listen on all
             available IPv4 and IPv6 interfaces.  If port is not specified, the default HTTP port as
-            listed in /etc/services is used.  Multiple listening addresses and ports can be speci?
-            fied as a whitespace- or comma-separated list.
+            listed in /etc/services is used.  Multiple listening addresses and ports can be
+            specified as a whitespace or comma -separated list.
 
 -b host[:port]
-            Use the specified host as backend server.  If port is not specified, 
+            Use the specified host as backend server.  If port is not specified,
 	    the default is 8080.
 
--C	    Print VCL code compiled to C language and exit. Specify the VCL file 
+-C	    Print VCL code compiled to C language and exit. Specify the VCL file
 	    to compile with the -f option.
 
 -d          Enables debugging mode: The parent process runs in the foreground with a CLI connection
@@ -85,8 +85,8 @@ OPTIONS
 -P file     Write the process's PID to the specified file.
 
 -p param=value
-            Set the parameter specified by param to the specified value.  See Run-Time 
-	    Parameters for a list of parameters. This option can be used multiple 
+            Set the parameter specified by param to the specified value.  See Run-Time
+	    Parameters for a list of parameters. This option can be used multiple
 	    times to specify multiple parameters.
 
 -S file     Path to a file containing a secret used for authorizing access to the management port.
@@ -108,11 +108,11 @@ OPTIONS
 
 -M address:port
             Connect to this port and offer the command line
-            interface. Think of it as a reverse shell. When running with 
+            interface. Think of it as a reverse shell. When running with
 	    -M and there is no backend defined the child process (the cache)
             will not start initially.
 
--t ttl      
+-t ttl
 	    Specifies a hard minimum time to live for cached
             documents.  This is a shortcut for specifying the
             default_ttl run-time parameter.
@@ -129,7 +129,7 @@ OPTIONS
             process should switch before it starts accepting
             connections.  This is a shortcut for specifying the user
             run- time parameter.
-	    
+
             If specifying both a user and a group, the user should be
             specified first.
 
@@ -250,7 +250,7 @@ The varnishd daemon was developed by Poul-Henning Kamp in cooperation
 with Verdens Gang AS, Varnish Software AS and Varnish Software.
 
 This manual page was written by Dag-Erling Sm?rgrav with updates by
-Stig Sandbeck Mathisen ?ssm at debian.org?
+Stig Sandbeck Mathisen <ssm at debian.org>
 
 
 COPYRIGHT
@@ -259,4 +259,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2011 Varnish Software AS
+* Copyright (c) 2007-2014 Varnish Software AS
diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst
index 017bb55..6446fb1 100644
--- a/doc/sphinx/reference/varnishhist.rst
+++ b/doc/sphinx/reference/varnishhist.rst
@@ -15,54 +15,54 @@ Varnish request histogram
 SYNOPSIS
 ========
 
-varnishhist [-b] [-C] [-c] [-d] [-I regex] [-i tag] [-m tag:regex ...] 
+varnishhist [-b] [-C] [-c] [-d] [-I regex] [-i tag] [-m tag:regex ...]
 [-n varnish_name] [-r file] [-V] [-w delay] [-X regex] [-x tag]
 
 DESCRIPTION
 ===========
 
 The varnishhist utility reads varnishd(1) shared memory logs and
-presents a continuously updated histogram show? ing the distribution
+presents a continuously updated histogram showing the distribution
 of the last N requests by their processing.  The value of N and the
-vertical scale are dis? played in the top left corner.  The horizontal
+vertical scale are displayed in the top left corner.  The horizontal
 scale is logarithmic.  Hits are marked with a pipe character ("|"),
 and misses are marked with a hash character ("#").
 
 The following options are available:
 
--b          Include log entries which result from communication with 
+-b          Include log entries which result from communication with
 	    a backend server.  If neither -b nor -c is
 	    specified, varnishhist acts as if they both were.
 
 -C          Ignore case when matching regular expressions.
 
--c          Include log entries which result from communication with 
-	    a client.  If neither -b nor -c is specified, 
+-c          Include log entries which result from communication with
+	    a client.  If neither -b nor -c is specified,
 	    varnishhist acts as if they both were.
 
--d          Process old log entries on startup.  Normally, varnishhist 
-	    will only process entries which are written to the 
+-d          Process old log entries on startup.  Normally, varnishhist
+	    will only process entries which are written to the
 	    log after it starts.
 
--I regex    Include log entries which match the specified 
-   	    regular expression.  If neither -I nor -i is specified, 
+-I regex    Include log entries which match the specified
+   	    regular expression.  If neither -I nor -i is specified,
 	    all log entries are included.
 
--i tag      Include log entries with the specified tag.  If neither 
+-i tag      Include log entries with the specified tag.  If neither
    	    -I nor -i is specified, all log entries are included.
 
 -m tag:regex only count transactions where tag matches regex. Multiple
             -m options are AND-ed together.
 
--n          Specifies the name of the varnishd instance to get logs 
+-n          Specifies the name of the varnishd instance to get logs
 	    from.  If -n is not specified, the host name is used.
 
 -r file     Read log entries from file instead of shared memory.
 
 -V          Display the version number and exit.
 
--w delay    Wait at least delay seconds between each update.  The 
-   	    default is 1.  file instead of displaying them.  The file 
+-w delay    Wait at least delay seconds between each update.  The
+   	    default is 1.  file instead of displaying them.  The file
 	    will be overwritten unless the -a option was specified.
 
 -X regex    Exclude log entries which match the specified regular expression.
@@ -75,7 +75,7 @@ SEE ALSO
 * varnishd(1)
 * varnishlog(1)
 * varnishncsa(1)
-* varnishstat(1) 
+* varnishstat(1)
 * varnishtop(1)
 
 HISTORY
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index fc186cc..9d93fe1 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -68,9 +68,9 @@ HISTORY
 =======
 
 The varnishlog utility was developed by Poul-Henning Kamp
-?phk at phk.freebsd.dk? in cooperation with Verdens Gang AS, Varnish
-Software AS and Varnish Software.  This manual page was initially
-written by Dag-Erling Sm?rgrav.
+<phk at phk.freebsd.dk> in cooperation with Verdens Gang AS and
+Varnish Software AS. This manual page was initially written by Dag-Erling
+Sm?rgrav.
 
 
 COPYRIGHT
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index f3b1f50..50dcfe6 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -115,7 +115,7 @@ Supported formatters are:
   Varnish:handling
     How the request was handled, whether it was a cache hit, miss,
     pass, pipe or error.
-	
+
   VCL_Log:key
     Output value set by std.log("key:value") in VCL.
 
@@ -140,7 +140,7 @@ HISTORY
 
 The varnishncsa utility was developed by Poul-Henning Kamp in
 cooperation with Verdens Gang AS and Varnish Software AS.  This manual page was
-written by Dag-Erling Sm?rgrav ?des at des.no?.
+written by Dag-Erling Sm?rgrav <des at des.no>.
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index 40951e1..67829c0 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -70,26 +70,26 @@ When using the -1 option, the columns in the output are, from left to right:
 
 When using the -x option, the output is::
 
-  <stat> 
-    <name>FIELD NAME</name> 
-    <value>FIELD VALUE</value> 
-    <description>FIELD DESCRIPTION</description> 
-  </stat> 
+  <stat>
+    <name>FIELD NAME</name>
+    <value>FIELD VALUE</value>
+    <description>FIELD DESCRIPTION</description>
+  </stat>
 
 With -j the output format is::
 
   {
     "timestamp": "YYYY-MM-DDTHH:mm:SS",
     "client_conn": {
-	"value": 0, "flag": "a", 
+	"value": 0, "flag": "a",
 	"description": "Client connections accepted"
     },
     "client_drop": {
-	"value": 0, "flag": "a", 
+	"value": 0, "flag": "a",
 	"description": "Connection dropped, no sess/wrk"
     },
     "LCK.backend.creat": {
-	"type": "LCK", "ident": "backend", "value": 1, 
+	"type": "LCK", "ident": "backend", "value": 1,
 	"flag": "a", "description": "Created locks"
     },
     [..]
@@ -114,9 +114,8 @@ HISTORY
 =======
 
 The varnishstat utility was originally developed by Poul-Henning Kamp
-?phk at phk.freebsd.dk? in cooperation with Verdens Gang AS, Varnish Software AS
-and Varnish Software. Manual page written by Dag-Erling Sm?rgrav,
-and Per Buer. 
+<phk at phk.freebsd.dk> in cooperation with Verdens Gang AS and Varnish
+Software AS. Manual page written by Dag-Erling Sm?rgrav and Per Buer.
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index 8a3124c..52c642b 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -128,10 +128,10 @@ HISTORY
 =======
 
 The varnishtest program was developed by Poul-Henning Kamp
-?phk at phk.freebsd.dk? in cooperation with Varnish Software AS.
+<phk at phk.freebsd.dk> in cooperation with Varnish Software AS.
 This manual page was originally written by Stig Sandbeck Mathisen
-?ssm at linpro.no? and updated by Kristian Lyngst?l
-(kristian at varnish-cache.org).
+<ssm at linpro.no> and updated by Kristian Lyngst?l
+<kristian at varnish-cache.org>.
 
 COPYRIGHT
 =========
@@ -139,4 +139,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2011 Varnish Software AS
+* Copyright (c) 2007-2014 Varnish Software AS



From lkarsten at varnish-software.com  Thu Mar 13 09:24:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 084f926 Use A4 by default, minor style cleanup
Message-ID: <E1WO1sE-000729-Bz@project.varnish-software.com>


commit 084f9267556da6c5509c8b98b64a9e5533f854d6
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Jan 29 14:18:27 2014 +0100

    Use A4 by default, minor style cleanup

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 217d49f..54c01f6 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -4,7 +4,7 @@
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
-PAPER         =
+PAPER         = a4
 BUILDDIR      = =build
 
 # Internal variables.
diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index 5cef666..d5415da 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -38,7 +38,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Varnish'
-copyright = u'2010, Varnish Project'
+copyright = u'2010-2014 (c) Varnish Software AS'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -96,7 +96,7 @@ html_theme = 'default'
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-# 
+#
 # topp background: #437EB2
 # left column: #EEEEEE;
 # h3: #222222;
@@ -106,7 +106,7 @@ html_theme = 'default'
 
 html_theme_options = {
     "bgcolor" : "white",
-    
+
     "relbarbgcolor" : "#437EB2",
     "relbartextcolor" : "white",
 
@@ -116,7 +116,7 @@ html_theme_options = {
 
     "textcolor" : "#222222",
     "linkcolor" : "#336590",
-    
+
 
     # "codebgcolor" : "#EEEEEE",
     "codetextcolor" : "#222222",
@@ -212,7 +212,7 @@ latex_documents = [
 #latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
+# latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
 #latex_appendices = []



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] ac647ac Port varnishtop to the new logging API
Message-ID: <E1WO1sE-00072I-D3@project.varnish-software.com>


commit ac647acd1a44ff770f7a76efe9e2beafa8286b52
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Mon Dec 2 15:13:30 2013 +0100

    Port varnishtop to the new logging API

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 124cb98..48a9a51 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -8,4 +8,5 @@ SUBDIRS = varnishadm varnishd varnishlog varnishtest varnishncsa
 if VARNISH_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop
 SUBDIRS += varnishstat
+SUBDIRS += varnishtop
 endif
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index 54e672f..3a78c4e 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -7,8 +7,14 @@ bin_PROGRAMS = varnishtop
 dist_man_MANS = varnishtop.1
 
 varnishtop_SOURCES = varnishtop.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c
+	$(top_srcdir)/lib/libvarnishtools/vut.c \
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/flopen.c \
+	$(top_srcdir)/lib/libvarnish/version.c \
+	$(top_srcdir)/lib/libvarnish/vpf.c \
+	$(top_srcdir)/lib/libvarnish/vtim.c \
+	$(top_srcdir)/lib/libvarnish/vsb.c
+
 
 varnishtop_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index e217f28..f2f9555 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -50,6 +50,7 @@
 #include "vcs.h"
 #include "vtree.h"
 #include "vsb.h"
+#include "vut.h"
 
 #if 0
 #define AC(x) assert((x) != ERR)
@@ -57,6 +58,9 @@
 #define AC(x) x
 #endif
 
+static const char progname[] = "varnishtop2";
+static float period = 60; /* seconds */
+
 struct top {
 	uint8_t			tag;
 	char			*rec_data;
@@ -105,61 +109,63 @@ top_cmp(const struct top *tp, const struct top *tp2)
 }
 
 
-static int
-accumulate(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
-    unsigned spec, const char *ptr, uint64_t bm)
+static int __match_proto__(VSLQ_dispatch_f)
+accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
+	void *priv)
 {
 	struct top *tp, t;
-	const char *q;
 	char *rd;
 	unsigned int u;
-	int i;
+	unsigned tag;
+	const char *b, *e, *p;
+	unsigned len;
+
+	struct VSL_transaction *tr;
+	for (tr = pt[0]; tr != NULL; tr = *++pt) {
+		while ((1 == VSL_Next(tr->c))) {
+			tag = VSL_TAG(tr->c->rec.ptr);
+			b = VSL_CDATA(tr->c->rec.ptr);
+			len = VSL_LEN(tr->c->rec.ptr);
+			assert(len > 0);
+			e = b + len;
+			u = 0;
+			for (p = b; p <= e; p++) {
+				u += *p;
+			}
+
+			t.hash = u;
+			t.tag = tag;
+			t.clen = len;
+			t.count = 0;
+			rd = calloc(len+1, 1);
+			AN(rd);
+			memcpy(rd, VSL_CDATA(tr->c->rec.ptr), len);
+            rd[len] = '\0';
+			t.rec_data = rd;
+
+			AZ(pthread_mutex_lock(&mtx));
+			tp = VRB_FIND(top_tree, &top_tree_head, &t);
+			if (tp) {
+				VRB_REMOVE(top_tree, &top_tree_head, tp);
+				tp->count += 1.0;
+				/* Reinsert to rebalance */
+				VRB_INSERT(top_tree, &top_tree_head, tp);
+                free(rd);
+			} else {
+				ntop++;
+				tp = calloc(sizeof *tp, 1);
+				assert(tp != NULL);
+				tp->hash = u;
+				tp->count = 1.0;
+				tp->clen = len;
+				tp->tag = tag;
+                tp->rec_data = rd;
+				VRB_INSERT(top_tree, &top_tree_head, tp);
+			}
+			AZ(pthread_mutex_unlock(&mtx));
 
-	(void)priv;
-	(void)fd;
-	(void)spec;
-	(void)bm;
-	// fprintf(stderr, "%p %08x %08x\n", p, p[0], p[1]);
-
-	u = 0;
-	q = ptr;
-	for (i = 0; i < len; i++, q++) {
-		if (f_flag && (*q == ':' || isspace(*q))) {
-			len = q - ptr;
-			break;
 		}
-		u += *q;
 	}
-	t.hash = u;
-	t.tag = tag;
-	t.clen = len;
-	rd = malloc(len);
-	AN(rd);
-	memcpy(rd, ptr, len);
-	t.rec_data = rd;
-
-	AZ(pthread_mutex_lock(&mtx));
-	tp = VRB_FIND(top_tree, &top_tree_head, &t);
-	if (tp) {
-		VRB_REMOVE(top_tree, &top_tree_head, tp);
-		tp->count += 1.0;
-		/* Reinsert to rebalance */
-		VRB_INSERT(top_tree, &top_tree_head, tp);
-	} else {
-		ntop++;
-		tp = calloc(sizeof *tp, 1);
-		assert(tp != NULL);
-		tp->rec_data = calloc(len + 1, 1);
-		assert(tp->rec_data != NULL);
-		tp->hash = u;
-		tp->count = 1.0;
-		tp->clen = len;
-		tp->tag = tag;
-		memcpy(tp->rec_data, ptr, len);
-		tp->rec_data[len] = '\0';
-		VRB_INSERT(top_tree, &top_tree_head, tp);
-	}
-	AZ(pthread_mutex_unlock(&mtx));
 
 	return (0);
 }
@@ -187,16 +193,18 @@ update(const struct VSM_data *vd, int period)
 	AC(mvprintw(0, 0, "list length %u", ntop));
 	for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) {
 		tp2 = VRB_NEXT(top_tree, &top_tree_head, tp);
+
 		if (++l < LINES) {
 			len = tp->clen;
 			if (len > COLS - 20)
 				len = COLS - 20;
 			AC(mvprintw(l, 0, "%9.2f %-*.*s %*.*s\n",
-			    tp->count, maxfieldlen, maxfieldlen,
-			    VSL_tags[tp->tag],
-			    len, len, tp->rec_data));
+				tp->count, maxfieldlen, maxfieldlen,
+				VSL_tags[tp->tag],
+				len, len, tp->rec_data));
 			t = tp->count;
 		}
+		(void)t;
 		tp->count += (1.0/3.0 - tp->count) / (double)n;
 		if (tp->count * 10 < t || l > LINES * 10) {
 			VRB_REMOVE(top_tree, &top_tree_head, tp);
@@ -209,27 +217,10 @@ update(const struct VSM_data *vd, int period)
 }
 
 static void *
-accumulate_thread(void *arg)
-{
-	struct VSM_data *vd = arg;
-	int i;
-
-	for (;;) {
-
-		i = VSL_Dispatch(vd, accumulate, NULL);
-		if (i < 0)
-			break;
-		if (i == 0)
-			usleep(50000);
-	}
-	return (arg);
-}
-
-static void
-do_curses(struct VSM_data *vd, int period)
+do_curses(void *arg)
 {
-	pthread_t thr;
 	int i;
+	struct VSM_data *vd = (struct VSM_data *)arg;
 
 	for (i = 0; i < 256; i++) {
 		if (VSL_tags[i] == NULL)
@@ -238,11 +229,6 @@ do_curses(struct VSM_data *vd, int period)
 			maxfieldlen = strlen(VSL_tags[i]);
 	}
 
-	if (pthread_create(&thr, NULL, accumulate_thread, vd) != 0) {
-		fprintf(stderr, "pthread_create(): %s\n", strerror(errno));
-		exit(1);
-	}
-
 	(void)initscr();
 	AC(raw());
 	AC(noecho());
@@ -269,54 +255,46 @@ do_curses(struct VSM_data *vd, int period)
 			AC(redrawwin(stdscr));
 			AC(refresh());
 			break;
-		case '\003': /* Ctrl-C */
-			AZ(raise(SIGINT));
-			break;
 		case '\032': /* Ctrl-Z */
 			AC(endwin());
 			AZ(raise(SIGTSTP));
 			break;
+		case '\003': /* Ctrl-C */
+			printf("got ctrl-C\r\n");
 		case '\021': /* Ctrl-Q */
 		case 'Q':
 		case 'q':
+			AZ(raise(SIGINT));
 			AC(endwin());
-			return;
+			return NULL;
 		default:
 			AC(beep());
 			break;
 		}
 	}
+	return NULL;
+
 }
 
 static void
 dump(void)
 {
 	struct top *tp, *tp2;
-	printf("%d\n", ntop);
-	printf("%p\n", VRB_MIN(top_tree, &top_tree_head));
 	for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) {
 		tp2 = VRB_NEXT(top_tree, &top_tree_head, tp);
 		if (tp->count <= 1.0)
-		break;
+			break;
 		printf("%9.2f %s %*.*s\n",
-		    tp->count, VSL_tags[tp->tag],
-		    tp->clen, tp->clen, tp->rec_data);
+			tp->count, VSL_tags[tp->tag],
+			tp->clen, tp->clen, tp->rec_data);
 	}
 }
 
 static void
-do_once(struct VSM_data *vd)
-{
-	while (VSL_Dispatch(vd, accumulate, NULL) > 0)
-		;
-	dump();
-}
-
-static void
 usage(void)
 {
 	fprintf(stderr,
-	    "usage: varnishtop %s [-1fV] [-n varnish_name]\n", VSL_USAGE);
+		"usage: varnishtop [-1fV] [-n varnish_name]\n");
 	exit(1);
 }
 
@@ -325,14 +303,15 @@ main(int argc, char **argv)
 {
 	struct VSM_data *vd;
 	int o, once = 0;
-	float period = 60; /* seconds */
+	pthread_t thr;
 
 	vd = VSM_New();
+	VUT_Init(progname);
 
-	while ((o = getopt(argc, argv, VSL_ARGS "1fVp:")) != -1) {
+	while ((o = getopt(argc, argv, "1fVp:")) != -1) {
 		switch (o) {
 		case '1':
-			AN(VSL_Arg(vd, 'd', NULL));
+			VUT_Arg('d', NULL);
 			once = 1;
 			break;
 		case 'f':
@@ -343,7 +322,7 @@ main(int argc, char **argv)
 			period = strtol(optarg, NULL, 0);
 			if (errno != 0)  {
 				fprintf(stderr,
-				    "Syntax error, %s is not a number", optarg);
+					"Syntax error, %s is not a number", optarg);
 				exit(1);
 			}
 			break;
@@ -354,7 +333,7 @@ main(int argc, char **argv)
 			fprintf(stderr, "-m is not supported\n");
 			exit(1);
 		default:
-			if (VSL_Arg(vd, o, optarg) > 0)
+			if (!VUT_Arg(o, optarg))
 				break;
 			usage();
 		}
@@ -365,10 +344,20 @@ main(int argc, char **argv)
 		exit (1);
 	}
 
-	if (once) {
-		do_once(vd);
-	} else {
-		do_curses(vd, period);
+	VUT.dispatch_f = &accumulate;
+	VUT.dispatch_priv = NULL;
+	if (!once){
+		if (pthread_create(&thr, NULL, do_curses, vd) != 0) {
+			fprintf(stderr, "pthread_create(): %s\n", strerror(errno));
+			exit(1);
+		}
 	}
+	VUT_Setup();
+	VUT_Main();
+	VUT_Fini();
+	if (once)
+		dump();
+	else
+		pthread_join(thr, NULL);
 	exit(0);
 }
diff --git a/configure.ac b/configure.ac
index e22fc4e..38634b9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -562,6 +562,7 @@ AC_CONFIG_FILES([
     bin/varnishd/Makefile
     bin/varnishlog/Makefile
     bin/varnishstat/Makefile
+    bin/varnishtop/Makefile
     bin/varnishtest/Makefile
     bin/varnishncsa/Makefile
     doc/Makefile



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] c105997 Add author
Message-ID: <E1WO1sE-00072S-EC@project.varnish-software.com>


commit c105997c8748940feefc9c47c49c8a0b3c830c52
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:30:55 2014 +0100

    Add author

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index f2f9555..05f9357 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -5,6 +5,7 @@
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
  * Author: Dag-Erling Sm?rgrav <des at des.no>
+ * Author: Guillaume Quintard <guillaume.quintard at gmail.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] ade4dff Fix whitespace
Message-ID: <E1WO1sE-00072d-FD@project.varnish-software.com>


commit ade4dff380ff5be428ea2914ab9065d62968e614
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:31:25 2014 +0100

    Fix whitespace

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 05f9357..8be6c36 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -141,7 +141,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			rd = calloc(len+1, 1);
 			AN(rd);
 			memcpy(rd, VSL_CDATA(tr->c->rec.ptr), len);
-            rd[len] = '\0';
+			rd[len] = '\0';
 			t.rec_data = rd;
 
 			AZ(pthread_mutex_lock(&mtx));
@@ -151,7 +151,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				tp->count += 1.0;
 				/* Reinsert to rebalance */
 				VRB_INSERT(top_tree, &top_tree_head, tp);
-                free(rd);
+				free(rd);
 			} else {
 				ntop++;
 				tp = calloc(sizeof *tp, 1);
@@ -160,7 +160,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				tp->count = 1.0;
 				tp->clen = len;
 				tp->tag = tag;
-                tp->rec_data = rd;
+				tp->rec_data = rd;
 				VRB_INSERT(top_tree, &top_tree_head, tp);
 			}
 			AZ(pthread_mutex_unlock(&mtx));



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] ac8c50e Don't trigger unused variables warnings
Message-ID: <E1WO1sE-00072m-G7@project.varnish-software.com>


commit ac8c50e163b192f2520800a217e2bac13021b9d7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:32:00 2014 +0100

    Don't trigger unused variables warnings

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 8be6c36..e79f682 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -121,6 +121,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	const char *b, *e, *p;
 	unsigned len;
 
+	(void)vsl;
+	(void)priv;
 	struct VSL_transaction *tr;
 	for (tr = pt[0]; tr != NULL; tr = *++pt) {
 		while ((1 == VSL_Next(tr->c))) {



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 367f8b5 Don't trigger name shadows global warning
Message-ID: <E1WO1sE-00072v-H2@project.varnish-software.com>


commit 367f8b5160812b39b4b13681e9f106a5f01dd688
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:32:30 2014 +0100

    Don't trigger name shadows global warning

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index e79f682..3392ce1 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -174,7 +174,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 }
 
 static void
-update(const struct VSM_data *vd, int period)
+update(const struct VSM_data *vd, int p)
 {
 	struct top *tp, *tp2;
 	int l, len;
@@ -189,7 +189,7 @@ update(const struct VSM_data *vd, int period)
 	last = now;
 
 	l = 1;
-	if (n < period)
+	if (n < p)
 		n++;
 	AC(erase());
 	AC(mvprintw(0, 0, "%*s", COLS - 1, VSM_Name(vd)));



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 07e159c Rename to varnishtop2 to varnishtop in usage output
Message-ID: <E1WO1sE-000734-ID@project.varnish-software.com>


commit 07e159ccfc6fbd8421ba89cb764940b950025bf5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:32:57 2014 +0100

    Rename to varnishtop2 to varnishtop in usage output

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 3392ce1..ed56833 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -59,7 +59,7 @@
 #define AC(x) x
 #endif
 
-static const char progname[] = "varnishtop2";
+static const char progname[] = "varnishtop";
 static float period = 60; /* seconds */
 
 struct top {



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 316d15b Rename varnishncsa2 to varnishncsa in usage output
Message-ID: <E1WO1sE-00073D-JL@project.varnish-software.com>


commit 316d15b37a6a07e1479cd2192f6ee262cd906500
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:37:58 2014 +0100

    Rename varnishncsa2 to varnishncsa in usage output

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index f6a5c87..c7157f4 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -82,7 +82,7 @@
 
 #define TIME_FMT "[%d/%b/%Y:%T %z]"
 #define FORMAT "%h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\""
-static const char progname[] = "varnishncsa2";
+static const char progname[] = "varnishncsa";
 
 struct format;
 struct fragment;



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 4492226 Apply VSL_Match on records.
Message-ID: <E1WO1sE-00073M-Kr@project.varnish-software.com>


commit 44922267c8fbea587e3143c13a635c71d25bb338
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 11:45:43 2014 +0100

    Apply VSL_Match on records.
    
    This allows using the -ixIX options to filter which records are counted.

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index ed56833..65aa6a9 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -120,12 +120,14 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	unsigned tag;
 	const char *b, *e, *p;
 	unsigned len;
+	struct VSL_transaction *tr;
 
-	(void)vsl;
 	(void)priv;
-	struct VSL_transaction *tr;
+
 	for (tr = pt[0]; tr != NULL; tr = *++pt) {
 		while ((1 == VSL_Next(tr->c))) {
+			if (!VSL_Match(vsl, tr->c))
+				continue;
 			tag = VSL_TAG(tr->c->rec.ptr);
 			b = VSL_CDATA(tr->c->rec.ptr);
 			len = VSL_LEN(tr->c->rec.ptr);



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 47d2153 Sort out options parsing for varnishtop
Message-ID: <E1WO1sE-00073V-M3@project.varnish-software.com>


commit 47d2153fb0040be763091932fcc051add175b4b9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 13:50:59 2014 +0100

    Sort out options parsing for varnishtop

diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index 3a78c4e..a38e293 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -7,6 +7,8 @@ bin_PROGRAMS = varnishtop
 dist_man_MANS = varnishtop.1
 
 varnishtop_SOURCES = varnishtop.c \
+	varnishtop_options.h \
+	varnishtop_options.c \
 	$(top_srcdir)/lib/libvarnishtools/vut.c \
 	$(top_srcdir)/lib/libvarnish/vas.c \
 	$(top_srcdir)/lib/libvarnish/flopen.c \
@@ -21,9 +23,31 @@ varnishtop_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	@CURSES_LIB@ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
-varnishtop.1: $(top_srcdir)/doc/sphinx/reference/varnishtop.rst
+noinst_PROGRAMS = varnishtop_opt2rst
+varnishtop_opt2rst_SOURCES = \
+	varnishtop_options.h \
+	varnishtop_options.c \
+	$(top_srcdir)/lib/libvarnishtools/opt2rst.c
+
+BUILT_SOURCES = varnishtop_options.rst varnishtop_synopsis.rst
+EXTRA_DIST = $(BUILT_SOURCES)
+MAINTAINERCLEANFILES = $(EXTRA_DIST)
+
+varnishtop_options.rst:
+	./varnishtop_opt2rst options > $@
+varnishtop_synopsis.rst:
+	./varnishtop_opt2rst synopsis > $@
+
+if HAVE_RST2MAN
+varnishtop_options.rst varnishtop_synopsis.rst: varnishtop_opt2rst
+endif
+
+varnishtop.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
+	varnishtop_options.rst \
+	varnishtop_synopsis.rst
 if HAVE_RST2MAN
-	${RST2MAN} $? $@
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
 else
 	@echo "========================================"
 	@echo "You need rst2man installed to make dist"
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 65aa6a9..7190257 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -45,8 +45,9 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vapi/vsl.h"
 #include "vapi/vsm.h"
+#include "vapi/vsl.h"
+#include "vapi/voptget.h"
 #include "vas.h"
 #include "vcs.h"
 #include "vtree.h"
@@ -296,11 +297,15 @@ dump(void)
 }
 
 static void
-usage(void)
+usage(int status)
 {
-	fprintf(stderr,
-		"usage: varnishtop [-1fV] [-n varnish_name]\n");
-	exit(1);
+	const char **opt;
+
+	fprintf(stderr, "Usage: %s <options>\n\n", progname);
+	fprintf(stderr, "Options:\n");
+	for (opt = vopt_usage; *opt != NULL; opt +=2)
+		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
+	exit(status);
 }
 
 int
@@ -313,7 +318,7 @@ main(int argc, char **argv)
 	vd = VSM_New();
 	VUT_Init(progname);
 
-	while ((o = getopt(argc, argv, "1fVp:")) != -1) {
+	while ((o = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (o) {
 		case '1':
 			VUT_Arg('d', NULL);
@@ -322,6 +327,9 @@ main(int argc, char **argv)
 		case 'f':
 			f_flag = 1;
 			break;
+		case 'h':
+			/* Usage help */
+			usage(0);
 		case 'p':
 			errno = 0;
 			period = strtol(optarg, NULL, 0);
@@ -331,16 +339,9 @@ main(int argc, char **argv)
 				exit(1);
 			}
 			break;
-		case 'V':
-			VCS_Message("varnishtop");
-			exit(0);
-		case 'm':
-			fprintf(stderr, "-m is not supported\n");
-			exit(1);
 		default:
 			if (!VUT_Arg(o, optarg))
-				break;
-			usage();
+				usage(1);
 		}
 	}
 
diff --git a/bin/varnishtop/varnishtop_options.c b/bin/varnishtop/varnishtop_options.c
new file mode 100644
index 0000000..58d132e
--- /dev/null
+++ b/bin/varnishtop/varnishtop_options.c
@@ -0,0 +1,34 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Option definitions for varnishtop
+ */
+
+#include <stdlib.h>
+#define VOPT_DEFINITION
+#define VOPT_INC "varnishtop_options.h"
+#include "vapi/voptget.h"
diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h
new file mode 100644
index 0000000..33135a8
--- /dev/null
+++ b/bin/varnishtop/varnishtop_options.h
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Option definitions for varnishtop
+ */
+
+#include "vapi/vapi_options.h"
+#include "vut_options.h"
+
+VSL_OPT_b
+VSL_OPT_c
+VSL_OPT_C
+VUT_OPT_d
+VUT_OPT_D
+VUT_OPT_h
+VSL_OPT_i
+VSL_OPT_I
+VSL_OPT_L
+VUT_OPT_n
+VUT_OPT_N
+VUT_OPT_q
+VUT_OPT_r
+VSL_OPT_T
+VSL_OPT_x
+VSL_OPT_X
+VUT_OPT_V



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 7630265 Reintroduce the -1, -f and -p options to varnishtop
Message-ID: <E1WO1sE-00073f-Mz@project.varnish-software.com>


commit 763026598b03334fa5d1f8d8ed15b4092d931d15
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 14:39:51 2014 +0100

    Reintroduce the -1, -f and -p options to varnishtop

diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h
index 33135a8..5256b1d 100644
--- a/bin/varnishtop/varnishtop_options.h
+++ b/bin/varnishtop/varnishtop_options.h
@@ -31,17 +31,41 @@
 #include "vapi/vapi_options.h"
 #include "vut_options.h"
 
+#define TOP_OPT_1							\
+	VOPT("1", "[-1]", "Run once",					\
+	    "Instead of a continously updated display, print the"	\
+	    " statistics once and exit. Implies ``-d``."		\
+	)
+
+#define TOP_OPT_f							\
+	VOPT("f", "[-f]", "First field only",				\
+	    "Sort and group only on the first field of each log entry."	\
+	    " This is useful when displaying e.g. stataddr entries,"	\
+	    " where the first field is the client IP address."		\
+	)
+
+#define TOP_OPT_p							\
+	VOPT("p:", "-p period", "Sampling period",			\
+	    "Specified the number of seconds to measure over, the"	\
+	    " default is 60 seconds. The first number in the list is"	\
+	    " the average number of requests seen over this time"	\
+	    " period."							\
+	)
+
+TOP_OPT_1
 VSL_OPT_b
 VSL_OPT_c
 VSL_OPT_C
 VUT_OPT_d
 VUT_OPT_D
+TOP_OPT_f
 VUT_OPT_h
 VSL_OPT_i
 VSL_OPT_I
 VSL_OPT_L
 VUT_OPT_n
 VUT_OPT_N
+TOP_OPT_p
 VUT_OPT_q
 VUT_OPT_r
 VSL_OPT_T



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 41b9a00 Reintroduce code to handle -f option
Message-ID: <E1WO1sE-00073o-O1@project.varnish-software.com>


commit 41b9a000e2999c7d078bd266b1fa0a58df65f78f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 15:09:00 2014 +0100

    Reintroduce code to handle -f option

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 7190257..3863119 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -131,13 +131,18 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				continue;
 			tag = VSL_TAG(tr->c->rec.ptr);
 			b = VSL_CDATA(tr->c->rec.ptr);
-			len = VSL_LEN(tr->c->rec.ptr);
-			assert(len > 0);
-			e = b + len;
+			e = b + VSL_LEN(tr->c->rec.ptr);
 			u = 0;
 			for (p = b; p <= e; p++) {
+				if (*p == '\0')
+					break;
+				if (f_flag && (*p == ':' || isspace(*p)))
+					break;
 				u += *p;
 			}
+			len = p - b;
+			if (len == 0)
+				continue;
 
 			t.hash = u;
 			t.tag = tag;



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 03b751b Update copyright
Message-ID: <E1WO1sE-00073x-Ox@project.varnish-software.com>


commit 03b751b16616895f3517bfdc79a678727d520062
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 15:09:21 2014 +0100

    Update copyright

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 3863119..23fcdec 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2014 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 5acc552 Update varnishtop documentation
Message-ID: <E1WO1sE-000746-Pu@project.varnish-software.com>


commit 5acc552c0f74f9874a99bdbe8c57d32aa9757de5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 15:09:56 2014 +0100

    Update varnishtop documentation

diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 86e6cf6..23298d5 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -1,12 +1,13 @@
-============
+==========
 varnishtop
-============
+==========
 
 -------------------------
 Varnish log entry ranking
 -------------------------
 
 :Author: Dag-Erling Sm?rgrav
+:Author: Martin Blix Grydeland
 :Date:   2010-05-31
 :Version: 1.0
 :Manual section: 1
@@ -16,9 +17,8 @@ Varnish log entry ranking
 SYNOPSIS
 ========
 
-varnishtop [-1] [-b] [-C] [-c] [-d] [-f] [-I regex] 
-[-i tag] [-n varnish_name] [-r file] [-V] [-X regex]
-[-x tag]
+.. include:: ../../../bin/varnishtop/varnishtop_synopsis.rst
+varnishtop |synopsis|
 
 DESCRIPTION
 ===========
@@ -32,48 +32,7 @@ recorded in the log.
 
 The following options are available:
 
--1          Instead of presenting of a continuously updated display, 
-	    print the statistics once and exit. Implies ``-d``.
-
--b          Include log entries which result from communication 
-	    with a backend server.  If neither ``-b`` nor ``-c`` is
-	    specified, varnishtop acts as if they both were.
-
--C          Ignore case when matching regular expressions.
-
--c          Include log entries which result from communication 
-	    with a client.  If neither ``-b`` nor ``-c`` is specified, 
-	    varnishtop acts as if they both were.
-
--d          Process old log entries on startup.  Normally, varnishtop 
-	    will only process entries which are written to the log 
-	    after it starts.
-
--f          Sort and group only on the first field of each log entry.  
-	    This is useful when displaying e.g. stataddr entries, 
-	    where the first field is the client IP address.
-
--I regex    Include log entries which match the specified regular 
-   	    expression.  If neither ``-I`` nor ``-i`` is specified, all log 
-	    entries are included.
-
--i tag      Include log entries with the specified tag.  If neither ``-I``
-   	    nor ``-i`` is specified, all log entries are included.
-
--p period   Specifies the number of seconds to measure over, the default
-            is 60 seconds. The first number in the list is the average
-            number of requests seen over this time period.
-
--n          Specifies the name of the varnishd instance to get logs from.  
-	    If ``-n`` is not specified, the host name is used.
-
--r file     Read log entries from file instead of shared memory.
-
--V          Display the version number and exit.
-
--X regex    Exclude log entries which match the specified regular expression.
-
--x tag      Exclude log entries with the specified tag.
+.. include:: ../../../bin/varnishtop/varnishtop_options.rst
 
 EXAMPLES
 ========
@@ -81,12 +40,12 @@ EXAMPLES
 The following example displays a continuously updated list of the most
 frequently requested URLs::
 
-  varnishtop -i RxURL
+  varnishtop -i ReqURL
 
 The following example displays a continuously updated list of the most
 commonly used user agents::
 
-  varnishtop -i RxHeader -C -I ^User-Agent
+  varnishtop -C -I ReqHeader:User-Agent
 
 SEE ALSO
 ========
@@ -112,4 +71,4 @@ This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2011 Varnish Software AS
+* Copyright (c) 2006-2014 Varnish Software AS



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 22dddf3 Update Makefile.phk to handle varnishtop
Message-ID: <E1WO1sE-00074F-R1@project.varnish-software.com>


commit 22dddf3c55ada965940aa339d7467a84037b1483
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 15:12:39 2014 +0100

    Update Makefile.phk to handle varnishtop

diff --git a/bin/Makefile.phk b/bin/Makefile.phk
index 933c9d8..5380231 100644
--- a/bin/Makefile.phk
+++ b/bin/Makefile.phk
@@ -6,7 +6,7 @@ SUBDIRS += varnishncsa
 SUBDIRS += varnishtest 
 SUBDIRS += varnishstat
 # SUBDIRS += varnishhist 
-# SUBDIRS += varnishtop
+SUBDIRS += varnishtop
 
 TOPDIR = $(CURDIR)/..
 include $(TOPDIR)/Makefile.inc.phk
diff --git a/bin/varnishtop/Makefile.phk b/bin/varnishtop/Makefile.phk
index 1b922f3..3c5684c 100644
--- a/bin/varnishtop/Makefile.phk
+++ b/bin/varnishtop/Makefile.phk
@@ -1,9 +1,15 @@
-PROG_SRC = varnishtop.c
+PROG_SRC += varnishtop.c
+PROG_SRC += varnishtop_options.c
 
-LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_VARNISHTOOLS}
 LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_VARNISH}
 LD_ADD	+= ${LIB_PCRE}
-LD_ADD	+= ${LIB_CURSES}
+LD_ADD	+= -lm
+
+TODO_ALL += build_man
 
 TOPDIR  = $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk
+
+.PHONY: build_man



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 2dc3e7e Fix formatting of -p option
Message-ID: <E1WO1sE-00074R-SA@project.varnish-software.com>


commit 2dc3e7ed476f4379b8be9c4662fd442c4845d399
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 17:28:32 2014 +0100

    Fix formatting of -p option

diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h
index 5256b1d..51c877d 100644
--- a/bin/varnishtop/varnishtop_options.h
+++ b/bin/varnishtop/varnishtop_options.h
@@ -45,7 +45,7 @@
 	)
 
 #define TOP_OPT_p							\
-	VOPT("p:", "-p period", "Sampling period",			\
+	VOPT("p:", "[-p period]", "Sampling period",			\
 	    "Specified the number of seconds to measure over, the"	\
 	    " default is 60 seconds. The first number in the list is"	\
 	    " the average number of requests seen over this time"	\



From martin at varnish-software.com  Thu Mar 13 09:24:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 22ac7f8 Enable grouping mode option in varnishtop
Message-ID: <E1WO1sE-00074X-T6@project.varnish-software.com>


commit 22ac7f849c019ccb42e970fe1bf4ccae56dd8f29
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 29 17:28:48 2014 +0100

    Enable grouping mode option in varnishtop

diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h
index 51c877d..592694a 100644
--- a/bin/varnishtop/varnishtop_options.h
+++ b/bin/varnishtop/varnishtop_options.h
@@ -59,6 +59,7 @@ VSL_OPT_C
 VUT_OPT_d
 VUT_OPT_D
 TOP_OPT_f
+VUT_OPT_g
 VUT_OPT_h
 VSL_OPT_i
 VSL_OPT_I



From tfheen at varnish-software.com  Thu Mar 13 09:24:22 2014
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] f1851f9 Drop obsolete VSM_Open
Message-ID: <E1WO1sE-00074g-U9@project.varnish-software.com>


commit f1851f9e19fea8677ec961ec3f2a774588cfb35a
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 30 08:09:16 2014 +0100

    Drop obsolete VSM_Open
    
    This is no longer needed and breaks -n

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 23fcdec..fbc8dc8 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -350,11 +350,6 @@ main(int argc, char **argv)
 		}
 	}
 
-	if (VSM_Open(vd)) {
-		fprintf(stderr, "%s\n", VSM_Error(vd));
-		exit (1);
-	}
-
 	VUT.dispatch_f = &accumulate;
 	VUT.dispatch_priv = NULL;
 	if (!once){



From tfheen at varnish-software.com  Thu Mar 13 09:24:22 2014
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] 34afd2c Remove debugging output
Message-ID: <E1WO1sE-00074p-V5@project.varnish-software.com>


commit 34afd2cd38970f210f8b0d606b4ac0fba4a5c24b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 30 08:09:33 2014 +0100

    Remove debugging output
    
    This upsets coverity and doesn't add value, so remove the printf
    line.

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index fbc8dc8..b5d5c7c 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -271,7 +271,6 @@ do_curses(void *arg)
 			AZ(raise(SIGTSTP));
 			break;
 		case '\003': /* Ctrl-C */
-			printf("got ctrl-C\r\n");
 		case '\021': /* Ctrl-Q */
 		case 'Q':
 		case 'q':



From daghf at varnish-software.com  Thu Mar 13 09:24:22 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 13 Mar 2014 10:24:22 +0100
Subject: [4.0] d43af81 Fix missing dependency in libvmod_debug makefile.
Message-ID: <E1WO1sE-00074y-W8@project.varnish-software.com>


commit d43af818662a8c5cf3330c850f42c79b62b07baf
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Jan 30 11:43:08 2014 +0100

    Fix missing dependency in libvmod_debug makefile.
    
    Fixes: #1418

diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index 18d5ecf..89df1b2 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -22,7 +22,7 @@ nodist_libvmod_debug_la_SOURCES = \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vmod_debug.lo: vcc_if.h
+vmod_debug.lo vmod_debug_obj.lo: vcc_if.h
 
 vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc



From daghf at varnish-software.com  Thu Mar 13 09:24:23 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] fafe433 Add missing vcc_if.h deps in libvmod_directors.
Message-ID: <E1WO1sF-000757-0q@project.varnish-software.com>


commit fafe4334b1ca7be8f22fb502f27fb98c06fec7ea
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Jan 30 11:55:38 2014 +0100

    Add missing vcc_if.h deps in libvmod_directors.

diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 5706ab8..d44839d 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -26,7 +26,7 @@ nodist_libvmod_directors_la_SOURCES = \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vdir.lo: vcc_if.h
+vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
 vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] ae00276 Improve -n and -N VSM handling.
Message-ID: <E1WO1sF-00075G-1x@project.varnish-software.com>


commit ae002768a6d38bd251b218e2c45d7dcf8b0efe63
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:32:49 2014 +0100

    Improve -n and -N VSM handling.
    
    This makes sure that VSM_Name always has something sensible to return.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index f036d99..f05f184 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -69,6 +69,7 @@ VSM_New(void)
 	if (vd == NULL)
 		return (vd);
 
+	REPLACE(vd->name, "");
 	vd->vsm_fd = -1;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -126,38 +127,45 @@ VSM_ResetError(struct VSM_data *vd)
 /*--------------------------------------------------------------------*/
 
 int
-VSM_n_Arg(struct VSM_data *vd, const char *opt)
+VSM_n_Arg(struct VSM_data *vd, const char *arg)
 {
+	char *name = NULL;
+	char *fname = NULL;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(opt);
 
-	if (vd->fname) {
+	if (vd->head)
+		return (vsm_diag(vd, "VSM_n_Arg: Already open\n"));
+	if (VIN_N_Arg(arg, &name, NULL, &fname))
+		return (vsm_diag(vd, "Invalid instance name: %s\n",
+			strerror(errno)));
+	AN(name);
+	AN(fname);
+
+	if (vd->name)
+		free(vd->name);
+	vd->name = name;
+	if (vd->fname)
 		free(vd->fname);
-		vd->fname = NULL;
-	}
+	vd->fname = fname;
 	vd->N_opt = 0;
-	REPLACE(vd->n_opt, opt);
-	if (VIN_N_Arg(vd->n_opt, NULL, NULL, &vd->fname))
-		return (vsm_diag(vd, "Invalid instance name: %s\n",
-		    strerror(errno)));
+
 	return (1);
 }
 
 /*--------------------------------------------------------------------*/
 
 int
-VSM_N_Arg(struct VSM_data *vd, const char *opt)
+VSM_N_Arg(struct VSM_data *vd, const char *arg)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(opt);
+	AN(arg);
 
-	if (vd->n_opt) {
-		free(vd->n_opt);
-		vd->n_opt = NULL;
-	}
-	REPLACE(vd->fname, opt);
+	if (vd->head)
+		return (vsm_diag(vd, "VSM_N_Arg: Already open\n"));
+	REPLACE(vd->name, arg);
+	REPLACE(vd->fname, arg);
 	vd->N_opt = 1;
 	return (1);
 }
@@ -170,7 +178,7 @@ VSM_Name(const struct VSM_data *vd)
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
-	return (vd->n_opt);
+	return (vd->name);
 }
 
 /*--------------------------------------------------------------------*/
@@ -182,11 +190,11 @@ VSM_Delete(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
 	VSM_Close(vd);
-	free(vd->n_opt);
-	free(vd->fname);
 	if (vd->vsc != NULL)
 		VSC_Delete(vd);
 	VSM_ResetError(vd);
+	free(vd->name);
+	free(vd->fname);
 	FREE_OBJ(vd);
 }
 
@@ -214,11 +222,13 @@ VSM_Open(struct VSM_data *vd)
 		/* Already open */
 		return (0);
 
-	if (!vd->n_opt && !vd->N_opt)
-		(void)VSM_n_Arg(vd, "");
-
-	AZ(vd->head);
-	AN(vd->fname);
+	if (vd->fname == NULL) {
+		/* Use default (hostname) */
+		i = VSM_n_Arg(vd, "");
+		if (i < 0)
+			return (i);
+		AN(vd->fname);
+	}
 
 	vd->vsm_fd = open(vd->fname, O_RDONLY);
 	if (vd->vsm_fd < 0)
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index dc1430b..1d73773 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -39,7 +39,7 @@ struct VSM_data {
 
 	struct vsb		*diag;
 
-	char			*n_opt;
+	char			*name;
 	char			*fname;
 	int			N_opt;
 



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] c385dbf API documentation fixes.
Message-ID: <E1WO1sF-00075R-39@project.varnish-software.com>


commit c385dbf63d57f23702a5fdaa2e5ebd3640ce58a7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:33:31 2014 +0100

    API documentation fixes.

diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index d92ab85..7db5199 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -93,7 +93,8 @@ void VSM_ResetError(struct VSM_data *vd);
 int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
 	/*
 	 * Configure which varnishd instance to access.
-	 * Can also be, and normally is done through VSC_Arg()/VSL_Arg().
+	 * Uses hostname if n_arg is NULL or "".
+	 *
 	 * Returns:
 	 *	 1 on success
 	 *	 <0 on failure, VSM_Error() returns diagnostic string
@@ -107,8 +108,6 @@ int VSM_N_Arg(struct VSM_data *vd, const char *N_arg);
 	 * bypasses abandonment checks and allows looking at stale VSM
 	 * files without a running Varnish instance.
 	 *
-	 * Can also be, and normally is done through VSC_Arg()/VSL_Arg().
-	 *
 	 * Returns:
 	 *	1 on success
 	 */



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] f09b51f Add a VUT.name desc string of the currently open VSM
 context.
Message-ID: <E1WO1sF-00075a-4D@project.varnish-software.com>


commit f09b51fbad14ec5c11e6422dd1efd429436a684d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:34:58 2014 +0100

    Add a VUT.name desc string of the currently open VSM context.

diff --git a/include/vut.h b/include/vut.h
index ba109f4..a0b9df2 100644
--- a/include/vut.h
+++ b/include/vut.h
@@ -35,6 +35,7 @@ typedef int VUT_cb_f(void);
 
 struct VUT {
 	const char	*progname;
+	char		*name;
 
 	/* Options */
 	int		d_opt;
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 4ca7fbc..f465b1f 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -173,6 +173,7 @@ void
 VUT_Init(const char *progname)
 {
 	VUT.progname = progname;
+	REPLACE(VUT.name, "");
 	VUT.g_arg = VSL_g_vxid;
 	AZ(VUT.vsl);
 	VUT.vsl = VSL_New();
@@ -189,9 +190,10 @@ VUT_Setup(void)
 	/* Input */
 	if (VUT.r_arg && VUT.vsm)
 		VUT_Error(1, "Can't have both -n and -r options");
-	if (VUT.r_arg)
+	if (VUT.r_arg) {
+		REPLACE(VUT.name, VUT.r_arg);
 		c = VSL_CursorFile(VUT.vsl, VUT.r_arg, 0);
-	else {
+	} else {
 		if (VUT.vsm == NULL)
 			/* Default uses VSM with n=hostname */
 			VUT.vsm = VSM_New();
@@ -199,6 +201,7 @@ VUT_Setup(void)
 		if (VSM_Open(VUT.vsm))
 			VUT_Error(1, "Can't open VSM file (%s)",
 			    VSM_Error(VUT.vsm));
+		REPLACE(VUT.name, VSM_Name(VUT.vsm));
 		c = VSL_CursorVSM(VUT.vsl, VUT.vsm,
 		    (VUT.d_opt ? VSL_COPT_TAILSTOP : VSL_COPT_TAIL)
 		    | VSL_COPT_BATCH);
@@ -242,6 +245,7 @@ VUT_Fini(void)
 {
 	free(VUT.r_arg);
 	free(VUT.P_arg);
+	free(VUT.name);
 
 	vut_vpf_remove();
 	AZ(VUT.pfh);



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] d1dd526 Use VUT.name as descriptive name of VSM context.
Message-ID: <E1WO1sF-00075r-5e@project.varnish-software.com>


commit d1dd52656a35e2ef61086f394201aa4179a8c85b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:36:21 2014 +0100

    Use VUT.name as descriptive name of VSM context.
    
    Fixes issue where a 2nd VSM context is used to get the current name,
    not taking any -n/-N arguments into account

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index b5d5c7c..4a6038e 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -182,7 +182,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 }
 
 static void
-update(const struct VSM_data *vd, int p)
+update(int p)
 {
 	struct top *tp, *tp2;
 	int l, len;
@@ -200,7 +200,7 @@ update(const struct VSM_data *vd, int p)
 	if (n < p)
 		n++;
 	AC(erase());
-	AC(mvprintw(0, 0, "%*s", COLS - 1, VSM_Name(vd)));
+	AC(mvprintw(0, 0, "%*s", COLS - 1, VUT.name));
 	AC(mvprintw(0, 0, "list length %u", ntop));
 	for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) {
 		tp2 = VRB_NEXT(top_tree, &top_tree_head, tp);
@@ -231,8 +231,8 @@ static void *
 do_curses(void *arg)
 {
 	int i;
-	struct VSM_data *vd = (struct VSM_data *)arg;
 
+	(void)arg;
 	for (i = 0; i < 256; i++) {
 		if (VSL_tags[i] == NULL)
 			continue;
@@ -249,7 +249,7 @@ do_curses(void *arg)
 	AC(erase());
 	for (;;) {
 		AZ(pthread_mutex_lock(&mtx));
-		update(vd, period);
+		update(period);
 		AZ(pthread_mutex_unlock(&mtx));
 
 		timeout(1000);
@@ -315,11 +315,9 @@ usage(int status)
 int
 main(int argc, char **argv)
 {
-	struct VSM_data *vd;
 	int o, once = 0;
 	pthread_t thr;
 
-	vd = VSM_New();
 	VUT_Init(progname);
 
 	while ((o = getopt(argc, argv, vopt_optstring)) != -1) {
@@ -349,20 +347,21 @@ main(int argc, char **argv)
 		}
 	}
 
-	VUT.dispatch_f = &accumulate;
-	VUT.dispatch_priv = NULL;
-	if (!once){
-		if (pthread_create(&thr, NULL, do_curses, vd) != 0) {
-			fprintf(stderr, "pthread_create(): %s\n", strerror(errno));
+	VUT_Setup();
+	if (!once) {
+		if (pthread_create(&thr, NULL, do_curses, NULL) != 0) {
+			fprintf(stderr, "pthread_create(): %s\n",
+			    strerror(errno));
 			exit(1);
 		}
 	}
-	VUT_Setup();
+	VUT.dispatch_f = &accumulate;
+	VUT.dispatch_priv = NULL;
 	VUT_Main();
-	VUT_Fini();
 	if (once)
 		dump();
 	else
 		pthread_join(thr, NULL);
+	VUT_Fini();
 	exit(0);
 }



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 0ade45c Long line fix
Message-ID: <E1WO1sF-00075x-6Y@project.varnish-software.com>


commit 0ade45c1838eef6653c8771c422d63621991ded4
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:38:50 2014 +0100

    Long line fix

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 4a6038e..3e94445 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -337,7 +337,7 @@ main(int argc, char **argv)
 			period = strtol(optarg, NULL, 0);
 			if (errno != 0)  {
 				fprintf(stderr,
-					"Syntax error, %s is not a number", optarg);
+				    "Syntax error, %s is not a number", optarg);
 				exit(1);
 			}
 			break;



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 139a4ce Display EOF upon hitting end of input
Message-ID: <E1WO1sF-000766-7q@project.varnish-software.com>


commit 139a4ce232ac7fe2fd2b96f41e25d44d58d6c2d1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:51:09 2014 +0100

    Display EOF upon hitting end of input

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 3e94445..3bb12b3 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -62,6 +62,7 @@
 
 static const char progname[] = "varnishtop";
 static float period = 60; /* seconds */
+static int end_of_file = 0;
 
 struct top {
 	uint8_t			tag;
@@ -200,7 +201,11 @@ update(int p)
 	if (n < p)
 		n++;
 	AC(erase());
-	AC(mvprintw(0, 0, "%*s", COLS - 1, VUT.name));
+	if (end_of_file)
+		AC(mvprintw(0, COLS - 1 - strlen(VUT.name) - 5, "%s (EOF)",
+			VUT.name));
+	else
+		AC(mvprintw(0, COLS - 1 - strlen(VUT.name), "%s", VUT.name));
 	AC(mvprintw(0, 0, "list length %u", ntop));
 	for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) {
 		tp2 = VRB_NEXT(top_tree, &top_tree_head, tp);
@@ -358,6 +363,7 @@ main(int argc, char **argv)
 	VUT.dispatch_f = &accumulate;
 	VUT.dispatch_priv = NULL;
 	VUT_Main();
+	end_of_file = 1;
 	if (once)
 		dump();
 	else



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 83196f4 Stop updating counters after hitting EOF
Message-ID: <E1WO1sF-00076F-8w@project.varnish-software.com>


commit 83196f45f6d9d44ee127f38b6889fa36146b2a71
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 30 13:53:42 2014 +0100

    Stop updating counters after hitting EOF

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 3bb12b3..761c3f3 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -220,7 +220,8 @@ update(int p)
 				len, len, tp->rec_data));
 			t = tp->count;
 		}
-		(void)t;
+		if (end_of_file)
+			continue;
 		tp->count += (1.0/3.0 - tp->count) / (double)n;
 		if (tp->count * 10 < t || l > LINES * 10) {
 			VRB_REMOVE(top_tree, &top_tree_head, tp);



From fgsch at lodoss.net  Thu Mar 13 09:24:23 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 2ccf1b6 Some minor improvements
Message-ID: <E1WO1sF-00076S-AU@project.varnish-software.com>


commit 2ccf1b62fff566763df7f483f5044268e6254cf4
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Jan 30 15:21:28 2014 +0000

    Some minor improvements

diff --git a/etc/example.vcl b/etc/example.vcl
index c9b9aad..c093552 100644
--- a/etc/example.vcl
+++ b/etc/example.vcl
@@ -1,14 +1,15 @@
 #
 # This is an example VCL file for Varnish.
+#
 # It does not do anything by default, delegating control to the builtin vcl.
 # The builtin VCL is called when there is no explicit explicit return
 # statement.
-
+#
 # See the VCL tutorial at https://www.varnish-cache.org/docs/trunk/tutorial/
-# See http://varnish-cache.org/trac/wiki/VCLExamples for more examples.
+# and http://varnish-cache.org/trac/wiki/VCLExamples for more examples.
 
-# Marker to tell the VCL compiler that this VCL has been adapted to the new 4.0
-# format.
+# Marker to tell the VCL compiler that this VCL has been adapted to the
+# new 4.0 format.
 vcl 4.0;
 
 # Default backend definition. Set this to point to your content server.
@@ -25,7 +26,6 @@ sub vcl_recv {
 sub vcl_backend_response {
     # Happens after we have read the response headers from the backend.
     # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
-#
 }
 
 sub vcl_deliver {
@@ -33,4 +33,3 @@ sub vcl_deliver {
     # response to the client.
     # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
 }
-



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 2f017f0 Add two counters to keep track of the exp inbox traffic
Message-ID: <E1WO1sF-00076Y-Bn@project.varnish-software.com>


commit 2f017f0d8f3c409a25722253c2480dbfa929048f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 30 21:13:44 2014 +0000

    Add two counters to keep track of the exp inbox traffic

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index f1b65f3..eeda901 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -117,6 +117,7 @@ exp_mail_it(struct objcore *oc)
 		VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
 	else
 		VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
+	VSC_C_main->exp_mailed++;
 	AZ(pthread_cond_signal(&exphdl->condvar));
 	Lck_Unlock(&exphdl->mtx);
 }
@@ -583,6 +584,7 @@ exp_thread(struct worker *wrk, void *priv)
 		oc = VTAILQ_FIRST(&ep->inbox);
 		if (oc != NULL) {
 			VTAILQ_REMOVE(&ep->inbox, oc, lru_list);
+			VSC_C_main->exp_received++;
 			tnext = 0;
 		} else if (tnext > t) {
 			VSL_Flush(&ep->vsl, 0);
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 9d7da8c..d72b325 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -561,6 +561,18 @@ VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g', diag,
 
 /*--------------------------------------------------------------------*/
 
+VSC_F(exp_mailed,			uint64_t, 0, 'c', diag,
+    "Number of objects mailed to expiry thread",
+	"Number of objects mailed to expiry thread for handling."
+)
+
+VSC_F(exp_received,			uint64_t, 0, 'c', diag,
+    "Number of objects received by expiry thread",
+	"Number of objects received by expiry thread for handling."
+)
+
+/*--------------------------------------------------------------------*/
+
 VSC_F(hcb_nolock,		uint64_t, 1, 'a', debug,
     "HCB Lookups without lock",
 	""



From fgsch at lodoss.net  Thu Mar 13 09:24:23 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] ea100e0 Update tests so they compile at the very least
Message-ID: <E1WO1sF-00076h-Cs@project.varnish-software.com>


commit ea100e0a8612811cb3d05cccdca2f609ade84b44
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Jan 31 10:40:57 2014 +0000

    Update tests so they compile at the very least

diff --git a/bin/varnishtest/tests.disabled/r00693.vtc b/bin/varnishtest/tests.disabled/r00693.vtc
index bf94f6b..4947dbc 100644
--- a/bin/varnishtest/tests.disabled/r00693.vtc
+++ b/bin/varnishtest/tests.disabled/r00693.vtc
@@ -41,7 +41,7 @@ varnish v1 -arg "-p workspace_client=3072" -vcl+backend {
 	}
 	sub vcl_hash {
 		hash_data(req.url);
-		return (hash);
+		return (lookup);
 	}
 } -start
 
diff --git a/bin/varnishtest/tests.disabled/r00978.vtc b/bin/varnishtest/tests.disabled/r00978.vtc
index 8d52e60..e02be57 100644
--- a/bin/varnishtest/tests.disabled/r00978.vtc
+++ b/bin/varnishtest/tests.disabled/r00978.vtc
@@ -23,7 +23,7 @@ varnish v1 -vcl+backend {
 		}
 	}
 	sub vcl_backend_response {
-		if (req.url == "/") {
+		if (bereq.url == "/") {
 			set beresp.do_esi = true;
 		}
 		set beresp.do_stream = true;
diff --git a/bin/varnishtest/tests.disabled/t00001.vtc b/bin/varnishtest/tests.disabled/t00001.vtc
index 4250b1a..a94e956 100644
--- a/bin/varnishtest/tests.disabled/t00001.vtc
+++ b/bin/varnishtest/tests.disabled/t00001.vtc
@@ -23,10 +23,10 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url == "/foo") {
+		if (bereq.url == "/foo") {
 			set beresp.do_gzip = true;
 		}
-		if (req.url == "/barf") {
+		if (bereq.url == "/barf") {
 			set beresp.do_gunzip = true;
 		}
 		set beresp.do_stream = true;



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 2e63682 Strengten link checks against loops in the linked
 records.
Message-ID: <E1WO1sF-00076q-EK@project.varnish-software.com>


commit 2e63682065c9d676f691e4900e57c3677ec4ebcc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 4 11:24:09 2014 +0100

    Strengten link checks against loops in the linked records.

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 5ff57e3..c0ed3c9 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -656,6 +656,7 @@ vtx_set_parent(struct vtx *parent, struct vtx *child)
 
 	CHECK_OBJ_NOTNULL(parent, VTX_MAGIC);
 	CHECK_OBJ_NOTNULL(child, VTX_MAGIC);
+	assert(parent != child);
 	AZ(parent->flags & VTX_F_COMPLETE);
 	AZ(child->flags & VTX_F_COMPLETE);
 	AZ(child->parent);
@@ -736,8 +737,10 @@ vtx_scan_begin(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	vtx->reason = reason;
 
 	if (p_vxid == 0)
-		/* No parent */
+		/* Zero means no parent */
 		return (0);
+	if (p_vxid == vtx->key.vxid)
+		return (vtx_diag_tag(vtx, ptr, "link to self"));
 
 	if (vslq->grouping == VSL_g_vxid)
 		return (0);	/* No links */
@@ -796,6 +799,11 @@ vtx_scan_link(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	if (vslq->grouping == VSL_g_request && vtx->type == VSL_t_sess)
 		return (0);	/* No links */
 
+	if (c_vxid == 0)
+		return (vtx_diag_tag(vtx, ptr, "illegal link vxid"));
+	if (c_vxid == vtx->key.vxid)
+		return (vtx_diag_tag(vtx, ptr, "link to self"));
+
 	/* Lookup and check child vtx */
 	c_vtx = vtx_lookup(vslq, c_vxid);
 	if (c_vtx == NULL) {



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 5659c4d Don't loop to your self in synthesized Begin records.
Message-ID: <E1WO1sF-000773-Fv@project.varnish-software.com>


commit 5659c4d0900a522e729a6c9e573ed032e6b587fd
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 4 11:25:04 2014 +0100

    Don't loop to your self in synthesized Begin records.
    
    This would cause the API to link this log record to itself, creating
    endless loops. Correct it by setting it to zero (meaning no parent).

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index c0ed3c9..0aee758 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -881,7 +881,7 @@ vtx_force(struct VSLQ *vslq, struct vtx *vtx, const char *reason)
 	vtx_scan(vslq, vtx);
 	if (!(vtx->flags & VTX_F_BEGIN))
 		vtx_synth_rec(vtx, SLT_Begin, "%s %u synth",
-		    vsl_t_names[vtx->type], vtx->key.vxid);
+		    vsl_t_names[vtx->type], 0);
 	vtx_diag(vtx, reason);
 	if (!(vtx->flags & VTX_F_END))
 		vtx_synth_rec(vtx, SLT_End, "synth");



From daghf at varnish-software.com  Thu Mar 13 09:24:23 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 336bd58 Fix WS_Printf off by one error/truncation handling.
Message-ID: <E1WO1sF-000779-H0@project.varnish-software.com>


commit 336bd5861a0340d0729c4b053dfc01e127f2f1da
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Feb 4 12:59:47 2014 +0100

    Fix WS_Printf off by one error/truncation handling.

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 5ed9040..8ef7aca 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -175,11 +175,12 @@ WS_Printf(struct ws *ws, const char *fmt, ...)
 	p = ws->f;
 	va_start(ap, fmt);
 	v = vsnprintf(p, u, fmt, ap);
-	if (v > u) {
+	va_end(ap);
+	if (v >= u) {
 		WS_Release(ws, 0);
 		p = NULL;
 	} else {
-		WS_Release(ws, v);
+		WS_Release(ws, v + 1);
 	}
 	return (p);
 }



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 2eddf20 Rewrite test-cases to not use req.backend
Message-ID: <E1WO1sF-00077I-I2@project.varnish-software.com>


commit 2eddf20ca7dffe58e94144982c5be403b7224de4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 5 09:32:26 2014 +0000

    Rewrite test-cases to not use req.backend

diff --git a/bin/varnishtest/tests/b00026.vtc b/bin/varnishtest/tests/b00026.vtc
index b094b0a..2f5915f 100644
--- a/bin/varnishtest/tests/b00026.vtc
+++ b/bin/varnishtest/tests/b00026.vtc
@@ -29,13 +29,16 @@ varnish v1 -vcl {
 
 	sub vcl_recv {
 		if (req.url  == "from_backend") {
-			set req.backend = b1;
 			return(pass);
 		}
-		set req.backend = b2;
 	}
 	sub vcl_backend_fetch {
 		set bereq.first_byte_timeout = 2s;
+		if (bereq.url  == "from_backend") {
+			set bereq.backend = b1;
+		} else {
+			set bereq.backend = b2;
+		}
 	}
 } -start
 varnish v1 -cliok "param.set first_byte_timeout 0.5"
diff --git a/bin/varnishtest/tests/c00009.vtc b/bin/varnishtest/tests/c00009.vtc
index 3d43334..7931e81 100644
--- a/bin/varnishtest/tests/c00009.vtc
+++ b/bin/varnishtest/tests/c00009.vtc
@@ -14,10 +14,13 @@ server s2 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		if (req.restarts == 0) {
-			set req.backend = s1;
+		set req.http.restarts = req.restarts;
+	}
+	sub vcl_backend_fetch {
+		if (bereq.http.restarts == "0") {
+			set bereq.backend = s1;
 		} else {
-			set req.backend = s2;
+			set bereq.backend = s2;
 		}
 	}
 
diff --git a/bin/varnishtest/tests/c00028.vtc b/bin/varnishtest/tests/c00028.vtc
index 4f8a0ff..91abe45 100644
--- a/bin/varnishtest/tests/c00028.vtc
+++ b/bin/varnishtest/tests/c00028.vtc
@@ -5,8 +5,8 @@ varnish v1 -vcl {
 		.host = "${bad_ip}"; .port = "9090";
 	}
 
-	sub vcl_recv {
-	    set req.backend = bad;
+	sub vcl_backend_fetch {
+	    set bereq.backend = bad;
 	}
 	sub vcl_error {
 	    return (restart);
diff --git a/bin/varnishtest/tests/c00038.vtc b/bin/varnishtest/tests/c00038.vtc
index 0764e85..e9402b6 100644
--- a/bin/varnishtest/tests/c00038.vtc
+++ b/bin/varnishtest/tests/c00038.vtc
@@ -17,11 +17,13 @@ varnish v1 -vcl+backend {
 		if (req.http.x-ignorebusy == "1") {
 			set req.hash_ignore_busy = true;
 		}
-		if (req.http.x-client == "1") {
-			set req.backend = s1;
+	}
+	sub vcl_backend_fetch {
+		if (bereq.http.x-client == "1") {
+			set bereq.backend = s1;
 		}
-		if (req.http.x-client == "2") {
-			set req.backend = s2;
+		if (bereq.http.x-client == "2") {
+			set bereq.backend = s2;
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/c00043.vtc b/bin/varnishtest/tests/c00043.vtc
index 46cc4e1..0960621 100644
--- a/bin/varnishtest/tests/c00043.vtc
+++ b/bin/varnishtest/tests/c00043.vtc
@@ -16,9 +16,9 @@ server s2 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_recv {
-		if (req.http.bar) {
-			set req.backend = s2;
+	sub vcl_backend_fetch {
+		if (bereq.http.bar) {
+			set bereq.backend = s2;
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/e00006.vtc b/bin/varnishtest/tests/e00006.vtc
index 79382fc..759a0f4 100644
--- a/bin/varnishtest/tests/e00006.vtc
+++ b/bin/varnishtest/tests/e00006.vtc
@@ -22,11 +22,11 @@ server s2 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_recv {
-		if (req.http.host == "bozz") {
-			set req.backend = s2;
+	sub vcl_backend_fetch {
+		if (bereq.http.host == "bozz") {
+			set bereq.backend = s2;
 		} else {
-			set req.backend = s1;
+			set bereq.backend = s1;
 		}
 	}
 	sub vcl_backend_response {
diff --git a/bin/varnishtest/tests/m00009.vtc b/bin/varnishtest/tests/m00009.vtc
index 4e97863..7fbeec1 100644
--- a/bin/varnishtest/tests/m00009.vtc
+++ b/bin/varnishtest/tests/m00009.vtc
@@ -34,8 +34,8 @@ varnish v1 -vcl+backend {
 		rr.add_backend(s4);
 	}
 
-	sub vcl_recv {
-		set req.backend = rr.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = rr.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/m00010.vtc b/bin/varnishtest/tests/m00010.vtc
index db2a07f..9d8ba1d 100644
--- a/bin/varnishtest/tests/m00010.vtc
+++ b/bin/varnishtest/tests/m00010.vtc
@@ -47,8 +47,8 @@ varnish v1 -vcl+backend {
 		rr3.add_backend(rr2.backend());
 	}
 
-	sub vcl_recv {
-		set req.backend = rr3.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = rr3.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00255.vtc b/bin/varnishtest/tests/r00255.vtc
index 1aedcc1..27b2bc8 100644
--- a/bin/varnishtest/tests/r00255.vtc
+++ b/bin/varnishtest/tests/r00255.vtc
@@ -10,7 +10,11 @@ server s1 {
 	    -body "012345\n"
 } -start
 
-varnish v1 -vcl+backend { sub vcl_recv {set req.backend = s1;} } -start
+varnish v1 -vcl+backend {
+	sub vcl_backend_fetch {
+		set bereq.backend = s1;
+	}
+} -start
 
 client c1 {
 	txreq -url "/"
diff --git a/bin/varnishtest/tests/r00263.vtc b/bin/varnishtest/tests/r00263.vtc
index 39f1ef5..c9036d5 100644
--- a/bin/varnishtest/tests/r00263.vtc
+++ b/bin/varnishtest/tests/r00263.vtc
@@ -16,8 +16,8 @@ varnish v1  -vcl {
 		cluster.add_backend(node3, 1);
 	}
 
-	sub vcl_recv {
-		set req.backend = cluster.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = cluster.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00306.vtc b/bin/varnishtest/tests/r00306.vtc
index 9e85b25..0d4c465 100644
--- a/bin/varnishtest/tests/r00306.vtc
+++ b/bin/varnishtest/tests/r00306.vtc
@@ -33,8 +33,8 @@ varnish v1 -vcl {
 		foo.add_backend(s2, 1);
 	}
 
-	sub vcl_recv {
-		set req.backend = foo.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = foo.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00433.vtc b/bin/varnishtest/tests/r00433.vtc
index 01462d4..494d8a8 100644
--- a/bin/varnishtest/tests/r00433.vtc
+++ b/bin/varnishtest/tests/r00433.vtc
@@ -23,11 +23,11 @@ server s2 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_recv {
-		if (req.url == "/foo") {
-			set req.backend = s1;
+	sub vcl_backend_fetch {
+		if (bereq.url == "/foo") {
+			set bereq.backend = s1;
 		} else {
-			set req.backend = s2;
+			set bereq.backend = s2;
 		}
 	}
 
diff --git a/bin/varnishtest/tests/r00722.vtc b/bin/varnishtest/tests/r00722.vtc
index ef7268c..739b8f0 100644
--- a/bin/varnishtest/tests/r00722.vtc
+++ b/bin/varnishtest/tests/r00722.vtc
@@ -22,8 +22,8 @@ varnish v1 -vcl+backend {
 		foo.add_backend(b2, 1);
 		foo.add_backend(b3, 1);
 	}
-	sub vcl_recv {
-		set req.backend = foo.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = foo.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00916.vtc b/bin/varnishtest/tests/r00916.vtc
index 2b3a9e0..04aa7c6 100644
--- a/bin/varnishtest/tests/r00916.vtc
+++ b/bin/varnishtest/tests/r00916.vtc
@@ -10,8 +10,8 @@ varnish v1 -errvcl {Symbol not found: 's-1' (expected type BACKEND)} {
 	sub s1 {
 	}
 	sub vcl_backend_response {
-	    if (req.backend == s-1){
-		set req.backend = s-1;
+	    if (bereq.backend == s-1){
+		set bereq.backend = s-1;
 	    }
 	}
 }
diff --git a/bin/varnishtest/tests/r01073.vtc b/bin/varnishtest/tests/r01073.vtc
index 58d8ae6..3e0c44e 100644
--- a/bin/varnishtest/tests/r01073.vtc
+++ b/bin/varnishtest/tests/r01073.vtc
@@ -19,11 +19,13 @@ varnish v1 -vcl+backend {
 		if (req.http.x-hash-always-miss == "1") {
 			set req.hash_always_miss = true;
 		}
-		if (req.http.x-client == "1") {
-			set req.backend = s1;
+	}
+	sub vcl_backend_fetch {
+		if (bereq.http.x-client == "1") {
+			set bereq.backend = s1;
 		}
-		if (req.http.x-client == "2") {
-			set req.backend = s2;
+		if (bereq.http.x-client == "2") {
+			set bereq.backend = s2;
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/r01144.vtc b/bin/varnishtest/tests/r01144.vtc
index dcd1505..284c848 100644
--- a/bin/varnishtest/tests/r01144.vtc
+++ b/bin/varnishtest/tests/r01144.vtc
@@ -11,9 +11,9 @@ varnish v1 -vcl+backend {
 		.host = "127.0.0.1";
 		.port = "54321";
 	}
-	sub vcl_recv {
-		if (req.url == "never") {
-			set req.backend = fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210;
+	sub vcl_backend_fetch {
+		if (bereq.url == "never") {
+			set bereq.backend = fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210;
 		}
 	}
 } -start
@@ -30,9 +30,9 @@ varnish v1 -errvcl {Expected ID got '0'} {
 		.host = "127.0.0.1";
 		.port = "54321";
 	}
-	sub vcl_recv {
-		if (req.url == "never") {
-			set req.backend = fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210;
+	sub vcl_backend_fetch {
+		if (bereq.url == "never") {
+			set bereq.backend = fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210;
 		}
 	}
 }
diff --git a/bin/varnishtest/tests/r01164.vtc b/bin/varnishtest/tests/r01164.vtc
index b9072e1..2951482 100644
--- a/bin/varnishtest/tests/r01164.vtc
+++ b/bin/varnishtest/tests/r01164.vtc
@@ -5,8 +5,8 @@ varnish v1 -vcl {
 		.host = "127.0.0.1";
 	}
 
-	sub vcl_recv {
-		set req.backend = storage_foo;
+	sub vcl_backend_fetch {
+		set bereq.backend = storage_foo;
 	}
 }
 
@@ -15,8 +15,8 @@ varnish v1 -vcl {
 		.host = "127.0.0.1";
 	}
 
-	sub vcl_recv {
-		set req.backend = storagefoo;
+	sub vcl_backend_fetch {
+		set bereq.backend = storagefoo;
 	}
 }
 
diff --git a/bin/varnishtest/tests/v00007.vtc b/bin/varnishtest/tests/v00007.vtc
index 313d970..196f92e 100644
--- a/bin/varnishtest/tests/v00007.vtc
+++ b/bin/varnishtest/tests/v00007.vtc
@@ -16,8 +16,8 @@ varnish v1 -vcl+backend {
 		foo.add_backend(s1, 1);
 	}
 
-	sub vcl_recv {
-		set req.backend = foo.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = foo.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/v00009.vtc b/bin/varnishtest/tests/v00009.vtc
index a356a51..af925fb 100644
--- a/bin/varnishtest/tests/v00009.vtc
+++ b/bin/varnishtest/tests/v00009.vtc
@@ -33,8 +33,8 @@ varnish v1 -vcl+backend {
 		batman.add_backend(s4);
 	}
 
-	sub vcl_recv {
-		set req.backend = batman.backend();
+	sub vcl_backend_fetch {
+		set bereq.backend = batman.backend();
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/v00014.vtc b/bin/varnishtest/tests/v00014.vtc
index 67d3431..3a48188 100644
--- a/bin/varnishtest/tests/v00014.vtc
+++ b/bin/varnishtest/tests/v00014.vtc
@@ -27,7 +27,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_recv {
-		if (std.healthy(req.backend)) {
+		if (std.healthy(default)) {
 			return(error(200,"Backend healthy"));
 		} else {
 			return(error(500,"Backend sick"));
diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index a155dd0..7a00d95 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -84,8 +84,9 @@ varnish v1 -errvcl {Expression has type DURATION, expected BOOL} {
 }
 
 varnish v1 -errvcl {Operator > not possible on BACKEND} {
+	backend a { .host = "127.0.0.1"; }
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { if (req.backend > b) { } }
+	sub vcl_recv { if (a > b) { } }
 }
 
 varnish v1 -errvcl {Symbol not found: 'req.hash' (expected type BOOL):} {
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index c264cca..720c816 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -29,7 +29,7 @@ varnish v1 -errvcl {Expected '=' got '>>'} {
 
 varnish v1 -errvcl {Expected '=' got '+='} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { set req.backend += b; }
+	sub vcl_backend_fetch { set bereq.backend += b; }
 }
 
 varnish v1 -errvcl {Expected '=' got '+='} {
diff --git a/bin/varnishtest/tests/v00022.vtc b/bin/varnishtest/tests/v00022.vtc
index 10f4c9f..5e5a17b 100644
--- a/bin/varnishtest/tests/v00022.vtc
+++ b/bin/varnishtest/tests/v00022.vtc
@@ -43,9 +43,11 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_recv {
-		set req.backend = foo.backend();
 		return(pass);
 	}
+	sub vcl_backend_fetch {
+		set bereq.backend = foo.backend();
+	}
 } -start
 
 # NB: Do not change the number 1
diff --git a/bin/varnishtest/tests/v00026.vtc b/bin/varnishtest/tests/v00026.vtc
index 40d3b33..e1b77b9 100644
--- a/bin/varnishtest/tests/v00026.vtc
+++ b/bin/varnishtest/tests/v00026.vtc
@@ -25,9 +25,11 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_recv {
-		set req.backend = h1.backend(req.url);
 		return(pass);
 	}
+	sub vcl_backend_fetch {
+		set bereq.backend = h1.backend(bereq.url);
+	}
 
 } -start
 
diff --git a/bin/varnishtest/tests/v00027.vtc b/bin/varnishtest/tests/v00027.vtc
index 72a7030..f014322 100644
--- a/bin/varnishtest/tests/v00027.vtc
+++ b/bin/varnishtest/tests/v00027.vtc
@@ -20,9 +20,9 @@ varnish v1 -vcl {
 		.port = "${s2_port}";
 	}
 
-	sub vcl_recv {
-		if (req.url != req.url) {
-			set req.backend = s1;
+	sub vcl_backend_fetch {
+		if (bereq.url != bereq.url) {
+			set bereq.backend = s1;
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index 42e9123..7bb1be8 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -26,9 +26,11 @@ varnish v1 -vcl+backend {
 	}
 
         sub vcl_recv {
-                set req.backend = fb1.backend();
                 return (pass);
         }
+        sub vcl_backend_fetch {
+                set bereq.backend = fb1.backend();
+        }
 
 } -start
 



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 7f97686 Rename req.backend to req.backend_hint and restrict it
 to vcl_recv{}
Message-ID: <E1WO1sF-00077S-Jj@project.varnish-software.com>


commit 7f9768602064589d122f72aea95e05c894c7e975
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 5 10:44:34 2014 +0000

    Rename req.backend to req.backend_hint and restrict it to vcl_recv{}
    
    Rename beresp.storage to beresp.storage_hint for consistency.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6e522bb..93ec51f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -664,7 +664,7 @@ struct req {
 	uint16_t		err_code;
 	const char		*err_reason;
 
-	struct director		*director;
+	struct director		*director_hint;
 	struct VCL_conf		*vcl;
 
 	char			*ws_req;	/* WS above request data */
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 2466404..65a24bf 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -144,7 +144,7 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 
 	bo->do_stream = 1;
 
-	bo->director = req->director;
+	bo->director = req->director_hint;
 	bo->vcl = req->vcl;
 	VCL_Ref(bo->vcl);
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index eb69164..b7eba44 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -357,7 +357,6 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	wrk = req->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req->http, HTTP_MAGIC);
-	AN(req->director);
 	AN(hash);
 
 	hsh_prealloc(wrk);
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 3389f6d..5d0f286 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -183,7 +183,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC);
 
 	AZ(req->obj);
-	req->director = NULL;
+	req->director_hint = NULL;
 	req->restarts = 0;
 
 	AZ(req->esi_level);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ec7713a..b2f3ef0 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -208,7 +208,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	bo->stats = NULL;
 	if (req->obj == NULL) {
 		req->doclose = SC_OVERLOAD;
-		req->director = NULL;
+		req->director_hint = NULL;
 		AZ(HSH_DerefObjCore(&wrk->stats, &bo->fetch_objcore));
 		bo->fetch_objcore = NULL;
 		http_Teardown(bo->beresp);
@@ -634,7 +634,7 @@ cnt_restart(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	req->director = NULL;
+	req->director_hint = NULL;
 	if (++req->restarts >= cache_param->max_restarts) {
 		VSLb(req->vsl, SLT_VCL_Error, "Too many restarts");
 		req->err_code = 503;
@@ -698,9 +698,9 @@ cnt_recv(struct worker *wrk, struct req *req)
 	}
 
 	/* By default we use the first backend */
-	AZ(req->director);
-	req->director = req->vcl->director[0];
-	AN(req->director);
+	AZ(req->director_hint);
+	req->director_hint = req->vcl->director[0];
+	AN(req->director_hint);
 
 	req->d_ttl = -1;
 	req->disable_esi = 0;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index dad35f3..5d875ed 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -279,7 +279,7 @@ VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx)
 }
 
 const char *
-VRT_r_beresp_storage(const struct vrt_ctx *ctx)
+VRT_r_beresp_storage_hint(const struct vrt_ctx *ctx)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
@@ -290,7 +290,7 @@ VRT_r_beresp_storage(const struct vrt_ctx *ctx)
 }
 
 void
-VRT_l_beresp_storage(const struct vrt_ctx *ctx, const char *str, ...)
+VRT_l_beresp_storage_hint(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
 	const char *b;
@@ -326,8 +326,8 @@ VRT_r_req_##nm(const struct vrt_ctx *ctx)				\
 	return(ctx->req->elem);						\
 }
 
-REQ_VAR_L(backend, director, struct director *,)
-REQ_VAR_R(backend, director, struct director *)
+REQ_VAR_L(backend_hint, director_hint, struct director *,)
+REQ_VAR_R(backend_hint, director_hint, struct director *)
 REQ_VAR_L(ttl, d_ttl, double, if (!(arg>0.0)) arg = 0;)
 REQ_VAR_R(ttl, d_ttl, double)
 
diff --git a/bin/varnishtest/tests/b00016.vtc b/bin/varnishtest/tests/b00016.vtc
index fc2478e..f4763ee 100644
--- a/bin/varnishtest/tests/b00016.vtc
+++ b/bin/varnishtest/tests/b00016.vtc
@@ -1,74 +1,42 @@
 varnishtest "Check naming of backends"
 
-varnish v1 -vcl {
-	backend foo {
-		.host = "${bad_ip}"; .port = "9080";
-	}
+server s1 {
+	rxreq
+	txresp
+} -start
 
-	sub vcl_recv {
-	    return(error(200,"ok"));
-	}
+varnish v1 -vcl+backend {
 
-	sub vcl_error {
-	    set obj.http.X-Backend-Name = req.backend;
+	sub vcl_backend_response {
+	    set beresp.http.X-Backend-Name = bereq.backend;
 	}
 } -start
 
 client c1 {
 	txreq -url "/"
 	rxresp
-	expect resp.http.X-Backend-Name == "foo"
+	expect resp.http.X-Backend-Name == "s1"
 } -run
 
-varnish v1 -vcl {
+varnish v1 -vcl+backend {
 	import ${vmod_directors};
 
-	backend b1 {
-		.host = "${bad_ip}"; .port = "9080";
-	}
 	sub vcl_init {
 		new bar = directors.random();
-		bar.add_backend(b1, 1);
-	}
-
-	sub vcl_recv {
-		set req.backend = bar.backend();
-		return(error(200,"ok"));
-	}
-
-	sub vcl_error {
-		set obj.http.X-Backend-Name = req.backend;
-	}
-}
-
-client c1 {
-	txreq -url "/"
-	rxresp
-	expect resp.http.X-Backend-Name == "bar"
-} -run
-
-varnish v1 -vcl {
-	import ${vmod_directors};
-	backend b1 {
-		.host = "${bad_ip}"; .port = "9080";
-	}
-	sub vcl_init {
-		new baz = directors.round_robin();
-		baz.add_backend(b1);
+		bar.add_backend(s1, 1);
 	}
 
 	sub vcl_recv {
-	    set req.backend = baz.backend();
-	    return (error(200,"ok"));
+		set req.backend_hint = bar.backend();
 	}
 
-	sub vcl_error {
-	    set obj.http.X-Backend-Name = req.backend;
+	sub vcl_backend_response {
+		set beresp.http.X-Backend-Name = bereq.backend;
 	}
 }
 
 client c1 {
 	txreq -url "/"
 	rxresp
-	expect resp.http.X-Backend-Name == "baz"
+	expect resp.http.X-Backend-Name == "s1"
 } -run
diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc
index fb415eb..9bd7950 100644
--- a/bin/varnishtest/tests/c00044.vtc
+++ b/bin/varnishtest/tests/c00044.vtc
@@ -24,7 +24,7 @@ varnish v1 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
-		set beresp.storage = "invalid";
+		set beresp.storage_hint = "invalid";
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 23967fa..c653cf0 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -16,7 +16,7 @@ varnish v1 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
-		set beresp.storage = "s0";
+		set beresp.storage_hint = "s0";
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00046.vtc b/bin/varnishtest/tests/c00046.vtc
index a2e1d2b..3abea2f 100644
--- a/bin/varnishtest/tests/c00046.vtc
+++ b/bin/varnishtest/tests/c00046.vtc
@@ -15,7 +15,7 @@ varnish v1 \
 	-arg "-smalloc,1m" \
 	-vcl+backend {
 	sub vcl_backend_response {
-		set beresp.storage = "s0";
+		set beresp.storage_hint = "s0";
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc
index 411c4b4..b839a0c 100644
--- a/bin/varnishtest/tests/p00008.vtc
+++ b/bin/varnishtest/tests/p00008.vtc
@@ -20,9 +20,9 @@ varnish v1 \
 	-arg "-sper2=persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 		sub vcl_backend_response {
-			set beresp.storage = "per1";
+			set beresp.storage_hint = "per1";
 			if (bereq.url ~ "silo2") {
-				set beresp.storage = "per2";
+				set beresp.storage_hint = "per2";
 			}
 		}
 	} -start
diff --git a/bin/varnishtest/tests/r01175.vtc b/bin/varnishtest/tests/r01175.vtc
index 5539b20..5624c20 100644
--- a/bin/varnishtest/tests/r01175.vtc
+++ b/bin/varnishtest/tests/r01175.vtc
@@ -7,7 +7,7 @@ server s1 {
 
 varnish v1 -arg "-s test=malloc,1M" -vcl+backend {
 	sub vcl_backend_response {
-		set beresp.storage = "test";
+		set beresp.storage_hint = "test";
 		set beresp.do_stream = false;
 	}
 } -start
diff --git a/bin/varnishtest/tests/r01283.vtc b/bin/varnishtest/tests/r01283.vtc
index 015b2fa..9b843ce 100644
--- a/bin/varnishtest/tests/r01283.vtc
+++ b/bin/varnishtest/tests/r01283.vtc
@@ -16,7 +16,7 @@ varnish v1 \
 	}
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
-		set beresp.storage = "Transient";
+		set beresp.storage_hint = "Transient";
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 68dc456..697f104 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -15,7 +15,7 @@ varnish v1 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
-		set beresp.storage = "Transient";
+		set beresp.storage_hint = "Transient";
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/v00033.vtc b/bin/varnishtest/tests/v00033.vtc
index d3eecc3..a10fac1 100644
--- a/bin/varnishtest/tests/v00033.vtc
+++ b/bin/varnishtest/tests/v00033.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl+backend {
 		    storage.nowhere.free_space +
 		    1 B + 1 KB + 1 MB + 1GB + 1TB;
 		if (bereq.url == "/foo") {
-			set beresp.storage = "Transient";
+			set beresp.storage_hint = "Transient";
 		}
 	}
 	sub vcl_deliver {
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index a36c2d4..8d8f14f 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -253,11 +253,13 @@ The client's IP address.
 		Does the client accept the gzip transfer encoding.
 		"""
 	),
-	('req.backend',
+	('req.backend_hint',
 		'BACKEND',
-		( 'client',),
-		( 'client',), """
-		The backend to use to service the request.
+		( 'recv',),
+		( 'recv',), """
+		Set bereq.backend to this if we attemt to fetch.
+		This variable is a convenience so the overall
+		policy can be set up once and for all in vcl_recv{}.
 		"""
 	),
 	('req.hash_ignore_busy',
@@ -455,11 +457,11 @@ The client's IP address.
 		IP of the backend this response was fetched from.
 		"""
 	),
-	('beresp.storage',
+	('beresp.storage_hint',
 		'STRING',
 		( 'backend_response',),
 		( 'backend_response',), """
-		Set to force Varnish to save this object to a
+		Hint to Varnish that you want to save this object to a
 		particular storage backend.
 		"""
 	),



From tfheen at err.no  Thu Mar 13 09:24:23 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] b52a310 Generate reference/params.rst as part of build process
Message-ID: <E1WO1sF-00077e-L9@project.varnish-software.com>


commit b52a3108c28a357c82be4318e84a7a2b3c24516b
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Wed Feb 5 12:28:58 2014 +0100

    Generate reference/params.rst as part of build process

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 54c01f6..692cf6f 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -30,7 +30,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html:
+html: reference/params.rst
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -176,6 +176,5 @@ dist-hook:
 distclean-local:
 	rm -rf $(BUILDDIR)
 
-# XXX: doesn't work...
-#reference/params.rst: $(top_builddir)/bin/varnishd/varnishd
-#	$(top_builddir)/bin/varnishd/varnishd -x dumprstparam > reference/params.rst
+reference/params.rst:
+	$(top_builddir)/bin/varnishd/varnishd -x dumprstparam > reference/params.rst
diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
deleted file mode 100644
index e475158..0000000
--- a/doc/sphinx/reference/params.rst
+++ /dev/null
@@ -1,1090 +0,0 @@
-
-.. The following is the autogenerated output from varnishd -x dumprstparam
-
-.. _ref_param_accept_filter:
-
-accept_filter
-~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-	* Flags: must_restart
-
-Enable kernel accept-filters, if supported by the kernel.
-
-.. _ref_param_acceptor_sleep_decay:
-
-acceptor_sleep_decay
-~~~~~~~~~~~~~~~~~~~~
-	* Default: 0.9
-	* Minimum: 0
-	* Maximum: 1
-	* Flags: experimental
-
-If we run out of resources, such as file descriptors or worker threads, the acceptor will sleep between accepts.
-This parameter (multiplicatively) reduces the sleep duration for each succesfull accept. (ie: 0.9 = reduce by 10%)
-
-.. _ref_param_acceptor_sleep_incr:
-
-acceptor_sleep_incr
-~~~~~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 0.001000
-	* Minimum: 0.000000
-	* Maximum: 1.000000
-	* Flags: experimental
-
-If we run out of resources, such as file descriptors or worker threads, the acceptor will sleep between accepts.
-This parameter control how much longer we sleep, each time we fail to accept a new connection in succession.
-
-.. _ref_param_acceptor_sleep_max:
-
-acceptor_sleep_max
-~~~~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 0.050000
-	* Minimum: 0.000000
-	* Maximum: 10.000000
-	* Flags: experimental
-
-If we run out of resources, such as file descriptors or worker threads, the acceptor will sleep between accepts.
-This parameter limits how long it can sleep between attempts to accept new connections.
-
-.. _ref_param_auto_restart:
-
-auto_restart
-~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Restart child process automatically if it dies. This is not related to restarts in VCL.
-
-.. _ref_param_ban_dups:
-
-ban_dups
-~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Detect and eliminate duplicate bans.
-
-.. _ref_param_ban_lurker_sleep:
-
-ban_lurker_sleep
-~~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 0.010000
-	* Minimum: 0.000000
-
-How long time does the ban lurker thread sleep between successful attempts to push the last item up the ban list.  It always sleeps a second when nothing can be done.
-A value of zero disables the ban lurker.
-
-.. _ref_param_between_bytes_timeout:
-
-between_bytes_timeout
-~~~~~~~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 60.000000
-	* Minimum: 0.000000
-
-Default timeout between bytes when receiving data from backend. We only wait for this many seconds between bytes before giving up. A value of 0 means it will never time out. VCL can override this default value for each backend request and backend request. This parameter does not apply to pipe mode.
-
-.. _ref_param_busyobj_worker_cache:
-
-busyobj_worker_cache
-~~~~~~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: off
-
-Cache free busyobj per worker thread. Disable this if you have very high hitrates and want to save the memory of one busyobj per worker thread.
-
-.. _ref_param_cc_command:
-
-cc_command
-~~~~~~~~~~
-	* Default: "exec clang -std=gnu99  -Qunused-arguments -D_THREAD_SAFE -pthread -fpic -shared -Wl,-x -o %o %s"
-	* Flags: must_reload
-
-Command used for compiling the C source code to a dlopen(3) loadable object.  Any occurrence of %s in the string will be replaced with the source file name, and %o will be replaced with the output file name.
-
-.. _ref_param_cli_buffer:
-
-cli_buffer
-~~~~~~~~~~
-	* Units: bytes
-	* Default: 8k
-	* Minimum: 4k
-
-Size of buffer for CLI command input.
-You may need to increase this if you have big VCL files and use the vcl.inline CLI command.
-NB: Must be specified with -p to have an effect.
-
-.. _ref_param_cli_limit:
-
-cli_limit
-~~~~~~~~~
-	* Units: bytes
-	* Default: 48k
-	* Minimum: 128b
-	* Maximum: 99999999b
-
-Maximum size of CLI response.  If the response exceeds this limit, the reponse code will be 201 instead of 200 and the last line will indicate the truncation.
-
-.. _ref_param_cli_timeout:
-
-cli_timeout
-~~~~~~~~~~~
-	* Units: seconds
-	* Default: 10
-	* Minimum: 0
-
-Timeout for CLI requests from the parent to the child process. If this timeout expires before the child responds, the master process will terminate the child process. The setting of auto_restart determines if it will get restarted automatically.
-
-.. _ref_param_clock_skew:
-
-clock_skew
-~~~~~~~~~~
-	* Units: s
-	* Default: 10
-	* Minimum: 0
-
-How much clockskew we are willing to accept between the backend and our own clock.
-
-.. _ref_param_connect_timeout:
-
-connect_timeout
-~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 3.500000
-	* Minimum: 0.000000
-
-Default connection timeout for backend connections. We only try to connect to the backend for this many seconds before giving up. VCL can override this default value for each backend and backend request.
-
-.. _ref_param_critbit_cooloff:
-
-critbit_cooloff
-~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 180.000000
-	* Minimum: 60.000000
-	* Maximum: 254.000000
-	* Flags: wizard
-
-How long time the critbit hasher keeps deleted objheads on the cooloff list.
-
-.. _ref_param_debug:
-
-debug
-~~~~~
-	* Default: none
-
-Enable/Disable various kinds of debugging.
-
-	*none*
-		Disable all debugging
-
-Use +/- prefix to set/reset individual bits:
-
-	*req_state*
-		VSL Request state engine
-
-	*workspace*
-		VSL Workspace operations
-
-	*waiter*
-		VSL Waiter internals
-
-	*waitinglist*
-		VSL Waitinglist events
-
-	*syncvsl*
-		Make VSL synchronous
-
-	*hashedge*
-		Edge cases in Hash
-
-	*vclrel*
-		Rapid VCL release
-
-	*lurker*
-		VSL Ban lurker
-
-	*esi_chop*
-		Chop ESI fetch to bits
-
-.. _ref_param_default_grace:
-
-default_grace
-~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 10.000000
-	* Minimum: 0.000000
-	* Flags: 
-
-Default grace period.  We will deliver an object this long after it has expired, provided another thread is attempting to get a new copy.
-
-.. _ref_param_default_keep:
-
-default_keep
-~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 0.000000
-	* Minimum: 0.000000
-	* Flags: 
-
-Default keep period.  We will keep a useless object around this long, making it available for conditional backend fetches.  That means that the object will be removed from the cache at the end of ttl+grace+keep.
-
-.. _ref_param_default_ttl:
-
-default_ttl
-~~~~~~~~~~~
-	* Units: seconds
-	* Default: 20.000000
-	* Minimum: 0.000000
-	* Flags: 
-
-The TTL assigned to objects if neither the backend nor the VCL code assigns one.
-
-.. _ref_param_feature:
-
-feature
-~~~~~~~
-	* Default: none
-
-Enable/Disable various minor features.
-
-	*none*
-		Disable all features.
-
-Use +/- prefix to enable/disable individual feature:
-
-	*short_panic*
-		Short panic message.
-
-	*wait_silo*
-		Wait for persistent silo.
-
-	*no_coredump*
-		No coredumps.
-
-	*esi_ignore_https*
-		Treat HTTPS as HTTP in ESI:includes
-
-	*esi_disable_xml_check*
-		Don't check of body looks like XML
-
-	*esi_ignore_other_elements*
-		Ignore non-esi XML-elements
-
-	*esi_remove_bom*
-		Remove UTF-8 BOM
-
-.. _ref_param_fetch_chunksize:
-
-fetch_chunksize
-~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 128k
-	* Minimum: 4k
-	* Flags: experimental
-
-The default chunksize used by fetcher. This should be bigger than the majority of objects with short TTLs.
-Internal limits in the storage_file module makes increases above 128kb a dubious idea.
-
-.. _ref_param_fetch_maxchunksize:
-
-fetch_maxchunksize
-~~~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 0.25G
-	* Minimum: 64k
-	* Flags: experimental
-
-The maximum chunksize we attempt to allocate from storage. Making this too large may cause delays and storage fragmentation.
-
-.. _ref_param_first_byte_timeout:
-
-first_byte_timeout
-~~~~~~~~~~~~~~~~~~
-	* Units: s
-	* Default: 60.000000
-	* Minimum: 0.000000
-
-Default timeout for receiving first byte from backend. We only wait for this many seconds for the first byte before giving up. A value of 0 means it will never time out. VCL can override this default value for each backend and backend request. This parameter does not apply to pipe.
-
-.. _ref_param_group:
-
-group
-~~~~~
-	* Default: nogroup (65533)
-	* Flags: must_restart
-
-The unprivileged group to run as.
-
-.. _ref_param_gzip_buffer:
-
-gzip_buffer
-~~~~~~~~~~~
-	* Units: bytes
-	* Default: 32k
-	* Minimum: 2k
-	* Flags: experimental
-
-Size of malloc buffer used for gzip processing.
-These buffers are used for in-transit data, for instance gunzip'ed data being sent to a client.Making this space to small results in more overhead, writes to sockets etc, making it too big is probably just a waste of memory.
-
-.. _ref_param_gzip_level:
-
-gzip_level
-~~~~~~~~~~
-	* Default: 6
-	* Minimum: 0
-	* Maximum: 9
-
-Gzip compression level: 0=debug, 1=fast, 9=best
-
-.. _ref_param_gzip_memlevel:
-
-gzip_memlevel
-~~~~~~~~~~~~~
-	* Default: 8
-	* Minimum: 1
-	* Maximum: 9
-
-Gzip memory level 1=slow/least, 9=fast/most compression.
-Memory impact is 1=1k, 2=2k, ... 9=256k.
-
-.. _ref_param_http_gzip_support:
-
-http_gzip_support
-~~~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Enable gzip support. When enabled Varnish request compressed objects from the backend and store them compressed. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to:
-  Accept-Encoding: gzip
-
-Clients that do not support gzip will have their Accept-Encoding header removed. For more information on how gzip is implemented please see the chapter on gzip in the Varnish reference.
-
-.. _ref_param_http_max_hdr:
-
-http_max_hdr
-~~~~~~~~~~~~
-	* Units: header lines
-	* Default: 64
-	* Minimum: 32
-	* Maximum: 65535
-
-Maximum number of HTTP header lines we allow in {req|resp|bereq|beresp}.http (obj.http is autosized to the exact number of headers).
-Cheap, ~20 bytes, in terms of workspace memory.
-Note that the first line occupies five header lines.
-
-.. _ref_param_http_range_support:
-
-http_range_support
-~~~~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Enable support for HTTP Range headers.
-
-.. _ref_param_http_req_hdr_len:
-
-http_req_hdr_len
-~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 8k
-	* Minimum: 40b
-
-Maximum length of any HTTP client request header we will allow.  The limit is inclusive its continuation lines.
-
-.. _ref_param_http_req_size:
-
-http_req_size
-~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 32k
-	* Minimum: 0.25k
-
-Maximum number of bytes of HTTP client request we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-The memory for the request is allocated from the client workspace (param: workspace_client) and this parameter limits how much of that the request is allowed to take up.
-
-.. _ref_param_http_resp_hdr_len:
-
-http_resp_hdr_len
-~~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 8k
-	* Minimum: 40b
-
-Maximum length of any HTTP backend response header we will allow.  The limit is inclusive its continuation lines.
-
-.. _ref_param_http_resp_size:
-
-http_resp_size
-~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 32k
-	* Minimum: 0.25k
-
-Maximum number of bytes of HTTP backend resonse we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-The memory for the request is allocated from the worker workspace (param: thread_pool_workspace) and this parameter limits how much of that the request is allowed to take up.
-
-.. _ref_param_idle_send_timeout:
-
-idle_send_timeout
-~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 60
-	* Minimum: 0
-	* Flags: delayed
-
-Time to wait with no data sent. If no data has been transmitted in this many
-seconds the session is closed.
-See setsockopt(2) under SO_SNDTIMEO for more information.
-
-.. _ref_param_listen_address:
-
-listen_address
-~~~~~~~~~~~~~~
-	* Default: :80
-	* Flags: must_restart
-
-Whitespace separated list of network endpoints where Varnish will accept requests.
-Possible formats: host, host:port, :port
-
-.. _ref_param_listen_depth:
-
-listen_depth
-~~~~~~~~~~~~
-	* Units: connections
-	* Default: 1024
-	* Minimum: 0
-	* Flags: must_restart
-
-Listen queue depth.
-
-.. _ref_param_log_local_address:
-
-log_local_address
-~~~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Log the local address on the TCP connection in the SessionOpen VSL record.
-Disabling this saves a getsockname(2) system call per TCP connection.
-
-.. _ref_param_lru_interval:
-
-lru_interval
-~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 2
-	* Minimum: 0
-	* Flags: experimental
-
-Grace period before object moves on LRU list.
-Objects are only moved to the front of the LRU list if they have not been moved there already inside this timeout period.  This reduces the amount of lock operations necessary for LRU list access.
-
-.. _ref_param_max_esi_depth:
-
-max_esi_depth
-~~~~~~~~~~~~~
-	* Units: levels
-	* Default: 5
-	* Minimum: 0
-
-Maximum depth of esi:include processing.
-
-.. _ref_param_max_restarts:
-
-max_restarts
-~~~~~~~~~~~~
-	* Units: restarts
-	* Default: 4
-	* Minimum: 0
-
-Upper limit on how many times a request can restart.
-Be aware that restarts are likely to cause a hit against the backend, so don't increase thoughtlessly.
-
-.. _ref_param_max_retries:
-
-max_retries
-~~~~~~~~~~~
-	* Units: retries
-	* Default: 4
-	* Minimum: 0
-
-Upper limit on how many times a backend fetch can retry.
-
-.. _ref_param_nuke_limit:
-
-nuke_limit
-~~~~~~~~~~
-	* Units: allocations
-	* Default: 50
-	* Minimum: 0
-	* Flags: experimental
-
-Maximum number of objects we attempt to nuke in orderto make space for a object body.
-
-.. _ref_param_pcre_match_limit:
-
-pcre_match_limit
-~~~~~~~~~~~~~~~~
-	* Default: 10000
-	* Minimum: 1
-
-The limit for the  number of internal matching function calls in a pcre_exec() execution.
-
-.. _ref_param_pcre_match_limit_recursion:
-
-pcre_match_limit_recursion
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-	* Default: 10000
-	* Minimum: 1
-
-The limit for the  number of internal matching function recursions in a pcre_exec() execution.
-
-.. _ref_param_ping_interval:
-
-ping_interval
-~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 3
-	* Minimum: 0
-	* Flags: must_restart
-
-Interval between pings from parent to child.
-Zero will disable pinging entirely, which makes it possible to attach a debugger to the child.
-
-.. _ref_param_pipe_timeout:
-
-pipe_timeout
-~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 60
-	* Minimum: 0
-
-Idle timeout for PIPE sessions. If nothing have been received in either direction for this many seconds, the session is closed.
-
-.. _ref_param_pool_req:
-
-pool_req
-~~~~~~~~
-	* Default: 10,100,10
-
-Parameters for per worker pool request memory pool.
-The three numbers are:
-
-	*min_pool*
-		minimum size of free pool.
-
-	*max_pool*
-		maximum size of free pool.
-
-	*max_age*
-		max age of free element.
-
-.. _ref_param_pool_sess:
-
-pool_sess
-~~~~~~~~~
-	* Default: 10,100,10
-
-Parameters for per worker pool session memory pool.
-The three numbers are:
-
-	*min_pool*
-		minimum size of free pool.
-
-	*max_pool*
-		maximum size of free pool.
-
-	*max_age*
-		max age of free element.
-
-.. _ref_param_pool_vbc:
-
-pool_vbc
-~~~~~~~~
-	* Default: 10,100,10
-
-Parameters for backend connection memory pool.
-The three numbers are:
-
-	*min_pool*
-		minimum size of free pool.
-
-	*max_pool*
-		maximum size of free pool.
-
-	*max_age*
-		max age of free element.
-
-.. _ref_param_pool_vbo:
-
-pool_vbo
-~~~~~~~~
-	* Default: 10,100,10
-
-Parameters for backend object fetch memory pool.
-The three numbers are:
-
-	*min_pool*
-		minimum size of free pool.
-
-	*max_pool*
-		maximum size of free pool.
-
-	*max_age*
-		max age of free element.
-
-.. _ref_param_prefer_ipv6:
-
-prefer_ipv6
-~~~~~~~~~~~
-	* Units: bool
-	* Default: off
-
-Prefer IPv6 address when connecting to backends which have both IPv4 and IPv6 addresses.
-
-.. _ref_param_rush_exponent:
-
-rush_exponent
-~~~~~~~~~~~~~
-	* Units: requests per request
-	* Default: 3
-	* Minimum: 2
-	* Flags: experimental
-
-How many parked request we start for each completed request on the object.
-NB: Even with the implict delay of delivery, this parameter controls an exponential increase in number of worker threads.
-
-.. _ref_param_send_timeout:
-
-send_timeout
-~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 600
-	* Minimum: 0
-	* Flags: delayed
-
-Send timeout for client connections. If the HTTP response hasn't been transmitted in this many
-seconds the session is closed.
-See setsockopt(2) under SO_SNDTIMEO for more information.
-
-.. _ref_param_session_max:
-
-session_max
-~~~~~~~~~~~
-	* Units: sessions
-	* Default: 100000
-	* Minimum: 1000
-
-Maximum number of sessions we will allocate from one pool before just dropping connections.
-This is mostly an anti-DoS measure, and setting it plenty high should not hurt, as long as you have the memory for it.
-
-.. _ref_param_shm_reclen:
-
-shm_reclen
-~~~~~~~~~~
-	* Units: bytes
-	* Default: 255b
-	* Minimum: 16b
-	* Maximum: 65535b
-
-Maximum number of bytes in SHM log record.
-Maximum is 65535 bytes.
-
-.. _ref_param_shortlived:
-
-shortlived
-~~~~~~~~~~
-	* Units: s
-	* Default: 10.000000
-	* Minimum: 0.000000
-
-Objects created with TTL shorter than this are always put in transient storage.
-
-.. _ref_param_sigsegv_handler:
-
-sigsegv_handler
-~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: off
-	* Flags: must_restart
-
-Install a signal handler which tries to dump debug information on segmentation faults.
-
-.. _ref_param_syslog_cli_traffic:
-
-syslog_cli_traffic
-~~~~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Log all CLI traffic to syslog(LOG_INFO).
-
-.. _ref_param_tcp_keepalive_intvl:
-
-tcp_keepalive_intvl
-~~~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 5
-	* Minimum: 1
-	* Maximum: 100
-	* Flags: experimental
-
-The number of seconds between TCP keep-alive probes. Note that this setting will only take effect when it is less thanthe system default.
-
-.. _ref_param_tcp_keepalive_probes:
-
-tcp_keepalive_probes
-~~~~~~~~~~~~~~~~~~~~
-	* Units: probes
-	* Default: 5
-	* Minimum: 1
-	* Maximum: 100
-	* Flags: experimental
-
-The maximum number of TCP keep-alive probes to send before giving up and killing the connection if no response is obtained from the other end. Note that this setting will only take effect when it is less than the system default.
-
-.. _ref_param_tcp_keepalive_time:
-
-tcp_keepalive_time
-~~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 600
-	* Minimum: 1
-	* Maximum: 7200
-	* Flags: experimental
-
-The number of seconds a connection needs to be idle before TCP begins sending out keep-alive probes. Note that this setting will only take effect when it is less than the system default.
-
-.. _ref_param_thread_pool_add_delay:
-
-thread_pool_add_delay
-~~~~~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 0.000000
-	* Minimum: 0.000000
-	* Flags: experimental
-
-Wait at least this long after creating a thread.
-
-Some (buggy) systems may need a short (sub-second) delay between creating threads.
-Set this to a few milliseconds if you see the 'threads_failed' counter grow too much.
-
-Setting this too high results in insuffient worker threads.
-
-.. _ref_param_thread_pool_destroy_delay:
-
-thread_pool_destroy_delay
-~~~~~~~~~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 1.000000
-	* Minimum: 0.010000
-	* Flags: delayed, experimental
-
-Wait this long after destroying a thread.
-
-This controls the decay of thread pools when idle(-ish).
-
-Minimum is 0.01 second.
-
-.. _ref_param_thread_pool_fail_delay:
-
-thread_pool_fail_delay
-~~~~~~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 0.200000
-	* Minimum: 0.010000
-	* Flags: experimental
-
-Wait at least this long after a failed thread creation before trying to create another thread.
-
-Failure to create a worker thread is often a sign that  the end is near, because the process is running out of some resource.  This delay tries to not rush the end on needlessly.
-
-If thread creation failures are a problem, check that thread_pool_max is not too high.
-
-It may also help to increase thread_pool_timeout and thread_pool_min, to reduce the rate at which treads are destroyed and later recreated.
-
-.. _ref_param_thread_pool_max:
-
-thread_pool_max
-~~~~~~~~~~~~~~~
-	* Units: threads
-	* Default: 5000
-	* Minimum: 10
-	* Flags: delayed
-
-The maximum number of worker threads in each pool.
-
-Do not set this higher than you have to, since excess worker threads soak up RAM and CPU and generally just get in the way of getting work done.
-
-Minimum is 10 threads.
-
-.. _ref_param_thread_pool_min:
-
-thread_pool_min
-~~~~~~~~~~~~~~~
-	* Units: threads
-	* Default: 100
-	* Minimum: 10
-	* Flags: delayed
-
-The minimum number of worker threads in each pool.
-
-Increasing this may help ramp up faster from low load situations or when threads have expired.
-
-Minimum is 10 threads.
-
-.. _ref_param_thread_pool_stack:
-
-thread_pool_stack
-~~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 48k
-	* Minimum: 2k
-	* Flags: experimental
-
-Worker thread stack size.
-This is likely rounded up to a multiple of 4k by the kernel.
-The kernel/OS has a lower limit which will be enforced.
-
-.. _ref_param_thread_pool_timeout:
-
-thread_pool_timeout
-~~~~~~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 300.000000
-	* Minimum: 10.000000
-	* Flags: delayed, experimental
-
-Thread idle threshold.
-
-Threads in excess of thread_pool_min, which have been idle for at least this long, will be destroyed.
-
-Minimum is 10 seconds.
-
-.. _ref_param_thread_pools:
-
-thread_pools
-~~~~~~~~~~~~
-	* Units: pools
-	* Default: 2
-	* Minimum: 1
-	* Flags: delayed, experimental
-
-Number of worker thread pools.
-
-Increasing number of worker pools decreases lock contention.
-
-Too many pools waste CPU and RAM resources, and more than one pool for each CPU is probably detrimal to performance.
-
-Can be increased on the fly, but decreases require a restart to take effect.
-
-.. _ref_param_thread_queue_limit:
-
-thread_queue_limit
-~~~~~~~~~~~~~~~~~~
-	* Default: 20
-	* Minimum: 0
-	* Flags: experimental
-
-Permitted queue length per thread-pool.
-
-This sets the number of requests we will queue, waiting for an available thread.  Above this limit sessions will be dropped instead of queued.
-
-.. _ref_param_thread_stats_rate:
-
-thread_stats_rate
-~~~~~~~~~~~~~~~~~
-	* Units: requests
-	* Default: 10
-	* Minimum: 0
-	* Flags: experimental
-
-Worker threads accumulate statistics, and dump these into the global stats counters if the lock is free when they finish a request.
-This parameters defines the maximum number of requests a worker thread may handle, before it is forced to dump its accumulated stats into the global counters.
-
-.. _ref_param_timeout_idle:
-
-timeout_idle
-~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 5.000000
-	* Minimum: 0.000000
-
-Idle timeout for client connections.
-A connection is considered idle, until we receive a non-white-space character on it.
-
-.. _ref_param_timeout_linger:
-
-timeout_linger
-~~~~~~~~~~~~~~
-	* Units: seconds
-	* Default: 0.050000
-	* Minimum: 0.000000
-	* Flags: experimental
-
-How long time the workerthread lingers on an idle session before handing it over to the waiter.
-When sessions are reused, as much as half of all reuses happen within the first 100 msec of the previous request completing.
-Setting this too high results in worker threads not doing anything for their keep, setting it too low just means that more sessions take a detour around the waiter.
-
-.. _ref_param_timeout_req:
-
-timeout_req
-~~~~~~~~~~~
-	* Units: seconds
-	* Default: 2.000000
-	* Minimum: 0.000000
-
-Max time to receive clients request header, measured from first non-white-space character to double CRNL.
-
-.. _ref_param_user:
-
-user
-~~~~
-	* Default: nobody (65534)
-	* Flags: must_restart
-
-The unprivileged user to run as.
-
-.. _ref_param_vcc_allow_inline_c:
-
-vcc_allow_inline_c
-~~~~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: off
-
-Allow inline C code in VCL.
-
-.. _ref_param_vcc_err_unref:
-
-vcc_err_unref
-~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Unreferenced VCL objects result in error.
-
-.. _ref_param_vcc_unsafe_path:
-
-vcc_unsafe_path
-~~~~~~~~~~~~~~~
-	* Units: bool
-	* Default: on
-
-Allow '/' in vmod & include paths.
-Allow 'import ... from ...'.
-
-.. _ref_param_vcl_dir:
-
-vcl_dir
-~~~~~~~
-	* Default: /opt/varnish/etc/varnish
-
-Directory from which relative VCL filenames (vcl.load and include) are opened.
-
-.. _ref_param_vmod_dir:
-
-vmod_dir
-~~~~~~~~
-	* Default: /opt/varnish/lib/varnish/vmods
-
-Directory where VCL modules are to be found.
-
-.. _ref_param_vsl_buffer:
-
-vsl_buffer
-~~~~~~~~~~
-	* Units: bytes
-	* Default: 4k
-	* Minimum: 1k
-
-Bytes of (req-/backend-)workspace dedicated to buffering VSL records.
-At a bare minimum, this must be longer than the longest HTTP header to be logged.
-Setting this too high costs memory, setting it too low will cause more VSL flushes and likely increase lock-contention on the VSL mutex.
-Minimum is 1k bytes.
-
-.. _ref_param_vsl_mask:
-
-vsl_mask
-~~~~~~~~
-	* Default: -VCL_trace,-WorkThread,-Hash
-
-Mask individual VSL messages from being logged.
-
-	*default*
-		Set default value
-
-Use +/- prefixe in front of VSL tag name, to mask/unmask individual VSL messages.
-
-.. _ref_param_vsl_space:
-
-vsl_space
-~~~~~~~~~
-	* Units: bytes
-	* Default: 80M
-	* Minimum: 1M
-	* Flags: must_restart
-
-The amount of space to allocate for the VSL fifo buffer in the VSM memory segment.  If you make this too small, varnish{ncsa|log} etc will not be able to keep up.  Making it too large just costs memory resources.
-
-.. _ref_param_vsm_space:
-
-vsm_space
-~~~~~~~~~
-	* Units: bytes
-	* Default: 1M
-	* Minimum: 1M
-	* Flags: must_restart
-
-The amount of space to allocate for stats counters in the VSM memory segment.  If you make this too small, some counters will be invisible.  Making it too large just costs memory resources.
-
-.. _ref_param_waiter:
-
-waiter
-~~~~~~
-	* Default: kqueue (possible values: kqueue, poll)
-	* Flags: must_restart, wizard
-
-Select the waiter kernel interface.
-
-.. _ref_param_workspace_backend:
-
-workspace_backend
-~~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 64k
-	* Minimum: 1k
-	* Flags: delayed
-
-Bytes of HTTP protocol workspace for backend HTTP req/resp.  If larger than 4k, use a multiple of 4k for VM efficiency.
-
-.. _ref_param_workspace_client:
-
-workspace_client
-~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 64k
-	* Minimum: 3k
-	* Flags: delayed
-
-Bytes of HTTP protocol workspace for clients HTTP req/resp.  If larger than 4k, use a multiple of 4k for VM efficiency.
-
-.. _ref_param_workspace_thread:
-
-workspace_thread
-~~~~~~~~~~~~~~~~
-	* Units: bytes
-	* Default: 2k
-	* Minimum: 0.25k
-	* Maximum: 8k
-	* Flags: delayed
-
-Bytes of auxillary workspace per thread.
-This workspace is used for certain temporary data structures during the operation of a worker thread.
-One use is for the io-vectors for writing requests and responses to sockets, having too little space will result in more writev(2) system calls, having too much just wastes the space.
-



From tfheen at err.no  Thu Mar 13 09:24:23 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 2abb545 Build man from man/ and add params.rst to BUILT_OBJECTS
Message-ID: <E1WO1sF-00077o-MB@project.varnish-software.com>


commit 2abb545829d9a494360b020cd5efcf5cb2a0a07c
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Wed Feb 5 13:14:16 2014 +0100

    Build man from man/ and add params.rst to BUILT_OBJECTS

diff --git a/Makefile.am b/Makefile.am
index 66c9e03..2eb3af4 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS = include lib bin man etc doc
+SUBDIRS = include lib bin etc doc man
 
 SUBDIRS += redhat
 
diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index e0a8461..8cd8cf5 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -8,8 +8,6 @@ AM_CPPFLAGS = \
 
 sbin_PROGRAMS = varnishd
 
-dist_man_MANS = varnishd.1
-
 varnishd_SOURCES = \
 	cache/cache_acceptor.c \
 	cache/cache_backend.c \
@@ -146,13 +144,3 @@ builtin_vcl.h:	builtin.vcl
 
 # Explicitly record dependency
 mgt/mgt_vcc.c:	builtin_vcl.h
-
-varnishd.1: $(top_srcdir)/doc/sphinx/reference/varnishd.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 692cf6f..df3d3c4 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -176,5 +176,8 @@ dist-hook:
 distclean-local:
 	rm -rf $(BUILDDIR)
 
-reference/params.rst:
+reference/params.rst: $(top_builddir)/bin/varnishd/varnishd
+	mkdir -p reference
 	$(top_builddir)/bin/varnishd/varnishd -x dumprstparam > reference/params.rst
+
+BUILT_SOURCES = reference/params.rst
diff --git a/man/Makefile.am b/man/Makefile.am
index 1ed3a5d..7d0b1f6 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -8,7 +8,7 @@ vsc2rst_SOURCES = vsc2rst.c \
 
 AM_CPPFLAGS = -I$(top_srcdir)/include
 
-dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7 vsl.7 vsl-query.7
+dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7 vsl.7 vsl-query.7 varnishd.1
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
@@ -63,3 +63,13 @@ else
 	@echo "========================================"
 	@false
 endif
+
+varnishd.1: $(top_srcdir)/doc/sphinx/reference/varnishd.rst
+if HAVE_RST2MAN
+	${RST2MAN} $^ $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif



From tfheen at err.no  Thu Mar 13 09:24:23 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] c191685 Ensure source tree is built before running make dist
Message-ID: <E1WO1sF-00077x-NR@project.varnish-software.com>


commit c191685566820de1ce5da2bcf49a5bc3d7537e5c
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Wed Feb 5 13:50:27 2014 +0100

    Ensure source tree is built before running make dist

diff --git a/Makefile.am b/Makefile.am
index 2eb3af4..2aa609f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -36,6 +36,11 @@ distcleancheck_listfiles = \
 	find . -type f -exec sh -c 'test -f $(srcdir)/$$1 || echo $$1' \
 		sh '{}' ';'
 
+# XXX: This is a hack to ensure we have a built source tree when
+# running make dist If we had used non-recursive make we could have
+# solved it better, but we don't, so use this at least for now.
+LICENSE: all
+
 cscope:
 	-rm -f cscope*
 	find . -name '*.[hcS]' > cscope.files



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 090e84c Fix an oversight which disabled TTL calculation on IMS
 fetches.
Message-ID: <E1WO1sF-000789-PM@project.varnish-software.com>


commit 090e84c5a28397ad7e4b2ac6fbfdbb58bc712882
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 5 14:27:37 2014 +0000

    Fix an oversight which disabled TTL calculation on IMS fetches.
    
    Also make sure that objects marked uncacheable on an IMS is treated
    as such.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9bbc6c1..fa58309 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -271,7 +271,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->do_esi)
 		bo->do_stream = 0;
-	if (bo->do_pass)
+	if (bo->do_pass || bo->uncacheable)
 		bo->fetch_objcore->flags |= OC_F_PASS;
 
 	assert(wrk->handling == VCL_RET_DELIVER);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index dcf4c55..5334291 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -117,6 +117,7 @@ RFC2616_Ttl(struct busyobj *bo)
 	case 300: /* Multiple Choices */
 	case 301: /* Moved Permanently */
 	case 302: /* Moved Temporarily */
+	case 304: /* Not Modified */
 	case 307: /* Temporary Redirect */
 	case 410: /* Gone */
 	case 404: /* Not Found */
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 5d875ed..2b1629a 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -181,6 +181,7 @@ VRT_l_beresp_uncacheable(const struct vrt_ctx *ctx, unsigned a)
 		VSLb(ctx->vsl, SLT_VCL_Error,
 		    "Ignoring attempt to reset beresp.uncacheable");
 	} else if (a) {
+		VSLb(ctx->vsl, SLT_Debug, "XXXX: UNCACHEABLE %u" , a);
 		ctx->bo->uncacheable = a;
 	}
 }



From tfheen at err.no  Thu Mar 13 09:24:23 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] b5929ad Move doc generation into doc/sphinx and man
Message-ID: <E1WO1sF-00078J-Qy@project.varnish-software.com>


commit b5929ad76db05b36876c91f990e2d043106c72b4
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Wed Feb 5 15:34:25 2014 +0100

    Move doc generation into doc/sphinx and man

diff --git a/.gitignore b/.gitignore
index 78cdba5..35dfbe7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -76,32 +76,19 @@ cscope.*out
 
 # Man-files and binaries
 /man/vsc2rst
-/man/vcl.7
-/man/vsl.7
-/man/vsl-query.7
-/man/varnish-cli.7
-/man/varnish-counters.7
+/man/*.7
+/man/*.1
+/doc/sphinx/include
 /bin/varnish*/varnish*_opt2rst
 /bin/varnishadm/varnishadm
-/bin/varnishadm/varnishadm.1
 /bin/varnishd/varnishd
-/bin/varnishd/varnishd.1
 /bin/varnishhist/varnishhist
-/bin/varnishhist/varnishhist.1
 /bin/varnishlog/varnishlog
-/bin/varnishlog/varnishlog.1
-/bin/varnishlog/varnishlog_*.rst
 /bin/varnishncsa/varnishncsa
-/bin/varnishncsa/varnishncsa.1
-/bin/varnishncsa/varnishncsa_*.rst
 /bin/varnishreplay/varnishreplay
-/bin/varnishreplay/varnishreplay.1
 /bin/varnishstat/varnishstat
-/bin/varnishstat/varnishstat.1
 /bin/varnishtest/varnishtest
-/bin/varnishtest/varnishtest.1
 /bin/varnishtop/varnishtop
-/bin/varnishtop/varnishtop.1
 
 # Doc-stuff generated from xml
 /doc/*.html
diff --git a/Makefile.am b/Makefile.am
index 2aa609f..389be99 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,8 +1,6 @@
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS = include lib bin etc doc man
-
-SUBDIRS += redhat
+SUBDIRS = include lib bin etc doc redhat
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = varnishapi.pc
@@ -22,6 +20,16 @@ DISTCHECK_CONFIGURE_FLAGS = \
 install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 
+if HAVE_RST2MAN
+SUBDIRS += man
+else
+dist-hook:
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif
+
 distcheck-hook:
 	V="@PACKAGE_VERSION@" ; \
 	V="$${V%%-*}" ; \
diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index 8fb28d7..f3050eb 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishadm
 
-dist_man_MANS = varnishadm.1
-
 varnishadm_SOURCES = \
 	varnishadm.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
@@ -19,13 +17,3 @@ varnishadm_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	${PTHREAD_LIBS} ${RT_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM}
-
-varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 5423155..9511a6e 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishhist
 
-dist_man_MANS = varnishhist.1
-
 varnishhist_SOURCES = varnishhist.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/version.c
@@ -15,13 +13,3 @@ varnishhist_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	-lm \
 	@CURSES_LIB@ ${RT_LIBS} ${PTHREAD_LIBS}
-
-varnishhist.1: $(top_srcdir)/doc/sphinx/reference/varnishhist.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index a757b61..1e6fa79 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishlog
 
-dist_man_MANS = varnishlog.1
-
 varnishlog_SOURCES = \
 	varnishlog.c \
 	varnishlog_options.h \
@@ -28,29 +26,3 @@ varnishlog_opt2rst_SOURCES = \
 	varnishlog_options.h \
 	varnishlog_options.c \
 	$(top_srcdir)/lib/libvarnishtools/opt2rst.c
-
-BUILT_SOURCES = varnishlog_options.rst varnishlog_synopsis.rst
-EXTRA_DIST = $(BUILT_SOURCES)
-MAINTAINERCLEANFILES = $(EXTRA_DIST)
-
-varnishlog_options.rst:
-	./varnishlog_opt2rst options > $@
-varnishlog_synopsis.rst:
-	./varnishlog_opt2rst synopsis > $@
-
-if HAVE_RST2MAN
-varnishlog_options.rst varnishlog_synopsis.rst: varnishlog_opt2rst
-endif
-
-varnishlog.1: \
-	$(top_srcdir)/doc/sphinx/reference/varnishlog.rst \
-	varnishlog_options.rst \
-	varnishlog_synopsis.rst
-if HAVE_RST2MAN
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am
index 6f3a7f9..12a3f4d 100644
--- a/bin/varnishncsa/Makefile.am
+++ b/bin/varnishncsa/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishncsa
 
-dist_man_MANS = varnishncsa.1
-
 varnishncsa_SOURCES = \
 	varnishncsa.c \
 	varnishncsa_options.h \
@@ -30,29 +28,3 @@ varnishncsa_opt2rst_SOURCES = \
 	varnishncsa_options.h \
 	varnishncsa_options.c \
 	$(top_srcdir)/lib/libvarnishtools/opt2rst.c
-
-BUILT_SOURCES = varnishncsa_options.rst varnishncsa_synopsis.rst
-EXTRA_DIST = $(BUILT_SOURCES)
-MAINTAINERCLEANFILES = $(EXTRA_DIST)
-
-varnishncsa_options.rst:
-	./varnishncsa_opt2rst options > $@
-varnishncsa_synopsis.rst:
-	./varnishncsa_opt2rst synopsis > $@
-
-if HAVE_RST2MAN
-varnishncsa_options.rst varnishncsa_synopsis.rst: varnishncsa_opt2rst
-endif
-
-varnishncsa.1: \
-	$(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \
-	varnishncsa_options.rst \
-	varnishncsa_synopsis.rst
-if HAVE_RST2MAN
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index 59c6a0d..72a293d 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishreplay
 
-dist_man_MANS = varnishreplay.1
-
 varnishreplay_SOURCES = \
 	varnishreplay.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
@@ -16,13 +14,3 @@ varnishreplay_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	${RT_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM}
-
-varnishreplay.1: $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index d22edac..fe17250 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishstat
 
-dist_man_MANS = varnishstat.1
-
 varnishstat_SOURCES = \
 	varnishstat.h \
 	\
@@ -19,13 +17,3 @@ varnishstat_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	@CURSES_LIB@ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
-
-varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index b51d822..5af8068 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -19,8 +19,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/lib/libvgz
 
 bin_PROGRAMS =	varnishtest
 
-dist_man_MANS = varnishtest.1
-
 varnishtest_SOURCES = \
 		vtc.c \
 		vtc.h \
@@ -47,13 +45,3 @@ varnishtest_CFLAGS = \
 
 EXTRA_DIST = $(top_srcdir)/bin/varnishtest/tests/*.vtc \
 	$(top_srcdir)/bin/varnishtest/tests/README
-
-varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index a38e293..b919dba 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -4,8 +4,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 
 bin_PROGRAMS = varnishtop
 
-dist_man_MANS = varnishtop.1
-
 varnishtop_SOURCES = varnishtop.c \
 	varnishtop_options.h \
 	varnishtop_options.c \
@@ -28,29 +26,3 @@ varnishtop_opt2rst_SOURCES = \
 	varnishtop_options.h \
 	varnishtop_options.c \
 	$(top_srcdir)/lib/libvarnishtools/opt2rst.c
-
-BUILT_SOURCES = varnishtop_options.rst varnishtop_synopsis.rst
-EXTRA_DIST = $(BUILT_SOURCES)
-MAINTAINERCLEANFILES = $(EXTRA_DIST)
-
-varnishtop_options.rst:
-	./varnishtop_opt2rst options > $@
-varnishtop_synopsis.rst:
-	./varnishtop_opt2rst synopsis > $@
-
-if HAVE_RST2MAN
-varnishtop_options.rst varnishtop_synopsis.rst: varnishtop_opt2rst
-endif
-
-varnishtop.1: \
-	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
-	varnishtop_options.rst \
-	varnishtop_synopsis.rst
-if HAVE_RST2MAN
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/configure.ac b/configure.ac
index 38634b9..f25b2cf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -554,6 +554,9 @@ fi
 VTC_TESTS="$(cd $srcdir/bin/varnishtest && echo tests/*.vtc)"
 AC_SUBST(VTC_TESTS)
 
+# Make sure this include dir exists
+AC_CONFIG_COMMANDS([mkdir], [$MKDIR_P doc/sphinx/include])
+
 # Generate output
 AC_CONFIG_FILES([
     Makefile
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index df3d3c4..7940d07 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -30,7 +30,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html: reference/params.rst
+html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -112,7 +112,6 @@ EXTRA_DIST = \
 	phk/vcl_expr.rst \
 	phk/wanton_destruction.rst \
 	reference/index.rst \
-	reference/params.rst \
 	reference/varnishadm.rst \
 	reference/varnish-cli.rst \
 	reference/varnishd.rst \
@@ -168,7 +167,6 @@ EXTRA_DIST = \
 	users-guide/vcl-variables.rst \
 	users-guide/websockets.rst
 
-
 dist-hook:
 	$(MAKE) html
 	cp -r $(BUILDDIR) $(distdir)
@@ -176,8 +174,30 @@ dist-hook:
 distclean-local:
 	rm -rf $(BUILDDIR)
 
-reference/params.rst: $(top_builddir)/bin/varnishd/varnishd
-	mkdir -p reference
-	$(top_builddir)/bin/varnishd/varnishd -x dumprstparam > reference/params.rst
-
-BUILT_SOURCES = reference/params.rst
+include/params.rst: $(top_builddir)/bin/varnishd/varnishd
+	$(top_builddir)/bin/varnishd/varnishd -x dumprstparam > include/params.rst
+BUILT_SOURCES = include/params.rst
+
+include/varnishncsa_options.rst: $(top_builddir)/bin/varnishncsa/varnishncsa_opt2rst
+	$(top_builddir)/bin/varnishncsa/varnishncsa_opt2rst options > $@
+include/varnishncsa_synopsis.rst: $(top_builddir)/bin/varnishncsa/varnishncsa_opt2rst
+	$(top_builddir)/bin/varnishncsa/varnishncsa_opt2rst synopsis > $@
+BUILT_SOURCES += include/varnishncsa_options.rst \
+	include/varnishncsa_synopsis.rst
+
+include/varnishlog_options.rst: $(top_builddir)/bin/varnishlog/varnishlog_opt2rst
+	$(top_builddir)/bin/varnishlog/varnishlog_opt2rst options > $@
+include/varnishlog_synopsis.rst: $(top_builddir)/bin/varnishlog/varnishlog_opt2rst
+	$(top_builddir)/bin/varnishlog/varnishlog_opt2rst synopsis > $@
+BUILT_SOURCES += include/varnishlog_options.rst \
+	include/varnishlog_synopsis.rst
+
+include/varnishtop_options.rst: $(top_builddir)/bin/varnishtop/varnishtop_opt2rst
+	$(top_builddir)/bin/varnishtop/varnishtop_opt2rst options > $@
+include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop_opt2rst
+	$(top_builddir)/bin/varnishtop/varnishtop_opt2rst synopsis > $@
+BUILT_SOURCES += include/varnishtop_options.rst \
+	 include/varnishtop_synopsis.rst
+
+EXTRA_DIST += $(BUILT_SOURCES)
+MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 48fd44c..f5599c5 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -230,7 +230,7 @@ workspace_client (=16k), thread_pool_workspace (=16k), http_resp_size
 thread_pool_stack (=64k) are reduced relative to the values listed
 here, in order to conserve VM space.
 
-.. include:: params.rst
+.. include:: ../include/params.rst
 
 SEE ALSO
 ========
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 9d93fe1..60ebc57 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -19,7 +19,7 @@ Display Varnish logs
 SYNOPSIS
 ========
 
-.. include:: ../../../bin/varnishlog/varnishlog_synopsis.rst
+.. include:: ../include/varnishlog_synopsis.rst
 varnishlog |synopsis| <query expression>
 
 OPTIONS
@@ -27,7 +27,7 @@ OPTIONS
 
 The following options are available:
 
-.. include:: ../../../bin/varnishlog/varnishlog_options.rst
+.. include:: ../include/varnishlog_options.rst
 
 -k num
 
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 50dcfe6..8189696 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -16,7 +16,7 @@ Display Varnish logs in Apache / NCSA combined log format
 SYNOPSIS
 ========
 
-.. include:: ../../../bin/varnishncsa/varnishncsa_synopsis.rst
+.. include:: ../include/varnishncsa_synopsis.rst
 varnishncsa |synopsis|
 
 DESCRIPTION
@@ -33,7 +33,7 @@ applicable transactions. Non-request transactions are ignored.
 
 The following options are available:
 
-.. include:: ../../../bin/varnishncsa/varnishncsa_options.rst
+.. include:: ../include/varnishncsa_options.rst
 
 FORMAT
 ======
diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 23298d5..7ebc384 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -17,7 +17,7 @@ Varnish log entry ranking
 SYNOPSIS
 ========
 
-.. include:: ../../../bin/varnishtop/varnishtop_synopsis.rst
+.. include:: ../include/varnishtop_synopsis.rst
 varnishtop |synopsis|
 
 DESCRIPTION
@@ -32,7 +32,7 @@ recorded in the log.
 
 The following options are available:
 
-.. include:: ../../../bin/varnishtop/varnishtop_options.rst
+.. include:: ../include/varnishtop_options.rst
 
 EXAMPLES
 ========
diff --git a/man/Makefile.am b/man/Makefile.am
index 7d0b1f6..2b26b0b 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -8,68 +8,59 @@ vsc2rst_SOURCES = vsc2rst.c \
 
 AM_CPPFLAGS = -I$(top_srcdir)/include
 
-dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7 vsl.7 vsl-query.7 varnishd.1
+dist_man_MANS = vcl.7 \
+	varnish-cli.7 \
+	varnish-counters.7 \
+	vsl.7 \
+	vsl-query.7 \
+	varnishadm.1 \
+	varnishd.1 \
+	varnishlog.1 \
+	varnishncsa.1 \
+	varnishreplay.1 \
+	varnishstat.1 \
+	varnishtest.1 \
+	varnishtop.1
+
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
 	$(top_srcdir)/bin/varnishd/builtin.vcl
-if HAVE_RST2MAN
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
 
 varnish-cli.7: $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst
-if HAVE_RST2MAN
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
 
-if HAVE_RST2MAN
 varnish-counters.7: vsc2rst
 	./vsc2rst | ${RST2MAN} - $@
-else
-varnish-counters.7: $(top_srcdir)/include/tbl/vsc_fields.h
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
 
 vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \
 	$(top_srcdir)/lib/libvarnishapi/vsl-tags.rst
-if HAVE_RST2MAN
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
 
 vsl-query.7: $(top_srcdir)/doc/sphinx/reference/vsl-query.rst
-if HAVE_RST2MAN
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
 
-varnishd.1: $(top_srcdir)/doc/sphinx/reference/varnishd.rst
-if HAVE_RST2MAN
-	${RST2MAN} $^ $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
+%.7: $(top_srcdir)/doc/sphinx/reference/%.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
+
+%.1: $(top_srcdir)/doc/sphinx/reference/%.rst
+	${RST2MAN} $< $@
+
+varnishncsa.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishncsa_options.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishncsa_synopsis.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
+
+varnishlog.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishlog.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishlog_options.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishlog_synopsis.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
+
+varnishtop.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishtop_options.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishtop_synopsis.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@



From martin at varnish-software.com  Thu Mar 13 09:24:23 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 862c6a1 Don't output trailing '\0' of SLT_Debug records
Message-ID: <E1WO1sF-00078T-SZ@project.varnish-software.com>


commit 862c6a174ac335a5aba048d9e4af733ab9c64f2f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Feb 5 15:40:22 2014 +0100

    Don't output trailing '\0' of SLT_Debug records
    
    SLT_Debug is special as it is both used as a binary log record and as
    a text record. Add a special case for it and don't escape-output the
    trailing zero.

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 363085a..6e0f20a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -255,6 +255,8 @@ VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	if (VSL_tagflags[tag] & SLT_F_BINARY) {
 		VSL_PRINT(fo, "%10u %-14s %c \"", vxid, VSL_tags[tag], type);
 		while (len-- > 0) {
+			if (len == 0 && tag == SLT_Debug && *data == '\0')
+				break;
 			if (*data >= ' ' && *data <= '~')
 				VSL_PRINT(fo, "%c", *data);
 			else
@@ -288,6 +290,8 @@ VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	if (VSL_tagflags[tag] & SLT_F_BINARY) {
 		VSL_PRINT(fo, "%-14s \"", VSL_tags[tag]);
 		while (len-- > 0) {
+			if (len == 0 && tag == SLT_Debug && *data == '\0')
+				break;
 			if (*data >= ' ' && *data <= '~')
 				VSL_PRINT(fo, "%c", *data);
 			else



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] be429e5 Make the CHANGE column show negative numbers when
 gauges drop
Message-ID: <E1WO1sF-00078g-U9@project.varnish-software.com>


commit be429e5e3b421d9a07be4bfa8025951841dea9c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 5 19:13:59 2014 +0000

    Make the CHANGE column show negative numbers when gauges drop

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 2b0137d..e722408 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -380,7 +380,7 @@ sample_points(void)
 		pt->t_cur = VTIM_mono();
 
 		if (pt->t_last)
-			pt->chg = (pt->cur - (intmax_t)pt->last) /
+			pt->chg = ((intmax_t)pt->cur - (intmax_t)pt->last) /
 			    (pt->t_cur - pt->t_last);
 
 		if (pt->flag == 'g') {



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] 760db1b Dump the objects on busyobj on panic in fetch code
Message-ID: <E1WO1sF-00078n-V5@project.varnish-software.com>


commit 760db1bd31e95f4573fc54583c51f84ec6add533
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 5 21:05:04 2014 +0000

    Dump the objects on busyobj on panic in fetch code

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 81b794d..8df8bfa 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -210,11 +210,11 @@ pan_http(const char *id, const struct http *h, int indent)
 /*--------------------------------------------------------------------*/
 
 static void
-pan_object(const struct object *o)
+pan_object(const char *typ, const struct object *o)
 {
 	const struct storage *st;
 
-	VSB_printf(pan_vsp, "  obj = %p {\n", o);
+	VSB_printf(pan_vsp, "  obj (%s) = %p {\n", typ, o);
 	VSB_printf(pan_vsp, "    vxid = %u,\n", o->vxid);
 	pan_http("obj", o->http, 4);
 	VSB_printf(pan_vsp, "    len = %jd,\n", (intmax_t)o->len);
@@ -284,6 +284,10 @@ pan_busyobj(const struct busyobj *bo)
 	if (bo->beresp->ws != NULL)
 		pan_http("beresp", bo->beresp, 4);
 	pan_ws(bo->ws_o, 4);
+	if (bo->fetch_obj)
+		pan_object("FETCH", bo->fetch_obj);
+	if (bo->ims_obj)
+		pan_object("IMS", bo->ims_obj);
 	VSB_printf(pan_vsp, "  }\n");
 }
 
@@ -337,7 +341,7 @@ pan_req(const struct req *req)
 	if (VALID_OBJ(req->obj, OBJECT_MAGIC)) {
 		if (req->obj->objcore->busyobj != NULL)
 			pan_busyobj(req->obj->objcore->busyobj);
-		pan_object(req->obj);
+		pan_object("REQ", req->obj);
 	}
 
 	VSB_printf(pan_vsp, "},\n");



From phk at FreeBSD.org  Thu Mar 13 09:24:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:23 +0100
Subject: [4.0] aeebcb5 Only grab the extra reference if we have a request
 thread that wants it.
Message-ID: <E1WO1sF-00078w-WA@project.varnish-software.com>


commit aeebcb565a4dfe4e2c3f720e5428b7b78bd0686a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 5 21:13:54 2014 +0000

    Only grab the extra reference if we have a request thread that wants it.
    
    Zeroed in on by:	scoof

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index fa58309..c6514fd 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -780,9 +780,11 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 	bo->vary = req->vary_b;
 	req->vary_b = NULL;
 
-	HSH_Ref(oc);
+	if (mode != VBF_BACKGROUND)
+		HSH_Ref(oc);
 	bo->fetch_objcore = oc;
 
+	AZ(bo->ims_obj);
 	if (oldobj != NULL) {
 		if (http_GetHdr(oldobj->http, H_Last_Modified, NULL) ||
 		   http_GetHdr(oldobj->http, H_ETag, NULL)) {



From fgsch at lodoss.net  Thu Mar 13 09:24:24 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 9878c6c Spelling
Message-ID: <E1WO1sG-000795-1C@project.varnish-software.com>


commit 9878c6c2c4a45984e307748aa5b15872c514c19e
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Feb 5 23:50:22 2014 +0000

    Spelling

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 8d8f14f..01a6243 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -257,7 +257,7 @@ The client's IP address.
 		'BACKEND',
 		( 'recv',),
 		( 'recv',), """
-		Set bereq.backend to this if we attemt to fetch.
+		Set bereq.backend to this if we attempt to fetch.
 		This variable is a convenience so the overall
 		policy can be set up once and for all in vcl_recv{}.
 		"""
@@ -1160,7 +1160,7 @@ for i in l:
 	fo.write("\n" + i[0] + "\n\n")
 	fo.write("\tType: " + i[1] + "\n\n")
 	rst_where(fo, "Readable from: ", i[2])
-	rst_where(fo, "Writeable frome: ", i[3])
+	rst_where(fo, "Writable from: ", i[3])
 	for j in i[4].split("\n"):
 		fo.write("\t" + j.strip() + "\n")
 



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 130ada9 Update tutorial for 4.0
Message-ID: <E1WO1sG-00079G-3F@project.varnish-software.com>


commit 130ada9d511ee84046661166f904dd27cc0a2d57
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 13:51:26 2014 +0100

    Update tutorial for 4.0

diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
index 69d1c28..476d325 100644
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -15,6 +15,8 @@ is probably /etc/varnish/default.vcl.
 If you've been following the tutorial there is probably a section of
 the configuration that looks like this:::
 
+  vcl 4.0;
+  
   backend default {
       .host = "www.varnish-cache.org";
       .port = "80";
@@ -31,10 +33,12 @@ server so now we need to tie it to the origin.
 For this example, let's pretend the origin server is running on
 localhost, port 8080.::
 
-          backend default {
-                .host = "127.0.0.1";
-    		.port = "8080";
-	  }
+  vcl 4.0;
+
+  backend default {
+    .host = "127.0.0.1";
+    .port = "8080";
+  }
 
 
 Varnish can have several backends defined and can you can even join
diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
index ed2ca12..58bf574 100644
--- a/doc/sphinx/tutorial/starting_varnish.rst
+++ b/doc/sphinx/tutorial/starting_varnish.rst
@@ -37,6 +37,8 @@ Fire up your favorite editor and edit /etc/varnish/default.vcl. Most
 of it is commented out but there is some text that is not. It will
 probably look like this::
 
+  vcl 4.0;
+  
   backend default {
       .host = "127.0.0.1";
       .port = "8080";
@@ -45,6 +47,8 @@ probably look like this::
 We'll change it and make it point to something that works. Hopefully
 www.varnish-cache.org is up. Let's use that. Replace the text with::
 
+  vcl 4.0;
+  
   backend default {
       .host = "www.varnish-cache.org";
       .port = "80";



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 63dce4f Reformat and rewrite for readability
Message-ID: <E1WO1sG-00079P-48@project.varnish-software.com>


commit 63dce4fed739d953729aeb7f170568bb8f9f7c17
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:18:35 2014 +0100

    Reformat and rewrite for readability

diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index e497e00..bd1ec46 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -4,24 +4,21 @@ The Big Varnish Picture
 =======================
 
 What is in this package called "Varnish", what are all the different
-bits and pieces named and will you need a hex-wrench for assembly ?
+bits and pieces named and will you need a hex-wrench for assembly?
 
 The two main parts of Varnish are the two processes in the varnishd
-program.
-
-The first process is called "the manager", and its job is to talk
-to you, the administrator, and make the things you ask for happen.
+program. The first process is called "the manager", and its job is to
+talk to you, the administrator, and make the things you ask for
+happen.
 
 The second process is called "the worker" or just "the child" and
 this is the process which does all the actual work with your HTTP
 traffic.
 
-When you start varnishd, you start the manager process, and once
-it is done handling all the command line flags, it will start the
-child process for you.
-
-Should the child process die, the manager will start it again for
-you, automatically and right away.
+When you start varnishd, you start the manager process, and once it is
+done handling all the command line flags, it will start the child
+process for you. Should the child process die, the manager will start
+it again for you, automatically and right away.
 
 The main reason for this division of labor is security:  The manager
 process will typically run with "root" permissions, in order to
@@ -29,12 +26,11 @@ open TCP socket port 80, but it starts the child process with minimal
 permissions, as a defensive measure.
 
 The manager process is interactive, it offers a CLI -- Command Line
-Interface, which can be used manually, from scripts or programs.
-
-The CLI offers almost full control of what Varnish actually does
-to your HTTP traffic, and we have gone to great lengths to ensure
-that you should not need to restart the Varnish processes, unless
-you need to change something very fundamental.
+Interface, which can be used manually, from scripts or programs. The
+CLI offers almost full control of what Varnish actually does to your
+HTTP traffic, and we have gone to great lengths to ensure that you
+should not need to restart the Varnish processes, unless you need to
+change something very fundamental.
 
 The CLI can be safely accessed remotely, using a simple and flexible
 PSK -- Pre Shared Key, access control scheme, so it is easy to
@@ -46,7 +42,6 @@ All this is covered in :ref:`users_running`.
 How the child process should deal with the HTTP requests, what to
 cache, which headers to remove etc, is al specified in a small
 programming language called VCL -- Varnish Configuration Language.
-
 The manager process will compile the VCL program and check it for
 errors, but it is the child process which runs the VCL program, for
 each and every HTTP request which comes in.
@@ -75,13 +70,10 @@ can do for your HTTP traffic, there really is no limit.
 :ref:`users_vcl` describes VCL and what it can do in great detail.
 
 Varnish uses a piece of shared memory to report its activity and
-status.
-
-For each HTTP request, a number of very detailed records will be
-appended to the log segment in this shared memory.
-
-Other processes can subscribe to log-records, filter them, and
-format them, for instance as NCSA style log records.
+status. For each HTTP request, a number of very detailed records will
+be appended to the log segment in this shared memory.  Other processes
+can subscribe to log-records, filter them, and format them, for
+instance as NCSA style log records.
 
 Another segment in shared memory is used for statistics counters,
 this allows real-time, down to microsecond resolution monitoring
@@ -95,24 +87,21 @@ writing
 
 :ref:`users_report` explains how all that work.
 
-Presumably the reason why you are interested in Varnish, is that
-you want your website to work better.
-
-There are many aspects of performance tuning a website, from
-relatively simple policy decisions about what to cache, to designing
-a geographically diverse multilevel CDNs using ESI and automatic
-failover.
+Presumably the reason why you are interested in Varnish, is that you
+want your website to work better. There are many aspects of
+performance tuning a website, from relatively simple policy decisions
+about what to cache, to designing a geographically diverse multilevel
+CDNs using ESI and automatic failover.
 
 :ref:`users_performance` will take you through the possibilities
 and facilities Varnish offers.
 
-Finally, Murphys Law must be contended with:  Things will go wrong,
-and more likely than not, they will do so at zero-zero-dark O'clock.
-
-...during a hurricane, when your phones battery is flat and your
+Finally, Murphys Law must be contended with: Things will go wrong, and
+more likely than not, they will do so at zero-zero-dark O'clock. Most
+likely during a hurricane, when your phones battery is flat and your
 wife had prepared a intimate evening to celebrate your aniversary.
 
-Yes, we've all been there, havn't we ?
+Yes, we've all been there, havn't we?
 
 When things go wrong :ref:`users_trouble` will hopefully be of some help.
 



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 6aff95f clarity
Message-ID: <E1WO1sG-00079Z-5X@project.varnish-software.com>


commit 6aff95f5382cd5aa6f4a98ceed7a13b9ac6491eb
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:19:12 2014 +0100

    clarity

diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst
index 2b5509a..a5e97e1 100644
--- a/doc/sphinx/users-guide/run_cli.rst
+++ b/doc/sphinx/users-guide/run_cli.rst
@@ -45,7 +45,7 @@ What can you do with the CLI
 The CLI gives you almost total control over varnishd:
 
 * load/use/discard VCL programs
-* ban cache content
+* ban (invalidate) cache content
 * change parameters
 * start/stop worker process
 
@@ -115,7 +115,7 @@ but they are banned from delivery.
 
 Instead of checking each and every cached object right away, we
 test each object against the regular expression only if and when
-some requests decides to deliver it.
+a HTTP request asks for it.
 
 Banning stuff is much cheaper than restarting Varnish to get rid
 of wronly cached content.



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 1be3f1a add two params for -r consideration - and a bit for
 reformatting
Message-ID: <E1WO1sG-00079m-7J@project.varnish-software.com>


commit 1be3f1a185dfce49457f4a5ae62d163cd9458c00
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:19:53 2014 +0100

    add two params for -r consideration - and a bit for reformatting

diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index a2557a2..fb4b4d4 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -31,13 +31,13 @@ line, in order to make them invulnerable to subsequent manipulation.
 
 The important decisions to make are:
 
-#. Who should have access to the Command Line Interface ?
+#. Who should have access to the Command Line Interface?
 
-#. Which parameters can they change ?
+#. Which parameters can they change?
 
-#. Will inline-C code be allowed ?
+#. Will inline-C code be allowed?
 
-#. If/how VMODs will be restricted ?
+#. If/how VMODs will be restricted?
 
 CLI interface access
 ^^^^^^^^^^^^^^^^^^^^
@@ -89,7 +89,7 @@ command on stdin/stdout, but since you started the process, it
 would be hard to prevent you getting CLI access, wouldn't it ?
 
 CLI interface authentication
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 By default the CLI interface is protected with a simple,  yet
 strong "Pre Shared Key" authentication method, which do not provide
@@ -150,6 +150,9 @@ HTTP service, but a few can do more damage than that:
 :ref:`ref_param_cc_command`
 	Execute arbitrary programs
 
+:ref:`ref_param_vcc_allow_inline_c`
+        Allow inline C in VCL, which would any C code from VCL to be executed by Varnish.
+
 Furthermore you may want to look at and lock down:
 
 :ref:`ref_param_syslog_cli_traffic`
@@ -158,6 +161,11 @@ Furthermore you may want to look at and lock down:
 :ref:`ref_param_vcc_unsafe_path`
 	Retrict VCL/VMODS to :ref:`ref_param_vcl_dir` and :ref:`ref_param_vmod_dir`
 
+:ref:`ref_param_vmod_dir` 
+        The directory where Varnish will will look
+        for modules. This could potentially be used to load rouge
+        modules into Varnish.
+
 The CLI interface
 -----------------
 



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 9b59003 remove cruft
Message-ID: <E1WO1sG-00079s-8L@project.varnish-software.com>


commit 9b59003fc7bc20d57a14cf07eff4bfc0939a2ebf
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:20:55 2014 +0100

    remove cruft

diff --git a/doc/sphinx/users-guide/operation-cli.rst b/doc/sphinx/users-guide/operation-cli.rst
deleted file mode 100644
index 5cb40a5..0000000
--- a/doc/sphinx/users-guide/operation-cli.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-Varnishadm
-----------
-
-You connect to it and everything becomes awesome.



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] e4c66f7 Move purging into the performance section and explain
 its role
Message-ID: <E1WO1sG-0007A2-9M@project.varnish-software.com>


commit e4c66f75b38930fd0e87ee2eecf23da1339f26fe
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:26:38 2014 +0100

    Move purging into the performance section and explain its role

diff --git a/doc/sphinx/users-guide/operation.rst b/doc/sphinx/users-guide/operation.rst
index dc45e7e..ed6f5e0 100644
--- a/doc/sphinx/users-guide/operation.rst
+++ b/doc/sphinx/users-guide/operation.rst
@@ -6,7 +6,6 @@ XXX: These are chapters which need to find a new home in the other sections.
 .. toctree::
    :maxdepth: 2
 
-   operation-cli
    purging
    compression
    esi
diff --git a/doc/sphinx/users-guide/performance.rst b/doc/sphinx/users-guide/performance.rst
index c28dea0..1dbb991 100644
--- a/doc/sphinx/users-guide/performance.rst
+++ b/doc/sphinx/users-guide/performance.rst
@@ -6,7 +6,16 @@ Varnish and Website Performance
 This section is about tuning the performance of your Varnish server,
 and about tuning the performance of your website using Varnish.
 
+The section is split into two. One deals with the various tools and
+functions of Varnish that you should be aware of and the other focuses
+on the how to purge content out of your cache. Purging of content is
+essential in a performance context because it allows you to extend the
+*time-to-live* (TTL) of your cached objects. Having a long TTL allows
+Varnish to keep the content in cache longer, meaning Varnish will make
+send fewer requests to your relativly slow backend.
+
 .. toctree::
    :maxdepth: 2
 
    increasing-your-hitrate
+   purging



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 1199126 Add compression to performance
Message-ID: <E1WO1sG-0007AB-Ah@project.varnish-software.com>


commit 1199126de3127df1fd46b6c2615f02f89feca3d8
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:29:24 2014 +0100

    Add compression to performance

diff --git a/doc/sphinx/users-guide/operation.rst b/doc/sphinx/users-guide/operation.rst
index ed6f5e0..76fbf5d 100644
--- a/doc/sphinx/users-guide/operation.rst
+++ b/doc/sphinx/users-guide/operation.rst
@@ -6,8 +6,6 @@ XXX: These are chapters which need to find a new home in the other sections.
 .. toctree::
    :maxdepth: 2
 
-   purging
-   compression
    esi
    vary
    cookies
diff --git a/doc/sphinx/users-guide/performance.rst b/doc/sphinx/users-guide/performance.rst
index 1dbb991..baf9c75 100644
--- a/doc/sphinx/users-guide/performance.rst
+++ b/doc/sphinx/users-guide/performance.rst
@@ -6,7 +6,7 @@ Varnish and Website Performance
 This section is about tuning the performance of your Varnish server,
 and about tuning the performance of your website using Varnish.
 
-The section is split into two. One deals with the various tools and
+The section is split in three sections. One deals with the various tools and
 functions of Varnish that you should be aware of and the other focuses
 on the how to purge content out of your cache. Purging of content is
 essential in a performance context because it allows you to extend the
@@ -14,8 +14,14 @@ essential in a performance context because it allows you to extend the
 Varnish to keep the content in cache longer, meaning Varnish will make
 send fewer requests to your relativly slow backend.
 
+The final section deals with compression of web content. Varnish can
+gzip content when fetching it from the backend and then deliver
+compressed. This will reduce the time it takes to download the content
+thereby increasing the performance of your website.
+
 .. toctree::
    :maxdepth: 2
 
    increasing-your-hitrate
    purging
+   compression



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] fd3383e change commet style
Message-ID: <E1WO1sG-0007AL-Bu@project.varnish-software.com>


commit fd3383e48270b559a6930ec13b8092bdffc8ec8b
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:30:46 2014 +0100

    change commet style

diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 8a12461..58b31f2 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -48,11 +48,11 @@ For ESI to work you need to activate ESI processing in VCL, like this::
 
     sub vcl_fetch {
     	if (req.url == "/test.html") {
-           set beresp.do_esi = true; /* Do ESI processing		*/
-           set beresp.ttl = 24 h;    /* Sets the TTL on the HTML above  */
+           set beresp.do_esi = true; // Do ESI processing
+           set beresp.ttl = 24 h;    // Sets the TTL on the HTML above
     	} elseif (req.url == "/cgi-bin/date.cgi") {
-           set beresp.ttl = 1m;      /* Sets a one minute TTL on	*/
-	       	       	 	     /*  the included object		*/
+           set beresp.ttl = 1m;      // Sets a one minute TTL on
+	       	       	 	     // the included object
         }
     }
 
@@ -83,4 +83,4 @@ Doing ESI on JSON and other non-XMLish content
 Please note that Varnish will peek at the included content. If it
 doesn't start with a "<" Varnish assumes you didn't really mean to
 include it and disregard it. You can alter this behaviour by setting
-the esi_syntax parameter (see ref:`ref-varnishd`).
\ No newline at end of file
+the esi_syntax parameter (see ref:`ref-varnishd`).



From martin at varnish-software.com  Thu Mar 13 09:24:24 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] e462839 Disable daemonize option for varnishtop
Message-ID: <E1WO1sG-0007AU-DG@project.varnish-software.com>


commit e462839eb0d11e8709c757c5cbbd7a914ce3b9e2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 6 14:50:47 2014 +0100

    Disable daemonize option for varnishtop
    
    This option got added by misstake. Disable it as it doesn't do any
    good for this application.

diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h
index 592694a..2d3353e 100644
--- a/bin/varnishtop/varnishtop_options.h
+++ b/bin/varnishtop/varnishtop_options.h
@@ -57,7 +57,6 @@ VSL_OPT_b
 VSL_OPT_c
 VSL_OPT_C
 VUT_OPT_d
-VUT_OPT_D
 TOP_OPT_f
 VUT_OPT_g
 VUT_OPT_h



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] daee5be Remove residue of former structure
Message-ID: <E1WO1sG-0007Ag-Ec@project.varnish-software.com>


commit daee5be0d9003df7c766f6a0d62774741c228bdf
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 14:43:49 2014 +0100

    Remove residue of former structure

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 462a781..b5fa17e 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -200,14 +200,3 @@ setting up redirects or by using the following VCL::
   }
 
 
-Ways of increasing your hitrate even more
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following chapters should give your ways of further increasing
-your hitrate, especially the chapter on Cookies.
-
- * :ref:`users-guide-cookies`
- * :ref:`users-guide-vary`
- * :ref:`users-guide-purging`
- * :ref:`users-guide-esi`
-



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 4f7f7e0 Rewrite intro
Message-ID: <E1WO1sG-0007Am-Fs@project.varnish-software.com>


commit 4f7f7e0d9c1ebb6cf89187c56365f892024e3fa1
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 15:21:32 2014 +0100

    Rewrite intro

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 1f41b0f..5241575 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -1,31 +1,28 @@
 .. _tutorial-intro:
 
-What is Varnish?
-----------------
-
-Varnish Cache is a web application accelerator. It can also be called
-a HTTP reverse proxy. It sits in front of a web server, accepts HTTP
-requests and tries to answer those by looking them up in it's
-cache. If it can't answer the request from cache it will hand the
-request over to a backend server.
-
-You can say that Varnish Cache is a specialized key-value store that
-stores HTTP responses. These are looked up with HTTP requests. If a
-match is found the content is delivered.
-
-Varnish Cache will typically look up a response by inspecting the HTTP
-Host header together with the URL.  Varnish Cache maintains an index,
-a hash, with all the host+url combinations that are kept in the cache.
-
-Some times Varnish will refuse to store the content in it's
-cache. This might be because the HTTP reponse has metadata that
-disables cacheing or that there might be a cookie involved. Varnish,
-in the default configuration, will refuse to cache content when there
-are cookies involved because it has no idea if the content is derived
-from the cookie or not.
-
-All this behaviour can be changed using VCL. See the Users Guide for
-more information on how to do that.
+The fundamentals of web proxy caching with Varnish
+--------------------------------------------------
+
+Varnish is a caching HTTP reverse proxy. It recieves requests from
+clients and tries to answer them from its cache. If it cannot answer
+the request from its cache it will forward the request to the backend,
+fetch the response, store it and deliver it to the client.
+
+Varnish decides whether it can store the content or not based on the
+response it's gets back from the backend. The backend can instruct
+Varnish to cache the content with the HTTP response header
+Cache-Control.
+
+Varnish will be very careful when it encounters cookies, either coming
+from the client or from the origin server. When Varnish sees a
+Set-Cookie header on a response it decides that the object is not
+cacheable. When there is a Cookie header in the request it will also
+refuse to serve a cached object and rather ask the backend for version
+of the object that is tailored to the request.
+
+This behaviour and most other behaviour can be changed using policies
+written in the Varnish Configuration Language. See the Users Guide
+for more information on how to do that.
 
 
 Performance



From perbu at varnish-software.com  Thu Mar 13 09:24:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 9335747 fix reference
Message-ID: <E1WO1sG-0007Ax-Gg@project.varnish-software.com>


commit 93357472bfdaa16b321346b067b93f80000facc3
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 6 15:22:16 2014 +0100

    fix reference

diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
index 856cbfa..aaf3bb5 100644
--- a/doc/sphinx/users-guide/operation-logging.rst
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -65,4 +65,4 @@ want to know are:
  Only list transactions where the tag matches a regular expression. If
  it matches you will get the whole transaction.
 
-For more information on this topic please see ref:`ref-varnishlog`.
+For more information on this topic please see :ref:`ref-varnishlog`.



From fgsch at lodoss.net  Thu Mar 13 09:24:24 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 1bc8f54 Fix dist
Message-ID: <E1WO1sG-0007B7-IB@project.varnish-software.com>


commit 1bc8f5449bd33a23f2e6c97d41af91091917bd4a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Feb 7 02:24:59 2014 +0000

    Fix dist

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 7940d07..3abd43f 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -143,7 +143,6 @@ EXTRA_DIST = \
 	users-guide/increasing-your-hitrate.rst \
 	users-guide/index.rst \
 	users-guide/intro.rst \
-	users-guide/operation-cli.rst \
 	users-guide/operation-logging.rst \
 	users-guide/operation.rst \
 	users-guide/operation-statistics.rst \



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 0f5f294 White space nit
Message-ID: <E1WO1sG-0007BG-Jc@project.varnish-software.com>


commit 0f5f29436c3cc225c66883cdfa0d681d7b0ccdda
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 6 18:05:14 2014 +0000

    White space nit

diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index df02a48..5a5f087 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -133,6 +133,6 @@ VFIL_nonblocking(int fd)
 	assert(i != -1);
 	i |= O_NONBLOCK;
 	i = fcntl(fd, F_SETFL, i);
-       assert(i != -1);
-       return (i);
+	assert(i != -1);
+	return (i);
 }



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] f01131b Fix printf format nit.
Message-ID: <E1WO1sG-0007BP-L4@project.varnish-software.com>


commit f01131b30ff664f8cdf9657947cc178cbbf32416
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 7 11:29:26 2014 +0000

    Fix printf format nit.
    
    Fixes #1429

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 99c54fa..0b71923 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -813,7 +813,7 @@ varnish_expect(const struct varnish *v, char * const *av) {
 		    av[0], sp.val, av[1], av[2]);
 	} else {
 		vtc_log(v->vl, 0, "Not true: %s (%ju) %s %s (%ju)",
-		    av[0], sp.val, av[1], av[2], ref);
+		    av[0], (uintmax_t)sp.val, av[1], av[2], (uintmax_t)ref);
 	}
 }
 



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] d427ba9 Always nuke the IMS-candidate object if we successfully
 IMS fetch it.
Message-ID: <E1WO1sG-0007BY-MI@project.varnish-software.com>


commit d427ba97b2b769941cc405c5b1bd843043d4755c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 7 11:39:53 2014 +0000

    Always nuke the IMS-candidate object if we successfully IMS fetch it.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c6514fd..9254c8e 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -660,6 +660,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		assert(al == bo->ims_obj->len);
 		assert(obj->len == al);
 		VBO_setstate(bo, BOS_FINISHED);
+		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
 	}
 	HSH_Complete(obj->objcore);
 	return (F_STP_DONE);



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 7ccfbdd Remove a noise debug vsl
Message-ID: <E1WO1sG-0007Bk-Nd@project.varnish-software.com>


commit 7ccfbdd73da6410edd3be697389b5f31fc4794f1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 7 11:56:45 2014 +0000

    Remove a noise debug vsl

diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 3c2f670..a92d29f 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -86,7 +86,6 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 		ol = oi->len;
 		while (1) {
 			nl = VBO_waitlen(oi->bo, ol);
-			VSL(SLT_Debug, 0, "STREAM %zd -> %zd", ol, nl);
 			if (nl != ol)
 				break;
 			if (oi->bo->state == BOS_FINISHED)



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 1418ca7 Make sure the thread pool herders wake up every so
 often so they notice if the params have changed.
Message-ID: <E1WO1sG-0007Bt-P1@project.varnish-software.com>


commit 1418ca74677af0fc9feae0fcd3090dcd4f531554
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 7 11:56:56 2014 +0000

    Make sure the thread pool herders wake up every so often so they
    notice if the params have changed.

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index b22fe09..75fac47 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -411,7 +411,8 @@ pool_herder(void *priv)
 
 		Lck_Lock(&pp->mtx);
 		if (!pp->dry) {
-			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx, 0);
+			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
+				VTIM_real() + 5);
 		} else {
 			/* XXX: unsafe counters */
 			VSC_C_main->threads_limited++;



From fgsch at lodoss.net  Thu Mar 13 09:24:24 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 6ef598f Make this work with bsd make as well
Message-ID: <E1WO1sG-0007C2-Qd@project.varnish-software.com>


commit 6ef598f51fc80db676c1f84a754093141f371a4f
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Feb 7 12:23:01 2014 +0000

    Make this work with bsd make as well
    
    Avoid using gnu make specific rules. While here sort entries and add
    missing dep for varnishd.

diff --git a/man/Makefile.am b/man/Makefile.am
index 2b26b0b..50e9b64 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -8,9 +8,10 @@ vsc2rst_SOURCES = vsc2rst.c \
 
 AM_CPPFLAGS = -I$(top_srcdir)/include
 
-dist_man_MANS = vcl.7 \
+dist_man_MANS = \
 	varnish-cli.7 \
 	varnish-counters.7 \
+	vcl.7 \
 	vsl.7 \
 	vsl-query.7 \
 	varnishadm.1 \
@@ -24,16 +25,16 @@ dist_man_MANS = vcl.7 \
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
-vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
-	$(top_srcdir)/bin/varnishd/builtin.vcl
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
-
 varnish-cli.7: $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
 
 varnish-counters.7: vsc2rst
 	./vsc2rst | ${RST2MAN} - $@
 
+vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
+	$(top_srcdir)/bin/varnishd/builtin.vcl
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
+
 vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \
 	$(top_srcdir)/lib/libvarnishapi/vsl-tags.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
@@ -41,11 +42,13 @@ vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \
 vsl-query.7: $(top_srcdir)/doc/sphinx/reference/vsl-query.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
 
-%.7: $(top_srcdir)/doc/sphinx/reference/%.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
+varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@
 
-%.1: $(top_srcdir)/doc/sphinx/reference/%.rst
-	${RST2MAN} $< $@
+varnishd.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishd.rst \
+	$(top_srcdir)/doc/sphinx/include/params.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@
 
 varnishncsa.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \
@@ -59,6 +62,15 @@ varnishlog.1: \
 	$(top_srcdir)/doc/sphinx/include/varnishlog_synopsis.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
 
+varnishreplay.1: $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst $@
+
+varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@
+
+varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
+
 varnishtop.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_options.rst \



From daghf at varnish-software.com  Thu Mar 13 09:24:24 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] a68bedb Fix varnishlog option doc typo.
Message-ID: <E1WO1sG-0007CB-SO@project.varnish-software.com>


commit a68bedb64414234f3943054eec5bf89c585d569b
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Fri Feb 7 13:52:38 2014 +0100

    Fix varnishlog option doc typo.

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 60ebc57..7f6d0bf 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -32,14 +32,14 @@ The following options are available:
 -k num
 
 	Only show the first num log transactions (or log records
-	in --raw mode)
+	in -g raw mode)
 
 	XXX: Not yet implemented
 
 -s num
 
 	Skip the first num log transactions (or log records if
-	in --raw mode)
+	in -g raw mode)
 
 	XXX: Not yet implemented
 



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 0ccbd83 Move the closing of VDI connection to also cover
 successful IMS fetches.
Message-ID: <E1WO1sG-0007CL-Tn@project.varnish-software.com>


commit 0ccbd8320692becd41f27061737dbfede593550d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Feb 9 23:05:28 2014 +0000

    Move the closing of VDI connection to also cover successful IMS fetches.
    
    Spotted by:	scoof

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9254c8e..819c39f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -498,14 +498,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	bo->t_body = VTIM_mono();
 
-	if (bo->vbc != NULL) {
-		if (bo->should_close)
-			VDI_CloseFd(&bo->vbc);
-		else
-			VDI_RecycleFd(&bo->vbc);
-		AZ(bo->vbc);
-	}
-
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
 
@@ -721,6 +713,14 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	}
 	assert(WRW_IsReleased(wrk));
 
+	if (bo->vbc != NULL) {
+		if (bo->should_close)
+			VDI_CloseFd(&bo->vbc);
+		else
+			VDI_RecycleFd(&bo->vbc);
+		AZ(bo->vbc);
+	}
+
 	if (bo->state == BOS_FAILED)
 		assert(bo->fetch_objcore->flags & OC_F_FAILED);
 



From phk at FreeBSD.org  Thu Mar 13 09:24:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:24 +0100
Subject: [4.0] 13e2f2c Name the mempool guardian threads.
Message-ID: <E1WO1sG-0007CV-VC@project.varnish-software.com>


commit 13e2f2c694c76b0e6994626c849c954f768deef2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 11 09:31:59 2014 +0000

    Name the mempool guardian threads.

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index fded87b..79d1841 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -51,7 +51,7 @@ VTAILQ_HEAD(memhead_s, memitem);
 struct mempool {
 	unsigned			magic;
 #define MEMPOOL_MAGIC			0x37a75a8d
-	char				name[8];
+	char				name[12];
 	struct memhead_s		list;
 	struct memhead_s		surplus;
 	struct lock			mtx;
@@ -101,6 +101,7 @@ mpl_guard(void *priv)
 	double last = 0;
 
 	CAST_OBJ_NOTNULL(mpl, priv, MEMPOOL_MAGIC);
+	THR_SetName(mpl->name);
 	mpl_slp = 0.15;	// random
 	while (1) {
 		VTIM_sleep(mpl_slp);
@@ -229,7 +230,7 @@ MPL_New(const char *name,
 
 	ALLOC_OBJ(mpl, MEMPOOL_MAGIC);
 	AN(mpl);
-	bprintf(mpl->name, "%s", name);
+	bprintf(mpl->name, "MPL_%s", name);
 	mpl->param = pp;
 	mpl->cur_size = cur_size;
 	VTAILQ_INIT(&mpl->list);
@@ -237,7 +238,7 @@ MPL_New(const char *name,
 	Lck_New(&mpl->mtx, lck_mempool);
 	/* XXX: prealloc min_pool */
 	mpl->vsc = VSM_Alloc(sizeof *mpl->vsc,
-	    VSC_CLASS, VSC_type_mempool, mpl->name);
+	    VSC_CLASS, VSC_type_mempool, mpl->name + 4);
 	AN(mpl->vsc);
 	AZ(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
 	AZ(pthread_detach(mpl->thread));



From phk at FreeBSD.org  Thu Mar 13 09:24:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] a88dbf9 Fix a memory leak in BAN processing: We didn't free the
 compiled regexp.
Message-ID: <E1WO1sH-0007Ce-0N@project.varnish-software.com>


commit a88dbf913c7c264bb994f1fd5810e8a6ab27c3cc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 11 10:32:01 2014 +0000

    Fix a memory leak in BAN processing:  We didn't free the compiled regexp.
    
    Also solve a abstract corner-case when malloc(3) fails.  We'll probably
    still come crashing down, but not for this reason any more.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 93ec51f..7ab3a1a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -811,6 +811,7 @@ struct ban *BAN_New(void);
 int BAN_AddTest(struct ban *, const char *, const char *, const char *);
 void BAN_Free(struct ban *b);
 char *BAN_Insert(struct ban *b);
+void BAN_Free_Errormsg(char *);
 void BAN_Init(void);
 void BAN_Shutdown(void);
 void BAN_NewObjCore(struct objcore *oc);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 1c6b0c8..99f2914 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -394,6 +394,7 @@ ban_parse_regexp(struct ban *b, const char *a3)
 	rc = pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, &sz);
 	AZ(rc);
 	ban_add_lump(b, re, sz);
+	pcre_free(re);
 	return (0);
 }
 
@@ -464,19 +465,27 @@ BAN_AddTest(struct ban *b, const char *a1, const char *a2, const char *a3)
  *      deleted.
  */
 
+static char ban_error_nomem[] = "Could not get memory";
+
 static char *
 ban_ins_error(const char *p)
 {
 	char *r = NULL;
-	static char nomem[] = "Could not get memory";
 
 	if (p != NULL)
 		r = strdup(p);
 	if (r == NULL)
-		r = nomem;
+		r = ban_error_nomem;
 	return (r);
 }
 
+void
+BAN_Free_Errormsg(char *p)
+{
+	if (p != ban_error_nomem)
+		free(p);
+}
+
 char *
 BAN_Insert(struct ban *b)
 {
@@ -1229,7 +1238,7 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 	p = BAN_Insert(b);
 	if (p != NULL) {
 		VCLI_Out(cli, "%s", p);
-		free(p);
+		BAN_Free_Errormsg(p);
 		VCLI_SetResult(cli, CLIS_PARAM);
 	}
 }
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 3d45e4c..ce536e5 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -446,7 +446,7 @@ VRT_ban_string(const struct vrt_ctx *ctx, const char *str)
 			if (a1 != NULL) {
 				VSLb(ctx->vsl, SLT_VCL_Error,
 				    "ban(): %s", a1);
-				free(a1);
+				BAN_Free_Errormsg(a1);
 			}
 			break;
 		}



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] fdabaa1 Spelling
Message-ID: <E1WO1sH-0007Cn-1t@project.varnish-software.com>


commit fdabaa15d33cd9b4689582777af476c0c8e1a20e
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Feb 16 09:18:00 2014 +0000

    Spelling

diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
index 0b78893..5e21511 100644
--- a/doc/sphinx/users-guide/compression.rst
+++ b/doc/sphinx/users-guide/compression.rst
@@ -36,7 +36,7 @@ vcl_fetch by setting do_gzip to true, like this::
   }
 
 Please make sure that you don't try to compress content that is
-incompressable, like jpgs, gifs and mp3. You'll only waste CPU
+uncompressable, like jpgs, gifs and mp3. You'll only waste CPU
 cycles. You can also uncompress objects before storing it in memory by
 setting do_gunzip to *true* but I have no idea why anybody would want
 to do that.
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index e78ceda..c7acd73 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -28,7 +28,7 @@ Setting this header can be as simple as::
    }
 
 There are different commercial and free offerings in doing grouping and
-identifiying clients in further detail than this. For a basic and community
+identifying clients in further detail than this. For a basic and community
 based regular expression set, see
 https://github.com/varnish/varnish-devicedetect/ .
 
diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index bd1ec46..79b4c06 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -99,9 +99,9 @@ and facilities Varnish offers.
 Finally, Murphys Law must be contended with: Things will go wrong, and
 more likely than not, they will do so at zero-zero-dark O'clock. Most
 likely during a hurricane, when your phones battery is flat and your
-wife had prepared a intimate evening to celebrate your aniversary.
+wife had prepared a intimate evening to celebrate your anniversary.
 
-Yes, we've all been there, havn't we?
+Yes, we've all been there, haven't we?
 
 When things go wrong :ref:`users_trouble` will hopefully be of some help.
 
diff --git a/doc/sphinx/users-guide/params.rst b/doc/sphinx/users-guide/params.rst
index 8985bdc..d3a10af 100644
--- a/doc/sphinx/users-guide/params.rst
+++ b/doc/sphinx/users-guide/params.rst
@@ -5,7 +5,7 @@ Parameters
 
 Varnish Cache has a set of parameter that affect its behaviour and
 performance. Most of these parameters can be set on the Varnish
-command line (through varnishadm) useing the param.set keyword.
+command line (through varnishadm) using the param.set keyword.
 
 Some parameters can, for security purposes be read only using the "-r"
 command line switch to varnishd.
diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index fb4b4d4..16ab277 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -9,7 +9,7 @@ this chapter:  We have protected Varnish as well as we can from
 anything which can come in through HTTP socket.
 
 If parts of your web infrastructure are outsourced or otherwise
-partitioned along adminitrative lines, you need to think about
+partitioned along administrative lines, you need to think about
 security.
 
 Varnish provides four levels of authority, roughly related to
diff --git a/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst b/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
index 8afdb50..b5724b4 100644
--- a/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
+++ b/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
@@ -3,7 +3,7 @@
 Altering the backend response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Here we override the TTL of a object comming from the backend if it
+Here we override the TTL of a object coming from the backend if it
 matches certain criteria::
 
   sub vcl_fetch {
diff --git a/doc/sphinx/users-guide/vcl-examples.rst b/doc/sphinx/users-guide/vcl-examples.rst
index 931ec27..e3a9489 100644
--- a/doc/sphinx/users-guide/vcl-examples.rst
+++ b/doc/sphinx/users-guide/vcl-examples.rst
@@ -3,7 +3,7 @@ VCL Examples
 ------------
 
 These are a short collection of examples that showcase some of the
-capabilites of the VCL language.
+capabilities of the VCL language.
 
 .. toctree::
 
diff --git a/doc/sphinx/users-guide/vcl-hashing.rst b/doc/sphinx/users-guide/vcl-hashing.rst
index 01b42d0..fdbe37f 100644
--- a/doc/sphinx/users-guide/vcl-hashing.rst
+++ b/doc/sphinx/users-guide/vcl-hashing.rst
@@ -3,7 +3,7 @@ Hashing
 
 Internally, when Varnish stores content in it's store it uses a hash
 key to find the object again. In the default setup this key is
-calculated based on the content of the *Host* header or the IP adress
+calculated based on the content of the *Host* header or the IP address
 of the server and the URL.
 
 Behold the default vcl::
@@ -19,8 +19,8 @@ Behold the default vcl::
  }
 
 As you can see it first chucks in req.url then req.http.host if it
-exsists. It is worth pointing out that Varnish doesn't lowercase the
-hostname or the URL before hashing it so in thery having Varnish.org/
+exists. It is worth pointing out that Varnish doesn't lowercase the
+hostname or the URL before hashing it so in theory having Varnish.org/
 and varnish.org/ would result in different cache entries. Browers
 however, tend to lowercase hostnames.
 
diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst
index 978f5e4..15999cc 100644
--- a/doc/sphinx/users-guide/vcl-variables.rst
+++ b/doc/sphinx/users-guide/vcl-variables.rst
@@ -15,7 +15,7 @@ In VCL, there several important objects.
 
 *beresp*
  The backend respons object. It contains the headers of the object 
- comming from the backend. Most of the work you do in vcl_fetch you 
+ coming from the backend. Most of the work you do in vcl_fetch you 
  do on the beresp object.
 
 *resp*
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index dc3fdc5..53d99e3 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -16,7 +16,7 @@ the request is being handled by altering the VCL code. You can direct
 certain requests to certains backends, you can alter the requests and
 the responses or have Varnish take various actions depending on
 arbitrary properties of the request or the response. This makes
-Varnish an extremly powerful HTTP processor, not just for caching.
+Varnish an extremely powerful HTTP processor, not just for caching.
 
 Varnish translates VCL into binary code which is then executed when
 requests arrive. The performance impact of VCL is negligible.



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 8a1448a Mention all the valid units of time on error
Message-ID: <E1WO1sH-0007D2-3y@project.varnish-software.com>


commit 8a1448ad1e3a3617c8cb44c68472349591df2b42
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Feb 16 13:24:36 2014 +0000

    Mention all the valid units of time on error

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index fab381a..08f91df 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -77,7 +77,7 @@ vcc_TimeUnit(struct vcc *tl)
 	else {
 		VSB_printf(tl->sb, "Unknown time unit ");
 		vcc_ErrToken(tl, tl->t);
-		VSB_printf(tl->sb, ".  Legal are 's', 'm', 'h' and 'd'\n");
+		VSB_printf(tl->sb, ".  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'\n");
 		vcc_ErrWhere(tl, tl->t);
 		return (1.0);
 	}



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] f66d9ae Start updating vcl(7) for 4.0
Message-ID: <E1WO1sH-0007D8-5R@project.varnish-software.com>


commit f66d9ae6c2688007345586a07c7107511761cc4c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 08:02:48 2014 +0000

    Start updating vcl(7) for 4.0

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9aea743..7e00e43 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -33,20 +33,18 @@ For an overview of the VCL syntax please see the users guide.
 Backend declarations
 --------------------
 
-A backend declaration creates and initializes a named backend object:
-::
+A backend declaration creates and initializes a named backend object::
 
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-  }
+    backend www {
+        .host = "www.example.com";
+        .port = "http";
+    }
 
-The backend object can later be used to select a backend at request time:
-::
+The backend object can later be used to select a backend at request time::
 
-  if (req.http.host ~ "(?i)^(www.)?example.com$") {
-    set req.backend = www;
-  }
+    if (req.http.host ~ "(?i)^(www.)?example.com$") {
+        set req.backend = www;
+    }
 
 To avoid overloading backend servers, .max_connections can be set to
 limit the maximum number of concurrent backend connections.
@@ -57,24 +55,15 @@ backend connection, .first_byte_timeout for the time to wait for the
 first byte from the backend and .between_bytes_timeout for time to
 wait between each received byte.
 
-These can be set in the declaration like this:
-::
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-    .connect_timeout = 1s;
-    .first_byte_timeout = 5s;
-    .between_bytes_timeout = 2s;
-  }
-
-To mark a backend as unhealthy after number of items have been added
-to its saintmode list ``.saintmode_threshold`` can be set to the maximum
-list size. Setting a value of 0 disables saint mode checking entirely
-for that backend.  The value in the backend declaration overrides the
-parameter.
-
+These can be set in the declaration like this::
 
+    backend www {
+        .host = "www.example.com";
+        .port = "http";
+        .connect_timeout = 1s;
+        .first_byte_timeout = 5s;
+        .between_bytes_timeout = 2s;
+    }
 
 .. _ref-vcl-director:
 
@@ -89,24 +78,23 @@ be used.
 There are several types of directors. The different director types
 use different algorithms to choose which backend to use.
 
-Configuring a director may look like this:
-::
-
-  director b2 random {
-    .retries = 5;
-    {
-      // We can refer to named backends
-      .backend = b1;
-      .weight  = 7;
-    }
-    {
-      // Or define them inline 
-      .backend  = {
-        .host = "fs2";
-      }
-    .weight         = 3;
+Configuring a director may look like this::
+
+    director b2 random {
+        .retries = 5;
+        {
+            # We can refer to named backends
+            .backend = b1;
+            .weight = 7;
+        }
+        {
+            # Or define them inline
+            .backend  = {
+                .host = "fs2";
+            }
+            .weight = 3;
+        }
     }
-  } 
 
 The family of random directors
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -171,8 +159,7 @@ The DNS director
 ~~~~~~~~~~~~~~~~
 
 The DNS director can use backends in two different ways. Either like the
-random or round-robin director or using .list:
-::
+random or round-robin director or using .list::
 
   director directorname dns {
           .list = {
@@ -207,27 +194,26 @@ round-robin manner.
 The fallback director
 ~~~~~~~~~~~~~~~~~~~~~
 
-The fallback director will pick the first backend that is healthy. It 
+The fallback director will pick the first backend that is healthy. It
 considers them in the order in which they are listed in its definition.
 
 The fallback director does not take any options.
 
-An example of a fallback director:
-::
+An example of a fallback director::
 
-  director b3 fallback {
-    { .backend = www1; }
-    { .backend = www2; } // will only be used if www1 is unhealthy.
-    { .backend = www3; } // will only be used if both www1 and www2
-                         // are unhealthy.
-  }
+    director b3 fallback {
+        { .backend = www1; }
+        { .backend = www2; } # will only be used if www1 is unhealthy.
+        { .backend = www3; } # will only be used if both www1 and www2
+                             # are unhealthy.
+    }
 
 Backend probes
 --------------
 
 Backends can be probed to see whether they should be considered
-healthy or not.  The return status can also be checked by using
-req.backend.healthy.
+healthy or not.  The return status can also be checked by using the
+healthy function from the std vmod.
 
 Probes take the following parameters:
 
@@ -241,7 +227,7 @@ Probes take the following parameters:
 .window
   How many of the latest polls we examine to determine backend health.
   Defaults to 8.
-.threshold 
+.threshold
   How many of the polls in .window must have succeeded for us to
   consider the backend healthy. If this is set to more than or equal
   to the threshold, the backend starts as healthy. Defaults to the
@@ -264,66 +250,61 @@ Probes take the following parameters:
   Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
-backend or director:
-::
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-    .probe = {
-      .url = "/test.jpg";
-      .timeout = 0.3 s;
-      .window = 8;
-      .threshold = 3;
-      .initial = 3;
+backend or director::
+
+    backend www {
+        .host = "www.example.com";
+        .port = "http";
+        .probe = {
+            .url = "/test.jpg";
+            .timeout = 0.3s;
+            .window = 8;
+            .threshold = 3;
+            .initial = 3;
+        }
     }
-  }
 
-Or it can be defined separately and then referenced:
-::
+Or it can be defined separately and then referenced::
 
-  probe healthcheck {
-     .url = "/status.cgi";
-     .interval = 60s;     
-     .timeout = 0.3 s;
-     .window = 8;
-     .threshold = 3;
-     .initial = 3;
-     .expected_response = 200;
-  }	
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-    .probe = healthcheck;
-  }
+    probe healthcheck {
+        .url = "/status.cgi";
+        .interval = 60s;
+        .timeout = 0.3s;
+        .window = 8;
+        .threshold = 3;
+        .initial = 3;
+        .expected_response = 200;
+    }
 
-If you have many backends this can simplify the config a lot.
+    backend www {
+        .host = "www.example.com";
+        .port = "http";
+        .probe = healthcheck;
+    }
 
+If you have many backends this can simplify the config a lot.
 
-It is also possible to specify the raw HTTP request:
-::
+It is also possible to specify the raw HTTP request::
 
-  probe rawprobe {
-      # NB: \r\n automatically inserted after each string!
-      .request =
-        "GET / HTTP/1.1"
-        "Host: www.foo.bar"
-        "Connection: close";
-  }
+    probe rawprobe {
+        # NB: \r\n automatically inserted after each string!
+        .request =
+          "GET / HTTP/1.1"
+          "Host: www.foo.bar"
+          "Connection: close";
+    }
 
 ACLs
 ----
 
 An ACL declaration creates and initializes a named access control list
-which can later be used to match client addresses:
-::
+which can later be used to match client addresses::
 
-  acl local {
-    "localhost";         // myself
-    "192.0.2.0"/24;      // and everyone on the local network
-    ! "192.0.2.23";      // except for the dialin router 
-  }
+    acl local {
+        "localhost";    # myself
+        "192.0.2.0"/24; # and everyone on the local network
+        ! "192.0.2.23"; # except for the dialin router
+    }
 
 If an ACL entry specifies a host name which Varnish is unable to
 resolve, it will match any address it is compared to.  Consequently,
@@ -331,12 +312,11 @@ if it is preceded by a negation mark, it will reject any address it is
 compared to, which may not be what you intended.  If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
-To match an IP address against an ACL, simply use the match operator:
-::
+To match an IP address against an ACL, simply use the match operator::
 
-  if (client.ip ~ local) {
-    return (pipe);
-  }
+    if (client.ip ~ local) {
+        return (pipe);
+    }
 
 Regular Expressions
 -------------------
@@ -346,54 +326,51 @@ complete description of PCRE please see the pcre(3) man page.
 
 To send flags to the PCRE engine, such as to turn on *case
 insensitivity* add the flag within parens following a question mark,
-like this:
-::
-
-  # If host is NOT example dot com..
-  if (req.http.host !~ "(?i)example.com$") {
-          ...
-  }
+like this::
 
+    # If host is NOT example dot com..
+    if (req.http.host !~ "(?i)example.com$") {
+        ...
+    }
 
 Functions
 ---------
 
 The following built-in functions are available:
 
+ban(expression)
+  Bans all objects in cache that match the expression.
+
 hash_data(str)
-  Adds a string to the hash input. In default.vcl hash_data() is
-  called on the host and URL of the *request*.
+  Adds a string to the hash input. In the built-in VCL hash_data()
+  is called on the host and URL of the *request*.
 
 regsub(str, regex, sub)
-  Returns a copy of str with the first occurrence of the regular 
+  Returns a copy of str with the first occurrence of the regular
   expression regex replaced with sub. Within sub, \\0 (which can
   also be spelled \\&) is replaced with the entire matched string,
-  and \\n is replaced with the contents of subgroup n in the 
+  and \\n is replaced with the contents of subgroup n in the
   matched string.
 
 regsuball(str, regex, sub)
   As regsub() but this replaces all occurrences.
 
-ban(ban expression)
-  Bans all objects in cache that match the expression.
-
 Subroutines
 ~~~~~~~~~~~
 
-A subroutine is used to group code for legibility or reusability:
-::
-  
-  sub pipe_if_local {
-    if (client.ip ~ local) {
-      return (pipe);
+A subroutine is used to group code for legibility or reusability::
+
+    sub pipe_if_local {
+        if (client.ip ~ local) {
+            return (pipe);
+        }
     }
-  }
 
 Subroutines in VCL do not take arguments, nor do they return values.
 
-To call a subroutine, use the call keyword followed by the subroutine's name:
+To call a subroutine, use the call keyword followed by the subroutine's name::
 
-call pipe_if_local;
+    call pipe_if_local;
 
 There are a number of special subroutines which hook into the Varnish
 workflow.  These subroutines may inspect and manipulate HTTP headers
@@ -416,72 +393,87 @@ vcl_recv
   been received and parsed.  Its purpose is to decide whether or not
   to serve the request, how to do it, and, if applicable, which backend
   to use.
-  
+
   The vcl_recv subroutine may terminate with calling return() on one of
   the following keywords:
 
-  error code [reason]
+  error(code [, reason])
     Return the specified error code to the client and abandon the request.
 
-  pass    
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  pipe    
-    Switch to pipe mode.  Control will eventually pass to vcl_pipe.
-
-  lookup  
+  hash
     Look up the requested object in the cache.  Control will
     eventually pass to vcl_hit or vcl_miss, depending on whether the
     object is in the cache.  The ``bereq.method`` value will be set
     to ``GET`` regardless of the value of ``req.method``.
 
-vcl_pipe
-  Called upon entering pipe mode.  In this mode, the request is passed
-  on to the backend, and any further data from either client or
-  backend is passed on unaltered until either end closes the
-  connection.
-  
-  The vcl_pipe subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
+  pass
+    Switch to pass mode.  Control will eventually pass to vcl_pass.
 
   pipe
-    Proceed with pipe mode.
+    Switch to pipe mode.  Control will eventually pass to vcl_pipe.
+
+  purge
+    ?
 
 vcl_pass
   Called upon entering pass mode.  In this mode, the request is passed
   on to the backend, and the backend's response is passed on to the
   client, but is not entered into the cache.  Subsequent requests
   submitted over the same client connection are handled normally.
-  
+
   The vcl_pass subroutine may terminate with calling return() with one of
   the following keywords:
-  
-  error code [reason]
+
+  error(code [, reason])
     Return the specified error code to the client and abandon the request.
 
-  pass
+  fetch
     Proceed with pass mode.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
+vcl_pipe
+  Called upon entering pipe mode.  In this mode, the request is passed
+  on to the backend, and any further data from either client or
+  backend is passed on unaltered until either end closes the
+  connection.
+
+  The vcl_pipe subroutine may terminate with calling return() with one of
+  the following keywords:
+
+  error(code [, reason])
+    Return the specified error code to the client and abandon the request.
+
+  pipe
+    Proceed with pipe mode.
+
+vcl_purge
+  ?
+
+  The vcl_purge subroutine may terminate with calling return() with one of
+  the following keywords:
+
+  error(code [, reason])
+    Return the specified error code to the client and abandon the request.
+
+..  fetch
+..    Not implemented.
+
 vcl_hash
   You may call hash_data() on the data you would like to add to the hash.
-  
+
   The vcl_hash subroutine may terminate with calling return() with one of
   the following keywords:
 
-  hash
+  lookup
     Proceed.
 
 vcl_hit
   Called after a cache lookup if the requested document was found in the cache.
-  
+
   The vcl_hit subroutine may terminate with calling return() with one of
   the following keywords:
 
@@ -489,69 +481,73 @@ vcl_hit
     Deliver the cached object to the client.  Control will eventually
     pass to vcl_deliver.
 
-  error code [reason]
+  error(code [, reason])
     Return the specified error code to the client and abandon the request.
 
+  fetch
+    ?
+
   pass
     Switch to pass mode.  Control will eventually pass to vcl_pass.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_miss
   Called after a cache lookup if the requested document was not found
   in the cache.  Its purpose is to decide whether or not to attempt to
   retrieve the document from the backend, and which backend to use.
-  
+
   The vcl_miss subroutine may terminate with calling return() with one of
   the following keywords:
 
-  error code [reason]
+  error(code [, reason])
     Return the specified error code to the client and abandon the request.
 
+  fetch
+    Retrieve the requested object from the backend.  Control will
+    eventually pass to vcl_backend_fetch.
+
   pass
     Switch to pass mode.  Control will eventually pass to vcl_pass.
 
+  restart
+    ?
+
+vcl_backend_fetch
+  ?
+
+  The vcl_backend_fetch subroutine may terminate with calling return()
+  with one of the following keywords:
+
+  abandon
+    ?
+
   fetch
     Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_fetch.
+    eventually pass to vcl_backend_response.
 
-vcl_fetch
+vcl_backend_response
   Called after a document has been successfully retrieved from the backend.
-  
-  The vcl_fetch subroutine may terminate with calling return() with
-  one of the following keywords:
+
+  The vcl_backend_response subroutine may terminate with calling return()
+  with one of the following keywords:
+
+  abandon
+    ?
 
   deliver
     Possibly insert the object into the cache, then deliver it to the
     client.  Control will eventually pass to vcl_deliver.
 
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
-
-  hit_for_pass 
-    Pass in fetch. Passes the object without caching it. This will
-    create a so-called hit_for_pass object which has the side effect
-    that the decision not to cache will be cached. This is to allow
-    would-be uncachable requests to be passed to the backend at the
-    same time. The same logic is not necessary in vcl_recv because
-    this happens before any potential queueing for an object takes
-    place.  Note that the TTL for the hit_for_pass object will be set
-    to what the current value of beresp.ttl is. Control will be
-    handled to vcl_deliver on the current request, but subsequent
-    requests will go directly to vcl_pass based on the hit_for_pass
-    object.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
-    error.
+  retry
+    ?
 
 vcl_deliver
   Called before a cached object is delivered to the client.
-  
+
   The vcl_deliver subroutine may terminate with one of the following
   keywords:
 
@@ -559,23 +555,23 @@ vcl_deliver
     Deliver the object to the client.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_error
-  Called when we hit an error, either explicitly or implicitly due to 
+  Called when we hit an error, either explicitly or implicitly due to
   backend or internal errors.
 
   The vcl_error subroutine may terminate by calling return with one of
   the following keywords:
- 
+
   deliver
     Deliver the error object to the client.
 
   restart
-    Restart the transaction. Increases the restart counter. If the number 
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation 
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
 vcl_fini
@@ -587,7 +583,6 @@ vcl_fini
   ok
     Normal return, VCL will be discarded.
 
-
 If one of these subroutines is left undefined or terminates without
 reaching a handling decision, control will be handed over to the
 builtin default.  See the EXAMPLES section for a listing of the
@@ -601,30 +596,29 @@ appear in the source.
 The default versions distributed with Varnish will be implicitly
 concatenated as a last resort at the end.
 
-Example:
-::
+Example::
 
-	# in file "main.vcl"
-	include "backends.vcl";
-	include "ban.vcl";
-
-	# in file "backends.vcl"
-	sub vcl_recv {
-	  if (req.http.host ~ "(?i)example.com") {
-	    set req.backend = foo;
-	  } elsif (req.http.host ~ "(?i)example.org") {
-	    set req.backend = bar;
-	  }
+    # in file "main.vcl"
+    include "backends.vcl";
+    include "ban.vcl";
+
+    # in file "backends.vcl"
+    sub vcl_recv {
+        if (req.http.host ~ "(?i)example.com") {
+            set req.backend_hint = foo;
+	} elsif (req.http.host ~ "(?i)example.org") {
+	    set req.backend_hint = bar;
 	}
+    }
 
-	# in file "ban.vcl"
-	sub vcl_recv {
-	  if (client.ip ~ admin_network) {
+    # in file "ban.vcl"
+    sub vcl_recv {
+        if (client.ip ~ admin_network) {
 	    if (req.http.Cache-Control ~ "no-cache") {
-	      ban("req.url ~ " + req.url);
-	    }
-	  }
-	}
+                ban("req.url ~ " + req.url);
+            }
+        }
+    }
 
 Variables
 ~~~~~~~~~
@@ -634,7 +628,7 @@ made available to the handler subroutines through global variables.
 
 The following variables are always available:
 
-now   
+now
   The current time, in seconds since the epoch. When used in string context
   it returns a formatted string.
 
@@ -654,26 +648,22 @@ client.ip
 client.identity
   Identification of the client, used to load balance in the client director.
 
-server.hostname
-  The host name of the server.
-
-server.identity 
-  The identity of the server, as set by the -i
-  parameter.  If the -i parameter is not passed to varnishd,
-  server.identity will be set to the name of the instance, as
-  specified by the -n parameter.
-
 server.ip
   The IP address of the socket on which the client connection was received.
 
-server.port
-  The port number of the socket on which the client connection was received.
+server.hostname
+  The host name of the server.
+
+server.identity
+  The identity of the server, as set by the -i parameter.
+  If the -i parameter is not passed to varnishd, server.identity will be
+  set to the name of the instance, as specified by the -n parameter.
 
 req.method
   The request type (e.g. "GET", "HEAD").
 
 req.request
-  Outdated way to spell req.method.
+  For backward compatibility. Same as req.method.
 
 req.url
   The requested URL.
@@ -681,31 +671,17 @@ req.url
 req.proto
   The HTTP protocol version used by the client.
 
-req.backend
-  The backend to use to service the request.
-
-req.backend.healthy
-  Whether the backend is healthy or not. Requires an active probe to be set
-  on the backend.
-
 req.http.header
   The corresponding HTTP header.
 
-req.hash_always_miss
-  Force a cache miss for this request. If set to true Varnish will disregard
-  any existing objects and always (re)fetch from the backend.
-
-req.hash_ignore_busy
-  Ignore any busy object during cache lookup. You would want to do 
-  this if you have two server looking up content from each other to 
-  avoid potential deadlocks.
+req.backend_hint
+  Set bereq.backend to this if we attempt to fetch. This variable is
+  a convenience so the overall policy can be set up once and for all
+  in vcl_recv.
 
 req.can_gzip
   Does the client accept the gzip transfer encoding.
 
-req.restarts
-  A count of how many times this request has been restarted.
-
 req.esi
   Boolean. Set to false to disable ESI processing regardless of any
   value in beresp.do_esi. Defaults to true. This variable is subject
@@ -714,8 +690,20 @@ req.esi
 req.esi_level
   A count of how many levels of ESI requests we're currently at.
 
-req.grace
-  Set to a period to enable grace.
+req.hash_always_miss
+  Force a cache miss for this request. If set to true Varnish will disregard
+  any existing objects and always (re)fetch from the backend.
+
+req.hash_ignore_busy
+  Ignore any busy object during cache lookup. You would want to do
+  this if you have two server looking up content from each other to
+  avoid potential deadlocks.
+
+req.restarts
+  A count of how many times this request has been restarted.
+
+req.ttl
+  ?
 
 req.xid
   Unique ID of this request.
@@ -727,7 +715,7 @@ bereq.method
   The request type (e.g. "GET", "HEAD").
 
 bereq.request
-  Outdated way to spell bereq.method.
+  For backward compatibility. Same as bereq.method.
 
 bereq.url
   The requested URL.
@@ -738,43 +726,29 @@ bereq.proto
 bereq.http.header
   The corresponding HTTP header.
 
+bereq.backend
+  ?
+
+bereq.between_bytes_timeout
+  The time in seconds to wait between each received byte from the
+  backend. Not available in pipe mode.
+
 bereq.connect_timeout
   The time in seconds to wait for a backend connection.
 
 bereq.first_byte_timeout
-  The time in seconds to wait for the first byte from the backend.  Not
+  The time in seconds to wait for the first byte from the backend. Not
   available in pipe mode.
 
-bereq.between_bytes_timeout
-  The time in seconds to wait between each received byte from the
-  backend.  Not available in pipe mode.
+bereq.retries
+  ?
+
+bereq.uncacheable
+  ?
 
 The following variables are available after the requested object has
 been retrieved from the backend, before it is entered into the cache. In
-other words, they are available in vcl_fetch:
-
-beresp.do_stream 
-  Deliver the object to the client directly without fetching the whole
-  object into Varnish. If this request is pass'ed it will not be
-  stored in memory. As of Varnish Cache 3.0 the object will marked as busy
-  as it is delivered so only client can access the object.
-
-beresp.do_esi
-  Boolean. ESI-process the object after fetching it. Defaults to
-  false. Set it to true to parse the object for ESI directives. Will
-  only be honored if req.esi is true.
-
-beresp.do_gzip 
-  Boolean. Gzip the object before storing it. Defaults to false. When
-  http_gzip_support is on Varnish will request already compressed
-  content from the backend and as such compression in Varnish is not needed.
-
-beresp.do_gunzip
-  Boolean. Unzip the object before storing it in the cache.  Defaults
-  to false.
-
-beresp.http.header
-  The corresponding HTTP header.
+other words, they are available in vcl_backend_response:
 
 beresp.proto
   The HTTP protocol version used the backend replied with.
@@ -782,31 +756,57 @@ beresp.proto
 beresp.status
   The HTTP status code returned by the server.
 
-beresp.response
+beresp.reason
   The HTTP status message returned by the server.
 
-beresp.ttl
-  The object's remaining time to live, in seconds. beresp.ttl is writable.
+beresp.response
+  For backward compatibility. Same as beresp.reason.
 
-beresp.grace
-  Set to a period to enable grace.
+beresp.http.header
+  The corresponding HTTP header.
 
-beresp.saintmode
-  Set to a period to enable saint mode.
+beresp.backend.ip
+  IP of the backend this response was fetched from.
 
 beresp.backend.name
   Name of the backend this response was fetched from.
 
-beresp.backend.ip
-  IP of the backend this response was fetched from.
+beresp.do_esi
+  Boolean. ESI-process the object after fetching it. Defaults to
+  false. Set it to true to parse the object for ESI directives. Will
+  only be honored if req.esi is true.
+
+beresp.do_gunzip
+  Boolean. Unzip the object before storing it in the cache. Defaults to
+  false.
+
+beresp.do_gzip
+  Boolean. Gzip the object before storing it. Defaults to false. When
+  http_gzip_support is on Varnish will request already compressed
+  content from the backend and as such compression in Varnish is not needed.
+
+beresp.do_stream
+  Deliver the object to the client directly without fetching the whole
+  object into varnish. If this request is pass'ed it will not be
+  stored in memory. As of Varnish Cache 3.0 the object will marked as busy
+  as it is delivered so only client can access the object.
+
+beresp.grace
+  Set to a period to enable grace.
 
-beresp.backend.port
-  Port of the backend this response was fetched from.
+beresp.keep
+  ?
 
-beresp.storage
-  Set to force Varnish to save this object to a particular storage
+beresp.storage_hint
+  Hint to Varnish that you want to save this object to a particular storage
   backend.
 
+beresp.ttl
+  The object's remaining time to live, in seconds. beresp.ttl is writable.
+
+beresp.uncacheable
+  ?
+
 After the object is entered into the cache, the following (mostly
 read-only) variables are available when the object has been located in
 cache, typically in vcl_hit, or when constructing a synthetic reply in
@@ -818,33 +818,35 @@ obj.proto
 obj.status
   The HTTP status code returned by the server.
 
-obj.response
+obj.reason
   The HTTP status message returned by the server.
 
-obj.ttl
-  The object's remaining time to live, in seconds. obj.ttl is writable.
+obj.response
+  For backward compatibility. Same as obj.reason.
 
-obj.last_use
-  The approximate time elapsed since the object was last requests, in
-  seconds. This variable is also available in vcl_deliver.
+obj.http.header
+  The corresponding HTTP header.
+
+obj.grace
+  The object's grace period in seconds. obj.grace is writable.
 
 obj.hits
-  The approximate number of times the object has been delivered. A value 
+  The approximate number of times the object has been delivered. A value
   of 0 indicates a cache miss. This variable is also available in
   vcl_deliver.
 
-obj.grace
-  The object's grace period in seconds. obj.grace is writable.
+obj.keep
+  ?
 
-obj.http.header
-  The corresponding HTTP header.
+obj.last_use
+  The approximate time elapsed since the object was last requests, in
+  seconds. This variable is also available in vcl_deliver.
 
-The following variables are available while determining the hash key
-of an object:
+obj.ttl
+  The object's remaining time to live, in seconds. obj.ttl is writable.
 
-req.hash
-  The hash key used to refer to an object in the cache.  Used when
-  both reading from and writing to the cache.
+obj.uncacheable
+  ?
 
 The following variables are available while preparing a response to the client:
 
@@ -854,32 +856,33 @@ resp.proto
 resp.status
   The HTTP status code that will be returned.
 
-resp.response
+resp.reason
   The HTTP status message that will be returned.
 
+resp.response
+  For backward compatibility. Same as resp.reason.
+
 resp.http.header
   The corresponding HTTP header.
 
-Values may be assigned to variables using the set keyword:
-::
+Values may be assigned to variables using the set keyword::
 
-  sub vcl_recv {
-    # Normalize the Host: header
-    if (req.http.host ~ "(?i)^(www.)?example.com$") {
-      set req.http.host = "www.example.com";
+    sub vcl_recv {
+        # Normalize the Host: header
+        if (req.http.host ~ "(?i)^(www.)?example.com$") {
+            set req.http.host = "www.example.com";
+        }
     }
-  }
 
-HTTP headers can be removed entirely using the remove keyword:
-::
+HTTP headers can be removed entirely using the unset keyword::
 
-  sub vcl_fetch {
-    # Don't cache cookies
-    remove beresp.http.Set-Cookie;
-  }
+    sub vcl_backend_response {
+        # Don't cache cookies
+        unset beresp.http.Set-Cookie;
+    }
 
-Grace and saint mode
---------------------
+Grace
+-----
 
 If the backend takes a long time to generate an object there is a risk
 of a thread pile up.  In order to prevent this you can enable *grace*.
@@ -891,134 +894,116 @@ object will be kept up to two minutes past their expiration time or a
 fresh object is generated.
 ::
 
-  sub vcl_recv {
-    set req.grace = 2m;
-  }
-  sub vcl_fetch {
-    set beresp.grace = 2m;
-  }
-
-Saint mode is similar to grace mode and relies on the same
-infrastructure but functions differently. You can add VCL code to
-vcl_fetch to see whether or not you *like* the response coming from
-the backend. If you find that the response is not appropriate you can
-set beresp.saintmode to a time limit and call *restart*. Varnish will
-then retry other backends to try to fetch the object again. 
-
-If there are no more backends or if you hit *max_restarts* and we have
-an object that is younger than what you set beresp.saintmode to be
-Varnish will serve the object, even if it is stale.
+    sub vcl_backend_response {
+        set beresp.grace = 2m;
+    }
 
 EXAMPLES
 ========
 
 The following code is the equivalent of the default configuration with
 the backend address set to "backend.example.com" and no backend port
-specified:
-::
+specified::
 
-  backend default {
-   .host = "backend.example.com";
-   .port = "http";
-  }
+    backend default {
+        .host = "backend.example.com";
+        .port = "http";
+    }
 
 .. include:: ../../../bin/varnishd/builtin.vcl
   :literal:
 
 The following example shows how to support multiple sites running on
 separate backends in the same Varnish instance, by selecting backends
-based on the request URL:
-::
+based on the request URL::
 
-  backend www {
-    .host = "www.example.com";
-    .port = "80";
-  }
-  
-  backend images {
-    .host = "images.example.com";
-    .port = "80";
-  }
-  
-  sub vcl_recv {
-    if (req.http.host ~ "(?i)^(www.)?example.com$") {
-      set req.http.host = "www.example.com";
-      set req.backend = www;
-    } elsif (req.http.host ~ "(?i)^images.example.com$") {
-      set req.backend = images;
-    } else {
-      error 404 "Unknown virtual host";
+    backend www {
+        .host = "www.example.com";
+        .port = "80";
+    }
+
+    backend images {
+        .host = "images.example.com";
+        .port = "80";
+    }
+
+    sub vcl_recv {
+        if (req.http.host ~ "(?i)^(www.)?example.com$") {
+            set req.http.host = "www.example.com";
+            set req.backend_hint = www;
+        } elsif (req.http.host ~ "(?i)^images.example.com$") {
+            set req.backend_hint = images;
+        } else {
+            return (error(404, "Unknown virtual host"));
+        }
     }
-  }
 
 The following snippet demonstrates how to force a minimum TTL for
 all documents.  Note that this is not the same as setting the
 default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL:
-::
-  
-  import std; # needed for std.log
+which the backend did not specify a TTL::
+
+    import std; # needed for std.log
 
-  sub vcl_fetch {
-    if (beresp.ttl < 120s) {
-      std.log("Adjusting TTL");
-      set beresp.ttl = 120s;
+    sub vcl_backend_response {
+        if (beresp.ttl < 120s) {
+            std.log("Adjusting TTL");
+            set beresp.ttl = 120s;
+        }
     }
-  }
 
 The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present:
-::
+documents even when cookies are present::
 
-  sub vcl_recv {
-    if (req.method == "GET" && req.http.cookie) {
-       return(lookup);
+    sub vcl_recv {
+        if (req.method == "GET" && req.http.cookie) {
+            return (hash);
+        }
+    }
+
+    sub vcl_backend_response {
+        if (beresp.http.Set-Cookie) {
+            return (deliver);
+        }
     }
-  }
-  
-  sub vcl_fetch {
-    if (beresp.http.Set-Cookie) {
-       return(deliver);
-   }
-  }
 
 The following code implements the HTTP PURGE method as used by Squid
-for object invalidation:
-::
+for object invalidation::
 
-  acl purge {
-    "localhost";
-    "192.0.2.1"/24;
-  }
+    acl purge {
+        "localhost";
+        "192.0.2.1"/24;
+    }
 
-  sub vcl_recv {
-    if (req.method == "PURGE") {
-      if (!client.ip ~ purge) {
-        error 405 "Not allowed.";
-      }
-      return(lookup);
+    sub vcl_recv {
+        if (req.method == "PURGE") {
+            if (!client.ip ~ purge) {
+                return (error(405, "Not allowed."));
+            }
+            return (hash);
+        }
     }
-  }
 
-  sub vcl_hit {
-    if (req.method == "PURGE") {
-      purge;
-      error 200 "Purged.";
+    sub vcl_hit {
+        if (req.method == "PURGE") {
+            purge;
+            return (error(200, "Purged."));
+        }
     }
-  }
 
-  sub vcl_miss {
-    if (req.method == "PURGE") {
-      purge;
-      error 200 "Purged.";
+    sub vcl_miss {
+        if (req.method == "PURGE") {
+            purge;
+            return (error(200, "Purged."));
+        }
     }
-  }
 
 SEE ALSO
 ========
 
 * varnishd(1)
-* vmod_std(7)
+* vmod_directors(3)
+* vmod_std(3)
 
 HISTORY
 =======



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] eeb8d58 Various fixes
Message-ID: <E1WO1sH-0007DI-7E@project.varnish-software.com>


commit eeb8d58715b023fdd5938790931bafd97c52f662
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 08:03:57 2014 +0000

    Various fixes
    
    Spelling. While here also fix indentation and make sure we are using
    the right name for the variables and not the backward compat ones.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 3a723e3..00f5f6c 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -105,7 +105,7 @@ sub vcl_hit {
 	return (deliver);
     }
     if (obj.ttl + obj.grace > 0s) {
-	// Object is in grace, delive it
+	// Object is in grace, deliver it
 	// Automatically triggers a background fetch
 	return (deliver);
     }
@@ -132,7 +132,7 @@ sub vcl_backend_response {
 		/*
 		 * Mark as "Hit-For-Pass" for the next 2 minutes
 		 */
-		set beresp.ttl = 120 s;
+		set beresp.ttl = 120s;
 		set beresp.uncacheable = true;
     }
     return (deliver);
@@ -150,7 +150,7 @@ sub vcl_deliver {
 }
 
 /*
- * We can come here "invisibly" with the following errors:  413, 417 & 503
+ * We can come here "invisibly" with the following errors: 413, 417 & 503
  */
 sub vcl_error {
     set obj.http.Content-Type = "text/html; charset=utf-8";
@@ -161,11 +161,11 @@ sub vcl_error {
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
   <head>
-    <title>"} + obj.status + " " + obj.response + {"</title>
+    <title>"} + obj.status + " " + obj.reason + {"</title>
   </head>
   <body>
-    <h1>Error "} + obj.status + " " + obj.response + {"</h1>
-    <p>"} + obj.response + {"</p>
+    <h1>Error "} + obj.status + " " + obj.reason + {"</h1>
+    <p>"} + obj.reason + {"</p>
     <h3>Guru Meditation:</h3>
     <p>XID: "} + req.xid + {"</p>
     <hr>
@@ -177,9 +177,9 @@ sub vcl_error {
 }
 
 sub vcl_init {
-	return (ok);
+    return (ok);
 }
 
 sub vcl_fini {
-	return (ok);
+    return (ok);
 }



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] ed27a26 Improve manpage formatting
Message-ID: <E1WO1sH-0007DS-8l@project.varnish-software.com>


commit ed27a2634576f9911e01b23db05065833a924abe
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 08:44:13 2014 +0000

    Improve manpage formatting
    
    Add a second parameter to $Module to specify the manpage section
    and change how the description is printed.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 959f203..be6cff6 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -93,11 +93,12 @@ class token(object):
 #######################################################################
 
 class vmod(object):
-	def __init__(self, nam, dnam):
+	def __init__(self, nam, dnam, sec):
 		if not is_c_name(nam):
 			raise Exception("Module name '%s' is illegal" % nam)
 		self.nam = nam
 		self.dnam = dnam
+		self.sec = sec
 		self.init = None
 		self.funcs = list()
 		self.objs = list()
@@ -243,11 +244,18 @@ class vmod(object):
 		self.doc_str.append(l)
 
 	def doc_dump(self, fo, suf):
-		i = "vmod_" + self.nam + " -- " + self.dnam
+		i = "vmod_" + self.nam
 		fo.write("=" * len(i) + "\n")
 		fo.write(i + "\n")
 		fo.write("=" * len(i) + "\n")
 		fo.write("\n")
+		i = self.dnam
+		fo.write("-" * len(i) + "\n")
+		fo.write(i + "\n")
+		fo.write("-" * len(i) + "\n")
+		fo.write("\n")
+		fo.write(":Manual section: %s\n" % self.sec)
+		fo.write("\n")
 		fo.write("SYNOPSIS\n")
 		fo.write("========\n")
 		fo.write("\n")
@@ -529,11 +537,12 @@ def parse_enum2(tl):
 
 def parse_module(tl):
 	nm = tl.get_token().str
+	sec = tl.get_token().str
 	s = ""
 	while len(tl.tl) > 0:
 		s += " " + tl.get_token().str
 	dnm = s[1:]
-	return vmod(nm, dnm)
+	return vmod(nm, dnm, sec)
 
 #######################################################################
 #



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 962aa94 Pass the section
Message-ID: <E1WO1sH-0007Dc-A0@project.varnish-software.com>


commit 962aa9422cb50f8f1dd5180f226ebeadab56f2c9
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 08:51:18 2014 +0000

    Pass the section

diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 4f74a24..71a98d6 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -25,7 +25,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-$Module debug Development, test and debug
+$Module debug 3 Development, test and debug
 
 DESCRIPTION
 ===========
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 37d8f56..609fc77 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -25,7 +25,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-$Module directors Backend traffic directors
+$Module directors 3 Backend traffic directors
 
 $Object round_robin()
 $Method VOID .add_backend(BACKEND)
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 1a66dbd..a777801 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -25,7 +25,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-$Module std Varnish Standard Module
+$Module std 3 Varnish Standard Module
 
 DESCRIPTION
 ===========



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 6131dc9 Hook the vmod_directors manpage
Message-ID: <E1WO1sH-0007Dl-BA@project.varnish-software.com>


commit 6131dc97727b6e5029d2ad771b53f3a6129a75ae
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 09:04:52 2014 +0000

    Hook the vmod_directors manpage

diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index d44839d..7b66787 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -6,12 +6,14 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/bin/varnishd \
 	-I$(top_builddir)/include
 
+dist_man_MANS = vmod_directors.3
+
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_directors
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
 vmod_LTLIBRARIES = libvmod_directors.la
 
-libvmod_directors_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared -rpath /nowhere
+libvmod_directors_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared
 
 libvmod_directors_la_SOURCES = \
 	vdir.c \
@@ -36,3 +38,13 @@ EXTRA_DIST = vmod.vcc
 CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
 	$(builddir)/vmod_directors.rst \
 	$(builddir)/vmod_directors.man.rst
+
+vmod_directors.3: $(vmod_srcdir)/vmod_directors.man.rst
+if HAVE_RST2MAN
+	${RST2MAN} $? $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 5c3ce45 Use the generated rst when creating the manpage
Message-ID: <E1WO1sH-0007Dv-Cr@project.varnish-software.com>


commit 5c3ce452c58e0c7cb8c55cea8071b92249efb013
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 09:05:33 2014 +0000

    Use the generated rst when creating the manpage

diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index cd8aebf..2e27af5 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -37,7 +37,7 @@ CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
 	$(builddir)/vmod_std.rst \
 	$(builddir)/vmod_std.man.rst
 
-vmod_std.3: $(top_srcdir)/doc/sphinx/reference/vmod_std.rst
+vmod_std.3: $(vmod_srcdir)/vmod_std.man.rst
 if HAVE_RST2MAN
 	${RST2MAN} $? $@
 else



From phk at FreeBSD.org  Thu Mar 13 09:24:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 70d557a Fix test-case to match updated error from compiler.
Message-ID: <E1WO1sH-0007E5-E2@project.varnish-software.com>


commit 70d557a65de408cb2177bb6047dbd305f073d330
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 17 09:28:55 2014 +0000

    Fix test-case to match updated error from compiler.
    
    Varnishtest not run by:		fgs

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 7a00d95..69e99e5 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -73,7 +73,7 @@ varnish v1 -vcl {
 
 }
 
-varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 's', 'm', 'h' and 'd'} {
+varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { set obj.ttl = 1. k; }
 }



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] e85e0aa Ensure rst files are built before generating the
 manpages
Message-ID: <E1WO1sH-0007EH-FV@project.varnish-software.com>


commit e85e0aae8197bdcec0aa15dae4b4c41753f08441
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 10:01:14 2014 +0000

    Ensure rst files are built before generating the manpages

diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 7b66787..087ecbb 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -28,7 +28,7 @@ nodist_libvmod_directors_la_SOURCES = \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
+vdir.lo fall_back.lo hash.lo random.lo round_robin.lo vmod_directors.rst vmod_directors.man.rst: vcc_if.h
 
 vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index 2e27af5..bd66ab1 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -26,7 +26,7 @@ nodist_libvmod_std_la_SOURCES = \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
+vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo vmod_std.rst vmod_std.man.rst: vcc_if.h
 
 vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc



From phk at FreeBSD.org  Thu Mar 13 09:24:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] c14936b Fix uintmax_t printf format arguments.
Message-ID: <E1WO1sH-0007EN-H6@project.varnish-software.com>


commit c14936bcfff366c8aae97d159d5fcd98050a3058
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 17 11:05:45 2014 +0000

    Fix uintmax_t printf format arguments.
    
    Fixes	#1432

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 0971c10..10f6d51 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -65,7 +65,7 @@ do_xml_cb(void *priv, const struct VSC_point * const pt)
 	if (strcmp(sec->fantom->ident, ""))
 		printf("\t\t<ident>%s</ident>\n", sec->fantom->ident);
 	printf("\t\t<name>%s</name>\n", pt->desc->name);
-	printf("\t\t<value>%ju</value>\n", val);
+	printf("\t\t<value>%ju</value>\n", (uintmax_t)val);
 	printf("\t\t<flag>%c</flag>\n", pt->desc->flag);
 	printf("\t\t<description>%s</description>\n", pt->desc->sdesc);
 	printf("\t</stat>\n");
@@ -119,7 +119,7 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 	if (strcmp(sec->fantom->ident, "")) printf("\"ident\": \"%s\", ",
 	    sec->fantom->ident);
 
-	printf("\"value\": %ju, ", val);
+	printf("\"value\": %ju, ", (uintmax_t)val);
 
 	printf("\"flag\": \"%c\", ", pt->desc->flag);
 	printf("\"description\": \"%s\"", pt->desc->sdesc);
@@ -180,9 +180,11 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 		op->pad = i + 1;
 	printf("%*.*s", op->pad - i, op->pad - i, "");
 	if (pt->desc->flag == 'a' || pt->desc->flag == 'c')
-		printf("%12ju %12.2f %s\n", val, val / op->up, pt->desc->sdesc);
+		printf("%12ju %12.2f %s\n",
+		    (uintmax_t)val, val / op->up, pt->desc->sdesc);
 	else
-		printf("%12ju %12s %s\n", val, ".  ", pt->desc->sdesc);
+		printf("%12ju %12s %s\n",
+		    (uintmax_t)val, ".  ", pt->desc->sdesc);
 	return (0);
 }
 



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] a5f678f Revert 5c3ce452, e85e0aae and 6131dc97 for now
Message-ID: <E1WO1sH-0007EY-IO@project.varnish-software.com>


commit a5f678f47328c100f646a4162551d1c8dfdfd12a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 12:23:45 2014 +0000

    Revert 5c3ce452, e85e0aae and 6131dc97 for now
    
    They break the build if parallelised and make jenkins upset

diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 087ecbb..6a85ad5 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -6,8 +6,6 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/bin/varnishd \
 	-I$(top_builddir)/include
 
-dist_man_MANS = vmod_directors.3
-
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_directors
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
@@ -28,7 +26,7 @@ nodist_libvmod_directors_la_SOURCES = \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vdir.lo fall_back.lo hash.lo random.lo round_robin.lo vmod_directors.rst vmod_directors.man.rst: vcc_if.h
+vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
 vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
@@ -38,13 +36,3 @@ EXTRA_DIST = vmod.vcc
 CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
 	$(builddir)/vmod_directors.rst \
 	$(builddir)/vmod_directors.man.rst
-
-vmod_directors.3: $(vmod_srcdir)/vmod_directors.man.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index bd66ab1..cd8aebf 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -26,7 +26,7 @@ nodist_libvmod_std_la_SOURCES = \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo vmod_std.rst vmod_std.man.rst: vcc_if.h
+vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
 
 vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
@@ -37,7 +37,7 @@ CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
 	$(builddir)/vmod_std.rst \
 	$(builddir)/vmod_std.man.rst
 
-vmod_std.3: $(vmod_srcdir)/vmod_std.man.rst
+vmod_std.3: $(top_srcdir)/doc/sphinx/reference/vmod_std.rst
 if HAVE_RST2MAN
 	${RST2MAN} $? $@
 else



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 1fbe580 Move vcl_hash close to vcl_recv
Message-ID: <E1WO1sH-0007Eh-JN@project.varnish-software.com>


commit 1fbe580804e0c58f5f541082b049c9a27d0187ac
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 12:28:17 2014 +0000

    Move vcl_hash close to vcl_recv

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 7e00e43..8cb7439 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -415,6 +415,15 @@ vcl_recv
   purge
     ?
 
+vcl_hash
+  You may call hash_data() on the data you would like to add to the hash.
+
+  The vcl_hash subroutine may terminate with calling return() with one of
+  the following keywords:
+
+  lookup
+    Proceed.
+
 vcl_pass
   Called upon entering pass mode.  In this mode, the request is passed
   on to the backend, and the backend's response is passed on to the
@@ -462,15 +471,6 @@ vcl_purge
 ..  fetch
 ..    Not implemented.
 
-vcl_hash
-  You may call hash_data() on the data you would like to add to the hash.
-
-  The vcl_hash subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  lookup
-    Proceed.
-
 vcl_hit
   Called after a cache lookup if the requested document was found in the cache.
 



From phk at FreeBSD.org  Thu Mar 13 09:24:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 8c66705 When changing the busyobj we should use broadcast so
 everybody hears.
Message-ID: <E1WO1sH-0007Eq-Kv@project.varnish-software.com>


commit 8c6670580d04f02016b9302445bc97cb6ed1a14d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 17 16:20:00 2014 +0000

    When changing the busyobj we should use broadcast so everybody hears.
    
    Spotted by:	gquintard

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 65a24bf..d6b43ae 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -222,7 +222,7 @@ VBO_extend(struct busyobj *bo, ssize_t l)
 	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 	st->len += l;
 	bo->fetch_obj->len += l;
-	AZ(pthread_cond_signal(&bo->cond));
+	AZ(pthread_cond_broadcast(&bo->cond));
 	Lck_Unlock(&bo->mtx);
 }
 
@@ -248,7 +248,7 @@ VBO_setstate(struct busyobj *bo, enum busyobj_state_e next)
 	VSLb(bo->vsl, SLT_Debug, "XXX BOS: %d -> %d", bo->state, next);
 	assert(next > bo->state);
 	bo->state = next;
-	AZ(pthread_cond_signal(&bo->cond));
+	AZ(pthread_cond_broadcast(&bo->cond));
 	Lck_Unlock(&bo->mtx);
 }
 



From phk at FreeBSD.org  Thu Mar 13 09:24:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 8f26bf5 Bump the cli_timeout up to 60 seconds to reduce false
 alarms.
Message-ID: <E1WO1sH-0007Ez-MS@project.varnish-software.com>


commit 8f26bf57a3130ecf3efa75ad3a422500619ed8aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 17 19:07:27 2014 +0000

    Bump the cli_timeout up to 60 seconds to reduce false alarms.
    
    There seems to be a tendency for systems to trip up the previous
    10 second cli_timeout during nightly housekeeping operations, possibly
    log rotation running out of cron.

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 578bd85..b6ae75c 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -286,7 +286,7 @@ struct parspec mgt_parspec[] = {
 		"Timeout for the childs replies to CLI requests from "
 		"the mgt_param.",
 		0,
-		"10", "seconds" },
+		"60", "seconds" },
 	{ "ping_interval", tweak_uint, &mgt_param.ping_interval,
 		"0", NULL,
 		"Interval between pings from parent to child.\n"



From fgsch at lodoss.net  Thu Mar 13 09:24:25 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 1e0e7b4 Remove backward compatibility
Message-ID: <E1WO1sH-0007FA-Nk@project.varnish-software.com>


commit 1e0e7b4cb1dc4330ee0cacae89b971c4c1caaa1a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 17 21:04:38 2014 +0000

    Remove backward compatibility
    
    This kills `remove', `.request' and `.response'.  Use `unset', `.method'
    and `.reason' instead.
    ok phk.

diff --git a/bin/varnishtest/tests/b00018.vtc b/bin/varnishtest/tests/b00018.vtc
index 3989b57..682c19c 100644
--- a/bin/varnishtest/tests/b00018.vtc
+++ b/bin/varnishtest/tests/b00018.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 	    set beresp.http.Foo = "bar";
 	    set beresp.status = 523;
-	    set beresp.response = "not ok";
+	    set beresp.reason = "not ok";
 	    set beresp.uncacheable = true;
 	    set beresp.ttl = 0s;
 	    return (deliver);
diff --git a/bin/varnishtest/tests/b00019.vtc b/bin/varnishtest/tests/b00019.vtc
index 6b33702..597e8b9 100644
--- a/bin/varnishtest/tests/b00019.vtc
+++ b/bin/varnishtest/tests/b00019.vtc
@@ -28,13 +28,13 @@ varnish v1 -vcl+backend {
 	sub vcl_error {
 		if (req.restarts == 2) {
 			set obj.status = 200;
-			set obj.response = "restart=2";
+			set obj.reason = "restart=2";
 		} elsif (req.restarts > 2) {
 			set obj.status = 501;
-			set obj.response = "restart>2";
+			set obj.reason = "restart>2";
 		} elsif (req.restarts < 2) {
 			set obj.status = 500;
-			set obj.response = "restart<2";
+			set obj.reason = "restart<2";
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/b00035.vtc b/bin/varnishtest/tests/b00035.vtc
deleted file mode 100644
index dbe8a6f..0000000
--- a/bin/varnishtest/tests/b00035.vtc
+++ /dev/null
@@ -1,26 +0,0 @@
-varnishtest "{be}req.request compat check"
-
-server s1 {
-	rxreq
-	expect req.method == "GETABCD"
-	txresp
-} -start
-
-varnish v1 -vcl+backend {
-
-	sub vcl_recv {
-		set req.method = req.request + "A";
-		set req.request = req.method + "B";
-		return (pass);
-	}
-
-	sub vcl_backend_fetch {
-		set bereq.method = bereq.request + "C";
-		set bereq.request = bereq.method + "D";
-	}
-} -start
-
-client c1 {
-	txreq
-	rxresp
-} -run
diff --git a/bin/varnishtest/tests/b00036.vtc b/bin/varnishtest/tests/b00036.vtc
index cb0b6e5..4cd6e17 100644
--- a/bin/varnishtest/tests/b00036.vtc
+++ b/bin/varnishtest/tests/b00036.vtc
@@ -9,7 +9,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		if (req.request == "PURGE") {
+		if (req.method == "PURGE") {
 			return (purge);
 		}
 	}
diff --git a/bin/varnishtest/tests/r00292.vtc b/bin/varnishtest/tests/r00292.vtc
index f6bc807..4972188 100644
--- a/bin/varnishtest/tests/r00292.vtc
+++ b/bin/varnishtest/tests/r00292.vtc
@@ -18,8 +18,8 @@ server s1 {
 varnish v1 -vcl+backend {
 
 	sub vcl_recv {
-		remove req.http.hdr1;
-		remove req.http.hdr5;
+		unset req.http.hdr1;
+		unset req.http.hdr5;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00494.vtc b/bin/varnishtest/tests/r00494.vtc
index 31a9d6e..cb0bbe8 100644
--- a/bin/varnishtest/tests/r00494.vtc
+++ b/bin/varnishtest/tests/r00494.vtc
@@ -11,7 +11,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.http.bar = beresp.http.foo;
-		remove beresp.http.foo;
+		unset beresp.http.foo;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index 72317de..ccd01b5 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.http.X-Banned == "check") {
-			remove req.http.X-Banned;
+			unset req.http.X-Banned;
 		} elseif (req.restarts == 0) {
 		        set req.http.X-Banned = "check";
 			if (req.http.x-pass) {
diff --git a/bin/varnishtest/tests/v00001.vtc b/bin/varnishtest/tests/v00001.vtc
index 4f77f25..1798ec9 100644
--- a/bin/varnishtest/tests/v00001.vtc
+++ b/bin/varnishtest/tests/v00001.vtc
@@ -1,4 +1,4 @@
-varnishtest "VCL/VRT: url/request/proto/response/status"
+varnishtest "VCL/VRT: url/request/proto/reason/status"
 
 
 server s1 {
@@ -28,14 +28,14 @@ varnish v1 -vcl+backend {
 	}
 	sub vcl_backend_response {
 		set beresp.http.foobar =
-		    beresp.proto + beresp.response + beresp.status;
+		    beresp.proto + beresp.reason + beresp.status;
 		set beresp.proto = "HTTP/1.2";
-		set beresp.response = "For circular files";
+		set beresp.reason = "For circular files";
 		set beresp.status = 903;
 	}
 	sub vcl_deliver {
 		set resp.proto = "HTTP/1.2";
-		set resp.response = "Naah, lets fail it";
+		set resp.reason = "Naah, lets fail it";
 		set resp.status = 904;
 		# XXX should be moved to it's own test
 		set resp.http.x-served-by-hostname = server.hostname;
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 8cb7439..9f384f8 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -662,9 +662,6 @@ server.identity
 req.method
   The request type (e.g. "GET", "HEAD").
 
-req.request
-  For backward compatibility. Same as req.method.
-
 req.url
   The requested URL.
 
@@ -714,9 +711,6 @@ request (either for a cache miss or for pass or pipe mode):
 bereq.method
   The request type (e.g. "GET", "HEAD").
 
-bereq.request
-  For backward compatibility. Same as bereq.method.
-
 bereq.url
   The requested URL.
 
@@ -759,9 +753,6 @@ beresp.status
 beresp.reason
   The HTTP status message returned by the server.
 
-beresp.response
-  For backward compatibility. Same as beresp.reason.
-
 beresp.http.header
   The corresponding HTTP header.
 
@@ -821,9 +812,6 @@ obj.status
 obj.reason
   The HTTP status message returned by the server.
 
-obj.response
-  For backward compatibility. Same as obj.reason.
-
 obj.http.header
   The corresponding HTTP header.
 
@@ -859,9 +847,6 @@ resp.status
 resp.reason
   The HTTP status message that will be returned.
 
-resp.response
-  For backward compatibility. Same as resp.reason.
-
 resp.http.header
   The corresponding HTTP header.
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 01a6243..466bf6b 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -578,11 +578,6 @@ The client's IP address.
 
 # Backwards compatibility:
 aliases = [
-	('req.request',		'req.method'),
-	('bereq.request',	'bereq.method'),
-	('beresp.response',	'beresp.reason'),
-	('resp.response',	'resp.reason'),
-	('obj.response',	'obj.reason'),
 ]
 
 stv_variables = (
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 7e9bf2e..b0f7d98 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -411,7 +411,6 @@ static struct action_table {
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "new",		parse_new, VCL_MET_INIT},
 	{ "purge",		parse_purge, VCL_MET_MISS | VCL_MET_HIT },
-	{ "remove",		parse_unset }, /* backward compatibility */
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },



From phk at FreeBSD.org  Thu Mar 13 09:24:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 603e1cb Add a currently unused vcl_backend_error{} method
Message-ID: <E1WO1sH-0007FK-PU@project.varnish-software.com>


commit 603e1cbd752961c3cdd6fd5fc4f2e279f676ddec
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 18 08:36:26 2014 +0000

    Add a currently unused vcl_backend_error{} method

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 00f5f6c..09c08cc 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -41,6 +41,9 @@
 
 vcl 4.0;
 
+#######################################################################
+# Client side
+
 sub vcl_recv {
     if (req.restarts == 0) {
 	if (req.http.x-forwarded-for) {
@@ -117,27 +120,6 @@ sub vcl_miss {
     return (fetch);
 }
 
-
-sub vcl_backend_fetch {
-    return (fetch);
-}
-
-sub vcl_backend_response {
-    if (beresp.ttl <= 0s ||
-        beresp.http.Set-Cookie ||
-        beresp.http.Surrogate-control ~ "no-store" ||
-        (!beresp.http.Surrogate-Control &&
-          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
-        beresp.http.Vary == "*") {
-		/*
-		 * Mark as "Hit-For-Pass" for the next 2 minutes
-		 */
-		set beresp.ttl = 120s;
-		set beresp.uncacheable = true;
-    }
-    return (deliver);
-}
-
 sub vcl_deliver {
     /*
      * These two write to the stored object causing extra page faults
@@ -176,6 +158,36 @@ sub vcl_error {
     return (deliver);
 }
 
+#######################################################################
+# Backend Fetch
+
+sub vcl_backend_fetch {
+    return (fetch);
+}
+
+sub vcl_backend_response {
+    if (beresp.ttl <= 0s ||
+        beresp.http.Set-Cookie ||
+        beresp.http.Surrogate-control ~ "no-store" ||
+        (!beresp.http.Surrogate-Control &&
+          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
+        beresp.http.Vary == "*") {
+		/*
+		 * Mark as "Hit-For-Pass" for the next 2 minutes
+		 */
+		set beresp.ttl = 120s;
+		set beresp.uncacheable = true;
+    }
+    return (deliver);
+}
+
+sub vcl_backend_error {
+	return (deliver);
+}
+
+#######################################################################
+# Housekeeping
+
 sub vcl_init {
     return (ok);
 }
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 466bf6b..0702465 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -78,6 +78,10 @@ tokens = {
 # Our methods and actions
 
 returns =(
+
+	###############################################################
+	# Client side
+
 	('recv',
 		"C",
 		('error', 'pass', 'pipe', 'hash', 'purge',)
@@ -106,6 +110,18 @@ returns =(
 		"C",
 		('error', 'restart', 'pass', 'fetch', 'deliver',)
 	),
+	('deliver',
+		"C",
+		('restart', 'deliver',)
+	),
+	('error',
+		"C",
+		('restart', 'deliver',)
+	),
+
+	###############################################################
+	# Backend-fetch 
+
 	('backend_fetch',
 		"B",
 		('fetch', 'abandon')
@@ -114,14 +130,14 @@ returns =(
 		"B",
 		('deliver', 'retry', 'abandon')
 	),
-	('deliver',
-		"C",
-		('restart', 'deliver',)
-	),
-	('error',
-		"C",
-		('restart', 'deliver',)
+	('backend_error',
+		"B",
+		('deliver', 'retry')
 	),
+
+	###############################################################
+	# Housekeeping
+
 	('init',
 		"",
 		('ok',)



From lkarsten at varnish-software.com  Thu Mar 13 09:24:25 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 86c21b6 Squash cosmetic compile error on ARMv7l
Message-ID: <E1WO1sH-0007FT-Qi@project.varnish-software.com>


commit 86c21b6f9abeb1aec918b3588642c720facb2014
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Feb 19 14:57:43 2014 +0100

    Squash cosmetic compile error on ARMv7l

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index c7157f4..7434070 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -853,7 +853,7 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 int
 main(int argc, char * const *argv)
 {
-	char opt;
+	signed char opt;
 	char *format = NULL;
 
 	memset(&CTX, 0, sizeof CTX);



From perbu at varnish-software.com  Thu Mar 13 09:24:25 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 79a0da1 Fix for OS X/Darvin
Message-ID: <E1WO1sH-0007Fd-SC@project.varnish-software.com>


commit 79a0da1f306631812fb5c666534a5473afadd34e
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Feb 19 15:16:47 2014 +0100

    Fix for OS X/Darvin

diff --git a/bin/varnishtest/tests/r00310.vtc b/bin/varnishtest/tests/r00310.vtc
index 6a5ce22..38c8a32 100644
--- a/bin/varnishtest/tests/r00310.vtc
+++ b/bin/varnishtest/tests/r00310.vtc
@@ -1,7 +1,7 @@
 varnishtest "Test obj.http.x-cache in vcl_hit"
 
 varnish v1 -errvcl {'obj.http.x-cache': cannot be set in method 'vcl_hit'.} {
-	backend foo { .host = "localhost"; }
+	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_hit {
 		set obj.http.x-cache = "hit";



From perbu at varnish-software.com  Thu Mar 13 09:24:25 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] b161676 remove cruft
Message-ID: <E1WO1sH-0007Fv-Ur@project.varnish-software.com>


commit b161676f4222e946ab7f1554f3b66807916e3b75
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:29:11 2014 +0100

    remove cruft

diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
index db3c7fb..acef0c6 100644
--- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
+++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
@@ -2,9 +2,8 @@
 Put Varnish on port 80
 ----------------------
 
-Until now we've been running with Varnish on a high port, for testing
-purposes. You probably don't want to do this in the long term so let's
-put Varnish on port 80.
+Until now we've been running with Varnish on a high port, which is
+great for testing purposes. Lets put Varnish on the default HTTP port.
 
 First we stop varnish::
 
@@ -16,7 +15,8 @@ Now we need to edit the configuration file that starts Varnish.
 Debian/Ubuntu
 ~~~~~~~~~~~~~
 
-On Debian/Ubuntu this is /etc/default/varnish. In the file you'll find some text that looks like this::
+On Debian/Ubuntu this is /etc/default/varnish. In the file you'll find
+some text that looks like this::
 
   DAEMON_OPTS="-a :6081 \
                -T localhost:6082 \



From perbu at varnish-software.com  Thu Mar 13 09:24:25 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 05d0d53 rewrite for readability
Message-ID: <E1WO1sH-0007Fm-Td@project.varnish-software.com>


commit 05d0d5306e558096d39f57641559ab4f9a08e834
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:28:48 2014 +0100

    rewrite for readability

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 5241575..3f65968 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -8,23 +8,23 @@ clients and tries to answer them from its cache. If it cannot answer
 the request from its cache it will forward the request to the backend,
 fetch the response, store it and deliver it to the client.
 
+When Varnish has a cached response ready it is typically delivered in
+a matter of microseconds, several orders of magnitude faster than your
+typical application server, so you want to make sure to have it answer
+as many of the requests as possible.
+
 Varnish decides whether it can store the content or not based on the
 response it's gets back from the backend. The backend can instruct
 Varnish to cache the content with the HTTP response header
-Cache-Control.
-
-Varnish will be very careful when it encounters cookies, either coming
-from the client or from the origin server. When Varnish sees a
-Set-Cookie header on a response it decides that the object is not
-cacheable. When there is a Cookie header in the request it will also
-refuse to serve a cached object and rather ask the backend for version
-of the object that is tailored to the request.
+Cache-Control. There are a few other conditions where Varnish will not
+cache, the most common one being cookies. Since cookies is a good
+indication that a web object is personlised, Varnish will with it's
+default configuration, not cache it.
 
 This behaviour and most other behaviour can be changed using policies
 written in the Varnish Configuration Language. See the Users Guide
 for more information on how to do that.
 
-
 Performance
 ~~~~~~~~~~~
 



From perbu at varnish-software.com  Thu Mar 13 09:24:25 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:25 +0100
Subject: [4.0] 74bc2ec Remove commentary
Message-ID: <E1WO1sH-0007G6-Vm@project.varnish-software.com>


commit 74bc2ec05e2b7d54c6b792c714a8839a4eb57063
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:39:04 2014 +0100

    Remove commentary

diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 7b909c9..ced7723 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -4,7 +4,6 @@
 The Varnish Users Guide
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-(The plan for ...)
 The Varnish documentation consists of three main documents:
 
 * :ref:`tutorial-index` explains the basics and gets you started with Varnish.



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] f180ee4 Remove commentary
Message-ID: <E1WO1sI-0007GF-12@project.varnish-software.com>


commit f180ee4a6f440e1870c3e8df67f81a29ffad11d3
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:39:18 2014 +0100

    Remove commentary

diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index 79b4c06..18b4a1d 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -105,4 +105,3 @@ Yes, we've all been there, haven't we?
 
 When things go wrong :ref:`users_trouble` will hopefully be of some help.
 
-And now, lets put som substance on this skeleton outline...



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 840f5cf spelling
Message-ID: <E1WO1sI-0007GR-2h@project.varnish-software.com>


commit 840f5cf7cc84115946b8268fe7de98fd106ea51a
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:39:35 2014 +0100

    spelling

diff --git a/doc/sphinx/users-guide/params.rst b/doc/sphinx/users-guide/params.rst
index d3a10af..a283d51 100644
--- a/doc/sphinx/users-guide/params.rst
+++ b/doc/sphinx/users-guide/params.rst
@@ -3,7 +3,7 @@
 Parameters
 ----------
 
-Varnish Cache has a set of parameter that affect its behaviour and
+Varnish Cache has a set of parameters that affect its behaviour and
 performance. Most of these parameters can be set on the Varnish
 command line (through varnishadm) using the param.set keyword.
 



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 794e231 Add reference to ACLs
Message-ID: <E1WO1sI-0007GY-4T@project.varnish-software.com>


commit 794e23178ac91d065f75f14ad17bb1492ce2b73c
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:40:15 2014 +0100

    Add reference to ACLs

diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index 16ab277..dfcfc6d 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -223,6 +223,5 @@ to various kinds of attacks and subversive activities.
 
 If you have "administrative" HTTP requests, for instance PURGE
 requests, we strongly recommend that you restrict them to trusted
-IP numbers/nets using VCL's Access Control Lists.
+IP numbers/nets using VCL's :ref:`vcl_syntax_acl`.
 
-.. (XXX: missing ref to ACL)



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 5a68ff7 add label
Message-ID: <E1WO1sI-0007Gj-5u@project.varnish-software.com>


commit 5a68ff7e4e9744decb01566bd1ccac8c6734e476
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:40:56 2014 +0100

    add label

diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index c979bb5..14e07ed 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -25,6 +25,8 @@ including ", newline and other control characters except for the NUL
 (0x00) character. If you really want NUL characters in a string there
 is a VMOD that makes it possible to create such strings.
 
+.. _vcl_syntax_acl:
+
 Access control lists (ACLs)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] bdcda11 remove redundant intro
Message-ID: <E1WO1sI-0007Gr-70@project.varnish-software.com>


commit bdcda11174040a31de91939cc3523edba21c56e1
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 19:41:24 2014 +0100

    remove redundant intro

diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index 53d99e3..0e56c3a 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -34,7 +34,6 @@ code commented out in default.vcl that ships with Varnish Cache.
 .. toctree::
    :maxdepth: 1
 
-   vcl-intro
    vcl-syntax
    vcl-built-in-subs
    vcl-variables



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 30f2ac8 Fixed some things. Lots more to do on this one
Message-ID: <E1WO1sI-0007H1-8s@project.varnish-software.com>


commit 30f2ac823b6ce0a8d4d15767de331322cdfa72ef
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Feb 20 20:23:08 2014 +0100

    Fixed some things. Lots more to do on this one

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 0ab6ced..2202af0 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -1,31 +1,26 @@
 
 .. _vcl-built-in-subs:
 
-Built in subroutines
---------------------
-
-
-vcl_init
-~~~~~~~~
+.. XXX This document needs substational review.
 
- Called when VCL is loaded, before any requests pass through it.
- Typically used to initialize VMODs.
 
-  return() values:
+Built in subroutines
+--------------------
 
-  ok
-    Normal return, VCL continues loading.
 
 vcl_recv
 ~~~~~~~~
 
- Called at the beginning of a request, after the complete request has
- been received and parsed.  Its purpose is to decide whether or not
- to serve the request, how to do it, and, if applicable, which backend
- to use.
+Called at the beginning of a request, after the complete request has
+been received and parsed.  Its purpose is to decide whether or not to
+serve the request, how to do it, and, if applicable, which backend to
+use.
 
- The vcl_recv subroutine may terminate with calling ``return()`` on one of
- the following keywords:
+It is also used to modify the request, something you'll probably find
+yourself doing frequently. 
+
+The vcl_recv subroutine may terminate with calling ``return()`` on one
+of the following keywords:
 
   error code [reason]
     Return the specified error code to the client and abandon the request.
@@ -43,71 +38,22 @@ vcl_recv
     to ``GET`` regardless of the value of ``req.method``.
 
 
-vcl_pipe
-~~~~~~~~
 
- Called upon entering pipe mode.  In this mode, the request is passed
- on to the backend, and any further data from either client or
- backend is passed on unaltered until either end closes the
- connection.
+vcl_backend_fetch
+~~~~~~~~~~~~~~~~~
 
- The vcl_pipe subroutine may terminate with calling return() with one of
- the following keywords:
+Called before sending the backend request. In this subroutine you
+typically alter the request before it gets to the backend.
 
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
+.. XXX Return statements?
 
-  pipe
-    Proceed with pipe mode.
 
-vcl_pass
-~~~~~~~~
-
- Called upon entering pass mode.  In this mode, the request is passed
- on to the backend, and the backend's response is passed on to the
- client, but is not entered into the cache.  Subsequent requests
- submitted over the same client connection are handled normally.
-
- The vcl_pass subroutine may terminate with calling return() with one
- of the following keywords:
-
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
-
-  pass
-    Proceed with pass mode.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_miss
-~~~~~~~~
-
-Called after a cache lookup if the requested document was not found in
-the cache.  Its purpose is to decide whether or not to attempt to
-retrieve the document from the backend, and which backend to use.
-
-The vcl_miss subroutine may terminate with calling return() with one
-of the following keywords:
-
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_fetch.
-
-vcl_fetch
-~~~~~~~~~
+vcl_backend_response
+~~~~~~~~~~~~~~~~~~~
 
 Called after a document has been successfully retrieved from the backend.
 
-The vcl_fetch subroutine may terminate with calling return() with one
+The vcl_backend_response subroutine may terminate with calling return() with one
 of the following keywords:
 
   deliver
@@ -151,13 +97,13 @@ keywords:
     of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
-vcl_error
-~~~~~~~~~
+vcl_backend_error
+~~~~~~~~~~~~~~~~~
 
 Called when we hit an error, either explicitly or implicitly due to
 backend or internal errors.
 
-The vcl_error subroutine may terminate by calling return with one of
+The vcl_backend_error subroutine may terminate by calling return with one of
 the following keywords:
 
   deliver
@@ -168,11 +114,85 @@ the following keywords:
     of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
+
+vcl_pipe
+~~~~~~~~
+
+Called upon entering pipe mode.  In this mode, the request is passed
+on to the backend, and any further data from either client or
+backend is passed on unaltered until either end closes the
+connection.
+
+ The vcl_pipe subroutine may terminate with calling return() with one of
+ the following keywords:
+
+  error code [reason]
+    Return the specified error code to the client and abandon the request.
+
+  pipe
+    Proceed with pipe mode.
+
+vcl_pass
+~~~~~~~~
+
+Called upon entering pass mode.  In this mode, the request is passed
+on to the backend, and the backend's response is passed on to the
+client, but is not entered into the cache.  Subsequent requests
+submitted over the same client connection are handled normally.
+
+The vcl_pass subroutine may terminate with calling return() with one
+of the following keywords:
+
+  error code [reason]
+    Return the specified error code to the client and abandon the request.
+
+  pass
+    Proceed with pass mode.
+
+  restart
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
+    error.
+
+vcl_miss
+~~~~~~~~
+
+Called after a cache lookup if the requested document was not found in
+the cache.  Its purpose is to decide whether or not to attempt to
+retrieve the document from the backend, and which backend to use.
+
+The vcl_miss subroutine may terminate with calling return() with one
+of the following keywords:
+
+  error code [reason]
+    Return the specified error code to the client and abandon the request.
+
+  pass
+    Switch to pass mode.  Control will eventually pass to vcl_pass.
+
+  fetch
+    Retrieve the requested object from the backend.  Control will
+    eventually pass to vcl_fetch.
+
+
+
+vcl_init
+~~~~~~~~
+
+Called when VCL is loaded, before any requests pass through it.
+Typically used to initialize VMODs.
+
+  return() values:
+
+  ok
+    Normal return, VCL continues loading.
+
+
 vcl_fini
 ~~~~~~~~
 
-  Called when VCL is discarded only after all requests have exited the VCL.
-  Typically used to clean up VMODs.
+Called when VCL is discarded only after all requests have exited the VCL.
+Typically used to clean up VMODs.
 
   return() values:
 



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 033c75b Update example.vcl
Message-ID: <E1WO1sI-0007HC-9o@project.varnish-software.com>


commit 033c75b7bb7a0692982c1c06dc2026c0382ca30b
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 10:54:08 2014 +0100

    Update example.vcl

diff --git a/etc/example.vcl b/etc/example.vcl
index c093552..b99ec90 100644
--- a/etc/example.vcl
+++ b/etc/example.vcl
@@ -5,7 +5,7 @@
 # The builtin VCL is called when there is no explicit explicit return
 # statement.
 #
-# See the VCL tutorial at https://www.varnish-cache.org/docs/trunk/tutorial/
+# See the VCL chapters in the Users Guide at https://www.varnish-cache.org/docs/
 # and http://varnish-cache.org/trac/wiki/VCLExamples for more examples.
 
 # Marker to tell the VCL compiler that this VCL has been adapted to the
@@ -20,16 +20,21 @@ backend default {
 
 sub vcl_recv {
     # Happens before we check if we have this in cache already.
-    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_recv
+    # 
+    # Typically you clean up the request here, removing cookies you don't need,
+    # rewriting the request, etc.
 }
 
 sub vcl_backend_response {
     # Happens after we have read the response headers from the backend.
-    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
+    # 
+    # Here you clean the response headers, removing silly Set-Cookie headers
+    # and other mistakes your backend does.
 }
 
 sub vcl_deliver {
     # Happens when we have all the pieces we need, and are about to send the
     # response to the client.
-    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
+    # 
+    # You can do accounting or modifying the final object here.
 }



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] f03a993 4.0 updates
Message-ID: <E1WO1sI-0007HL-Bp@project.varnish-software.com>


commit f03a993b0f04894f99d8931c06eaabdcb60c694d
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 12:07:48 2014 +0100

    4.0 updates

diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst
index 15999cc..f21e30e 100644
--- a/doc/sphinx/users-guide/vcl-variables.rst
+++ b/doc/sphinx/users-guide/vcl-variables.rst
@@ -11,15 +11,17 @@ In VCL, there several important objects.
  do on or with the req object.
 
 *bereq*
- The backend request object. 
+ The backend request object. Varnish contructs this before sending it to the 
+ backend.
 
 *beresp*
- The backend respons object. It contains the headers of the object 
- coming from the backend. Most of the work you do in vcl_fetch you 
- do on the beresp object.
+ The backend response object. It contains the headers of the object 
+ coming from the backend. If you want to modify the reponse coming from the 
+ server you modify this object in vcl_backend_reponse. 
 
 *resp*
- The HTTP response right before it is delivered to the client. 
+ The HTTP response right before it is delivered to the client. It is
+ typically modified in vcl_deliver.
 
 *obj* 
  The object as it is stored in cache. Mostly read only.



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 03893a9 4.0 updates
Message-ID: <E1WO1sI-0007HV-DM@project.varnish-software.com>


commit 03893a974183b5468d72bab9b032c0c453843af1
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 12:08:20 2014 +0100

    4.0 updates

diff --git a/doc/sphinx/users-guide/vcl-actions.rst b/doc/sphinx/users-guide/vcl-actions.rst
index c981051..fc5a0da 100644
--- a/doc/sphinx/users-guide/vcl-actions.rst
+++ b/doc/sphinx/users-guide/vcl-actions.rst
@@ -21,7 +21,7 @@ The most common actions to return are these:
 *lookup*
   When you return lookup from vcl_recv you tell Varnish to deliver content 
   from cache even if the request othervise indicates that the request 
-  should be passed. You can't return lookup from vcl_fetch.
+  should be passed. 
 
 *pipe*
   Pipe can be returned from vcl_recv as well. Pipe short circuits the
@@ -34,3 +34,12 @@ The most common actions to return are these:
 
 *deliver*
  Deliver the cached object to the client.  Usually returned from vcl_fetch. 
+
+*restart*
+ Restart processing of the request. You can restart the processing of
+ the whole transaction. Changes to the req object are retained.
+
+*retry*
+ Retry the request against the backend. This can be called from
+ vcl_backend_response or vcl_backend_error if you don't like the response 
+ that the backend delivered.



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] e5840a2 rename
Message-ID: <E1WO1sI-0007Hf-Eg@project.varnish-software.com>


commit e5840a2f6d82f305a528912632cad3e08a4f61ff
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:04:15 2014 +0100

    rename

diff --git a/doc/sphinx/users-guide/orphans.rst b/doc/sphinx/users-guide/orphans.rst
new file mode 100644
index 0000000..76fbf5d
--- /dev/null
+++ b/doc/sphinx/users-guide/orphans.rst
@@ -0,0 +1,11 @@
+Orphans
+=======
+
+XXX: These are chapters which need to find a new home in the other sections.
+
+.. toctree::
+   :maxdepth: 2
+
+   esi
+   vary
+   cookies



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] b84796f 4.0 syntax changes
Message-ID: <E1WO1sI-0007Ho-Fh@project.varnish-software.com>


commit b84796f178b9501539e0ab553f0fcb9302b5ab12
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:05:07 2014 +0100

    4.0 syntax changes

diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
index 5e21511..d611807 100644
--- a/doc/sphinx/users-guide/compression.rst
+++ b/doc/sphinx/users-guide/compression.rst
@@ -29,7 +29,7 @@ backend sends content in clear text it will be stored like that.
 You can make Varnish compress content before storing it in cache in
 vcl_fetch by setting do_gzip to true, like this::
 
-   sub vcl_fetch {
+   sub vcl_backend_response {
         if (beresp.http.content-type ~ "text") {
                 set beresp.do_gzip = true;
         }



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 96b94ac Add -b flag to help debugging
Message-ID: <E1WO1sI-0007Hy-HM@project.varnish-software.com>


commit 96b94ac5d3e2dbf81ce9f6f051ba422274341af3
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:05:36 2014 +0100

    Add -b flag to help debugging

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index b5fa17e..fe1161d 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -23,7 +23,7 @@ Tool: varnishtop
 ~~~~~~~~~~~~~~~~
 
 You can use varnishtop to identify what URLs are hitting the backend
-the most. ``varnishtop -i txurl`` is an essential command. You can see
+the most. ``varnishtop -b -i txurl`` is an essential command. You can see
 some other examples of varnishtop usage in :ref:`users-guide-statistics`.
 
 



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] eafac81 call the orphans orphans
Message-ID: <E1WO1sI-0007I8-Ir@project.varnish-software.com>


commit eafac811e45e916ae168e200140b9d853bbdb689
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:05:59 2014 +0100

    call the orphans orphans

diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index ced7723..efe6d25 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -45,7 +45,7 @@ Detailed Table of Contents:
    report
    performance
    troubleshooting
-   operation
+   orphans
 
 .. customizing (which is a non ideal title)
 



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 9fc490c minor nitpicking
Message-ID: <E1WO1sI-0007IH-K4@project.varnish-software.com>


commit 9fc490c3e0d1590cd7af9fad1011783d6690d41b
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:06:34 2014 +0100

    minor nitpicking

diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
index aaf3bb5..547a987 100644
--- a/doc/sphinx/users-guide/operation-logging.rst
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -5,10 +5,12 @@ Logging in Varnish
 
 One of the really nice features in Varnish is how logging
 works. Instead of logging to normal log file Varnish logs to a shared
-memory segment. When the end of the segment is reached we start over,
-overwriting old data. This is much, much faster than logging to a file
-and it doesn't require disk space. Besides it gives you much, much
-more information when you need it.
+memory segment, called the VSL - the Varnish Shared Log. When the end
+of the segment is reached we start over, overwriting old data. 
+
+This is much, much faster than logging to a file and it doesn't
+require disk space. Besides it gives you much, much more information
+when you need it.
 
 The flip side is that if you forget to have a program actually write the
 logs to disk they will disappear.
@@ -40,12 +42,12 @@ app. You'll see lines like these.::
    11 RxHeader     c Host: localhost:8080
    11 RxHeader     c Connection: keep-alive
 
-The first column is an arbitrary number, it defines the request. Lines
-with the same number are part of the same HTTP transaction. The second
-column is the *tag* of the log message. All log entries are tagged
-with a tag indicating what sort of activity is being logged. Tags
-starting with Rx indicate Varnish is recieving data and Tx indicates
-sending data.
+The first column is an arbitrary number, it identifies the
+session. Lines with the same number are coming from the same session
+and are being handled by the same thread. The second column is the
+*tag* of the log message. All log entries are tagged with a tag
+indicating what sort of activity is being logged. Tags starting with
+Rx indicate Varnish is recieving data and Tx indicates sending data.
 
 The third column tell us whether this is is data coming or going to
 the client (c) or to/from the backend (b). The forth column is the



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] a66b2ff Remove varnishsizes, add reference to man pages
Message-ID: <E1WO1sI-0007IQ-LM@project.varnish-software.com>


commit a66b2ffb0e6f8adad7d3b1cc5e5c20248e83641a
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:07:22 2014 +0100

    Remove varnishsizes, add reference to man pages

diff --git a/doc/sphinx/users-guide/operation-statistics.rst b/doc/sphinx/users-guide/operation-statistics.rst
index 8736c5e..c891342 100644
--- a/doc/sphinx/users-guide/operation-statistics.rst
+++ b/doc/sphinx/users-guide/operation-statistics.rst
@@ -23,6 +23,8 @@ is being asked the most. ``varnishtop -i RxHeader -I
 Accept-Encoding`` will show the most popular Accept-Encoding header
 the client are sending you.
 
+For more information please see :ref:`ref-varnishtop`.
+
 varnishhist
 ~~~~~~~~~~~
 
@@ -33,13 +35,7 @@ vertical scale are displayed in the top left corner.  The horizontal
 scale is logarithmic.  Hits are marked with a pipe character ("|"),
 and misses are marked with a hash character ("#").
 
-
-varnishsizes
-~~~~~~~~~~~~
-
-Varnishsizes does the same as varnishhist, except it shows the size of
-the objects and not the time take to complete the request. This gives
-you a good overview of how big the objects you are serving are.
+For more information please see :ref:`ref-varnishhist`.
 
 
 varnishstat
@@ -55,3 +51,4 @@ graphs of these counters. One such program is Munin. Munin can be
 found at http://munin-monitoring.org/ . There is a plugin for munin in
 the Varnish source code.
 
+For more information please see :ref:`ref-varnishstat`.



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] df8c054 comment the VCL for clarity
Message-ID: <E1WO1sI-0007Ib-MO@project.varnish-software.com>


commit df8c054419e6867eb178f68a94d536293b83fc85
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:07:44 2014 +0100

    comment the VCL for clarity

diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 5604d96..38de2d3 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -39,6 +39,7 @@ following VCL in place::
 		  if (!client.ip ~ purge) {
 			  error 405 "Not allowed.";
 		  }
+                  # jump to hit/miss
 		  return (lookup);
 	  }
   }
@@ -133,7 +134,7 @@ object is not available in the ban lurker thread.
 
 You can use the following template to write ban lurker friendly bans::
 
-  sub vcl_fetch {
+  sub vcl_backend_response {
     set beresp.http.x-url = req.url;
   }
 



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 5ef8de8 4.0 updates to vcl + language
Message-ID: <E1WO1sI-0007Ik-NT@project.varnish-software.com>


commit 5ef8de8c9e35adec2c27323cd694d2428767a199
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:08:16 2014 +0100

    4.0 updates to vcl + language

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index dc9a3cb..d5b1aae 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -115,8 +115,14 @@ Directors
 
 You can also group several backend into a group of backends. These
 groups are called directors. This will give you increased performance
-and resilience. You can define several backends and group them
-together in a director.::
+and resilience. 
+
+You can define several backends and group them together in a
+director. This requires you to load a VMOD, a Varnish module, and then to 
+call certain actions in vcl_init.::
+
+
+         import directors;    # load the directors
 
 	 backend server1 {
 	     .host = "192.168.0.10";
@@ -125,24 +131,21 @@ together in a director.::
 	     .host = "192.168.0.10";
 	 }
 
-Now we create the director.::
-
-       	director example_director round-robin {
-        {
-                .backend = server1;
+         sub vcl_init {
+                new bar = directors.round_robin();
+                bar.add_backend(server1);
+                bar.add_backend(server2);
         }
-	# server2
-        {
-                .backend = server2;
-        }
-	# foo
+	
+        sub vcl_recv {
+                # send all traffic to the bar director:
+	        req.backend = bar.backend;
 	}
 
-
 This director is a round-robin director. This means the director will
 distribute the incoming requests on a round-robin basis. There is
 also a *random* director which distributes requests in a, you guessed
-it, random fashion.
+it, random fashion. 
 
 But what if one of your servers goes down? Can Varnish direct all the
 requests to the healthy server? Sure it can. This is where the Health
@@ -202,7 +205,9 @@ initial
 
 Now we define the director.::
 
-  director example_director round-robin {
+   import directors;
+
+   director example_director round-robin {
         {
                 .backend = server1;
         }



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 7f443a3 rename
Message-ID: <E1WO1sI-0007It-OT@project.varnish-software.com>


commit 7f443a3ae77acabfab7d4d05fdbe45fa09189c47
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Feb 21 21:09:21 2014 +0100

    rename

diff --git a/doc/sphinx/users-guide/operation.rst b/doc/sphinx/users-guide/operation.rst
deleted file mode 100644
index 76fbf5d..0000000
--- a/doc/sphinx/users-guide/operation.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Orphans
-=======
-
-XXX: These are chapters which need to find a new home in the other sections.
-
-.. toctree::
-   :maxdepth: 2
-
-   esi
-   vary
-   cookies



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 6737d14 Ignore various autogenerated files
Message-ID: <E1WO1sI-0007J4-Pd@project.varnish-software.com>


commit 6737d14391b3e30ecd54cebe89d450c3e9ebe79d
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Feb 24 09:42:41 2014 +0100

    Ignore various autogenerated files

diff --git a/.gitignore b/.gitignore
index 35dfbe7..a6d7219 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ Makefile.in
 *.la
 *~
 *.sw[op]
+*.trs
 
 
 # Various auto-tools artifacts
@@ -78,6 +79,8 @@ cscope.*out
 /man/vsc2rst
 /man/*.7
 /man/*.1
+/bin/*/*.1
+/bin/*/*.rst
 /doc/sphinx/include
 /bin/varnish*/varnish*_opt2rst
 /bin/varnishadm/varnishadm



From perbu at varnish-software.com  Thu Mar 13 09:24:26 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] b47c027 4.0 updates
Message-ID: <E1WO1sI-0007JD-Qb@project.varnish-software.com>


commit b47c02728e029d9bc8a9a03afe913fb6c297b132
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Feb 24 10:49:02 2014 +0100

    4.0 updates

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index fe1161d..9d1dc67 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -146,7 +146,8 @@ header. You could easily add support for this header in VCL.
 In vcl_fetch::
 
   if (beresp.http.Pragma ~ "nocache") {
-     return(hit_for_pass);
+        set beresp.uncacheable = true;
+	set beresp.ttl = 120s; // how long not to cache this url.
   }
 
 Authorization
diff --git a/doc/sphinx/users-guide/vcl-actions.rst b/doc/sphinx/users-guide/vcl-actions.rst
index fc5a0da..691baa4 100644
--- a/doc/sphinx/users-guide/vcl-actions.rst
+++ b/doc/sphinx/users-guide/vcl-actions.rst
@@ -10,14 +10,6 @@ The most common actions to return are these:
  and from the backend server. It won't be cached. pass can be returned from
  vcl_recv
 
-*hit_for_pass*
-  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
-  will create a hitforpass object in the cache. This has the side-effect of
-  caching the decision not to cache. This is to allow would-be uncachable
-  requests to be passed to the backend at the same time. The same logic is
-  not necessary in vcl_recv because this happens before any potential
-  queueing for an object takes place.
-
 *lookup*
   When you return lookup from vcl_recv you tell Varnish to deliver content 
   from cache even if the request othervise indicates that the request 
@@ -33,7 +25,7 @@ The most common actions to return are these:
   header before actually returning pipe. 
 
 *deliver*
- Deliver the cached object to the client.  Usually returned from vcl_fetch. 
+ Deliver the object to the client.  Usually returned from vcl_backend_response. 
 
 *restart*
  Restart processing of the request. You can restart the processing of
diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst
index f21e30e..20fcd4e 100644
--- a/doc/sphinx/users-guide/vcl-variables.rst
+++ b/doc/sphinx/users-guide/vcl-variables.rst
@@ -12,7 +12,7 @@ In VCL, there several important objects.
 
 *bereq*
  The backend request object. Varnish contructs this before sending it to the 
- backend.
+ backend. It is based on the req object.
 
 *beresp*
  The backend response object. It contains the headers of the object 



From martin at varnish-software.com  Thu Mar 13 09:24:26 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 2f580e2 Fix std.duration documentation of qualifiers
Message-ID: <E1WO1sI-0007JM-Rr@project.varnish-software.com>


commit 2f580e27c311060d4fc92fb600ec4aa865c4907d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Feb 24 13:36:46 2014 +0100

    Fix std.duration documentation of qualifiers
    
    std.duration conversion function has a mandatory qualifier. Change the
    wording from 'can' to 'must' specify the qualifier.
    
    Add a test case for a string without a qualifier.

diff --git a/bin/varnishtest/tests/m00005.vtc b/bin/varnishtest/tests/m00005.vtc
index 2ea397a..aa7143d 100644
--- a/bin/varnishtest/tests/m00005.vtc
+++ b/bin/varnishtest/tests/m00005.vtc
@@ -85,4 +85,10 @@ client c1 {
 	expect resp.http.ttl ==  1000002.000
 	expect resp.bodylen == 1
 
+	txreq -url "/1"  -hdr "ttl: 100"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.ttl == 1000002.000
+	expect resp.bodylen == 1
+
 } -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index a777801..a55371c 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -111,9 +111,10 @@ Example
 $Function DURATION duration(STRING, DURATION)
 
 Description
-	Converts the string *s* to seconds. *s* can be quantified with
-	the usual s (seconds), m (minutes), h (hours), d (days) and w
-	(weeks) units. If *s* fails to parse, *fallback* will be returned.
+	Converts the string *s* to seconds. *s* must be quantified
+	with the usual s (seconds), m (minutes), h (hours), d (days)
+	and w (weeks) units. If *s* fails to parse, *fallback* will be
+	returned.
 Example
 	set beresp.ttl = std.duration("1w", 3600s);
 



From martin at varnish-software.com  Thu Mar 13 09:24:26 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] d35ed6e Add 'ms' conversion qualifier to std.duration
Message-ID: <E1WO1sI-0007JV-T3@project.varnish-software.com>


commit d35ed6e2617203a0ce9d5cd27bb2286720efab73
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Feb 24 13:48:38 2014 +0100

    Add 'ms' conversion qualifier to std.duration
    
    Add 'ms' conversion qualifier to std.duration, and update
    documentation and test case.
    
    Fixes: #1434

diff --git a/bin/varnishtest/tests/m00005.vtc b/bin/varnishtest/tests/m00005.vtc
index aa7143d..53f5ae3 100644
--- a/bin/varnishtest/tests/m00005.vtc
+++ b/bin/varnishtest/tests/m00005.vtc
@@ -19,6 +19,12 @@ varnish v1 -vcl+backend {
 } -start
 
 client c1 {
+	txreq -url "/1"  -hdr "ttl: 10ms "
+	rxresp
+	expect resp.status == 200
+	expect resp.http.ttl == 1000001.010
+	expect resp.bodylen == 1
+
 	txreq -url "/1"  -hdr "ttl: 10s "
 	rxresp
 	expect resp.status == 200
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index a55371c..baeb48f 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -112,9 +112,9 @@ $Function DURATION duration(STRING, DURATION)
 
 Description
 	Converts the string *s* to seconds. *s* must be quantified
-	with the usual s (seconds), m (minutes), h (hours), d (days)
-	and w (weeks) units. If *s* fails to parse, *fallback* will be
-	returned.
+	with the usual ms (milliseconds), s (seconds), m (minutes), h
+	(hours), d (days) and w (weeks) units. If *s* fails to parse,
+	*fallback* will be returned.
 Example
 	set beresp.ttl = std.duration("1w", 3600s);
 
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index f9cba84..b9020d1 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -76,7 +76,13 @@ vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
 	/* NB: Keep this list synchronized with VCC */
 	switch (*e++) {
 	case 's': break;
-	case 'm': r *= 60.; break;
+	case 'm':
+		if (*e == 's') {
+			r *= 1e-3;
+			e++;
+		} else
+			r *= 60.;
+		break;
 	case 'h': r *= 60.*60.; break;
 	case 'd': r *= 60.*60.*24.; break;
 	case 'w': r *= 60.*60.*24.*7.; break;



From lkarsten at varnish-software.com  Thu Mar 13 09:24:26 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 339b68f Rename =build/ to build/
Message-ID: <E1WO1sI-0007Jg-Ty@project.varnish-software.com>


commit 339b68f92bca636bd8cb786fc68cb5ea437202fe
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Feb 24 14:14:42 2014 +0100

    Rename =build/ to build/
    
    That silly equality sign constantly needed to be escaped and
    is just unecessesary finger gymnastics. Die, please.

diff --git a/.gitignore b/.gitignore
index a6d7219..3df191e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -95,7 +95,7 @@ cscope.*out
 
 # Doc-stuff generated from xml
 /doc/*.html
-/doc/sphinx/=build/
+/doc/sphinx/build/
 /doc/sphinx/conf.py
 /doc/sphinx/reference/vcl_var.rst
 
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 3abd43f..8e891c8 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -5,7 +5,7 @@
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         = a4
-BUILDDIR      = =build
+BUILDDIR      = build
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index d5415da..b9fce6a 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -37,8 +37,8 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Varnish'
-copyright = u'2010-2014 (c) Varnish Software AS'
+project = u'Varnish Cache'
+copyright = u'2010-2014, Varnish Software AS'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the



From lkarsten at varnish-software.com  Thu Mar 13 09:24:26 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 714dae2 Make the TOC look sane PDF/latex version
Message-ID: <E1WO1sI-0007Jp-Uy@project.varnish-software.com>


commit 714dae2c06bbbfd7ced61ac7c8384e2d18d15a00
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Feb 24 14:16:35 2014 +0100

    Make the TOC look sane PDF/latex version

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index b9fce6a..d825d6b 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -199,8 +199,8 @@ htmlhelp_basename = 'Varnishdoc'
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'Varnish.tex', u'Varnish Documentation',
-   u'Varnish Project', 'manual'),
+  ('index', 'Varnish.tex', u'Varnish Administrator documentation',
+   u'Varnish Cache Project', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 3d1e28f..b29121e 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -1,8 +1,4 @@
-.. Varnish documentation master file, created by
-   sphinx-quickstart on Tue Apr 20 13:02:15 2010.
-
 .. We use the following header indicators
-
 .. For titles:
 
 .. H1
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index efe6d25..1000a6f 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -32,10 +32,6 @@ from transaction level to aggregate statistics.
 
 :ref:`users_trouble` is for locating and fixing trouble with Varnish.
 
-
-Detailed Table of Contents:
----------------------------
-
 .. toctree::
    :maxdepth: 2
 



From lkarsten at varnish-software.com  Thu Mar 13 09:24:26 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:26 +0100
Subject: [4.0] 2d1a8c1 Assorted minor documentation fixes
Message-ID: <E1WO1sI-0007Jz-W4@project.varnish-software.com>


commit 2d1a8c11ea88e3fc394ba160183282ddbdcbd672
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Feb 24 16:14:38 2014 +0100

    Assorted minor documentation fixes

diff --git a/doc/sphinx/installation/bugs.rst b/doc/sphinx/installation/bugs.rst
index b1cb06a..16f0c51 100644
--- a/doc/sphinx/installation/bugs.rst
+++ b/doc/sphinx/installation/bugs.rst
@@ -28,7 +28,7 @@ Varnish crashes
 Plain and simple: **boom**
 
 Varnish is split over two processes, the manager and the child.  The child
-does all the work, and the manager hangs around to resurect it, if it
+does all the work, and the manager hangs around to resurect it if it
 crashes.
 
 Therefore, the first thing to do if you see a Varnish crash, is to examine
@@ -77,9 +77,10 @@ general forms:
 	(yet) written the padded-box error handling code for.
 
 	The most likely cause here, is that you need a larger workspace
-	for HTTP headers and Cookies. (XXX: which params to tweak)
+	for HTTP headers and Cookies.
 
 	Please try that before reporting a bug.
+..  (TODO: which params to tweak)
 
 "Assert error in ..."
 	This is something bad that should never happen, and a bug
diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 5e62e77..4fb8362 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -33,23 +33,24 @@ families and jobs to deal with also.
 
 You are more than welcome to just hang out, and while we don't mind
 the occational intrusion of the real world into the flow, keep
-it mostly on topic, and dont paste random links unless they are
+it mostly on topic, and don't paste random links unless they are
 *really* spectacular and intelligent.
 
 Mailing Lists
 =============
 
-Getting on or off our mailinglist happens through MailMan_.
+Getting on or off our mailing lists happens through MailMan_.
 
 If you are going to use Varnish, subscribing to our ``varnish-announce``
-mailing list is probably a very good idea.  The typical pattern is that
+mailing list is probably a very good idea. The typical pattern is that
 people spend some time getting Varnish running, and then more or less
-forget about it.   Therefore the announce list is a good way to be
+forget about it. Therefore the announce list is a good way to be
 reminded about new releases, bad bugs or security holes.
 
+
 The ``varnish-misc`` mailing list is for general banter, questions,
 suggestions, ideas and so on.  If you are new to Varnish it may pay
-off to subscribe to -misc, simply to have an ear to the telegraph-pole
+off to subscribe to it, simply to have an ear to the telegraph-pole
 and learn some smart tricks.  This is a good place to ask for help
 with more complex issues, that require quoting of files and long
 explanations.
@@ -59,6 +60,7 @@ thread changes, please change the subject to match, some of us deal
 with hundreds of emails per day, after spam-filters, and we need all
 the help we can get to pick the interesting ones.
 
+
 The ``varnish-dev`` mailing list is used by the developers and is
 usually quite focused on source-code and such.  Everybody on
 the -dev list is also on -misc, so cross-posting only serves to annoy
diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index b1e0a23..60d43e0 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -5,7 +5,7 @@ Varnish Installation
 %%%%%%%%%%%%%%%%%%%%
 
 This document explains how to get Varnish onto your system, where
-to get help, how report bugs etc.  In other words, it is a manual
+to get help, how report bugs and so on. In other words, it is a manual
 about pretty much everything else than actually using Varnish to
 move traffic.
 
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index ef31f86..45c5e4e 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -13,7 +13,7 @@ are most comfortable with.
 Source or packages?
 -------------------
 
-Installing Varnish on most relevant operating systems can usually 
+Installing Varnish on most relevant operating systems can usually
 be done with with the systems package manager, typical examples
 being:
 
@@ -36,8 +36,9 @@ on `repo.varnish-cache.org <http://repo.varnish-cache.org/>`_.  See the
 Varnish is included in the `EPEL
 <http://fedoraproject.org/wiki/EPEL>`_ repository, however due to
 incompatible syntax changes in newer versions of Varnish, only older
-versions are available. We recommend that you install the latest
-version from our repository.
+versions are available.
+
+We recommend that you install the latest version from our repository.
 
 Debian/Ubuntu
 -------------
@@ -69,7 +70,7 @@ git repository by doing.
 Please note that a git checkout will need some more build-dependencies
 than listed below, in particular the Python Docutils and Sphinx.
 
-Build dependencies on Debian / Ubuntu 
+Build dependencies on Debian / Ubuntu
 --------------------------------------
 
 In order to build Varnish from source you need a number of packages
@@ -77,7 +78,7 @@ installed. On a Debian or Ubuntu system these are:
 
 * autotools-dev
 * automake1.11
-* libtool 
+* libtool
 * autoconf
 * libncurses-dev
 * groff-base
@@ -89,7 +90,7 @@ installed. On a Debian or Ubuntu system these are:
 If you're building from git, you also need the following:
 
 * python-docutils
-* python-sphinx (optional, if you want the HTML docs built)
+* python-sphinx (optional, if you want to build the documentation)
 
 Build dependencies on Red Hat / CentOS
 --------------------------------------
@@ -97,8 +98,8 @@ Build dependencies on Red Hat / CentOS
 To build Varnish on a Red Hat or CentOS system you need the following
 packages installed:
 
-* automake 
-* autoconf 
+* automake
+* autoconf
 * libtool
 * ncurses-devel
 * groff
@@ -109,10 +110,10 @@ packages installed:
 If you're building from git, you also need the following:
 
 * docutils
-* python-sphinx (optional, if you want the HTML docs built)
+* python-sphinx (optional, if you want to build the documentation)
 
-Configuring and compiling
--------------------------
+Compiling Varnish
+-----------------
 
 Next, configuration: The configuration will need the dependencies
 above satisfied. Once that is taken care of::
@@ -133,7 +134,7 @@ tea while it runs, it takes some minutes::
 
 Don't worry if a single or two tests fail, some of the tests are a
 bit too timing sensitive (Please tell us which so we can fix it) but
-if a lot of them fails, and in particular if the ``b00000.vtc`` test 
+if a lot of them fails, and in particular if the ``b00000.vtc`` test
 fails, something is horribly wrong, and you will get nowhere without
 figuring out what.
 
@@ -146,6 +147,6 @@ And finally, the true test of a brave heart::
 
 Varnish will now be installed in /usr/local. The varnishd binary is in
 /usr/local/sbin/varnishd and its default configuration will be
-/usr/local/etc/varnish/default.vcl. 
+/usr/local/etc/varnish/default.vcl.
 
-You can now proceed to the :ref:`tutorial-index`. 
+You can now proceed to the :ref:`tutorial-index`.
diff --git a/doc/sphinx/installation/prerequisites.rst b/doc/sphinx/installation/prerequisites.rst
index ad92ee0..736bcd8 100644
--- a/doc/sphinx/installation/prerequisites.rst
+++ b/doc/sphinx/installation/prerequisites.rst
@@ -4,6 +4,7 @@ Prerequisites
 
 In order for you to install Varnish you must have the following:
 
+  * A recent, preferably server grade, computer.
   * A fairly modern and 64 bit version of either
     - Linux
     - FreeBSD
diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst
index 65dec6a..dc25087 100644
--- a/doc/sphinx/users-guide/command-line.rst
+++ b/doc/sphinx/users-guide/command-line.rst
@@ -62,7 +62,7 @@ If you go with -f, you can start with a VCL file containing just::
 
 which is exactly what -b does.
 
-In both cases the default VCL code is appended.
+In both cases the built-in VCL code is appended.
 
 Other options
 ^^^^^^^^^^^^^
@@ -74,9 +74,9 @@ By default Varnish will use 100 megabytes of malloc(3) storage
 for caching objects, if you want to cache more than that, you
 should look at the '-s' argument.
 
-If you run a really big site, you may want to tune the size of
-the tread-pools and other parameters with the '-p' argument,
-but we generally advice not to do that, unless you need to.
+If you run a really big site, you may want to tune the number of
+worker threads and other parameters with the '-p' argument,
+but we generally advice not to do that unless you need to.
 
 Before you go into production, you may also want to re-visit the
 chapter
diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index 18b4a1d..6a67fc5 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -48,10 +48,10 @@ each and every HTTP request which comes in.
 
 Because the VCL is compiled to C code, and the C code is compiled
 to machine instructions, even very complex VCL programs execute in
-a a few microseconds, without impacting performance at all.
+a few microseconds, without impacting performance at all.
 
 And don't fret if you are not really a programmer, VCL is very
-simple to do simpel things with::
+simple to do simple things with::
 
 	sub vcl_recv {
 		# Remove the cookie header to enable caching
@@ -73,7 +73,7 @@ Varnish uses a piece of shared memory to report its activity and
 status. For each HTTP request, a number of very detailed records will
 be appended to the log segment in this shared memory.  Other processes
 can subscribe to log-records, filter them, and format them, for
-instance as NCSA style log records.
+instance as Apache/NCSA style log records.
 
 Another segment in shared memory is used for statistics counters,
 this allows real-time, down to microsecond resolution monitoring
diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst
index a5e97e1..b534971 100644
--- a/doc/sphinx/users-guide/run_cli.rst
+++ b/doc/sphinx/users-guide/run_cli.rst
@@ -14,7 +14,7 @@ same machine as varnishd is running::
 If you want to run varnishadm from a remote system, you can do it
 two ways.
 
-You can ssh into the varnishd computer and run varnishadm::
+You can SSH into the varnishd computer and run varnishadm::
 
 	ssh $http_front_end varnishadm help
 
@@ -27,7 +27,7 @@ And then on the remote system run varnishadm::
 
 	varnishadm -T $http_front_end -S /etc/copy_of_varnish_secret help
 
-but as you can see, ssh is much more convenient.
+but as you can see, SSH is much more convenient.
 
 If you run varnishadm without arguments, it will read CLI commands from
 stdin, if you give it arguments, it will treat those as the single
@@ -36,7 +36,7 @@ CLI command to execute.
 The CLI always returns a status code to tell how it went:  '200'
 means OK, anything else means there were some kind of trouble.
 
-Varnishadm will exit with status 1 and print the status code on
+varnishadm will exit with status 1 and print the status code on
 standard error if it is not 200.
 
 What can you do with the CLI
@@ -86,9 +86,8 @@ to the previous VCL program again::
 	varnish> vcl.use old_name
 
 The switch is instantaneous, all new requests will start using the
-VCL you activated right away, but for consistency, the requests
-currently being processed, complete using whatever VCL they started
-with.
+VCL you activated right away. The requests currently being processed complete
+using whatever VCL they started with.
 
 It is good idea to design an emergency-VCL before you need it,
 and always have it loaded, so you can switch to it with a single
@@ -120,9 +119,10 @@ a HTTP request asks for it.
 Banning stuff is much cheaper than restarting Varnish to get rid
 of wronly cached content.
 
-In addition to handling such special occations, banning can be used
-in many creative ways to keep the cache up to date, more about
-that in: (XXX: xref)
+.. In addition to handling such special occations, banning can be used
+.. in many creative ways to keep the cache up to date, more about
+.. that in: (TODO: xref)
+
 
 Change parameters
 ^^^^^^^^^^^^^^^^^
@@ -136,7 +136,7 @@ but they can also be examined and changed on the fly from the CLI::
                             Default is off
                             Prefer IPv6 address when connecting to backends
                             which have both IPv4 and IPv6 addresses.
-	
+
 	varnish> param.set prefer_ipv6 true
 	200
 
diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 6f0fe9b..b12dba0 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -19,10 +19,11 @@ malloc
 syntax: malloc[,size]
 
 Malloc is a memory based backend. Each object will be allocated from
-memory. If your system runs low on memory swap will be used. Be aware
-that the size limitation only limits the actual storage and that
-approximately 1k of memory per object will be used for various
-internal structures.
+memory. If your system runs low on memory swap will be used.
+
+Be aware that the size limitation only limits the actual storage and that
+approximately 1k of memory per object will be used for various internal
+structures.
 
 The size parameter specifies the maximum amount of memory varnishd
 will allocate.  The size is assumed to be in bytes, unless followed by
@@ -36,19 +37,19 @@ one of the following suffixes:
 
       T, t    The size is expressed in tebibytes.
 
-The default size is unlimited. 
+The default size is unlimited.
 
-Mallocs performance is bound by memory speed so it is very fast. If
+malloc's performance is bound by memory speed so it is very fast. If
 the dataset is bigger than what can fit in memory performance will
-depend on the operating system and how well it doesn paging. 
+depend on the operating system and how well it does paging.
 
 file
 ~~~~
 
 syntax: file[,path[,size[,granularity]]]
 
-The file backend stores objects in memory backed by a file on disk
-with mmap. 
+The file backend stores objects in memory backed by an unlinked file on disk
+with mmap.
 
 The path parameter specifies either the path to the backing file or
 the path to a directory in which varnishd will create the backing
@@ -84,8 +85,8 @@ allocation.  All allocations are rounded up to this size.  The
 is assumed to be in bytes, unless followed by one of the
 suffixes described for size except for %.
 
-The default size is the VM page size.  The size should be reduced if
-you have many small objects.
+The default granularity is the VM page size.  The size should be reduced if you
+have many small objects.
 
 File performance is typically limited by the write speed of the
 device, and depending on use, the seek time.
@@ -127,10 +128,11 @@ and can make previously banned objects reappear.
 
 Transient Storage
 -----------------
-      
+
 If you name any of your storage backend "Transient" it will be
 used for transient (short lived) objects. By default Varnish
 would use an unlimited malloc backend for this.
 
 Varnish will consider an object short lived if the TTL is below the
 parameter "shortlived".
+
diff --git a/doc/sphinx/users-guide/vcl-intro.rst b/doc/sphinx/users-guide/vcl-intro.rst
index bc90411..095896c 100644
--- a/doc/sphinx/users-guide/vcl-intro.rst
+++ b/doc/sphinx/users-guide/vcl-intro.rst
@@ -21,7 +21,7 @@ Varnish will execute the built in VCL code. You will see this VCL
 code commented out in default.vcl.
 
 99% of all the changes you'll need to do will be done in two of these
-subroutines. *vcl_recv* and *vcl_fetch*.
+subroutines. *vcl_recv* and *vcl_backend_response*.
 
 .. _users-guide-vcl_fetch_actions:
 
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index 0e56c3a..defe566 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -4,12 +4,11 @@ VCL - Varnish Configuration Language
 ------------------------------------
 
 This section is about getting Varnish to do what you want to
-your HTTP traffic, using the Varnish Configuration Language.
+your HTTP traffic, using the Varnish Configuration Language (VCL).
 
 Varnish has a great configuration system. Most other systems use
 configuration directives, where you basically turn on and off lots of
-switches. Varnish uses a domain specific language called Varnish
-Configuration Language, or VCL for short. 
+switches. Varnish uses a domain specific language called VCL for this.
 
 Every inbound request flows through Varnish and you can influence how
 the request is being handled by altering the VCL code. You can direct
@@ -26,8 +25,8 @@ are executed at different times. One is executed when we get the
 request, another when files are fetched from the backend server.
 
 If you don't call an action in your subroutine and it reaches the end
-Varnish will execute some built in VCL code. You will see this VCL
-code commented out in default.vcl that ships with Varnish Cache.
+Varnish will execute some built-in VCL code. You will see this VCL
+code commented out in builtin.vcl that ships with Varnish Cache.
 
 .. _users-guide-vcl_fetch_actions:
 
@@ -45,4 +44,4 @@ code commented out in default.vcl that ships with Varnish Cache.
    vcl-examples
    websockets
    devicedetection
-   
+



From fgsch at lodoss.net  Thu Mar 13 09:24:27 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 349256d Sync with vmod.vcc
Message-ID: <E1WO1sJ-0007K9-0j@project.varnish-software.com>


commit 349256d65a7b908cdf986bfc53b7acd4ab1b3918
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 24 16:08:29 2014 +0000

    Sync with vmod.vcc

diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index 36a2678..41c38e6 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -23,7 +23,7 @@ DESCRIPTION
 Vmod_std contains basic functions which are part and parcel of Varnish,
 but which for reasons of architecture fit better in a VMOD.
 
-One specific class of functions in vmod_std is the conversions functions
+One particular class of functions in vmod_std is the conversions functions
 which all have the form::
 
 	TYPE type(STRING, TYPE)
@@ -127,9 +127,10 @@ duration
 Prototype
 	DURATION duration(STRING s, DURATION fallback)
 Description
-	Converts the string *s* to seconds. *s* can be quantified with
-	the usual s (seconds), m (minutes), h (hours), d (days) and w
-	(weeks) units. If *s* fails to parse, *fallback* will be returned.
+	Converts the string *s* to seconds. *s* must be quantified
+	with ms (milliseconds), s (seconds), m (minutes), h (hours),
+	d (days) or w (weeks) units. If *s* fails to parse,
+	*fallback* will be returned.
 Example
 	set beresp.ttl = std.duration("1w", 3600s);
 
@@ -154,6 +155,22 @@ Description
 Example
 	if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { ... }
 
+healthy
+-------
+Prototype
+	BOOL healthy(BACKEND backend)
+ 
+Description
+	Returns true if the backend is healthy.
+
+port
+----
+Prototype
+	INT port(IP ip)
+
+Description
+	Returns the port number of an IP address.
+
 
 SEE ALSO
 ========



From fgsch at lodoss.net  Thu Mar 13 09:24:27 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] da2566c Minor cosmetic changes
Message-ID: <E1WO1sJ-0007KN-29@project.varnish-software.com>


commit da2566c2ae9297cf5f0fd28be1098fce2000b7d8
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 24 16:20:58 2014 +0000

    Minor cosmetic changes

diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index baeb48f..dcb12ec 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -112,8 +112,8 @@ $Function DURATION duration(STRING, DURATION)
 
 Description
 	Converts the string *s* to seconds. *s* must be quantified
-	with the usual ms (milliseconds), s (seconds), m (minutes), h
-	(hours), d (days) and w (weeks) units. If *s* fails to parse,
+	with ms (milliseconds), s (seconds), m (minutes), h (hours),
+	d (days) or w (weeks) units. If *s* fails to parse,
 	*fallback* will be returned.
 Example
 	set beresp.ttl = std.duration("1w", 3600s);
@@ -138,12 +138,12 @@ Example
 $Function BOOL healthy(BACKEND)
 
 Description
-	Returns true if the backend is healthy
+	Returns true if the backend is healthy.
 
 $Function INT port(IP)
 
 Description
-	Returns the port number of an IP address
+	Returns the port number of an IP address.
 
 
 SEE ALSO



From lkarsten at varnish-software.com  Thu Mar 13 09:24:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 543302c Wording, less genitive in the heading
Message-ID: <E1WO1sJ-0007KT-37@project.varnish-software.com>


commit 543302c4d3c74389ce78b81ff2e423dbacf00f0d
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Feb 25 12:10:20 2014 +0100

    Wording, less genitive in the heading

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index b29121e..e83d1dc 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -12,20 +12,16 @@
 .. ~~~~~~~~~~~~~~~~~~~~~~
 .. ...................... 
 
-Welcome to Varnish's documentation!
-===================================
+Welcome to the Varnish documentation!
+=====================================
 
-Varnish is a state of the art web accelerator. Its mission is to sit
-in front of a web server and cache content. It makes your web site
-go faster.
+Varnish is a state of the art web accelerator. It has its mission in front of a
+web server and cache content. It makes your web site go faster.
 
 We suggest you start by reading the installation guide
 :ref:`install-index`. Once you have Varnish up and running go through
 our tutorial - :ref:`tutorial-index`.
 
-
-Contents:
-
 .. toctree::
    :maxdepth: 1
 



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] c2ebc12 Introduce two new states in backend-fetch state
 machine: retry and error
Message-ID: <E1WO1sJ-0007Kc-47@project.varnish-software.com>


commit c2ebc12633e6cc577e5ffa6b2cf23eb4e1af9ba4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 18 12:19:23 2014 +0000

    Introduce two new states in backend-fetch state machine: retry and error

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 819c39f..31379cf 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -45,6 +45,22 @@
  */
 
 static void
+make_it_503(struct busyobj *bo)
+{
+
+	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
+	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
+	http_SetHeader(bo->beresp, "Content-Length: 0");
+	http_SetHeader(bo->beresp, "Connection: close");
+	bo->exp.ttl = 0;
+	bo->exp.grace = 0;
+	bo->exp.keep = 0;
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static void
 vbf_release_req(struct busyobj *bo)
 {
 	assert(bo->state == BOS_INVALID);
@@ -103,7 +119,31 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
- * Copy run bereq by VCL::vcl_backend_fetch{}
+ * Start a new VSL transaction and try again
+ */
+
+static enum fetch_step
+vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
+{
+	unsigned owid, wid;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	// XXX: BereqEnd + BereqAcct ?
+	wid = VXID_Get(&wrk->vxid_pool);
+	VSLb(bo->vsl, SLT_Link, "bereq %u retry", wid);
+	VSLb(bo->vsl, SLT_End, "%s", "");
+	VSL_Flush(bo->vsl, 0);
+	owid = bo->vsl->wid & VSL_IDENTMASK;
+	bo->vsl->wid = wid | VSL_BACKENDMARKER;
+	VSLb(bo->vsl, SLT_Begin, "bereq %u retry", owid);
+
+	return (F_STP_STARTFETCH);
+}
+
+/*--------------------------------------------------------------------
+ * Setup bereq from bereq0, run vcl_backend_fetch
  */
 
 static enum fetch_step
@@ -138,21 +178,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	return (F_STP_FETCHHDR);
 }
 
-/*--------------------------------------------------------------------
- */
-
-static void
-make_it_503(struct busyobj *bo)
-{
-
-	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
-	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
-	http_SetHeader(bo->beresp, "Content-Length: 0");
-	http_SetHeader(bo->beresp, "Connection: close");
-	bo->exp.ttl = 0;
-	bo->exp.grace = 0;
-	bo->exp.keep = 0;
-}
 
 /*--------------------------------------------------------------------
  */
@@ -161,7 +186,6 @@ static enum fetch_step
 vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 {
 	int i, do_ims, fail;
-	unsigned owid, wid;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -247,17 +271,9 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 		if (bo->vbc)
 			VDI_CloseFd(&bo->vbc);
 		bo->retries++;
-		if (bo->retries <= cache_param->max_retries) {
-			// XXX: BereqEnd + BereqAcct ?
-			wid = VXID_Get(&wrk->vxid_pool);
-			VSLb(bo->vsl, SLT_Link, "bereq %u retry", wid);
-			VSLb(bo->vsl, SLT_End, "%s", "");
-			VSL_Flush(bo->vsl, 0);
-			owid = bo->vsl->wid & VSL_IDENTMASK;
-			bo->vsl->wid = wid | VSL_BACKENDMARKER;
-			VSLb(bo->vsl, SLT_Begin, "bereq %u retry", owid);
-			return (F_STP_STARTFETCH);
-		}
+		if (bo->retries <= cache_param->max_retries)
+			return (F_STP_RETRY);
+
 		VSLb(bo->vsl, SLT_VCL_Error,
 		    "Too many retries, delivering 503");
 		make_it_503(bo);
@@ -548,13 +564,6 @@ VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d",
 	return (F_STP_DONE);
 }
 
-static enum fetch_step
-vbf_stp_done(void)
-{
-	WRONG("Just plain wrong");
-	return (F_STP_DONE);
-}
-
 /*--------------------------------------------------------------------
  */
 
@@ -659,8 +668,28 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ * Create synth object
  */
 
+static enum fetch_step
+vbf_stp_error(struct worker *wrk, struct busyobj *bo)
+{
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	WRONG("");
+	return (F_STP_DONE);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static enum fetch_step
+vbf_stp_done(void)
+{
+	WRONG("Just plain wrong");
+	return (F_STP_DONE);
+}
+
 static const char *
 vbf_step_name(enum fetch_step stp)
 {
@@ -675,7 +704,6 @@ vbf_step_name(enum fetch_step stp)
 	}
 }
 
-
 static void
 vbf_fetch_thread(struct worker *wrk, void *priv)
 {
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index d99738c..9ef7f6c 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -50,10 +50,12 @@ REQ_STEP(error,		ERROR,		(wrk, req))
 
 #ifdef FETCH_STEP
 FETCH_STEP(mkbereq,	MKBEREQ,	(wrk, bo))
+FETCH_STEP(retry,	RETRY,		(wrk, bo))
 FETCH_STEP(startfetch,	STARTFETCH,	(wrk, bo))
 FETCH_STEP(fetchhdr,	FETCHHDR,	(wrk, bo))
 FETCH_STEP(condfetch,	CONDFETCH,	(wrk, bo))
 FETCH_STEP(fetch,	FETCH,		(wrk, bo))
+FETCH_STEP(error,	ERROR,		(wrk, bo))
 FETCH_STEP(done,	DONE,		())
 #endif
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 0702465..29dcb9e 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -120,7 +120,7 @@ returns =(
 	),
 
 	###############################################################
-	# Backend-fetch 
+	# Backend-fetch
 
 	('backend_fetch',
 		"B",



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] fb97c67 Extract the bereq->obj creation code to a separate
 function
Message-ID: <E1WO1sJ-0007Km-5I@project.varnish-software.com>


commit fb97c67b70fa87dcc2091c05c104bb1e68cff37a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 18 13:17:18 2014 +0000

    Extract the bereq->obj creation code to a separate function

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 31379cf..f044347 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -71,6 +71,119 @@ vbf_release_req(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+static int
+vbf_bereq2obj(struct worker *wrk, struct busyobj *bo)
+{
+	unsigned l;
+	char *b;
+	struct vsb *vary = NULL;
+	int varyl = 0;
+	uint16_t nhttp;
+	struct object *obj;
+	struct http *hp, *hp2;
+
+	l = 0;
+
+	/* Create Vary instructions */
+	if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
+		varyl = VRY_Create(bo, &vary);
+		if (varyl > 0) {
+			AN(vary);
+			assert(varyl == VSB_len(vary));
+			l += varyl;
+		} else if (varyl < 0) {
+			/*
+			 * Vary parse error
+			 * Complain about it, and make this a pass.
+			 */
+			VSLb(bo->vsl, SLT_Error,
+			    "Illegal 'Vary' header from backend, "
+			    "making this a pass.");
+			bo->uncacheable = 1;
+			AZ(vary);
+		} else
+			/* No vary */
+			AZ(vary);
+	}
+
+	l += http_EstimateWS(bo->beresp,
+	    bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
+
+	if (bo->uncacheable)
+		bo->fetch_objcore->flags |= OC_F_PASS;
+
+	if (bo->uncacheable ||
+	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
+		bo->storage_hint = TRANSIENT_STORAGE;
+
+	AZ(bo->stats);
+	bo->stats = &wrk->stats;
+	AN(bo->fetch_objcore);
+	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
+#if 0
+	// XXX: we shouldn't retry if we're already on Transient
+	if (obj == NULL &&
+	    (bo->storage_hint == NULL ||
+	    strcmp(bo->storage_hint, TRANSIENT_STORAGE))) {
+#else
+	if (obj == NULL) {
+#endif
+		/*
+		 * Try to salvage the transaction by allocating a
+		 * shortlived object on Transient storage.
+		 */
+		if (bo->exp.ttl > cache_param->shortlived)
+			bo->exp.ttl = cache_param->shortlived;
+		bo->exp.grace = 0.0;
+		bo->exp.keep = 0.0;
+		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
+	}
+	if (obj == NULL)
+		return (-1);
+
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+
+	bo->storage_hint = NULL;
+
+	AZ(bo->fetch_obj);
+	bo->fetch_obj = obj;
+
+	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
+		obj->gziped = 1;
+
+	if (vary != NULL) {
+		obj->vary = (void *)WS_Copy(obj->http->ws,
+		    VSB_data(vary), varyl);
+		AN(obj->vary);
+		(void)VRY_Validate(obj->vary);
+		VSB_delete(vary);
+	}
+
+	obj->vxid = bo->vsl->wid;
+	obj->response = bo->err_code;
+	WS_Assert(bo->ws_o);
+
+	/* Filter into object */
+	hp = bo->beresp;
+	hp2 = obj->http;
+
+	hp2->logtag = SLT_ObjMethod;
+	http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
+	http_CopyHome(hp2);
+
+	if (http_GetHdr(hp, H_Last_Modified, &b))
+		obj->last_modified = VTIM_parse(b);
+	else
+		obj->last_modified = floor(bo->exp.t_origin);
+
+	assert(WRW_IsReleased(wrk));
+
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Copy req->bereq
  */
 
@@ -300,12 +413,6 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 {
-	struct http *hp, *hp2;
-	char *b;
-	uint16_t nhttp;
-	unsigned l;
-	struct vsb *vary = NULL;
-	int varyl = 0;
 	struct object *obj;
 	ssize_t est = -1;
 
@@ -388,97 +495,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->htc.body_status == BS_NONE)
 		bo->do_stream = 0;
 
-	l = 0;
-
-	/* Create Vary instructions */
-	if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
-		varyl = VRY_Create(bo, &vary);
-		if (varyl > 0) {
-			AN(vary);
-			assert(varyl == VSB_len(vary));
-			l += varyl;
-		} else if (varyl < 0) {
-			/*
-			 * Vary parse error
-			 * Complain about it, and make this a pass.
-			 */
-			VSLb(bo->vsl, SLT_Error,
-			    "Illegal 'Vary' header from backend, "
-			    "making this a pass.");
-			bo->uncacheable = 1;
-			AZ(vary);
-		} else
-			/* No vary */
-			AZ(vary);
-	}
-
-	l += http_EstimateWS(bo->beresp,
-	    bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
-
-	if (bo->uncacheable)
-		bo->fetch_objcore->flags |= OC_F_PASS;
-
-	if (bo->uncacheable ||
-	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
-		bo->storage_hint = TRANSIENT_STORAGE;
-
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
-	AN(bo->fetch_objcore);
-	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
-	if (obj == NULL) {
-		/*
-		 * Try to salvage the transaction by allocating a
-		 * shortlived object on Transient storage.
-		 */
-		if (bo->exp.ttl > cache_param->shortlived)
-			bo->exp.ttl = cache_param->shortlived;
-		bo->exp.grace = 0.0;
-		bo->exp.keep = 0.0;
-		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
-	}
-	if (obj == NULL) {
+	if (vbf_bereq2obj(wrk, bo)) {
 		bo->stats = NULL;
 		(void)VFP_Error(bo, "Could not get storage");
 		VDI_CloseFd(&bo->vbc);
 		return (F_STP_DONE);
 	}
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-
-	bo->storage_hint = NULL;
-
-	AZ(bo->fetch_obj);
-	bo->fetch_obj = obj;
-
-	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
-		obj->gziped = 1;
-
-	if (vary != NULL) {
-		obj->vary = (void *)WS_Copy(obj->http->ws,
-		    VSB_data(vary), varyl);
-		AN(obj->vary);
-		(void)VRY_Validate(obj->vary);
-		VSB_delete(vary);
-	}
 
-	obj->vxid = bo->vsl->wid;
-	obj->response = bo->err_code;
-	WS_Assert(bo->ws_o);
-
-	/* Filter into object */
-	hp = bo->beresp;
-	hp2 = obj->http;
-
-	hp2->logtag = SLT_ObjMethod;
-	http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(hp2);
-
-	if (http_GetHdr(hp, H_Last_Modified, &b))
-		obj->last_modified = VTIM_parse(b);
-	else
-		obj->last_modified = floor(bo->exp.t_origin);
-
-	assert(WRW_IsReleased(wrk));
+	obj = bo->fetch_obj;
 
 	/*
 	 * Ready to fetch the body
@@ -514,9 +538,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	bo->t_body = VTIM_mono();
 
-	http_Teardown(bo->bereq);
-	http_Teardown(bo->beresp);
-
 	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
 	    bo->htc.body_status, body_status_2str(bo->htc.body_status));
 
@@ -606,7 +627,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	obj->gzip_last = bo->ims_obj->gzip_last;
 	obj->gzip_stop = bo->ims_obj->gzip_stop;
 
-	/* XXX: ESI */
+	XXXAZ(bo->ims_obj->esidata);
 
 	if (bo->ims_obj->vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
@@ -676,6 +697,24 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 {
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	AN(bo->fetch_objcore->flags &= OC_F_BUSY);
+
+	// XXX: reset all beresp flags ?
+
+	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
+	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
+	http_SetHeader(bo->beresp, "Content-Length: 0");
+	http_SetHeader(bo->beresp, "Connection: close");
+
+	bo->exp.ttl = 0;
+	bo->exp.grace = 0;
+	bo->exp.keep = 0;
+
+	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
+
+	xxxassert(wrk->handling == VCL_RET_DELIVER);
+
 	WRONG("");
 	return (F_STP_DONE);
 }
@@ -749,6 +788,9 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		AZ(bo->vbc);
 	}
 
+	http_Teardown(bo->bereq);
+	http_Teardown(bo->beresp);
+
 	if (bo->state == BOS_FAILED)
 		assert(bo->fetch_objcore->flags & OC_F_FAILED);
 



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 2ddc91f Make bereq.* vars available in vcl_backend_error{}
Message-ID: <E1WO1sJ-0007Kw-6H@project.varnish-software.com>


commit 2ddc91fa4c49a219cc9890eed3803d46371e1861
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 18 13:49:20 2014 +0000

    Make bereq.* vars available in vcl_backend_error{}

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 29dcb9e..f313bd5 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -367,36 +367,36 @@ The client's IP address.
 	),
 	('beresp.proto',
 		'STRING',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		The HTTP protocol version used the backend replied with.
 		"""
 	),
 	('beresp.status',
 		'INT',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		The HTTP status code returned by the server.
 		"""
 	),
 	('beresp.reason',
 		'STRING',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		The HTTP status message returned by the server.
 		"""
 	),
 	('beresp.http.',
 		'HEADER',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		The corresponding HTTP header.
 		"""
 	),
 	('beresp.do_esi',
 		'BOOL',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		Boolean. ESI-process the object after fetching it.
 		Defaults to false. Set it to true to parse the
 		object for ESI directives. Will only be honored if
@@ -405,8 +405,8 @@ The client's IP address.
 	),
 	('beresp.do_stream',
 		'BOOL',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		Deliver the object to the client directly without
 		fetching the whole object into varnish. If this
 		request is pass'ed it will not be stored in memory.
@@ -416,8 +416,8 @@ The client's IP address.
 	),
 	('beresp.do_gzip',
 		'BOOL',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		Boolean. Gzip the object before storing it. Defaults
 		to false. When http_gzip_support is on Varnish will
 		request already compressed content from the backend
@@ -426,57 +426,57 @@ The client's IP address.
 	),
 	('beresp.do_gunzip',
 		'BOOL',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		Boolean. Unzip the object before storing it in the
 		cache.  Defaults to false.
 		"""
 	),
 	('beresp.uncacheable',
 		'BOOL',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		"""
 	),
 	('beresp.ttl',
 		'DURATION',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		The object's remaining time to live, in seconds.
 		beresp.ttl is writable.
 		"""
 	),
 	('beresp.grace',
 		'DURATION',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		Set to a period to enable grace.
 		"""
 	),
 	('beresp.keep',
 		'DURATION',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		"""
 	),
 	('beresp.backend.name',
 		'STRING',
-		( 'backend_response',),
+		( 'backend_response', 'backend_error'),
 		( ), """
 		Name of the backend this response was fetched from.
 		"""
 	),
 	('beresp.backend.ip',
 		'IP',
-		( 'backend_response',),
+		( 'backend_response', 'backend_error'),
 		( ), """
 		IP of the backend this response was fetched from.
 		"""
 	),
 	('beresp.storage_hint',
 		'STRING',
-		( 'backend_response',),
-		( 'backend_response',), """
+		( 'backend_response', 'backend_error'),
+		( 'backend_response', 'backend_error'), """
 		Hint to Varnish that you want to save this object to a
 		particular storage backend.
 		"""



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] a6277de Shift the 503 creation into the error step.
Message-ID: <E1WO1sJ-0007L6-7L@project.varnish-software.com>


commit a6277de361e1d17b5aabb0fd36baf705b2e418f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 18 14:13:27 2014 +0000

    Shift the 503 creation into the error step.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f044347..51cb160 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -41,21 +41,6 @@
 #include "vcl.h"
 #include "vtim.h"
 
-/*--------------------------------------------------------------------
- */
-
-static void
-make_it_503(struct busyobj *bo)
-{
-
-	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
-	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
-	http_SetHeader(bo->beresp, "Content-Length: 0");
-	http_SetHeader(bo->beresp, "Connection: close");
-	bo->exp.ttl = 0;
-	bo->exp.grace = 0;
-	bo->exp.keep = 0;
-}
 
 /*--------------------------------------------------------------------
  */
@@ -298,7 +283,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 {
-	int i, do_ims, fail;
+	int i, do_ims;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -330,14 +315,11 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	if (i) {
 		AZ(bo->vbc);
-		make_it_503(bo);
-		fail = 1;
-	} else {
-		AN(bo->vbc);
-		http_VSL_log(bo->beresp);
-		fail = 0;
-	}
+		return (F_STP_ERROR);
+	} 
 
+	AN(bo->vbc);
+	http_VSL_log(bo->beresp);
 
 	/*
 	 * These two headers can be spread over multiple actual headers
@@ -353,8 +335,6 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	 * NB: Also sets other wrk variables
 	 */
 	bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
-	if (i && bo->htc.body_status == BS_LENGTH)
-		bo->htc.body_status = BS_NONE;
 
 	bo->err_code = http_GetStatus(bo->beresp);
 
@@ -381,22 +361,18 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
 
 	if (wrk->handling == VCL_RET_RETRY) {
-		if (bo->vbc)
-			VDI_CloseFd(&bo->vbc);
+		AN (bo->vbc);
+		VDI_CloseFd(&bo->vbc);
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries)
 			return (F_STP_RETRY);
-
 		VSLb(bo->vsl, SLT_VCL_Error,
 		    "Too many retries, delivering 503");
-		make_it_503(bo);
-		wrk->handling = VCL_RET_DELIVER;
+		return (F_STP_ERROR);
 	}
 
 	assert(bo->state == BOS_REQ_DONE);
 	VBO_setstate(bo, BOS_COMMITTED);
-	if (fail)
-		(void)VFP_Error(bo, "Fetch failed");
 
 	if (bo->do_esi)
 		bo->do_stream = 0;
@@ -414,7 +390,7 @@ static enum fetch_step
 vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 {
 	struct object *obj;
-	ssize_t est = -1;
+	ssize_t est;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -465,8 +441,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	/* But we can't do both at the same time */
 	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
 
-	if (bo->vbc != NULL)
-		est = V1F_Setup_Fetch(bo);
+	AN(bo->vbc);
+	est = V1F_Setup_Fetch(bo);
 
 	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
 		RFC2616_Weaken_Etag(bo->beresp);
@@ -715,7 +691,21 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	xxxassert(wrk->handling == VCL_RET_DELIVER);
 
-	WRONG("");
+	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
+	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
+
+	if (vbf_bereq2obj(wrk, bo)) {
+		INCOMPL();
+	}
+
+	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
+
+	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
+		EXP_Insert(bo->fetch_obj->objcore);
+		AN(bo->fetch_obj->objcore->ban);
+	}
+	VBO_setstate(bo, BOS_FINISHED);
+	HSH_Complete(bo->fetch_obj->objcore);
 	return (F_STP_DONE);
 }
 
diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc
index 9905e96..029f782 100644
--- a/bin/varnishtest/tests/b00015.vtc
+++ b/bin/varnishtest/tests/b00015.vtc
@@ -21,7 +21,7 @@ client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 503
-	expect resp.http.X-varnish == "1005"
+	expect resp.http.X-varnish == "1004"
 } -run
 
 delay .1
@@ -46,7 +46,7 @@ client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 302
-	expect resp.http.X-varnish == "1009"
+	expect resp.http.X-varnish == "1007"
 } -run
 
 delay .1
@@ -55,7 +55,7 @@ client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 302
-	expect resp.http.X-varnish == "1012 1010"
+	expect resp.http.X-varnish == "1010 1008"
 } -run
 
 delay .1
@@ -79,7 +79,7 @@ client c1 {
 	txreq -url "/2"
 	rxresp
 	expect resp.status == 502
-	expect resp.http.X-varnish == "1014"
+	expect resp.http.X-varnish == "1012"
 } -run
 
 delay .1
@@ -88,7 +88,7 @@ client c1 {
 	txreq -url "/2"
 	rxresp
 	expect resp.status == 502
-	expect resp.http.X-varnish == "1017 1015"
+	expect resp.http.X-varnish == "1015 1013"
 } -run
 
 delay .1
diff --git a/bin/varnishtest/tests/r01395.vtc b/bin/varnishtest/tests/r01395.vtc
index c3243f6..acad0fe 100644
--- a/bin/varnishtest/tests/r01395.vtc
+++ b/bin/varnishtest/tests/r01395.vtc
@@ -5,8 +5,8 @@ varnish v1 -vcl {
 		.host = "${bad_ip}"; .port = "9090";
 	}
 
-	sub vcl_error {
-		set obj.status = 299;
+	sub vcl_backend_error {
+		set beresp.status = 299;
 	}
 } -start
 



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] ed7b310 Only attempt to salvage an object into Transient if it
 wasn't already destined for Transient in the first place.
Message-ID: <E1WO1sJ-0007LG-8P@project.varnish-software.com>


commit ed7b3105e82ff289d0ee8132fcacac0dd7cfe645
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 19 08:05:59 2014 +0000

    Only attempt to salvage an object into Transient if it wasn't already
    destined for Transient in the first place.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 51cb160..661d802 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -107,14 +107,9 @@ vbf_bereq2obj(struct worker *wrk, struct busyobj *bo)
 	bo->stats = &wrk->stats;
 	AN(bo->fetch_objcore);
 	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
-#if 0
-	// XXX: we shouldn't retry if we're already on Transient
 	if (obj == NULL &&
 	    (bo->storage_hint == NULL ||
 	    strcmp(bo->storage_hint, TRANSIENT_STORAGE))) {
-#else
-	if (obj == NULL) {
-#endif
 		/*
 		 * Try to salvage the transaction by allocating a
 		 * shortlived object on Transient storage.
@@ -316,7 +311,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	if (i) {
 		AZ(bo->vbc);
 		return (F_STP_ERROR);
-	} 
+	}
 
 	AN(bo->vbc);
 	http_VSL_log(bo->beresp);
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 697f104..6299926 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -36,8 +36,8 @@ client c1 {
 	delay 1
 } -run
 
-# Two failures, one for obj2 and two for the attempts at sending error
-varnish v1 -expect SMA.Transient.c_fail == 3
+# Two failures, one for obj2 and one for the attempts at sending error
+varnish v1 -expect SMA.Transient.c_fail == 2
 
 client c1 {
 	# Check that Varnish is still alive



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 128ca35 remove a couple of pointless copies of the HTTP status.
Message-ID: <E1WO1sJ-0007LR-AB@project.varnish-software.com>


commit 128ca35f6a48b5bea340d9ad734dd7c5394dd8ce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 19 08:35:05 2014 +0000

    remove a couple of pointless copies of the HTTP status.
    
    Pick it out of struct http when needed.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7ab3a1a..623e5ba 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -524,7 +524,6 @@ struct busyobj {
 	int			retries;
 	unsigned		refcount;
 	double			t_fetch;
-	uint16_t		err_code;
 	struct req		*req;
 
 	uint8_t			*vary;
@@ -591,7 +590,6 @@ struct object {
 	struct objcore		*objcore;
 
 	uint8_t			*vary;
-	uint16_t		response;
 
 	/* XXX: make bitmap */
 	uint8_t			gziped;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 661d802..4e4b229 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -142,7 +142,6 @@ vbf_bereq2obj(struct worker *wrk, struct busyobj *bo)
 	}
 
 	obj->vxid = bo->vsl->wid;
-	obj->response = bo->err_code;
 	WS_Assert(bo->ws_o);
 
 	/* Filter into object */
@@ -331,8 +330,6 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	 */
 	bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
 
-	bo->err_code = http_GetStatus(bo->beresp);
-
 	/*
 	 * What does RFC2616 think about TTL ?
 	 */
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 66982b2..2c681d6 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -93,7 +93,7 @@ v1d_dorange(struct req *req, const char *r)
 	ssize_t low, high, has_low;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	assert(req->obj->response == 200);
+	assert(http_GetStatus(req->obj->http) == 200);
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -271,7 +271,7 @@ V1D_Deliver(struct req *req)
 	    req->wantbody &&
 	    !(req->res_mode & (RES_ESI|RES_ESI_CHILD)) &&
 	    cache_param->http_range_support &&
-	    req->obj->response == 200) {
+	    http_GetStatus(req->obj->http) == 200) {
 		http_SetHeader(req->resp, "Accept-Ranges: bytes");
 		if (http_GetHdr(req->http, H_Range, &r))
 			v1d_dorange(req, r);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index b2f3ef0..cb78866 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -147,7 +147,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	if (!(req->obj->objcore->flags & OC_F_PASS)
 	    && req->esi_level == 0
-	    && req->obj->response == 200
+	    && http_GetStatus(req->obj->http) == 200
 	    && req->http->conds && RFC2616_Do_Cond(req))
 		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
 
@@ -304,7 +304,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	req->obj = oc_getobj(&wrk->stats, req->objcore);
 	req->objcore = NULL;
-	req->err_code = req->obj->response;
+	req->err_code = http_GetStatus(req->obj->http);
 	req->req_step = R_STP_DELIVER;
 	return (REQ_FSM_MORE);
 }



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 6675a65 Ensure that bitfields get set.
Message-ID: <E1WO1sJ-0007La-BB@project.varnish-software.com>


commit 6675a655c54c1ae55445d3c5b40839cc14ca11c9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 19 09:19:45 2014 +0000

    Ensure that bitfields get set.

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 2b1629a..2c79c0e 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -159,7 +159,7 @@ VRT_l_bereq_uncacheable(const struct vrt_ctx *ctx, unsigned a)
 		VSLb(ctx->vsl, SLT_VCL_Error,
 		    "Ignoring attempt to reset bereq.uncacheable");
 	} else if (a) {
-		ctx->bo->do_pass = a;
+		ctx->bo->do_pass = 1;
 	}
 }
 
@@ -182,7 +182,7 @@ VRT_l_beresp_uncacheable(const struct vrt_ctx *ctx, unsigned a)
 		    "Ignoring attempt to reset beresp.uncacheable");
 	} else if (a) {
 		VSLb(ctx->vsl, SLT_Debug, "XXXX: UNCACHEABLE %u" , a);
-		ctx->bo->uncacheable = a;
+		ctx->bo->uncacheable = 1;
 	}
 }
 



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] d2a7d10 Use the correct flag to controll EXP_Touch() calls
Message-ID: <E1WO1sJ-0007Lj-C6@project.varnish-software.com>


commit d2a7d10fb6747720ad8a896939cdb9a69cd07271
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 21 08:36:45 2014 +0000

    Use the correct flag to controll EXP_Touch() calls

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index cb78866..ba3ef0d 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -100,7 +100,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	assert(req->obj->objcore->refcnt > 0);
 
 	req->t_resp = W_TIM_real(wrk);
-	if (!(req->obj->objcore->flags & OC_F_PRIVATE))
+	if (req->obj->objcore->flags & OC_F_EXP)
 		EXP_Touch(req->obj->objcore, req->t_resp);
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] d1f1a6d Gently shuffle things around a bit.
Message-ID: <E1WO1sJ-0007Ls-DD@project.varnish-software.com>


commit d1f1a6dd90d9f73ca33d6a76b42be7e3a9431ea0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 21 09:00:16 2014 +0000

    Gently shuffle things around a bit.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index eeda901..1471a64 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -234,6 +234,8 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
+	AN(oc->flags & OC_F_EXP);
+
 	if (!isnan(ttl))
 		o->exp.ttl = now + ttl - o->exp.t_origin;
 	if (!isnan(grace))
@@ -253,7 +255,6 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
-	AN(oc->flags & OC_F_EXP);
 
 	if (!isnan(now) && when <= now)
 		oc->flags |= OC_F_DYING;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 4e4b229..cdea235 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -56,10 +56,11 @@ vbf_release_req(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ * Turn the beresp into a obj
  */
 
 static int
-vbf_bereq2obj(struct worker *wrk, struct busyobj *bo)
+vbf_beresp2obj(struct worker *wrk, struct busyobj *bo)
 {
 	unsigned l;
 	char *b;
@@ -157,8 +158,6 @@ vbf_bereq2obj(struct worker *wrk, struct busyobj *bo)
 	else
 		obj->last_modified = floor(bo->exp.t_origin);
 
-	assert(WRW_IsReleased(wrk));
-
 	return (0);
 }
 
@@ -302,16 +301,16 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 		i = V1F_fetch_hdr(wrk, bo, bo->req);
 	}
 
-	if (bo->do_pass && bo->req != NULL)
-		vbf_release_req(bo); /* XXX : retry ?? */
-
-	AZ(bo->req);
-
 	if (i) {
 		AZ(bo->vbc);
 		return (F_STP_ERROR);
 	}
 
+	if (bo->do_pass && bo->req != NULL)
+		vbf_release_req(bo); /* XXX : retry ?? */
+
+	AZ(bo->req);
+
 	AN(bo->vbc);
 	http_VSL_log(bo->beresp);
 
@@ -463,13 +462,15 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->htc.body_status == BS_NONE)
 		bo->do_stream = 0;
 
-	if (vbf_bereq2obj(wrk, bo)) {
+	if (vbf_beresp2obj(wrk, bo)) {
 		bo->stats = NULL;
 		(void)VFP_Error(bo, "Could not get storage");
 		VDI_CloseFd(&bo->vbc);
 		return (F_STP_DONE);
 	}
 
+	assert(WRW_IsReleased(wrk));
+
 	obj = bo->fetch_obj;
 
 	/*
@@ -536,11 +537,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (!bo->do_stream && bo->state != BOS_FAILED)
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 
-	if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(obj->objcore);
-		AN(obj->objcore->ban);
-	}
-
 	HSH_Complete(obj->objcore);
 
 	assert(bo->refcount >= 1);
@@ -615,10 +611,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	VBO_setstate(bo, BOS_FETCHING);
 	HSH_Unbusy(&wrk->stats, obj->objcore);
 
-	if (!(obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(obj->objcore);
-		AN(obj->objcore->ban);
-	}
 
 	st = NULL;
 	al = 0;
@@ -683,19 +675,17 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	xxxassert(wrk->handling == VCL_RET_DELIVER);
 
+	if (bo->req != NULL)
+		vbf_release_req(bo);
+
 	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
 	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
 
-	if (vbf_bereq2obj(wrk, bo)) {
+	if (vbf_beresp2obj(wrk, bo)) {
 		INCOMPL();
 	}
 
 	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
-
-	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(bo->fetch_obj->objcore);
-		AN(bo->fetch_obj->objcore->ban);
-	}
 	VBO_setstate(bo, BOS_FINISHED);
 	HSH_Complete(bo->fetch_obj->objcore);
 	return (F_STP_DONE);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index b7eba44..9d5f9d1 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -678,8 +678,12 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	if (oh->waitinglist != NULL)
 		hsh_rush(ds, oh);
 	Lck_Unlock(&oh->mtx);
-	if (!(oc->flags & OC_F_PRIVATE))
+	if (!(oc->flags & OC_F_PRIVATE)) {
 		BAN_NewObjCore(oc);
+		EXP_Insert(oc);
+		AN(oc->flags & OC_F_EXP);
+		AN(oc->ban);
+	}
 }
 
 /*---------------------------------------------------------------------



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] f1708eb Deal with BS_ERROR much earlier
Message-ID: <E1WO1sJ-0007M4-EK@project.varnish-software.com>


commit f1708ebab3c417c6c75ce771d4952e0de3c842a2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 21 09:35:39 2014 +0000

    Deal with BS_ERROR much earlier

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cdea235..de527da 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -329,6 +329,13 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	 */
 	bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
 
+	if (bo->htc.body_status == BS_ERROR) {
+		AN (bo->vbc);
+		VDI_CloseFd(&bo->vbc);
+		VSLb(bo->vsl, SLT_VCL_Error, "Body cannot be fetched");
+		return (F_STP_ERROR);
+	}
+
 	/*
 	 * What does RFC2616 think about TTL ?
 	 */
@@ -488,19 +495,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	else if (bo->state != BOS_FAILED)
 		WRONG("Wrong bo->state");
 
-	switch (bo->htc.body_status) {
-	case BS_NONE:
-		break;
-	case BS_ERROR:
-		/* XXX: Why not earlier ? */
-		bo->should_close |=
-		    VFP_Error(bo, "error incompatible Transfer-Encoding");
-		break;
-	default:
-		if (bo->vbc == NULL)
-			(void)VFP_Error(bo, "Backend connection gone");
-		else
-			VFP_Fetch_Body(bo, est);
+	if (bo->htc.body_status != BS_NONE) {
+		assert(bo->htc.body_status  != BS_ERROR);
+		VFP_Fetch_Body(bo, est);
 	}
 
 	bo->stats = NULL;



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] e734b2a Now that busyobj has its own worker thread,
 borrow the stats throughout the show.
Message-ID: <E1WO1sJ-0007MD-Fh@project.varnish-software.com>


commit e734b2a4e9e89b169765b66289ea99651c65d3fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 21 09:50:39 2014 +0000

    Now that busyobj has its own worker thread, borrow the stats
    throughout the show.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index de527da..0b9bf21 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -60,7 +60,7 @@ vbf_release_req(struct busyobj *bo)
  */
 
 static int
-vbf_beresp2obj(struct worker *wrk, struct busyobj *bo)
+vbf_beresp2obj(struct busyobj *bo)
 {
 	unsigned l;
 	char *b;
@@ -104,8 +104,6 @@ vbf_beresp2obj(struct worker *wrk, struct busyobj *bo)
 	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
 		bo->storage_hint = TRANSIENT_STORAGE;
 
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
 	AN(bo->fetch_objcore);
 	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
 	if (obj == NULL &&
@@ -469,8 +467,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->htc.body_status == BS_NONE)
 		bo->do_stream = 0;
 
-	if (vbf_beresp2obj(wrk, bo)) {
-		bo->stats = NULL;
+	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo, "Could not get storage");
 		VDI_CloseFd(&bo->vbc);
 		return (F_STP_DONE);
@@ -500,8 +497,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		VFP_Fetch_Body(bo, est);
 	}
 
-	bo->stats = NULL;
-
 	bo->t_body = VTIM_mono();
 
 	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
@@ -572,7 +567,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		vl = 0;
 	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
 
-	bo->stats = &wrk->stats;
 	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
 	if (obj == NULL) {
 		(void)VFP_Error(bo, "Could not get storage");
@@ -634,7 +628,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	} while (bo->state < BOS_FAILED &&
 	    (ois == OIS_DATA || ois == OIS_STREAM));
 	ObjIterEnd(&oi);
-	bo->stats = NULL;
 	if (bo->state != BOS_FAILED) {
 		assert(al == bo->ims_obj->len);
 		assert(obj->len == al);
@@ -678,7 +671,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
 	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
 
-	if (vbf_beresp2obj(wrk, bo)) {
+	if (vbf_beresp2obj(bo)) {
 		INCOMPL();
 	}
 
@@ -731,6 +724,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	bo->t_hdr = NAN;
 	bo->t_body = NAN;
 
+	bo->stats = &wrk->stats;
+
 	while (stp != F_STP_DONE) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		bo->step = stp;
@@ -749,6 +744,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	}
 	assert(WRW_IsReleased(wrk));
 
+	bo->stats = NULL;
+
 	if (bo->vbc != NULL) {
 		if (bo->should_close)
 			VDI_CloseFd(&bo->vbc);



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 6801c39 Eliminate some debugging which has overstayed its
 usefulness
Message-ID: <E1WO1sJ-0007MM-Gn@project.varnish-software.com>


commit 6801c3965be74f78b7986146709d366ab10e2de9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 21 10:01:08 2014 +0000

    Eliminate some debugging which has overstayed its usefulness

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0b9bf21..5b89bf1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -536,8 +536,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->state != BOS_FAILED)
 		VBO_setstate(bo, BOS_FINISHED);
 
-VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d",
-    bo->refcount, bo->fetch_obj->objcore->refcnt);
 	return (F_STP_DONE);
 }
 
@@ -691,20 +689,6 @@ vbf_stp_done(void)
 	return (F_STP_DONE);
 }
 
-static const char *
-vbf_step_name(enum fetch_step stp)
-{
-	switch (stp) {
-#define FETCH_STEP(l, U, arg)						\
-		case F_STP_##U:						\
-			return (#U);
-#include "tbl/steps.h"
-#undef FETCH_STEP
-	default:
-		return ("F-step ?");
-	}
-}
-
 static void
 vbf_fetch_thread(struct worker *wrk, void *priv)
 {
@@ -739,8 +723,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		default:
 			WRONG("Illegal fetch_step");
 		}
-		VSLb(bo->vsl, SLT_Debug, "%s -> %s",
-		    vbf_step_name(bo->step), vbf_step_name(stp));
 	}
 	assert(WRW_IsReleased(wrk));
 



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 911194b Collect code for increased clarity
Message-ID: <E1WO1sJ-0007MV-I7@project.varnish-software.com>


commit 911194b2d03a57637b7ce4fb00dd6479f21e7703
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 24 14:08:55 2014 +0000

    Collect code for increased clarity

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5b89bf1..890b247 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -422,21 +422,19 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->do_gunzip && !bo->is_gzip)
 		bo->do_gunzip = 0;
 
-	/* If we do gunzip, remove the C-E header */
-	if (bo->do_gunzip)
-		http_Unset(bo->beresp, H_Content_Encoding);
-
 	/* We wont gzip unless it is ungziped */
 	if (bo->do_gzip && !bo->is_gunzip)
 		bo->do_gzip = 0;
 
-	/* If we do gzip, add the C-E header */
-	if (bo->do_gzip)
-		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
-
 	/* But we can't do both at the same time */
 	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
 
+	/* Fix Content-Encoding, as appropriate */
+	if (bo->do_gzip)
+		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
+	else if (bo->do_gunzip)
+		http_Unset(bo->beresp, H_Content_Encoding);
+
 	AN(bo->vbc);
 	est = V1F_Setup_Fetch(bo);
 



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 6895fcb Use the same storage allocation policy for 304/IMS as
 for regular fetches.
Message-ID: <E1WO1sJ-0007Mg-JC@project.varnish-software.com>


commit 6895fcb6801994073068d9d6d48b3f6a84f77868
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 25 08:28:16 2014 +0000

    Use the same storage allocation policy for 304/IMS as for regular
    fetches.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 890b247..914c6d4 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -56,6 +56,52 @@ vbf_release_req(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ * Allocate an object, with fall-back to Transient.
+ * XXX: This somewhat overlaps the stuff in stevedore.c
+ * XXX: Should this be merged over there ?
+ */
+
+static struct object *
+vbf_allocobj(struct busyobj *bo, unsigned l, uint16_t nhttp)
+{
+	struct object *obj;
+	const char *storage_hint;
+	double lifetime;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
+
+	lifetime = bo->exp.ttl + bo->exp.grace + bo->exp.keep;
+
+	if (bo->uncacheable || lifetime < cache_param->shortlived)
+		storage_hint = TRANSIENT_STORAGE;
+	else
+		storage_hint = bo->storage_hint;
+
+	bo->storage_hint = NULL;
+
+	obj = STV_NewObject(bo, storage_hint, l, nhttp);
+
+	if (obj != NULL)
+		return (obj);
+
+	if (storage_hint != NULL && !strcmp(storage_hint, TRANSIENT_STORAGE))
+		return (NULL);
+
+	/*
+	 * Try to salvage the transaction by allocating a shortlived object
+	 * on Transient storage.
+	 */
+
+	if (bo->exp.ttl > cache_param->shortlived)
+		bo->exp.ttl = cache_param->shortlived;
+	bo->exp.grace = 0.0;
+	bo->exp.keep = 0.0;
+	obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
+	return (obj);
+}
+
+/*--------------------------------------------------------------------
  * Turn the beresp into a obj
  */
 
@@ -100,32 +146,13 @@ vbf_beresp2obj(struct busyobj *bo)
 	if (bo->uncacheable)
 		bo->fetch_objcore->flags |= OC_F_PASS;
 
-	if (bo->uncacheable ||
-	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
-		bo->storage_hint = TRANSIENT_STORAGE;
-
-	AN(bo->fetch_objcore);
-	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
-	if (obj == NULL &&
-	    (bo->storage_hint == NULL ||
-	    strcmp(bo->storage_hint, TRANSIENT_STORAGE))) {
-		/*
-		 * Try to salvage the transaction by allocating a
-		 * shortlived object on Transient storage.
-		 */
-		if (bo->exp.ttl > cache_param->shortlived)
-			bo->exp.ttl = cache_param->shortlived;
-		bo->exp.grace = 0.0;
-		bo->exp.keep = 0.0;
-		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
-	}
+	obj = vbf_allocobj(bo, l, nhttp);
+
 	if (obj == NULL)
 		return (-1);
 
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 
-	bo->storage_hint = NULL;
-
 	AZ(bo->fetch_obj);
 	bo->fetch_obj = obj;
 
@@ -563,7 +590,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		vl = 0;
 	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
 
-	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
+	obj = vbf_allocobj(bo, l, nhttp);
 	if (obj == NULL) {
 		(void)VFP_Error(bo, "Could not get storage");
 		VDI_CloseFd(&bo->vbc);



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 8c8b500 Drop the BOS_COMMITTED state,
 and rename BOS_FETCHING to BOS_STREAM which is really what client
 threads care about.
Message-ID: <E1WO1sJ-0007Mq-KV@project.varnish-software.com>


commit 8c8b5007403f80ecb4cca537d1fb8f7bc0648e1f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 25 09:27:23 2014 +0000

    Drop the BOS_COMMITTED state, and rename BOS_FETCHING to BOS_STREAM
    which is really what client threads care about.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 623e5ba..5db6e22 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -503,8 +503,7 @@ oc_getlru(const struct objcore *oc)
 enum busyobj_state_e {
 	BOS_INVALID = 0,	/* don't touch (yet) */
 	BOS_REQ_DONE,		/* beresp.* can be examined */
-	BOS_COMMITTED,		/* beresp.* can be examined */
-	BOS_FETCHING,		/* beresp.* can be examined */
+	BOS_STREAM,		/* beresp.* can be examined */
 	BOS_FINISHED,		/* object is complete */
 	BOS_FAILED,		/* something went wrong */
 };
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 914c6d4..b64b2bb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -395,7 +395,6 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	}
 
 	assert(bo->state == BOS_REQ_DONE);
-	VBO_setstate(bo, BOS_COMMITTED);
 
 	if (bo->do_esi)
 		bo->do_stream = 0;
@@ -509,13 +508,13 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	assert(bo->refcount >= 1);
 
 	AZ(WS_Overflowed(bo->ws_o));
-	if (bo->do_stream)
-		HSH_Unbusy(&wrk->stats, obj->objcore);
 
-	if (bo->state == BOS_COMMITTED)
-		VBO_setstate(bo, BOS_FETCHING);
-	else if (bo->state != BOS_FAILED)
-		WRONG("Wrong bo->state");
+	assert (bo->state == BOS_REQ_DONE);
+
+	if (bo->do_stream) {
+		HSH_Unbusy(&wrk->stats, obj->objcore);
+		VBO_setstate(bo, BOS_STREAM);
+	}
 
 	if (bo->htc.body_status != BS_NONE) {
 		assert(bo->htc.body_status  != BS_ERROR);
@@ -530,7 +529,10 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 	} else {
-		assert(bo->state == BOS_FETCHING);
+		if (bo->do_stream)
+			assert(bo->state == BOS_STREAM);
+		else
+			assert(bo->state == BOS_REQ_DONE);
 
 		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
 
@@ -622,7 +624,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 
 	AZ(WS_Overflowed(bo->ws_o));
-	VBO_setstate(bo, BOS_FETCHING);
+	VBO_setstate(bo, BOS_STREAM);
 	HSH_Unbusy(&wrk->stats, obj->objcore);
 
 
@@ -849,9 +851,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 	if (mode == VBF_BACKGROUND) {
 		VBO_waitstate(bo, BOS_REQ_DONE);
 	} else {
-		VBO_waitstate(bo, BOS_FETCHING);
-		if (!bo->do_stream)
-			VBO_waitstate(bo, BOS_FINISHED);
+		VBO_waitstate(bo, BOS_STREAM);
 		assert(bo->state != BOS_FAILED || (oc->flags & OC_F_FAILED));
 	}
 	VBO_DerefBusyObj(wrk, &bo);
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index ea928e4..b21ca15 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -61,7 +61,7 @@ VFP_Error(struct busyobj *bo, const char *fmt, ...)
 	va_list ap;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	assert(bo->state >= BOS_COMMITTED);
+	assert(bo->state >= BOS_REQ_DONE);
 	if (bo->state < BOS_FAILED) {
 		va_start(ap, fmt);
 		VSLbv(bo->vsl, SLT_FetchError, fmt, ap);



From phk at FreeBSD.org  Thu Mar 13 09:24:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 25b38f9 Add the internal bo->failed state we need to allow
 retries separate from the external BOS_FAILED state.
Message-ID: <E1WO1sJ-0007Mz-Lk@project.varnish-software.com>


commit 25b38f92702e74d0014c36033b24755c79dddada
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 25 10:29:24 2014 +0000

    Add the internal bo->failed state we need to allow retries separate
    from the external BOS_FAILED state.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5db6e22..7c6079e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -500,6 +500,9 @@ oc_getlru(const struct objcore *oc)
  * streaming delivery will make use of.
  */
 
+/*
+ * The macro-states we expose outside the fetch code
+ */
 enum busyobj_state_e {
 	BOS_INVALID = 0,	/* don't touch (yet) */
 	BOS_REQ_DONE,		/* beresp.* can be examined */
@@ -514,7 +517,6 @@ struct busyobj {
 	struct lock		mtx;
 	pthread_cond_t		cond;
 	char			*end;
-	enum fetch_step		step;
 
 	/*
 	 * All fields from refcount and down are zeroed when the busyobj
@@ -532,6 +534,7 @@ struct busyobj {
 	intptr_t		vfps_priv[N_VFPS];
 	int			vfp_nxt;
 
+	int			failed;
 	enum busyobj_state_e	state;
 
 	struct ws		ws[1];
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b64b2bb..0c621df 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -494,7 +494,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo, "Could not get storage");
 		VDI_CloseFd(&bo->vbc);
-		return (F_STP_DONE);
+		return (F_STP_ERROR);
 	}
 
 	assert(WRW_IsReleased(wrk));
@@ -526,7 +526,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
 	    bo->htc.body_status, body_status_2str(bo->htc.body_status));
 
-	if (bo->state == BOS_FAILED) {
+	if (bo->failed) {
 		wrk->stats.fetch_failed++;
 	} else {
 		if (bo->do_stream)
@@ -553,16 +553,29 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		}
 	}
 
-	if (!bo->do_stream && bo->state != BOS_FAILED)
-		HSH_Unbusy(&wrk->stats, obj->objcore);
+	if (!bo->do_stream) {
+		if (bo->failed) {
+			if (bo->fetch_obj != NULL) {
+				oc_freeobj(bo->fetch_objcore);
+				bo->fetch_obj = NULL;
+				bo->stats->n_object--;
+			}
+			return (F_STP_ERROR);
+		} else {
+			HSH_Unbusy(&wrk->stats, obj->objcore);
+			VBO_setstate(bo, BOS_FINISHED);
+		}
+	} else if (bo->failed) {
+		HSH_Fail(bo->fetch_objcore);
+		VBO_setstate(bo, BOS_FAILED);
+	} else {
+		VBO_setstate(bo, BOS_FINISHED);
+	}
 
 	HSH_Complete(obj->objcore);
 
 	assert(bo->refcount >= 1);
 
-	if (bo->state != BOS_FAILED)
-		VBO_setstate(bo, BOS_FINISHED);
-
 	return (F_STP_DONE);
 }
 
@@ -650,14 +663,16 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 			if (st->len == st->space)
 				st = NULL;
 		}
-	} while (bo->state < BOS_FAILED &&
-	    (ois == OIS_DATA || ois == OIS_STREAM));
+	} while (!bo->failed && (ois == OIS_DATA || ois == OIS_STREAM));
 	ObjIterEnd(&oi);
-	if (bo->state != BOS_FAILED) {
+	if (!bo->failed) {
 		assert(al == bo->ims_obj->len);
 		assert(obj->len == al);
 		VBO_setstate(bo, BOS_FINISHED);
 		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+	} else {
+		HSH_Fail(bo->fetch_objcore);
+		VBO_setstate(bo, BOS_FAILED);
 	}
 	HSH_Complete(obj->objcore);
 	return (F_STP_DONE);
@@ -673,7 +688,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	AN(bo->fetch_objcore->flags &= OC_F_BUSY);
+	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
 	// XXX: reset all beresp flags ?
 
@@ -697,7 +712,9 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
 
 	if (vbf_beresp2obj(bo)) {
-		INCOMPL();
+		VBO_setstate(bo, BOS_FAILED);
+		HSH_Fail(bo->fetch_objcore);
+		return (F_STP_DONE);
 	}
 
 	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
@@ -739,7 +756,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	while (stp != F_STP_DONE) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-		bo->step = stp;
 		switch(stp) {
 #define FETCH_STEP(l, U, arg)						\
 		case F_STP_##U:						\
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index b21ca15..8861e1d 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -62,12 +62,11 @@ VFP_Error(struct busyobj *bo, const char *fmt, ...)
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	assert(bo->state >= BOS_REQ_DONE);
-	if (bo->state < BOS_FAILED) {
+	if (!bo->failed) {
 		va_start(ap, fmt);
 		VSLbv(bo->vsl, SLT_FetchError, fmt, ap);
 		va_end(ap);
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
+		bo->failed = 1;
 	}
 	return (VFP_ERROR);
 }
@@ -214,7 +213,7 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 			bo->should_close = 1;
 			break;
 		}
-		assert(bo->state != BOS_FAILED);
+		AZ(bo->failed);
 		if (st == NULL) {
 			st = VFP_GetStorage(bo, est);
 			est = 0;
@@ -228,7 +227,7 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		assert(st == VTAILQ_LAST(&bo->fetch_obj->store, storagehead));
 		l = st->space - st->len;
-		assert(bo->state != BOS_FAILED);
+		AZ(bo->failed);
 		vfps = VFP_Suck(bo, st->ptr + st->len, &l);
 		if (l > 0 && vfps != VFP_ERROR) {
 			assert(!VTAILQ_EMPTY(&bo->fetch_obj->store));
@@ -239,7 +238,7 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 	} while (vfps == VFP_OK);
 
 	if (vfps == VFP_ERROR) {
-		assert(bo->state == BOS_FAILED);
+		AN(bo->failed);
 		(void)VFP_Error(bo, "Fetch Pipeline failed to process");
 		bo->should_close = 1;
 	}
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 6299926..123ddb2 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -19,6 +19,10 @@ varnish v1 \
 	}
 } -start
 
+varnish v1 -cliok "param.set debug +syncvsl"
+
+delay .1
+
 client c1 {
 	# Fill transient
 	txreq -url "/obj1"
@@ -26,6 +30,8 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+delay .1
+
 varnish v1 -expect SMA.Transient.g_bytes > 1048000
 varnish v1 -expect SMA.Transient.g_space < 200
 
@@ -36,8 +42,8 @@ client c1 {
 	delay 1
 } -run
 
-# Two failures, one for obj2 and one for the attempts at sending error
-varnish v1 -expect SMA.Transient.c_fail == 2
+# Three failures, one for obj2, one for vcl_backend_error{} and for vcl_error{}
+varnish v1 -expect SMA.Transient.c_fail == 3
 
 client c1 {
 	# Check that Varnish is still alive



From lkarsten at varnish-software.com  Thu Mar 13 09:24:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] fd2f519 Fix faulty example. Reformat tabs/spaces
Message-ID: <E1WO1sJ-0007N9-Mj@project.varnish-software.com>


commit fd2f5191b21721ed3c6a6f7e56f0605333bd9c5c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Feb 25 12:39:40 2014 +0100

    Fix faulty example. Reformat tabs/spaces
    
    Fixes #1438.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index d5b1aae..ee21f4d 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -11,17 +11,17 @@ your favorite text editor and open the relevant VCL file.
 
 Somewhere in the top there will be a section that looks a bit like this.::
 
-	  # backend default {
-	  #     .host = "127.0.0.1";
-	  #     .port = "8080";
-	  # }
+    # backend default {
+    #     .host = "127.0.0.1";
+    #     .port = "8080";
+    # }
 
 We comment in this bit of text making the text look like.::
 
-          backend default {
-                .host = "127.0.0.1";
-    		.port = "8080";
-	  }
+    backend default {
+        .host = "127.0.0.1";
+        .port = "8080";
+    }
 
 Now, this piece of configuration defines a backend in Varnish called
 *default*. When Varnish needs to get content from this backend it will
@@ -46,27 +46,27 @@ site. Lets say our Java application should handle URL beginning with
 We manage to get the thing up and running on port 8000. Now, lets have
 a look at the default.vcl.::
 
-  backend default {
-      .host = "127.0.0.1";
-      .port = "8080";
-  }
+    backend default {
+        .host = "127.0.0.1";
+        .port = "8080";
+    }
 
 We add a new backend.::
 
-  backend java {
-      .host = "127.0.0.1";
-      .port = "8000";
-  }
+    backend java {
+        .host = "127.0.0.1";
+        .port = "8000";
+    }
 
 Now we need tell where to send the difference URL. Lets look at vcl_recv.::
 
-  sub vcl_recv {
-      if (req.url ~ "^/java/") {
-          set req.backend = java;
-      } else {
-          set req.backend = default.
-      }
-  }
+    sub vcl_recv {
+        if (req.url ~ "^/java/") {
+            set req.backend = java;
+        } else {
+            set req.backend = default.
+        }
+    }
 
 It's quite simple, really. Lets stop and think about this for a
 moment. As you can see you can define how you choose backends based on
@@ -86,25 +86,24 @@ hosts you just need to inspect req.http.host.
 
 You can have something like this:::
 
-  sub vcl_recv {
-     if (req.http.host ~ "foo.com") {
-       set req.backend = foo;
-     } elsif (req.http.host ~ "bar.com") {
-       set req.backend = bar;
-     }
-  }
+    sub vcl_recv {
+        if (req.http.host ~ "foo.com") {
+            set req.backend = foo;
+        } elsif (req.http.host ~ "bar.com") {
+            set req.backend = bar;
+        }
+    }
 
 Note that the first regular expressions will match foo.com,
 www.foo.com, zoop.foo.com and any other host ending in foo.com. In
 this example this is intentional but you might want it to be a bit
 more tight, maybe relying on the == operator in stead, like this:::
 
-
-  sub vcl_recv {
-    if (req.http.host == "foo.com" or req.http.host == "www.foo.com") {
-      set req.backend = foo;
+    sub vcl_recv {
+        if (req.http.host == "foo.com" or req.http.host == "www.foo.com") {
+            set req.backend = foo;
+        }
     }
-  }
 
 
 .. _users-guide-advanced_backend_servers-directors:
@@ -115,37 +114,37 @@ Directors
 
 You can also group several backend into a group of backends. These
 groups are called directors. This will give you increased performance
-and resilience. 
+and resilience.
 
 You can define several backends and group them together in a
-director. This requires you to load a VMOD, a Varnish module, and then to 
+director. This requires you to load a VMOD, a Varnish module, and then to
 call certain actions in vcl_init.::
 
 
-         import directors;    # load the directors
+    import directors;    # load the directors
 
-	 backend server1 {
-	     .host = "192.168.0.10";
-	 }
-	 backend server2{
-	     .host = "192.168.0.10";
-	 }
-
-         sub vcl_init {
-                new bar = directors.round_robin();
-                bar.add_backend(server1);
-                bar.add_backend(server2);
-        }
-	
-        sub vcl_recv {
-                # send all traffic to the bar director:
-	        req.backend = bar.backend;
+    backend server1 {
+	    .host = "192.168.0.10";
+	}
+	backend server2{
+	    .host = "192.168.0.10";
 	}
 
+    sub vcl_init {
+        new bar = directors.round_robin();
+        bar.add_backend(server1);
+        bar.add_backend(server2);
+    }
+
+    sub vcl_recv {
+        # send all traffic to the bar director:
+	    req.backend = bar.backend();
+    }
+
 This director is a round-robin director. This means the director will
 distribute the incoming requests on a round-robin basis. There is
 also a *random* director which distributes requests in a, you guessed
-it, random fashion. 
+it, random fashion.
 
 But what if one of your servers goes down? Can Varnish direct all the
 requests to the healthy server? Sure it can. This is where the Health
@@ -159,64 +158,59 @@ Health checks
 Lets set up a director with two backends and health checks. First lets
 define the backends.::
 
-       backend server1 {
-         .host = "server1.example.com";
-	 .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	   }
-         }
-       backend server2 {
-  	  .host = "server2.example.com";
-  	  .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	  }
+    backend server1 {
+        .host = "server1.example.com";
+        .probe = {
+            .url = "/";
+            .interval = 5s;
+            .timeout = 1 s;
+            .window = 5;
+            .threshold = 3;
         }
+    }
+
+    backend server2 {
+  	    .host = "server2.example.com";
+  	    .probe = {
+            .url = "/";
+            .interval = 5s;
+            .timeout = 1 s;
+            .window = 5;
+            .threshold = 3;
+	    }
+    }
 
 Whats new here is the probe. Varnish will check the health of each
-backend with a probe. The options are
+backend with a probe. The options are:
 
 url
- What URL should Varnish request.
+    What URL should Varnish request.
 
 interval
- How often should we poll
+    How often should we poll
 
 timeout
- What is the timeout of the probe
+    What is the timeout of the probe
 
 window
- Varnish will maintain a *sliding window* of the results. Here the
- window has five checks.
+    Varnish will maintain a *sliding window* of the results. Here the
+    window has five checks.
 
 threshold
- How many of the .window last polls must be good for the backend to be declared healthy.
+    How many of the .window last polls must be good for the backend to be declared healthy.
 
 initial
- How many of the of the probes a good when Varnish starts - defaults
- to the same amount as the threshold.
+    How many of the of the probes a good when Varnish starts - defaults
+    to the same amount as the threshold.
 
 Now we define the director.::
 
-   import directors;
-
-   director example_director round-robin {
-        {
-                .backend = server1;
-        }
-        # server2
-        {
-                .backend = server2;
-        }
+    import directors;
 
-        }
+    director example_director round-robin {
+        { .backend = server1; }
+        { .backend = server2; }
+    }
 
 You use this director just as you would use any other director or
 backend. Varnish will not send traffic to hosts that are marked as



From lkarsten at varnish-software.com  Thu Mar 13 09:24:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 625f53b Prettier formatting of RST lists.
Message-ID: <E1WO1sJ-0007NJ-Nn@project.varnish-software.com>


commit 625f53b391b1acaeeb6774259c362e772d59b006
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Feb 25 12:47:34 2014 +0100

    Prettier formatting of RST lists.
    
    Remove the last few tabs that was forgotten on the last commit.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index ee21f4d..6e72416 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -124,11 +124,11 @@ call certain actions in vcl_init.::
     import directors;    # load the directors
 
     backend server1 {
-	    .host = "192.168.0.10";
-	}
-	backend server2{
-	    .host = "192.168.0.10";
-	}
+        .host = "192.168.0.10";
+    }
+    backend server2 {
+        .host = "192.168.0.10";
+    }
 
     sub vcl_init {
         new bar = directors.round_robin();
@@ -138,7 +138,7 @@ call certain actions in vcl_init.::
 
     sub vcl_recv {
         # send all traffic to the bar director:
-	    req.backend = bar.backend();
+        req.backend = bar.backend();
     }
 
 This director is a round-robin director. This means the director will
@@ -170,14 +170,14 @@ define the backends.::
     }
 
     backend server2 {
-  	    .host = "server2.example.com";
-  	    .probe = {
+        .host = "server2.example.com";
+        .probe = {
             .url = "/";
             .interval = 5s;
             .timeout = 1 s;
             .window = 5;
             .threshold = 3;
-	    }
+        }
     }
 
 Whats new here is the probe. Varnish will check the health of each



From lkarsten at varnish-software.com  Thu Mar 13 09:24:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 7633860 Remove extra meta-header.
Message-ID: <E1WO1sJ-0007NS-Oo@project.varnish-software.com>


commit 76338608b75334bf504667a562585f6dce0ca5cd
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Feb 25 13:54:24 2014 +0100

    Remove extra meta-header.
    
    We never know when to update this header. All the important parts of it is
    available in the git log. Since we don't really do versioning of documentation,
    it was really just taking up space.
    
    I've tried my best to keep the author names from the headers in the history
    section.

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 33e9a67..f967465 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -6,11 +6,6 @@ Varnish CLI
 Varnish Command Line Interface
 ------------------------------
 
-:Author: Per Buer
-:Date:   2011-03-23
-:Version: 0.1
-:Manual section: 7
-
 DESCRIPTION
 ===========
 
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index 95113fa..54e42ec 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -2,16 +2,9 @@
 varnishadm
 ==========
 
-----------------------------------
 Control a running Varnish instance
 ----------------------------------
 
-:Author: Cecilie Fritzvold
-:Author: Per Buer
-:Date:   2010-05-31
-:Version: 0.3
-:Manual section: 1
-
 SYNOPSIS
 ========
 
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index f5599c5..0f7d7fa 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -1,20 +1,13 @@
 .. _ref-varnishd:
 
-=========
- varnishd
-=========
+========
+varnishd
+========
 
 -----------------------
 HTTP accelerator daemon
 -----------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Author: Stig Sandbeck Mathisen
-:Author: Per Buer
-:Date:   2010-05-31
-:Version: 1.0
-:Manual section: 1
-
 
 SYNOPSIS
 ========
diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst
index 6446fb1..f513294 100644
--- a/doc/sphinx/reference/varnishhist.rst
+++ b/doc/sphinx/reference/varnishhist.rst
@@ -6,12 +6,6 @@ varnishhist
 Varnish request histogram
 -------------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Date:   2010-05-31
-:Version: 1.0
-:Manual section: 1
-
-
 SYNOPSIS
 ========
 
@@ -80,8 +74,9 @@ SEE ALSO
 
 HISTORY
 =======
-The varnishhist utility was developed by Poul-Henning Kamp in cooperation with Verdens Gang
-AS and Varnish Software AS.  This manual page was written by Dag-Erling Sm?rgrav.
+The varnishhist utility was developed by Poul-Henning Kamp in cooperation with
+Verdens Gang AS and Varnish Software AS. This manual page was written by
+Dag-Erling Sm?rgrav.
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 7f6d0bf..b916752 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -8,14 +8,6 @@ varnishlog
 Display Varnish logs
 --------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Author: Per Buer
-:Author: Martin Blix Grydeland
-:Date:   2013-05-15
-:Version: 0.3
-:Manual section: 1
-
-
 SYNOPSIS
 ========
 
@@ -70,7 +62,7 @@ HISTORY
 The varnishlog utility was developed by Poul-Henning Kamp
 <phk at phk.freebsd.dk> in cooperation with Verdens Gang AS and
 Varnish Software AS. This manual page was initially written by Dag-Erling
-Sm?rgrav.
+Sm?rgrav, and later updated by Per Buer and Martin Blix Grydeland.
 
 
 COPYRIGHT
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 8189696..2902f6b 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -6,13 +6,6 @@ varnishncsa
 Display Varnish logs in Apache / NCSA combined log format
 ---------------------------------------------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Author: Martin Blix Grydeland
-:Date:   2010-05-31
-:Version: 1.0
-:Manual section: 1
-
-
 SYNOPSIS
 ========
 
@@ -139,8 +132,10 @@ HISTORY
 =======
 
 The varnishncsa utility was developed by Poul-Henning Kamp in
-cooperation with Verdens Gang AS and Varnish Software AS.  This manual page was
-written by Dag-Erling Sm?rgrav <des at des.no>.
+cooperation with Verdens Gang AS and Varnish Software AS. This manual page was
+initially written by Dag-Erling Sm?rgrav <des at des.no>, and later updated
+by Martin Blix Grydeland.
+
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/varnishreplay.rst b/doc/sphinx/reference/varnishreplay.rst
index b8b89e9..884e7b7 100644
--- a/doc/sphinx/reference/varnishreplay.rst
+++ b/doc/sphinx/reference/varnishreplay.rst
@@ -6,13 +6,6 @@ varnishreplay
 HTTP traffic replay tool
 ------------------------
 
-:Author: Cecilie Fritzvold
-:Author: Per Buer
-:Date:   2010-05-31
-:Version: 1.0
-:Manual section: 1
-
-
 SYNOPSIS
 ========
 varnishreplay [-D] -a address:port -r file
diff --git a/doc/sphinx/reference/varnishsizes.rst b/doc/sphinx/reference/varnishsizes.rst
index cc36277..91c37a9 100644
--- a/doc/sphinx/reference/varnishsizes.rst
+++ b/doc/sphinx/reference/varnishsizes.rst
@@ -6,13 +6,6 @@ varnishsizes
 Varnish object size request histogram
 -------------------------------------
 
-:Author: Dag Erling Sm?rgrav
-:Author: Kristian Lyngst?l
-:Author: Per Buer
-:Date:   2010-05-31
-:Version: 1.0
-:Manual section: 1
-
 
 SYNOPSIS
 ========
diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index 67829c0..3505526 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -8,14 +8,6 @@ varnishstat
 Varnish Cache statistics
 ---------------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Author: Per Buer
-:Author: Lasse Karstensen
-:Date:   2011-11-07
-:Version: 1.1
-:Manual section: 1
-
-
 SYNOPSIS
 ========
 
@@ -115,7 +107,9 @@ HISTORY
 
 The varnishstat utility was originally developed by Poul-Henning Kamp
 <phk at phk.freebsd.dk> in cooperation with Verdens Gang AS and Varnish
-Software AS. Manual page written by Dag-Erling Sm?rgrav and Per Buer.
+Software AS. Manual page written by Dag-Erling Sm?rgrav, Per Buer
+and Lasse Karstensen.
+
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index 52c642b..03f47da 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -6,13 +6,6 @@ varnishtest
 Test program for Varnish
 ------------------------
 
-:Author: Stig Sandbeck Mathisen
-:Author: Kristian Lyngst?l
-:Date:   2011-11-15
-:Version: 1.1
-:Manual section: 1
-
-
 SYNOPSIS
 ========
 
diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 7ebc384..4692143 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -6,13 +6,6 @@ varnishtop
 Varnish log entry ranking
 -------------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Author: Martin Blix Grydeland
-:Date:   2010-05-31
-:Version: 1.0
-:Manual section: 1
-
-
 
 SYNOPSIS
 ========
@@ -62,7 +55,9 @@ HISTORY
 The varnishtop utility was originally developed by Poul-Henning Kamp
 in cooperation with Verdens Gang AS and Varnish Software AS, and later
 substantially rewritten by Dag-Erling Sm?rgrav.  This manual page was
-written by Dag-Erling Sm?rgrav.
+written by Dag-Erling Sm?rgrav, and later updated by Martin Blix
+Grydeland.
+
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9f384f8..436d58d 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -8,14 +8,6 @@ VCL
 Varnish Configuration Language
 ------------------------------
 
-:Author: Dag-Erling Sm?rgrav
-:Author: Poul-Henning Kamp
-:Author: Kristian Lyngst?l
-:Author: Per Buer
-:Date:   2010-06-02
-:Version: 1.0
-:Manual section: 7
-
 DESCRIPTION
 ===========
 
@@ -995,8 +987,9 @@ HISTORY
 
 VCL was developed by Poul-Henning Kamp in cooperation with Verdens
 Gang AS, Redpill Linpro and Varnish Software.  This manual page was
-written by Dag-Erling Sm?rgrav and later edited by Poul-Henning Kamp
-and Per Buer.
+written by Dag-Erling Sm?rgrav and later edited by Poul-Henning Kamp,
+Kristian Lyngst?l and Per Buer.
+
 
 COPYRIGHT
 =========
diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index 41c38e6..b8f2aa3 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -6,12 +6,6 @@ vmod_std
 Varnish Standard Module
 -----------------------
 
-:Author: Per Buer
-:Date:   2011-05-19
-:Version: 1.0
-:Manual section: 3
-
-
 SYNOPSIS
 ========
 



From lkarsten at varnish-software.com  Thu Mar 13 09:24:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 0f66e81 Reformatting tab/space and width.
Message-ID: <E1WO1sJ-0007Nf-QD@project.varnish-software.com>


commit 0f66e81e5e858e5b43aa8f1e1008661cf571df40
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Feb 25 14:08:39 2014 +0100

    Reformatting tab/space and width.
    
    Also removed one last meta-header forgotten on the last commit.

diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index 54e42ec..5bb9f6e 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -29,22 +29,23 @@ OPTIONS
 =======
 
 -t timeout
-	Wait no longer than this many seconds for an operation to finish.
+    Wait no longer than this many seconds for an operation to finish.
 
 -S secret_file
-        Specify the authentication secret file. This should be the same -S 
-        argument as was given to varnishd. Only processes which can read 
-        the contents of this file, will be able to authenticate the CLI connection.
-
--T address:port   
-        Connect to the management interface at the specified address and port.
+    Specify the authentication secret file. This should be the same -S
+    argument as was given to varnishd. Only processes which can read
+    the contents of this file, will be able to authenticate the CLI connection.
 
 -n name
-	Connect to the instance of varnishd with this name.
+    Connect to the instance of varnishd with this name.
+
+-T address:port
+    Connect to the management interface at the specified address and port.
+
 
 The syntax and operation of the actual CLI interface is described in
 the varnish-cli(7) manual page. Parameteres are described in
-varnishd(1) manual page.  
+varnishd(1) manual page.
 
 Additionally, a summary of commands can be obtained by issuing the
 *help* command, and a summary of parameters can be obtained by issuing
@@ -61,9 +62,9 @@ EXAMPLES
 
 Some ways you can use varnishadm::
 
-           varnishadm -T localhost:999 -S /var/db/secret vcl.use foo
-           echo vcl.use foo | varnishadm -T localhost:999 -S /var/db/secret
-           echo vcl.use foo | ssh vhost varnishadm -T localhost:999 -S /var/db/secret
+   varnishadm -T localhost:999 -S /var/db/secret vcl.use foo
+   echo vcl.use foo | varnishadm -T localhost:999 -S /var/db/secret
+   echo vcl.use foo | ssh vhost varnishadm -T localhost:999 -S /var/db/secret
 
 SEE ALSO
 ========
@@ -74,7 +75,7 @@ HISTORY
 =======
 
 The varnishadm utility and this manual page were written by Cecilie
-Fritzvold. Converted to reStructured and updated in 2010 by Per
+Fritzvold. Converted to reStructuredText and updated in 2010 by Per
 Buer.
 
 COPYRIGHT
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 0f7d7fa..a93ecac 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -30,7 +30,7 @@ OPTIONS
 =======
 
 -a address[:port][,address[:port][...]
-	    Listen for client requests on the specified address and port.  The address can be a host
+            Listen for client requests on the specified address and port.  The address can be a host
             name (?localhost?), an IPv4 dotted-quad (?127.0.0.1?), or an IPv6 address enclosed in
             square brackets (?[::1]?).  If address is not specified, varnishd will listen on all
             available IPv4 and IPv6 interfaces.  If port is not specified, the default HTTP port as
@@ -39,10 +39,10 @@ OPTIONS
 
 -b host[:port]
             Use the specified host as backend server.  If port is not specified,
-	    the default is 8080.
+	        the default is 8080.
 
--C	    Print VCL code compiled to C language and exit. Specify the VCL file
-	    to compile with the -f option.
+-C	        Print VCL code compiled to C language and exit. Specify the VCL file
+	        to compile with the -f option.
 
 -d          Enables debugging mode: The parent process runs in the foreground with a CLI connection
             on stdin/stdout, and the child process must be started explicitly with a CLI command.
@@ -78,9 +78,9 @@ OPTIONS
 -P file     Write the process's PID to the specified file.
 
 -p param=value
-            Set the parameter specified by param to the specified value.  See Run-Time
-	    Parameters for a list of parameters. This option can be used multiple
-	    times to specify multiple parameters.
+            Set the parameter specified by param to the specified value.  See
+            Run-Time Parameters for a list of parameters. This option can be
+            used multiple times to specify multiple parameters.
 
 -S file     Path to a file containing a secret used for authorizing access to the management port.
 
@@ -100,15 +100,13 @@ OPTIONS
             Interface for a list of management commands.
 
 -M address:port
-            Connect to this port and offer the command line
-            interface. Think of it as a reverse shell. When running with
-	    -M and there is no backend defined the child process (the cache)
-            will not start initially.
+            Connect to this port and offer the command line interface.
+            Think of it as a reverse shell. When running with -M and there is
+            no backend defined the child process (the cache) will not start
+            initially.
 
--t ttl
-	    Specifies a hard minimum time to live for cached
-            documents.  This is a shortcut for specifying the
-            default_ttl run-time parameter.
+-t ttl      Specifies a hard minimum time to live for cached documents. This
+            is a shortcut for specifying the default_ttl run-time parameter.
 
 -r param[,param...]
             Make the listed parameters read only. This gives the
@@ -120,8 +118,8 @@ OPTIONS
 
 -u user     Specifies the name of an unprivileged user to which the child
             process should switch before it starts accepting
-            connections.  This is a shortcut for specifying the user
-            run- time parameter.
+            connections. This is a shortcut for specifying the user
+            runtime parameter.
 
             If specifying both a user and a group, the user should be
             specified first.
@@ -135,19 +133,19 @@ Hash Algorithms
 The following hash algorithms are available:
 
 simple_list
-  A simple doubly-linked list.  Not recommended for production use.
+    A simple doubly-linked list.  Not recommended for production use.
 
 classic[,buckets]
-  A standard hash table.  This is the default.  The hash key is the
-  CRC32 of the object's URL modulo the size of the hash table.  Each
-  table entry points to a list of elements which share the same hash
-  key. The buckets parameter specifies the number of entries in the
-  hash table.  The default is 16383.
+    A standard hash table.  This is the default.  The hash key is the
+    CRC32 of the object's URL modulo the size of the hash table.  Each
+    table entry points to a list of elements which share the same hash
+    key. The buckets parameter specifies the number of entries in the
+    hash table.  The default is 16383.
 
 critbit
-  A self-scaling tree structure. The default hash algorithm in 2.1. In
-  comparison to a more traditional B tree the critbit tree is almost
-  completely lockless.
+    A self-scaling tree structure. The default hash algorithm in 2.1. In
+    comparison to a more traditional B tree the critbit tree is almost
+    completely lockless.
 
 Storage Types
 -------------
@@ -243,7 +241,7 @@ The varnishd daemon was developed by Poul-Henning Kamp in cooperation
 with Verdens Gang AS, Varnish Software AS and Varnish Software.
 
 This manual page was written by Dag-Erling Sm?rgrav with updates by
-Stig Sandbeck Mathisen <ssm at debian.org>
+Stig Sandbeck Mathisen <ssm at debian.org>.
 
 
 COPYRIGHT
diff --git a/doc/sphinx/reference/varnishsizes.rst b/doc/sphinx/reference/varnishsizes.rst
index 91c37a9..af2e05b 100644
--- a/doc/sphinx/reference/varnishsizes.rst
+++ b/doc/sphinx/reference/varnishsizes.rst
@@ -76,6 +76,7 @@ The varnishsizes utility was developed by Kristian Lyngst?l based on
 varnishhist.  This manual page was written by Kristian Lyngst?l,
 Dag-Erling Sm?rgrav and Per Buer.
 
+
 COPYRIGHT
 =========
 
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index 03f47da..a3a0eec 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -126,6 +126,7 @@ This manual page was originally written by Stig Sandbeck Mathisen
 <ssm at linpro.no> and updated by Kristian Lyngst?l
 <kristian at varnish-cache.org>.
 
+
 COPYRIGHT
 =========
 
diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index 9547a59..f188113 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -4,11 +4,6 @@
 Varnish VSL Query Expressions
 =============================
 
-:Author: Martin Blix Grydeland
-:Date: 2013-09-26
-:Version: 0.1
-:Manual section: 7
-
 OVERVIEW
 ========
 
@@ -238,3 +233,9 @@ QUERY EXPRESSION EXAMPLES
   their ESI subrequests. (Assumes request grouping mode). ::
 
     BerespStatus >= 500 or {2+}ReqEnd[5] > 1.
+
+HISTORY
+=======
+
+This document was written by Martin Blix Grydeland.
+
diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst
index 7f79209..78c3456 100644
--- a/doc/sphinx/reference/vsl.rst
+++ b/doc/sphinx/reference/vsl.rst
@@ -8,12 +8,6 @@ VSL
 Shared Memory Logging
 ---------------------
 
-:Author: Poul-Henning Kamp
-:Author: Martin Blix Grydeland
-:Date: 2013-10-16
-:Version: 1.0
-:Manual section: 7
-
 OVERVIEW
 ========
 
@@ -26,6 +20,14 @@ VSL tags
 
 .. include:: ../../../lib/libvarnishapi/vsl-tags.rst
 
+
+HISTORY
+=======
+
+This document was initially written by Poul-Henning Kamp, and later updated by
+Martin Blix Grydeland.
+
+
 SEE ALSO
 ========
 * varnishlog(1)



From martin at varnish-software.com  Thu Mar 13 09:24:27 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] c28824d Port varnishhist.c to the new api
Message-ID: <E1WO1sJ-0007Np-RB@project.varnish-software.com>


commit c28824d7a943beac75045681d6d2e2caac44291d
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 10:07:37 2014 +0100

    Port varnishhist.c to the new api

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 03bfd88..bf61060 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -48,12 +48,16 @@
 
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
+#include "vapi/voptget.h"
 #include "vas.h"
 #include "vcs.h"
+#include "vut.h"
 
 #define HIST_N 2000 /* how far back we remember */
 #define HIST_RES 100 /* bucket resolution */
 
+static const char progname[] = "varnishhist";
+
 static int hist_low;
 static int hist_high;
 static int hist_range;
@@ -61,15 +65,13 @@ static int hist_buckets;
 
 static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
 
+static int end_of_file = 0;
 static int delay = 1;
 static unsigned rr_hist[HIST_N];
 static unsigned nhist;
 static unsigned next_hist;
 static unsigned *bucket_miss;
 static unsigned *bucket_hit;
-static unsigned char hh[FD_SETSIZE];
-static uint64_t bitmap[FD_SETSIZE];
-static double  values[FD_SETSIZE];
 static char *format;
 static int match_tag;
 
@@ -105,7 +107,9 @@ struct profile {
 	int field;
 	int hist_low;
 	int hist_high;
-} profiles[] = {
+} 
+
+profiles[] = {
 	{
 		.name = "responsetime",
 		.tag = SLT_ReqEnd,
@@ -126,7 +130,7 @@ struct profile {
 static struct profile *active_profile;
 
 static void
-update(struct VSM_data *vd)
+update(void)
 {
 	int w = COLS / hist_range;
 	int n = w * hist_range;
@@ -146,7 +150,10 @@ update(struct VSM_data *vd)
 		mvprintw(LINES - 1, w * i, "|1e%d", j);
 	}
 
-	mvprintw(0, 0, "%*s", COLS - 1, VSM_Name(vd));
+	if (end_of_file)
+		mvprintw(0, 0, "%*s", COLS - 1, "EOF");
+	else
+		mvprintw(0, 0, "%*s", COLS - 1, VUT.name);
 
 	/* count our flock */
 	for (i = 0; i < n; ++i)
@@ -177,113 +184,77 @@ update(struct VSM_data *vd)
 	refresh();
 }
 
-static int
-h_hist(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
-    unsigned spec, const char *ptr, uint64_t bm)
+static int /*__match_proto__ (VSLQ_dispatch_f)*/
+accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
+	void *priv)
 {
 	int i, j;
-	struct VSM_data *vd = priv;
-	(void)spec;
-
-	if (fd >= FD_SETSIZE)
-		/* oops */
-		return (0);
-
-	bitmap[fd] |= bm;
+	unsigned tag, hit;
+	double value;
+	struct VSL_transaction *tr;
+	for (tr = pt[0]; tr != NULL; tr = *++pt) {
+		value = -1;
+		hit = 0;
+		while ((1 == VSL_Next(tr->c))) {
+			if (!VSL_Match(vsl, tr->c))
+				continue;
+			/* get the value we want, and register if it's a hit*/
+			tag = VSL_TAG(tr->c->rec.ptr);
+			if (tag == match_tag) {
+				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format, &value);
+				assert(i == 1);
+			} else if (tag == SLT_Hit)
+				hit = 1;
+
+			/* select bucket */
+			i = HIST_RES * (log(value) / log_ten);
+			if (i < hist_low * HIST_RES)
+				i = hist_low * HIST_RES;
+			if (i >= hist_high * HIST_RES)
+				i = hist_high * HIST_RES - 1;
+			i -= hist_low * HIST_RES;
+			assert(i >= 0);
+			assert(i < hist_buckets);
+			pthread_mutex_lock(&mtx);
+
+			/* phase out old data */
+			if (nhist == HIST_N) {
+				j = rr_hist[next_hist];
+				if (j < 0)  {
+					assert(bucket_miss[-j] > 0);
+					bucket_miss[-j]--;
+				} else {
+					assert(bucket_hit[j] > 0);
+					bucket_hit[j]--;
+				}
+			} else {
+				++nhist;
+			}
 
-	if (tag == SLT_Hit) {
-		hh[fd] = 1;
-		return (0);
-	}
-	if (tag == match_tag) {
-		char buf[1024]; /* size? */
-		assert(len < sizeof(buf));
-		memcpy(buf, ptr, len);
-		buf[len] = '\0';
-		i = sscanf(buf, format, &values[fd]);
-		assert(i == 1);
-	}
+			/* phase in new data */
+			if (hit) {
+				bucket_hit[i]++;
+				rr_hist[next_hist] = i;
+			} else {
+				bucket_miss[i]++;
+				rr_hist[next_hist] = -i;
+			}
+			if (++next_hist == HIST_N) {
+				next_hist = 0;
+			}
+			pthread_mutex_unlock(&mtx);
 
-	if (tag != SLT_ReqEnd)
-		return (0);
 
-	if (!VSL_Matched(vd, bitmap[fd])) {
-		bitmap[fd] = 0;
-		hh[fd] = 0;
-		return (0);
-	}
-
-	/* select bucket */
-	i = HIST_RES * (log(values[fd]) / log_ten);
-	if (i < hist_low * HIST_RES)
-		i = hist_low * HIST_RES;
-	if (i >= hist_high * HIST_RES)
-		i = hist_high * HIST_RES - 1;
-	i -= hist_low * HIST_RES;
-	assert(i >= 0);
-	assert(i < hist_buckets);
-	pthread_mutex_lock(&mtx);
-
-	/* phase out old data */
-	if (nhist == HIST_N) {
-		j = rr_hist[next_hist];
-		if (j < 0)  {
-			assert(bucket_miss[-j] > 0);
-			bucket_miss[-j]--;
-		} else {
-			assert(bucket_hit[j] > 0);
-			bucket_hit[j]--;
 		}
-	} else {
-		++nhist;
-	}
-
-	/* phase in new data */
-	if (hh[fd] || i == 0) {
-		bucket_hit[i]++;
-		rr_hist[next_hist] = i;
-	} else {
-		bucket_miss[i]++;
-		rr_hist[next_hist] = -i;
 	}
-	if (++next_hist == HIST_N) {
-		next_hist = 0;
-	}
-	hh[fd] = 0;
-	bitmap[fd] = 0;
-
-	pthread_mutex_unlock(&mtx);
-
 	return (0);
 }
 
 static void *
-accumulate_thread(void *arg)
+do_curses(void *arg)
 {
-	struct VSM_data *vd = arg;
-	int i;
-
-	for (;;) {
-		i = VSL_Dispatch(vd, h_hist, vd);
-		if (i < 0)
-			break;
-		if (i == 0)
-			usleep(50000);
-	}
-	return (arg);
-}
-
-static void
-do_curses(struct VSM_data *vd)
-{
-	pthread_t thr;
 	int ch;
 
-	if (pthread_create(&thr, NULL, accumulate_thread, vd) != 0) {
-		fprintf(stderr, "pthread_create(): %s\n", strerror(errno));
-		exit(1);
-	}
-
 	initscr();
 	raw();
 	noecho();
@@ -293,7 +264,7 @@ do_curses(struct VSM_data *vd)
 	erase();
 	for (;;) {
 		pthread_mutex_lock(&mtx);
-		update(vd);
+		update();
 		pthread_mutex_unlock(&mtx);
 
 		timeout(delay * 1000);
@@ -321,7 +292,7 @@ do_curses(struct VSM_data *vd)
 		case 'Q':
 		case 'q':
 			endwin();
-			return;
+			pthread_exit(NULL);
 		case '0':
 		case '1':
 		case '2':
@@ -339,34 +310,37 @@ do_curses(struct VSM_data *vd)
 			break;
 		}
 	}
+	pthread_exit(NULL);
 }
 
 /*--------------------------------------------------------------------*/
 
 static void
-usage(void)
+usage(int status)
 {
 	fprintf(stderr, "usage: varnishhist "
 	    "%s [-p profile] [-f field_num] "
-	    "[-R max] [-r min] [-V] [-w delay]\n", VSL_USAGE);
-	exit(1);
+	    "[-R max] [-r min] [-V] [-w delay]\n", "varnishhist");
+	exit(status);
 }
 
 int
 main(int argc, char **argv)
 {
-	int o, i;
-	struct VSM_data *vd;
+	int i;
 	const char *profile = "responsetime";
+	pthread_t thr;
 	int fnum = -1;
 	hist_low = -1;
 	hist_high = -1;
 	match_tag = -1;
 
-	vd = VSM_New();
+	VUT_Init(progname);
+	if (0)
+		(void)usage;
 
-	while ((o = getopt(argc, argv, VSL_ARGS "Vw:r:R:f:p:")) != -1) {
-		switch (o) {
+	while ((i = getopt(argc, argv, vopt_optstring)) != -1) {
+		switch (i) {
 		case 'V':
 			VCS_Message("varnishhist");
 			exit(0);
@@ -393,9 +367,8 @@ main(int argc, char **argv)
 			profile = optarg;
 			break;
 		default:
-			if (VSL_Arg(vd, o, optarg) > 0)
-				break;
-			usage();
+			if (!VUT_Arg(i, optarg))
+				usage(1);
 		}
 	}
 	if (profile) {
@@ -437,13 +410,19 @@ main(int argc, char **argv)
 	}
 	strcpy(format + 4*(fnum-1), "%lf");
 
-	if (VSM_Open(vd)) {
-		fprintf(stderr, "%s\n", VSM_Error(vd));
-		exit(1);
-	}
-
 	log_ten = log(10.0);
 
-	do_curses(vd);
+	VUT_Setup();
+	if (pthread_create(&thr, NULL, do_curses, NULL) != 0) {
+		fprintf(stderr, "pthread_create(): %s\n",
+				strerror(errno));
+		exit(1);
+	}
+	VUT.dispatch_f = &accumulate;
+	VUT.dispatch_priv = NULL;
+	VUT_Main();
+	end_of_file = 1;
+	pthread_join(thr, NULL);
+	VUT_Fini();
 	exit(0);
 }



From martin at varnish-software.com  Thu Mar 13 09:24:27 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] f23c07c Update autotools to compile varnishhist
Message-ID: <E1WO1sJ-0007Nz-SN@project.varnish-software.com>


commit f23c07c77537454e2cb59f480a14d5529b0dd777
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 10:08:20 2014 +0100

    Update autotools to compile varnishhist

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 48a9a51..a6defb4 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -6,7 +6,7 @@
 SUBDIRS = varnishadm varnishd varnishlog varnishtest varnishncsa
 
 if VARNISH_CURSES
-#SUBDIRS += varnishhist varnishstat varnishtop
+SUBDIRS += varnishhist
 SUBDIRS += varnishstat
 SUBDIRS += varnishtop
 endif
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 9511a6e..a24f29a 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -5,8 +5,14 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 bin_PROGRAMS = varnishhist
 
 varnishhist_SOURCES = varnishhist.c \
+	varnishhist_options.h \
+	varnishhist_options.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c
+	$(top_builddir)/lib/libvarnish/version.c \
+	$(top_srcdir)/lib/libvarnish/vpf.c \
+	$(top_srcdir)/lib/libvarnish/vtim.c \
+	$(top_srcdir)/lib/libvarnish/flopen.c \
+	$(top_srcdir)/lib/libvarnishtools/vut.c
 
 varnishhist_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/configure.ac b/configure.ac
index f25b2cf..5c46de8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -566,6 +566,7 @@ AC_CONFIG_FILES([
     bin/varnishlog/Makefile
     bin/varnishstat/Makefile
     bin/varnishtop/Makefile
+    bin/varnishhist/Makefile
     bin/varnishtest/Makefile
     bin/varnishncsa/Makefile
     doc/Makefile



From martin at varnish-software.com  Thu Mar 13 09:24:27 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] ce26d62 Handle the options like the other tools
Message-ID: <E1WO1sJ-0007O8-Th@project.varnish-software.com>


commit ce26d62cf3e695c5339e513d629f8fa694201c96
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 14:46:45 2014 +0100

    Handle the options like the other tools

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index bf61060..626eba6 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -318,9 +318,12 @@ do_curses(void *arg)
 static void
 usage(int status)
 {
-	fprintf(stderr, "usage: varnishhist "
-	    "%s [-p profile] [-f field_num] "
-	    "[-R max] [-r min] [-V] [-w delay]\n", "varnishhist");
+	const char **opt;
+
+	fprintf(stderr, "Usage: %s <options>\n\n", progname);
+	fprintf(stderr, "Options:\n");
+	for (opt = vopt_usage; *opt != NULL; opt +=2)
+		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
 	exit(status);
 }
 
@@ -328,12 +331,12 @@ int
 main(int argc, char **argv)
 {
 	int i;
+	char *colon;
 	const char *profile = "responsetime";
 	pthread_t thr;
 	int fnum = -1;
-	hist_low = -1;
-	hist_high = -1;
-	match_tag = -1;
+	struct profile cli_p;
+	cli_p.name = 0;
 
 	VUT_Init(progname);
 	if (0)
@@ -341,30 +344,30 @@ main(int argc, char **argv)
 
 	while ((i = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (i) {
-		case 'V':
-			VCS_Message("varnishhist");
-			exit(0);
-		case 'i':
-			match_tag = VSL_Name2Tag(optarg, -1);
+		case 'P':
+			colon = strchr(optarg, ':');
+			/* no colon, take the profile as a name*/
+			if (colon == NULL) {
+				profile = optarg;
+				break;
+			}
+			/* else it's a definition, we hope */
+			if (sscanf(colon+1, "%d:%d:%d", 
+					&cli_p.field, &cli_p.hist_low, &cli_p.hist_high) != 3) {
+				fprintf(stderr, "%s is neither a profile name nor definition (SLT_Tag:field:min:max)\n", optarg);
+				exit(1);
+			}
+
+			match_tag = VSL_Name2Tag(optarg, colon - optarg);
 			if (match_tag < 0) {
-				fprintf(stderr, "No such tag %s\n", optarg);
+				fprintf(stderr, "No such tag in %s\n", optarg);
 				exit(1);
 			}
-			break;
-		case 'w':
-			delay = atoi(optarg);
-			break;
-		case 'f':
-			fnum = atoi(optarg);
-			break;
-		case 'R':
-			hist_high = atoi(optarg);
-			break;
-		case 'r':
-			hist_low = atoi(optarg);
-			break;
-		case 'p':
-			profile = optarg;
+			cli_p.name = "custom";
+			cli_p.tag = match_tag;
+			profile = NULL;
+			active_profile = &cli_p;
+
 			break;
 		default:
 			if (!VUT_Arg(i, optarg))
@@ -383,21 +386,10 @@ main(int argc, char **argv)
 		fprintf(stderr, "No such profile %s\n", profile);
 		exit(1);
 	}
-	if (match_tag < 0) {
-		match_tag = active_profile->tag;
-	}
-
-	if (fnum < 0) {
-		fnum = active_profile->field;
-	}
-
-	if (hist_low < 0) {
-		hist_low = active_profile->hist_low;
-	}
-
-	if (hist_high < 0) {
-		hist_high = active_profile->hist_high;
-	}
+	match_tag = active_profile->tag;
+	fnum = active_profile->field;
+	hist_low = active_profile->hist_low;
+	hist_high = active_profile->hist_high;
 
 	hist_range = hist_high - hist_low;
 	hist_buckets = hist_range * HIST_RES;
diff --git a/bin/varnishhist/varnishhist_options.c b/bin/varnishhist/varnishhist_options.c
new file mode 100644
index 0000000..718fd3d
--- /dev/null
+++ b/bin/varnishhist/varnishhist_options.c
@@ -0,0 +1,34 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Option definitions for varnishtop
+ */
+
+#include <stdlib.h>
+#define VOPT_DEFINITION
+#define VOPT_INC "varnishhist_options.h"
+#include "vapi/voptget.h"
diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
new file mode 100644
index 0000000..e362d32
--- /dev/null
+++ b/bin/varnishhist/varnishhist_options.h
@@ -0,0 +1,68 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Option definitions for varnishtop
+ */
+
+#include "vapi/vapi_options.h"
+#include "vut_options.h"
+
+#define HIS_OPT_p							\
+	VOPT("p:", "[-p period]", "Refresh period",			\
+	    "Specified the number of seconds between screen refreshes"	\
+	    " default is 1 second, and can be changed during the run"	\
+	    " by pressing the [1-9] keys"	\
+	)
+
+#define HIS_OPT_P							\
+	VOPT("P:", "[-P <size|responsetime|tag:field_num:min:max>]", "Profile definition",			\
+		"Either specify \"size\" or \"responstime\" profile or create a new one" \
+	    " Define the tag we'll look for, and the field number of the value"	\
+	    " we are interested in. min and max are the boundaries of the graph"	\
+	    " (these are power of tens)"	\
+	)
+
+VSL_OPT_b
+VSL_OPT_c
+VSL_OPT_C
+VUT_OPT_d
+VUT_OPT_D
+VUT_OPT_g
+VUT_OPT_h
+VSL_OPT_i
+VSL_OPT_I
+VSL_OPT_L
+VUT_OPT_n
+VUT_OPT_N
+HIS_OPT_p
+HIS_OPT_P
+VUT_OPT_q
+VUT_OPT_r
+VSL_OPT_T
+VSL_OPT_x
+VSL_OPT_X
+VUT_OPT_V



From martin at varnish-software.com  Thu Mar 13 09:24:27 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] ff35db6 Fix white spaces
Message-ID: <E1WO1sJ-0007OI-Um@project.varnish-software.com>


commit ff35db66d5a3ef78bcef7ffe73a9951a4605bb70
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 15:02:02 2014 +0100

    Fix white spaces

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 626eba6..1705518 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -220,7 +220,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			/* phase out old data */
 			if (nhist == HIST_N) {
 				j = rr_hist[next_hist];
-				if (j < 0)  {
+				if (j < 0) {
 					assert(bucket_miss[-j] > 0);
 					bucket_miss[-j]--;
 				} else {
@@ -376,7 +376,7 @@ main(int argc, char **argv)
 	}
 	if (profile) {
 		for (active_profile = profiles; active_profile->name;
-		    active_profile++) {
+			active_profile++) {
 			if (strcmp(active_profile->name, profile) == 0) {
 				break;
 			}



From martin at varnish-software.com  Thu Mar 13 09:24:27 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:27 +0100
Subject: [4.0] 1346cd8 Build varnishhist documentation
Message-ID: <E1WO1sJ-0007OR-Vx@project.varnish-software.com>


commit 1346cd8b6f5aaf9bb5b47c16b301267c53e261bf
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 15:02:32 2014 +0100

    Build varnishhist documentation

diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index a24f29a..986966f 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -19,3 +19,35 @@ varnishhist_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	-lm \
 	@CURSES_LIB@ ${RT_LIBS} ${PTHREAD_LIBS}
+
+noinst_PROGRAMS = varnishhist_opt2rst
+varnishhist_opt2rst_SOURCES = \
+	varnishhist_options.h \
+	varnishhist_options.c \
+	$(top_srcdir)/lib/libvarnishtools/opt2rst.c
+
+BUILT_SOURCES = varnishhist_options.rst varnishhist_synopsis.rst
+EXTRA_DIST = $(BUILT_SOURCES)
+MAINTAINERCLEANFILES = $(EXTRA_DIST)
+
+varnishhist_options.rst:
+	./varnishhist_opt2rst options > $@
+varnishhist_synopsis.rst:
+	./varnishhist_opt2rst synopsis > $@
+
+if HAVE_RST2MAN
+varnishhist_options.rst varnishhist_synopsis.rst: varnishhist_opt2rst
+endif
+
+varnishhist.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishhist.rst \
+	varnishhist_options.rst \
+	varnishhist_synopsis.rst
+if HAVE_RST2MAN
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif
diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index e362d32..2b71835 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -40,7 +40,7 @@
 
 #define HIS_OPT_P							\
 	VOPT("P:", "[-P <size|responsetime|tag:field_num:min:max>]", "Profile definition",			\
-		"Either specify \"size\" or \"responstime\" profile or create a new one" \
+		"Either specify \"size\" or \"responstime\" profile or create a new one." \
 	    " Define the tag we'll look for, and the field number of the value"	\
 	    " we are interested in. min and max are the boundaries of the graph"	\
 	    " (these are power of tens)"	\
diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst
index f513294..901c7f4 100644
--- a/doc/sphinx/reference/varnishhist.rst
+++ b/doc/sphinx/reference/varnishhist.rst
@@ -9,8 +9,8 @@ Varnish request histogram
 SYNOPSIS
 ========
 
-varnishhist [-b] [-C] [-c] [-d] [-I regex] [-i tag] [-m tag:regex ...]
-[-n varnish_name] [-r file] [-V] [-w delay] [-X regex] [-x tag]
+.. include:: ../../../bin/varnishhist/varnishhist_synopsis.rst
+varnishhist |synopsis|
 
 DESCRIPTION
 ===========
@@ -24,44 +24,7 @@ and misses are marked with a hash character ("#").
 
 The following options are available:
 
--b          Include log entries which result from communication with
-	    a backend server.  If neither -b nor -c is
-	    specified, varnishhist acts as if they both were.
-
--C          Ignore case when matching regular expressions.
-
--c          Include log entries which result from communication with
-	    a client.  If neither -b nor -c is specified,
-	    varnishhist acts as if they both were.
-
--d          Process old log entries on startup.  Normally, varnishhist
-	    will only process entries which are written to the
-	    log after it starts.
-
--I regex    Include log entries which match the specified
-   	    regular expression.  If neither -I nor -i is specified,
-	    all log entries are included.
-
--i tag      Include log entries with the specified tag.  If neither
-   	    -I nor -i is specified, all log entries are included.
-
--m tag:regex only count transactions where tag matches regex. Multiple
-            -m options are AND-ed together.
-
--n          Specifies the name of the varnishd instance to get logs
-	    from.  If -n is not specified, the host name is used.
-
--r file     Read log entries from file instead of shared memory.
-
--V          Display the version number and exit.
-
--w delay    Wait at least delay seconds between each update.  The
-   	    default is 1.  file instead of displaying them.  The file
-	    will be overwritten unless the -a option was specified.
-
--X regex    Exclude log entries which match the specified regular expression.
-
--x tag      Exclude log entries with the specified tag.
+.. include:: ../../../bin/varnishhist/varnishhist_options.rst
 
 SEE ALSO
 ========



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] cce5ff4 Silence unused-parameter warnings
Message-ID: <E1WO1sK-0007Od-12@project.varnish-software.com>


commit cce5ff42b190e541d095dae9ef3e74ae117d0ab3
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 15:41:09 2014 +0100

    Silence unused-parameter warnings

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1705518..c838fef 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -192,6 +192,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	unsigned tag, hit;
 	double value;
 	struct VSL_transaction *tr;
+	(void)priv;
 	for (tr = pt[0]; tr != NULL; tr = *++pt) {
 		value = -1;
 		hit = 0;
@@ -254,6 +255,7 @@ static void *
 do_curses(void *arg)
 {
 	int ch;
+	(void)arg;
 
 	initscr();
 	raw();



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 01e6b2e Silence sign-compare warnings
Message-ID: <E1WO1sK-0007Om-2M@project.varnish-software.com>


commit 01e6b2e348b35ce4fe06929ca99f1c0c931d06ea
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Tue Feb 4 15:56:41 2014 +0100

    Silence sign-compare warnings

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index c838fef..3845db3 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -132,11 +132,12 @@ static struct profile *active_profile;
 static void
 update(void)
 {
-	int w = COLS / hist_range;
-	int n = w * hist_range;
+	unsigned w = COLS / hist_range;
+	unsigned n = w * hist_range;
 	unsigned bm[n], bh[n];
 	unsigned max;
-	int i, j, scale;
+	unsigned i, j, scale;
+	int k, l;
 
 	erase();
 
@@ -145,9 +146,9 @@ update(void)
 	n = w * hist_range;
 	for (i = 0; i < n; ++i)
 		(void)mvaddch(LINES - 2, i, '-');
-	for (i = 0, j = hist_low; i < hist_range; ++i, ++j) {
-		(void)mvaddch(LINES - 2, w * i, '+');
-		mvprintw(LINES - 1, w * i, "|1e%d", j);
+	for (k = 0, l = hist_low; k < hist_range; ++k, ++l) {
+		(void)mvaddch(LINES - 2, w * k, '+');
+		mvprintw(LINES - 1, w * k, "|1e%d", l);
 	}
 
 	if (end_of_file)
@@ -158,16 +159,17 @@ update(void)
 	/* count our flock */
 	for (i = 0; i < n; ++i)
 		bm[i] = bh[i] = 0;
-	for (i = 0, max = 1; i < hist_buckets; ++i) {
-		j = i * n / hist_buckets;
-		bm[j] += bucket_miss[i];
-		bh[j] += bucket_hit[i];
-		if (bm[j] + bh[j] > max)
-			max = bm[j] + bh[j];
+	for (k = 0, max = 1; k < hist_buckets; ++k) {
+		l = k * n / hist_buckets;
+		bm[l] += bucket_miss[k];
+		bh[l] += bucket_hit[k];
+		if (bm[l] + bh[l] > max)
+			max = bm[l] + bh[l];
 	}
 
 	/* scale */
-	for (i = 0; max / scales[i] > LINES - 3; ++i)
+	assert(LINES - 3 >= 0);
+	for (i = 0; max / scales[i] > (unsigned)(LINES - 3); ++i)
 		/* nothing */ ;
 	scale = scales[i];
 
@@ -201,7 +203,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				continue;
 			/* get the value we want, and register if it's a hit*/
 			tag = VSL_TAG(tr->c->rec.ptr);
-			if (tag == match_tag) {
+			assert(match_tag >= 0);
+			if (tag == (unsigned)match_tag) {
 				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format, &value);
 				assert(i == 1);
 			} else if (tag == SLT_Hit)



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 238d27c Only commit once, at the end of transaction
Message-ID: <E1WO1sK-0007Ov-3k@project.varnish-software.com>


commit 238d27c571c744f4f7a2c832c04d757d34f57a4f
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 17:03:35 2014 +0100

    Only commit once, at the end of transaction

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 3845db3..92aa3f4 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -209,6 +209,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				assert(i == 1);
 			} else if (tag == SLT_Hit)
 				hit = 1;
+			if (tag != SLT_ReqEnd && value == -1)
+				continue;
 
 			/* select bucket */
 			i = HIST_RES * (log(value) / log_ten);



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] b230de2 Limit the grouping method
Message-ID: <E1WO1sK-0007P4-5D@project.varnish-software.com>


commit b230de211b544485ce987b091106c29439e14901
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 18:19:12 2014 +0100

    Limit the grouping method

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 92aa3f4..ec30da5 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -381,6 +381,12 @@ main(int argc, char **argv)
 				usage(1);
 		}
 	}
+	/* Check for valid grouping mode */
+	assert(VUT.g_arg < VSL_g__MAX);
+	if (VUT.g_arg != VSL_g_vxid && VUT.g_arg != VSL_g_request)
+		VUT_Error(1, "Invalid grouping mode: %s (only vxid and request are supported)",
+				VSLQ_grouping[VUT.g_arg]);
+
 	if (profile) {
 		for (active_profile = profiles; active_profile->name;
 			active_profile++) {



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] e6645af Remove -i, -I, -x, -X options invarnishhist
Message-ID: <E1WO1sK-0007PF-6C@project.varnish-software.com>


commit e6645af6b642107a671120eb29376e165f5ce0c2
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 17:17:22 2014 +0100

    Remove -i, -I, -x, -X options invarnishhist

diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index 2b71835..65301e5 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -53,8 +53,6 @@ VUT_OPT_d
 VUT_OPT_D
 VUT_OPT_g
 VUT_OPT_h
-VSL_OPT_i
-VSL_OPT_I
 VSL_OPT_L
 VUT_OPT_n
 VUT_OPT_N
@@ -63,6 +61,4 @@ HIS_OPT_P
 VUT_OPT_q
 VUT_OPT_r
 VSL_OPT_T
-VSL_OPT_x
-VSL_OPT_X
 VUT_OPT_V



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 476c8fd Remove useless call to VSL_Match
Message-ID: <E1WO1sK-0007PO-7e@project.varnish-software.com>


commit 476c8fd7bac32486d85a1e395bea77647cd86b10
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 17:18:31 2014 +0100

    Remove useless call to VSL_Match

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index ec30da5..0824b48 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -199,8 +199,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 		value = -1;
 		hit = 0;
 		while ((1 == VSL_Next(tr->c))) {
-			if (!VSL_Match(vsl, tr->c))
-				continue;
 			/* get the value we want, and register if it's a hit*/
 			tag = VSL_TAG(tr->c->rec.ptr);
 			assert(match_tag >= 0);



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] b733d48 Force filtering to client requests
Message-ID: <E1WO1sK-0007PX-8d@project.varnish-software.com>


commit b733d48c62be51ca81273b326aafa7fe1e76c262
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 17:22:06 2014 +0100

    Force filtering to client requests

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 0824b48..1fed60b 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -347,6 +347,8 @@ main(int argc, char **argv)
 	if (0)
 		(void)usage;
 
+	/* only client requests */
+	assert(VUT_Arg('c', NULL));
 	while ((i = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (i) {
 		case 'P':
diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index 65301e5..5e595a3 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -46,8 +46,6 @@
 	    " (these are power of tens)"	\
 	)
 
-VSL_OPT_b
-VSL_OPT_c
 VSL_OPT_C
 VUT_OPT_d
 VUT_OPT_D



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] c37045f Keep only reqs and bereqs
Message-ID: <E1WO1sK-0007Ph-AC@project.varnish-software.com>


commit c37045f9a1d5740a6c4900a77b8f240bc982b8b2
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 17:37:21 2014 +0100

    Keep only reqs and bereqs

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1fed60b..01fc66d 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -198,6 +198,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	for (tr = pt[0]; tr != NULL; tr = *++pt) {
 		value = -1;
 		hit = 0;
+		if (tr->type != VSL_t_bereq && tr->type != VSL_t_req)
+			continue;
 		while ((1 == VSL_Next(tr->c))) {
 			/* get the value we want, and register if it's a hit*/
 			tag = VSL_TAG(tr->c->rec.ptr);



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 28b62f1 Fix Ctrl-C handling in varnishhist
Message-ID: <E1WO1sK-0007Pq-BI@project.varnish-software.com>


commit 28b62f14c1c5a4767d13763995776aa3a0590cf5
Author: Guillaume Quintard <guillaume.quintard at gmail.com>
Date:   Thu Feb 6 17:40:09 2014 +0100

    Fix Ctrl-C handling in varnishhist

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 01fc66d..51c87ec 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -288,16 +288,15 @@ do_curses(void *arg)
 			redrawwin(stdscr);
 			refresh();
 			break;
-		case '\003': /* Ctrl-C */
-			raise(SIGINT);
-			break;
 		case '\032': /* Ctrl-Z */
 			endwin();
 			raise(SIGTSTP);
 			break;
+		case '\003': /* Ctrl-C */
 		case '\021': /* Ctrl-Q */
 		case 'Q':
 		case 'q':
+			raise(SIGINT);
 			endwin();
 			pthread_exit(NULL);
 		case '0':



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] a870df8 Add attribution
Message-ID: <E1WO1sK-0007Pz-CX@project.varnish-software.com>


commit a870df8d69e1bb95c2a8f733f7c6f4a3fd7f1e8b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 15:53:48 2014 +0100

    Add attribution

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 51c87ec..3bc1a3a 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -5,6 +5,7 @@
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
  * Author: Dag-Erling Sm?rgrav <des at des.no>
+ * Author: Guillaume Quintard <guillaume.quintard at gmail.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] a3c4079 Line break and whitespace fixes
Message-ID: <E1WO1sK-0007Q8-Dq@project.varnish-software.com>


commit a3c40796003fc7cedf8e86c5684d76fd20d1a3c4
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 4 15:58:01 2014 +0100

    Line break and whitespace fixes

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 3bc1a3a..87558e1 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -108,8 +108,7 @@ struct profile {
 	int field;
 	int hist_low;
 	int hist_high;
-} 
-
+}
 profiles[] = {
 	{
 		.name = "responsetime",
@@ -206,7 +205,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			tag = VSL_TAG(tr->c->rec.ptr);
 			assert(match_tag >= 0);
 			if (tag == (unsigned)match_tag) {
-				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format, &value);
+				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format,
+				    &value);
 				assert(i == 1);
 			} else if (tag == SLT_Hit)
 				hit = 1;
@@ -361,9 +361,11 @@ main(int argc, char **argv)
 				break;
 			}
 			/* else it's a definition, we hope */
-			if (sscanf(colon+1, "%d:%d:%d", 
-					&cli_p.field, &cli_p.hist_low, &cli_p.hist_high) != 3) {
-				fprintf(stderr, "%s is neither a profile name nor definition (SLT_Tag:field:min:max)\n", optarg);
+			if (sscanf(colon+1, "%d:%d:%d",	&cli_p.field,
+				&cli_p.hist_low, &cli_p.hist_high) != 3) {
+				fprintf(stderr, "%s is neither a profile name"
+				    "nor definition (SLT_Tag:field:min:max)\n",
+				    optarg);
 				exit(1);
 			}
 



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 9b9498c Silence unused argument warning
Message-ID: <E1WO1sK-0007QK-F7@project.varnish-software.com>


commit 9b9498cc4bc933f43ab7aed3d0a7ff71bf622f07
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 11:50:44 2014 +0100

    Silence unused argument warning

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 87558e1..3cf128e 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -194,7 +194,10 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	unsigned tag, hit;
 	double value;
 	struct VSL_transaction *tr;
+
+	(void)vsl;
 	(void)priv;
+
 	for (tr = pt[0]; tr != NULL; tr = *++pt) {
 		value = -1;
 		hit = 0;



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] b6eacbd Fix varnishhist Makefile
Message-ID: <E1WO1sK-0007QT-G7@project.varnish-software.com>


commit b6eacbde019ec778b6cbb6bbba5921e844d1ea04
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 11:51:04 2014 +0100

    Fix varnishhist Makefile

diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 986966f..748721b 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -7,8 +7,8 @@ bin_PROGRAMS = varnishhist
 varnishhist_SOURCES = varnishhist.c \
 	varnishhist_options.h \
 	varnishhist_options.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c \
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/version.c \
 	$(top_srcdir)/lib/libvarnish/vpf.c \
 	$(top_srcdir)/lib/libvarnish/vtim.c \
 	$(top_srcdir)/lib/libvarnish/flopen.c \
@@ -25,29 +25,3 @@ varnishhist_opt2rst_SOURCES = \
 	varnishhist_options.h \
 	varnishhist_options.c \
 	$(top_srcdir)/lib/libvarnishtools/opt2rst.c
-
-BUILT_SOURCES = varnishhist_options.rst varnishhist_synopsis.rst
-EXTRA_DIST = $(BUILT_SOURCES)
-MAINTAINERCLEANFILES = $(EXTRA_DIST)
-
-varnishhist_options.rst:
-	./varnishhist_opt2rst options > $@
-varnishhist_synopsis.rst:
-	./varnishhist_opt2rst synopsis > $@
-
-if HAVE_RST2MAN
-varnishhist_options.rst varnishhist_synopsis.rst: varnishhist_opt2rst
-endif
-
-varnishhist.1: \
-	$(top_srcdir)/doc/sphinx/reference/varnishhist.rst \
-	varnishhist_options.rst \
-	varnishhist_synopsis.rst
-if HAVE_RST2MAN
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 57b22d5 Fix line breaks
Message-ID: <E1WO1sK-0007Qc-HI@project.varnish-software.com>


commit 57b22d58377226ae257af57cf85595fe5310a057
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 4 16:47:59 2014 +0100

    Fix line breaks

diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index 5e595a3..6c9baa2 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -39,11 +39,13 @@
 	)
 
 #define HIS_OPT_P							\
-	VOPT("P:", "[-P <size|responsetime|tag:field_num:min:max>]", "Profile definition",			\
-		"Either specify \"size\" or \"responstime\" profile or create a new one." \
-	    " Define the tag we'll look for, and the field number of the value"	\
-	    " we are interested in. min and max are the boundaries of the graph"	\
-	    " (these are power of tens)"	\
+	VOPT("P:", "[-P <size|responsetime|tag:field_num:min:max>]",	\
+	    "Profile definition",					\
+	    "Either specify \"size\" or \"responstime\" profile or"	\
+	    " create a new one. Define the tag we'll look for, and the"	\
+	    " field number of the value we are interested in. min and"	\
+	    " max are the boundaries of the graph (these are power of"	\
+	    " tens)."							\
 	)
 
 VSL_OPT_C



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 5f13f64 Fix up -p documentation
Message-ID: <E1WO1sK-0007Qm-Ik@project.varnish-software.com>


commit 5f13f64be13fa9f68ca059fc69746743307bb9a0
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 4 16:48:43 2014 +0100

    Fix up -p documentation

diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index 6c9baa2..db5a870 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -33,9 +33,9 @@
 
 #define HIS_OPT_p							\
 	VOPT("p:", "[-p period]", "Refresh period",			\
-	    "Specified the number of seconds between screen refreshes"	\
-	    " default is 1 second, and can be changed during the run"	\
-	    " by pressing the [1-9] keys"	\
+	    "Specified the number of seconds between screen refreshes."	\
+	    " Default is 1 second, and can be changed at runtime by"	\
+	    " pressing the [1-9] keys."					\
 	)
 
 #define HIS_OPT_P							\



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 343418d Change varnishtop -> varnishhist
Message-ID: <E1WO1sK-0007Qw-Jy@project.varnish-software.com>


commit 343418dade50e6aa7035005d10532a88b99c7c85
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 4 16:49:42 2014 +0100

    Change varnishtop -> varnishhist

diff --git a/bin/varnishhist/varnishhist_options.c b/bin/varnishhist/varnishhist_options.c
index 718fd3d..c4f5d27 100644
--- a/bin/varnishhist/varnishhist_options.c
+++ b/bin/varnishhist/varnishhist_options.c
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Option definitions for varnishtop
+ * Option definitions for varnishhist
  */
 
 #include <stdlib.h>
diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index db5a870..11479dd 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Option definitions for varnishtop
+ * Option definitions for varnishhist
  */
 
 #include "vapi/vapi_options.h"



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 885e5f1 Update linebreaks and error string formatting
Message-ID: <E1WO1sK-0007R5-LK@project.varnish-software.com>


commit 885e5f1fedca8bbce581325ab9c941db1e329992
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 13:03:16 2014 +0100

    Update linebreaks and error string formatting

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 3cf128e..4fc6866 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -366,15 +366,16 @@ main(int argc, char **argv)
 			/* else it's a definition, we hope */
 			if (sscanf(colon+1, "%d:%d:%d",	&cli_p.field,
 				&cli_p.hist_low, &cli_p.hist_high) != 3) {
-				fprintf(stderr, "%s is neither a profile name"
-				    "nor definition (SLT_Tag:field:min:max)\n",
-				    optarg);
+				fprintf(stderr, "-P: '%s' is not a valid"
+				    " profile name or definition\n", optarg);
 				exit(1);
 			}
 
 			match_tag = VSL_Name2Tag(optarg, colon - optarg);
 			if (match_tag < 0) {
-				fprintf(stderr, "No such tag in %s\n", optarg);
+				fprintf(stderr,
+				    "-P: '%s' is not a valid tag name\n",
+				    optarg);
 				exit(1);
 			}
 			cli_p.name = "custom";
@@ -391,8 +392,9 @@ main(int argc, char **argv)
 	/* Check for valid grouping mode */
 	assert(VUT.g_arg < VSL_g__MAX);
 	if (VUT.g_arg != VSL_g_vxid && VUT.g_arg != VSL_g_request)
-		VUT_Error(1, "Invalid grouping mode: %s (only vxid and request are supported)",
-				VSLQ_grouping[VUT.g_arg]);
+		VUT_Error(1, "Invalid grouping mode: %s"
+		    " (only vxid and request are supported)",
+		    VSLQ_grouping[VUT.g_arg]);
 
 	if (profile) {
 		for (active_profile = profiles; active_profile->name;
@@ -403,7 +405,7 @@ main(int argc, char **argv)
 		}
 	}
 	if (! active_profile->name) {
-		fprintf(stderr, "No such profile %s\n", profile);
+		fprintf(stderr, "-P: No such profile '%s'\n", profile);
 		exit(1);
 	}
 	match_tag = active_profile->tag;



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 18266c8 Make field match failure ignore the record rather than
 assert
Message-ID: <E1WO1sK-0007RE-Mo@project.varnish-software.com>


commit 18266c8b46c0919e949e71df8897dbcf850b242a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 13:12:21 2014 +0100

    Make field match failure ignore the record rather than assert

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 4fc6866..1767db0 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -210,7 +210,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			if (tag == (unsigned)match_tag) {
 				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format,
 				    &value);
-				assert(i == 1);
+				if (i != 1)
+					continue;
 			} else if (tag == SLT_Hit)
 				hit = 1;
 			if (tag != SLT_ReqEnd && value == -1)



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] d0dfe0a Rearrange the varnishhist loop
Message-ID: <E1WO1sK-0007RY-Ov@project.varnish-software.com>


commit d0dfe0ae50f74af49c8e0d5bcb41b0cc76973830
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 14:41:23 2014 +0100

    Rearrange the varnishhist loop
    
    Rearrange the varnishhist loop so that we can skip restarts from being
    counted and make it more readable.

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index faec821..b77d381 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -190,8 +190,7 @@ static int /*__match_proto__ (VSLQ_dispatch_f)*/
 accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	void *priv)
 {
-	int i, j;
-	unsigned tag, hit;
+	int i, j, tag, skip, match, hit;
 	double value;
 	struct VSL_transaction *tr;
 
@@ -205,62 +204,75 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 		if (tr->reason == VSL_r_esi)
 			/* Skip ESI requests */
 			continue;
-		value = -1;
+
 		hit = 0;
-		while ((1 == VSL_Next(tr->c))) {
+		skip = 0;
+		match = 0;
+		while (skip == 0 && 1 == VSL_Next(tr->c)) {
 			/* get the value we want, and register if it's a hit*/
 			tag = VSL_TAG(tr->c->rec.ptr);
-			assert(match_tag >= 0);
-			if (tag == (unsigned)match_tag) {
+
+			switch (tag) {
+			case SLT_Hit:
+				hit = 1;
+				break;
+			case SLT_VCL_return:
+				if (!strcasecmp(VSL_CDATA(tr->c->rec.ptr),
+					"restart"))
+					skip = 1;
+				break;
+			default:
+				if (tag != match_tag)
+					break;
 				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format,
 				    &value);
 				if (i != 1)
-					continue;
-			} else if (tag == SLT_Hit)
-				hit = 1;
-			if (tag != SLT_ReqEnd && value == -1)
-				continue;
-
-			/* select bucket */
-			i = HIST_RES * (log(value) / log_ten);
-			if (i < hist_low * HIST_RES)
-				i = hist_low * HIST_RES;
-			if (i >= hist_high * HIST_RES)
-				i = hist_high * HIST_RES - 1;
-			i -= hist_low * HIST_RES;
-			assert(i >= 0);
-			assert(i < hist_buckets);
-			pthread_mutex_lock(&mtx);
-
-			/* phase out old data */
-			if (nhist == HIST_N) {
-				j = rr_hist[next_hist];
-				if (j < 0) {
-					assert(bucket_miss[-j] > 0);
-					bucket_miss[-j]--;
-				} else {
-					assert(bucket_hit[j] > 0);
-					bucket_hit[j]--;
-				}
-			} else {
-				++nhist;
+					break;
+				match = 1;
+				break;
 			}
+		}
 
-			/* phase in new data */
-			if (hit) {
-				bucket_hit[i]++;
-				rr_hist[next_hist] = i;
+		if (skip || !match)
+			continue;
+
+		/* select bucket */
+		i = HIST_RES * (log(value) / log_ten);
+		if (i < hist_low * HIST_RES)
+			i = hist_low * HIST_RES;
+		if (i >= hist_high * HIST_RES)
+			i = hist_high * HIST_RES - 1;
+		i -= hist_low * HIST_RES;
+		assert(i >= 0);
+		assert(i < hist_buckets);
+		pthread_mutex_lock(&mtx);
+
+		/* phase out old data */
+		if (nhist == HIST_N) {
+			j = rr_hist[next_hist];
+			if (j < 0) {
+				assert(bucket_miss[-j] > 0);
+				bucket_miss[-j]--;
 			} else {
-				bucket_miss[i]++;
-				rr_hist[next_hist] = -i;
-			}
-			if (++next_hist == HIST_N) {
-				next_hist = 0;
+				assert(bucket_hit[j] > 0);
+				bucket_hit[j]--;
 			}
-			pthread_mutex_unlock(&mtx);
-
+		} else {
+			++nhist;
+		}
 
+		/* phase in new data */
+		if (hit) {
+			bucket_hit[i]++;
+			rr_hist[next_hist] = i;
+		} else {
+			bucket_miss[i]++;
+			rr_hist[next_hist] = -i;
 		}
+		if (++next_hist == HIST_N) {
+			next_hist = 0;
+		}
+		pthread_mutex_unlock(&mtx);
 	}
 	return (0);
 }



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 3a1ba09 Only look at client requests in varnishhist.
Message-ID: <E1WO1sK-0007RP-Nw@project.varnish-software.com>


commit 3a1ba09193055cc43d3c9896edaec7939a7e37f5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 14:05:19 2014 +0100

    Only look at client requests in varnishhist.
    
    Without the -b -c options enabled, looking at backend requests becomes
    ill-defined. Disable anything but non-esi client requests for
    now. This can be extended in the future.

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1767db0..faec821 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -199,10 +199,14 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	(void)priv;
 
 	for (tr = pt[0]; tr != NULL; tr = *++pt) {
+		if (tr->type != VSL_t_req)
+			/* Only look at client requests */
+			continue;
+		if (tr->reason == VSL_r_esi)
+			/* Skip ESI requests */
+			continue;
 		value = -1;
 		hit = 0;
-		if (tr->type != VSL_t_bereq && tr->type != VSL_t_req)
-			continue;
 		while ((1 == VSL_Next(tr->c))) {
 			/* get the value we want, and register if it's a hit*/
 			tag = VSL_TAG(tr->c->rec.ptr);



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 549ecd9 Disable daemon mode for varnishhist
Message-ID: <E1WO1sK-0007Rh-Px@project.varnish-software.com>


commit 549ecd98eceb8536985641b8d3d8b50725830b5b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 14:52:37 2014 +0100

    Disable daemon mode for varnishhist

diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index 11479dd..bab592a 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -50,7 +50,6 @@
 
 VSL_OPT_C
 VUT_OPT_d
-VUT_OPT_D
 VUT_OPT_g
 VUT_OPT_h
 VSL_OPT_L



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 8c022f2 Redefine -g option to reflect the valid grouping types.
Message-ID: <E1WO1sK-0007Rq-RK@project.varnish-software.com>


commit 8c022f2def00ae8389ee00b681b40fd81c56abe4
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 15:00:27 2014 +0100

    Redefine -g option to reflect the valid grouping types.

diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index bab592a..0a0e2b3 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -31,6 +31,12 @@
 #include "vapi/vapi_options.h"
 #include "vut_options.h"
 
+#define HIS_OPT_g							\
+	VOPT("g:", "[-g <request|vxid>]", "Grouping mode",		\
+	    "The grouping of the log records. The default is to group"	\
+	    " by vxid."							\
+	)
+
 #define HIS_OPT_p							\
 	VOPT("p:", "[-p period]", "Refresh period",			\
 	    "Specified the number of seconds between screen refreshes."	\
@@ -50,7 +56,7 @@
 
 VSL_OPT_C
 VUT_OPT_d
-VUT_OPT_g
+HIS_OPT_g
 VUT_OPT_h
 VSL_OPT_L
 VUT_OPT_n



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 168be72 Update copyright
Message-ID: <E1WO1sK-0007S1-Sa@project.varnish-software.com>


commit 168be724894759602b33d401994be94945e9488b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 15:53:29 2014 +0100

    Update copyright

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index b77d381..c7406da 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2014 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 57a7f98 Add the missing build bits for varnishhist
Message-ID: <E1WO1sK-0007SA-To@project.varnish-software.com>


commit 57a7f98eaa38cb6b9e2061a2d220fe9728cfd97b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 16:15:40 2014 +0100

    Add the missing build bits for varnishhist

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 8e891c8..5be4551 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -123,6 +123,7 @@ EXTRA_DIST = \
 	reference/varnishstat.rst \
 	reference/varnishtest.rst \
 	reference/varnishtop.rst \
+	reference/varnishhist.rst \
 	reference/vcl.rst \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
@@ -198,5 +199,12 @@ include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop_opt2r
 BUILT_SOURCES += include/varnishtop_options.rst \
 	 include/varnishtop_synopsis.rst
 
+include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist_opt2rst
+	$(top_builddir)/bin/varinshhist/varnishhist_opt2rst options > $@
+include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_opt2rst
+	$(top_builddir)/bin/varinshhist/varnishhist_opt2rst synopsis > $@
+BUILT_SOURCES += include/varnishhist_options.rst \
+	 include/varnishhist_synopsis.rst
+
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/man/Makefile.am b/man/Makefile.am
index 50e9b64..3321a2f 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -21,7 +21,8 @@ dist_man_MANS = \
 	varnishreplay.1 \
 	varnishstat.1 \
 	varnishtest.1 \
-	varnishtop.1
+	varnishtop.1 \
+	varnishhist.1
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
@@ -76,3 +77,9 @@ varnishtop.1: \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_synopsis.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
+
+varnishhist.1: \
+	$(top_srcdir)/doc/sphinx/reference/varnishhist.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishhist_options.rst \
+	$(top_srcdir)/doc/sphinx/include/varnishhist_synopsis.rst
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@



From martin at varnish-software.com  Thu Mar 13 09:24:28 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:28 +0100
Subject: [4.0] 56cbab9 Fix spelling error
Message-ID: <E1WO1sK-0007SJ-Ux@project.varnish-software.com>


commit 56cbab96034e29f1492f1e4b66162076a31553d5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 16:20:59 2014 +0100

    Fix spelling error

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 5be4551..631a3bb 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -200,7 +200,7 @@ BUILT_SOURCES += include/varnishtop_options.rst \
 	 include/varnishtop_synopsis.rst
 
 include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist_opt2rst
-	$(top_builddir)/bin/varinshhist/varnishhist_opt2rst options > $@
+	$(top_builddir)/bin/varnishhist/varnishhist_opt2rst options > $@
 include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_opt2rst
 	$(top_builddir)/bin/varinshhist/varnishhist_opt2rst synopsis > $@
 BUILT_SOURCES += include/varnishhist_options.rst \



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 9e3ce13 And another spelling error
Message-ID: <E1WO1sL-0007SS-0H@project.varnish-software.com>


commit 9e3ce1315ff82857d3aa0aac44d0515b05a58998
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 16:22:54 2014 +0100

    And another spelling error

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 631a3bb..26033c2 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -202,7 +202,7 @@ BUILT_SOURCES += include/varnishtop_options.rst \
 include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist_opt2rst
 	$(top_builddir)/bin/varnishhist/varnishhist_opt2rst options > $@
 include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_opt2rst
-	$(top_builddir)/bin/varinshhist/varnishhist_opt2rst synopsis > $@
+	$(top_builddir)/bin/varnishhist/varnishhist_opt2rst synopsis > $@
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 



From perbu at varnish-software.com  Thu Mar 13 09:24:29 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 5885903 Seems we don't need to ignore these
Message-ID: <E1WO1sL-0007Sc-1V@project.varnish-software.com>


commit 58859030a9d13f283dbdbc6d106463cc114ee709
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Feb 25 20:09:25 2014 +0100

    Seems we don't need to ignore these

diff --git a/.gitignore b/.gitignore
index 3df191e..dd684da 100644
--- a/.gitignore
+++ b/.gitignore
@@ -79,8 +79,6 @@ cscope.*out
 /man/vsc2rst
 /man/*.7
 /man/*.1
-/bin/*/*.1
-/bin/*/*.rst
 /doc/sphinx/include
 /bin/varnish*/varnish*_opt2rst
 /bin/varnishadm/varnishadm



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 3f79aa1 Fix include directory path
Message-ID: <E1WO1sL-0007Sl-2i@project.varnish-software.com>


commit 3f79aa1a30ecba3742d5964b7d5666318044a97d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 25 22:46:44 2014 +0100

    Fix include directory path
    
    Fix the varnishhist documentation include file paths.

diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst
index 901c7f4..8071384 100644
--- a/doc/sphinx/reference/varnishhist.rst
+++ b/doc/sphinx/reference/varnishhist.rst
@@ -9,7 +9,7 @@ Varnish request histogram
 SYNOPSIS
 ========
 
-.. include:: ../../../bin/varnishhist/varnishhist_synopsis.rst
+.. include:: ../include/varnishhist_synopsis.rst
 varnishhist |synopsis|
 
 DESCRIPTION
@@ -24,7 +24,7 @@ and misses are marked with a hash character ("#").
 
 The following options are available:
 
-.. include:: ../../../bin/varnishhist/varnishhist_options.rst
+.. include:: ../include/varnishhist_options.rst
 
 SEE ALSO
 ========



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 7c511ff Remove operation.rst from the dist lists to fix build
Message-ID: <E1WO1sL-0007Sv-3l@project.varnish-software.com>


commit 7c511ff549e14aee6cd853e1f35721cd532a7278
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Feb 26 11:07:24 2014 +0100

    Remove operation.rst from the dist lists to fix build

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 26033c2..ac09cc0 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -145,7 +145,6 @@ EXTRA_DIST = \
 	users-guide/index.rst \
 	users-guide/intro.rst \
 	users-guide/operation-logging.rst \
-	users-guide/operation.rst \
 	users-guide/operation-statistics.rst \
 	users-guide/params.rst \
 	users-guide/performance.rst \
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index b14e6cb..18197a6 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -155,7 +155,6 @@ EXTRA_DIST = \
 	users-guide/intro.rst \
 	users-guide/operation-cli.rst \
 	users-guide/operation-logging.rst \
-	users-guide/operation.rst \
 	users-guide/operation-statistics.rst \
 	users-guide/params.rst \
 	users-guide/performance.rst \



From perbu at varnish-software.com  Thu Mar 13 09:24:29 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 2b9db41 Rewrite for 4.0 - might still need review
Message-ID: <E1WO1sL-0007TH-68@project.varnish-software.com>


commit 2b9db41c981d1218c1a83e9b6e6174fe3513f671
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Feb 26 12:29:27 2014 +0100

    Rewrite for 4.0 - might still need review

diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 14e07ed..15b6da3 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -95,4 +95,5 @@ To call a subroutine, use the call keyword followed by the subroutine's name:
 call pipe_if_local;
 
 Varnish has quite a few built in subroutines that are called for each
-transaction as it flows through Varnish. See :ref:`vcl-built-in-subs`.
+transaction as it flows through Varnish. These builtin subroutines are all named vcl_*. Your own subroutines cannot start their name with vcl_.
+See :ref:`vcl-built-in-subs`.



From perbu at varnish-software.com  Thu Mar 13 09:24:29 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 56adaeb -b is redundant
Message-ID: <E1WO1sL-0007TR-8K@project.varnish-software.com>


commit 56adaeb29445c6be2cf8dc4fc53e8a5dca5825b3
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Feb 26 12:30:21 2014 +0100

    -b is redundant

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 9d1dc67..431c0fc 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -23,8 +23,10 @@ Tool: varnishtop
 ~~~~~~~~~~~~~~~~
 
 You can use varnishtop to identify what URLs are hitting the backend
-the most. ``varnishtop -b -i txurl`` is an essential command. You can see
-some other examples of varnishtop usage in :ref:`users-guide-statistics`.
+the most. ``varnishtop -i txurl`` is an essential command, showing you
+the top txurl requests Varnish is sending towards the backend. You can
+see some other examples of varnishtop usage in
+:ref:`users-guide-statistics`.
 
 
 Tool: varnishlog



From perbu at varnish-software.com  Thu Mar 13 09:24:29 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 0aac416 4.0 rewrite
Message-ID: <E1WO1sL-0007Ta-A5@project.varnish-software.com>


commit 0aac4167512cad131023bfdefb84072922d15a47
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Feb 26 12:30:50 2014 +0100

    4.0 rewrite

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 2202af0..7a31aef 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -22,7 +22,7 @@ yourself doing frequently.
 The vcl_recv subroutine may terminate with calling ``return()`` on one
 of the following keywords:
 
-  error code [reason]
+  error 
     Return the specified error code to the client and abandon the request.
 
   pass
@@ -31,56 +31,117 @@ of the following keywords:
   pipe
     Switch to pipe mode.  Control will eventually pass to vcl_pipe.
 
-  lookup
-    Look up the requested object in the cache.  Control will
-    eventually pass to vcl_hit or vcl_miss, depending on whether the
-    object is in the cache.  The ``bereq.method`` value will be set
-    to ``GET`` regardless of the value of ``req.method``.
+  hash
+    Continue processing the object as a potential candidate for
+    caching. Passes the control over to vcl_hash.
 
+  purge
+    Calls vcl_purge where the object will be purged.
 
 
-vcl_backend_fetch
-~~~~~~~~~~~~~~~~~
+vcl_pipe
+~~~~~~~~
 
-Called before sending the backend request. In this subroutine you
-typically alter the request before it gets to the backend.
+Called upon entering pipe mode.  In this mode, the request is passed
+on to the backend, and any further data from either client or backend
+is passed on unaltered until either end closes the
+connection. Basically, Varnish will degrade into a simple TCP proxy,
+shuffling bytes back and forth.
 
-.. XXX Return statements?
+The vcl_pipe subroutine may terminate with calling return() with one
+of the following keywords:
 
+  error code [reason]
+    Return the specified error code to the client and abandon the request.
 
-vcl_backend_response
-~~~~~~~~~~~~~~~~~~~
+  pipe
+    Proceed with pipe mode.
 
-Called after a document has been successfully retrieved from the backend.
+vcl_pass
+~~~~~~~~
 
-The vcl_backend_response subroutine may terminate with calling return() with one
-of the following keywords:
+Called upon entering pass mode.  In this mode, the request is passed
+on to the backend, and the backend's response is passed on to the
+client, but is not entered into the cache.  Subsequent requests
+submitted over the same client connection are handled normally.
 
-  deliver
-    Possibly insert the object into the cache, then deliver it to the
-    client.  Control will eventually pass to vcl_deliver.
+The vcl_pass subroutine may terminate with calling return() with one
+of the following keywords:
 
-  error code [reason]
+  error [reason]
     Return the specified error code to the client and abandon the request.
 
-  hit_for_pass
-    Pass in fetch. Passes the object without caching it. This will
-    create a so-called hit_for_pass object which has the side effect
-    that the decision not to cache will be cached. This is to allow
-    would-be uncachable requests to be passed to the backend at the
-    same time. The same logic is not necessary in vcl_recv because
-    this happens before any potential queueing for an object takes
-    place.  Note that the TTL for the hit_for_pass object will be set
-    to what the current value of beresp.ttl is. Control will be
-    handled to vcl_deliver on the current request, but subsequent
-    requests will go directly to vcl_pass based on the hit_for_pass
-    object.
+  pass
+    Proceed with pass mode.
+
+  restart
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
+    error.
+
+
+vcl_hit
+~~~~~~~
+
+Called is a cache lookup is successful. 
 
   restart
     Restart the transaction. Increases the restart counter. If the number
     of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
+  deliver
+    Deliver the object. Control passes to vcl_deliver.
+
+  error [reason]
+    Return the specified error code to the client and abandon the request.
+
+
+vcl_miss
+~~~~~~~~
+
+Called after a cache lookup if the requested document was not found in
+the cache.  Its purpose is to decide whether or not to attempt to
+retrieve the document from the backend, and which backend to use.
+
+The vcl_miss subroutine may terminate with calling return() with one
+of the following keywords:
+
+  error [reason]
+    Return the specified error code to the client and abandon the request.
+
+  pass
+    Switch to pass mode.  Control will eventually pass to vcl_pass.
+
+  fetch
+    Retrieve the requested object from the backend.  Control will
+    eventually pass to vcl_fetch.
+
+vcl_hash
+~~~~~~~~
+
+Called after vcl_recv to create a hash value for the request. This is
+used as a key to look up the object in Varnish.
+
+  lookup
+    Look up the object in cache.
+
+
+vcl_purge
+~~~~~~~~~
+
+Purge the object and all it's variants. Variants created when the
+backend issues a Vary response.
+
+  fetch
+    Execute the purge.
+
+  error
+    Fail the purge request. Typically you would call error here it
+    request doesn't pass the ACLs.
+
+
+
 vcl_deliver
 ~~~~~~~~~~~
 
@@ -97,83 +158,82 @@ keywords:
     of restarts is higher than *max_restarts* Varnish emits a guru meditation
     error.
 
-vcl_backend_error
-~~~~~~~~~~~~~~~~~
 
-Called when we hit an error, either explicitly or implicitly due to
-backend or internal errors.
+.. XXX
+.. vcl_error
+.. ~~~~~~~~~
 
-The vcl_backend_error subroutine may terminate by calling return with one of
-the following keywords:
+.. Not sure if we're going to keep this around.
 
-  deliver
-    Deliver the error object to the client.
 
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
+vcl_backend_fetch
+~~~~~~~~~~~~~~~~~
 
+Called before sending the backend request. In this subroutine you
+typically alter the request before it gets to the backend.
 
-vcl_pipe
-~~~~~~~~
+  fetch
+    Fetch the object from the backend.
 
-Called upon entering pipe mode.  In this mode, the request is passed
-on to the backend, and any further data from either client or
-backend is passed on unaltered until either end closes the
-connection.
+  abandon
+    Abandon the backend request and generates an error.
+  
 
- The vcl_pipe subroutine may terminate with calling return() with one of
- the following keywords:
+vcl_backend_response
+~~~~~~~~~~~~~~~~~~~~
 
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
+Called after an response has been successfully retrieved from the
+backend. The response is availble as beresp. Note that Varnish might
+not be talking to an actual client, so operations that require a
+client to be present are not allowed. Specifically there is no req
+object and restarts are not allowed.
 
-  pipe
-    Proceed with pipe mode.
+The vcl_backend_response subroutine may terminate with calling return() with one
+of the following keywords:
 
-vcl_pass
-~~~~~~~~
+  deliver
+    Possibly insert the object into the cache, then deliver it to the
+    Control will eventually pass to vcl_deliver. Caching is dependant
+    on beresp.cacheable.
 
-Called upon entering pass mode.  In this mode, the request is passed
-on to the backend, and the backend's response is passed on to the
-client, but is not entered into the cache.  Subsequent requests
-submitted over the same client connection are handled normally.
+  error [reason]
+    Return the specified error code to the client and abandon the request.
 
-The vcl_pass subroutine may terminate with calling return() with one
-of the following keywords:
+  retry
+    Retry the backend transaction. Increases the retries counter. If the number
+    of retries is higher than *max_retries* Varnish emits a guru meditation
+    error.
 
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
+vcl_backend_error
+~~~~~~~~~~~~~~~~~
 
-  pass
-    Proceed with pass mode.
+This subroutine is called if we fail the backend fetch. 
 
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
+  deliver
+    Deliver the error.
 
-vcl_miss
-~~~~~~~~
+  retry
+    Retry the backend transaction. Increases the retries counter. If the number
+    of retries is higher than *max_retries* Varnish emits a guru meditation
+    error.
 
-Called after a cache lookup if the requested document was not found in
-the cache.  Its purpose is to decide whether or not to attempt to
-retrieve the document from the backend, and which backend to use.
 
-The vcl_miss subroutine may terminate with calling return() with one
-of the following keywords:
+vcl_backend_error
+~~~~~~~~~~~~~~~~~
 
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
+Called when we hit an error, either explicitly or implicitly due to
+backend or internal errors.
 
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
+The vcl_backend_error subroutine may terminate by calling return with one of
+the following keywords:
 
-  fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_fetch.
+  deliver
+    Deliver the error object to the client.
 
+  retry
+    Retry the backend transaction. Increases the retries counter. If the number
+    of retries is higher than *max_retries* Varnish emits a guru meditation
+    error.
 
 
 vcl_init



From fgsch at lodoss.net  Thu Mar 13 09:24:29 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] e551669 Make rst2man mandatory
Message-ID: <E1WO1sL-0007Tl-Bp@project.varnish-software.com>


commit e5516699439d560f5df326a55a427b3a9f2bdce5
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Feb 26 14:20:00 2014 +0000

    Make rst2man mandatory
    
    This simplifies the autofoo magic and fixes the manpage installation
    on the dist tarball when rst2man was not installed.

diff --git a/Makefile.am b/Makefile.am
index 389be99..7624ec5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS = include lib bin etc doc redhat
+SUBDIRS = include lib bin etc doc man redhat
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = varnishapi.pc
@@ -20,16 +20,6 @@ DISTCHECK_CONFIGURE_FLAGS = \
 install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 
-if HAVE_RST2MAN
-SUBDIRS += man
-else
-dist-hook:
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
-
 distcheck-hook:
 	V="@PACKAGE_VERSION@" ; \
 	V="$${V%%-*}" ; \
diff --git a/configure.ac b/configure.ac
index 5c46de8..6f43695 100644
--- a/configure.ac
+++ b/configure.ac
@@ -50,15 +50,15 @@ CC="$PTHREAD_CC"
 
 AC_PROG_INSTALL
 AC_PROG_MAKE_SET
+
 AC_ARG_WITH([rst2man],
-               AS_HELP_STRING([--with-rst2man=PATH],
-                              [Location of rst2man (auto)]),
-               [RST2MAN="$withval"],
-               [AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], "no")
-	        if test "x$RST2MAN" = "xno"; then
-		   AC_MSG_WARN([rst2man not found - not building man pages])
-		fi])
-AM_CONDITIONAL(HAVE_RST2MAN,[test "x$RST2MAN" != "xno"])
+  AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
+  [RST2MAN="$withval"],
+  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], []))
+if test -z "$RST2MAN"; then
+  AC_MSG_ERROR(
+    [rst2man is needed to build Varnish, please install python-docutils.])
+fi
 
 AC_ARG_WITH([rst2html],
                AS_HELP_STRING([--with-rst2html=PATH],
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 9c96d39..f0a23fa 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -73,10 +73,8 @@ vsl2rst_SOURCES = \
 	$(top_srcdir)/include/tbl/vsl_tags.h \
 	$(top_srcdir)/include/tbl/vsl_tags_http.h
 
-if HAVE_RST2MAN
 vsl-tags.rst: vsl2rst
 	./vsl2rst > $@
-endif
 
 vxp_fixed_token.c vxp_tokens.h: \
 	$(srcdir)/generate.py
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index cd8aebf..8967bf5 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -38,11 +38,4 @@ CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
 	$(builddir)/vmod_std.man.rst
 
 vmod_std.3: $(top_srcdir)/doc/sphinx/reference/vmod_std.rst
-if HAVE_RST2MAN
 	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/man/Makefile.am b/man/Makefile.am
index 3321a2f..9343063 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -1,8 +1,6 @@
 #
 
-if HAVE_RST2MAN
 noinst_PROGRAMS = vsc2rst
-endif
 vsc2rst_SOURCES = vsc2rst.c \
 	$(top_srcdir)/include/tbl/vsc_fields.h
 



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 755836a Warn on VSL record batches containing records with VXID
 missmatch
Message-ID: <E1WO1sL-0007Tv-D9@project.varnish-software.com>


commit 755836a1c58f5ab6c6b2f895a077af33c88e088a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 27 10:17:54 2014 +0100

    Warn on VSL record batches containing records with VXID missmatch

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 0aee758..709b9fe 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -847,6 +847,11 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 
 	while (vslc_vtx_next(&vtx->c.cursor) == 1) {
 		ptr = vtx->c.cursor.rec.ptr;
+		if (VSL_ID(ptr) != vtx->key.vxid) {
+			(void)vtx_diag_tag(vtx, ptr, "vxid missmatch");
+			continue;
+		}
+
 		tag = VSL_TAG(ptr);
 		assert(tag != SLT__Batch);
 
@@ -1026,7 +1031,7 @@ static int
 vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr, const char *reason)
 {
 
-	vtx_synth_rec(vtx, SLT_VSL, "%s (%s: %.*s)", reason,
+	vtx_synth_rec(vtx, SLT_VSL, "%s (%u:%s \"%.*s\")", reason, VSL_ID(ptr),
 	    VSL_tags[VSL_TAG(ptr)], (int)VSL_LEN(ptr), VSL_CDATA(ptr));
 	return (-1);
 }



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 95dd968 Don't assert on multiple End records seen in a record
 set.
Message-ID: <E1WO1sL-0007U4-EJ@project.varnish-software.com>


commit 95dd9683ab3d4c4bb7423c7701721493bd00ea48
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 27 10:18:55 2014 +0100

    Don't assert on multiple End records seen in a record set.
    
    Spotted by: Dag

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 709b9fe..3acdbd2 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -842,10 +842,8 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 	const uint32_t *ptr;
 	enum VSL_tag_e tag;
 
-	if (vtx->flags & VTX_F_END)
-		return;
-
-	while (vslc_vtx_next(&vtx->c.cursor) == 1) {
+	while (!(vtx->flags & VTX_F_COMPLETE) &&
+	    vslc_vtx_next(&vtx->c.cursor) == 1) {
 		ptr = vtx->c.cursor.rec.ptr;
 		if (VSL_ID(ptr) != vtx->key.vxid) {
 			(void)vtx_diag_tag(vtx, ptr, "vxid missmatch");
@@ -867,12 +865,14 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 
 		case SLT_End:
 			vtx->flags |= VTX_F_END;
-			vtx_mark_complete(vslq, vtx);
 			break;
 		default:
 			break;
 		}
 	}
+
+	if (vtx->flags & VTX_F_END && !(vtx->flags & VTX_F_COMPLETE))
+		vtx_mark_complete(vslq, vtx);
 }
 
 /* Force a vtx into complete status by synthing the necessary outstanding



From lkarsten at varnish-software.com  Thu Mar 13 09:24:29 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 92d71fb s/req.backend/req.backend_hint/g
Message-ID: <E1WO1sL-0007UD-Fa@project.varnish-software.com>


commit 92d71fb8eeaf5d7e8599c7ecb90da6f603d0941c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Feb 26 13:12:59 2014 +0100

    s/req.backend/req.backend_hint/g

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 436d58d..57072d0 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -35,7 +35,7 @@ A backend declaration creates and initializes a named backend object::
 The backend object can later be used to select a backend at request time::
 
     if (req.http.host ~ "(?i)^(www.)?example.com$") {
-        set req.backend = www;
+        set req.backend_hint = www;
     }
 
 To avoid overloading backend servers, .max_connections can be set to
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index c7acd73..e60b438 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -229,7 +229,7 @@ special needs in VCL, you can use the X-UA-Device header like this::
         # call some detection engine
 
         if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
-            set req.backend = mobile;
+            set req.backend_hint = mobile;
         }
     }
     sub vcl_hash {
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 6e72416..099e2a6 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -62,9 +62,9 @@ Now we need tell where to send the difference URL. Lets look at vcl_recv.::
 
     sub vcl_recv {
         if (req.url ~ "^/java/") {
-            set req.backend = java;
+            set req.backend_hint = java;
         } else {
-            set req.backend = default.
+            set req.backend_hint = default.
         }
     }
 
@@ -88,9 +88,9 @@ You can have something like this:::
 
     sub vcl_recv {
         if (req.http.host ~ "foo.com") {
-            set req.backend = foo;
+            set req.backend_hint = foo;
         } elsif (req.http.host ~ "bar.com") {
-            set req.backend = bar;
+            set req.backend_hint = bar;
         }
     }
 
@@ -101,7 +101,7 @@ more tight, maybe relying on the == operator in stead, like this:::
 
     sub vcl_recv {
         if (req.http.host == "foo.com" or req.http.host == "www.foo.com") {
-            set req.backend = foo;
+            set req.backend_hint = foo;
         }
     }
 
@@ -138,7 +138,7 @@ call certain actions in vcl_init.::
 
     sub vcl_recv {
         # send all traffic to the bar director:
-        req.backend = bar.backend();
+        req.backend_hint = bar.backend();
     }
 
 This director is a round-robin director. This means the director will
diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 97e3a0d..68a41d0 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -32,7 +32,7 @@ Since the client director was already a special case of the hash director, it ha
 	}
 
 	sub vcl_recv {
-		set req.backend = h.backend(client.ip);
+		set req.backend_hint = h.backend(client.ip);
 	}
 
 error() is now a return value



From lkarsten at varnish-software.com  Thu Mar 13 09:24:29 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] d8f74bc Reindent with 4 spaces, 2 for continuations
Message-ID: <E1WO1sL-0007UM-HT@project.varnish-software.com>


commit d8f74bc538393bb8c67de3660c7126dc144487f2
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Feb 26 13:36:57 2014 +0100

    Reindent with 4 spaces, 2 for continuations

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 09c08cc..a6a126d 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -46,12 +46,12 @@ vcl 4.0;
 
 sub vcl_recv {
     if (req.restarts == 0) {
-	if (req.http.x-forwarded-for) {
-	    set req.http.X-Forwarded-For =
-		req.http.X-Forwarded-For + ", " + client.ip;
-	} else {
-	    set req.http.X-Forwarded-For = client.ip;
-	}
+        if (req.http.x-forwarded-for) {
+            set req.http.X-Forwarded-For =
+                req.http.X-Forwarded-For + ", " + client.ip;
+        } else {
+            set req.http.X-Forwarded-For = client.ip;
+        }
     }
     if (req.method != "GET" &&
       req.method != "HEAD" &&
@@ -104,13 +104,13 @@ sub vcl_purge {
 
 sub vcl_hit {
     if (obj.ttl >= 0s) {
-	// A pure unadultered hit, deliver it
-	return (deliver);
+        // A pure unadultered hit, deliver it
+        return (deliver);
     }
     if (obj.ttl + obj.grace > 0s) {
-	// Object is in grace, deliver it
-	// Automatically triggers a background fetch
-	return (deliver);
+        // Object is in grace, deliver it
+        // Automatically triggers a background fetch
+        return (deliver);
     }
     // fetch & deliver once we get the result
     return (fetch);
@@ -167,22 +167,22 @@ sub vcl_backend_fetch {
 
 sub vcl_backend_response {
     if (beresp.ttl <= 0s ||
-        beresp.http.Set-Cookie ||
-        beresp.http.Surrogate-control ~ "no-store" ||
-        (!beresp.http.Surrogate-Control &&
-          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
-        beresp.http.Vary == "*") {
-		/*
-		 * Mark as "Hit-For-Pass" for the next 2 minutes
-		 */
-		set beresp.ttl = 120s;
-		set beresp.uncacheable = true;
+      beresp.http.Set-Cookie ||
+      beresp.http.Surrogate-control ~ "no-store" ||
+      (!beresp.http.Surrogate-Control &&
+        beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
+      beresp.http.Vary == "*") {
+        /*
+        * Mark as "Hit-For-Pass" for the next 2 minutes
+        */
+        set beresp.ttl = 120s;
+        set beresp.uncacheable = true;
     }
     return (deliver);
 }
 
 sub vcl_backend_error {
-	return (deliver);
+    return (deliver);
 }
 
 #######################################################################



From phk at FreeBSD.org  Thu Mar 13 09:24:29 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] a95525a Remove an almost correct assert.
Message-ID: <E1WO1sL-0007UX-Ie@project.varnish-software.com>


commit a95525aff978039148712306134ad60d9b9c0b67
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 27 10:01:11 2014 +0000

    Remove an almost correct assert.
    
    The real condition is that either there is no busyobj or it is in
    state BOS_FINISHED, being rapidly dismantled.
    
    The locking of testing that would be to complex and expensive so we
    eliminate the assert instead, and strengthen another busyobj related
    check a little bit in compensation.
    
    Fixes #1440

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index d6b43ae..db1c7a1 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -244,9 +244,10 @@ VBO_waitlen(struct busyobj *bo, ssize_t l)
 void
 VBO_setstate(struct busyobj *bo, enum busyobj_state_e next)
 {
-	Lck_Lock(&bo->mtx);
-	VSLb(bo->vsl, SLT_Debug, "XXX BOS: %d -> %d", bo->state, next);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	assert(bo->do_stream || next != BOS_STREAM);
 	assert(next > bo->state);
+	Lck_Lock(&bo->mtx);
 	bo->state = next;
 	AZ(pthread_cond_broadcast(&bo->cond));
 	Lck_Unlock(&bo->mtx);
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 2c681d6..c9f373a 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -294,7 +294,6 @@ V1D_Deliver(struct req *req)
 	if (!req->wantbody) {
 		/* This was a HEAD or conditional request */
 	} else if (req->res_mode & RES_ESI) {
-		AZ(req->obj->objcore->busyobj);
 		ESI_Deliver(req);
 	} else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) {
 		while (req->obj->objcore->busyobj)



From lkarsten at varnish-software.com  Thu Mar 13 09:24:29 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] b3ff41f Explain default groupmode in short help.
Message-ID: <E1WO1sL-0007Ug-KK@project.varnish-software.com>


commit b3ff41f5da0c0a05683d9e5d59c2f619c78f55bb
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Feb 27 12:28:25 2014 +0100

    Explain default groupmode in short help.
    
    Main reason to duplicate is that it should be visible
    in "varnishlog --help". (and friends)

diff --git a/include/vut_options.h b/include/vut_options.h
index 1f19024..de3bab2 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -41,7 +41,7 @@
 	)
 
 #define VUT_OPT_g							\
-	VOPT("g:", "[-g <session|request|vxid|raw>]", "Grouping mode",	\
+	VOPT("g:", "[-g <session|request|vxid|raw>]", "Grouping mode (default: vxid)",	\
 	    "The grouping of the log records. The default is to group"	\
 	    " by vxid."							\
 	)



From fgsch at lodoss.net  Thu Mar 13 09:24:29 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 42cec19 Remove --without-rst2man
Message-ID: <E1WO1sL-0007Up-LP@project.varnish-software.com>


commit 42cec19d525f56297626831782643d776866e87a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Feb 27 11:29:06 2014 +0000

    Remove --without-rst2man
    
    This is mandatory now.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 451d131..e883314 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -98,9 +98,9 @@ export CFLAGS="$CFLAGS -Wp,-D_FORTIFY_SOURCE=0"
 # Remove "--disable static" if you want to build static libraries
 # jemalloc is not compatible with Red Hat's ppc64 RHEL kernel :-(
 %ifarch ppc64 ppc
-	%configure --disable-static --localstatedir=/var/lib --without-jemalloc  --without-rst2man --without-rst2html
+	%configure --disable-static --localstatedir=/var/lib --without-jemalloc --without-rst2html
 %else
-	%configure --disable-static --localstatedir=/var/lib --without-rst2man --without-rst2html
+	%configure --disable-static --localstatedir=/var/lib --without-rst2html
 %endif
 
 # We have to remove rpath - not allowed in Fedora



From lkarsten at varnish-software.com  Thu Mar 13 09:24:29 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 0eb0c12 Separate files forgotten in b3ff41f5
Message-ID: <E1WO1sL-0007Uz-My@project.varnish-software.com>


commit 0eb0c12a6620fc9e40521b89e5c3e29b2ec9354a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Feb 27 13:08:30 2014 +0100

    Separate files forgotten in b3ff41f5
    
    Try to be consistent in the help text across the different tools.

diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h
index 0a0e2b3..fdd217e 100644
--- a/bin/varnishhist/varnishhist_options.h
+++ b/bin/varnishhist/varnishhist_options.h
@@ -32,7 +32,7 @@
 #include "vut_options.h"
 
 #define HIS_OPT_g							\
-	VOPT("g:", "[-g <request|vxid>]", "Grouping mode",		\
+	VOPT("g:", "[-g <request|vxid>]", "Grouping mode (default: vxid)",		\
 	    "The grouping of the log records. The default is to group"	\
 	    " by vxid."							\
 	)
diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h
index 346b7fc..dc8ea30 100644
--- a/bin/varnishncsa/varnishncsa_options.h
+++ b/bin/varnishncsa/varnishncsa_options.h
@@ -41,7 +41,7 @@
 	)
 
 #define NCSA_OPT_g							\
-	VOPT("g:", "[-g <request|vxid>]", "Grouping mode",		\
+	VOPT("g:", "[-g <request|vxid>]", "Grouping mode (default: vxid)",		\
 	    "The grouping of the log records. The default is to group"	\
 	    " by vxid."							\
 	)



From fgsch at lodoss.net  Thu Mar 13 09:24:29 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] aa8c6e3 Move python-docutils to BuildRequires
Message-ID: <E1WO1sL-0007V9-OL@project.varnish-software.com>


commit aa8c6e3b7ed8eadc126520c4a596a5331856aa7c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Feb 27 12:12:10 2014 +0000

    Move python-docutils to BuildRequires
    
    Missed in 42cec19d

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index e883314..81ad6f1 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -14,9 +14,9 @@ Source0: %{name}-%{version}%{?vd_rc}.tar.gz
 #Source0: %{name}-trunk.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
-# You will need at least automake autoconf libtool python-docutils
-#BuildRequires: automake autoconf libtool python-docutils
-BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel
+# You will need at least automake autoconf libtool
+#BuildRequires: automake autoconf libtool
+BuildRequires: ncurses-devel groff pcre-devel pkgconfig python-docutils libedit-devel
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] f278f8b Warn on duplicate Begin log records
Message-ID: <E1WO1sL-0007VI-PT@project.varnish-software.com>


commit f278f8b2ac33b51d8a35b5ad26f58dd5bbfd7d5a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 27 12:35:50 2014 +0100

    Warn on duplicate Begin log records

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 3acdbd2..8a7f6e6 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -855,8 +855,12 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 
 		switch (tag) {
 		case SLT_Begin:
-			(void)vtx_scan_begin(vslq, vtx, ptr);
-			vtx->flags |= VTX_F_BEGIN;
+			if (vtx->flags & VTX_F_BEGIN)
+				(void)vtx_diag_tag(vtx, ptr, "duplicate begin");
+			else {
+				(void)vtx_scan_begin(vslq, vtx, ptr);
+				vtx->flags |= VTX_F_BEGIN;
+			}
 			break;
 
 		case SLT_Link:
@@ -864,15 +868,15 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 			break;
 
 		case SLT_End:
+			AZ(vtx->flags & VTX_F_END);
 			vtx->flags |= VTX_F_END;
+			vtx_mark_complete(vslq, vtx);
 			break;
+
 		default:
 			break;
 		}
 	}
-
-	if (vtx->flags & VTX_F_END && !(vtx->flags & VTX_F_COMPLETE))
-		vtx_mark_complete(vslq, vtx);
 }
 
 /* Force a vtx into complete status by synthing the necessary outstanding



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 2386b35 Allow link to parent on ESI subrequests and in
 -grequest mode
Message-ID: <E1WO1sL-0007VR-Qn@project.varnish-software.com>


commit 2386b3512523282ce3a42b2bad806c718897015a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 27 15:18:31 2014 +0100

    Allow link to parent on ESI subrequests and in -grequest mode

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 8a7f6e6..bf4bcd0 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -744,7 +744,8 @@ vtx_scan_begin(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 
 	if (vslq->grouping == VSL_g_vxid)
 		return (0);	/* No links */
-	if (vslq->grouping == VSL_g_request && vtx->type == VSL_t_req)
+	if (vslq->grouping == VSL_g_request && vtx->type == VSL_t_req &&
+	    vtx->reason == VSL_r_rxreq)
 		return (0);	/* No links */
 
 	if (vtx->parent != NULL) {



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] df1cdb8 Avoid duplicate Begin/End records on ESI subrequests
Message-ID: <E1WO1sL-0007Vc-Ru@project.varnish-software.com>


commit df1cdb82cb1c103c1ae610fc6e6ebb4e75de054f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 27 15:34:38 2014 +0100

    Avoid duplicate Begin/End records on ESI subrequests
    
    When starting an ESI subrequest, make sure that the Begin/End headers
    are only output once, formatted for ESI.
    
    Fixes: #1441

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index e76b437..65ad51d 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -61,6 +61,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 
 	req = SES_GetReq(wrk, preq->sp);
 	req->req_body_status = REQ_BODY_NONE;
+	AN(req->vsl->wid & VSL_CLIENTMARKER);
 	VSLb(req->vsl, SLT_Begin, "req %u esi", preq->vsl->wid & VSL_IDENTMASK);
 	VSLb(preq->vsl, SLT_Link, "req %u esi", req->vsl->wid & VSL_IDENTMASK);
 	req->esi_level = preq->esi_level + 1;
@@ -115,8 +116,8 @@ ved_include(struct req *preq, const char *src, const char *host)
 		(void)usleep(10000);
 	}
 
-	VSLb(req->vsl, SLT_End, "%s", "");
-	req->vsl->wid = 0;
+	/* Make sure the VSL id has been released */
+	AZ(req->vsl->wid);
 
 	/* Reset the workspace */
 	WS_Reset(wrk->aws, wrk_ws_wm);	/* XXX ? */
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 83ed4cd..96a2507 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -159,6 +159,9 @@ ses_sess_pool_task(struct worker *wrk, void *arg)
 
 	req = SES_GetReq(wrk, sp);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AN(req->vsl->wid & VSL_CLIENTMARKER);
+	VSLb(req->vsl, SLT_Begin, "req %u rxreq", sp->vxid & VSL_IDENTMASK);
+	VSL(SLT_Link, sp->vxid, "req %u rxreq", req->vsl->wid & VSL_IDENTMASK);
 
 	sp->sess_step = S_STP_NEWREQ;
 	ses_req_pool_task(wrk, req);
@@ -389,9 +392,6 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 	sz = cache_param->vsl_buffer;
 	VSL_Setup(req->vsl, p, sz);
 	req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER;
-	VSLb(req->vsl, SLT_Begin, "req %u rxreq", sp->vxid & VSL_IDENTMASK);
-	VSL(SLT_Link, req->sp->vxid, "req %u rxreq",
-	    req->vsl->wid & VSL_IDENTMASK);
 	p += sz;
 	p = (void*)PRNDUP(p);
 
diff --git a/bin/varnishtest/tests/r01441.vtc b/bin/varnishtest/tests/r01441.vtc
new file mode 100644
index 0000000..ec9553d
--- /dev/null
+++ b/bin/varnishtest/tests/r01441.vtc
@@ -0,0 +1,40 @@
+varnishtest "Session grouping on ESI"
+
+server s1 {
+	rxreq
+	expect req.url == "/"
+	txresp -body "<esi:include src=\"/include\"/>"
+
+	rxreq
+	expect req.url == "/include"
+	txresp -body "included"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		if (bereq.url == "/") {
+			set beresp.do_esi = true;
+		}
+	}
+} -start
+
+logexpect l1 -v v1 -g session {
+	expect 0 1000	Begin sess 0 HTTP/1
+	expect * =	End
+	expect 0 1001	Begin req 1000 rxreq
+	expect * =	End
+	expect 0 1002	Begin bereq 1001 fetch
+	expect * =	End
+	expect 0 1003	Begin req 1001 esi
+	expect * =	End
+	expect 0 1004	Begin bereq 1003 fetch
+	expect * =	End
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+} -run
+
+logexpect l1 -wait



From fgsch at lodoss.net  Thu Mar 13 09:24:29 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 9dbe0ec Rename =build to build
Message-ID: <E1WO1sL-0007Vl-Sv@project.varnish-software.com>


commit 9dbe0ec2bcf6a3fa2d48f40812b13a38fbb92cc3
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Feb 27 18:22:55 2014 +0000

    Rename =build to build
    
    Forgotten in 339b68f9

diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index 18197a6..fea357f 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -5,7 +5,7 @@
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
-BUILDDIR      = =build
+BUILDDIR      = build
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index d825d6b..f68a20f 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_patterns = ['=build','reference/params.rst']
+exclude_patterns = ['build','reference/params.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 81ad6f1..6f35ecc 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -118,9 +118,9 @@ make %{?_smp_mflags} V=1
 	redhat/varnish.initrc redhat/varnishlog.initrc redhat/varnishncsa.initrc
 %endif
 
-rm -rf doc/sphinx/\=build/html/_sources
-mv doc/sphinx/\=build/html doc
-rm -rf doc/sphinx/\=build
+rm -rf doc/sphinx/build/html/_sources
+mv doc/sphinx/build/html doc
+rm -rf doc/sphinx/build
 
 %check
 # rhel5 on ppc64 is just too strange



From phk at FreeBSD.org  Thu Mar 13 09:24:29 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 51ab3f2 Collapse two fetch states and make the X-Varnish
 heading for the backend available in vcl_backend_fetch{}
Message-ID: <E1WO1sL-0007Vu-Tv@project.varnish-software.com>


commit 51ab3f207c2b6a3c3ac1c3411be232730801db39
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 3 09:03:51 2014 +0000

    Collapse two fetch states and make the X-Varnish heading for the backend
    available in vcl_backend_fetch{}

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0c621df..c52ba05 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -265,6 +265,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 {
+	int i, do_ims;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -277,12 +278,12 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
 	HTTP_Copy(bo->bereq, bo->bereq0);
 
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
+
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
 	bo->uncacheable = bo->do_pass;
-
-	http_PrintfHeader(bo->bereq,
-	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
 	if (wrk->handling == VCL_RET_ABANDON) {
 		if (bo->req != NULL)
 			vbf_release_req(bo);
@@ -291,22 +292,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_DONE);
 	}
 	assert (wrk->handling == VCL_RET_FETCH);
-	return (F_STP_FETCHHDR);
-}
-
-
-/*--------------------------------------------------------------------
- */
-
-static enum fetch_step
-vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
-{
-	int i, do_ims;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	xxxassert (wrk->handling == VCL_RET_FETCH);
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 9ef7f6c..47da3fc 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -52,7 +52,6 @@ REQ_STEP(error,		ERROR,		(wrk, req))
 FETCH_STEP(mkbereq,	MKBEREQ,	(wrk, bo))
 FETCH_STEP(retry,	RETRY,		(wrk, bo))
 FETCH_STEP(startfetch,	STARTFETCH,	(wrk, bo))
-FETCH_STEP(fetchhdr,	FETCHHDR,	(wrk, bo))
 FETCH_STEP(condfetch,	CONDFETCH,	(wrk, bo))
 FETCH_STEP(fetch,	FETCH,		(wrk, bo))
 FETCH_STEP(error,	ERROR,		(wrk, bo))



From phk at FreeBSD.org  Thu Mar 13 09:24:29 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 2edbe43 Always release the req early if non-pass fetch,
 don't release it if it is pass, we may beed the body for retries.
Message-ID: <E1WO1sL-0007W3-V6@project.varnish-software.com>


commit 2edbe43feda671dd432152e82e4cc376e1b79cb4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 3 09:30:47 2014 +0000

    Always release the req early if non-pass fetch, don't release it if
    it is pass, we may beed the body for retries.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c52ba05..8672375 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -35,26 +35,10 @@
 #include <stdlib.h>
 
 #include "cache.h"
-
 #include "hash/hash_slinger.h"
-
 #include "vcl.h"
 #include "vtim.h"
 
-
-/*--------------------------------------------------------------------
- */
-
-static void
-vbf_release_req(struct busyobj *bo)
-{
-	assert(bo->state == BOS_INVALID);
-	AN(bo->req);
-	bo->req = NULL;
-	http_CopyHome(bo->bereq);
-	VBO_setstate(bo, BOS_REQ_DONE);
-}
-
 /*--------------------------------------------------------------------
  * Allocate an object, with fall-back to Transient.
  * XXX: This somewhat overlaps the stuff in stevedore.c
@@ -187,7 +171,7 @@ vbf_beresp2obj(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
- * Copy req->bereq
+ * Copy req->bereq and release req if not pass fetch
  */
 
 static enum fetch_step
@@ -199,6 +183,7 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
 
+	assert(bo->state == BOS_INVALID);
 	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
@@ -207,18 +192,16 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod);
 	http_FilterReq(bo->bereq0, bo->req->http,
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
+
 	if (!bo->do_pass) {
-		// XXX: Forcing GET should happen in vcl_miss{} ?
 		http_ForceGet(bo->bereq0);
-		if (cache_param->http_gzip_support) {
-			/*
-			 * We always ask the backend for gzip, even if the
-			 * client doesn't grok it.  We will uncompress for
-			 * the minority of clients which don't.
-			 */
+		if (cache_param->http_gzip_support)
 			http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
-		}
+		AN(bo->req);
+		bo->req = NULL;
+		http_CopyHome(bo->bereq0);
 	}
+
 	if (bo->ims_obj != NULL) {
 		if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
 			http_PrintfHeader(bo->bereq0,
@@ -231,6 +214,7 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 		}
 	}
 
+	VBO_setstate(bo, BOS_REQ_DONE);
 	return (F_STP_STARTFETCH);
 }
 
@@ -275,6 +259,11 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
 
+	if (bo->do_pass)
+		AN(bo->req);
+	else
+		AZ(bo->req);
+
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
 	HTTP_Copy(bo->bereq, bo->bereq0);
 
@@ -285,8 +274,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	bo->uncacheable = bo->do_pass;
 	if (wrk->handling == VCL_RET_ABANDON) {
-		if (bo->req != NULL)
-			vbf_release_req(bo);
 		HSH_Fail(bo->fetch_objcore);
 		VBO_setstate(bo, BOS_FAILED);
 		return (F_STP_DONE);
@@ -295,9 +282,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 
-	if (!bo->do_pass && bo->req != NULL)
-		vbf_release_req(bo); /* XXX: retry ?? */
-
 	assert(bo->state <= BOS_REQ_DONE);
 
 	i = V1F_fetch_hdr(wrk, bo, bo->req);
@@ -316,11 +300,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_ERROR);
 	}
 
-	if (bo->do_pass && bo->req != NULL)
-		vbf_release_req(bo); /* XXX : retry ?? */
-
-	AZ(bo->req);
-
 	AN(bo->vbc);
 	http_VSL_log(bo->beresp);
 
@@ -690,9 +669,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	xxxassert(wrk->handling == VCL_RET_DELIVER);
 
-	if (bo->req != NULL)
-		vbf_release_req(bo);
-
 	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
 	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
 



From martin at varnish-software.com  Thu Mar 13 09:24:29 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 13 Mar 2014 10:24:29 +0100
Subject: [4.0] 2f05fe4 Serve ESI-includes from a different backend.
Message-ID: <E1WO1sM-0007WF-09@project.varnish-software.com>


commit 2f05fe44f072373bd11da06b01586110412fc15c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 3 12:51:22 2014 +0100

    Serve ESI-includes from a different backend.
    
    Serve ESI-includes from a different backend so that we don't race
    against when the backend connection is ready for reuse.
    
    Fixes: #1440

diff --git a/bin/varnishtest/tests/e00019.vtc b/bin/varnishtest/tests/e00019.vtc
index 1aeaa06..1010eab 100644
--- a/bin/varnishtest/tests/e00019.vtc
+++ b/bin/varnishtest/tests/e00019.vtc
@@ -22,13 +22,25 @@ server s1 {
 	chunked {<esi:comment/>}
 
 	chunkedlen 0
+} -start
 
+# The included object gets served from a different backend.
+# This is to avoid a race between when a backend connection
+# gets put up for reuse because of background fetches in
+# Varnish 4
+server s2 {
 	rxreq
 	expect req.url == "bar/foo"
 	txresp -body {<INCL>}
 } -start
 
 varnish v1 -vcl+backend {
+	sub vcl_backend_fetch {
+		if (bereq.url != "bar") {
+			set bereq.backend = s2;
+		}
+	}
+
 	sub vcl_backend_response {
 		if (bereq.url == "bar") {
 			set beresp.do_esi = true;



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] f10e0a5 Regenerated from inline help. This should be automatic
 at some point. Also, the inline help might be a bit more verbose.
Message-ID: <E1WO1sM-0007WO-1S@project.varnish-software.com>


commit f10e0a58125c8a7251e868bf4a37b2b1ca2fbd50
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Mar 3 10:43:18 2014 +0100

    Regenerated from inline help. This should be automatic at some point.
    Also, the inline help might be a bit more verbose.

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index f967465..2e17c2e 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -71,112 +71,74 @@ be entered with the \\xnn syntax.
 Commands
 --------
 
-backend.list
-      Lists the defined backends including health state.
-
-backend.set_health matcher state
-      Sets the health state on a specific backend, overriding the state
-      determined by a probe.  This is useful if you want to take a
-      certain backend out of circulation.
-
-      *state* can be 'auto', 'sick' or 'healthy'.
-
-ban   *field operator argument* [&& field operator argument [...]]
-      Immediately invalidate all documents matching the ban
-      expression.  See *Ban Expressions* for more documentation and
-      examples.
-
-ban.list
-      All requests for objects from the cache are matched against
-      items on the ban list.  If an object in the cache is older than
-      a matching ban list item, it is considered "banned", and will be
-      fetched from the backend instead.
-
-      When a ban expression is older than all the objects in the
-      cache, it is removed from the list.
+help [command]
+  Show command/protocol help
 
-      ban.list displays the ban list. The output looks something like
-      this::
+ping [timestamp]
+  Keep connection alive
 
-        0x7fea4fcb0580 1303835108.618863   131G   req.url ~ /some/url
+auth response
+  Authenticate.
 
-      The first field is the address of the ban.
+quit
+  Close connection
 
-      The second is the time of entry into the list, given
-      as a high precision timestamp.
+banner
+  Print welcome banner.
 
-      The third field describes many objects point to this ban. When
-      an object is compared to a ban the object is marked with a
-      reference to the newest ban it was tested against. This isn't
-      really useful unless you're debugging.
+status
+  Check status of Varnish cache process.
 
-      A "G" marks that the ban is "Gone". Meaning it has been marked
-      as a duplicate or it is no longer valid. It stays in the list
-      for effiency reasons.
+start
+  Start the Varnish cache process.
 
-      Then follows the actual ban it self.
+stop
+  Stop the Varnish cache process
 
-help [command]
-      Display a list of available commands.
-      If the command is specified, display help for this command.
+vcl.load <configname> <filename>
+  Compile and load the VCL file under the name provided.
 
-param.set param value
-      Set the parameter specified by param to the specified value.
-      See Run-Time Parameters for a list of parameters.
+vcl.inline <configname> <quoted_VCLstring>
+  Compile and load the VCL data under the name provided.
 
-param.show [-l] [param]
-      Display a list if run-time parameters and their values.
+vcl.use <configname>
+  Switch to the named configuration immediately.
 
-      If the -l option is specified, the list includes a brief
-      explanation of each parameter.
+vcl.discard <configname>
+  Unload the named configuration (when possible).
 
-      If a param is specified, display only the value and explanation
-      for this parameter.
+vcl.list
+  List all loaded configuration.
 
-ping  [timestamp]
-      Ping the Varnish cache process, keeping the connection alive.
+vcl.show <configname>
+  Display the source code for the specified configuration.
 
-quit
-      Close the connection to the Varnish admin port.
+param.show [-l] [<param>]
+  Show parameters and their values.
 
-start
-      Start the Varnish cache process if it is not already running.
+param.set <param> <value>
+  Set parameter value.
 
-status
-      Check the status of the Varnish cache process.
+panic.show
+  Return the last panic, if any.
 
-stop
-      Stop the Varnish cache process.
+panic.clear
+  Clear the last panic, if any.
 
 storage.list
-      Lists the defined storage backends.
-
-vcl.discard configname
-      Discard the configuration specified by configname.  This will
-      have no effect if the specified configuration has a non-zero
-      reference count.
+  List storage devices
 
-vcl.inline configname vcl
-      Create a new configuration named configname with the VCL code
-      specified by vcl, which must be a quoted string.
-
-vcl.list
-      List available configurations and their respective reference
-      counts.  The active configuration is indicated with an asterisk
-      ("*").
-
-vcl.load configname filename
-      Create a new configuration named configname with the contents of
-      the specified file.
+backend.list
+  List all backends
 
-vcl.show configname
-      Display the source code for the specified configuration.
+backend.set_health matcher state
+  Set health status on a backend
 
-vcl.use configname
-      Start using the configuration specified by configname for all
-      new requests.  Existing requests will continue using whichever
-      configuration was in use when they arrived.
+ban <field> <operator> <arg> [&& <field> <oper> <arg>]...
+  All objects where the all the conditions match will be marked obsolete.
 
+ban.list
+  List the active bans.
 
 
 Ban Expressions



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 805fa87 First stab at a mostly syntax-oriented VCL manpage.
 Feedback is welcome.
Message-ID: <E1WO1sM-0007Wb-2q@project.varnish-software.com>


commit 805fa87a555972f724a3672439f72220c17f5429
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Mar 3 13:28:49 2014 +0100

    First stab at a mostly syntax-oriented VCL manpage.
    Feedback is welcome.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 57072d0..96a4aab 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -19,273 +19,232 @@ When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
 is then dynamically linked into the server process.
 
-For an overview of the VCL syntax please see the users guide.
+This document focuses on the syntax of the VCL language. Full a full
+description of syntax and semantics, with ample examples, please see
+the users guide at https://www.varnish-cache.org/doc/
+
+VCL consists of the following elements:
+ * Operators
+ * Conditionals
+ * Strings, booleans, time, duration, ints
+ * Regular expressions
+
+In addition VCL has the following contructs:
+ * Include
+ * Backend definitions
+ * Probes
+ * Access control lists - ACLs
+ * Import statement
+ * Functions
+ * Subroutines
+
+Note that are no loops or iterators of any kind in VCL.
+
+Operators
+---------
 
+The following operators are available in VCL:
 
-Backend declarations
---------------------
+  =
+    Assignment operator.
 
-A backend declaration creates and initializes a named backend object::
+  ==
+    Comparison.
 
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-    }
+  ~
+    Match. Can either be used with regular expressions or ACLs.
 
-The backend object can later be used to select a backend at request time::
+  !
+    Negation.
 
-    if (req.http.host ~ "(?i)^(www.)?example.com$") {
-        set req.backend_hint = www;
-    }
+  &&
+    Logical and
 
-To avoid overloading backend servers, .max_connections can be set to
-limit the maximum number of concurrent backend connections.
+  ||
+    Logical or
 
-The timeout parameters can be overridden in the backend declaration.
-The timeout parameters are .connect_timeout for the time to wait for a
-backend connection, .first_byte_timeout for the time to wait for the
-first byte from the backend and .between_bytes_timeout for time to
-wait between each received byte.
 
-These can be set in the declaration like this::
+Conditionals
+------------
 
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-        .connect_timeout = 1s;
-        .first_byte_timeout = 5s;
-        .between_bytes_timeout = 2s;
-    }
+VCL has *if* statments.
 
-.. _ref-vcl-director:
 
-Directors
----------
+Strings, booleans, time, duration and ints
+------------------------------------------
 
-A director is a logical group of backend servers clustered together
-for redundancy. The basic role of the director is to let Varnish
-choose a backend server amongst several so if one is down another can
-be used.
+These are the data types in Varnish. You can *set* or *unset* these.
 
-There are several types of directors. The different director types
-use different algorithms to choose which backend to use.
+Example::
 
-Configuring a director may look like this::
+  set req.http.user-agent = "unknown";
 
-    director b2 random {
-        .retries = 5;
-        {
-            # We can refer to named backends
-            .backend = b1;
-            .weight = 7;
-        }
-        {
-            # Or define them inline
-            .backend  = {
-                .host = "fs2";
-            }
-            .weight = 3;
-        }
-    }
 
-The family of random directors
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Strings
+~~~~~~~
 
-There are three directors that share the same logic, called the random
-director, client director and hash director. They each distribute traffic
-among the backends assigned to it using a random distribution seeded with
-either the client identity, a random number or the cache hash (typically
-url). Beyond the initial seed, they act the same.
+Basic strings are enclosed in " ... ", and may not contain
+newlines. Long strings are enclosed in {" ... "}. They may contain any
+character including ", newline and other control characters except for
+the NUL (0x00) character
 
-Each backend requires a .weight option which sets the amount of traffic
-each backend will get compared to the others. Equal weight means equal
-traffic. A backend with lower weight than an other will get proportionally
-less traffic.
+Booleans
+~~~~~~~~
 
-The director has an optional .retries option which defaults to the number
-of backends the director has. The director will attempt .retries times to
-find a healthy backend if the first attempt fails. Each attempt re-uses the
-previous seed in an iterative manner. For the random director this detail
-is of no importance as it will give different results each time. For the
-hash and client director, this means the same URL or the same client will
-fail to the same server consistently.
+Booleans can be either true or false.
 
-The random director
-...................
+Time
+----
 
-This uses a random number to seed the backend selection.
+VCL has time. The function *now* returns a time. A duration can be
+added to a time to make another time. In string context they return a
+formatted string.
 
-The client director
-...................
+Durations
+---------
 
-The client director picks a backend based on the clients
-*identity*. You can set the VCL variable *client.identity* to identify
-the client by picking up the value of a session cookie or similar.
+Durations are defined by a number and a designation. The number can be a real so 1.5w is allowed.
 
-The hash director
-.................
+  s
+    seconds
 
-The hash director will pick a backend based on the URL hash
-value.
+  m
+    minutes
 
-This is useful is you are using Varnish to load balance in front of
-other Varnish caches or other web accelerators as objects won't be
-duplicated across caches.
+  h
+    hours
 
-It will use the value of req.hash, just as the normal cache lookup methods.
+  d
+    days
 
+  w
+    weeks
 
-The round-robin director
-~~~~~~~~~~~~~~~~~~~~~~~~
+  y
+    years.
 
-The round-robin director does not take any options.
 
-It will use the first backend for the first request, the second backend for
-the second request and so on, and start from the top again when it gets to
-the end.
+Ints
+----
 
-If a backend is unhealthy or Varnish fails to connect, it will be skipped.
-The round-robin director will try all the backends once before giving up.
+Certain fields are integers, used as expected. In string context they
+return a string.
 
-The DNS director
-~~~~~~~~~~~~~~~~
 
-The DNS director can use backends in two different ways. Either like the
-random or round-robin director or using .list::
+Regular Expressions
+-------------------
 
-  director directorname dns {
-          .list = {
-                  .host_header = "www.example.com";
-                  .port = "80";
-                  .connect_timeout = 0.4s;
-                  "192.168.15.0"/24;
-                  "192.168.16.128"/25;
-          }
-          .ttl = 5m;
-          .suffix = "internal.example.net";
-  }
+Varnish uses PCRE - Perl-compatible regular expressions. For a
+complete description of PCRE please see the pcre(3) man page.
 
-This will specify 384 backends, all using port 80 and a connection timeout
-of 0.4s. Options must come before the list of IPs in the .list statement.
-The .list-method does not support IPv6. It is not a white-list, it is an
-actual list of backends that will be created internally in Varnish - the
-larger subnet the more overhead.
+To send flags to the PCRE engine, such as to turn on *case insensitivity* 
+add the flag within parens following a question mark,
+like this::
 
-The .ttl defines the cache duration of the DNS lookups.
+    # If host is NOT example dot com..
+    if (req.http.host !~ "(?i)example.com$") {
+        ...
+    }
 
-The above example will append "internal.example.net" to the incoming Host
-header supplied by the client, before looking it up. All settings are
-optional.
 
-Health checks are not thoroughly supported.
+Include statement
+-----------------
 
-DNS round robin balancing is supported. If a hostname resolves to multiple
-backends, the director will divide the traffic between all of them in a
-round-robin manner.
+To include a VCL file in another file use the include keyword::
 
-The fallback director
-~~~~~~~~~~~~~~~~~~~~~
+  include "foo.vcl";
 
-The fallback director will pick the first backend that is healthy. It
-considers them in the order in which they are listed in its definition.
 
-The fallback director does not take any options.
+Backend definition
+------------------
 
-An example of a fallback director::
+A backend declaration creates and initializes a named backend
+object. A declaration start with the keyword *backend* followed by the
+name of the backend. The actual declaration is in curly brackets, in a
+key/value fashion.::
 
-    director b3 fallback {
-        { .backend = www1; }
-        { .backend = www2; } # will only be used if www1 is unhealthy.
-        { .backend = www3; } # will only be used if both www1 and www2
-                             # are unhealthy.
+    backend name {
+        .attribute = "value";
     }
 
-Backend probes
---------------
-
-Backends can be probed to see whether they should be considered
-healthy or not.  The return status can also be checked by using the
-healthy function from the std vmod.
-
-Probes take the following parameters:
-
-.url
-  Specify a URL to request from the backend.
-  Defaults to "/".
-.request
-  Specify a full HTTP request using multiple strings. .request will
-  have \\r\\n automatically inserted after every string.
-  If specified, .request will take precedence over .url.
-.window
-  How many of the latest polls we examine to determine backend health.
-  Defaults to 8.
-.threshold
-  How many of the polls in .window must have succeeded for us to
-  consider the backend healthy. If this is set to more than or equal
-  to the threshold, the backend starts as healthy. Defaults to the
-  value of threshold - 1. In this case, the backend starts as sick and
-  requires one poll to pass to become healthy.
-  Defaults to threshold - 1.
-.initial
-  How many of the polls in .window are considered good when Varnish
-  starts. Defaults to the value of threshold - 1. In this case, the
-  backend starts as sick and requires one single poll to be considered
-  healthy.
-.expected_response
-  The expected backend HTTP response code.
-  Defaults to 200.
-.interval
-  Defines how often the probe should check the backend.
-  Default is every 5 seconds.
-.timeout
-  How fast each probe times out.
-  Default is 2 seconds.
-
-A backend with a probe can be defined like this, together with the
-backend or director::
-
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-        .probe = {
-            .url = "/test.jpg";
-            .timeout = 0.3s;
-            .window = 8;
-            .threshold = 3;
-            .initial = 3;
-        }
-    }
+The only mandatory attribute is host. The attributes will inherit
+their defaults from the global parameters. The following attributes
+are availble:
 
-Or it can be defined separately and then referenced::
+  host (mandatory)
+    The host to be used. IP address or a hostname that resolves to a
+    single IP address.
 
-    probe healthcheck {
-        .url = "/status.cgi";
-        .interval = 60s;
-        .timeout = 0.3s;
-        .window = 8;
-        .threshold = 3;
-        .initial = 3;
-        .expected_response = 200;
-    }
+  port
+    The port on the backend that Varnish should connect to.
 
-    backend www {
-        .host = "www.example.com";
-        .port = "http";
-        .probe = healthcheck;
-    }
+  host_header
+    A host header to add.
+
+  connect_timeout
+    Timeout for connections.
+
+  first_byte_timeout
+    Timeout for first byte.
+
+  between_bytes_timeout
+    Timeout between bytes.
 
-If you have many backends this can simplify the config a lot.
+  probe
+    Attach a probe to the backend. See Probes.
 
-It is also possible to specify the raw HTTP request::
+  max_connections
+    Maximum number of open connections towards this backend. If
+    Varnish reaches the maximum Varnish it will start failing
+    connections.
 
-    probe rawprobe {
-        # NB: \r\n automatically inserted after each string!
-        .request =
-          "GET / HTTP/1.1"
-          "Host: www.foo.bar"
-          "Connection: close";
+Probes
+------
+
+Probes will query the backend for status on a regular basis and mark
+the backend as down it they fail. A probe is defined as this:::
+
+    probe name {
+         .attribute = "value";
     }
 
+There are no mandatory options. These are the options you can set:
+
+  url
+    The URL to query. Defaults to "/".
+
+  request
+    Specify a full HTTP request using multiple strings. .request will
+    have \r\n automatically inserted after every string. If specified,
+    .request will take precedence over .url.
+
+  expected_response
+    The expected HTTP response code. Defaults to 200.
+            
+  timeout
+    The timeout for the probe. Default it 2s.
+
+  interval
+    How often the probe is run. Default is 5s.
+
+  initial
+    How many of the polls in .window are considered good when Varnish
+    starts. Defaults to the value of threshold - 1. In this case, the
+    backend starts as sick and requires one single poll to be
+    conqsidered healthy.
+            
+  window
+    How many of the latest polls we examine to determine backend health. Defaults to 8.
+            
+  threshold
+    How many of the polls in .window must have succeeded for us to
+    consider the backend healthy. If this is set to more than or equal
+    to the threshold, the backend starts as healthy. Defaults to the
+    value of threshold - 1. In this case, the backend starts as sick
+    and requires one poll to pass to become healthy. Defaults to
+    threshold - 1.
+
+
 ACLs
 ----
 
@@ -310,45 +269,9 @@ To match an IP address against an ACL, simply use the match operator::
         return (pipe);
     }
 
-Regular Expressions
--------------------
-
-Varnish uses PCRE - Perl-compatible regular expressions. For a
-complete description of PCRE please see the pcre(3) man page.
-
-To send flags to the PCRE engine, such as to turn on *case
-insensitivity* add the flag within parens following a question mark,
-like this::
-
-    # If host is NOT example dot com..
-    if (req.http.host !~ "(?i)example.com$") {
-        ...
-    }
-
-Functions
----------
-
-The following built-in functions are available:
-
-ban(expression)
-  Bans all objects in cache that match the expression.
-
-hash_data(str)
-  Adds a string to the hash input. In the built-in VCL hash_data()
-  is called on the host and URL of the *request*.
-
-regsub(str, regex, sub)
-  Returns a copy of str with the first occurrence of the regular
-  expression regex replaced with sub. Within sub, \\0 (which can
-  also be spelled \\&) is replaced with the entire matched string,
-  and \\n is replaced with the contents of subgroup n in the
-  matched string.
-
-regsuball(str, regex, sub)
-  As regsub() but this replaces all occurrences.
 
 Subroutines
-~~~~~~~~~~~
+-----------
 
 A subroutine is used to group code for legibility or reusability::
 
@@ -358,622 +281,71 @@ A subroutine is used to group code for legibility or reusability::
         }
     }
 
-Subroutines in VCL do not take arguments, nor do they return values.
+Subroutines in VCL do not take arguments, nor do they return
+values. The built in subroutines all have names beginning with vcl_,
+which is reserved.
 
 To call a subroutine, use the call keyword followed by the subroutine's name::
 
     call pipe_if_local;
 
-There are a number of special subroutines which hook into the Varnish
-workflow.  These subroutines may inspect and manipulate HTTP headers
-and various other aspects of each request, and to a certain extent
-decide how the request should be handled.  Each subroutine terminates
-by calling one of a small number of keywords which indicates the
-desired outcome.
-
-vcl_init
-  Called when VCL is loaded, before any requests pass through it.
-  Typically used to initialize VMODs.
-
-  return() values:
-
-  ok
-    Normal return, VCL continues loading.
-
-vcl_recv
-  Called at the beginning of a request, after the complete request has
-  been received and parsed.  Its purpose is to decide whether or not
-  to serve the request, how to do it, and, if applicable, which backend
-  to use.
-
-  The vcl_recv subroutine may terminate with calling return() on one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  hash
-    Look up the requested object in the cache.  Control will
-    eventually pass to vcl_hit or vcl_miss, depending on whether the
-    object is in the cache.  The ``bereq.method`` value will be set
-    to ``GET`` regardless of the value of ``req.method``.
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  pipe
-    Switch to pipe mode.  Control will eventually pass to vcl_pipe.
-
-  purge
-    ?
-
-vcl_hash
-  You may call hash_data() on the data you would like to add to the hash.
-
-  The vcl_hash subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  lookup
-    Proceed.
-
-vcl_pass
-  Called upon entering pass mode.  In this mode, the request is passed
-  on to the backend, and the backend's response is passed on to the
-  client, but is not entered into the cache.  Subsequent requests
-  submitted over the same client connection are handled normally.
-
-  The vcl_pass subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  fetch
-    Proceed with pass mode.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_pipe
-  Called upon entering pipe mode.  In this mode, the request is passed
-  on to the backend, and any further data from either client or
-  backend is passed on unaltered until either end closes the
-  connection.
-
-  The vcl_pipe subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  pipe
-    Proceed with pipe mode.
-
-vcl_purge
-  ?
-
-  The vcl_purge subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-..  fetch
-..    Not implemented.
-
-vcl_hit
-  Called after a cache lookup if the requested document was found in the cache.
-
-  The vcl_hit subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  deliver
-    Deliver the cached object to the client.  Control will eventually
-    pass to vcl_deliver.
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
+Return statements
+~~~~~~~~~~~~~~~~~
 
-  fetch
-    ?
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_miss
-  Called after a cache lookup if the requested document was not found
-  in the cache.  Its purpose is to decide whether or not to attempt to
-  retrieve the document from the backend, and which backend to use.
-
-  The vcl_miss subroutine may terminate with calling return() with one of
-  the following keywords:
-
-  error(code [, reason])
-    Return the specified error code to the client and abandon the request.
-
-  fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_backend_fetch.
-
-  pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
-
-  restart
-    ?
-
-vcl_backend_fetch
-  ?
-
-  The vcl_backend_fetch subroutine may terminate with calling return()
-  with one of the following keywords:
-
-  abandon
-    ?
-
-  fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_backend_response.
-
-vcl_backend_response
-  Called after a document has been successfully retrieved from the backend.
-
-  The vcl_backend_response subroutine may terminate with calling return()
-  with one of the following keywords:
-
-  abandon
-    ?
-
-  deliver
-    Possibly insert the object into the cache, then deliver it to the
-    client.  Control will eventually pass to vcl_deliver.
-
-  retry
-    ?
-
-vcl_deliver
-  Called before a cached object is delivered to the client.
-
-  The vcl_deliver subroutine may terminate with one of the following
-  keywords:
-
-  deliver
-    Deliver the object to the client.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_error
-  Called when we hit an error, either explicitly or implicitly due to
-  backend or internal errors.
-
-  The vcl_error subroutine may terminate by calling return with one of
-  the following keywords:
-
-  deliver
-    Deliver the error object to the client.
-
-  restart
-    Restart the transaction. Increases the restart counter. If the number
-    of restarts is higher than *max_restarts* Varnish emits a guru meditation
-    error.
-
-vcl_fini
-  Called when VCL is discarded only after all requests have exited the VCL.
-  Typically used to clean up VMODs.
-
-  return() values:
-
-  ok
-    Normal return, VCL will be discarded.
-
-If one of these subroutines is left undefined or terminates without
-reaching a handling decision, control will be handed over to the
-builtin default.  See the EXAMPLES section for a listing of the
-default code.
+The subroutine executions ends when a return(*action*) statement is
+made. The *action* specifies how execution should proceed. The context
+defines which actions are availble. See the user guide for information
+on what actions are available where.
 
 Multiple subroutines
 ~~~~~~~~~~~~~~~~~~~~
+
 If multiple subroutines with the name of one of the builtin
 ones are defined, they are concatenated in the order in which they
 appear in the source.
-The default versions distributed with Varnish will be implicitly
-concatenated as a last resort at the end.
 
-Example::
+The default versions distributed with Varnish will be implicitly
+concatenated.
 
-    # in file "main.vcl"
-    include "backends.vcl";
-    include "ban.vcl";
-
-    # in file "backends.vcl"
-    sub vcl_recv {
-        if (req.http.host ~ "(?i)example.com") {
-            set req.backend_hint = foo;
-	} elsif (req.http.host ~ "(?i)example.org") {
-	    set req.backend_hint = bar;
-	}
-    }
 
-    # in file "ban.vcl"
-    sub vcl_recv {
-        if (client.ip ~ admin_network) {
-	    if (req.http.Cache-Control ~ "no-cache") {
-                ban("req.url ~ " + req.url);
-            }
-        }
-    }
 
 Variables
-~~~~~~~~~
-
-Although subroutines take no arguments, the necessary information is
-made available to the handler subroutines through global variables.
-
-The following variables are always available:
-
-now
-  The current time, in seconds since the epoch. When used in string context
-  it returns a formatted string.
-
-The following variables are available in backend declarations:
-
-.host
-  Host name or IP address of a backend.
-
-.port
-  Service name or port number of a backend.
-
-The following variables are available while processing a request:
-
-client.ip
-  The client's IP address.
-
-client.identity
-  Identification of the client, used to load balance in the client director.
-
-server.ip
-  The IP address of the socket on which the client connection was received.
-
-server.hostname
-  The host name of the server.
-
-server.identity
-  The identity of the server, as set by the -i parameter.
-  If the -i parameter is not passed to varnishd, server.identity will be
-  set to the name of the instance, as specified by the -n parameter.
-
-req.method
-  The request type (e.g. "GET", "HEAD").
-
-req.url
-  The requested URL.
-
-req.proto
-  The HTTP protocol version used by the client.
-
-req.http.header
-  The corresponding HTTP header.
-
-req.backend_hint
-  Set bereq.backend to this if we attempt to fetch. This variable is
-  a convenience so the overall policy can be set up once and for all
-  in vcl_recv.
-
-req.can_gzip
-  Does the client accept the gzip transfer encoding.
-
-req.esi
-  Boolean. Set to false to disable ESI processing regardless of any
-  value in beresp.do_esi. Defaults to true. This variable is subject
-  to change in future versions, you should avoid using it.
-
-req.esi_level
-  A count of how many levels of ESI requests we're currently at.
-
-req.hash_always_miss
-  Force a cache miss for this request. If set to true Varnish will disregard
-  any existing objects and always (re)fetch from the backend.
-
-req.hash_ignore_busy
-  Ignore any busy object during cache lookup. You would want to do
-  this if you have two server looking up content from each other to
-  avoid potential deadlocks.
-
-req.restarts
-  A count of how many times this request has been restarted.
-
-req.ttl
-  ?
-
-req.xid
-  Unique ID of this request.
-
-The following variables are available while preparing a backend
-request (either for a cache miss or for pass or pipe mode):
-
-bereq.method
-  The request type (e.g. "GET", "HEAD").
-
-bereq.url
-  The requested URL.
-
-bereq.proto
-  The HTTP protocol version used to talk to the server.
-
-bereq.http.header
-  The corresponding HTTP header.
-
-bereq.backend
-  ?
-
-bereq.between_bytes_timeout
-  The time in seconds to wait between each received byte from the
-  backend. Not available in pipe mode.
-
-bereq.connect_timeout
-  The time in seconds to wait for a backend connection.
-
-bereq.first_byte_timeout
-  The time in seconds to wait for the first byte from the backend. Not
-  available in pipe mode.
-
-bereq.retries
-  ?
-
-bereq.uncacheable
-  ?
-
-The following variables are available after the requested object has
-been retrieved from the backend, before it is entered into the cache. In
-other words, they are available in vcl_backend_response:
-
-beresp.proto
-  The HTTP protocol version used the backend replied with.
-
-beresp.status
-  The HTTP status code returned by the server.
-
-beresp.reason
-  The HTTP status message returned by the server.
-
-beresp.http.header
-  The corresponding HTTP header.
-
-beresp.backend.ip
-  IP of the backend this response was fetched from.
-
-beresp.backend.name
-  Name of the backend this response was fetched from.
-
-beresp.do_esi
-  Boolean. ESI-process the object after fetching it. Defaults to
-  false. Set it to true to parse the object for ESI directives. Will
-  only be honored if req.esi is true.
-
-beresp.do_gunzip
-  Boolean. Unzip the object before storing it in the cache. Defaults to
-  false.
-
-beresp.do_gzip
-  Boolean. Gzip the object before storing it. Defaults to false. When
-  http_gzip_support is on Varnish will request already compressed
-  content from the backend and as such compression in Varnish is not needed.
-
-beresp.do_stream
-  Deliver the object to the client directly without fetching the whole
-  object into varnish. If this request is pass'ed it will not be
-  stored in memory. As of Varnish Cache 3.0 the object will marked as busy
-  as it is delivered so only client can access the object.
-
-beresp.grace
-  Set to a period to enable grace.
-
-beresp.keep
-  ?
-
-beresp.storage_hint
-  Hint to Varnish that you want to save this object to a particular storage
-  backend.
-
-beresp.ttl
-  The object's remaining time to live, in seconds. beresp.ttl is writable.
-
-beresp.uncacheable
-  ?
-
-After the object is entered into the cache, the following (mostly
-read-only) variables are available when the object has been located in
-cache, typically in vcl_hit, or when constructing a synthetic reply in
-vcl_error:
-
-obj.proto
-  The HTTP protocol version used when the object was retrieved.
-
-obj.status
-  The HTTP status code returned by the server.
-
-obj.reason
-  The HTTP status message returned by the server.
-
-obj.http.header
-  The corresponding HTTP header.
-
-obj.grace
-  The object's grace period in seconds. obj.grace is writable.
-
-obj.hits
-  The approximate number of times the object has been delivered. A value
-  of 0 indicates a cache miss. This variable is also available in
-  vcl_deliver.
-
-obj.keep
-  ?
-
-obj.last_use
-  The approximate time elapsed since the object was last requests, in
-  seconds. This variable is also available in vcl_deliver.
-
-obj.ttl
-  The object's remaining time to live, in seconds. obj.ttl is writable.
-
-obj.uncacheable
-  ?
-
-The following variables are available while preparing a response to the client:
-
-resp.proto
-  The HTTP protocol version to use for the response.
-
-resp.status
-  The HTTP status code that will be returned.
+---------
 
-resp.reason
-  The HTTP status message that will be returned.
+In VCL you have access to certain variable objects. These contain
+requests and responses currently beeing worked on. What variables are
+availble depends on context.
 
-resp.http.header
-  The corresponding HTTP header.
+.. include:: vcl_var.rst
 
-Values may be assigned to variables using the set keyword::
 
-    sub vcl_recv {
-        # Normalize the Host: header
-        if (req.http.host ~ "(?i)^(www.)?example.com$") {
-            set req.http.host = "www.example.com";
-        }
-    }
+Functions
+---------
 
-HTTP headers can be removed entirely using the unset keyword::
+The following built-in functions are available:
 
-    sub vcl_backend_response {
-        # Don't cache cookies
-        unset beresp.http.Set-Cookie;
-    }
+ban(expression)
+  Bans all objects in cache that match the expression.
 
-Grace
------
+hash_data(str)
+  Adds a string to the hash input. In the built-in VCL hash_data()
+  is called on the host and URL of the *request*.
 
-If the backend takes a long time to generate an object there is a risk
-of a thread pile up.  In order to prevent this you can enable *grace*.
-This allows Varnish to serve an expired version of the object while a
-fresh object is being generated by the backend.
+regsub(str, regex, sub)
+  Returns a copy of str with the first occurrence of the regular
+  expression regex replaced with sub. Within sub, \\0 (which can
+  also be spelled \\&) is replaced with the entire matched string,
+  and \\n is replaced with the contents of subgroup n in the
+  matched string.
 
-The following vcl code will make Varnish serve expired objects.  All
-object will be kept up to two minutes past their expiration time or a
-fresh object is generated.
-::
+regsuball(str, regex, sub)
+  As regsub() but this replaces all occurrences.
 
-    sub vcl_backend_response {
-        set beresp.grace = 2m;
-    }
 
 EXAMPLES
 ========
 
-The following code is the equivalent of the default configuration with
-the backend address set to "backend.example.com" and no backend port
-specified::
-
-    backend default {
-        .host = "backend.example.com";
-        .port = "http";
-    }
-
-.. include:: ../../../bin/varnishd/builtin.vcl
-  :literal:
-
-The following example shows how to support multiple sites running on
-separate backends in the same Varnish instance, by selecting backends
-based on the request URL::
-
-    backend www {
-        .host = "www.example.com";
-        .port = "80";
-    }
-
-    backend images {
-        .host = "images.example.com";
-        .port = "80";
-    }
-
-    sub vcl_recv {
-        if (req.http.host ~ "(?i)^(www.)?example.com$") {
-            set req.http.host = "www.example.com";
-            set req.backend_hint = www;
-        } elsif (req.http.host ~ "(?i)^images.example.com$") {
-            set req.backend_hint = images;
-        } else {
-            return (error(404, "Unknown virtual host"));
-        }
-    }
-
-The following snippet demonstrates how to force a minimum TTL for
-all documents.  Note that this is not the same as setting the
-default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL::
-
-    import std; # needed for std.log
-
-    sub vcl_backend_response {
-        if (beresp.ttl < 120s) {
-            std.log("Adjusting TTL");
-            set beresp.ttl = 120s;
-        }
-    }
-
-The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present::
-
-    sub vcl_recv {
-        if (req.method == "GET" && req.http.cookie) {
-            return (hash);
-        }
-    }
-
-    sub vcl_backend_response {
-        if (beresp.http.Set-Cookie) {
-            return (deliver);
-        }
-    }
-
-The following code implements the HTTP PURGE method as used by Squid
-for object invalidation::
-
-    acl purge {
-        "localhost";
-        "192.0.2.1"/24;
-    }
-
-    sub vcl_recv {
-        if (req.method == "PURGE") {
-            if (!client.ip ~ purge) {
-                return (error(405, "Not allowed."));
-            }
-            return (hash);
-        }
-    }
-
-    sub vcl_hit {
-        if (req.method == "PURGE") {
-            purge;
-            return (error(200, "Purged."));
-        }
-    }
-
-    sub vcl_miss {
-        if (req.method == "PURGE") {
-            purge;
-            return (error(200, "Purged."));
-        }
-    }
+For examples, please see the guide guide.
 
 SEE ALSO
 ========
@@ -986,10 +358,9 @@ HISTORY
 =======
 
 VCL was developed by Poul-Henning Kamp in cooperation with Verdens
-Gang AS, Redpill Linpro and Varnish Software.  This manual page was
-written by Dag-Erling Sm?rgrav and later edited by Poul-Henning Kamp,
-Kristian Lyngst?l and Per Buer.
-
+Gang AS, Redpill Linpro and Varnish Software.  This manual page is
+written by Per Buer, Poul-Henning Kamp, Martin Blix Grydeland,
+Kristian Lyngst?l and possibly others.
 
 COPYRIGHT
 =========



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 0038eeb Forgot about microseconds and reals
Message-ID: <E1WO1sM-0007Wk-3p@project.varnish-software.com>


commit 0038eebb241d2d2e938a2fe2fc6b474c5eea7ab8
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Mar 3 13:50:43 2014 +0100

    Forgot about microseconds and reals

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 96a4aab..5a7fe6b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -105,6 +105,9 @@ Durations
 
 Durations are defined by a number and a designation. The number can be a real so 1.5w is allowed.
 
+  ms
+    milliseconds
+
   s
     seconds
 
@@ -130,6 +133,12 @@ Ints
 Certain fields are integers, used as expected. In string context they
 return a string.
 
+Reals
+-----
+
+VCL understands real numbers. As with integers, when used in a string
+context they will return a string.
+
 
 Regular Expressions
 -------------------



From fgsch at lodoss.net  Thu Mar 13 09:24:30 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] f479083 Better error on missing import
Message-ID: <E1WO1sM-0007Wv-5b@project.varnish-software.com>


commit f47908316d9ff23fdc2a438f68e92d27bd194853
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Mar 3 16:01:56 2014 +0000

    Better error on missing import
    
    Fixes: #1436

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 69e99e5..b23d21e 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -98,3 +98,9 @@ varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_hash { if (2 == 3) { } }
 }
+
+varnish v1 -errvcl {Object not found: 'foo.bar'} {
+	sub vcl_init {
+		new bar = foo.bar();
+	}
+}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index b0f7d98..35f88f8 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -190,6 +190,13 @@ parse_new(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	sy2 = VCC_FindSymbol(tl, tl->t, SYM_OBJECT);
+	if (sy2 == NULL) {
+		VSB_printf(tl->sb, "Object not found: ");
+		vcc_ErrToken(tl, tl->t);
+		VSB_printf(tl->sb, " at ");
+		vcc_ErrWhere(tl, tl->t);
+		return;
+	}
 	XXXAN(sy2);
 
 	/*lint -save -e448 */



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 34e2bba Also make IMS fetches possible in the interval
 ttl+grace and keep.
Message-ID: <E1WO1sM-0007X4-6e@project.varnish-software.com>


commit 34e2bba2c8c51fc1bc1f6b06440860011a16cbb5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 4 10:21:51 2014 +0000

    Also make IMS fetches possible in the interval ttl+grace and keep.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ba3ef0d..8696dad 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -388,7 +388,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	}
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AZ (oc->flags & OC_F_BUSY);
+	AZ(oc->flags & OC_F_BUSY);
 	AZ(req->objcore);
 
 	o = oc_getobj(&wrk->stats, oc);
@@ -427,11 +427,11 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_DELIVER;
 		return (REQ_FSM_MORE);
 	case VCL_RET_FETCH:
-		(void)HSH_DerefObj(&wrk->stats, &req->obj);
 		req->objcore = boc;
 		if (req->objcore != NULL)
 			req->req_step = R_STP_MISS;
 		else {
+			(void)HSH_DerefObj(&wrk->stats, &req->obj);
 			/*
 			 * We don't have a busy object, so treat this
 			 * like a pass
@@ -486,18 +486,21 @@ DOT
 static enum req_fsm_nxt
 cnt_miss(struct worker *wrk, struct req *req)
 {
+	struct object *o;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	AZ(req->obj);
+
+	o = req->obj;
+	req->obj = NULL;
 
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
 	switch (wrk->handling) {
 	case VCL_RET_FETCH:
 		wrk->stats.cache_miss++;
-		VBF_Fetch(wrk, req, req->objcore, NULL, VBF_NORMAL);
+		VBF_Fetch(wrk, req, req->objcore, o, VBF_NORMAL);
 		req->req_step = R_STP_FETCH;
 		return (REQ_FSM_MORE);
 	case VCL_RET_ERROR:
@@ -513,6 +516,8 @@ cnt_miss(struct worker *wrk, struct req *req)
 		WRONG("Illegal return from vcl_miss{}");
 	}
 	free(req->vary_b);
+	if (o != NULL)
+		(void)HSH_DerefObj(&wrk->stats, &o);
 	AZ(HSH_DerefObjCore(&wrk->stats, &req->objcore));
 	return (REQ_FSM_MORE);
 }



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] c48e48a Fix IMS fetching of ESI objects.
Message-ID: <E1WO1sM-0007XD-7s@project.varnish-software.com>


commit c48e48ad2cbf2bb4b8e957baeeed2784123dffd1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 4 10:22:43 2014 +0000

    Fix IMS fetching of ESI objects.
    
    Fixes #1442

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 8672375..2d5f5ae 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -584,7 +584,14 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	obj->gzip_last = bo->ims_obj->gzip_last;
 	obj->gzip_stop = bo->ims_obj->gzip_stop;
 
-	XXXAZ(bo->ims_obj->esidata);
+	if (bo->ims_obj->esidata != NULL) {
+		obj->esidata = STV_alloc(bo, bo->ims_obj->esidata->len);
+		XXXAN(obj->esidata);
+		xxxassert(obj->esidata->space >= bo->ims_obj->esidata->len);
+		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr,
+		    bo->ims_obj->esidata->len);
+		obj->esidata->len = bo->ims_obj->esidata->len;
+	}
 
 	if (bo->ims_obj->vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
@@ -601,10 +608,10 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 
 	AZ(WS_Overflowed(bo->ws_o));
-	VBO_setstate(bo, BOS_STREAM);
+	if (bo->do_stream)
+		VBO_setstate(bo, BOS_STREAM);
 	HSH_Unbusy(&wrk->stats, obj->objcore);
 
-
 	st = NULL;
 	al = 0;
 
diff --git a/bin/varnishtest/tests/r01442.vtc b/bin/varnishtest/tests/r01442.vtc
new file mode 100644
index 0000000..939d5c9
--- /dev/null
+++ b/bin/varnishtest/tests/r01442.vtc
@@ -0,0 +1,31 @@
+varnishtest "ESI + IMS"
+
+server s1 {
+	rxreq
+	txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+		-body {<html><esi:remove>bla bla bla</esi:remove></html>}
+	rxreq
+	expect req.http.if-modified-since == "Thu, 26 Jun 2008 12:00:01 GMT"
+	txresp -status "304" -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" -nolen
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+		set beresp.grace = 0s;
+		set beresp.keep = 60s;
+		set beresp.ttl = 1s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+	delay 3
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+} -run
+



From lkarsten at varnish-software.com  Thu Mar 13 09:24:30 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 70b1004 Run-through of the first chapter in the docs
Message-ID: <E1WO1sM-0007XN-9a@project.varnish-software.com>


commit 70b1004ce4adbc52f5c872972cba9738862c9114
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Mar 4 14:43:37 2014 +0100

    Run-through of the first chapter in the docs
    
    I'm happy with the first chapter for 4.0 release.

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index e83d1dc..dde63a6 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -15,12 +15,24 @@
 Welcome to the Varnish documentation!
 =====================================
 
+Introduction
+------------
+
 Varnish is a state of the art web accelerator. It has its mission in front of a
-web server and cache content. It makes your web site go faster.
+web server and cache content.
+
+It makes your web site go faster.
 
 We suggest you start by reading the installation guide
 :ref:`install-index`. Once you have Varnish up and running go through
-our tutorial - :ref:`tutorial-index`.
+our tutorial - :ref:`tutorial-index`, and finally the :ref:`users_guide_index`.
+
+If you need to look up how a specific Varnish tool works, the
+:ref:`reference-index` can help you out. Changes from previous versions are in
+the :ref:`whats-new-index` chapter. In closing we have :ref:`phk`, a collection
+of blog posts related to Varnish and HTTP.
+
+
 
 .. toctree::
    :maxdepth: 1
diff --git a/doc/sphinx/installation/bugs.rst b/doc/sphinx/installation/bugs.rst
index 16f0c51..98c0e19 100644
--- a/doc/sphinx/installation/bugs.rst
+++ b/doc/sphinx/installation/bugs.rst
@@ -28,11 +28,11 @@ Varnish crashes
 Plain and simple: **boom**
 
 Varnish is split over two processes, the manager and the child.  The child
-does all the work, and the manager hangs around to resurect it if it
+does all the work, and the manager hangs around to resurrect it if it
 crashes.
 
 Therefore, the first thing to do if you see a Varnish crash, is to examine
-your syslogs, to see if it has happened before.  (One site is rumoured
+your syslogs to see if it has happened before.  (One site is rumoured
 to have had Varnish restarting every 10 minutes and *still* provide better
 service than their CMS system.)
 
@@ -138,11 +138,11 @@ transactions recorded with ``varnishlog`` and your explanation of
 what is wrong about what Varnish does.
 
 Be aware, that often Varnish does exactly what you asked it to, rather
-than what you intended it to do, so it sounds like a bug that would
+than what you intended it to do. If it sounds like a bug that would
 have tripped up everybody else, take a moment to read though your
 VCL and see if it really does what you think.
 
-You can also try setting the ``vcl_trace`` parameter, that will
-generate log records with like and char number for each statement
-executed in your VCL program.
+You can also try setting the ``vcl_trace`` parameter, that will generate log
+records with like and character number for each statement executed in your VCL
+program.
 
diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 4fb8362..aaea9ec 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -7,7 +7,7 @@ we try to help as much as time permits and have tried to streamline
 this process as much as possible.
 
 But before you grab hold of us, spend a moment composing your thoughts and
-formulate your question, there is nothing as pointless as simply telling
+formulate your question. There is nothing as pointless as simply telling
 us "Varnish does not work for me" with no further information to give
 any clue to why.
 
@@ -18,15 +18,15 @@ much faster that way.
 IRC Channel
 ===========
 
-The most immediate way to get hold of us, is to join our IRC channel:
+The most immediate way to get hold of us is to join our IRC channel:
 
 	``#varnish on server irc.linpro.no``
 
-The main timezone of the channel is Europe+America.
+The main timezone of the channel is Europe work hours.
 
 If you can explain your problem in a few clear sentences, without too
 much copy&paste, IRC is a good way to try to get help. If you do need
-to paste log files, VCL and so on, please use a pastebin_.
+to paste log files, VCL and so on, please use a pastebin_ service.
 
 If the channel is all quiet, try again some time later, we do have lives,
 families and jobs to deal with also.
@@ -39,7 +39,7 @@ it mostly on topic, and don't paste random links unless they are
 Mailing Lists
 =============
 
-Getting on or off our mailing lists happens through MailMan_.
+Getting on or off our mailing lists happens through Mailman_.
 
 If you are going to use Varnish, subscribing to our ``varnish-announce``
 mailing list is probably a very good idea. The typical pattern is that
@@ -66,6 +66,10 @@ usually quite focused on source-code and such.  Everybody on
 the -dev list is also on -misc, so cross-posting only serves to annoy
 those people.
 
+.. XXX: we should introduce the wiki (if we care about it) before
+.. we start referring to it (below). Make a wiki chapter?
+
+
 Trouble Tickets
 ===============
 
@@ -90,6 +94,6 @@ an email to phk at FreeBSD.org.
 	sales at varnish-software.com
 
 
-.. _Mailman: http://lists.varnish-cache.org/mailman/listinfo
+.. _mailman: http://lists.varnish-cache.org/mailman/listinfo
 .. _pastebin: http://gist.github.com/
 .. _"Shopping-List" wiki page: http://varnish-cache.org/wiki/PostTwoShoppingList
diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index 60d43e0..0b0c665 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -5,10 +5,13 @@ Varnish Installation
 %%%%%%%%%%%%%%%%%%%%
 
 This document explains how to get Varnish onto your system, where
-to get help, how report bugs and so on. In other words, it is a manual
-about pretty much everything else than actually using Varnish to
+to get help, how report bugs and so on.
+
+In other words, it is a manual about pretty much everything else than actually using Varnish to
 move traffic.
 
+.. XXX: rewrite the last paragraph.
+
 .. toctree::
 
 	prerequisites.rst
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 45c5e4e..a32eb68 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -4,7 +4,7 @@ Installing Varnish
 ==================
 
 With open source software, you can choose to install binary packages
-or compile stuff from source-code. To install a package or compile
+or compile it yourself from source code. To install a package or compile
 from source is a matter of personal taste. If you don't know which
 method to choose, read the whole document and choose the method you
 are most comfortable with.
@@ -29,7 +29,7 @@ CentOS/RedHat
 -------------
 
 We try to keep the latest version available as prebuilt RPMs (el5 and el6)
-on `repo.varnish-cache.org <http://repo.varnish-cache.org/>`_.  See the
+on `repo.varnish-cache.org <http://repo.varnish-cache.org/>`_.  See the online
 `RedHat installation instructions
 <http://www.varnish-cache.org/installation/redhat>`_ for more information.
 
@@ -46,7 +46,7 @@ Debian/Ubuntu
 Varnish is distributed with both Debian and Ubuntu. In order to get
 Varnish up and running type `sudo apt-get install varnish`. Please
 note that this might not be the latest version of Varnish.  If you
-need a later version of Varnish, please follow the installation
+need a later version of Varnish, please follow the online installation
 instructions for `Debian
 <http://www.varnish-cache.org/installation/debian>`_ or `Ubuntu
 <http://www.varnish-cache.org/installation/ubuntu>`_.
@@ -92,10 +92,10 @@ If you're building from git, you also need the following:
 * python-docutils
 * python-sphinx (optional, if you want to build the documentation)
 
-Build dependencies on Red Hat / CentOS
+Build dependencies on RedHat / CentOS
 --------------------------------------
 
-To build Varnish on a Red Hat or CentOS system you need the following
+To build Varnish on a RedHat or CentOS system you need the following
 packages installed:
 
 * automake
@@ -115,8 +115,8 @@ If you're building from git, you also need the following:
 Compiling Varnish
 -----------------
 
-Next, configuration: The configuration will need the dependencies
-above satisfied. Once that is taken care of::
+The configuration will need the dependencies above satisfied. Once that is
+taken care of::
 
 	cd varnish-cache
 	sh autogen.sh
@@ -124,7 +124,7 @@ above satisfied. Once that is taken care of::
 	make
 
 The ``configure`` script takes some arguments, but more likely than
-not, you can forget about that for now, almost everything in Varnish
+not you can forget about that for now, almost everything in Varnish
 are run time parameters.
 
 Before you install, you may want to run the test suite, make a cup of
@@ -143,10 +143,11 @@ Installing
 
 And finally, the true test of a brave heart::
 
-	make install
+	sudo make install
 
 Varnish will now be installed in /usr/local. The varnishd binary is in
 /usr/local/sbin/varnishd and its default configuration will be
 /usr/local/etc/varnish/default.vcl.
 
 You can now proceed to the :ref:`tutorial-index`.
+
diff --git a/doc/sphinx/installation/platformnotes.rst b/doc/sphinx/installation/platformnotes.rst
index bacc57e..a7d3fb4 100644
--- a/doc/sphinx/installation/platformnotes.rst
+++ b/doc/sphinx/installation/platformnotes.rst
@@ -2,6 +2,10 @@
 Platform specific notes
 ------------------------
 
+On some platforms it is necessary to adjust the operating system before running
+Varnish on it. The systems and steps known to us are described in this section.
+
+
 Transparent hugepages on Redhat Enterprise Linux 6
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -12,7 +16,8 @@ It is recommended to disable transparent hugepages on affected systems::
 
     $ echo "never" > /sys/kernel/mm/redhat_transparent_hugepage/enabled
 
-On Debian/Ubuntu systems running 3.2 kernels the default value is "madvise" and does not need to changed.
+On Debian/Ubuntu systems running 3.2 kernels the default value is "madvise" and
+does not need to be changed.
 
 
 OpenVZ
@@ -38,12 +43,12 @@ in the Varnish startup script.
 TCP keep-alive configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-On some systems, Varnish is not able to set the TCP keep-alive values
-per socket, and therefor the tcp_keepalive_* Varnish runtime
-parameters are not available. On these platforms it can be benefitial
-to tune the system wide values for these in order to more reliably
-detect remote close for sessions spending long time on
-waitinglists. This will help free up resources faster.
+On some Solaris, FreeBSD and OS X systems, Varnish is not able to set the TCP
+keep-alive values per socket, and therefor the tcp_keepalive_* Varnish runtime
+parameters are not available. On these platforms it can be benefitial to tune
+the system wide values for these in order to more reliably detect remote close
+for sessions spending long time on waitinglists. This will help free up
+resources faster.
 
 Systems to not support TCP keep-alive values per socket include:
 



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] b9e6d62 Add code to do the right "backwards" merge of IMS
 responses.
Message-ID: <E1WO1sM-0007XY-B8@project.varnish-software.com>


commit b9e6d624e0f07d1de469dcc7f9b1df385482c7c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 5 08:23:51 2014 +0000

    Add code to do the right "backwards" merge of IMS responses.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7c6079e..f18c69a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -192,6 +192,7 @@ struct http {
 	txt			*hd;
 	unsigned char		*hdf;
 #define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
+#define HDF_MARKER		(1 << 1)	/* Marker bit */
 	uint16_t		shd;		/* Size of hd space */
 	uint16_t		nhd;		/* Next free hd */
 	uint16_t		status;
@@ -997,6 +998,7 @@ void http_CopyHome(const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
+void http_Merge(const struct http *fm, struct http *to);
 
 /* cache_http1_proto.c */
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2d5f5ae..20b60fe 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -339,8 +339,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->ims_obj != NULL && bo->beresp->status == 304) {
 		bo->beresp->status = 200;
-		http_PrintfHeader(bo->beresp, "Content-Length: %jd",
-		    (intmax_t)bo->ims_obj->len);
+		http_Merge(bo->ims_obj->http, bo->beresp);
 		do_ims = 1;
 	} else
 		do_ims = 0;
@@ -567,7 +566,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		l += vl;
 	} else
 		vl = 0;
-	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
+	l += http_EstimateWS(bo->beresp, 0, &nhttp);
 
 	obj = vbf_allocobj(bo, l, nhttp);
 	if (obj == NULL) {
@@ -603,7 +602,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 	obj->http->logtag = SLT_ObjMethod;
 	/* XXX: we should have our own HTTP_A_CONDFETCH */
-	http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
+	http_FilterResp(bo->beresp, obj->http, HTTPH_A_INS);
 	http_CopyHome(obj->http);
 
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ec28396..2a9b8de 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -254,7 +254,6 @@ http_CollectHdr(struct http *hp, const char *hdr)
 	WS_ReleaseP(hp->ws, b + 1);
 }
 
-
 /*--------------------------------------------------------------------*/
 
 static unsigned
@@ -300,7 +299,6 @@ http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
 	return (1);
 }
 
-
 /*--------------------------------------------------------------------
  * Find a given data element in a header according to RFC2616's #rule
  * (section 2.1, p15)
@@ -639,6 +637,30 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 }
 
 /*--------------------------------------------------------------------
+ * Merge two HTTP headers the "wrong" way.
+ */
+
+void
+http_Merge(const struct http *fm, struct http *to)
+{
+	unsigned u, v;
+	const char *p;
+
+	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
+		fm->hdf[u] |= HDF_MARKER;
+	for (v = HTTP_HDR_FIRST; v < to->nhd; v++) {
+		p = strchr(to->hd[v].b, ':');
+		AN(p);
+		u = http_findhdr(fm, p - to->hd[v].b, to->hd[v].b);
+		if (u)
+			fm->hdf[u] &= ~HDF_MARKER;
+	}
+	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
+		if (fm->hdf[u] & HDF_MARKER)
+			http_SetHeader(to, fm->hd[u].b);
+}
+
+/*--------------------------------------------------------------------
  * This function copies any header fields which reference foreign
  * storage into our own WS.
  */
diff --git a/bin/varnishtest/tests/c00060.vtc b/bin/varnishtest/tests/c00060.vtc
new file mode 100644
index 0000000..ece157e
--- /dev/null
+++ b/bin/varnishtest/tests/c00060.vtc
@@ -0,0 +1,45 @@
+varnishtest "IMS header merging"
+
+server s1 {
+	rxreq
+	txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+		-hdr "Foobar: foo" \
+		-hdr "Snafu: 1" \
+		-bodylen 13
+	rxreq
+	expect req.http.if-modified-since == "Thu, 26 Jun 2008 12:00:01 GMT"
+	txresp -status "304" \
+		-hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+		-hdr "Snafu: 2" \
+		-nolen
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.grace = 0s;
+		set beresp.keep = 60s;
+		set beresp.ttl = 1s;
+		if (beresp.http.foobar == "foo") {
+			set beresp.http.foobar = "foo0";
+		}
+		if (beresp.http.snafu == "2") {
+			set beresp.http.snafu = "2a";
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+	expect resp.http.foobar == foo0
+	expect resp.http.snafu == 1
+	delay 3
+	txreq
+	rxresp
+	expect resp.bodylen == 13
+	expect resp.http.foobar == foo0
+	expect resp.http.snafu == 2a
+} -run
+



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 774d865 Add link to the man page
Message-ID: <E1WO1sM-0007Xi-C7@project.varnish-software.com>


commit 774d86583d2a48d7bf34e63e49d29cf167c25f85
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 4 13:54:05 2014 +0100

    Add link to the man page

diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 15b6da3..889ea97 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -9,6 +9,9 @@ preferences.
 
 Note that VCL doesn't contain any loops or jump statements.
 
+This document gives an outline of the most important parts of the
+syntax. For a full documentation of VCL syntax please see
+:ref:`reference-vcl` in the reference.
 
 Strings
 ~~~~~~~



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 10fb858 First stab at director documentation
Message-ID: <E1WO1sM-0007Xt-Dr@project.varnish-software.com>


commit 10fb858033ad4709131d5eda7a4f31eb3c7768de
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 5 14:01:13 2014 +0100

    First stab at director documentation

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 609fc77..4905858 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -27,18 +27,125 @@
 
 $Module directors 3 Backend traffic directors
 
+DESCRIPTION
+===========
+
+Vmod_direcors enables load balancing in Varnish. The module are serves
+as an example on how one could extend the load balancing capabilites
+of Varnish.
+
+To enable load balancing you must import this vmod (directors) in your
+VCL:::
+
+  import directors;
+
+Then you define your backends. Once you have the backends declared you
+can add them to a director. This happens in executed VCL code. If you
+want to emulate the previous behaviour of Varnish 3.0 you can just
+initialize the directors in vcl_init, like this:::
+
+    sub vcl_init {
+        new bar = directors.round_robin();
+        bar.add_backend(server1);
+        bar.add_backend(server2);
+    }
+
+As you can see there is nothing keeping you from manipulting the
+directors elsewhere in VCL. So, you could have VCL code that would
+add more backends to a director when a certin URL is called.
+
 $Object round_robin()
+
+Description
+       Create a round robin director. This director will pick backends
+       in a round robin fashion.
+Example
+       new bar = directors.round_robin();
+
 $Method VOID .add_backend(BACKEND)
+
+Description
+       Adds a backend to the director.
+Example
+       rrdir.add_backend(backend1);
+
 $Method BACKEND .backend()
 
+Description
+       Picks a backend from the director.
+Example
+       req.backend = rrdir.backend();
+
 $Object fallback()
+
+Description
+        Creates a fallback director. This director will asdasoidwqjwqdasdas
+Example
+        new foo = directors.fallback();
+
 $Method VOID .add_backend(BACKEND)
+
+Description
+       Adds a backend to the director.
+Example
+       bar.add_backend(backend1);
+
 $Method BACKEND .backend()
 
+Description
+       Picks a backend from the director.
+Example
+       req.backend = rrdir.backend();
+
 $Object random()
+
+Description
+       Adds a random director. This director chooses backend based on
+       a random number. As you add backends to the director each
+       backends gets a weight, which is used to when requests are
+       being distributed. So, a backend with a weight of 1 would get
+       more or less 1% of the traffic of a backend in the same
+       director with a weight of 100.
+Example
+       new rand_dir = director.random();
+
 $Method VOID .add_backend(BACKEND, REAL)
+
+Description
+       Adds a backend to the director with weight.
+Example
+       bar.add_backend(backend1, 3.14);
+
 $Method BACKEND .backend()
 
+Description
+       Picks a backend from the director.
+Example
+       req.backend = rrdir.backend();
+
 $Object hash()
+
+Description
+       Creates a hash director. The hash director chooses the backend
+       bashed on hashing an arbitrary string. If you provide it with a
+       session cookie, you'll have the client connecting to the same
+       backend every time.
+Example
+       new hdir = director.hash();
+
 $Method VOID .add_backend(BACKEND, REAL)
+
+Description
+       Adds a backend to the director.
+Example
+       hdir.add_backend(backend1);
+
 $Method BACKEND .backend(STRING_LIST)
+
+Description
+       Picks a backend from the director. Use the string or list of
+       strings provided to pick the backend.
+Example
+       # pick a backend based on the cookie header from the client
+       req.backend = hdir.backend(req.http.cookie);
+



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] eb0513a remove the vcl vars from the index. it is included in
 the vcl man page
Message-ID: <E1WO1sM-0007Y2-Er@project.varnish-software.com>


commit eb0513acf8b25ee21671cab0d754c04785219caa
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 5 14:16:52 2014 +0100

    remove the vcl vars from the index.
    it is included in the vcl man page

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index e8377f0..1537fb4 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -24,7 +24,6 @@ The Varnish Reference Manual
 	vmod_std.rst
 	vsl.rst
 	vsl-query.rst
-	vcl_var.rst
 
 .. todo::
         The programs:



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 825ecef req.backend is gone now. req.backend_hint is it.
Message-ID: <E1WO1sM-0007YC-Fu@project.varnish-software.com>


commit 825ecef42dabfc43076624dc0e5de97bf6f67d35
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 5 14:19:07 2014 +0100

    req.backend is gone now. req.backend_hint is it.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 4905858..67ad069 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -74,7 +74,7 @@ $Method BACKEND .backend()
 Description
        Picks a backend from the director.
 Example
-       req.backend = rrdir.backend();
+       req.backend_hint = rrdir.backend();
 
 $Object fallback()
 
@@ -95,7 +95,7 @@ $Method BACKEND .backend()
 Description
        Picks a backend from the director.
 Example
-       req.backend = rrdir.backend();
+       req.backend_hint = rrdir.backend();
 
 $Object random()
 
@@ -121,7 +121,7 @@ $Method BACKEND .backend()
 Description
        Picks a backend from the director.
 Example
-       req.backend = rrdir.backend();
+       req.backend_hint = rrdir.backend();
 
 $Object hash()
 
@@ -147,5 +147,5 @@ Description
        strings provided to pick the backend.
 Example
        # pick a backend based on the cookie header from the client
-       req.backend = hdir.backend(req.http.cookie);
+       req.backend_hint = hdir.backend(req.http.cookie);
 



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 990197a Reuse the same beresp->obj function for both cond and
 normal fetches.
Message-ID: <E1WO1sM-0007YL-Gw@project.varnish-software.com>


commit 990197ad3fbd8b4bb30d9f9133c57ef35c9de27e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 5 14:54:55 2014 +0000

    Reuse the same beresp->obj function for both cond and normal fetches.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 20b60fe..27a2df8 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -140,9 +140,6 @@ vbf_beresp2obj(struct busyobj *bo)
 	AZ(bo->fetch_obj);
 	bo->fetch_obj = obj;
 
-	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
-		obj->gziped = 1;
-
 	if (vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
 		    VSB_data(vary), varyl);
@@ -464,6 +461,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	obj = bo->fetch_obj;
 
+	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
+		obj->gziped = 1;
+
 	/*
 	 * Ready to fetch the body
 	 */
@@ -548,40 +548,18 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 {
-	unsigned l;
-	uint16_t nhttp;
 	struct object *obj;
 	struct objiter *oi;
 	void *sp;
-	ssize_t sl, al, tl, vl;
+	ssize_t sl, al, tl;
 	struct storage *st;
 	enum objiter_status ois;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	l = 0;
-	if (bo->ims_obj->vary != NULL) {
-		vl = VRY_Validate(bo->ims_obj->vary);
-		l += vl;
-	} else
-		vl = 0;
-	l += http_EstimateWS(bo->beresp, 0, &nhttp);
-
-	obj = vbf_allocobj(bo, l, nhttp);
-	if (obj == NULL) {
-		(void)VFP_Error(bo, "Could not get storage");
-		VDI_CloseFd(&bo->vbc);
-		return (F_STP_DONE);
-	}
-
-	AZ(bo->fetch_obj);
-	bo->fetch_obj = obj;
-
-	obj->gziped = bo->ims_obj->gziped;
-	obj->gzip_start = bo->ims_obj->gzip_start;
-	obj->gzip_last = bo->ims_obj->gzip_last;
-	obj->gzip_stop = bo->ims_obj->gzip_stop;
+	AZ(vbf_beresp2obj(bo));
+	obj = bo->fetch_obj;
 
 	if (bo->ims_obj->esidata != NULL) {
 		obj->esidata = STV_alloc(bo, bo->ims_obj->esidata->len);
@@ -592,19 +570,10 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		obj->esidata->len = bo->ims_obj->esidata->len;
 	}
 
-	if (bo->ims_obj->vary != NULL) {
-		obj->vary = (void *)WS_Copy(obj->http->ws,
-		    bo->ims_obj->vary, vl);
-		assert(vl == VRY_Validate(obj->vary));
-	}
-
-	obj->vxid = bo->vsl->wid;
-
-	obj->http->logtag = SLT_ObjMethod;
-	/* XXX: we should have our own HTTP_A_CONDFETCH */
-	http_FilterResp(bo->beresp, obj->http, HTTPH_A_INS);
-	http_CopyHome(obj->http);
-
+	obj->gziped = bo->ims_obj->gziped;
+	obj->gzip_start = bo->ims_obj->gzip_start;
+	obj->gzip_last = bo->ims_obj->gzip_last;
+	obj->gzip_stop = bo->ims_obj->gzip_stop;
 
 	AZ(WS_Overflowed(bo->ws_o));
 	if (bo->do_stream)



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 35baa30 Make ESI a separate chapter
Message-ID: <E1WO1sM-0007YU-IE@project.varnish-software.com>


commit 35baa30ebf557a87fe9307f60392ce1b28e81bca
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:07:48 2014 +0100

    Make ESI a separate chapter

diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 58b31f2..9035224 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -1,7 +1,7 @@
 .. _users-guide-esi:
 
-Edge Side Includes
-------------------
+Content composition with Edge Side Includes
+-------------------------------------------
 
 Varnish can cache create web pages by putting different pages
 together. These *fragments* can have individual cache policies. If you



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 03fd530 Move the content from cookies and vary into increasing
 perf
Message-ID: <E1WO1sM-0007Yf-Jo@project.varnish-software.com>


commit 03fd530f057bc98e4421a9f1ac77e5e14e8028a9
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:08:09 2014 +0100

    Move the content from cookies and vary into increasing perf

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 431c0fc..3e1adc5 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -114,6 +114,104 @@ your expectations.
 
 Let's take a look at the important headers you should be aware of:
 
+.. _users-guide-cookies:
+
+Cookies
+-------
+
+Varnish will, in the default configuration, not cache a object coming
+from the backend with a Set-Cookie header present. Also, if the client
+sends a Cookie header, Varnish will bypass the cache and go directly to
+the backend.
+
+This can be overly conservative. A lot of sites use Google Analytics
+(GA) to analyze their traffic. GA sets a cookie to track you. This
+cookie is used by the client side javascript and is therefore of no
+interest to the server. 
+
+Cookies from the client
+~~~~~~~~~~~~~~~~~~~~~~~
+
+For a lot of web application it makes sense to completely disregard the
+cookies unless you are accessing a special part of the web site. This
+VCL snippet in vcl_recv will disregard cookies unless you are
+accessing /admin/::
+
+  if ( !( req.url ~ ^/admin/) ) {
+    unset req.http.Cookie;
+  }
+
+Quite simple. If, however, you need to do something more complicated,
+like removing one out of several cookies, things get
+difficult. Unfortunately Varnish doesn't have good tools for
+manipulating the Cookies. We have to use regular expressions to do the
+work. If you are familiar with regular expressions you'll understand
+whats going on. If you don't I suggest you either pick up a book on
+the subject, read through the *pcrepattern* man page or read through
+one of many online guides.
+
+Let me show you what Varnish Software uses. We use some cookies for
+Google Analytics tracking and similar tools. The cookies are all set
+and used by Javascript. Varnish and Drupal doesn't need to see those
+cookies and since Varnish will cease caching of pages when the client
+sends cookies we will discard these unnecessary cookies in VCL. 
+
+In the following VCL we discard all cookies that start with a
+underscore::
+
+  // Remove has_js and Google Analytics __* cookies.
+  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
+  // Remove a ";" prefix, if present.
+  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
+
+Let me show you an example where we remove everything except the
+cookies named COOKIE1 and COOKIE2 and you can marvel at it::
+
+  sub vcl_recv {
+    if (req.http.Cookie) {
+      set req.http.Cookie = ";" + req.http.Cookie;
+      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
+      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
+      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
+      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
+
+      if (req.http.Cookie == "") {
+          remove req.http.Cookie;
+      }
+    }
+  }
+
+A somewhat simpler example that can accomplish almost the same can be
+found below. Instead of filtering out the other cookies it picks out
+the one cookie that is needed, copies it to another header and then
+copies it back, deleting the original cookie header.::
+
+  sub vcl_recv {
+         # save the original cookie header so we can mangle it
+        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
+        # using a capturing sub pattern, extract the continuous string of 
+        # alphanumerics that immediately follows "PHPSESSID="
+        set req.http.X-Varnish-PHP_SID = 
+           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
+        set req.http.Cookie = req.X-Varnish-PHP_SID;
+        remove req.X-Varnish-PHP_SID;
+   }   
+
+There are other scary examples of what can be done in VCL in the
+Varnish Cache Wiki.
+
+
+Cookies coming from the backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your backend server sets a cookie using the Set-Cookie header
+Varnish will not cache the page in the default configuration.  A
+hit-for-pass object (see :ref:`user-guide-vcl_actions`) is created.
+So, if the backend server acts silly and sets unwanted cookies just unset
+the Set-Cookie header and all should be fine. 
+
+
+
 Cache-Control
 ~~~~~~~~~~~~~
 
@@ -145,7 +243,7 @@ Pragma
 An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
 header. You could easily add support for this header in VCL.
 
-In vcl_fetch::
+In vcl_backend_response::
 
   if (beresp.http.Pragma ~ "nocache") {
         set beresp.uncacheable = true;
@@ -168,7 +266,7 @@ somewhat cumbersome backend.
 You need VCL to identify the objects you want and then you set the
 beresp.ttl to whatever you want::
 
-  sub vcl_fetch {
+  sub vcl_backend_response {
       if (req.url ~ "^/legacy_broken_cms/") {
           set beresp.ttl = 5d;
       }
@@ -203,3 +301,80 @@ setting up redirects or by using the following VCL::
   }
 
 
+.. _users-guide-vary:
+
+HTTP Vary
+---------
+
+*HTTP Vary is not a trivial concept. It is by far the most
+misunderstood HTTP header.*
+
+A lot of the response headers tell the client something about the HTTP
+object being delivered. Clients can request different variants a an
+HTTP object, based on their preference. Their preferences might cover
+stuff like encoding or language. When a client prefers UK English this
+is indicated through "Accept-Language: en-uk". Caches need to keep
+these different variants apart and this is done through the HTTP
+response header "Vary".
+
+When a backend server issues a "Vary: Accept-Language" it tells
+Varnish that its needs to cache a separate version for every different
+Accept-Language that is coming from the clients.
+
+If two clients say they accept the languages "en-us, en-uk" and "da,
+de" respectively, Varnish will cache and serve two different versions
+of the page if the backend indicated that Varnish needs to vary on the
+Accept-Language header.
+
+Please note that the headers that Vary refer to need to match
+*exactly* for there to be a match. So Varnish will keep two copies of
+a page if one of them was created for "en-us, en-uk" and the other for
+"en-us,en-uk". Just the lack of space will force Varnish to cache
+another version.
+
+To achieve a high hitrate whilst using Vary is there therefor crucial
+to normalize the headers the backends varies on. Remember, just a
+difference in case can force different cache entries.
+
+The following VCL code will normalize the Accept-Language headers, to
+one of either "en","de" or "fr"::
+
+    if (req.http.Accept-Language) {
+        if (req.http.Accept-Language ~ "en") {
+            set req.http.Accept-Language = "en";
+        } elsif (req.http.Accept-Language ~ "de") {
+            set req.http.Accept-Language = "de";
+        } elsif (req.http.Accept-Language ~ "fr") {
+            set req.http.Accept-Language = "fr";
+        } else {
+            # unknown language. Remove the accept-language header and 
+	    # use the backend default.
+            remove req.http.Accept-Language
+        }
+    }
+
+The code sets the Accept-Encoding header from the client to either
+gzip, deflate with a preference for gzip.
+
+Vary parse errors
+~~~~~~~~~~~~~~~~~
+
+Varnish will return a 503 internal server error page when it fails to
+parse the Vary server header, or if any of the client headers listed
+in the Vary header exceeds the limit of 65k characters. An SLT_Error
+log entry is added in these cases.
+
+Pitfall - Vary: User-Agent
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some applications or application servers send *Vary: User-Agent* along
+with their content. This instructs Varnish to cache a separate copy
+for every variation of User-Agent there is. There are plenty. Even a
+single patchlevel of the same browser will generate at least 10
+different User-Agent headers based just on what operating system they
+are running. 
+
+So if you *really* need to Vary based on User-Agent be sure to
+normalize the header or your hit rate will suffer badly. Use the above
+code as a template.
+



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 4f487ab Update index, remove orphans, add esi
Message-ID: <E1WO1sM-0007Yp-LO@project.varnish-software.com>


commit 4f487ab8e568c2ab80c739cc4ad3fa8a11cdd758
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:08:24 2014 +0100

    Update index, remove orphans, add esi

diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 1000a6f..7cd0c25 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -40,6 +40,7 @@ from transaction level to aggregate statistics.
    vcl
    report
    performance
+   esi   
    troubleshooting
    orphans
 



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] dcf8e96 rebreak lines
Message-ID: <E1WO1sM-0007Yy-Mk@project.varnish-software.com>


commit dcf8e960f9e3dc35e6434d9968bfe0566cf020db
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:09:02 2014 +0100

    rebreak lines

diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index 0b0c665..7952958 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -7,8 +7,8 @@ Varnish Installation
 This document explains how to get Varnish onto your system, where
 to get help, how report bugs and so on.
 
-In other words, it is a manual about pretty much everything else than actually using Varnish to
-move traffic.
+In other words, it is a manual about pretty much everything else than
+actually using Varnish to move traffic.
 
 .. XXX: rewrite the last paragraph.
 



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] e8384a5 remove orphans - content absorbed elsewhere
Message-ID: <E1WO1sM-0007Z9-Nx@project.varnish-software.com>


commit e8384a53979b44526573a868a444a32fd84bc2ec
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:10:05 2014 +0100

    remove orphans - content absorbed elsewhere

diff --git a/doc/sphinx/users-guide/cookies.rst b/doc/sphinx/users-guide/cookies.rst
deleted file mode 100644
index c57478b..0000000
--- a/doc/sphinx/users-guide/cookies.rst
+++ /dev/null
@@ -1,96 +0,0 @@
-.. _users-guide-cookies:
-
-Cookies
--------
-
-Varnish will, in the default configuration, not cache a object coming
-from the backend with a Set-Cookie header present. Also, if the client
-sends a Cookie header, Varnish will bypass the cache and go directly to
-the backend.
-
-This can be overly conservative. A lot of sites use Google Analytics
-(GA) to analyze their traffic. GA sets a cookie to track you. This
-cookie is used by the client side javascript and is therefore of no
-interest to the server. 
-
-Cookies from the client
-~~~~~~~~~~~~~~~~~~~~~~~
-
-For a lot of web application it makes sense to completely disregard the
-cookies unless you are accessing a special part of the web site. This
-VCL snippet in vcl_recv will disregard cookies unless you are
-accessing /admin/::
-
-  if ( !( req.url ~ ^/admin/) ) {
-    unset req.http.Cookie;
-  }
-
-Quite simple. If, however, you need to do something more complicated,
-like removing one out of several cookies, things get
-difficult. Unfortunately Varnish doesn't have good tools for
-manipulating the Cookies. We have to use regular expressions to do the
-work. If you are familiar with regular expressions you'll understand
-whats going on. If you don't I suggest you either pick up a book on
-the subject, read through the *pcrepattern* man page or read through
-one of many online guides.
-
-Let me show you what Varnish Software uses. We use some cookies for
-Google Analytics tracking and similar tools. The cookies are all set
-and used by Javascript. Varnish and Drupal doesn't need to see those
-cookies and since Varnish will cease caching of pages when the client
-sends cookies we will discard these unnecessary cookies in VCL. 
-
-In the following VCL we discard all cookies that start with a
-underscore::
-
-  // Remove has_js and Google Analytics __* cookies.
-  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
-  // Remove a ";" prefix, if present.
-  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
-
-Let me show you an example where we remove everything except the
-cookies named COOKIE1 and COOKIE2 and you can marvel at it::
-
-  sub vcl_recv {
-    if (req.http.Cookie) {
-      set req.http.Cookie = ";" + req.http.Cookie;
-      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
-      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
-
-      if (req.http.Cookie == "") {
-          remove req.http.Cookie;
-      }
-    }
-  }
-
-A somewhat simpler example that can accomplish almost the same can be
-found below. Instead of filtering out the other cookies it picks out
-the one cookie that is needed, copies it to another header and then
-copies it back, deleting the original cookie header.::
-
-  sub vcl_recv {
-         # save the original cookie header so we can mangle it
-        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
-        # using a capturing sub pattern, extract the continuous string of 
-        # alphanumerics that immediately follows "PHPSESSID="
-        set req.http.X-Varnish-PHP_SID = 
-           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
-        set req.http.Cookie = req.X-Varnish-PHP_SID;
-        remove req.X-Varnish-PHP_SID;
-   }   
-
-There are other scary examples of what can be done in VCL in the
-Varnish Cache Wiki.
-
-
-Cookies coming from the backend
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If your backend server sets a cookie using the Set-Cookie header
-Varnish will not cache the page in the default configuration.  A
-hit-for-pass object (see :ref:`user-guide-vcl_actions`) is created.
-So, if the backend server acts silly and sets unwanted cookies just unset
-the Set-Cookie header and all should be fine. 
-
diff --git a/doc/sphinx/users-guide/orphans.rst b/doc/sphinx/users-guide/orphans.rst
deleted file mode 100644
index 76fbf5d..0000000
--- a/doc/sphinx/users-guide/orphans.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Orphans
-=======
-
-XXX: These are chapters which need to find a new home in the other sections.
-
-.. toctree::
-   :maxdepth: 2
-
-   esi
-   vary
-   cookies
diff --git a/doc/sphinx/users-guide/vary.rst b/doc/sphinx/users-guide/vary.rst
deleted file mode 100644
index 00c946c..0000000
--- a/doc/sphinx/users-guide/vary.rst
+++ /dev/null
@@ -1,77 +0,0 @@
-.. _users-guide-vary:
-
-HTTP Vary
----------
-
-*HTTP Vary is not a trivial concept. It is by far the most
-misunderstood HTTP header.*
-
-A lot of the response headers tell the client something about the HTTP
-object being delivered. Clients can request different variants a an
-HTTP object, based on their preference. Their preferences might cover
-stuff like encoding or language. When a client prefers UK English this
-is indicated through "Accept-Language: en-uk". Caches need to keep
-these different variants apart and this is done through the HTTP
-response header "Vary".
-
-When a backend server issues a "Vary: Accept-Language" it tells
-Varnish that its needs to cache a separate version for every different
-Accept-Language that is coming from the clients.
-
-If two clients say they accept the languages "en-us, en-uk" and "da,
-de" respectively, Varnish will cache and serve two different versions
-of the page if the backend indicated that Varnish needs to vary on the
-Accept-Language header.
-
-Please note that the headers that Vary refer to need to match
-*exactly* for there to be a match. So Varnish will keep two copies of
-a page if one of them was created for "en-us, en-uk" and the other for
-"en-us,en-uk". Just the lack of space will force Varnish to cache
-another version.
-
-To achieve a high hitrate whilst using Vary is there therefor crucial
-to normalize the headers the backends varies on. Remember, just a
-difference in case can force different cache entries.
-
-The following VCL code will normalize the Accept-Language headers, to
-one of either "en","de" or "fr"::
-
-    if (req.http.Accept-Language) {
-        if (req.http.Accept-Language ~ "en") {
-            set req.http.Accept-Language = "en";
-        } elsif (req.http.Accept-Language ~ "de") {
-            set req.http.Accept-Language = "de";
-        } elsif (req.http.Accept-Language ~ "fr") {
-            set req.http.Accept-Language = "fr";
-        } else {
-            # unknown language. Remove the accept-language header and 
-	    # use the backend default.
-            remove req.http.Accept-Language
-        }
-    }
-
-The code sets the Accept-Encoding header from the client to either
-gzip, deflate with a preference for gzip.
-
-Vary parse errors
-~~~~~~~~~~~~~~~~~
-
-Varnish will return a 503 internal server error page when it fails to
-parse the Vary server header, or if any of the client headers listed
-in the Vary header exceeds the limit of 65k characters. An SLT_Error
-log entry is added in these cases.
-
-Pitfall - Vary: User-Agent
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Some applications or application servers send *Vary: User-Agent* along
-with their content. This instructs Varnish to cache a separate copy
-for every variation of User-Agent there is. There are plenty. Even a
-single patchlevel of the same browser will generate at least 10
-different User-Agent headers based just on what operating system they
-are running. 
-
-So if you *really* need to Vary based on User-Agent be sure to
-normalize the header or your hit rate will suffer badly. Use the above
-code as a template.
-



From perbu at varnish-software.com  Thu Mar 13 09:24:30 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 46b5fac Duplicate.
Message-ID: <E1WO1sM-0007ZJ-PQ@project.varnish-software.com>


commit 46b5fac332390aeb410360188757f661d50f4157
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 6 14:29:53 2014 +0100

    Duplicate.

diff --git a/doc/sphinx/users-guide/websockets.rst b/doc/sphinx/users-guide/websockets.rst
deleted file mode 100644
index 7217b88..0000000
--- a/doc/sphinx/users-guide/websockets.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-
-Implementing websocket support
-------------------------------
-
-Websockets is a technology for creating a bidirectional stream-based channel over HTTP.
-
-To run websockets through Varnish you need to pipe it, and copy the Upgrade header. Use the following
-VCL config to do so::
-
-    sub vcl_pipe {
-         if (req.http.upgrade) {
-             set bereq.http.upgrade = req.http.upgrade;
-         }
-    }
-    sub vcl_recv {
-         if (req.http.Upgrade ~ "(?i)websocket") {
-             return (pipe);
-         }
-    }
-



From fgsch at lodoss.net  Thu Mar 13 09:24:30 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 839cdf3 Correct spelling
Message-ID: <E1WO1sM-0007ZS-Qn@project.varnish-software.com>


commit 839cdf399a581d5cd05c46eb68f367dcecb3a104
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 5 21:32:04 2014 +0000

    Correct spelling

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index bf4bcd0..0502c36 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -847,7 +847,7 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 	    vslc_vtx_next(&vtx->c.cursor) == 1) {
 		ptr = vtx->c.cursor.rec.ptr;
 		if (VSL_ID(ptr) != vtx->key.vxid) {
-			(void)vtx_diag_tag(vtx, ptr, "vxid missmatch");
+			(void)vtx_diag_tag(vtx, ptr, "vxid mismatch");
 			continue;
 		}
 



From fgsch at lodoss.net  Thu Mar 13 09:24:30 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 41db0bc Don't expose certain variables in vcl_pipe
Message-ID: <E1WO1sM-0007Zd-Ro@project.varnish-software.com>


commit 41db0bc0b31b46cea33e6d93f598fa210eeb343d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 6 15:36:54 2014 +0000

    Don't expose certain variables in vcl_pipe
    
    Accessing bereq.first_byte_timeout and bereq.between_bytes_timeout
    in vcl_pipe makes no sense so disallow it.
    
    Fixes: #1435

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index b23d21e..4314dce 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -104,3 +104,17 @@ varnish v1 -errvcl {Object not found: 'foo.bar'} {
 		new bar = foo.bar();
 	}
 }
+
+varnish v1 -errvcl {'bereq.first_byte_timeout': cannot be set} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_pipe {
+		set bereq.first_byte_timeout = 10s;
+	}
+}
+
+varnish v1 -errvcl {'bereq.between_bytes_timeout': cannot be set} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_pipe {
+		set bereq.between_bytes_timeout = 10s;
+	}
+}
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index f313bd5..8c9461a 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -351,16 +351,16 @@ The client's IP address.
 	),
 	('bereq.first_byte_timeout',
 		'DURATION',
-		( 'pipe', 'backend', ),
-		( 'pipe', 'backend', ), """
+		( 'backend', ),
+		( 'backend', ), """
 		The time in seconds to wait for the first byte from
 		the backend.  Not available in pipe mode.
 		"""
 	),
 	('bereq.between_bytes_timeout',
 		'DURATION',
-		( 'pipe', 'backend', ),
-		( 'pipe', 'backend', ), """
+		( 'backend', ),
+		( 'backend', ), """
 		The time in seconds to wait between each received byte from the
 		backend.  Not available in pipe mode.
 		"""



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 818f00c Make vmod_priv_fini() a real function and name it
 VRT_priv_fini() for consistency.
Message-ID: <E1WO1sM-0007Zm-TB@project.varnish-software.com>


commit 818f00c637486c17168412fc7c188d9051a31cf6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 07:30:35 2014 +0000

    Make vmod_priv_fini() a real function and name it VRT_priv_fini()
    for consistency.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index ce536e5..f78f140 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -498,6 +498,16 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+void
+VRT_priv_fini(const struct vmod_priv *p)
+{
+	if (p->priv != (void*)0 && p->free != (void*)0)
+		p->free(p->priv);
+}
+
+/*--------------------------------------------------------------------
  * Simple stuff
  */
 
diff --git a/include/vrt.h b/include/vrt.h
index 9b3664a..dc84852 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -246,13 +246,7 @@ struct vmod_priv {
 
 typedef int vmod_init_f(struct vmod_priv *,  const struct VCL_conf *);
 
-static inline void
-vmod_priv_fini(const struct vmod_priv *p)
-{
-
-	if (p->priv != (void*)0 && p->free != (void*)0)
-		p->free(p->priv);
-}
+void VRT_priv_fini(const struct vmod_priv *p);
 
 /* Stevedore related functions */
 int VRT_Stv(const char *nm);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 08f91df..c143cf1 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -561,7 +561,7 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 			bprintf(buf, "vmod_priv_%u", tl->unique++);
 			ifp = New_IniFin(tl);
 			Fh(tl, 0, "static struct vmod_priv %s;\n", buf);
-			VSB_printf(ifp->fin, "\tvmod_priv_fini(&%s);", buf);
+			VSB_printf(ifp->fin, "\tVRT_priv_fini(&%s);", buf);
 			e2 = vcc_mk_expr(VOID, "&%s", buf);
 			p += strlen(p) + 1;
 		} else if (fmt == ENUM) {
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index ac88b59..044cd24 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -119,7 +119,7 @@ vcc_ParseImport(struct vcc *tl)
 	VSB_printf(ifp->ini, "\t\treturn(1);");
 
 	/* XXX: zero the function pointer structure ?*/
-	VSB_printf(ifp->fin, "\tvmod_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
 	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
 
 	ifp = NULL;
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 7109644..146f46b 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -126,6 +126,6 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 		p += 2;
 		q += 1;
 	}
-	vmod_priv_fini(b);
+	VRT_priv_fini(b);
 	return (s);
 }



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 42baff6 Don't merge Content-Encoding from IMS object before VCL,
 in order to not confuse VCL by "backend" suddenly changing gzip status,
 but do unconditionally smash it in there afterwards, no matter what VCL does,
 so it will be correct relative to the existing body.
Message-ID: <E1WO1sM-0007Zv-UU@project.varnish-software.com>


commit 42baff6df2736e86fb7c73a90686b83756575dd1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 09:21:50 2014 +0000

    Don't merge Content-Encoding from IMS object before VCL, in order
    to not confuse VCL by "backend" suddenly changing gzip status, but do
    unconditionally smash it in there afterwards, no matter what VCL
    does, so it will be correct relative to the existing body.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f18c69a..5b4028d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -594,8 +594,9 @@ struct object {
 
 	uint8_t			*vary;
 
-	/* XXX: make bitmap */
-	uint8_t			gziped;
+	unsigned		gziped:1;
+	unsigned		changed_gzip:1;
+
 	/* Bit positions in the gzip stream */
 	ssize_t			gzip_start;
 	ssize_t			gzip_last;
@@ -998,7 +999,7 @@ void http_CopyHome(const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
-void http_Merge(const struct http *fm, struct http *to);
+void http_Merge(const struct http *fm, struct http *to, int not_ce);
 
 /* cache_http1_proto.c */
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 27a2df8..b42674a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -336,7 +336,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->ims_obj != NULL && bo->beresp->status == 304) {
 		bo->beresp->status = 200;
-		http_Merge(bo->ims_obj->http, bo->beresp);
+		http_Merge(bo->ims_obj->http, bo->beresp,
+		    bo->ims_obj->changed_gzip);
 		do_ims = 1;
 	} else
 		do_ims = 0;
@@ -464,6 +465,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
 		obj->gziped = 1;
 
+	if (bo->do_gzip || bo->do_gunzip)
+		obj->changed_gzip = 1;
+
 	/*
 	 * Ready to fetch the body
 	 */
@@ -554,10 +558,22 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	ssize_t sl, al, tl;
 	struct storage *st;
 	enum objiter_status ois;
+	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	if (bo->ims_obj->changed_gzip) {
+		/*
+		 * If we modified the gzip status of the IMS object, that
+		 * must control the C-E header, if any.
+		 */
+		http_Unset(bo->beresp, H_Content_Encoding);
+		if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p))
+			http_PrintfHeader(bo->beresp,
+			    "Content-Encoding: %s", p);
+	}
+
 	AZ(vbf_beresp2obj(bo));
 	obj = bo->fetch_obj;
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 2a9b8de..6898987 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -641,13 +641,19 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
  */
 
 void
-http_Merge(const struct http *fm, struct http *to)
+http_Merge(const struct http *fm, struct http *to, int not_ce)
 {
 	unsigned u, v;
 	const char *p;
 
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
+	if (not_ce) {
+		u = http_findhdr(fm,
+		    H_Content_Encoding[0] - 1, H_Content_Encoding + 1);
+		if (u > 0)
+			fm->hdf[u] &= ~HDF_MARKER;
+	}
 	for (v = HTTP_HDR_FIRST; v < to->nhd; v++) {
 		p = strchr(to->hd[v].b, ':');
 		AN(p);
diff --git a/bin/varnishtest/tests/g00006.vtc b/bin/varnishtest/tests/g00006.vtc
new file mode 100644
index 0000000..740d4bd
--- /dev/null
+++ b/bin/varnishtest/tests/g00006.vtc
@@ -0,0 +1,72 @@
+varnishtest "IMS'ing g[un]zip'ed objects"
+
+server s1 {
+	rxreq
+	expect req.url == /1
+	txresp -hdr  "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -bodylen 20
+
+	rxreq
+	expect req.url == /1
+	expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT"
+	txresp -status 304 -nolen
+
+	rxreq
+	expect req.url == /2
+	txresp -hdr  "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -gzipbody "012345678901234567"
+
+	rxreq
+	expect req.url == /2
+	expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT"
+	txresp -status 304 -hdr "Content-Encoding: gzip,rot13" -nolen
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.http.foobar = beresp.http.content-encoding;
+		if (bereq.url == "/1") {
+			set beresp.do_gzip = true;
+		} else {
+			set beresp.do_gunzip = true;
+		}
+		set beresp.ttl = 1s;
+		set beresp.grace = 0s;
+		set beresp.keep = 60s;
+	}
+} -start
+
+client c1 {
+	txreq -url /1 -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	expect resp.http.foobar == ""
+	gunzip
+	expect resp.bodylen == 20
+
+	delay 1
+
+	txreq -url /1 -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	expect resp.http.foobar == ""
+	gunzip
+	expect resp.bodylen == 20
+
+	delay .2
+
+	txreq -url /2
+	rxresp
+	expect resp.http.content-encoding == "<undef>"
+	expect resp.http.foobar == "gzip"
+	expect resp.bodylen == 18
+
+	delay 1
+
+	txreq -url /2
+	rxresp
+	expect resp.http.content-encoding == "<undef>"
+	# Here we see the C-E of the IMS OBJ
+	expect resp.http.foobar == "gzip,rot13"
+	expect resp.bodylen == 18
+
+} -run



From phk at FreeBSD.org  Thu Mar 13 09:24:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:30 +0100
Subject: [4.0] 3edd1f7 Make synth {} work in vcl_backend_response{}
Message-ID: <E1WO1sM-0007a7-Vh@project.varnish-software.com>


commit 3edd1f7b0d83f9baaa20c01109e8c0e4bd43c20b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 14:36:45 2014 +0000

    Make synth {} work in vcl_backend_response{}

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index a6a126d..a0eb015 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -182,6 +182,26 @@ sub vcl_backend_response {
 }
 
 sub vcl_backend_error {
+    set beresp.http.Content-Type = "text/html; charset=utf-8";
+    set beresp.http.Retry-After = "5";
+    synthetic {"
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html>
+  <head>
+    <title>"} + beresp.status + " " + beresp.reason + {"</title>
+  </head>
+  <body>
+    <h1>Error "} + beresp.status + " " + beresp.reason + {"</h1>
+    <p>"} + beresp.reason + {"</p>
+    <h3>Guru Meditation:</h3>
+    <p>XID: "} + bereq.xid + {"</p>
+    <hr>
+    <p>Varnish cache server</p>
+  </body>
+</html>
+"};
     return (deliver);
 }
 
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5b4028d..a954316 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -579,6 +579,8 @@ struct busyobj {
 
 	/* Workspace for object only needed during fetch */
 	struct ws		ws_o[1];
+
+	struct vsb		*synth_body;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b42674a..9368385 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -640,28 +640,33 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 {
+	struct storage *st;
+	ssize_t l;
+
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
+	AZ(bo->synth_body);
+	bo->synth_body = VSB_new_auto();
+	AN(bo->synth_body);
+
 	// XXX: reset all beresp flags ?
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
-	http_SetHeader(bo->beresp, "Content-Length: 0");
-	http_SetHeader(bo->beresp, "Connection: close");
 
+	bo->exp.t_origin = VTIM_real();
 	bo->exp.ttl = 0;
 	bo->exp.grace = 0;
 	bo->exp.keep = 0;
 
 	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
-	xxxassert(wrk->handling == VCL_RET_DELIVER);
+	AZ(VSB_finish(bo->synth_body));
 
-	http_PrintfHeader(bo->beresp, "Content-Length: %jd", (intmax_t)0);
-	http_PrintfHeader(bo->beresp, "X-XXXPHK: yes");
+	xxxassert(wrk->handling == VCL_RET_DELIVER);
 
 	if (vbf_beresp2obj(bo)) {
 		VBO_setstate(bo, BOS_FAILED);
@@ -669,9 +674,29 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_DONE);
 	}
 
+	l = VSB_len(bo->synth_body);
+	if (l > 0) {
+		st = VFP_GetStorage(bo, l);
+		if (st != NULL) {
+			if (st->space < l) {
+				VSLb(bo->vsl, SLT_Error,
+				    "No space for %zd bytes of synth body", l);
+			} else {
+				memcpy(st->ptr, VSB_data(bo->synth_body), l);
+				st->len = l;
+				VBO_extend(bo, l);
+			}
+		}
+	}
+	VSB_delete(bo->synth_body);
+	bo->synth_body = NULL;
+
 	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
+
 	VBO_setstate(bo, BOS_FINISHED);
+
 	HSH_Complete(bo->fetch_obj->objcore);
+
 	return (F_STP_DONE);
 }
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index f78f140..95a37b0 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -377,9 +377,14 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 
 	(void)flags;
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
-	vsb = SMS_Makesynth(ctx->req->obj);
+	if (ctx->method == VCL_MET_BACKEND_ERROR) {
+		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+		vsb = ctx->bo->synth_body;
+	} else {
+		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+		CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
+		vsb = SMS_Makesynth(ctx->req->obj);
+	}
 	AN(vsb);
 
 	va_start(ap, str);
@@ -391,8 +396,10 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	SMS_Finish(ctx->req->obj);
-	http_Unset(ctx->req->obj->http, H_Content_Length);
+	if (ctx->method != VCL_MET_BACKEND_ERROR) {
+		SMS_Finish(ctx->req->obj);
+		http_Unset(ctx->req->obj->http, H_Content_Length);
+	}
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 2c79c0e..b6e2835 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -466,7 +466,7 @@ VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0, ctx->bo->exp.t_origin,)
 VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0, ctx->bo->exp.t_origin,)
 
 /*--------------------------------------------------------------------
- * req.xid
+ * [be]req.xid
  */
 
 const char *
@@ -480,6 +480,17 @@ VRT_r_req_xid(const struct vrt_ctx *ctx)
 	    ctx->req->vsl->wid & VSL_IDENTMASK));
 }
 
+const char *
+VRT_r_bereq_xid(const struct vrt_ctx *ctx)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+
+	return (WS_Printf(ctx->bo->bereq->ws, "%u",
+	    ctx->bo->vsl->wid & VSL_IDENTMASK));
+}
+
 /*--------------------------------------------------------------------*/
 
 #define REQ_BOOL(hash_var)					\
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 8c9461a..c3f26ff 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -296,6 +296,13 @@ The client's IP address.
 		always (re)fetch from the backend.
 		"""
 	),
+	('bereq.xid',
+		'STRING',
+		( 'backend',),
+		( ), """
+		Unique ID of this request.
+		"""
+	),
 	('bereq.retries',
 		'INT',
 		( 'backend',),
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 35f88f8..0026ad4 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -421,7 +421,8 @@ static struct action_table {
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },
-	{ "synthetic",		parse_synthetic, VCL_MET_ERROR },
+	{ "synthetic",		parse_synthetic,
+		VCL_MET_ERROR | VCL_MET_BACKEND_ERROR },
 	{ "unset",		parse_unset },
 	{ NULL,			NULL }
 };



From phk at FreeBSD.org  Thu Mar 13 09:24:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 5cf517e Change the way we build synth content for vcl_error{}.
Message-ID: <E1WO1sN-0007aH-13@project.varnish-software.com>


commit 5cf517e858b212a37a106a8ae78ffabe13a0dd5b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 19:59:24 2014 +0000

    Change the way we build synth content for vcl_error{}.
    
    This eliminates the need for the SMS stevedore.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 8cd8cf5..d9a8e62 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -78,7 +78,6 @@ varnishd_SOURCES = \
 	storage/storage_persistent_mgt.c \
 	storage/storage_persistent_silo.c \
 	storage/storage_persistent_subr.c \
-	storage/storage_synth.c \
 	storage/storage_umem.c \
 	waiter/mgt_waiter.c \
 	waiter/cache_waiter.c \
diff --git a/bin/varnishd/Makefile.phk b/bin/varnishd/Makefile.phk
index 1cbd5c2..f8bf48a 100644
--- a/bin/varnishd/Makefile.phk
+++ b/bin/varnishd/Makefile.phk
@@ -73,7 +73,6 @@ PROG_SRC += storage/storage_persistent.c
 PROG_SRC += storage/storage_persistent_mgt.c
 PROG_SRC += storage/storage_persistent_silo.c
 PROG_SRC += storage/storage_persistent_subr.c
-PROG_SRC += storage/storage_synth.c
 PROG_SRC += storage/storage_umem.c
 
 PROG_SRC += waiter/cache_waiter.c
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a954316..e0f45c9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -722,6 +722,9 @@ struct req {
 
 	/* Temporary accounting */
 	struct acct		acct_req;
+
+	/* Synth content in vcl_error */
+	struct vsb		*synth_body;
 };
 
 /*--------------------------------------------------------------------
@@ -1226,11 +1229,6 @@ int STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len);
 void STV_BanExport(const uint8_t *bans, unsigned len);
 struct storage *STV_alloc_transient(size_t size);
 
-/* storage_synth.c */
-struct vsb *SMS_Makesynth(struct object *obj);
-void SMS_Finish(struct object *obj);
-void SMS_Init(void);
-
 /* storage_persistent.c */
 void SMP_Init(void);
 void SMP_Ready(void);
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 8e116c9..a5674c0 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -226,7 +226,6 @@ child_main(void)
 
 	VCA_Init();
 
-	SMS_Init();
 	SMP_Init();
 	STV_open();
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8696dad..f5ae189 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -190,6 +190,8 @@ cnt_error(struct worker *wrk, struct req *req)
 	struct http *h;
 	struct busyobj *bo;
 	char date[40];
+	ssize_t l;
+	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -205,7 +207,6 @@ cnt_error(struct worker *wrk, struct req *req)
 	    TRANSIENT_STORAGE, cache_param->http_resp_size,
 	    (uint16_t)cache_param->http_max_hdr);
 	req->obj = bo->fetch_obj;
-	bo->stats = NULL;
 	if (req->obj == NULL) {
 		req->doclose = SC_OVERLOAD;
 		req->director_hint = NULL;
@@ -213,6 +214,7 @@ cnt_error(struct worker *wrk, struct req *req)
 		bo->fetch_objcore = NULL;
 		http_Teardown(bo->beresp);
 		http_Teardown(bo->bereq);
+		bo->stats = NULL;
 		VBO_DerefBusyObj(wrk, &bo);
 		return (REQ_FSM_DONE);
 	}
@@ -239,13 +241,25 @@ cnt_error(struct worker *wrk, struct req *req)
 		http_PutResponse(h, req->err_reason);
 	else
 		http_PutResponse(h, http_StatusMessage(req->err_code));
+
+	AZ(req->synth_body);
+	req->synth_body = VSB_new_auto();
+	AN(req->synth_body);
+
 	VCL_error_method(req->vcl, wrk, req, NULL, req->http->ws);
 
+	http_Unset(req->obj->http, H_Content_Length);
+
+	AZ(VSB_finish(req->synth_body));
+
 	/* Stop the insanity before it turns "Hotel California" on us */
 	if (req->restarts >= cache_param->max_restarts)
 		wrk->handling = VCL_RET_DELIVER;
 
 	if (wrk->handling == VCL_RET_RESTART) {
+		VSB_delete(req->synth_body);
+		req->synth_body = NULL;
+		bo->stats = NULL;
 		VBO_DerefBusyObj(wrk, &bo);
 		HSH_Drop(wrk, &req->obj);
 		req->req_step = R_STP_RESTART;
@@ -258,10 +272,28 @@ cnt_error(struct worker *wrk, struct req *req)
 	req->wantbody = 1;
 
 	assert(wrk->handling == VCL_RET_DELIVER);
+
+	l = VSB_len(req->synth_body);
+	if (l > 0) {
+		st = STV_alloc(bo, l);
+		if (st != NULL) {
+			VTAILQ_INSERT_TAIL(&req->obj->store, st, list);
+			if (st->space >= l) {
+				memcpy(st->ptr, VSB_data(req->synth_body), l);
+				st->len = l;
+				req->obj->len = l;
+			}
+		}
+	}
+
+	VSB_delete(req->synth_body);
+	req->synth_body = NULL;
+
 	req->err_code = 0;
 	req->err_reason = NULL;
 	http_Teardown(bo->bereq);
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	bo->stats = NULL;
 	VBO_DerefBusyObj(wrk, &bo);
 	req->req_step = R_STP_DELIVER;
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 95a37b0..c423eb2 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -369,21 +369,19 @@ VRT_Rollback(const struct vrt_ctx *ctx)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
+VRT_synth_page(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
 	const char *p;
 	struct vsb *vsb;
 
-	(void)flags;
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (ctx->method == VCL_MET_BACKEND_ERROR) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 		vsb = ctx->bo->synth_body;
 	} else {
 		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-		CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
-		vsb = SMS_Makesynth(ctx->req->obj);
+		vsb = ctx->req->synth_body;
 	}
 	AN(vsb);
 
@@ -396,10 +394,6 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	if (ctx->method != VCL_MET_BACKEND_ERROR) {
-		SMS_Finish(ctx->req->obj);
-		http_Unset(ctx->req->obj->http, H_Content_Length);
-	}
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage/storage_synth.c b/bin/varnishd/storage/storage_synth.c
deleted file mode 100644
index ba3fa24..0000000
--- a/bin/varnishd/storage/storage_synth.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Storage method for synthetic content, based on vsb.
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache/cache.h"
-#include "storage/storage.h"
-
-
-static struct lock		sms_mtx;
-
-static void
-sms_free(struct storage *sto)
-{
-
-	CHECK_OBJ_NOTNULL(sto, STORAGE_MAGIC);
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nobj--;
-	VSC_C_main->sms_nbytes -= sto->len;
-	VSC_C_main->sms_bfree += sto->len;
-	Lck_Unlock(&sms_mtx);
-	VSB_delete(sto->priv);
-	free(sto);
-}
-
-void
-SMS_Init(void)
-{
-
-	Lck_New(&sms_mtx, lck_sms);
-}
-
-static struct stevedore sms_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"synth",
-	.free	=	sms_free,
-};
-
-struct vsb *
-SMS_Makesynth(struct object *obj)
-{
-	struct storage *sto;
-	struct vsb *vsb;
-
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	STV_Freestore(obj);
-	obj->len = 0;
-
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nreq++;
-	VSC_C_main->sms_nobj++;
-	Lck_Unlock(&sms_mtx);
-
-	sto = calloc(sizeof *sto, 1);
-	XXXAN(sto);
-	vsb = VSB_new_auto();
-	XXXAN(vsb);
-	sto->priv = vsb;
-	sto->len = 0;
-	sto->space = 0;
-	sto->stevedore = &sms_stevedore;
-	sto->magic = STORAGE_MAGIC;
-
-	VTAILQ_INSERT_TAIL(&obj->store, sto, list);
-	return (vsb);
-}
-
-void
-SMS_Finish(struct object *obj)
-{
-	struct storage *sto;
-	struct vsb *vsb;
-
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	sto = VTAILQ_FIRST(&obj->store);
-	assert(sto->stevedore == &sms_stevedore);
-	vsb = sto->priv;
-	AZ(VSB_finish(vsb));
-
-	sto->ptr = (void*)VSB_data(vsb);
-	sto->len = VSB_len(vsb);
-	sto->space = VSB_len(vsb);
-	obj->len = sto->len;
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nbytes += sto->len;
-	VSC_C_main->sms_balloc += sto->len;
-	Lck_Unlock(&sms_mtx);
-}
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index 633e6e1..0912e64 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -7,12 +7,18 @@ varnish v1 -vcl {
 	sub vcl_recv {
 		return (error(888));
 	}
+
+	sub vcl_error {
+		synthetic "Custom vcl_error's synth output";
+		return (deliver);
+	}
 } -start
 
 client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 888
+	expect resp.bodylen == 31
 	expect resp.http.connection == close
 } -run
 
diff --git a/include/vrt.h b/include/vrt.h
index dc84852..8e5acba 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -222,7 +222,7 @@ void VRT_memmove(void *dst, const void *src, unsigned len);
 void VRT_Rollback(const struct vrt_ctx *);
 
 /* Synthetic pages */
-void VRT_synth_page(const struct vrt_ctx *, unsigned flags, const char *, ...);
+void VRT_synth_page(const struct vrt_ctx *, const char *, ...);
 
 /* Backend related */
 void VRT_init_dir(struct cli *, struct director **, int idx, const void *priv);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 0026ad4..ff5f0f4 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -396,7 +396,7 @@ parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_synth_page(ctx, 0, ");
+	Fb(tl, 1, "VRT_synth_page(ctx, ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");



From phk at FreeBSD.org  Thu Mar 13 09:24:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 2eabf10 Require the "synthetic" argument to be function-like:
Message-ID: <E1WO1sN-0007aR-2V@project.varnish-software.com>


commit 2eabf10ffa3b7d8c75bf01336f671e872a2832ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 10 20:27:12 2014 +0000

    Require the "synthetic" argument to be function-like:
    
    	synthetic ( STRING_LIST ) ;

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index a0eb015..96d74ea 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -137,7 +137,7 @@ sub vcl_deliver {
 sub vcl_error {
     set obj.http.Content-Type = "text/html; charset=utf-8";
     set obj.http.Retry-After = "5";
-    synthetic {"
+    synthetic( {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
@@ -154,7 +154,7 @@ sub vcl_error {
     <p>Varnish cache server</p>
   </body>
 </html>
-"};
+"} );
     return (deliver);
 }
 
@@ -184,7 +184,7 @@ sub vcl_backend_response {
 sub vcl_backend_error {
     set beresp.http.Content-Type = "text/html; charset=utf-8";
     set beresp.http.Retry-After = "5";
-    synthetic {"
+    synthetic( {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
@@ -201,7 +201,7 @@ sub vcl_backend_error {
     <p>Varnish cache server</p>
   </body>
 </html>
-"};
+"} );
     return (deliver);
 }
 
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index 0912e64..faac1e8 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_error {
-		synthetic "Custom vcl_error's synth output";
+		synthetic("Custom vcl_error's synth output");
 		return (deliver);
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00936.vtc b/bin/varnishtest/tests/r00936.vtc
index 99e1d65..7e6ad64 100644
--- a/bin/varnishtest/tests/r00936.vtc
+++ b/bin/varnishtest/tests/r00936.vtc
@@ -9,7 +9,8 @@ varnish v1 -errvcl {'synthetic': not a valid action in method 'vcl_recv'} {
 	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_recv {
-		synthetic "HELLOO"; return (error(503));
+		synthetic("HELLOO");
+		return (error(503));
 	}
 }
 
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
index 8560ce3..fc1ebd2 100644
--- a/bin/varnishtest/tests/r01287.vtc
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -8,7 +8,7 @@ varnish v1 -vcl+backend {
 		return (error(200, "OK"));
 	}
 	sub vcl_error {
-		synthetic obj.http.blank;
+		synthetic(obj.http.blank);
 		return (deliver);
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 720c816..9e97762 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -100,7 +100,7 @@ varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
 
 varnish v1 -errvcl {Unknown token '<<' when looking for STRING_LIST} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_error { synthetic << "foo"; }
+	sub vcl_error { synthetic( << "foo"; }
 }
 
 varnish v1 -errvcl {Syntax has changed, use:} {
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index ff5f0f4..4e3f12c 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -396,10 +396,18 @@ parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
 
+	ExpectErr(tl, '(');
+	ERRCHK(tl);
+	vcc_NextToken(tl);
+
 	Fb(tl, 1, "VRT_synth_page(ctx, ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");
+
+	ExpectErr(tl, ')');
+	vcc_NextToken(tl);
+	ERRCHK(tl);
 }
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Thu Mar 13 09:24:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 33d80ab Implement "return(retry)" in vcl_backend_error{}
Message-ID: <E1WO1sN-0007ac-3v@project.varnish-software.com>


commit 33d80aba55165ea35af1a182b2b6a45303075be0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 09:07:53 2014 +0000

    Implement "return(retry)" in vcl_backend_error{}

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9368385..c3cefac 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -666,7 +666,20 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	AZ(VSB_finish(bo->synth_body));
 
-	xxxassert(wrk->handling == VCL_RET_DELIVER);
+	if (wrk->handling == VCL_RET_RETRY) {
+		VSB_delete(bo->synth_body);
+		bo->synth_body = NULL;
+	    	if (bo->retries++ < cache_param->max_retries) {
+			return (F_STP_RETRY);
+		}
+		bo->synth_body = NULL;
+		HSH_Fail(bo->fetch_objcore);
+		VBO_setstate(bo, BOS_FAILED);
+		HSH_Complete(bo->fetch_objcore);
+		return (F_STP_DONE);
+	}
+
+	assert(wrk->handling == VCL_RET_DELIVER);
 
 	if (vbf_beresp2obj(bo)) {
 		VBO_setstate(bo, BOS_FAILED);
diff --git a/bin/varnishtest/tests/c00061.vtc b/bin/varnishtest/tests/c00061.vtc
new file mode 100644
index 0000000..ecb01a4
--- /dev/null
+++ b/bin/varnishtest/tests/c00061.vtc
@@ -0,0 +1,24 @@
+varnishtest "retry in vcl_backend_error"
+
+varnish v1 -vcl {
+
+	backend b1 { .host = "${bad_ip}"; }
+
+	sub vcl_backend_error {
+		return (retry);
+	}
+
+	sub vcl_error {
+		set obj.status = 504;
+	}
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set connect_timeout 1"
+varnish v1 -cliok "param.set max_retries 2"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 504
+} -run



From fgsch at lodoss.net  Thu Mar 13 09:24:31 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 6b2faff Remove user-guides/cookies.rst from here as well
Message-ID: <E1WO1sN-0007al-50@project.varnish-software.com>


commit 6b2faff67e21d6ff4cf2a8e75a243bbf8d537d6d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 09:50:47 2014 +0000

    Remove user-guides/cookies.rst from here as well

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index ac09cc0..3a6bb2b 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -138,7 +138,6 @@ EXTRA_DIST = \
 	tutorial/now_what.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
-	users-guide/cookies.rst \
 	users-guide/devicedetection.rst \
 	users-guide/esi.rst \
 	users-guide/increasing-your-hitrate.rst \
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index fea357f..d89f4e5 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -147,7 +147,6 @@ EXTRA_DIST = \
 	tutorial/now_what.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
-	users-guide/cookies.rst \
 	users-guide/devicedetection.rst \
 	users-guide/esi.rst \
 	users-guide/increasing-your-hitrate.rst \



From fgsch at lodoss.net  Thu Mar 13 09:24:31 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 03ff391 Remove more nonexistent rst files to please make dist
 and jenkins
Message-ID: <E1WO1sN-0007au-6H@project.varnish-software.com>


commit 03ff3917bff138419484fa27e6c53cf274824e60
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 10:19:26 2014 +0000

    Remove more nonexistent rst files to please make dist and jenkins

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 3a6bb2b..f36f569 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -153,7 +153,6 @@ EXTRA_DIST = \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
-	users-guide/vary.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
 	users-guide/vcl-examples.rst \
@@ -162,8 +161,7 @@ EXTRA_DIST = \
 	users-guide/vcl.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst \
-	users-guide/websockets.rst
+	users-guide/vcl-variables.rst
 
 dist-hook:
 	$(MAKE) html
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index d89f4e5..611ff18 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -163,7 +163,6 @@ EXTRA_DIST = \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
-	users-guide/vary.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
 	users-guide/vcl-examples.rst \
@@ -172,9 +171,7 @@ EXTRA_DIST = \
 	users-guide/vcl.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst \
-	users-guide/websockets.rst
-
+	users-guide/vcl-variables.rst
 
 dist-hook:
 	$(MAKE) html



From phk at FreeBSD.org  Thu Mar 13 09:24:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 94e99cb Add a V1D_Deliver_Synth() for delivering synth
 vcl_error{} responses.
Message-ID: <E1WO1sN-0007b3-7F@project.varnish-software.com>


commit 94e99cb511c5d50108636197cb0001d5b9b32c92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 10:04:01 2014 +0000

    Add a V1D_Deliver_Synth() for delivering synth vcl_error{} responses.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e0f45c9..94fdf32 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -855,6 +855,8 @@ int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
 
 /* cache_http1_deliver.c */
 void V1D_Deliver(struct req *);
+void V1D_Deliver_Synth(struct req *req);
+
 
 static inline int
 VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len)
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index c9f373a..5a3da11 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -271,7 +271,7 @@ V1D_Deliver(struct req *req)
 	    req->wantbody &&
 	    !(req->res_mode & (RES_ESI|RES_ESI_CHILD)) &&
 	    cache_param->http_range_support &&
-	    http_GetStatus(req->obj->http) == 200) {
+	    http_GetStatus(req->resp) == 200) {
 		http_SetHeader(req->resp, "Accept-Ranges: bytes");
 		if (http_GetHdr(req->http, H_Range, &r))
 			v1d_dorange(req, r);
@@ -318,3 +318,101 @@ V1D_Deliver(struct req *req)
 	if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0)
 		SES_Close(req->sp, SC_REM_CLOSE);
 }
+
+void
+V1D_Deliver_Synth(struct req *req)
+{
+	char *r;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(req->obj);
+	AN(req->synth_body);
+
+	req->res_mode = 0;
+	if (req->resp->status == 304) {
+		req->res_mode &= ~RES_LEN;
+		http_Unset(req->resp, H_Content_Length);
+		req->wantbody = 0;
+	} else {
+		req->res_mode |= RES_LEN;
+		http_Unset(req->resp, H_Content_Length);
+		http_PrintfHeader(req->resp, "Content-Length: %zd",
+		    VSB_len(req->synth_body));
+	}
+
+	if (req->esi_level > 0) {
+		/* Included ESI object, always CHUNKED or EOF */
+		req->res_mode &= ~RES_LEN;
+		req->res_mode |= RES_ESI_CHILD;
+	}
+
+	if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
+		/* We havn't chosen yet, do so */
+		if (!req->wantbody) {
+			/* Nothing */
+		} else if (req->http->protover >= 11) {
+			req->res_mode |= RES_CHUNKED;
+		} else {
+			req->res_mode |= RES_EOF;
+			req->doclose = SC_TX_EOF;
+		}
+	}
+	VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
+
+	if (!(req->res_mode & RES_LEN))
+		http_Unset(req->resp, H_Content_Length);
+
+	if (req->res_mode & RES_GUNZIP)
+		http_Unset(req->resp, H_Content_Encoding);
+
+	if (req->res_mode & RES_CHUNKED)
+		http_SetHeader(req->resp, "Transfer-Encoding: chunked");
+
+	http_SetHeader(req->resp,
+	    req->doclose ? "Connection: close" : "Connection: keep-alive");
+
+	req->vdps[0] = v1d_bytes;
+	req->vdp_nxt = 0;
+
+	if (
+	    req->wantbody &&
+	    !(req->res_mode & RES_ESI_CHILD) &&
+	    cache_param->http_range_support &&
+	    http_GetStatus(req->resp) == 200) {
+		http_SetHeader(req->resp, "Accept-Ranges: bytes");
+		if (http_GetHdr(req->http, H_Range, &r))
+			v1d_dorange(req, r);
+	}
+
+	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp);
+
+	/*
+	 * Send HTTP protocol header, unless interior ESI object
+	 */
+	if (!(req->res_mode & RES_ESI_CHILD))
+		req->acct_req.hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+
+	if (req->res_mode & RES_CHUNKED)
+		WRW_Chunked(req->wrk);
+
+	if (!req->wantbody) {
+		/* This was a HEAD or conditional request */
+#if 0
+	XXX: Missing pretend GZIP for esi-children
+	} else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) {
+		while (req->obj->objcore->busyobj)
+			(void)usleep(10000);
+		ESI_DeliverChild(req);
+#endif
+	} else {
+		(void)VDP_bytes(req, VDP_FLUSH, VSB_data(req->synth_body),
+		    VSB_len(req->synth_body));
+		(void)VDP_bytes(req, VDP_FINISH,  NULL, 0);
+	}
+
+	if (req->res_mode & RES_CHUNKED && !(req->res_mode & RES_ESI_CHILD))
+		WRW_EndChunk(req->wrk);
+
+	if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0)
+		SES_Close(req->sp, SC_REM_CLOSE);
+}



From phk at FreeBSD.org  Thu Mar 13 09:24:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] a0e68b3 Change vcl_error{} to do a direct response without
 going the detour over a bogo-object.
Message-ID: <E1WO1sN-0007bD-8M@project.varnish-software.com>


commit a0e68b31df6eeba8634e1a436cccb176e889a33f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 10:48:53 2014 +0000

    Change vcl_error{} to do a direct response without going the detour
    over a bogo-object.
    
    This means that vcl_error{} looses access to obj.* variables, but
    gains access to resp.* vaiables instead.
    
    A couple of now obsolute test-cases retired.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 96d74ea..da30346 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -135,19 +135,19 @@ sub vcl_deliver {
  * We can come here "invisibly" with the following errors: 413, 417 & 503
  */
 sub vcl_error {
-    set obj.http.Content-Type = "text/html; charset=utf-8";
-    set obj.http.Retry-After = "5";
+    set resp.http.Content-Type = "text/html; charset=utf-8";
+    set resp.http.Retry-After = "5";
     synthetic( {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
   <head>
-    <title>"} + obj.status + " " + obj.reason + {"</title>
+    <title>"} + resp.status + " " + resp.reason + {"</title>
   </head>
   <body>
-    <h1>Error "} + obj.status + " " + obj.reason + {"</h1>
-    <p>"} + obj.reason + {"</p>
+    <h1>Error "} + resp.status + " " + resp.reason + {"</h1>
+    <p>"} + resp.reason + {"</p>
     <h3>Guru Meditation:</h3>
     <p>XID: "} + req.xid + {"</p>
     <hr>
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c3cefac..3aabd8d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -669,9 +669,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		VSB_delete(bo->synth_body);
 		bo->synth_body = NULL;
-	    	if (bo->retries++ < cache_param->max_retries) {
+		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
-		}
 		bo->synth_body = NULL;
 		HSH_Fail(bo->fetch_objcore);
 		VBO_setstate(bo, BOS_FAILED);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f5ae189..467ea8d 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -187,56 +187,29 @@ DOT }
 static enum req_fsm_nxt
 cnt_error(struct worker *wrk, struct req *req)
 {
-	struct http *h;
-	struct busyobj *bo;
 	char date[40];
-	ssize_t l;
-	struct storage *st;
+	struct http *h;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AZ(req->objcore);
-	AZ(req->obj);
 
 	req->acct_req.error++;
-	bo = VBO_GetBusyObj(wrk, req);
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
-	bo->fetch_objcore = HSH_Private(wrk);
-	bo->fetch_obj = STV_NewObject(bo,
-	    TRANSIENT_STORAGE, cache_param->http_resp_size,
-	    (uint16_t)cache_param->http_max_hdr);
-	req->obj = bo->fetch_obj;
-	if (req->obj == NULL) {
-		req->doclose = SC_OVERLOAD;
-		req->director_hint = NULL;
-		AZ(HSH_DerefObjCore(&wrk->stats, &bo->fetch_objcore));
-		bo->fetch_objcore = NULL;
-		http_Teardown(bo->beresp);
-		http_Teardown(bo->bereq);
-		bo->stats = NULL;
-		VBO_DerefBusyObj(wrk, &bo);
-		return (REQ_FSM_DONE);
-	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	AZ(req->objcore);
-	req->obj->vxid = bo->vsl->wid;
-	req->obj->exp.t_origin = req->t_req;
 
-	h = req->obj->http;
+	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
+	h = req->resp;
+	req->t_resp = VTIM_real();
 
 	if (req->err_code < 100 || req->err_code > 999)
 		req->err_code = 501;
 
+	http_ClrHeader(h);
 	http_PutProtocol(h, "HTTP/1.1");
 	http_PutStatus(h, req->err_code);
-	VTIM_format(W_TIM_real(wrk), date);
+	VTIM_format(req->t_resp, date);
 	http_PrintfHeader(h, "Date: %s", date);
 	http_SetHeader(h, "Server: Varnish");
-
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	HSH_Ref(req->obj->objcore);
-
+	http_PrintfHeader(req->resp,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 	if (req->err_reason != NULL)
 		http_PutResponse(h, req->err_reason);
 	else
@@ -248,56 +221,27 @@ cnt_error(struct worker *wrk, struct req *req)
 
 	VCL_error_method(req->vcl, wrk, req, NULL, req->http->ws);
 
-	http_Unset(req->obj->http, H_Content_Length);
+	http_Unset(h, H_Content_Length);
 
 	AZ(VSB_finish(req->synth_body));
 
-	/* Stop the insanity before it turns "Hotel California" on us */
-	if (req->restarts >= cache_param->max_restarts)
-		wrk->handling = VCL_RET_DELIVER;
-
 	if (wrk->handling == VCL_RET_RESTART) {
+		http_ClrHeader(h);
 		VSB_delete(req->synth_body);
 		req->synth_body = NULL;
-		bo->stats = NULL;
-		VBO_DerefBusyObj(wrk, &bo);
-		HSH_Drop(wrk, &req->obj);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-
-	/* We always close when we take this path */
-	req->doclose = SC_TX_ERROR;
-	req->wantbody = 1;
-
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	l = VSB_len(req->synth_body);
-	if (l > 0) {
-		st = STV_alloc(bo, l);
-		if (st != NULL) {
-			VTAILQ_INSERT_TAIL(&req->obj->store, st, list);
-			if (st->space >= l) {
-				memcpy(st->ptr, VSB_data(req->synth_body), l);
-				st->len = l;
-				req->obj->len = l;
-			}
-		}
-	}
+	V1D_Deliver_Synth(req);
 
 	VSB_delete(req->synth_body);
 	req->synth_body = NULL;
 
 	req->err_code = 0;
 	req->err_reason = NULL;
-	http_Teardown(bo->bereq);
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	bo->stats = NULL;
-	VBO_DerefBusyObj(wrk, &bo);
-	req->req_step = R_STP_DELIVER;
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	return (REQ_FSM_MORE);
+	return (REQ_FSM_DONE);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b6e2835..8a999c9 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -62,7 +62,7 @@ vrt_do_string(const struct http *hp, int fld,
 	va_end(ap);
 }
 
-#define VRT_DO_HDR(obj, hdr, fld)					\
+#define VRT_HDR_L(obj, hdr, fld)					\
 void									\
 VRT_l_##obj##_##hdr(const struct vrt_ctx *ctx, const char *p, ...)	\
 {									\
@@ -72,8 +72,9 @@ VRT_l_##obj##_##hdr(const struct vrt_ctx *ctx, const char *p, ...)	\
 	va_start(ap, p);						\
 	vrt_do_string(ctx->http_##obj, fld, #obj "." #hdr, p, ap);	\
 	va_end(ap);							\
-}									\
-									\
+}
+
+#define VRT_HDR_R(obj, hdr, fld)					\
 const char *								\
 VRT_r_##obj##_##hdr(const struct vrt_ctx *ctx)				\
 {									\
@@ -82,7 +83,11 @@ VRT_r_##obj##_##hdr(const struct vrt_ctx *ctx)				\
 	return (ctx->http_##obj->hd[fld].b);				\
 }
 
-#define VRT_DO_STATUS(obj)						\
+#define VRT_HDR_LR(obj, hdr, fld)					\
+	VRT_HDR_L(obj, hdr, fld)					\
+	VRT_HDR_R(obj, hdr, fld)
+
+#define VRT_STATUS_L(obj)						\
 void									\
 VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 {									\
@@ -91,8 +96,9 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 	CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC);			\
 	assert(num >= 100 && num <= 999);				\
 	ctx->http_##obj->status = (uint16_t)num;			\
-}									\
-									\
+}
+
+#define VRT_STATUS_R(obj)						\
 long									\
 VRT_r_##obj##_status(const struct vrt_ctx *ctx)				\
 {									\
@@ -102,22 +108,26 @@ VRT_r_##obj##_status(const struct vrt_ctx *ctx)				\
 	return(ctx->http_##obj->status);				\
 }
 
-VRT_DO_HDR(req,    method,	HTTP_HDR_METHOD)
-VRT_DO_HDR(req,    url,		HTTP_HDR_URL)
-VRT_DO_HDR(req,    proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,    proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,    reason,	HTTP_HDR_RESPONSE)
-VRT_DO_STATUS(obj)
-VRT_DO_HDR(resp,   proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(resp,   reason,	HTTP_HDR_RESPONSE)
-VRT_DO_STATUS(resp)
-
-VRT_DO_HDR(bereq,  method,	HTTP_HDR_METHOD)
-VRT_DO_HDR(bereq,  url,		HTTP_HDR_URL)
-VRT_DO_HDR(bereq,  proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp, proto,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp, reason,	HTTP_HDR_RESPONSE)
-VRT_DO_STATUS(beresp)
+VRT_HDR_LR(req,    method,	HTTP_HDR_METHOD)
+VRT_HDR_LR(req,    url,		HTTP_HDR_URL)
+VRT_HDR_LR(req,    proto,	HTTP_HDR_PROTO)
+
+VRT_HDR_R(obj,    proto,	HTTP_HDR_PROTO)
+VRT_HDR_R(obj,    reason,	HTTP_HDR_RESPONSE)
+VRT_STATUS_R(obj)
+
+VRT_HDR_LR(resp,   proto,	HTTP_HDR_PROTO)
+VRT_HDR_LR(resp,   reason,	HTTP_HDR_RESPONSE)
+VRT_STATUS_L(resp)
+VRT_STATUS_R(resp)
+
+VRT_HDR_LR(bereq,  method,	HTTP_HDR_METHOD)
+VRT_HDR_LR(bereq,  url,		HTTP_HDR_URL)
+VRT_HDR_LR(bereq,  proto,	HTTP_HDR_PROTO)
+VRT_HDR_LR(beresp, proto,	HTTP_HDR_PROTO)
+VRT_HDR_LR(beresp, reason,	HTTP_HDR_RESPONSE)
+VRT_STATUS_L(beresp)
+VRT_STATUS_R(beresp)
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index faac1e8..dbe673b 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -1,4 +1,4 @@
-varnishtest "Check that we close one error"
+varnishtest "Check that we do not close one error"
 
 varnish v1 -vcl {
 	backend foo {
@@ -19,7 +19,7 @@ client c1 {
 	rxresp
 	expect resp.status == 888
 	expect resp.bodylen == 31
-	expect resp.http.connection == close
+	expect resp.http.connection != close
 } -run
 
 varnish v1 -expect s_error == 1
diff --git a/bin/varnishtest/tests/b00019.vtc b/bin/varnishtest/tests/b00019.vtc
index 597e8b9..59ceb61 100644
--- a/bin/varnishtest/tests/b00019.vtc
+++ b/bin/varnishtest/tests/b00019.vtc
@@ -27,14 +27,14 @@ varnish v1 -vcl+backend {
 
 	sub vcl_error {
 		if (req.restarts == 2) {
-			set obj.status = 200;
-			set obj.reason = "restart=2";
+			set resp.status = 200;
+			set resp.reason = "restart=2";
 		} elsif (req.restarts > 2) {
-			set obj.status = 501;
-			set obj.reason = "restart>2";
+			set resp.status = 501;
+			set resp.reason = "restart>2";
 		} elsif (req.restarts < 2) {
-			set obj.status = 500;
-			set obj.reason = "restart<2";
+			set resp.status = 500;
+			set resp.reason = "restart<2";
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc
index 8b945e1..a7b6c57 100644
--- a/bin/varnishtest/tests/b00030.vtc
+++ b/bin/varnishtest/tests/b00030.vtc
@@ -13,7 +13,7 @@ varnish v1 -vcl+backend {
 		return (error(200,req.ttl));
 	}
 	sub vcl_error {
-		set obj.http.x-timestamp = now;
+		set resp.http.x-timestamp = now;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00024.vtc b/bin/varnishtest/tests/c00024.vtc
index 2a1e542..6167914 100644
--- a/bin/varnishtest/tests/c00024.vtc
+++ b/bin/varnishtest/tests/c00024.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl+backend {
 		if (req.restarts < 1) {
 			return (restart);
 		} else {
-			set obj.status = 201;
+			set resp.status = 201;
 		}
 	 }
  } -start
diff --git a/bin/varnishtest/tests/c00061.vtc b/bin/varnishtest/tests/c00061.vtc
index ecb01a4..c99f324 100644
--- a/bin/varnishtest/tests/c00061.vtc
+++ b/bin/varnishtest/tests/c00061.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_error {
-		set obj.status = 504;
+		set resp.status = 504;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00310.vtc b/bin/varnishtest/tests/r00310.vtc
index 38c8a32..9d0c431 100644
--- a/bin/varnishtest/tests/r00310.vtc
+++ b/bin/varnishtest/tests/r00310.vtc
@@ -1,6 +1,6 @@
 varnishtest "Test obj.http.x-cache in vcl_hit"
 
-varnish v1 -errvcl {'obj.http.x-cache': cannot be set in method 'vcl_hit'.} {
+varnish v1 -errvcl {Variable 'obj.http.x-cache' is read only.} {
 	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_hit {
diff --git a/bin/varnishtest/tests/r00769.vtc b/bin/varnishtest/tests/r00769.vtc
index 838ef17..451029c 100644
--- a/bin/varnishtest/tests/r00769.vtc
+++ b/bin/varnishtest/tests/r00769.vtc
@@ -17,9 +17,9 @@ varnish v1 -vcl+backend {
 		return (deliver);
 	}
 	sub vcl_error {
-		if (obj.status == 700) {
-			set obj.status=404;
-			set obj.http.X-status = obj.status;
+		if (resp.status == 700) {
+			set resp.status=404;
+			set resp.http.X-status = resp.status;
 			return (deliver);
 		}
 	}
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index ccd01b5..d9cbdba 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -28,7 +28,7 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_error {
-		if (obj.status == 988) { return (restart); }
+		if (resp.status == 988) { return (restart); }
 	}
 
 	sub vcl_miss {
diff --git a/bin/varnishtest/tests/r01031.vtc b/bin/varnishtest/tests/r01031.vtc
deleted file mode 100644
index 27bbab2..0000000
--- a/bin/varnishtest/tests/r01031.vtc
+++ /dev/null
@@ -1,30 +0,0 @@
-varnishtest "Test overflowing the response through sp->err_reason"
-
-varnish v1 -vcl {
-	backend blatti {
-		.host = "127.0.0.1";
-	}
-
-	sub vcl_recv {
-		return (error(200,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
-	}
-	sub vcl_error {
-		return(deliver);
-	}
-} -start
-
-client c1 {
-	txreq -req GET
-	rxresp
-	expect resp.status == 200
-	expect resp.msg == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-} -run
-
-varnish v1 -cliok "param.set http_resp_size 256"
-
-client c2 {
-	txreq -req GET
-	rxresp
-	expect resp.status == 200
-	expect resp.msg == "Lost Response"
-} -run
diff --git a/bin/varnishtest/tests/r01283.vtc b/bin/varnishtest/tests/r01283.vtc
deleted file mode 100644
index 9b843ce..0000000
--- a/bin/varnishtest/tests/r01283.vtc
+++ /dev/null
@@ -1,48 +0,0 @@
-varnishtest "#1283 - Test failure to allocate object for error (transient full)"
-
-server s1 {
-	rxreq
-	txresp -bodylen 1048198
-} -start
-
-varnish v1 \
-	-arg "-p nuke_limit=0" \
-	-arg "-sTransient=malloc,1m" \
-	-vcl+backend {
-	sub vcl_recv {
-		if (req.http.x-do-error) {
-			return (error(500));
-		}
-	}
-	sub vcl_backend_response {
-		set beresp.do_stream = false;
-		set beresp.storage_hint = "Transient";
-	}
-} -start
-
-client c1 {
-	# Fill transient
-	txreq
-	rxresp
-	expect resp.status == 200
-} -run
-
-varnish v1 -expect SMA.Transient.g_bytes > 1048000
-varnish v1 -expect SMA.Transient.g_space < 200
-varnish v1 -expect SMA.Transient.c_fail == 0
-
-client c1 {
-	# Trigger vcl_error. Don't wait for reply as Varnish will not send one
-	# due to Transient full
-	txreq -hdr "X-Do-Error: true"
-	delay 1
-} -run
-
-varnish v1 -expect SMA.Transient.c_fail == 1
-
-client c1 {
-	# Check that Varnish is still alive
-	txreq
-	rxresp
-	expect resp.status == 200
-} -run
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 123ddb2..f89b579 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -42,8 +42,8 @@ client c1 {
 	delay 1
 } -run
 
-# Three failures, one for obj2, one for vcl_backend_error{} and for vcl_error{}
-varnish v1 -expect SMA.Transient.c_fail == 3
+# Three failures, one for obj2, one for vcl_backend_error{}
+varnish v1 -expect SMA.Transient.c_fail == 2
 
 client c1 {
 	# Check that Varnish is still alive
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
index fc1ebd2..7e329ac 100644
--- a/bin/varnishtest/tests/r01287.vtc
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -8,7 +8,7 @@ varnish v1 -vcl+backend {
 		return (error(200, "OK"));
 	}
 	sub vcl_error {
-		synthetic(obj.http.blank);
+		synthetic(resp.http.blank);
 		return (deliver);
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc
index 162c66a..3a3d842 100644
--- a/bin/varnishtest/tests/v00011.vtc
+++ b/bin/varnishtest/tests/v00011.vtc
@@ -52,6 +52,5 @@ logexpect l1 -wait
 client c1 {
 	txreq
 	rxresp
-	expect resp.http.X-Varnish == "1007"
-
+	expect resp.http.X-Varnish == "1006"
 } -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c3f26ff..48b0175 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -490,8 +490,8 @@ The client's IP address.
 	),
 	('obj.proto',
 		'STRING',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		The HTTP protocol version used when the object was retrieved.
 		"""
 	),
@@ -520,30 +520,30 @@ The client's IP address.
 	),
 	('obj.http.',
 		'HEADER',
-		( 'hit', 'error',),
-		( 'error',), """
+		( 'hit', ),
+		( ), """
 		The corresponding HTTP header.
 		"""
 	),
 	('obj.ttl',
 		'DURATION',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		The object's remaining time to live, in seconds.
 		obj.ttl is writable.
 		"""
 	),
 	('obj.grace',
 		'DURATION',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		The object's grace period in seconds. obj.grace is writable.
 		"""
 	),
 	('obj.keep',
 		'DURATION',
-		( 'hit', 'error',),
-		( 'hit', 'error',), """
+		( 'hit', ),
+		( 'hit', ), """
 		"""
 	),
 	('obj.last_use',
@@ -557,35 +557,35 @@ The client's IP address.
 	),
 	('obj.uncacheable',
 		'BOOL',
-		( 'hit', 'deliver', 'error',),
+		( 'hit', ),
 		( ), """
 		"""
 	),
 	('resp.proto',
 		'STRING',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The HTTP protocol version to use for the response.
 		"""
 	),
 	('resp.status',
 		'INT',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The HTTP status code that will be returned.
 		"""
 	),
 	('resp.reason',
 		'STRING',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The HTTP status message that will be returned.
 		"""
 	),
 	('resp.http.',
 		'HEADER',
-		( 'deliver',),
-		( 'deliver',), """
+		( 'deliver', 'error', ),
+		( 'deliver', 'error', ), """
 		The corresponding HTTP header.
 		"""
 	),



From fgsch at lodoss.net  Thu Mar 13 09:24:31 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 7bef976 Use `Symbol' instead of `Object'
Message-ID: <E1WO1sN-0007bN-9b@project.varnish-software.com>


commit 7bef97657858f646b0723bcd8a5f01a9d1496f94
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 11:59:38 2014 +0000

    Use `Symbol' instead of `Object'
    
    This matches the rest of the code.

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 4314dce..b42106d 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -99,7 +99,7 @@ varnish v1 -vcl {
 	sub vcl_hash { if (2 == 3) { } }
 }
 
-varnish v1 -errvcl {Object not found: 'foo.bar'} {
+varnish v1 -errvcl {Symbol not found: 'foo.bar'} {
 	sub vcl_init {
 		new bar = foo.bar();
 	}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 4e3f12c..5ed9146 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -191,7 +191,7 @@ parse_new(struct vcc *tl)
 	ExpectErr(tl, ID);
 	sy2 = VCC_FindSymbol(tl, tl->t, SYM_OBJECT);
 	if (sy2 == NULL) {
-		VSB_printf(tl->sb, "Object not found: ");
+		VSB_printf(tl->sb, "Symbol not found: ");
 		vcc_ErrToken(tl, tl->t);
 		VSB_printf(tl->sb, " at ");
 		vcc_ErrWhere(tl, tl->t);



From fgsch at lodoss.net  Thu Mar 13 09:24:31 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 9e298b4 Sync with reality
Message-ID: <E1WO1sN-0007bX-Gg@project.varnish-software.com>


commit 9e298b48ffaf7c4b7f58ac4c56ac9407a2d27405
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 13:46:08 2014 +0000

    Sync with reality
    
    Only varnishreplay is left now.

diff --git a/bin/Makefile.am b/bin/Makefile.am
index a6defb4..1f33d4e 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -2,8 +2,8 @@
 
 # Disabling building of the tools while api is in flux
 
-#SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishadm varnishd varnishlog varnishtest varnishncsa
+#SUBDIRS = varnishreplay
+SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishtest
 
 if VARNISH_CURSES
 SUBDIRS += varnishhist



From fgsch at lodoss.net  Thu Mar 13 09:24:31 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] 8e7d2f1 Update list of rst files
Message-ID: <E1WO1sN-0007bh-Mh@project.varnish-software.com>


commit 8e7d2f19cda7e988c611eb4696fd405e37a0b095
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 13:47:03 2014 +0000

    Update list of rst files

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index f36f569..64e6020 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -96,46 +96,48 @@ EXTRA_DIST = \
 	installation/help.rst \
 	installation/index.rst \
 	installation/install.rst \
-	installation/prerequisites.rst \
 	installation/platformnotes.rst \
+	installation/prerequisites.rst \
 	phk/autocrap.rst \
 	phk/backends.rst \
 	phk/barriers.rst \
 	phk/gzip.rst \
+	phk/http20.rst \
 	phk/index.rst \
 	phk/ipv6suckage.rst \
 	phk/platforms.rst \
+	phk/spdy.rst \
 	phk/sphinx.rst \
 	phk/ssl.rst \
+	phk/thetoolsweworkwith.rst \
 	phk/thoughts.rst \
 	phk/three-zero.rst \
+	phk/varnish_does_not_hash.rst \
 	phk/vcl_expr.rst \
 	phk/wanton_destruction.rst \
 	reference/index.rst \
-	reference/varnishadm.rst \
 	reference/varnish-cli.rst \
+	reference/varnishadm.rst \
 	reference/varnishd.rst \
 	reference/varnishhist.rst \
 	reference/varnishlog.rst \
 	reference/varnishncsa.rst \
-	reference/varnishreplay.rst \
-	reference/varnishsizes.rst \
 	reference/varnishstat.rst \
 	reference/varnishtest.rst \
 	reference/varnishtop.rst \
-	reference/varnishhist.rst \
 	reference/vcl.rst \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
-	reference/vsm.rst \
 	reference/vsl-query.rst \
 	reference/vsl.rst \
+	reference/vsm.rst \
+	tutorial/backend_servers.rst \
 	tutorial/index.rst \
 	tutorial/introduction.rst \
-	tutorial/starting_varnish.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/backend_servers.rst \
 	tutorial/now_what.rst \
+	tutorial/peculiarities.rst \
+	tutorial/putting_varnish_on_port_80.rst \
+	tutorial/starting_varnish.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
 	users-guide/devicedetection.rst \
@@ -149,19 +151,27 @@ EXTRA_DIST = \
 	users-guide/performance.rst \
 	users-guide/purging.rst \
 	users-guide/report.rst \
+	users-guide/run_cli.rst \
+	users-guide/run_security.rst \
 	users-guide/running.rst \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
+	users-guide/vcl-built-in-subs.rst \
+	users-guide/vcl-example-acls.rst \
+	users-guide/vcl-example-manipulating-headers.rst \
+	users-guide/vcl-example-manipulating-responses.rst \
+	users-guide/vcl-example-websockets.rst \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl.rst \
+	users-guide/vcl-intro.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst
+	users-guide/vcl-variables.rst \
+	users-guide/vcl.rst
 
 dist-hook:
 	$(MAKE) html
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index 611ff18..fbfc35f 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -105,46 +105,48 @@ EXTRA_DIST = \
 	installation/help.rst \
 	installation/index.rst \
 	installation/install.rst \
-	installation/prerequisites.rst \
 	installation/platformnotes.rst \
+	installation/prerequisites.rst \
 	phk/autocrap.rst \
 	phk/backends.rst \
 	phk/barriers.rst \
 	phk/gzip.rst \
+	phk/http20.rst \
 	phk/index.rst \
 	phk/ipv6suckage.rst \
 	phk/platforms.rst \
+	phk/spdy.rst \
 	phk/sphinx.rst \
 	phk/ssl.rst \
+	phk/thetoolsweworkwith.rst \
 	phk/thoughts.rst \
 	phk/three-zero.rst \
+	phk/varnish_does_not_hash.rst \
 	phk/vcl_expr.rst \
 	phk/wanton_destruction.rst \
 	reference/index.rst \
-	reference/params.rst \
-	reference/varnishadm.rst \
 	reference/varnish-cli.rst \
+	reference/varnishadm.rst \
 	reference/varnishd.rst \
 	reference/varnishhist.rst \
 	reference/varnishlog.rst \
 	reference/varnishncsa.rst \
-	reference/varnishreplay.rst \
-	reference/varnishsizes.rst \
 	reference/varnishstat.rst \
 	reference/varnishtest.rst \
 	reference/varnishtop.rst \
 	reference/vcl.rst \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
-	reference/vsm.rst \
 	reference/vsl-query.rst \
 	reference/vsl.rst \
+	reference/vsm.rst \
+	tutorial/backend_servers.rst \
 	tutorial/index.rst \
 	tutorial/introduction.rst \
-	tutorial/starting_varnish.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/backend_servers.rst \
 	tutorial/now_what.rst \
+	tutorial/peculiarities.rst \
+	tutorial/putting_varnish_on_port_80.rst \
+	tutorial/starting_varnish.rst \
 	users-guide/command-line.rst \
 	users-guide/compression.rst \
 	users-guide/devicedetection.rst \
@@ -152,26 +154,33 @@ EXTRA_DIST = \
 	users-guide/increasing-your-hitrate.rst \
 	users-guide/index.rst \
 	users-guide/intro.rst \
-	users-guide/operation-cli.rst \
 	users-guide/operation-logging.rst \
 	users-guide/operation-statistics.rst \
 	users-guide/params.rst \
 	users-guide/performance.rst \
 	users-guide/purging.rst \
 	users-guide/report.rst \
+	users-guide/run_cli.rst \
+	users-guide/run_security.rst \
 	users-guide/running.rst \
 	users-guide/sizing-your-cache.rst \
 	users-guide/storage-backends.rst \
 	users-guide/troubleshooting.rst \
 	users-guide/vcl-actions.rst \
 	users-guide/vcl-backends.rst \
+	users-guide/vcl-built-in-subs.rst \
+	users-guide/vcl-example-acls.rst \
+	users-guide/vcl-example-manipulating-headers.rst \
+	users-guide/vcl-example-manipulating-responses.rst \
+	users-guide/vcl-example-websockets.rst \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl.rst \
+	users-guide/vcl-intro.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst
+	users-guide/vcl-variables.rst \
+	users-guide/vcl.rst
 
 dist-hook:
 	$(MAKE) html



From phk at FreeBSD.org  Thu Mar 13 09:24:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:31 +0100
Subject: [4.0] e6a2d41 Correctly account the length of vcl_backend_error{}
 synthetic body.
Message-ID: <E1WO1sN-0007bu-SY@project.varnish-software.com>


commit e6a2d417bd256f7a6970174036bafb0419030d76
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 11 14:22:33 2014 +0000

    Correctly account the length of vcl_backend_error{} synthetic body.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3aabd8d..297f6f2 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -695,7 +695,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 				    "No space for %zd bytes of synth body", l);
 			} else {
 				memcpy(st->ptr, VSB_data(bo->synth_body), l);
-				st->len = l;
 				VBO_extend(bo, l);
 			}
 		}



From fgsch at lodoss.net  Thu Mar 13 09:24:32 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:32 +0100
Subject: [4.0] 1ea3585 Unhook varnishreplay for the time being
Message-ID: <E1WO1sO-0007c3-1k@project.varnish-software.com>


commit 1ea35850b69ae1fda724e5787f383017b986a548
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Mar 11 14:49:27 2014 +0000

    Unhook varnishreplay for the time being
    
    This will be revisited post 4.0. Reorder while I'm here.

diff --git a/man/Makefile.am b/man/Makefile.am
index 9343063..581a9de 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -14,13 +14,12 @@ dist_man_MANS = \
 	vsl-query.7 \
 	varnishadm.1 \
 	varnishd.1 \
+	varnishhist.1 \
 	varnishlog.1 \
 	varnishncsa.1 \
-	varnishreplay.1 \
 	varnishstat.1 \
 	varnishtest.1 \
-	varnishtop.1 \
-	varnishhist.1
+	varnishtop.1
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 



From phk at FreeBSD.org  Thu Mar 13 09:24:32 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:32 +0100
Subject: [4.0] 12b2362 These two test-cases cannot work if streaming is
 allowed.
Message-ID: <E1WO1sO-0007cC-7t@project.varnish-software.com>


commit 12b236205d184be5f2cb5acbd0e9d45ed604a58c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 10:10:13 2014 +0000

    These two test-cases cannot work if streaming is allowed.

diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc
index 7bc36ae..b9578aa 100644
--- a/bin/varnishtest/tests/c00013.vtc
+++ b/bin/varnishtest/tests/c00013.vtc
@@ -4,13 +4,22 @@ server s1 {
 	rxreq
 	expect req.url == "/foo"
 	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay .2
 	sema r1 sync 2
+	delay .2
 	send "line1\n"
+	delay .2
 	sema r1 sync 2
 	send "line2\n"
 } -start
 
-varnish v1 -vcl+backend { } -start
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
 
 client c1 {
 	txreq -url "/foo" -hdr "client: c1"
@@ -24,7 +33,7 @@ sema r1 sync 2
 
 client c2 {
 	txreq -url "/foo" -hdr "client: c2"
-	delay .1
+	delay .2
 	sema r1 sync 2
 	rxresp
 	expect resp.status == 200
diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc
index a8b2d03..fffbbd7 100644
--- a/bin/varnishtest/tests/c00014.vtc
+++ b/bin/varnishtest/tests/c00014.vtc
@@ -15,6 +15,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
+		set beresp.do_stream = false;
 		set beresp.uncacheable = true;
 	}
 } -start



From phk at FreeBSD.org  Thu Mar 13 09:24:32 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:32 +0100
Subject: [4.0] 56bf8ab Add a debugging vsl to indicate when we enable
 streaming.
Message-ID: <E1WO1sO-0007cL-Cn@project.varnish-software.com>


commit 56bf8ab581345565bc81fea2b2d5ded71987081e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 10:14:01 2014 +0000

    Add a debugging vsl to indicate when we enable streaming.
    
    This should possibly be an official VSL record ?

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 297f6f2..1172a61 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -480,6 +480,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->do_stream) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
+		VSLb(bo->vsl, SLT_Debug, "Allow streaming");
 		VBO_setstate(bo, BOS_STREAM);
 	}
 



From phk at FreeBSD.org  Thu Mar 13 09:24:32 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:32 +0100
Subject: [4.0] 3781556 Allow streaming for secondary requests.
Message-ID: <E1WO1sO-0007cU-IK@project.varnish-software.com>


commit 3781556d8294e17bc903657d2bdd2690a3e204f0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 10:14:33 2014 +0000

    Allow streaming for secondary requests.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9d5f9d1..882c84b 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -404,7 +404,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		if (oc->flags & (OC_F_FAILED | OC_F_DYING))
 			continue;
 
-		if (oc->flags & OC_F_BUSY || oc->busyobj != NULL) {
+		if (oc->flags & OC_F_BUSY) {
 			CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC);
 			if (req->hash_ignore_busy)
 				continue;



From phk at FreeBSD.org  Thu Mar 13 09:24:32 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 10:24:32 +0100
Subject: [4.0] 86b11dd Have SLT_Fetch_Body report if we're streaming or not.
Message-ID: <E1WO1sO-0007cd-Mf@project.varnish-software.com>


commit 86b11dd76e3e984d1574dce0523e4ef3b970858f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 12 12:44:39 2014 +0000

    Have SLT_Fetch_Body report if we're streaming or not.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1172a61..74724e0 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -480,10 +480,13 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->do_stream) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
-		VSLb(bo->vsl, SLT_Debug, "Allow streaming");
 		VBO_setstate(bo, BOS_STREAM);
 	}
 
+	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
+	    bo->htc.body_status, body_status_2str(bo->htc.body_status),
+	    bo->do_stream ? "stream" : "-");
+
 	if (bo->htc.body_status != BS_NONE) {
 		assert(bo->htc.body_status  != BS_ERROR);
 		VFP_Fetch_Body(bo, est);
@@ -491,9 +494,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	bo->t_body = VTIM_mono();
 
-	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
-	    bo->htc.body_status, body_status_2str(bo->htc.body_status));
-
 	if (bo->failed) {
 		wrk->stats.fetch_failed++;
 	} else {
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 5dee431..9217582 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -255,13 +255,13 @@ SLTM(TTL, 0, "TTL set on object",
 )
 
 SLTM(Fetch_Body, 0, "Body fetched from backend",
-	"Finished fetching body from backend.\n\n"
+	"Ready to fetch body from backend.\n\n"
 	"The format is::\n\n"
-	"\t%d(%s) cls %d\n"
-	"\t|  |       |\n"
-	"\t|  |       +- 1 if the backend connection was closed\n"
-	"\t|  +--------- Text description of body status\n"
-	"\t+------------ Body status\n"
+	"\t%d (%s) %s\n"
+	"\t|   |    |\n"
+	"\t|   |    +---- 'stream' or '-'\n"
+	"\t|   +--------- Text description of body fetch mode\n"
+	"\t+------------- Body fetch mode\n"
 	"\n"
 )
 



From fgsch at lodoss.net  Thu Mar 13 09:24:32 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 10:24:32 +0100
Subject: [4.0] 3ae1b7a Spelling
Message-ID: <E1WO1sO-0007cm-TW@project.varnish-software.com>


commit 3ae1b7a718c25e3e1ad0fea22a5f7f6ff8508cb5
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 12 09:23:16 2014 +0000

    Spelling

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 67ad069..6970ae0 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -30,7 +30,7 @@ $Module directors 3 Backend traffic directors
 DESCRIPTION
 ===========
 
-Vmod_direcors enables load balancing in Varnish. The module are serves
+Vmod_direcors enables load balancing in Varnish. The module serves
 as an example on how one could extend the load balancing capabilites
 of Varnish.
 
@@ -50,7 +50,7 @@ initialize the directors in vcl_init, like this:::
         bar.add_backend(server2);
     }
 
-As you can see there is nothing keeping you from manipulting the
+As you can see there is nothing keeping you from manipulating the
 directors elsewhere in VCL. So, you could have VCL code that would
 add more backends to a director when a certin URL is called.
 
@@ -127,7 +127,7 @@ $Object hash()
 
 Description
        Creates a hash director. The hash director chooses the backend
-       bashed on hashing an arbitrary string. If you provide it with a
+       based on hashing an arbitrary string. If you provide it with a
        session cookie, you'll have the client connecting to the same
        backend every time.
 Example



From phk at FreeBSD.org  Thu Mar 13 10:09:50 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 11:09:50 +0100
Subject: [master] c18e940 Go over and abstract the completion (good or bad) of
 backend fetches to reduce code duplication a little bit.
Message-ID: <E1WO2aE-0004UX-5j@project.varnish-software.com>


commit c18e940b2264b72bc7967564e153fb6ff1f621cc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 10:09:17 2014 +0000

    Go over and abstract the completion (good or bad) of backend fetches
    to reduce code duplication a little bit.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cac3acb..e0c9350 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -270,11 +270,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
 	bo->uncacheable = bo->do_pass;
-	if (wrk->handling == VCL_RET_ABANDON) {
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-		return (F_STP_DONE);
-	}
+	if (wrk->handling == VCL_RET_ABANDON)
+		return (F_STP_FAIL);
+
 	assert (wrk->handling == VCL_RET_FETCH);
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
@@ -363,6 +361,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		bo->fetch_objcore->flags |= OC_F_PASS;
 
 	assert(wrk->handling == VCL_RET_DELIVER);
+
+	bo->t_body = VTIM_mono();
+
 	return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
 }
 
@@ -492,58 +493,26 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		VFP_Fetch_Body(bo, est);
 	}
 
-	bo->t_body = VTIM_mono();
-
-	if (bo->failed) {
-		wrk->stats.fetch_failed++;
-	} else {
-		if (bo->do_stream)
-			assert(bo->state == BOS_STREAM);
-		else
-			assert(bo->state == BOS_REQ_DONE);
-
-		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
-
-		{
-		/* Sanity check fetch methods accounting */
-			ssize_t uu;
-			struct storage *st;
-
-			uu = 0;
-			VTAILQ_FOREACH(st, &obj->store, list)
-				uu += st->len;
-			if (bo->do_stream)
-				/* Streaming might have started freeing stuff */
-				assert(uu <= obj->len);
-
-			else
-				assert(uu == obj->len);
-		}
-	}
-
-	if (!bo->do_stream) {
-		if (bo->failed) {
-			if (bo->fetch_obj != NULL) {
-				oc_freeobj(bo->fetch_objcore);
-				bo->fetch_obj = NULL;
-				bo->stats->n_object--;
-			}
-			return (F_STP_ERROR);
-		} else {
-			HSH_Unbusy(&wrk->stats, obj->objcore);
-			VBO_setstate(bo, BOS_FINISHED);
+	if (bo->failed && !bo->do_stream) {
+		assert(bo->state < BOS_STREAM);
+		if (bo->fetch_obj != NULL) {
+			oc_freeobj(bo->fetch_objcore);
+			bo->fetch_obj = NULL;
+			bo->stats->n_object--;
 		}
-	} else if (bo->failed) {
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-	} else {
-		VBO_setstate(bo, BOS_FINISHED);
+		return (F_STP_ERROR);
 	}
 
-	HSH_Complete(obj->objcore);
-
-	assert(bo->refcount >= 1);
+	if (bo->failed)
+		return (F_STP_FAIL);
 
+	if (bo->do_stream)
+		assert(bo->state == BOS_STREAM);
+	else {
+		assert(bo->state == BOS_REQ_DONE);
+		HSH_Unbusy(&wrk->stats, obj->objcore);
+	}
+	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
 
@@ -584,10 +553,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		if (obj->esidata == NULL || obj->esidata->space < sl) {
 			VSLb(bo->vsl, SLT_Error,
 			    "No space for %zd bytes of ESI data", sl);
-			HSH_Fail(bo->fetch_objcore);
-			VBO_setstate(bo, BOS_FAILED);
-			HSH_Complete(obj->objcore);
-			return (F_STP_DONE);
+			return (F_STP_FAIL);
 		}
 		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
 		obj->esidata->len = sl;
@@ -627,16 +593,13 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		}
 	} while (!bo->failed && (ois == OIS_DATA || ois == OIS_STREAM));
 	ObjIterEnd(&oi);
-	if (!bo->failed) {
-		assert(al == bo->ims_obj->len);
-		assert(obj->len == al);
-		VBO_setstate(bo, BOS_FINISHED);
-		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
-	} else {
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-	}
-	HSH_Complete(obj->objcore);
+	if (bo->failed)
+		return (F_STP_FAIL);
+
+	assert(al == bo->ims_obj->len);
+	assert(obj->len == al);
+	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
 
@@ -679,19 +642,13 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
 		bo->synth_body = NULL;
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-		HSH_Complete(bo->fetch_objcore);
-		return (F_STP_DONE);
+		return (F_STP_FAIL);
 	}
 
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	if (vbf_beresp2obj(bo)) {
-		VBO_setstate(bo, BOS_FAILED);
-		HSH_Fail(bo->fetch_objcore);
-		return (F_STP_DONE);
-	}
+	if (vbf_beresp2obj(bo))
+		return (F_STP_FAIL);
 
 	l = VSB_len(bo->synth_body);
 	if (l > 0) {
@@ -709,12 +666,28 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	VSB_delete(bo->synth_body);
 	bo->synth_body = NULL;
 
-	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
-
+	HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
 	VBO_setstate(bo, BOS_FINISHED);
+	return (F_STP_DONE);
+}
 
-	HSH_Complete(bo->fetch_obj->objcore);
+/*--------------------------------------------------------------------
+ */
 
+static enum fetch_step
+vbf_stp_fail(struct worker *wrk, struct busyobj *bo)
+{
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
+
+	assert(bo->state < BOS_FINISHED);
+	bo->fetch_objcore->flags |= OC_F_PRIVATE;
+	if (bo->fetch_objcore->flags & OC_F_BUSY)
+		HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
+	HSH_Fail(bo->fetch_objcore);
+	wrk->stats.fetch_failed++;
+	VBO_setstate(bo, BOS_FAILED);
 	return (F_STP_DONE);
 }
 
@@ -738,6 +711,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
 	THR_SetBusyobj(bo);
 	stp = F_STP_MKBEREQ;
@@ -751,6 +725,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	while (stp != F_STP_DONE) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+		assert(bo->refcount >= 1);
 		switch(stp) {
 #define FETCH_STEP(l, U, arg)						\
 		case F_STP_##U:						\
@@ -764,6 +739,9 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	}
 	assert(WRW_IsReleased(wrk));
 
+	AZ(bo->fetch_objcore->flags & OC_F_BUSY);
+	HSH_Complete(bo->fetch_objcore);
+
 	bo->stats = NULL;
 
 	if (bo->vbc != NULL) {
@@ -777,8 +755,25 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
 
-	if (bo->state == BOS_FAILED)
-		assert(bo->fetch_objcore->flags & OC_F_FAILED);
+	if (bo->state == BOS_FINISHED) {
+		assert(!(bo->fetch_objcore->flags & OC_F_FAILED));
+		VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
+		{
+		/* Sanity check fetch methods accounting */
+			ssize_t uu;
+			struct storage *st;
+
+			uu = 0;
+			VTAILQ_FOREACH(st, &bo->fetch_obj->store, list)
+				uu += st->len;
+			if (bo->do_stream)
+				/* Streaming might have started freeing stuff */
+				assert(uu <= bo->fetch_obj->len);
+
+			else
+				assert(uu == bo->fetch_obj->len);
+		}
+	}
 
 	if (bo->ims_obj != NULL)
 		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 47da3fc..d0efae7 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -55,6 +55,7 @@ FETCH_STEP(startfetch,	STARTFETCH,	(wrk, bo))
 FETCH_STEP(condfetch,	CONDFETCH,	(wrk, bo))
 FETCH_STEP(fetch,	FETCH,		(wrk, bo))
 FETCH_STEP(error,	ERROR,		(wrk, bo))
+FETCH_STEP(fail,	FAIL,		(wrk, bo))
 FETCH_STEP(done,	DONE,		())
 #endif
 



From tfheen at err.no  Thu Mar 13 12:07:51 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 13:07:51 +0100
Subject: [master] 80a4432 Language, typos, XXX rather than asdasoidwqjwqdasdas
Message-ID: <E1WO4QR-0002w4-7h@project.varnish-software.com>


commit 80a443224bbf29ba62145051e73cb7ea115e2966
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Thu Mar 13 13:07:49 2014 +0100

    Language, typos, XXX rather than asdasoidwqjwqdasdas

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 6970ae0..f7ec8f4 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -30,8 +30,8 @@ $Module directors 3 Backend traffic directors
 DESCRIPTION
 ===========
 
-Vmod_direcors enables load balancing in Varnish. The module serves
-as an example on how one could extend the load balancing capabilites
+Vmod_directors enables load balancing in Varnish. The module serves
+as an example on how one could extend the load balancing capabilities
 of Varnish.
 
 To enable load balancing you must import this vmod (directors) in your
@@ -41,7 +41,7 @@ VCL:::
 
 Then you define your backends. Once you have the backends declared you
 can add them to a director. This happens in executed VCL code. If you
-want to emulate the previous behaviour of Varnish 3.0 you can just
+want to emulate the previous behavior of Varnish 3.0 you can just
 initialize the directors in vcl_init, like this:::
 
     sub vcl_init {
@@ -52,7 +52,7 @@ initialize the directors in vcl_init, like this:::
 
 As you can see there is nothing keeping you from manipulating the
 directors elsewhere in VCL. So, you could have VCL code that would
-add more backends to a director when a certin URL is called.
+add more backends to a director when a certain URL is called.
 
 $Object round_robin()
 
@@ -79,7 +79,10 @@ Example
 $Object fallback()
 
 Description
-        Creates a fallback director. This director will asdasoidwqjwqdasdas
+        Creates a fallback director. This director will
+
+	XXX: Add description of fallback director.
+
 Example
         new foo = directors.fallback();
 



From phk at FreeBSD.org  Thu Mar 13 13:45:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 14:45:00 +0100
Subject: [4.0] f2a0ace Correctly bail if there is insuficient storage for ESI
 data clone on IMS fetch.
Message-ID: <E1WO5wS-0005VM-Gz@project.varnish-software.com>


commit f2a0acec1566a38bdd41101773bf053cb6d103f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 08:55:14 2014 +0000

    Correctly bail if there is insuficient storage for ESI data clone
    on IMS fetch.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 74724e0..cac3acb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -579,12 +579,18 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	obj = bo->fetch_obj;
 
 	if (bo->ims_obj->esidata != NULL) {
-		obj->esidata = STV_alloc(bo, bo->ims_obj->esidata->len);
-		XXXAN(obj->esidata);
-		xxxassert(obj->esidata->space >= bo->ims_obj->esidata->len);
-		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr,
-		    bo->ims_obj->esidata->len);
-		obj->esidata->len = bo->ims_obj->esidata->len;
+		sl = bo->ims_obj->esidata->len;
+		obj->esidata = STV_alloc(bo, sl);
+		if (obj->esidata == NULL || obj->esidata->space < sl) {
+			VSLb(bo->vsl, SLT_Error,
+			    "No space for %zd bytes of ESI data", sl);
+			HSH_Fail(bo->fetch_objcore);
+			VBO_setstate(bo, BOS_FAILED);
+			HSH_Complete(obj->objcore);
+			return (F_STP_DONE);
+		}
+		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
+		obj->esidata->len = sl;
 	}
 
 	obj->gziped = bo->ims_obj->gziped;



From phk at FreeBSD.org  Thu Mar 13 13:45:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 14:45:00 +0100
Subject: [4.0] c18e940 Go over and abstract the completion (good or bad) of
 backend fetches to reduce code duplication a little bit.
Message-ID: <E1WO5wS-0005VV-IP@project.varnish-software.com>


commit c18e940b2264b72bc7967564e153fb6ff1f621cc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 10:09:17 2014 +0000

    Go over and abstract the completion (good or bad) of backend fetches
    to reduce code duplication a little bit.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cac3acb..e0c9350 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -270,11 +270,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
 	bo->uncacheable = bo->do_pass;
-	if (wrk->handling == VCL_RET_ABANDON) {
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-		return (F_STP_DONE);
-	}
+	if (wrk->handling == VCL_RET_ABANDON)
+		return (F_STP_FAIL);
+
 	assert (wrk->handling == VCL_RET_FETCH);
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
@@ -363,6 +361,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		bo->fetch_objcore->flags |= OC_F_PASS;
 
 	assert(wrk->handling == VCL_RET_DELIVER);
+
+	bo->t_body = VTIM_mono();
+
 	return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
 }
 
@@ -492,58 +493,26 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		VFP_Fetch_Body(bo, est);
 	}
 
-	bo->t_body = VTIM_mono();
-
-	if (bo->failed) {
-		wrk->stats.fetch_failed++;
-	} else {
-		if (bo->do_stream)
-			assert(bo->state == BOS_STREAM);
-		else
-			assert(bo->state == BOS_REQ_DONE);
-
-		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
-
-		{
-		/* Sanity check fetch methods accounting */
-			ssize_t uu;
-			struct storage *st;
-
-			uu = 0;
-			VTAILQ_FOREACH(st, &obj->store, list)
-				uu += st->len;
-			if (bo->do_stream)
-				/* Streaming might have started freeing stuff */
-				assert(uu <= obj->len);
-
-			else
-				assert(uu == obj->len);
-		}
-	}
-
-	if (!bo->do_stream) {
-		if (bo->failed) {
-			if (bo->fetch_obj != NULL) {
-				oc_freeobj(bo->fetch_objcore);
-				bo->fetch_obj = NULL;
-				bo->stats->n_object--;
-			}
-			return (F_STP_ERROR);
-		} else {
-			HSH_Unbusy(&wrk->stats, obj->objcore);
-			VBO_setstate(bo, BOS_FINISHED);
+	if (bo->failed && !bo->do_stream) {
+		assert(bo->state < BOS_STREAM);
+		if (bo->fetch_obj != NULL) {
+			oc_freeobj(bo->fetch_objcore);
+			bo->fetch_obj = NULL;
+			bo->stats->n_object--;
 		}
-	} else if (bo->failed) {
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-	} else {
-		VBO_setstate(bo, BOS_FINISHED);
+		return (F_STP_ERROR);
 	}
 
-	HSH_Complete(obj->objcore);
-
-	assert(bo->refcount >= 1);
+	if (bo->failed)
+		return (F_STP_FAIL);
 
+	if (bo->do_stream)
+		assert(bo->state == BOS_STREAM);
+	else {
+		assert(bo->state == BOS_REQ_DONE);
+		HSH_Unbusy(&wrk->stats, obj->objcore);
+	}
+	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
 
@@ -584,10 +553,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		if (obj->esidata == NULL || obj->esidata->space < sl) {
 			VSLb(bo->vsl, SLT_Error,
 			    "No space for %zd bytes of ESI data", sl);
-			HSH_Fail(bo->fetch_objcore);
-			VBO_setstate(bo, BOS_FAILED);
-			HSH_Complete(obj->objcore);
-			return (F_STP_DONE);
+			return (F_STP_FAIL);
 		}
 		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
 		obj->esidata->len = sl;
@@ -627,16 +593,13 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		}
 	} while (!bo->failed && (ois == OIS_DATA || ois == OIS_STREAM));
 	ObjIterEnd(&oi);
-	if (!bo->failed) {
-		assert(al == bo->ims_obj->len);
-		assert(obj->len == al);
-		VBO_setstate(bo, BOS_FINISHED);
-		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
-	} else {
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-	}
-	HSH_Complete(obj->objcore);
+	if (bo->failed)
+		return (F_STP_FAIL);
+
+	assert(al == bo->ims_obj->len);
+	assert(obj->len == al);
+	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
 
@@ -679,19 +642,13 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
 		bo->synth_body = NULL;
-		HSH_Fail(bo->fetch_objcore);
-		VBO_setstate(bo, BOS_FAILED);
-		HSH_Complete(bo->fetch_objcore);
-		return (F_STP_DONE);
+		return (F_STP_FAIL);
 	}
 
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	if (vbf_beresp2obj(bo)) {
-		VBO_setstate(bo, BOS_FAILED);
-		HSH_Fail(bo->fetch_objcore);
-		return (F_STP_DONE);
-	}
+	if (vbf_beresp2obj(bo))
+		return (F_STP_FAIL);
 
 	l = VSB_len(bo->synth_body);
 	if (l > 0) {
@@ -709,12 +666,28 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	VSB_delete(bo->synth_body);
 	bo->synth_body = NULL;
 
-	HSH_Unbusy(&wrk->stats, bo->fetch_obj->objcore);
-
+	HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
 	VBO_setstate(bo, BOS_FINISHED);
+	return (F_STP_DONE);
+}
 
-	HSH_Complete(bo->fetch_obj->objcore);
+/*--------------------------------------------------------------------
+ */
 
+static enum fetch_step
+vbf_stp_fail(struct worker *wrk, struct busyobj *bo)
+{
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
+
+	assert(bo->state < BOS_FINISHED);
+	bo->fetch_objcore->flags |= OC_F_PRIVATE;
+	if (bo->fetch_objcore->flags & OC_F_BUSY)
+		HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
+	HSH_Fail(bo->fetch_objcore);
+	wrk->stats.fetch_failed++;
+	VBO_setstate(bo, BOS_FAILED);
 	return (F_STP_DONE);
 }
 
@@ -738,6 +711,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
 	THR_SetBusyobj(bo);
 	stp = F_STP_MKBEREQ;
@@ -751,6 +725,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	while (stp != F_STP_DONE) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+		assert(bo->refcount >= 1);
 		switch(stp) {
 #define FETCH_STEP(l, U, arg)						\
 		case F_STP_##U:						\
@@ -764,6 +739,9 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	}
 	assert(WRW_IsReleased(wrk));
 
+	AZ(bo->fetch_objcore->flags & OC_F_BUSY);
+	HSH_Complete(bo->fetch_objcore);
+
 	bo->stats = NULL;
 
 	if (bo->vbc != NULL) {
@@ -777,8 +755,25 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
 
-	if (bo->state == BOS_FAILED)
-		assert(bo->fetch_objcore->flags & OC_F_FAILED);
+	if (bo->state == BOS_FINISHED) {
+		assert(!(bo->fetch_objcore->flags & OC_F_FAILED));
+		VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
+		{
+		/* Sanity check fetch methods accounting */
+			ssize_t uu;
+			struct storage *st;
+
+			uu = 0;
+			VTAILQ_FOREACH(st, &bo->fetch_obj->store, list)
+				uu += st->len;
+			if (bo->do_stream)
+				/* Streaming might have started freeing stuff */
+				assert(uu <= bo->fetch_obj->len);
+
+			else
+				assert(uu == bo->fetch_obj->len);
+		}
+	}
 
 	if (bo->ims_obj != NULL)
 		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 47da3fc..d0efae7 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -55,6 +55,7 @@ FETCH_STEP(startfetch,	STARTFETCH,	(wrk, bo))
 FETCH_STEP(condfetch,	CONDFETCH,	(wrk, bo))
 FETCH_STEP(fetch,	FETCH,		(wrk, bo))
 FETCH_STEP(error,	ERROR,		(wrk, bo))
+FETCH_STEP(fail,	FAIL,		(wrk, bo))
 FETCH_STEP(done,	DONE,		())
 #endif
 



From tfheen at err.no  Thu Mar 13 13:45:00 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 13 Mar 2014 14:45:00 +0100
Subject: [4.0] 80a4432 Language, typos, XXX rather than asdasoidwqjwqdasdas
Message-ID: <E1WO5wS-0005Vc-JS@project.varnish-software.com>


commit 80a443224bbf29ba62145051e73cb7ea115e2966
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Thu Mar 13 13:07:49 2014 +0100

    Language, typos, XXX rather than asdasoidwqjwqdasdas

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 6970ae0..f7ec8f4 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -30,8 +30,8 @@ $Module directors 3 Backend traffic directors
 DESCRIPTION
 ===========
 
-Vmod_direcors enables load balancing in Varnish. The module serves
-as an example on how one could extend the load balancing capabilites
+Vmod_directors enables load balancing in Varnish. The module serves
+as an example on how one could extend the load balancing capabilities
 of Varnish.
 
 To enable load balancing you must import this vmod (directors) in your
@@ -41,7 +41,7 @@ VCL:::
 
 Then you define your backends. Once you have the backends declared you
 can add them to a director. This happens in executed VCL code. If you
-want to emulate the previous behaviour of Varnish 3.0 you can just
+want to emulate the previous behavior of Varnish 3.0 you can just
 initialize the directors in vcl_init, like this:::
 
     sub vcl_init {
@@ -52,7 +52,7 @@ initialize the directors in vcl_init, like this:::
 
 As you can see there is nothing keeping you from manipulating the
 directors elsewhere in VCL. So, you could have VCL code that would
-add more backends to a director when a certin URL is called.
+add more backends to a director when a certain URL is called.
 
 $Object round_robin()
 
@@ -79,7 +79,10 @@ Example
 $Object fallback()
 
 Description
-        Creates a fallback director. This director will asdasoidwqjwqdasdas
+        Creates a fallback director. This director will
+
+	XXX: Add description of fallback director.
+
 Example
         new foo = directors.fallback();
 



From lkarsten at varnish-software.com  Thu Mar 13 13:45:00 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 13 Mar 2014 14:45:00 +0100
Subject: [4.0] ba3c037 Bump release number
Message-ID: <E1WO5wS-0005Vl-KQ@project.varnish-software.com>


commit ba3c0370e2c6adb143c4dfa73f0ec7e929603b67
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 13 14:44:53 2014 +0100

    Bump release number

diff --git a/configure.ac b/configure.ac
index 6f43695..5b582b9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [4.0.0-tp2], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [4.0.0-tp3], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])



From fgsch at lodoss.net  Thu Mar 13 14:01:57 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 15:01:57 +0100
Subject: [4.0] 262f7de There is no `y' (year) suffix
Message-ID: <E1WO6Cr-0007TU-9u@project.varnish-software.com>


commit 262f7de116e6804159d3eaf62d6f46ed7cd674cd
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 13 14:01:39 2014 +0000

    There is no `y' (year) suffix

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 5a7fe6b..ce0872c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -123,9 +123,6 @@ Durations are defined by a number and a designation. The number can be a real so
   w
     weeks
 
-  y
-    years.
-
 
 Ints
 ----



From fgsch at lodoss.net  Thu Mar 13 14:14:21 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 15:14:21 +0100
Subject: [4.0] 99e23f5 More spelling fixes
Message-ID: <E1WO6Or-0003KH-5r@project.varnish-software.com>


commit 99e23f5d59eab38f6d91055821dd09bc7e6da541
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 13 14:14:08 2014 +0000

    More spelling fixes

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index ce0872c..ab22ba4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -29,7 +29,7 @@ VCL consists of the following elements:
  * Strings, booleans, time, duration, ints
  * Regular expressions
 
-In addition VCL has the following contructs:
+In addition VCL has the following constructs:
  * Include
  * Backend definitions
  * Probes
@@ -67,7 +67,7 @@ The following operators are available in VCL:
 Conditionals
 ------------
 
-VCL has *if* statments.
+VCL has *if* statements.
 
 
 Strings, booleans, time, duration and ints
@@ -319,8 +319,8 @@ Variables
 ---------
 
 In VCL you have access to certain variable objects. These contain
-requests and responses currently beeing worked on. What variables are
-availble depends on context.
+requests and responses currently being worked on. What variables are
+available depends on context.
 
 .. include:: vcl_var.rst
 



From fgsch at lodoss.net  Thu Mar 13 14:27:14 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 15:27:14 +0100
Subject: [4.0] 0a39f22 A few more spelling fixes
Message-ID: <E1WO6bK-0007V6-5n@project.varnish-software.com>


commit 0a39f2226bf2f09748599cd934bd52f9da090b7b
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 13 14:27:07 2014 +0000

    A few more spelling fixes

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index ab22ba4..a4b6f7b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -19,7 +19,7 @@ When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
 is then dynamically linked into the server process.
 
-This document focuses on the syntax of the VCL language. Full a full
+This document focuses on the syntax of the VCL language. For a full
 description of syntax and semantics, with ample examples, please see
 the users guide at https://www.varnish-cache.org/doc/
 
@@ -175,7 +175,7 @@ key/value fashion.::
 
 The only mandatory attribute is host. The attributes will inherit
 their defaults from the global parameters. The following attributes
-are availble:
+are available:
 
   host (mandatory)
     The host to be used. IP address or a hostname that resolves to a
@@ -237,7 +237,7 @@ There are no mandatory options. These are the options you can set:
     How many of the polls in .window are considered good when Varnish
     starts. Defaults to the value of threshold - 1. In this case, the
     backend starts as sick and requires one single poll to be
-    conqsidered healthy.
+    considered healthy.
             
   window
     How many of the latest polls we examine to determine backend health. Defaults to 8.
@@ -300,7 +300,7 @@ Return statements
 
 The subroutine executions ends when a return(*action*) statement is
 made. The *action* specifies how execution should proceed. The context
-defines which actions are availble. See the user guide for information
+defines which actions are available. See the user guide for information
 on what actions are available where.
 
 Multiple subroutines



From phk at FreeBSD.org  Thu Mar 13 14:46:32 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 15:46:32 +0100
Subject: [master] 4f66a21 rename "rxhdrs" to "rxreqhdrs" and add "rxresphdrs"
Message-ID: <E1WO6u0-0003Pa-2X@project.varnish-software.com>


commit 4f66a21d84a687eaf71a23f8e86a0c867a1a5aa5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 14:46:06 2014 +0000

    rename "rxhdrs" to "rxreqhdrs" and add "rxresphdrs"

diff --git a/bin/varnishtest/tests/r01356.vtc b/bin/varnishtest/tests/r01356.vtc
index 1aaa337..341f2c8 100644
--- a/bin/varnishtest/tests/r01356.vtc
+++ b/bin/varnishtest/tests/r01356.vtc
@@ -1,7 +1,7 @@
 varnishtest "#1356, req.body failure"
 
 server s1 {
-	rxhdrs
+	rxreqhdrs
 	expect_close
 } -start
 
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 271540b..4388f73 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -233,8 +233,12 @@ cmd_http_expect(CMD_ARGS)
 	AN(av[2]);
 	AZ(av[3]);
 	lhs = cmd_var_resolve(hp, av[0]);
+	if (lhs == NULL)
+		lhs = "<missing>";
 	cmp = av[1];
 	rhs = cmd_var_resolve(hp, av[2]);
+	if (rhs == NULL)
+		rhs = "<missing>";
 	if (!strcmp(cmp, "==")) {
 		retval = strcmp(lhs, rhs) == 0;
 	} else if (!strcmp(cmp, "<")) {
@@ -541,6 +545,25 @@ cmd_http_rxresp(CMD_ARGS)
 	vtc_log(hp->vl, 4, "bodylen = %s", hp->bodylen);
 }
 
+static void
+cmd_http_rxresphdrs(CMD_ARGS)
+{
+	struct http *hp;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+	ONLY_CLIENT(hp, av);
+	assert(!strcmp(av[0], "rxresphdrs"));
+	av++;
+
+	for(; *av != NULL; av++)
+		vtc_log(hp->vl, 0, "Unknown http rxreq spec: %s\n", *av);
+	http_rxhdr(hp);
+	http_splitheader(hp, 0);
+}
+
+
 /**********************************************************************
  * Ungzip rx'ed body
  */
@@ -802,14 +825,14 @@ cmd_http_rxreq(CMD_ARGS)
 }
 
 static void
-cmd_http_rxhdrs(CMD_ARGS)
+cmd_http_rxreqhdrs(CMD_ARGS)
 {
 	struct http *hp;
 
 	(void)cmd;
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
-	assert(!strcmp(av[0], "rxhdrs"));
+	assert(!strcmp(av[0], "rxreqhdrs"));
 	av++;
 
 	for(; *av != NULL; av++)
@@ -1169,12 +1192,13 @@ static const struct cmds http_cmds[] = {
 	{ "txreq",		cmd_http_txreq },
 
 	{ "rxreq",		cmd_http_rxreq },
-	{ "rxhdrs",		cmd_http_rxhdrs },
+	{ "rxreqhdrs",		cmd_http_rxreqhdrs },
 	{ "rxchunk",		cmd_http_rxchunk },
 	{ "rxbody",		cmd_http_rxbody },
 
 	{ "txresp",		cmd_http_txresp },
 	{ "rxresp",		cmd_http_rxresp },
+	{ "rxresphdrs",		cmd_http_rxresphdrs },
 	{ "gunzip",		cmd_http_gunzip_body },
 	{ "expect",		cmd_http_expect },
 	{ "send",		cmd_http_send },



From fgsch at lodoss.net  Thu Mar 13 14:56:03 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 15:56:03 +0100
Subject: [master] bf996bc There is no `y' (year) suffix
Message-ID: <E1WO73D-0004oW-UM@project.varnish-software.com>


commit bf996bc2551d924d22c818c0c4f4f2401f0c1b34
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 13 14:01:39 2014 +0000

    There is no `y' (year) suffix

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 5a7fe6b..ce0872c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -123,9 +123,6 @@ Durations are defined by a number and a designation. The number can be a real so
   w
     weeks
 
-  y
-    years.
-
 
 Ints
 ----



From fgsch at lodoss.net  Thu Mar 13 14:56:04 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 15:56:04 +0100
Subject: [master] e757cc1 More spelling fixes
Message-ID: <E1WO73E-0004oc-15@project.varnish-software.com>


commit e757cc1a23e0bff04b937e8318f6a5749bfe86ed
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 13 14:14:08 2014 +0000

    More spelling fixes

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index ce0872c..ab22ba4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -29,7 +29,7 @@ VCL consists of the following elements:
  * Strings, booleans, time, duration, ints
  * Regular expressions
 
-In addition VCL has the following contructs:
+In addition VCL has the following constructs:
  * Include
  * Backend definitions
  * Probes
@@ -67,7 +67,7 @@ The following operators are available in VCL:
 Conditionals
 ------------
 
-VCL has *if* statments.
+VCL has *if* statements.
 
 
 Strings, booleans, time, duration and ints
@@ -319,8 +319,8 @@ Variables
 ---------
 
 In VCL you have access to certain variable objects. These contain
-requests and responses currently beeing worked on. What variables are
-availble depends on context.
+requests and responses currently being worked on. What variables are
+available depends on context.
 
 .. include:: vcl_var.rst
 



From fgsch at lodoss.net  Thu Mar 13 14:56:04 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 13 Mar 2014 15:56:04 +0100
Subject: [master] 899b87e A few more spelling fixes
Message-ID: <E1WO73E-0004ol-2v@project.varnish-software.com>


commit 899b87ee73edee4bb036d41c42163c80fab6383d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 13 14:27:07 2014 +0000

    A few more spelling fixes

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index ab22ba4..a4b6f7b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -19,7 +19,7 @@ When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
 is then dynamically linked into the server process.
 
-This document focuses on the syntax of the VCL language. Full a full
+This document focuses on the syntax of the VCL language. For a full
 description of syntax and semantics, with ample examples, please see
 the users guide at https://www.varnish-cache.org/doc/
 
@@ -175,7 +175,7 @@ key/value fashion.::
 
 The only mandatory attribute is host. The attributes will inherit
 their defaults from the global parameters. The following attributes
-are availble:
+are available:
 
   host (mandatory)
     The host to be used. IP address or a hostname that resolves to a
@@ -237,7 +237,7 @@ There are no mandatory options. These are the options you can set:
     How many of the polls in .window are considered good when Varnish
     starts. Defaults to the value of threshold - 1. In this case, the
     backend starts as sick and requires one single poll to be
-    conqsidered healthy.
+    considered healthy.
             
   window
     How many of the latest polls we examine to determine backend health. Defaults to 8.
@@ -300,7 +300,7 @@ Return statements
 
 The subroutine executions ends when a return(*action*) statement is
 made. The *action* specifies how execution should proceed. The context
-defines which actions are availble. See the user guide for information
+defines which actions are available. See the user guide for information
 on what actions are available where.
 
 Multiple subroutines



From phk at FreeBSD.org  Thu Mar 13 15:03:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 16:03:25 +0100
Subject: [master] 717f0fb Spotted while looking for 1449:
Message-ID: <E1WO7AL-0007qX-I3@project.varnish-software.com>


commit 717f0fb9995b25ebead1549f32ebd06e65fe7da7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 15:02:43 2014 +0000

    Spotted while looking for 1449:
    
    Make sure we do not send a regular "end-chunk" to the
    client if the streaming fetch fails.

diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 5a3da11..231df94 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -157,7 +157,7 @@ v1d_dorange(struct req *req, const char *r)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static enum objiter_status
 v1d_WriteDirObj(struct req *req)
 {
 	enum objiter_status ois;
@@ -190,12 +190,14 @@ v1d_WriteDirObj(struct req *req)
 	} while (ois == OIS_DATA || ois == OIS_STREAM);
 	(void)VDP_bytes(req, VDP_FINISH,  NULL, 0);
 	ObjIterEnd(&oi);
+	return (ois);
 }
 
 void
 V1D_Deliver(struct req *req)
 {
 	char *r;
+	enum objiter_status ois;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
@@ -291,6 +293,7 @@ V1D_Deliver(struct req *req)
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
 
+	ois = OIS_DONE;
 	if (!req->wantbody) {
 		/* This was a HEAD or conditional request */
 	} else if (req->res_mode & RES_ESI) {
@@ -305,17 +308,19 @@ V1D_Deliver(struct req *req)
 		VDP_push(req, VDP_gunzip);
 		req->vgz = VGZ_NewUngzip(req->vsl, "U D -");
 		AZ(VGZ_WrwInit(req->vgz));
-		v1d_WriteDirObj(req);
+		ois = v1d_WriteDirObj(req);
 		(void)VGZ_Destroy(&req->vgz);
 		VDP_pop(req, VDP_gunzip);
 	} else {
-		v1d_WriteDirObj(req);
+		ois = v1d_WriteDirObj(req);
 	}
 
-	if (req->res_mode & RES_CHUNKED && !(req->res_mode & RES_ESI_CHILD))
+	if (ois == OIS_DONE &&
+	    (req->res_mode & RES_CHUNKED) &&
+	    !(req->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(req->wrk);
 
-	if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0)
+	if ((WRW_FlushRelease(req->wrk) || ois != OIS_DONE) && req->sp->fd >= 0)
 		SES_Close(req->sp, SC_REM_CLOSE);
 }
 
diff --git a/bin/varnishtest/tests/c00062.vtc b/bin/varnishtest/tests/c00062.vtc
new file mode 100644
index 0000000..643a2f6
--- /dev/null
+++ b/bin/varnishtest/tests/c00062.vtc
@@ -0,0 +1,29 @@
+varnishtest "Check that aborted backend body aborts client in streaming mode"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked {<HTML>}
+	sema r1 sync 2
+	chunked {<HTML>}
+	sema r1 sync 2
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend {
+
+} -start
+
+
+client c1 {
+	txreq
+	rxresphdrs
+	expect resp.status == 200
+	rxchunk
+	sema r1 sync 2
+	rxchunk
+	sema r1 sync 2
+	expect_close
+} -run
+
+
+
diff --git a/bin/varnishtest/tests/r01086.vtc b/bin/varnishtest/tests/r01086.vtc
index 667cc4d..d861a0d 100644
--- a/bin/varnishtest/tests/r01086.vtc
+++ b/bin/varnishtest/tests/r01086.vtc
@@ -65,9 +65,9 @@ varnish v1 -vcl+backend {
 
 client c1 {
 	txreq -hdr "Cookie: FOO"
-	rxresp
+	rxresphdrs
 	expect resp.status == 200
-	expect resp.bodylen == 0
+	expect_close
 } -run
 
 delay .1



From phk at FreeBSD.org  Thu Mar 13 15:53:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 13 Mar 2014 16:53:31 +0100
Subject: [master] e177b46 Add a test-case to check that vcl_backend_error{}
 objects can be cached.
Message-ID: <E1WO7wp-0005OQ-Sj@project.varnish-software.com>


commit e177b4620e3a8e734c751c431541c5b7c27b29d7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 13 15:53:12 2014 +0000

    Add a test-case to check that vcl_backend_error{} objects can be cached.

diff --git a/bin/varnishtest/tests/c00063.vtc b/bin/varnishtest/tests/c00063.vtc
new file mode 100644
index 0000000..0da903e
--- /dev/null
+++ b/bin/varnishtest/tests/c00063.vtc
@@ -0,0 +1,32 @@
+varnishtest "cache backend synth object"
+
+varnish v1 -vcl {
+
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_backend_error {
+		set beresp.ttl = 1s;
+		set beresp.grace = 3s;
+		set beresp.http.foobar = "BLA" + bereq.xid;
+		synthetic(beresp.http.foobar);
+		return (deliver);
+	}
+} -start
+
+varnish v1 -cliok "param.set connect_timeout 1.0"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.foobar == "BLA1002"
+
+	delay 2
+	txreq
+	rxresp
+	expect resp.http.foobar == "BLA1002"
+
+	delay 3
+	txreq
+	rxresp
+	expect resp.http.foobar == "BLA1004"
+} -run



From phk at FreeBSD.org  Fri Mar 14 09:25:15 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 14 Mar 2014 10:25:15 +0100
Subject: [master] b135934 Fix a race when we Unbusy and abandonned oc which
 has no object.
Message-ID: <E1WOOMd-0003of-86@project.varnish-software.com>


commit b135934f9eace9bc281b3e727f37dfd302170fa8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 14 09:24:41 2014 +0000

    Fix a race when we Unbusy and abandonned oc which has no object.
    
    Fixes	#1449

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e0c9350..b5cb463 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -682,10 +682,9 @@ vbf_stp_fail(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
 	assert(bo->state < BOS_FINISHED);
-	bo->fetch_objcore->flags |= OC_F_PRIVATE;
+	HSH_Fail(bo->fetch_objcore);
 	if (bo->fetch_objcore->flags & OC_F_BUSY)
 		HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
-	HSH_Fail(bo->fetch_objcore);
 	wrk->stats.fetch_failed++;
 	VBO_setstate(bo, BOS_FAILED);
 	return (F_STP_DONE);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 882c84b..dbcfaee 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -629,6 +629,13 @@ HSH_Fail(struct objcore *oc)
 	oh = oc->objhead;
 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
 
+	/*
+	 * We have to have either a busy bit, so that HSH_Lookup
+	 * will not consider this oc, or an object hung of the oc
+	 * so that it can consider it.
+	 */
+	assert((oc->flags & OC_F_BUSY) || (oc->methods != NULL));
+
 	Lck_Lock(&oh->mtx);
 	oc->flags |= OC_F_FAILED;
 	Lck_Unlock(&oh->mtx);
@@ -678,7 +685,7 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	if (oh->waitinglist != NULL)
 		hsh_rush(ds, oh);
 	Lck_Unlock(&oh->mtx);
-	if (!(oc->flags & OC_F_PRIVATE)) {
+	if (!(oc->flags & OC_F_PRIVATE) && oc->methods != NULL) {
 		BAN_NewObjCore(oc);
 		EXP_Insert(oc);
 		AN(oc->flags & OC_F_EXP);



From perbu at varnish-software.com  Fri Mar 14 10:57:33 2014
From: perbu at varnish-software.com (Per Buer)
Date: Fri, 14 Mar 2014 11:57:33 +0100
Subject: [master] 02843b8 Fix URL
Message-ID: <E1WOPnx-0005kR-Qc@project.varnish-software.com>


commit 02843b8a0d70e5fe73d522f0ac8cde3e7a5032af
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Mar 14 11:57:31 2014 +0100

    Fix URL

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index a4b6f7b..6f91b91 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -21,7 +21,7 @@ is then dynamically linked into the server process.
 
 This document focuses on the syntax of the VCL language. For a full
 description of syntax and semantics, with ample examples, please see
-the users guide at https://www.varnish-cache.org/doc/
+the users guide at https://www.varnish-cache.org/docs/
 
 VCL consists of the following elements:
  * Operators



From tfheen at err.no  Fri Mar 14 18:25:59 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Fri, 14 Mar 2014 19:25:59 +0100
Subject: [master] 79c241e Typos and itsos
Message-ID: <E1WOWnv-0006iH-Jd@project.varnish-software.com>


commit 79c241e54e08a742e591b20bde253ec16f9f346d
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Fri Mar 14 19:25:47 2014 +0100

    Typos and itsos

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 3f65968..50a7296 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -3,7 +3,7 @@
 The fundamentals of web proxy caching with Varnish
 --------------------------------------------------
 
-Varnish is a caching HTTP reverse proxy. It recieves requests from
+Varnish is a caching HTTP reverse proxy. It receives requests from
 clients and tries to answer them from its cache. If it cannot answer
 the request from its cache it will forward the request to the backend,
 fetch the response, store it and deliver it to the client.
@@ -14,11 +14,11 @@ typical application server, so you want to make sure to have it answer
 as many of the requests as possible.
 
 Varnish decides whether it can store the content or not based on the
-response it's gets back from the backend. The backend can instruct
+response it gets back from the backend. The backend can instruct
 Varnish to cache the content with the HTTP response header
 Cache-Control. There are a few other conditions where Varnish will not
 cache, the most common one being cookies. Since cookies is a good
-indication that a web object is personlised, Varnish will with it's
+indication that a web object is personalised, Varnish will with its
 default configuration, not cache it.
 
 This behaviour and most other behaviour can be changed using policies
@@ -29,7 +29,7 @@ Performance
 ~~~~~~~~~~~
 
 Varnish has a modern architecture and is written with performance in
-mind.  It is usually bound by the speed of the network, effectivly
+mind.  It is usually bound by the speed of the network, effectively
 turning performance into a non-issue. You get to focus on how your web
 application work and you can allow yourself, to some degree, to care
 less about performance and scalability.
@@ -37,8 +37,8 @@ less about performance and scalability.
 Flexibility
 ~~~~~~~~~~~
 
-One of the key features of Varnish Cache, in addition to it's
-performance, is the flexibility of it's configuration language,
+One of the key features of Varnish Cache, in addition to its
+performance, is the flexibility of its configuration language,
 VCL. VCL enables you to write policies on how incoming requests should
 be handled. 
 
@@ -68,7 +68,7 @@ You can get in touch with us trough many channels. For real time chat
 you can reach us on IRC trough the server irc.linpro.net on the
 #varnish and #varnish-hacking channels.
 The are two mailing lists available. One for user questions and one
-for development discussions. See varnish-cache.org/mailinglist for
+for development discussions. See https://www.varnish-cache.org/mailinglist for
 information and signup.  There is also a web forum on the same site.
 
 Now that you have a vague idea on what Varnish Cache is, let see if we



From phk at FreeBSD.org  Mon Mar 17 09:46:40 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 17 Mar 2014 10:46:40 +0100
Subject: [master] 44d4609 Don't unbusy failed objects,
 it is pointless and introduces a race.
Message-ID: <E1WPU80-0003Vc-Jw@project.varnish-software.com>


commit 44d46093dd34b4db102964a49bc90d7039f65db2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 17 09:46:22 2014 +0000

    Don't unbusy failed objects, it is pointless and introduces a race.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b5cb463..2f5bb9c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -683,8 +683,6 @@ vbf_stp_fail(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->state < BOS_FINISHED);
 	HSH_Fail(bo->fetch_objcore);
-	if (bo->fetch_objcore->flags & OC_F_BUSY)
-		HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
 	wrk->stats.fetch_failed++;
 	VBO_setstate(bo, BOS_FAILED);
 	return (F_STP_DONE);
@@ -738,9 +736,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	}
 	assert(WRW_IsReleased(wrk));
 
-	AZ(bo->fetch_objcore->flags & OC_F_BUSY);
-	HSH_Complete(bo->fetch_objcore);
-
 	bo->stats = NULL;
 
 	if (bo->vbc != NULL) {
@@ -756,6 +751,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	if (bo->state == BOS_FINISHED) {
 		assert(!(bo->fetch_objcore->flags & OC_F_FAILED));
+		HSH_Complete(bo->fetch_objcore);
 		VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
 		{
 		/* Sanity check fetch methods accounting */



From lkarsten at varnish-software.com  Mon Mar 17 11:57:23 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 17 Mar 2014 12:57:23 +0100
Subject: [master] e7466ac Reserve h key for a future help screen
Message-ID: <E1WPWAV-0008L9-4M@project.varnish-software.com>


commit e7466ac4183f6271560ffd0ce48617fc29607419
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Mar 17 12:53:02 2014 +0100

    Reserve h key for a future help screen
    
    Move showing of debug counters from 'h' key to 'd' inside a running
    varnishstat.

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index e722408..6e8bae8 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -821,7 +821,7 @@ handle_keypress(int ch)
 		current = n_ptarray - 1;
 		page_start = current - l_points + 1;
 		break;
-	case 'h':
+	case 'd':
 		hide_unseen = 1 - hide_unseen;
 		rebuild = 1;
 		break;



From lkarsten at varnish-software.com  Mon Mar 17 12:08:05 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 17 Mar 2014 13:08:05 +0100
Subject: [master] 793d67a Helpful pointers for future contributors
Message-ID: <E1WPWKr-0000sP-KI@project.varnish-software.com>


commit 793d67ad00916ab55d089e932673a8000dffb62a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Mar 17 13:06:36 2014 +0100

    Helpful pointers for future contributors
    
    Mostly an attempt at pointing people that send pull requests on Github
    in the right direction.

diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 100644
index 0000000..cfccc09
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1,15 @@
+Contributing to Varnish Cache
+=============================
+
+Varnish Cache is developed with Git hosted on our own infrastructure.
+
+Official development tree is here:
+
+    git://git.varnish-cache.org/varnish-cache
+
+Patches can be sent to varnish-dev at varnish-cache.org.
+
+There is a READ-ONLY git-mirror on Github.  Please do not send contributions
+there, we need them by email to -dev@ for now.
+
+



From lkarsten at varnish-software.com  Mon Mar 17 13:19:51 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 17 Mar 2014 14:19:51 +0100
Subject: [master] a4d1d5d Small example on how to do synthetic responses
Message-ID: <E1WPXSJ-0007jp-CH@project.varnish-software.com>


commit a4d1d5d2ad8dd3d9a1c585e1d5e009bb81352944
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Mar 17 14:19:46 2014 +0100

    Small example on how to do synthetic responses

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 68a41d0..a0da998 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -41,6 +41,24 @@ You must now explicitly return an error::
 
 	return(error(999, "Response));
 
+
+Synthetic responses in vcl_error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Setting headers on synthetic response bodies made in vcl_error are now done on
+resp.http instead of obj.http.
+
+The synthetic keyword is now a function::
+
+    if (resp.status == 799) {
+        set resp.status = 200;
+        set resp.http.Content-Type = "text/plain; charset=utf-8";
+        synthetic({"You are "} + client.ip);
+        return (deliver);
+    }
+
+
+
 hit_for_pass objects are created using beresp.uncacheable
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Example::



From phk at FreeBSD.org  Mon Mar 17 16:06:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 17 Mar 2014 17:06:30 +0100
Subject: [master] 4e3169c Dump some info on fetch_objcore during panics
Message-ID: <E1WPa3a-0007z7-4u@project.varnish-software.com>


commit 4e3169c626d0591189c4de67f9b08e6a7f9aa44e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 17 16:05:42 2014 +0000

    Dump some info on fetch_objcore during panics

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 8df8bfa..a01e771 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -228,6 +228,19 @@ pan_object(const char *typ, const struct object *o)
 /*--------------------------------------------------------------------*/
 
 static void
+pan_objcore(const char *typ, const struct objcore *oc)
+{
+
+	VSB_printf(pan_vsp, "  objcore (%s) = %p {\n", typ, oc);
+	VSB_printf(pan_vsp, "    refcnt = %d\n", oc->refcnt);
+	VSB_printf(pan_vsp, "    flags = 0x%x\n", oc->flags);
+	VSB_printf(pan_vsp, "    objhead = %p\n", oc->objhead);
+	VSB_printf(pan_vsp, "  }\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
 pan_vcl(const struct VCL_conf *vcl)
 {
 	int i;
@@ -284,6 +297,8 @@ pan_busyobj(const struct busyobj *bo)
 	if (bo->beresp->ws != NULL)
 		pan_http("beresp", bo->beresp, 4);
 	pan_ws(bo->ws_o, 4);
+	if (bo->fetch_objcore)
+		pan_objcore("FETCH", bo->fetch_objcore);
 	if (bo->fetch_obj)
 		pan_object("FETCH", bo->fetch_obj);
 	if (bo->ims_obj)



From phk at FreeBSD.org  Mon Mar 17 16:06:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 17 Mar 2014 17:06:30 +0100
Subject: [master] 99851a1 Try to close the #1449 race definitively.
Message-ID: <E1WPa3a-0007zD-66@project.varnish-software.com>


commit 99851a10c7314dd6a4eb6a0de60161732b1b7b55
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 17 16:06:06 2014 +0000

    Try to close the #1449 race definitively.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 1471a64..a1fcbc3 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -137,7 +137,7 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 
 	AZ(oc->flags & (OC_F_OFFLRU | OC_F_INSERT | OC_F_MOVE));
 	AZ(oc->flags & OC_F_DYING);
-	AZ(oc->flags & OC_F_BUSY);
+	// AN(oc->flags & OC_F_BUSY);
 
 	if (lru == NULL)
 		lru = oc_getlru(oc);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2f5bb9c..42471ed 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -565,9 +565,10 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	obj->gzip_stop = bo->ims_obj->gzip_stop;
 
 	AZ(WS_Overflowed(bo->ws_o));
-	if (bo->do_stream)
+	if (bo->do_stream) {
+		HSH_Unbusy(&wrk->stats, obj->objcore);
 		VBO_setstate(bo, BOS_STREAM);
-	HSH_Unbusy(&wrk->stats, obj->objcore);
+	}
 
 	st = NULL;
 	al = 0;
@@ -596,6 +597,9 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->failed)
 		return (F_STP_FAIL);
 
+	if (!bo->do_stream)
+		HSH_Unbusy(&wrk->stats, obj->objcore);
+
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
 	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
@@ -802,6 +806,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 
 	bo = VBO_GetBusyObj(wrk, req);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	THR_SetBusyobj(bo);
 
 	switch(mode) {
 	case VBF_PASS:		how = "pass"; break;
@@ -853,7 +858,12 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 		VBO_waitstate(bo, BOS_REQ_DONE);
 	} else {
 		VBO_waitstate(bo, BOS_STREAM);
-		assert(bo->state != BOS_FAILED || (oc->flags & OC_F_FAILED));
+		if (bo->state == BOS_FAILED) {
+			AN((oc->flags & OC_F_FAILED));
+		} else {
+			AZ(bo->fetch_objcore->flags & OC_F_BUSY);
+		}
 	}
+	THR_SetBusyobj(NULL);
 	VBO_DerefBusyObj(wrk, &bo);
 }
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index dbcfaee..07c5931 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -672,9 +672,17 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	oh = oc->objhead;
 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
 
+	AN(oc->methods);
 	AN(oc->flags & OC_F_BUSY);
 	assert(oh->refcnt > 0);
 
+	if (!(oc->flags & OC_F_PRIVATE)) {
+		BAN_NewObjCore(oc);
+		EXP_Insert(oc);
+		AN(oc->flags & OC_F_EXP);
+		AN(oc->ban);
+	}
+
 	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
@@ -685,12 +693,6 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	if (oh->waitinglist != NULL)
 		hsh_rush(ds, oh);
 	Lck_Unlock(&oh->mtx);
-	if (!(oc->flags & OC_F_PRIVATE) && oc->methods != NULL) {
-		BAN_NewObjCore(oc);
-		EXP_Insert(oc);
-		AN(oc->flags & OC_F_EXP);
-		AN(oc->ban);
-	}
 }
 
 /*---------------------------------------------------------------------



From phk at FreeBSD.org  Tue Mar 18 09:26:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 10:26:00 +0100
Subject: [master] 8aec118 Comment out an assert to get jenkins happier
Message-ID: <E1WPqHY-0007l0-Cb@project.varnish-software.com>


commit 8aec1184ecb4d72ccaf70e9d8bf40139cf72a024
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 09:25:43 2014 +0000

    Comment out an assert to get jenkins happier

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index a1fcbc3..dfec9ee 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -427,7 +427,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inbox p=%p e=%.9f f=0x%x", oc,
 	    oc->timer_when, oc->flags);
 
-	AZ(oc->flags & OC_F_BUSY);
+	// AZ(oc->flags & OC_F_BUSY);
 
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);



From phk at FreeBSD.org  Tue Mar 18 09:35:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 10:35:00 +0100
Subject: [master] 07ff86d Add varnishhist
Message-ID: <E1WPqQG-0008Ek-S0@project.varnish-software.com>


commit 07ff86df5aabda77abf3f013dd2e16fd4a32c5a8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 09:34:54 2014 +0000

    Add varnishhist

diff --git a/bin/Makefile.phk b/bin/Makefile.phk
index 5380231..07ed226 100644
--- a/bin/Makefile.phk
+++ b/bin/Makefile.phk
@@ -5,7 +5,7 @@ SUBDIRS += varnishncsa
 # varnishreplay
 SUBDIRS += varnishtest 
 SUBDIRS += varnishstat
-# SUBDIRS += varnishhist 
+SUBDIRS += varnishhist 
 SUBDIRS += varnishtop
 
 TOPDIR = $(CURDIR)/..



From tfheen at err.no  Tue Mar 18 10:01:14 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Tue, 18 Mar 2014 11:01:14 +0100
Subject: [master] 166c557 Typo in makefile.phk
Message-ID: <E1WPqpe-0001Ga-Ho@project.varnish-software.com>


commit 166c557144414588c5ab0557428f5a280669c53c
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Tue Mar 18 11:00:39 2014 +0100

    Typo in makefile.phk

diff --git a/bin/varnishd/Makefile.phk b/bin/varnishd/Makefile.phk
index f8bf48a..9922d9f 100644
--- a/bin/varnishd/Makefile.phk
+++ b/bin/varnishd/Makefile.phk
@@ -104,7 +104,7 @@ include $(TOPDIR)/Makefile.inc.phk
 #
 # Turn the builtin.vcl file into a C-string we can include in the program.
 #
-builtint_vcl.h:	builtin.vcl
+builtin_vcl.h:	builtin.vcl
 	echo '/*' > $@
 	echo ' * NB:  This file is machine generated, DO NOT EDIT!' >> $@
 	echo ' *' >> $@



From phk at FreeBSD.org  Tue Mar 18 10:35:02 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 11:35:02 +0100
Subject: [master] 70fda4f Comment out another busy check
Message-ID: <E1WPrMM-0001pG-0Q@project.varnish-software.com>


commit 70fda4f4d85465c9672c208bab5a5343e35d92b6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 10:34:39 2014 +0000

    Comment out another busy check

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 94fdf32..eac7546 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -456,7 +456,7 @@ oc_getobj(struct dstat *ds, struct objcore *oc)
 {
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AZ(oc->flags & OC_F_BUSY);
+	// AZ(oc->flags & OC_F_BUSY);
 	AN(oc->methods);
 	AN(oc->methods->getobj);
 	return (oc->methods->getobj(ds, oc));



From phk at FreeBSD.org  Tue Mar 18 10:41:42 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 11:41:42 +0100
Subject: [master] 44b8341 White space fix
Message-ID: <E1WPrSo-0002Je-LE@project.varnish-software.com>


commit 44b8341390a05677d79a5c04cfce61d17a3bbc02
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 10:36:35 2014 +0000

    White space fix

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 48b0175..523f05f 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -161,8 +161,8 @@ sp_variables = [
 		'IP',
 		( 'both',),
 		( ), """
-The client's IP address.
-"""
+		The client's IP address.
+		"""
 	),
 	('client.identity',
 		'STRING',



From phk at FreeBSD.org  Tue Mar 18 11:13:02 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 12:13:02 +0100
Subject: [master] 2247fc8 More commenting out busy checks
Message-ID: <E1WPrx8-0004u4-VX@project.varnish-software.com>


commit 2247fc840d9fe8e44b31cddd81be0d8f646b361c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 11:12:52 2014 +0000

    More commenting out busy checks

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index dfec9ee..4f21152 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -513,7 +513,7 @@ exp_expire(struct exp_priv *ep, double now)
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	Lck_Lock(&lru->mtx);
-	AZ(oc->flags & OC_F_BUSY);
+	// AZ(oc->flags & OC_F_BUSY);
 	oc->flags |= OC_F_DYING;
 	if (oc->flags & OC_F_OFFLRU)
 		oc = NULL;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 42471ed..5b5be58 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -203,11 +203,10 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 		if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
 			http_PrintfHeader(bo->bereq0,
 			    "If-Modified-Since: %s", p);
-		} else if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) {
+		}
+		if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) {
 			http_PrintfHeader(bo->bereq0,
 			    "If-None-Match: %s", p);
-		} else {
-			WRONG("Shouldn't have bo->ims_obj");
 		}
 	}
 



From tfheen at err.no  Tue Mar 18 11:53:55 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Tue, 18 Mar 2014 12:53:55 +0100
Subject: [master] ae54868 Move X-Forwarded-For handled into C code
Message-ID: <E1WPsah-0000A4-K3@project.varnish-software.com>


commit ae548683b8f91d0a92799f6c746b80773a4c9f05
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Tue Mar 18 12:18:33 2014 +0100

    Move X-Forwarded-For handled into C code
    
    Remove adding of X-Forwarded-For from builtin.vcl and put it into the
    C code that runs before vcl_recv.  This makes it more likely that it's
    properly set when the user VCL runs.
    
    Fixes: #1454

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index da30346..ac795fd 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -45,14 +45,6 @@ vcl 4.0;
 # Client side
 
 sub vcl_recv {
-    if (req.restarts == 0) {
-        if (req.http.x-forwarded-for) {
-            set req.http.X-Forwarded-For =
-                req.http.X-Forwarded-For + ", " + client.ip;
-        } else {
-            set req.http.X-Forwarded-For = client.ip;
-        }
-    }
     if (req.method != "GET" &&
       req.method != "HEAD" &&
       req.method != "PUT" &&
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 467ea8d..5df16e6 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -660,6 +660,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 {
 	unsigned recv_handling;
 	struct SHA256Context sha256ctx;
+	char *xff;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -688,6 +689,16 @@ cnt_recv(struct worker *wrk, struct req *req)
 	req->hash_always_miss = 0;
 	req->hash_ignore_busy = 0;
 	req->client_identity = NULL;
+	if (req->restarts == 0) {
+		if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) {
+			http_Unset(req->http, H_X_Forwarded_For);
+			http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", xff,
+					  req->sp->client_addr_str);
+		} else {
+			http_PrintfHeader(req->http, "X-Forwarded-For: %s",
+					  req->sp->client_addr_str);
+		}
+	}
 
 	http_CollectHdr(req->http, H_Cache_Control);
 
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index 758c7e6..1db7784 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -94,5 +94,6 @@ HTTPH("Vary",			H_Vary,			0					  )	/* RFC2616 14.44 */
 HTTPH("Via",			H_Via,			0					  )	/* RFC2616 14.45 */
 HTTPH("Warning",		H_Warning,		0					  )	/* RFC2616 14.46 */
 HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	0					  )	/* RFC2616 14.47 */
+HTTPH("X-Forwarded-For",	H_X_Forwarded_For,	0					  )	/* Not RFC */
 
 /*lint -restore */



From tfheen at err.no  Tue Mar 18 11:53:55 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Tue, 18 Mar 2014 12:53:55 +0100
Subject: [master] 880c3c2 Set Connection: close before calling vcl_pipe
Message-ID: <E1WPsah-0000AA-L8@project.varnish-software.com>


commit 880c3c2a564aeeaad508dab136611973ca71d6aa
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Tue Mar 18 12:43:54 2014 +0100

    Set Connection: close before calling vcl_pipe
    
    It makes sense to default to setting Connection: close for vcl_pipe.
    Do this before calling vcl_pipe so it's overrideable.
    
    Fixes #1455

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 5df16e6..f43416f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -579,6 +579,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
+	http_SetHeader(bo->bereq, "Connection: close");
 
 	VCL_pipe_method(req->vcl, wrk, req, bo, req->http->ws);
 



From phk at FreeBSD.org  Tue Mar 18 13:25:20 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 14:25:20 +0100
Subject: [master] fa955a8 Make this test more robust
Message-ID: <E1WPu1A-0005d5-7K@project.varnish-software.com>


commit fa955a8134f64510db446967f8ea45736870824a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 13:25:06 2014 +0000

    Make this test more robust

diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index fcc9594..543f93f 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -43,7 +43,7 @@ client c1 {
 varnish v1 -expect bans_lurker_tests_tested == 0
 
 delay 1.0
-varnish v1 -expect bans_lurker_tests_tested == 1
+varnish v1 -expect bans_lurker_tests_tested >= 1
 
 varnish v1 -cliok "param.set ban_lurker_sleep 5.01"
 



From phk at FreeBSD.org  Tue Mar 18 16:31:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 17:31:55 +0100
Subject: [master] 05a462d Rename vcl_error{} to vcl_synth{}
Message-ID: <E1WPwvj-0006n1-G6@project.varnish-software.com>


commit 05a462d98c3ee38c862c9006924f8555d133f95d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 16:31:34 2014 +0000

    Rename vcl_error{} to vcl_synth{}

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index ac795fd..cdac8d9 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -126,7 +126,7 @@ sub vcl_deliver {
 /*
  * We can come here "invisibly" with the following errors: 413, 417 & 503
  */
-sub vcl_error {
+sub vcl_synth {
     set resp.http.Content-Type = "text/html; charset=utf-8";
     set resp.http.Retry-After = "5";
     synthetic( {"
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f43416f..792b1ec 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -193,7 +193,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	req->acct_req.error++;
+	req->acct_req.synth++;
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	h = req->resp;
@@ -219,7 +219,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	req->synth_body = VSB_new_auto();
 	AN(req->synth_body);
 
-	VCL_error_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_synth_method(req->vcl, wrk, req, NULL, req->http->ws);
 
 	http_Unset(h, H_Content_Length);
 
@@ -421,7 +421,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	case VCL_RET_RESTART:
 		req->req_step = R_STP_RESTART;
 		break;
-	case VCL_RET_ERROR:
+	case VCL_RET_SYNTH:
 		req->req_step = R_STP_ERROR;
 		break;
 	case VCL_RET_PASS:
@@ -479,7 +479,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 		VBF_Fetch(wrk, req, req->objcore, o, VBF_NORMAL);
 		req->req_step = R_STP_FETCH;
 		return (REQ_FSM_MORE);
-	case VCL_RET_ERROR:
+	case VCL_RET_SYNTH:
 		req->req_step = R_STP_ERROR;
 		break;
 	case VCL_RET_RESTART:
@@ -527,7 +527,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 
 	VCL_pass_method(req->vcl, wrk, req, NULL, req->http->ws);
 	switch (wrk->handling) {
-	case VCL_RET_ERROR:
+	case VCL_RET_SYNTH:
 		req->req_step = R_STP_ERROR;
 		break;
 	case VCL_RET_RESTART:
@@ -583,7 +583,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 
 	VCL_pipe_method(req->vcl, wrk, req, bo, req->http->ws);
 
-	if (wrk->handling == VCL_RET_ERROR)
+	if (wrk->handling == VCL_RET_SYNTH)
 		INCOMPL();
 	assert(wrk->handling == VCL_RET_PIPE);
 
@@ -754,7 +754,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 	case VCL_RET_PASS:
 		req->req_step = R_STP_PASS;
 		return (REQ_FSM_MORE);
-	case VCL_RET_ERROR:
+	case VCL_RET_SYNTH:
 		req->req_step = R_STP_ERROR;
 		return (REQ_FSM_MORE);
 	default:
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index dbe673b..81776bb 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -5,11 +5,11 @@ varnish v1 -vcl {
 		.host = "${bad_ip}"; .port = "9080";
 	}
 	sub vcl_recv {
-		return (error(888));
+		return (synth(888));
 	}
 
-	sub vcl_error {
-		synthetic("Custom vcl_error's synth output");
+	sub vcl_synth {
+		synthetic("Custom vcl_synth's synth output");
 		return (deliver);
 	}
 } -start
@@ -22,4 +22,4 @@ client c1 {
 	expect resp.http.connection != close
 } -run
 
-varnish v1 -expect s_error == 1
+varnish v1 -expect s_synth == 1
diff --git a/bin/varnishtest/tests/b00019.vtc b/bin/varnishtest/tests/b00019.vtc
index 59ceb61..33d3dbc 100644
--- a/bin/varnishtest/tests/b00019.vtc
+++ b/bin/varnishtest/tests/b00019.vtc
@@ -25,7 +25,7 @@ varnish v1 -vcl+backend {
 	       return (restart);
 	}
 
-	sub vcl_error {
+	sub vcl_synth {
 		if (req.restarts == 2) {
 			set resp.status = 200;
 			set resp.reason = "restart=2";
diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc
index a7b6c57..1115613 100644
--- a/bin/varnishtest/tests/b00030.vtc
+++ b/bin/varnishtest/tests/b00030.vtc
@@ -10,9 +10,9 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		return (error(200,req.ttl));
+		return (synth(200,req.ttl));
 	}
-	sub vcl_error {
+	sub vcl_synth {
 		set resp.http.x-timestamp = now;
 	}
 } -start
diff --git a/bin/varnishtest/tests/c00022.vtc b/bin/varnishtest/tests/c00022.vtc
index 841d2cf..39ce5dc 100644
--- a/bin/varnishtest/tests/c00022.vtc
+++ b/bin/varnishtest/tests/c00022.vtc
@@ -22,11 +22,11 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "PURGE") {
 			ban ("req.url == " + req.url);
-			return (error(410));
+			return (synth(410));
 		}
 		if (req.method == "PURGESTR") {
 			ban ("" + req.http.ban);
-			return (error(410));
+			return (synth(410));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/c00024.vtc b/bin/varnishtest/tests/c00024.vtc
index 6167914..80b3c59 100644
--- a/bin/varnishtest/tests/c00024.vtc
+++ b/bin/varnishtest/tests/c00024.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test restart in vcl_error"
+varnishtest "Test restart in vcl_synth"
 
 server s1 {
         rxreq
@@ -8,10 +8,10 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.restarts == 0) {
-			return (error(701, "FOO"));
+			return (synth(701, "FOO"));
 		}
 	}
-	sub vcl_error {
+	sub vcl_synth {
 		if (req.restarts < 1) {
 			return (restart);
 		} else {
diff --git a/bin/varnishtest/tests/c00028.vtc b/bin/varnishtest/tests/c00028.vtc
index 91abe45..0ad1466 100644
--- a/bin/varnishtest/tests/c00028.vtc
+++ b/bin/varnishtest/tests/c00028.vtc
@@ -8,7 +8,7 @@ varnish v1 -vcl {
 	sub vcl_backend_fetch {
 	    set bereq.backend = bad;
 	}
-	sub vcl_error {
+	sub vcl_synth {
 	    return (restart);
 	}
  } -start
diff --git a/bin/varnishtest/tests/c00061.vtc b/bin/varnishtest/tests/c00061.vtc
index c99f324..ef2ae76 100644
--- a/bin/varnishtest/tests/c00061.vtc
+++ b/bin/varnishtest/tests/c00061.vtc
@@ -8,7 +8,7 @@ varnish v1 -vcl {
 		return (retry);
 	}
 
-	sub vcl_error {
+	sub vcl_synth {
 		set resp.status = 504;
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00558.vtc b/bin/varnishtest/tests/r00558.vtc
index 3a8ba97..c889e79 100644
--- a/bin/varnishtest/tests/r00558.vtc
+++ b/bin/varnishtest/tests/r00558.vtc
@@ -7,7 +7,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		return (error(501));
+		return (synth(501));
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00561.vtc b/bin/varnishtest/tests/r00561.vtc
index 1efc0a7..416a97d 100644
--- a/bin/varnishtest/tests/r00561.vtc
+++ b/bin/varnishtest/tests/r00561.vtc
@@ -1,4 +1,4 @@
-varnishtest "Junk request should not go to vcl_error"
+varnishtest "Junk request should not go to vcl_synth"
 
 server s1 {
 	rxreq
@@ -6,7 +6,7 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_error {
+	sub vcl_synth {
 		return (restart);
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00769.vtc b/bin/varnishtest/tests/r00769.vtc
index 451029c..6778648 100644
--- a/bin/varnishtest/tests/r00769.vtc
+++ b/bin/varnishtest/tests/r00769.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.url ~ "^/test1") {
-			return (error(700));
+			return (synth(700));
 		}
 	}
 	sub vcl_backend_response {
@@ -16,7 +16,7 @@ varnish v1 -vcl+backend {
 		set beresp.http.X-status = beresp.status;
 		return (deliver);
 	}
-	sub vcl_error {
+	sub vcl_synth {
 		if (resp.status == 700) {
 			set resp.status=404;
 			set resp.http.X-status = resp.status;
diff --git a/bin/varnishtest/tests/r00896.vtc b/bin/varnishtest/tests/r00896.vtc
index c713a59..0e0c086 100644
--- a/bin/varnishtest/tests/r00896.vtc
+++ b/bin/varnishtest/tests/r00896.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 	    if (req.http.host ~ "^(abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij)") {
-		return (error(500,"not ok"));
+		return (synth(500,"not ok"));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00936.vtc b/bin/varnishtest/tests/r00936.vtc
index 7e6ad64..ddb2f79 100644
--- a/bin/varnishtest/tests/r00936.vtc
+++ b/bin/varnishtest/tests/r00936.vtc
@@ -10,7 +10,7 @@ varnish v1 -errvcl {'synthetic': not a valid action in method 'vcl_recv'} {
 
 	sub vcl_recv {
 		synthetic("HELLOO");
-		return (error(503));
+		return (synth(503));
 	}
 }
 
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index d9cbdba..e094e3b 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -27,19 +27,19 @@ varnish v1 -vcl+backend {
 		}
 	}
 
-	sub vcl_error {
+	sub vcl_synth {
 		if (resp.status == 988) { return (restart); }
 	}
 
 	sub vcl_miss {
 		if (req.http.X-Banned == "check") {
-			return (error(988,"restarting"));
+			return (synth(988,"restarting"));
 		}
 	}
 
 	sub vcl_pass {
 		if (req.http.X-Banned == "check") {
-			return (error(988,"restarting"));
+			return (synth(988,"restarting"));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/r01027.vtc b/bin/varnishtest/tests/r01027.vtc
index d6d0991..716e6f0 100644
--- a/bin/varnishtest/tests/r01027.vtc
+++ b/bin/varnishtest/tests/r01027.vtc
@@ -1,9 +1,9 @@
 varnishtest "Test if you can error in vcl_deliver"
 
-varnish v1 -errvcl {Invalid return "error"} {
+varnish v1 -errvcl {Invalid return "synth"} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_deliver {
-		return (error(201,"ok"));
+		return (synth(201,"ok"));
 	}
 }
 
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index 543f93f..a84e3a5 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -16,7 +16,7 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "BAN") {
 			ban("obj.http.url ~ /");
-			return (error(201,"banned"));
+			return (synth(201,"banned"));
 		}
 		return (hash);
 	}
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
index 7e329ac..bfb220a 100644
--- a/bin/varnishtest/tests/r01287.vtc
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -5,9 +5,9 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		return (error(200, "OK"));
+		return (synth(200, "OK"));
 	}
-	sub vcl_error {
+	sub vcl_synth {
 		synthetic(resp.http.blank);
 		return (deliver);
 	}
diff --git a/bin/varnishtest/tests/r01367.vtc b/bin/varnishtest/tests/r01367.vtc
index 3a57258..18c9ca5 100644
--- a/bin/varnishtest/tests/r01367.vtc
+++ b/bin/varnishtest/tests/r01367.vtc
@@ -6,7 +6,7 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_error {
+	sub vcl_synth {
 		return (restart);
 	}
 } -start
diff --git a/bin/varnishtest/tests/r01395.vtc b/bin/varnishtest/tests/r01395.vtc
index acad0fe..591bb2e 100644
--- a/bin/varnishtest/tests/r01395.vtc
+++ b/bin/varnishtest/tests/r01395.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test vcl_error is called even if vcl_backend_fetch failed"
+varnishtest "Test vcl_synth is called even if vcl_backend_fetch failed"
 
 varnish v1 -vcl {
 	backend default {
diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc
index 3a3d842..0f470ec 100644
--- a/bin/varnishtest/tests/v00011.vtc
+++ b/bin/varnishtest/tests/v00011.vtc
@@ -18,7 +18,7 @@ varnish v1 -vcl+backend {
 			ban("req.url == bar //");
 			ban("foo == bar //");
 			ban("req.url ~ ^/$");
-			return (error(209,"foo"));
+			return (synth(209,"foo"));
 		}
 	}
 
diff --git a/bin/varnishtest/tests/v00014.vtc b/bin/varnishtest/tests/v00014.vtc
index 3a48188..8f776b7 100644
--- a/bin/varnishtest/tests/v00014.vtc
+++ b/bin/varnishtest/tests/v00014.vtc
@@ -28,9 +28,9 @@ varnish v1 -vcl {
 
 	sub vcl_recv {
 		if (std.healthy(default)) {
-			return(error(200,"Backend healthy"));
+			return(synth(200,"Backend healthy"));
 		} else {
-			return(error(500,"Backend sick"));
+			return(synth(500,"Backend sick"));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 9e97762..004ec15 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -8,8 +8,8 @@ varnish v1 -vcl {
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_miss { return(error(100,req.url)); }
-	sub vcl_hit { return(error(100,"the butter please")); }
+	sub vcl_miss { return(synth(100,req.url)); }
+	sub vcl_hit { return(synth(100,"the butter please")); }
 }
 
 varnish v1 -errvcl {Variable 'now' is read only.} {
@@ -39,7 +39,7 @@ varnish v1 -errvcl {Expected '=' got '+='} {
 
 varnish v1 -errvcl {Expected ';' got 'if'} {
 	backend b { .host = "127.0.0.1"; }
-	/* XXX: This should not really be an error */
+	/* XXX: This should not really be an synth */
 	sub vcl_recv { set req.url = "foo" if "bar"; }
 }
 
@@ -100,15 +100,10 @@ varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
 
 varnish v1 -errvcl {Unknown token '<<' when looking for STRING_LIST} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_error { synthetic( << "foo"; }
+	sub vcl_synth { synthetic( << "foo"; }
 }
 
 varnish v1 -errvcl {Syntax has changed, use:} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { error 200; }
-}
-
-varnish v1 -errvcl {Syntax has changed, use:} {
-	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { return(error); }
+	sub vcl_recv { return(synth); }
 }
diff --git a/bin/varnishtest/tests/v00024.vtc b/bin/varnishtest/tests/v00024.vtc
index f112559..e502eee 100644
--- a/bin/varnishtest/tests/v00024.vtc
+++ b/bin/varnishtest/tests/v00024.vtc
@@ -9,7 +9,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.http.etag == req.http.if-none-match) {
-			return(error(400,"FOO"));
+			return(synth(400,"FOO"));
 		}
 	}
 } -start
diff --git a/include/tbl/acct_fields.h b/include/tbl/acct_fields.h
index 1184240..ac205a7 100644
--- a/include/tbl/acct_fields.h
+++ b/include/tbl/acct_fields.h
@@ -35,6 +35,6 @@ ACCT(req)
 ACCT(pipe)
 ACCT(pass)
 ACCT(fetch)
-ACCT(error)
+ACCT(synth)
 ACCT(hdrbytes)
 ACCT(bodybytes)
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index d72b325..3874ec1 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -380,8 +380,8 @@ VSC_F(s_fetch,			uint64_t, 1, 'a', info,
     "Total fetch",
 	""
 )
-VSC_F(s_error,			uint64_t, 1, 'a', info,
-    "Total error",
+VSC_F(s_synth,			uint64_t, 1, 'a', info,
+    "Total synth",
 	""
 )
 VSC_F(s_hdrbytes,		uint64_t, 1, 'a', info,
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 523f05f..7b9ab4e 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -84,15 +84,15 @@ returns =(
 
 	('recv',
 		"C",
-		('error', 'pass', 'pipe', 'hash', 'purge',)
+		('synth', 'pass', 'pipe', 'hash', 'purge',)
 	),
 	('pipe',
 		"C",
-		('error', 'pipe',)
+		('synth', 'pipe',)
 	),
 	('pass',
 		"C",
-		('error', 'restart', 'fetch',)
+		('synth', 'restart', 'fetch',)
 	),
 	('hash',
 		"C",
@@ -100,21 +100,21 @@ returns =(
 	),
 	('purge',
 		"C",
-		('error', 'fetch',)
+		('synth', 'fetch',)
 	),
 	('miss',
 		"C",
-		('error', 'restart', 'pass', 'fetch',)
+		('synth', 'restart', 'pass', 'fetch',)
 	),
 	('hit',
 		"C",
-		('error', 'restart', 'pass', 'fetch', 'deliver',)
+		('synth', 'restart', 'pass', 'fetch', 'deliver',)
 	),
 	('deliver',
 		"C",
 		('restart', 'deliver',)
 	),
-	('error',
+	('synth',
 		"C",
 		('restart', 'deliver',)
 	),
@@ -254,8 +254,8 @@ sp_variables = [
 	),
 	('req.esi',
 		'BOOL',
-		( 'recv', 'backend_response', 'deliver', 'error',),
-		( 'recv', 'backend_response', 'deliver', 'error',), """
+		( 'recv', 'backend_response', 'deliver', 'synth',),
+		( 'recv', 'backend_response', 'deliver', 'synth',), """
 		Boolean. Set to false to disable ESI processing
 		regardless of any value in beresp.do_esi. Defaults
 		to true. This variable is subject to change in
@@ -497,15 +497,15 @@ sp_variables = [
 	),
 	('obj.status',
 		'INT',
-		( 'error',),
-		( 'error',), """
+		( 'synth',),
+		( 'synth',), """
 		The HTTP status code returned by the server.
 		"""
 	),
 	('obj.reason',
 		'STRING',
-		( 'error',),
-		( 'error',), """
+		( 'synth',),
+		( 'synth',), """
 		The HTTP status message returned by the server.
 		"""
 	),
@@ -563,29 +563,29 @@ sp_variables = [
 	),
 	('resp.proto',
 		'STRING',
-		( 'deliver', 'error', ),
-		( 'deliver', 'error', ), """
+		( 'deliver', 'synth', ),
+		( 'deliver', 'synth', ), """
 		The HTTP protocol version to use for the response.
 		"""
 	),
 	('resp.status',
 		'INT',
-		( 'deliver', 'error', ),
-		( 'deliver', 'error', ), """
+		( 'deliver', 'synth', ),
+		( 'deliver', 'synth', ), """
 		The HTTP status code that will be returned.
 		"""
 	),
 	('resp.reason',
 		'STRING',
-		( 'deliver', 'error', ),
-		( 'deliver', 'error', ), """
+		( 'deliver', 'synth', ),
+		( 'deliver', 'synth', ), """
 		The HTTP status message that will be returned.
 		"""
 	),
 	('resp.http.',
 		'HEADER',
-		( 'deliver', 'error', ),
-		( 'deliver', 'error', ), """
+		( 'deliver', 'synth', ),
+		( 'deliver', 'synth', ), """
 		The corresponding HTTP header.
 		"""
 	),
@@ -1011,6 +1011,7 @@ def one_var(nm, spec):
 		if nm == i[0]:
 			fh.write("VCL_" + typ +
 			    " VRT_r_%s(const struct vrt_ctx *);\n" % cnam )
+	print(spec)
 	restrict(fo, spec[2])
 
 	if len(spec[3]) == 0:
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 5ed9146..602beff 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -54,21 +54,6 @@ parse_call(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_error(struct vcc *tl)
-{
-
-	VSB_printf(tl->sb,
-	    "Syntax has changed, use:\n"
-	    "\treturn(error(999));\n"
-	    "or\n"
-	    "\treturn(error(999, \"Response text\"));\n");
-	vcc_ErrWhere(tl, tl->t);
-	return;
-}
-
-/*--------------------------------------------------------------------*/
-
 static const struct arith {
 	enum var_type		type;
 	unsigned		oper;
@@ -311,14 +296,14 @@ parse_return(struct vcc *tl)
 	ExpectErr(tl, ID);
 
 	/* 'error' gets special handling, to allow optional status/response */
-	if (vcc_IdIs(tl->t, "error")) {
+	if (vcc_IdIs(tl->t, "synth")) {
 		vcc_NextToken(tl);
 		if (tl->t->tok == ')') {
 			VSB_printf(tl->sb,
 			    "Syntax has changed, use:\n"
-			    "\treturn(error(999));\n"
+			    "\treturn(synth(999));\n"
 			    "or\n"
-			    "\treturn(error(999, \"Response text\"));\n");
+			    "\treturn(synth(999, \"Response text\"));\n");
 			vcc_ErrWhere(tl, tl->t);
 			return;
 		}
@@ -340,9 +325,9 @@ parse_return(struct vcc *tl)
 		ExpectErr(tl, ')');
 		vcc_NextToken(tl);
 		Fb(tl, 1, ");\n");
-		Fb(tl, 1, "VRT_handling(ctx, VCL_RET_ERROR);\n");
+		Fb(tl, 1, "VRT_handling(ctx, VCL_RET_SYNTH);\n");
 		Fb(tl, 1, "return (1);\n");
-		vcc_ProcAction(tl->curproc, VCL_RET_ERROR, tl->t);
+		vcc_ProcAction(tl->curproc, VCL_RET_SYNTH, tl->t);
 		ExpectErr(tl, ')');
 		vcc_NextToken(tl);
 		return;
@@ -422,7 +407,6 @@ static struct action_table {
 	/* Keep list sorted from here */
 	{ "ban",		parse_ban },
 	{ "call",		parse_call },
-	{ "error",		parse_error },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "new",		parse_new, VCL_MET_INIT},
 	{ "purge",		parse_purge, VCL_MET_MISS | VCL_MET_HIT },
@@ -430,7 +414,7 @@ static struct action_table {
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },
 	{ "synthetic",		parse_synthetic,
-		VCL_MET_ERROR | VCL_MET_BACKEND_ERROR },
+		VCL_MET_SYNTH | VCL_MET_BACKEND_ERROR },
 	{ "unset",		parse_unset },
 	{ NULL,			NULL }
 };



From nils.goroll at uplex.de  Tue Mar 18 15:51:56 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 18 Mar 2014 16:51:56 +0100
Subject: [master] 40112d5 reference to wiki page
Message-ID: <E1WPwJ2-0007VR-OL@project.varnish-software.com>


commit 40112d580e27501df3e297ff2f5cd586a02b4a2c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Mar 18 16:51:44 2014 +0100

    reference to wiki page

diff --git a/CONTRIBUTING b/CONTRIBUTING
index cfccc09..2c2db8c 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -1,6 +1,9 @@
 Contributing to Varnish Cache
 =============================
 
+To contribute to Varnish, read https://www.varnish-cache.org/trac/wiki/Contributing
+as a staring point and follow relevant parts linked there.
+
 Varnish Cache is developed with Git hosted on our own infrastructure.
 
 Official development tree is here:



From nils.goroll at uplex.de  Tue Mar 18 16:05:06 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 18 Mar 2014 17:05:06 +0100
Subject: [master] 2e12bb7 another reference to the Contributing wiki page
Message-ID: <E1WPwVm-0000FZ-Gr@project.varnish-software.com>


commit 2e12bb7fde08aa294cb1e57947f0fc90d515bf71
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Mar 18 17:05:02 2014 +0100

    another reference to the Contributing wiki page

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 50a7296..ad8784a 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -61,6 +61,9 @@ the Varnish Governing Board which currently consist of Poul-Henning
 Kamp (Architect), Rogier Mulhuijzen (Fastly) and Kristian Lyngst?l
 (Varnish Software).
 
+Please see https://www.varnish-cache.org/trac/wiki/Contributing as
+a starting point if you would like to contribute to Varnish.
+
 Getting in touch
 ----------------
 



From nils.goroll at uplex.de  Tue Mar 18 16:10:20 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 18 Mar 2014 17:10:20 +0100
Subject: [master] dbc9a28 solaris platforms
Message-ID: <E1WPwaq-0002iR-9C@project.varnish-software.com>


commit dbc9a2822e965aabbfe29f43b6f7273bf1576492
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Mar 18 17:07:21 2014 +0100

    solaris platforms

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index ad8784a..9e6c541 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -51,7 +51,8 @@ Supported platforms
 
 Varnish is written to run on modern versions of Linux and FreeBSD and
 the best experience is had on those platforms. Thanks to our
-contributors it also runs on NetBSD, OpenBSD and OS X.
+contributors it also runs on NetBSD, OpenBSD, OS X and various
+Solaris-descendants like Oracle Solaris, OmniOS and SmartOS.
 
 About the Varnish development process
 -------------------------------------



From nils.goroll at uplex.de  Tue Mar 18 16:10:20 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 18 Mar 2014 17:10:20 +0100
Subject: [master] 7ebc542 platform notes about solaris
Message-ID: <E1WPwaq-0002iX-Bl@project.varnish-software.com>


commit 7ebc54253fc14604a6ca7c28e13d12371f6e72f9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Mar 18 17:10:15 2014 +0100

    platform notes about solaris

diff --git a/doc/sphinx/phk/platforms.rst b/doc/sphinx/phk/platforms.rst
index 2b60443..6f448e6 100644
--- a/doc/sphinx/phk/platforms.rst
+++ b/doc/sphinx/phk/platforms.rst
@@ -64,12 +64,12 @@ Platform specific bug reports without patches will likely live a quiet life.
 
 *Mac OS/X*
 
-*Solaris*.
+*Solaris-decendants* (Oracle Solaris, OmniOS, Joyent SmartOS)
 
-Yes, we'd like to bump Solaris to tier-A but I have to say that the
-uncertainty about the future for OpenSolaris, and lack of time to
-care and feed the somewhat altmodishe socket-API on Solaris, does
-keep the enthusiasm bounded.
+Mac OS/X is regarded as a developer platform, not as a production
+platform.
+
+Solaris-decendants are regarded as a production platform.
 
 NetBSD, AIX and HP-UX are conceivably candidates for this level, but
 so far I have not heard much, if any, user interest.
@@ -108,3 +108,4 @@ are right about Oracles intentions, Solaris may get demoted.
 Until next time,
 
 Poul-Henning, 2010-08-03
+Edited Nils, 2014-03-18 with Poul-Hennings concent



From phk at FreeBSD.org  Tue Mar 18 16:42:51 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Mar 2014 17:42:51 +0100
Subject: [master] 3c33b93 Forgot to change error -> synth here.
Message-ID: <E1WPx6J-0007aK-3G@project.varnish-software.com>


commit 3c33b93461793ad69c5a3aa644b44f3c64e07855
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 18 16:42:36 2014 +0000

    Forgot to change error -> synth here.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index cdac8d9..5ac9664 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -91,7 +91,7 @@ sub vcl_hash {
 }
 
 sub vcl_purge {
-    return (error(200, "Purged"));
+    return (synth(200, "Purged"));
 }
 
 sub vcl_hit {



From lkarsten at varnish-software.com  Tue Mar 18 16:45:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 18 Mar 2014 17:45:31 +0100
Subject: [master] f89015a Add major version in Via header
Message-ID: <E1WPx8t-00080v-9O@project.varnish-software.com>


commit f89015acfc28b6fe301b22a56174e9b85785d29a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Mar 18 16:12:22 2014 +0100

    Add major version in Via header
    
    To help track our adoption rate of the new Varnish versions, add the major
    version in the default Via header added by us.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 792b1ec..6650c31 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -123,7 +123,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	http_PrintfHeader(req->resp, "Age: %.0f",
 	    req->t_resp - req->obj->exp.t_origin);
 
-	http_SetHeader(req->resp, "Via: 1.1 varnish");
+	http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
 
 	if (cache_param->http_gzip_support && req->obj->gziped &&
 	    !RFC2616_Req_Gzip(req->http))



From fgsch at lodoss.net  Wed Mar 19 00:53:03 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 19 Mar 2014 01:53:03 +0100
Subject: [master] 101a5f7 Fix spelling and reformat a bit
Message-ID: <E1WQ4kh-0000oH-8F@project.varnish-software.com>


commit 101a5f7e8452bfae879b1e012ebc8e0ccf049b0a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 19 00:07:01 2014 +0000

    Fix spelling and reformat a bit

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 2c2db8c..8199602 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -1,8 +1,9 @@
 Contributing to Varnish Cache
 =============================
 
-To contribute to Varnish, read https://www.varnish-cache.org/trac/wiki/Contributing
-as a staring point and follow relevant parts linked there.
+To contribute to Varnish, read
+https://www.varnish-cache.org/trac/wiki/Contributing as a starting
+point and follow relevant parts linked there.
 
 Varnish Cache is developed with Git hosted on our own infrastructure.
 
@@ -12,7 +13,7 @@ Official development tree is here:
 
 Patches can be sent to varnish-dev at varnish-cache.org.
 
-There is a READ-ONLY git-mirror on Github.  Please do not send contributions
-there, we need them by email to -dev@ for now.
+There is a READ-ONLY git-mirror on Github.  Please do not send
+contributions there, we need them by email to varnish-dev@ for now.
 
 



From fgsch at lodoss.net  Wed Mar 19 00:53:03 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 19 Mar 2014 01:53:03 +0100
Subject: [master] 243e690 Better example
Message-ID: <E1WQ4kh-0000oN-9K@project.varnish-software.com>


commit 243e6909fd4c742158b46eb1e2d9b59b5dd25ccd
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 19 00:07:36 2014 +0000

    Better example
    
    Don't use {} in one-liners

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index a0da998..5f52318 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -53,7 +53,7 @@ The synthetic keyword is now a function::
     if (resp.status == 799) {
         set resp.status = 200;
         set resp.http.Content-Type = "text/plain; charset=utf-8";
-        synthetic({"You are "} + client.ip);
+        synthetic("You are " + client.ip);
         return (deliver);
     }
 



From tfheen at err.no  Wed Mar 19 05:42:39 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Wed, 19 Mar 2014 06:42:39 +0100
Subject: [master] 83319e4 Remove obj.last_use
Message-ID: <E1WQ9Gx-0005bf-RC@project.varnish-software.com>


commit 83319e42e9e5a274679a6aa578635b1335407658
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Tue Mar 18 17:25:10 2014 +0100

    Remove obj.last_use

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 5ac9664..92f4641 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -118,7 +118,6 @@ sub vcl_deliver {
      * Enable them only if you need them.
      *
      * set obj.hits = obj.hits + 1;
-     * set obj.last_use = now;
      */
     return (deliver);
 }
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index eac7546..ce90e58 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -610,7 +610,6 @@ struct object {
 
 	/* VCL only variables */
 	long			hits;
-	double			last_use;
 
 	double			last_modified;
 
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index 55145cd..8aa022b 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -200,7 +200,6 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	OFOF(struct object, http);
 	OFOF(struct object, store);
 	OFOF(struct object, esidata);
-	OFOF(struct object, last_use);
 #endif
 #undef OFOF
 #endif
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 8a999c9..0bcd5d5 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -592,8 +592,6 @@ VRT_r_obj_##field(const struct vrt_ctx *ctx)				\
 
 VOBJ_L(long, hits)
 VOBJ_R(long, hits)
-VOBJ_L(double, last_use)
-VOBJ_R(double, last_use)
 
 unsigned
 VRT_r_obj_uncacheable(const struct vrt_ctx *ctx)
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index c2788a9..30ae453 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -286,7 +286,6 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 	HTTP_Setup(o->http, bo->ws_o, bo->vsl, SLT_ObjMethod);
 	o->http->magic = HTTP_MAGIC;
 	o->exp = bo->exp;
-	o->last_use = bo->t_fetch;
 	VTAILQ_INIT(&o->store);
 	bo->stats->n_object++;
 
diff --git a/bin/varnishtest/tests/v00013.vtc b/bin/varnishtest/tests/v00013.vtc
index fa3afda..0bb2858 100644
--- a/bin/varnishtest/tests/v00013.vtc
+++ b/bin/varnishtest/tests/v00013.vtc
@@ -1,4 +1,4 @@
-varnishtest "Check obj.hits and obj.last_use"
+varnishtest "Check obj.hits"
 
 server s1 {
 	rxreq
@@ -13,9 +13,7 @@ varnish v1 -vcl+backend {
 
 	sub vcl_deliver {
 		set resp.http.foo = obj.hits;
-		set resp.http.bar = now - obj.last_use;
 		set obj.hits = obj.hits + 1;
-		set obj.last_use = now;
 	}
 } -start
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 7b9ab4e..9d8239b 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -546,15 +546,6 @@ sp_variables = [
 		( 'hit', ), """
 		"""
 	),
-	('obj.last_use',
-		'TIME',
-		( 'hit', 'deliver',),
-		( 'hit', 'deliver',), """
-		The approximate time elapsed since the object was
-		last requests, in seconds. This variable is also
-		available in vcl_deliver.
-		"""
-	),
 	('obj.uncacheable',
 		'BOOL',
 		( 'hit', ),



From tfheen at err.no  Wed Mar 19 09:12:02 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Wed, 19 Mar 2014 10:12:02 +0100
Subject: [master] fb4c1f8 Rework a bit of the build system for vmod docs
Message-ID: <E1WQCXa-0004ZK-F3@project.varnish-software.com>


commit fb4c1f855f0d21bf56799f57bf4232b736b90a05
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Wed Mar 19 10:08:09 2014 +0100

    Rework a bit of the build system for vmod docs
    
    Move the vmod man pages to man/ and build and ship the vmod_director
    page.  Also make sure we use the vmod_std docs from the .vcc file, not
    the old copy in docs/

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 64e6020..716e8da 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -127,7 +127,6 @@ EXTRA_DIST = \
 	reference/varnishtop.rst \
 	reference/vcl.rst \
 	reference/vmod.rst \
-	reference/vmod_std.rst \
 	reference/vsl-query.rst \
 	reference/vsl.rst \
 	reference/vsm.rst \
@@ -212,5 +211,13 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
+include/vmod_std.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+	cp $? $@
+BUILT_SOURCES += include/vmod_std.rst
+
+include/vmod_directors.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+	cp $? $@
+BUILT_SOURCES += include/vmod_directors.rst
+
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 1537fb4..20ac975 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,7 +21,8 @@ The Varnish Reference Manual
 	varnishtop.rst
 	vsm.rst
 	vmod.rst
-	vmod_std.rst
+	../include/vmod_std.rst
+	../include/vmod_directors.rst
 	vsl.rst
 	vsl-query.rst
 
diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
deleted file mode 100644
index b8f2aa3..0000000
--- a/doc/sphinx/reference/vmod_std.rst
+++ /dev/null
@@ -1,188 +0,0 @@
-========
-vmod_std
-========
-
------------------------
-Varnish Standard Module
------------------------
-
-SYNOPSIS
-========
-
-import std [from "path"] ;
-
-DESCRIPTION
-===========
-
-Vmod_std contains basic functions which are part and parcel of Varnish,
-but which for reasons of architecture fit better in a VMOD.
-
-One particular class of functions in vmod_std is the conversions functions
-which all have the form::
-
-	TYPE type(STRING, TYPE)
-
-These functions attempt to convert STRING to the TYPE, and if that fails,
-they return the second argument, which must have the given TYPE.
-
-FUNCTIONS
-=========
-
-toupper
--------
-
-Prototype
-	STRING toupper(STRING s)
-Description
-	Converts the string *s* to upper case.
-Example
-	set beresp.http.x-scream = std.toupper("yes!");
-
-tolower
--------
-Prototype
-	STRING tolower(STRING s)
-Description
-	Converts the string *s* to lower case.
-Example
-	set beresp.http.x-nice = std.tolower("VerY");
-
-set_ip_tos
-----------
-Prototype
-	VOID set_ip_tos(INT i)
-Description
-	Sets the Type-of-Service flag for the current session. Please
-	note that the TOS flag is not removed by the end of the
-	request so probably want to set it on every request should you
-	utilize it.
-Example
-	| if (req.url ~ ^/slow/) {
-	|    std.set_ip_tos(0x0);
-	| }
-
-random
-------
-Prototype
-	REAL random(REAL a, REAL b)
-Description
-	Returns a random REAL number between *a* and *b*.
-Example
-	set beresp.http.x-random-number = std.random(1, 100);
-
-log
----
-Prototype
-	VOID log(STRING string)
-Description
-	Logs *string* to the shared memory log, using VSL tag *SLT_VCL_Log*.
-Example
-	std.log("Something fishy is going on with the vhost " + req.host);
-
-syslog
-------
-Prototype
-	VOID syslog(INT priority, STRING string)
-Description
-	Logs *string* to syslog marked with *priority*.  See your
-	system's syslog.h file for the legal values of *priority*.
-Example
-	std.syslog(8 + 1, "Something is wrong");
-
-fileread
---------
-Prototype
-	STRING fileread(STRING filename)
-Description
-	Reads a file and returns a string with the content. Please
-	note that it is not recommended to send variables to this
-	function the caching in the function doesn't take this into
-	account. Also, files are not re-read.
-Example
-	set beresp.http.x-served-by = std.fileread("/etc/hostname");
-
-collect
--------
-Prototype
-	VOID collect(HEADER header)
-Description
-	Collapses the header, joining the headers into one.
-Example
-	std.collect(req.http.cookie);
-	This will collapse several Cookie: headers into one, long
-	cookie header.
-
-
-CONVERSION FUNCTIONS
-====================
-
-duration
---------
-Prototype
-	DURATION duration(STRING s, DURATION fallback)
-Description
-	Converts the string *s* to seconds. *s* must be quantified
-	with ms (milliseconds), s (seconds), m (minutes), h (hours),
-	d (days) or w (weeks) units. If *s* fails to parse,
-	*fallback* will be returned.
-Example
-	set beresp.ttl = std.duration("1w", 3600s);
-
-integer
---------
-Prototype
-	INT integer(STRING s, INT fallback)
-Description
-	Converts the string *s* to an integer.  If *s* fails to parse,
-	*fallback* will be returned.
-Example
-	if (std.integer(beresp.http.x-foo, 0) > 5) { ... }
-
-ip
---
-Prototype
-	IP ip(STRING s, IP fallback)
-Description
-	Converts string *s* to the first IP number returned by
-	the system library function getaddrinfo(3).  If conversion
-	fails, *fallback* will be returned.
-Example
-	if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { ... }
-
-healthy
--------
-Prototype
-	BOOL healthy(BACKEND backend)
- 
-Description
-	Returns true if the backend is healthy.
-
-port
-----
-Prototype
-	INT port(IP ip)
-
-Description
-	Returns the port number of an IP address.
-
-
-SEE ALSO
-========
-
-* vcl(7)
-* varnishd(1)
-
-HISTORY
-=======
-
-The Varnish standard module was released along with Varnish Cache 3.0.
-This manual page was written by Per Buer with help from Martin Blix
-Grydeland.
-
-COPYRIGHT
-=========
-
-This document is licensed under the same licence as Varnish
-itself. See LICENCE for details.
-
-* Copyright (c) 2011-2014 Varnish Software
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 6a85ad5..05912e0 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -28,7 +28,7 @@ nodist_libvmod_directors_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
-vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index 8967bf5..07dda95 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -7,8 +7,6 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/bin/varnishd \
 	-I$(top_builddir)/include
 
-dist_man_MANS = vmod_std.3
-
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_std
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
@@ -36,6 +34,3 @@ EXTRA_DIST = vmod.vcc
 CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
 	$(builddir)/vmod_std.rst \
 	$(builddir)/vmod_std.man.rst
-
-vmod_std.3: $(top_srcdir)/doc/sphinx/reference/vmod_std.rst
-	${RST2MAN} $? $@
diff --git a/man/Makefile.am b/man/Makefile.am
index 581a9de..c25804e 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -19,7 +19,9 @@ dist_man_MANS = \
 	varnishncsa.1 \
 	varnishstat.1 \
 	varnishtest.1 \
-	varnishtop.1
+	varnishtop.1 \
+	vmod_directors.3 \
+	vmod_std.3
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
@@ -80,3 +82,9 @@ varnishhist.1: \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_synopsis.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
+
+vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst
+	${RST2MAN} $? $@
+
+vmod_directors.3: $(top_srcdir)/lib/libvmod_directors/vmod_directors.man.rst
+	${RST2MAN} $? $@



From martin at varnish-software.com  Wed Mar 19 09:23:50 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 19 Mar 2014 10:23:50 +0100
Subject: [master] 0f90ee2 Reset busyobj state to BOS_INVALID in VBO_GetBusyObj
Message-ID: <E1WQCj0-0007nb-Vs@project.varnish-software.com>


commit 0f90ee223daf97d4e6d83a78ae919a95e590bcca
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Mar 19 10:23:11 2014 +0100

    Reset busyobj state to BOS_INVALID in VBO_GetBusyObj
    
    Fixes: #1458

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index db1c7a1..d8be0f6 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -109,6 +109,7 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(bo->refcount);
 
+	bo->state = BOS_INVALID;
 	bo->refcount = 1;
 
 	p = (void*)(bo + 1);



From perbu at varnish-software.com  Wed Mar 19 10:28:29 2014
From: perbu at varnish-software.com (Per Buer)
Date: Wed, 19 Mar 2014 11:28:29 +0100
Subject: [master] 493b979 Document the version marker
Message-ID: <E1WQDjZ-0004V6-HD@project.varnish-software.com>


commit 493b979fc5b01ce2203ad91f85a107e22281bcda
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Mar 19 11:28:09 2014 +0100

    Document the version marker

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 6f91b91..43295bb 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -40,6 +40,10 @@ In addition VCL has the following constructs:
 
 Note that are no loops or iterators of any kind in VCL.
 
+Each VCL file must start by declaring its version. For Varnish 4.0 you
+must include a "vcl 4.0;" at the top of the file.
+
+
 Operators
 ---------
 



From nils.goroll at uplex.de  Wed Mar 19 18:15:24 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 19 Mar 2014 19:15:24 +0100
Subject: [master] 1b34638 split DEVELOPER_CFLAGS to multiple lines for
 legibility
Message-ID: <E1WQL1Q-0007rh-4M@project.varnish-software.com>


commit 1b34638e87c3694138fa9ac160eeec3e289ad8f1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Mar 19 18:44:10 2014 +0100

    split DEVELOPER_CFLAGS to multiple lines for legibility

diff --git a/configure.ac b/configure.ac
index 6f43695..55c62ab 100644
--- a/configure.ac
+++ b/configure.ac
@@ -465,7 +465,29 @@ AX_CHECK_COMPILE_FLAG([-Werror=unused-result],
       OCFLAGS="${OCFLAGS} -Wno-unused-result"])])
 
 # This corresponds to FreeBSD's WARNS level 6
-DEVELOPER_CFLAGS="-Werror -Wall -Wno-format-y2k -W -Wno-unused-parameter -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch -Wshadow -Wunused-parameter -Wcast-align -Wchar-subscripts -Wnested-externs -Wno-pointer-sign -Wextra -Wno-missing-field-initializers -Wno-sign-compare"
+DEVELOPER_CFLAGS=" \
+	-Werror \
+	-Wall \
+	-Wno-format-y2k \
+	-W \
+	-Wno-unused-parameter \
+	-Wstrict-prototypes \
+	-Wmissing-prototypes \
+	-Wpointer-arith \
+	-Wreturn-type \
+	-Wcast-qual \
+	-Wwrite-strings \
+	-Wswitch \
+	-Wshadow \
+	-Wunused-parameter \
+	-Wcast-align \
+	-Wchar-subscripts \
+	-Wnested-externs \
+	-Wno-pointer-sign \
+	-Wextra \
+	-Wno-missing-field-initializers \
+	-Wno-sign-compare \
+"
 
 # These are not compliable yet
 DEVELOPER_GCC_CFLAGS="-Wold-style-definition -Wredundant-decls "



From nils.goroll at uplex.de  Wed Mar 19 18:15:24 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 19 Mar 2014 19:15:24 +0100
Subject: [master] 1cebffd dont try to build documentation for tools we cant
 build
Message-ID: <E1WQL1Q-0007rn-Aq@project.varnish-software.com>


commit 1cebffd4196dcb50f04ec519b0d0b97d8f497f0c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Mar 19 19:13:00 2014 +0100

    dont try to build documentation for tools we cant build

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 716e8da..bb44131 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -197,6 +197,9 @@ include/varnishlog_synopsis.rst: $(top_builddir)/bin/varnishlog/varnishlog_opt2r
 BUILT_SOURCES += include/varnishlog_options.rst \
 	include/varnishlog_synopsis.rst
 
+# tools which only get built when curses is available
+# XXX add varnishstat here when _opt2rst is ready
+if VARNISH_CURSES
 include/varnishtop_options.rst: $(top_builddir)/bin/varnishtop/varnishtop_opt2rst
 	$(top_builddir)/bin/varnishtop/varnishtop_opt2rst options > $@
 include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop_opt2rst
@@ -210,6 +213,8 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op
 	$(top_builddir)/bin/varnishhist/varnishhist_opt2rst synopsis > $@
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
+endif
+
 
 include/vmod_std.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
 	cp $? $@
diff --git a/man/Makefile.am b/man/Makefile.am
index c25804e..ae3e40d 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -14,12 +14,10 @@ dist_man_MANS = \
 	vsl-query.7 \
 	varnishadm.1 \
 	varnishd.1 \
-	varnishhist.1 \
 	varnishlog.1 \
 	varnishncsa.1 \
 	varnishstat.1 \
 	varnishtest.1 \
-	varnishtop.1 \
 	vmod_directors.3 \
 	vmod_std.3
 
@@ -71,6 +69,12 @@ varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst
 varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
 
+# tools which only get built when curses is available
+# XXX add varnishstat here when _opt2rst is ready
+if VARNISH_CURSES
+dist_man_MANS += varnishtop.1 \
+	varnishhist.1
+
 varnishtop.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_options.rst \
@@ -82,6 +86,8 @@ varnishhist.1: \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_synopsis.rst
 	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
+endif
+
 
 vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst
 	${RST2MAN} $? $@



From nils.goroll at uplex.de  Wed Mar 19 18:59:19 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 19 Mar 2014 19:59:19 +0100
Subject: [master] 2001b7e DEVELOPER_CFLAGS: - use -fstack-protector only if both compiler and linker support it - use -Wno-missing-field-initializers vs -Wno-extra depending on compiler support - -Wno-pointer-sign only if compiler supports it - add -Wno-address to avoid "the address of ‘buf’ will never be NULL" warning in   cache_backend_cfg.c (and in other places where REPLACE() is used with local   buf variable)
Message-ID: <E1WQLhv-0004zd-VJ@project.varnish-software.com>


commit 2001b7efbb114a9c4de5184c87e3c31fc4bee14e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Mar 19 19:57:34 2014 +0100

    DEVELOPER_CFLAGS:
    - use -fstack-protector only if both compiler and linker support it
    - use -Wno-missing-field-initializers vs -Wno-extra depending on compiler support
    - -Wno-pointer-sign only if compiler supports it
    - add -Wno-address to avoid "the address of ?buf? will never be NULL" warning in
      cache_backend_cfg.c (and in other places where REPLACE() is used with local
      buf variable)

diff --git a/configure.ac b/configure.ac
index 55c62ab..e2a0b98 100644
--- a/configure.ac
+++ b/configure.ac
@@ -470,7 +470,6 @@ DEVELOPER_CFLAGS=" \
 	-Wall \
 	-Wno-format-y2k \
 	-W \
-	-Wno-unused-parameter \
 	-Wstrict-prototypes \
 	-Wmissing-prototypes \
 	-Wpointer-arith \
@@ -483,10 +482,9 @@ DEVELOPER_CFLAGS=" \
 	-Wcast-align \
 	-Wchar-subscripts \
 	-Wnested-externs \
-	-Wno-pointer-sign \
 	-Wextra \
-	-Wno-missing-field-initializers \
 	-Wno-sign-compare \
+	-Wno-address \
 "
 
 # These are not compliable yet
@@ -505,7 +503,10 @@ AC_ARG_ENABLE(stack-protector,
        [enable_stack_protector=yes])
 
 if test "x$enable_stack_protector" != "xno"; then
-       DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -fstack-protector"
+	AX_CHECK_COMPILE_FLAG([-fstack-protector],
+		AX_CHECK_LINK_FLAG([-fstack-protector],
+			[DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -fstack-protector"], [], []),
+		[], [])
 fi
 
 # --enable-developer-warnings
@@ -515,6 +516,16 @@ AC_ARG_ENABLE(developer-warnings,
 	[enable_developer_warnings=no])
 
 if test "x$enable_developer_warnings" != "xno"; then
+	# compiler flags not available on gcc3
+	AX_CHECK_COMPILE_FLAG([-Wno-pointer-sign],
+		[DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -Wno-pointer-sign"], [], [])
+	# no known way to specifically disabling missing-field-initializers warnings
+	# keeping the rest of Wextra
+	AX_CHECK_COMPILE_FLAG([-Wno-missing-field-initializers],
+		[DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -Wno-missing-field-initializers"],
+		[DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -Wno-extra"],
+		[])
+
 	CFLAGS="${CFLAGS} ${DEVELOPER_CFLAGS}"
 	OCFLAGS="${OCFLAGS} ${DEVELOPER_CFLAGS}"
 fi
diff --git a/m4/ax_check_link_flag.m4 b/m4/ax_check_link_flag.m4
new file mode 100644
index 0000000..db899dd
--- /dev/null
+++ b/m4/ax_check_link_flag.m4
@@ -0,0 +1,73 @@
+# ===========================================================================
+#    http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT])
+#
+# DESCRIPTION
+#
+#   Check whether the given FLAG works with the linker or gives an error.
+#   (Warnings, however, are ignored)
+#
+#   ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on
+#   success/failure.
+#
+#   If EXTRA-FLAGS is defined, it is added to the linker's default flags
+#   when the check is done.  The check is thus made with the flags: "LDFLAGS
+#   EXTRA-FLAGS FLAG".  This can for example be used to force the linker to
+#   issue an error when a bad flag is given.
+#
+#   INPUT gives an alternative input source to AC_LINK_IFELSE.
+#
+#   NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this
+#   macro in sync with AX_CHECK_{PREPROC,COMPILE}_FLAG.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Guido U. Draheim <guidod at gmx.de>
+#   Copyright (c) 2011 Maarten Bosmans <mkbosmans at gmail.com>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 3
+
+AC_DEFUN([AX_CHECK_LINK_FLAG],
+[AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl
+AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [
+  ax_check_save_flags=$LDFLAGS
+  LDFLAGS="$LDFLAGS $4 $1"
+  AC_LINK_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])],
+    [AS_VAR_SET(CACHEVAR,[yes])],
+    [AS_VAR_SET(CACHEVAR,[no])])
+  LDFLAGS=$ax_check_save_flags])
+AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes],
+  [m4_default([$2], :)],
+  [m4_default([$3], :)])
+AS_VAR_POPDEF([CACHEVAR])dnl
+])dnl AX_CHECK_LINK_FLAGS



From phk at FreeBSD.org  Wed Mar 19 22:41:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 19 Mar 2014 23:41:22 +0100
Subject: [master] bba23ef Move obj.hits to objhead, make it read-only
Message-ID: <E1WQPAo-00062F-Pm@project.varnish-software.com>


commit bba23ef151986671cef6d1ad8a126d30d093daa0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 19 21:33:44 2014 +0000

    Move obj.hits to objhead, make it read-only

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ce90e58..ac615a4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -609,8 +609,6 @@ struct object {
 	struct exp		exp;
 
 	/* VCL only variables */
-	long			hits;
-
 	double			last_modified;
 
 	struct http		*http;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 07c5931..d7263e1 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -433,6 +433,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			assert(oh->refcnt > 1);
 			assert(oc->objhead == oh);
 			oc->refcnt++;
+			if (oh->hits < LONG_MAX)
+				oh->hits++;
 			Lck_Unlock(&oh->mtx);
 			assert(HSH_DerefObjHead(&wrk->stats, &oh));
 			*ocp = oc;
@@ -460,12 +462,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		} else {
 			retval = HSH_EXP;
 		}
-
+		if (oh->hits < LONG_MAX)
+			oh->hits++;
 		Lck_Unlock(&oh->mtx);
 		if (retval == HSH_EXP)
 			assert(HSH_DerefObjHead(&wrk->stats, &oh));
-		if (!cache_param->obj_readonly && exp_o->hits < INT_MAX)
-			exp_o->hits++;
 		*ocp = exp_oc;
 		return (retval);
 	}
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 0bcd5d5..b51d2de 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -36,6 +36,7 @@
 
 #include "cache.h"
 #include "common/heritage.h"
+#include "hash/hash_slinger.h"
 
 #include "cache_backend.h"
 #include "vrt.h"
@@ -570,29 +571,17 @@ VRT_r_server_hostname(const struct vrt_ctx *ctx)
 
 /*--------------------------------------------------------------------*/
 
-#define VOBJ_L(type, field)						\
-void									\
-VRT_l_obj_##field(const struct vrt_ctx *ctx, type a)			\
-{									\
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);				\
-	CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);			\
-	ctx->req->obj->field = a;					\
-}
-
-#define VOBJ_R(type, field)						\
-type									\
-VRT_r_obj_##field(const struct vrt_ctx *ctx)				\
-{									\
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);				\
-	CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);			\
-	return (ctx->req->obj->field);					\
+long
+VRT_r_obj_hits(const struct vrt_ctx *ctx)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req->obj->objcore, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req->obj->objcore->objhead, OBJHEAD_MAGIC);
+	return (ctx->req->obj->objcore->objhead->hits);
 }
 
-VOBJ_L(long, hits)
-VOBJ_R(long, hits)
-
 unsigned
 VRT_r_obj_uncacheable(const struct vrt_ctx *ctx)
 {
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index f38c84d..40d3aef 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -94,6 +94,8 @@ struct objhead {
 	uint8_t			digest[DIGEST_LEN];
 	struct waitinglist	*waitinglist;
 
+	long			hits;
+
 	/*----------------------------------------------------
 	 * The fields below are for the sole private use of
 	 * the hash implementation(s).
diff --git a/bin/varnishtest/tests/v00013.vtc b/bin/varnishtest/tests/v00013.vtc
index 0bb2858..c428add 100644
--- a/bin/varnishtest/tests/v00013.vtc
+++ b/bin/varnishtest/tests/v00013.vtc
@@ -13,7 +13,6 @@ varnish v1 -vcl+backend {
 
 	sub vcl_deliver {
 		set resp.http.foo = obj.hits;
-		set obj.hits = obj.hits + 1;
 	}
 } -start
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 9d8239b..c63274e 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -512,7 +512,7 @@ sp_variables = [
 	('obj.hits',
 		'INT',
 		( 'hit', 'deliver',),
-		( 'hit', 'deliver',), """
+		( ), """
 		The approximate number of times the object has been
 		delivered. A value of 0 indicates a cache miss.
 		This variable is also available in vcl_deliver.



From phk at FreeBSD.org  Wed Mar 19 22:41:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 19 Mar 2014 23:41:22 +0100
Subject: [master] 4a81bc6 Remove obsolete comment about obj.hits
Message-ID: <E1WQPAo-00062M-R1@project.varnish-software.com>


commit 4a81bc626e01e7e329b69184c0a6e95be8afbaa8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 19 21:34:29 2014 +0000

    Remove obsolete comment about obj.hits

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 92f4641..8e5ca36 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -113,12 +113,6 @@ sub vcl_miss {
 }
 
 sub vcl_deliver {
-    /*
-     * These two write to the stored object causing extra page faults
-     * Enable them only if you need them.
-     *
-     * set obj.hits = obj.hits + 1;
-     */
     return (deliver);
 }
 



From phk at FreeBSD.org  Wed Mar 19 22:41:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 19 Mar 2014 23:41:22 +0100
Subject: [master] d260e98 Fix the three obj.{proto, reason,
 status} variables to match reality.
Message-ID: <E1WQPAo-00062V-SJ@project.varnish-software.com>


commit d260e9874bc89d550f89462f7ea237a672c0f265
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 19 22:18:44 2014 +0000

    Fix the three obj.{proto,reason,status} variables to match reality.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c63274e..7c7e9db 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -491,21 +491,21 @@ sp_variables = [
 	('obj.proto',
 		'STRING',
 		( 'hit', ),
-		( 'hit', ), """
+		( ), """
 		The HTTP protocol version used when the object was retrieved.
 		"""
 	),
 	('obj.status',
 		'INT',
-		( 'synth',),
-		( 'synth',), """
+		( 'hit',),
+		( ), """
 		The HTTP status code returned by the server.
 		"""
 	),
 	('obj.reason',
 		'STRING',
-		( 'synth',),
-		( 'synth',), """
+		( 'hit',),
+		( ), """
 		The HTTP status message returned by the server.
 		"""
 	),



From fgsch at lodoss.net  Wed Mar 19 23:59:13 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 20 Mar 2014 00:59:13 +0100
Subject: [master] af27926 Update.. somewhat
Message-ID: <E1WQQO9-0003ss-W1@project.varnish-software.com>


commit af27926a9111247472faff6a5924b84293349f48
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 19 22:54:59 2014 +0000

    Update.. somewhat

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 95987e1..580caff 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -22,7 +22,7 @@ For instance::
 The "std" vmod is one you get with Varnish, it will always be there
 and we will put "boutique" functions in it, such as the "toupper"
 function shown above.  The full contents of the "std" module is
-documented in vmod_std(7).
+documented in vmod_std(3).
 
 This part of the manual is about how you go about writing your own
 VMOD, how the language interface between C and VCC works, where you
@@ -43,15 +43,15 @@ The vmod.vcc file
 
 The interface between your VMOD and the VCL compiler ("VCC") and the
 VCL runtime ("VRT") is defined in the vmod.vcc file which a python
-script called "vmod.py" turns into thaumaturgically challenged C
+script called "vmodtool.py" turns into thaumaturgically challenged C
 data structures that does all the hard work.
 
 The std VMODs vmod.vcc file looks somewhat like this::
 
-	$Module std
+	$Module std 3
 	$Init init_function
-	$Function STRING toupper(PRIV_CALL, STRING_LIST)
-	$Function STRING tolower(PRIV_VCL, STRING_LIST)
+	$Function STRING toupper(STRING_LIST)
+	$Function STRING tolower(STRING_LIST)
 	$Function VOID set_ip_tos(INT)
 
 The first line gives the name of the module, nothing special there.
@@ -75,7 +75,7 @@ primary action, something functions which return a value can not::
 		std.set_ip_tos(32);
 	}
 
-Running vmod.py on the vmod.vcc file, produces an "vcc_if.c" and
+Running vmodtool.py on the vmod.vcc file, produces an "vcc_if.c" and
 "vcc_if.h" files, which you must use to build your shared library
 file.
 
@@ -88,14 +88,18 @@ the functions you want to export to VCL.
 
 For the std VMOD, the compiled vcc_if.h file looks like this::
 
-	struct sess;
+	struct vrt_ctx;
 	struct VCL_conf;
-	const char * vmod_toupper(struct sess *, struct vmod_priv *, const char *, ...);
-	const char * vmod_tolower(struct sess *, struct vmod_priv *, const char *, ...);
-	int meta_function(void **, const struct VCL_conf *);
+	struct vmod_priv;
+
+	VCL_STRING vmod_toupper(const struct vrt_ctx *, const char *, ...);
+	VCL_STRING vmod_tolower(const struct vrt_ctx *, const char *, ...);
+	VCL_VOID vmod_set_ip_tos(const struct vrt_ctx *, VCL_INT);
+
+	int init_function(struct vmod_priv *, const struct VCL_conf *);
 
 Those are your C prototypes.  Notice the ``vmod_`` prefix on the function
-names and the C-types as return types and arguments.
+names and the C-types as arguments.
 
 VCL and C data types
 ====================
@@ -319,5 +323,5 @@ That means that the VMOD init, and any object init/fini functions
 are already serialized in sensible order, and won't need any locking,
 unless they access VMOD specific global state, shared with other VCLs.
 
-Trafic in other VCLs which also import this VMOD, will be happening
+Traffic in other VCLs which also import this VMOD, will be happening
 while housekeeping is going on.



From fgsch at lodoss.net  Wed Mar 19 23:59:14 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 20 Mar 2014 00:59:14 +0100
Subject: [master] 0f38ff3 Add missing dependency
Message-ID: <E1WQQOA-0003sy-0t@project.varnish-software.com>


commit 0f38ff3b4bd06203cf7d209b21411bc08856c228
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 19 22:59:34 2014 +0000

    Add missing dependency

diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index 07dda95..2e9832a 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -26,7 +26,7 @@ nodist_libvmod_std_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
 
-vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc



From fgsch at lodoss.net  Wed Mar 19 23:59:14 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 20 Mar 2014 00:59:14 +0100
Subject: [master] 4362464 Typo
Message-ID: <E1WQQOA-0003t7-25@project.varnish-software.com>


commit 4362464111e676d127c3ee10972aa5aec177b3a2
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 19 23:01:27 2014 +0000

    Typo

diff --git a/include/tbl/sess_close.h b/include/tbl/sess_close.h
index b132b39..522eb02 100644
--- a/include/tbl/sess_close.h
+++ b/include/tbl/sess_close.h
@@ -31,7 +31,7 @@
 
 SESS_CLOSE(REM_CLOSE,	"Client Closed")
 SESS_CLOSE(REQ_CLOSE,	"Client requested close")
-SESS_CLOSE(REQ_HTTP10,	"proto < HTTP.1.1")
+SESS_CLOSE(REQ_HTTP10,	"Proto < HTTP/1.1")
 SESS_CLOSE(RX_BODY,	"Failure receiving req.body")
 SESS_CLOSE(RX_JUNK,	"Received junk data")
 SESS_CLOSE(RX_OVERFLOW,	"Received buffer overflow")



From fgsch at lodoss.net  Wed Mar 19 23:59:14 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 20 Mar 2014 00:59:14 +0100
Subject: [master] f9fd935 Update after moving vmod docs to man/
Message-ID: <E1WQQOA-0003tG-3R@project.varnish-software.com>


commit f9fd935793e534bebe26afded486038774fbca30
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Mar 19 23:05:34 2014 +0000

    Update after moving vmod docs to man/

diff --git a/.gitignore b/.gitignore
index dd684da..a37080a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,14 +71,14 @@ cscope.*out
 # Misc. generated files for included vmods.
 /lib/libvmod_*/vcc_if.c
 /lib/libvmod_*/vcc_if.h
-/lib/libvmod_*/vmod_*.3
 /lib/libvmod_*/vmod_*.rst
 
 
 # Man-files and binaries
 /man/vsc2rst
-/man/*.7
 /man/*.1
+/man/*.3
+/man/*.7
 /doc/sphinx/include
 /bin/varnish*/varnish*_opt2rst
 /bin/varnishadm/varnishadm



From martin at varnish-software.com  Thu Mar 20 09:31:33 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 20 Mar 2014 10:31:33 +0100
Subject: [master] 5261bdb Revert "Reset busyobj state to BOS_INVALID in
 VBO_GetBusyObj"
Message-ID: <E1WQZK1-0002TF-M6@project.varnish-software.com>


commit 5261bdb4e83ee71789f8db64af91f7010c15ce6f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 20 10:30:39 2014 +0100

    Revert "Reset busyobj state to BOS_INVALID in VBO_GetBusyObj"
    
    This reverts commit 0f90ee223daf97d4e6d83a78ae919a95e590bcca.
    
    After looking at this again, I see that the busyobj is cleared as it
    should as part of VBO_DerefBusyObj(). Reverting the patch.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index d8be0f6..db1c7a1 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -109,7 +109,6 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(bo->refcount);
 
-	bo->state = BOS_INVALID;
 	bo->refcount = 1;
 
 	p = (void*)(bo + 1);



From lkarsten at varnish-software.com  Thu Mar 20 10:28:51 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 11:28:51 +0100
Subject: [master] cd2e297 Reflect new C: close default in vcl_pipe
Message-ID: <E1WQaDT-00072Q-RV@project.varnish-software.com>


commit cd2e297fabde5356d906abb9d44f803263bcd730
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 11:20:50 2014 +0100

    Reflect new C: close default in vcl_pipe
    
    We now set Connection: Close on all piped backend requests, to avoid some of
    the debugging pain seen over the last 2-3 years.
    
    Basic problem is that request backend routing done in vcl_recv would not be
    honored for subsequent requests after one was piped.
    
    This was, at least for new users, a rather nasty surprise.
    
    Original change in commit 880c3c2.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 8e5ca36..3949689 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -67,12 +67,11 @@ sub vcl_recv {
 }
 
 sub vcl_pipe {
-    # Note that only the first request to the backend will have
-    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
-    # have it set for all requests, make sure to have:
-    # set bereq.http.connection = "close";
-    # here.  It is not set by default as it might break some broken web
-    # applications, like IIS with NTLM authentication.
+    # By default Connection: close is set on all piped requests, to stop
+    # connection reuse from sending future requests directly to the
+    # (potentially) wrong backend. If you do want this to happen, you can undo
+    # it here.
+    # unset bereq.http.connection;
     return (pipe);
 }
 



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 47565d7 Emphasise the values.
Message-ID: <E1WQaV1-0008Tt-1H@project.varnish-software.com>


commit 47565d7e237745c9619707a4ce6d0aa3e1f96e47
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:27:16 2014 +0100

    Emphasise the values.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 43295bb..321c251 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -95,7 +95,7 @@ the NUL (0x00) character
 Booleans
 ~~~~~~~~
 
-Booleans can be either true or false.
+Booleans can be either *true* or *false*.
 
 Time
 ----



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] afbdcbb Note that directors stack.
Message-ID: <E1WQaV1-0008Tz-2e@project.varnish-software.com>


commit afbdcbb88b37873e5ffac4e1c9a30713cc1e928c
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:27:29 2014 +0100

    Note that directors stack.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 321c251..942d1ac 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -208,6 +208,9 @@ are available:
     Varnish reaches the maximum Varnish it will start failing
     connections.
 
+Backends can be used with *directors*. Please see the
+vmod_directors(3) man page for more information.
+
 Probes
 ------
 



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 6284634 Update purging docs for V4.
Message-ID: <E1WQaV1-0008U8-3g@project.varnish-software.com>


commit 6284634299582a1caed72b9fc38396d303be6c2f
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:27:55 2014 +0100

    Update purging docs for V4.

diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 38de2d3..5d7deda 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -40,31 +40,16 @@ following VCL in place::
 			  error 405 "Not allowed.";
 		  }
                   # jump to hit/miss
-		  return (lookup);
+		  return (purge);
 	  }
   }
 
-  sub vcl_hit {
-	  if (req.method == "PURGE") {
-	          purge;
-		  error 200 "Purged.";
-	  }
-  }
-
-  sub vcl_miss {
-	  if (req.method == "PURGE") {
-	          purge;
-		  error 200 "Purged.";
-	  }
-  }
-
-As you can see we have used two new VCL subroutines, vcl_hit and
-vcl_miss. When we call lookup Varnish will try to lookup the object in
-its cache. It will either hit an object or miss it and so the
-corresponding subroutine is called. In vcl_hit the object that is
-stored in cache is available and we can set the TTL. The purge in
-vcl_miss is necessary to purge all variants in the cases where you hit an
-object, but miss a particular variant.
+As you can see we have used a new action - return(purge). This ends
+execution of vcl_recv and jumps to vcl_hash. This is just like we
+handle a regular request. When vcl_hash calls return(lookup) varnish
+will purge the object and then call vcl_purge. Here you have the
+option of adding any particular actions you want Varnish to take once
+it has purge the object.
 
 So for example.com to invalidate their front page they would call out
 to Varnish like this::
@@ -121,7 +106,7 @@ You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
 
 		  # Throw a synthetic page so the
                   # request won't go to the backend.
-		  error 200 "Ban added";
+		  return(synth(200m"Ban added"));
 	  }
   }
 



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 06762cf How to call purge from recv.
Message-ID: <E1WQaV1-0008UH-5E@project.varnish-software.com>


commit 06762cf62ac4284b0c33a503b9d57f8b22306368
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:28:28 2014 +0100

    How to call purge from recv.

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 7a31aef..f71dfc1 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -36,8 +36,8 @@ of the following keywords:
     caching. Passes the control over to vcl_hash.
 
   purge
-    Calls vcl_purge where the object will be purged.
-
+    Purge the object and it's variants. Control passes through 
+    vcl_hash to vcl_purge.
 
 vcl_pipe
 ~~~~~~~~



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 526ca86 Directors can stack.
Message-ID: <E1WQaV1-0008UQ-6A@project.varnish-software.com>


commit 526ca86006db9b10ed678ceb22da7fc34e047b6b
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:28:42 2014 +0100

    Directors can stack.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index f7ec8f4..24e8839 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -54,6 +54,8 @@ As you can see there is nothing keeping you from manipulating the
 directors elsewhere in VCL. So, you could have VCL code that would
 add more backends to a director when a certain URL is called.
 
+Note that directors can use other directors as backends.
+
 $Object round_robin()
 
 Description



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] cda2ecf Error is gone.
Message-ID: <E1WQaV1-0008UZ-7G@project.varnish-software.com>


commit cda2ecfae0eb12328d613be2d9bfcc9f6b953ee0
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:29:46 2014 +0100

    Error is gone.

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index f71dfc1..4772374 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -93,7 +93,7 @@ Called is a cache lookup is successful.
   deliver
     Deliver the object. Control passes to vcl_deliver.
 
-  error [reason]
+  synth(error code, reason)
     Return the specified error code to the client and abandon the request.
 
 
@@ -107,7 +107,7 @@ retrieve the document from the backend, and which backend to use.
 The vcl_miss subroutine may terminate with calling return() with one
 of the following keywords:
 
-  error [reason]
+  synth(error code, reason)
     Return the specified error code to the client and abandon the request.
 
   pass
@@ -124,21 +124,19 @@ Called after vcl_recv to create a hash value for the request. This is
 used as a key to look up the object in Varnish.
 
   lookup
-    Look up the object in cache.
+    Look up the object in cache. Control passes to vcl_miss, vcl_hit
+    or vcl_purge.
+
+
 
 
 vcl_purge
 ~~~~~~~~~
 
-Purge the object and all it's variants. Variants created when the
-backend issues a Vary response.
-
-  fetch
-    Execute the purge.
+Called after the purge has been executed and all it's variant have been evited. 
 
-  error
-    Fail the purge request. Typically you would call error here it
-    request doesn't pass the ACLs.
+  synth
+    Produce a response.
 
 
 



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] c890439 Fix error syntax.
Message-ID: <E1WQaV1-0008Ui-8Q@project.varnish-software.com>


commit c890439f01e916c3385f34d6c91992563b21c638
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:29:57 2014 +0100

    Fix error syntax.

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 4772374..73934c7 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -194,7 +194,7 @@ of the following keywords:
     Control will eventually pass to vcl_deliver. Caching is dependant
     on beresp.cacheable.
 
-  error [reason]
+  error(error code, reason)
     Return the specified error code to the client and abandon the request.
 
   retry



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 7f5dbd3 Update for V4 syntax.
Message-ID: <E1WQaV1-0008Ur-9h@project.varnish-software.com>


commit 7f5dbd315095d5d1d8519b88deaad43c237f242a
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:42:37 2014 +0100

    Update for V4 syntax.

diff --git a/doc/sphinx/users-guide/vcl-example-acls.rst b/doc/sphinx/users-guide/vcl-example-acls.rst
index 3d0226a..b460bbe 100644
--- a/doc/sphinx/users-guide/vcl-example-acls.rst
+++ b/doc/sphinx/users-guide/vcl-example-acls.rst
@@ -15,20 +15,10 @@ the IP address of the client against an ACL with the match operator.::
   sub vcl_recv {
     if (req.method == "PURGE") {
       if (client.ip ~ local) {
-         return(lookup);
+         return(purge);
+      } else {
+         return(synth(403, "Access denied."));
       }
     } 
   }
   
-  sub vcl_hit {
-     if (req.method == "PURGE") {
-       set obj.ttl = 0s;
-       error 200 "Purged.";
-      }
-  }
-
-  sub vcl_miss {
-    if (req.method == "PURGE") {
-      error 404 "Not in cache.";
-    }
-  }



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 1a25542 Another error/synth.
Message-ID: <E1WQaV1-0008V0-Ab@project.varnish-software.com>


commit 1a25542ffd040759aecf49c91eab4b2489d7fa74
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:42:50 2014 +0100

    Another error/synth.

diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 5d7deda..4aad344 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -37,7 +37,7 @@ following VCL in place::
 
 	  if (req.method == "PURGE") {
 		  if (!client.ip ~ purge) {
-			  error 405 "Not allowed.";
+			  return(synth(405,"Not allowed.");
 		  }
                   # jump to hit/miss
 		  return (purge);



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] 7b5f540 Remove saint mode.
Message-ID: <E1WQaV1-0008V9-Bb@project.varnish-software.com>


commit 7b5f54055666b21c03ddd79053645d7e735a8e54
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:44:33 2014 +0100

    Remove saint mode.

diff --git a/doc/sphinx/users-guide/vcl-saint-and-grace.rst b/doc/sphinx/users-guide/vcl-saint-and-grace.rst
index bb5fd35..f318aa3 100644
--- a/doc/sphinx/users-guide/vcl-saint-and-grace.rst
+++ b/doc/sphinx/users-guide/vcl-saint-and-grace.rst
@@ -56,48 +56,3 @@ So, to sum up, grace mode solves two problems:
  * it serves stale content to avoid request pile-up.
  * it serves stale content if the backend is not healthy.
 
-Saint mode
-~~~~~~~~~~
-
-Sometimes servers get flaky. They start throwing out random
-errors. You can instruct Varnish to try to handle this in a
-more-than-graceful way - enter *Saint mode*. Saint mode enables you to
-discard a certain page from one backend server and either try another
-server or serve stale content from cache. Lets have a look at how this
-can be enabled in VCL::
-
-  sub vcl_fetch {
-    if (beresp.status == 500) { 
-      set beresp.saintmode = 10s;
-      return(restart);
-    }
-    set beresp.grace = 5m;
-  } 
-
-When we set beresp.saintmode to 10 seconds Varnish will not ask *that*
-server for URL for 10 seconds. A blacklist, more or less. Also a
-restart is performed so if you have other backends capable of serving
-that content Varnish will try those. When you are out of backends
-Varnish will serve the content from its stale cache.
-
-This can really be a life saver.
-
-Known limitations on grace- and saint mode
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If your request fails while it is being fetched you're thrown into
-vcl_error. vcl_error has access to a rather limited set of data so you
-can't enable saint mode or grace mode here. This will be addressed in a
-future release but a work-around available.
-
-* Declare a backend that is always sick.
-* Set a magic marker in vcl_error
-* Restart the transaction
-* Note the magic marker in vcl_recv and set the backend to the one mentioned
-* Varnish will now serve stale data is any is available
-
-
-God mode
-~~~~~~~~
-Not implemented yet. :-)
-



From perbu at varnish-software.com  Thu Mar 20 10:46:59 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 11:46:59 +0100
Subject: [master] dd54b29 Remove saint mode references.
Message-ID: <E1WQaV1-0008VI-Cg@project.varnish-software.com>


commit dd54b29a468e4b70af00bd004f7be4b3d8d2c986
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 11:46:49 2014 +0100

    Remove saint mode references.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index bb44131..f98a044 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -167,7 +167,7 @@ EXTRA_DIST = \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
 	users-guide/vcl-intro.rst \
-	users-guide/vcl-saint-and-grace.rst \
+	users-guide/vcl-grace.rst \
 	users-guide/vcl-syntax.rst \
 	users-guide/vcl-variables.rst \
 	users-guide/vcl.rst
diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
new file mode 100644
index 0000000..f318aa3
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -0,0 +1,58 @@
+.. _users-guide-handling_misbehaving_servers:
+
+Misbehaving servers
+-------------------
+
+A key feature of Varnish is its ability to shield you from misbehaving
+web- and application servers.
+
+
+
+Grace mode
+~~~~~~~~~~
+
+When several clients are requesting the same page Varnish will send
+one request to the backend and place the others on hold while fetching
+one copy from the back end. In some products this is called request
+coalescing and Varnish does this automatically.
+
+If you are serving thousands of hits per second the queue of waiting
+requests can get huge. There are two potential problems - one is a
+thundering herd problem - suddenly releasing a thousand threads to
+serve content might send the load sky high. Secondly - nobody likes to
+wait. To deal with this we can instruct Varnish to keep
+the objects in cache beyond their TTL and to serve the waiting
+requests somewhat stale content.
+
+So, in order to serve stale content we must first have some content to
+serve. So to make Varnish keep all objects for 30 minutes beyond their
+TTL use the following VCL::
+
+  sub vcl_fetch {
+    set beresp.grace = 30m;
+  }
+
+Varnish still won't serve the stale objects. In order to enable
+Varnish to actually serve the stale object we must enable this on the
+request. Lets us say that we accept serving 15s old object.::
+
+  sub vcl_recv {
+    set req.grace = 15s;
+  }
+
+You might wonder why we should keep the objects in the cache for 30
+minutes if we are unable to serve them? Well, if you have enabled
+:ref:`users-guide-advanced_backend_servers-health` you can check if the
+backend is sick and if it is we can serve the stale content for a bit
+longer.::
+
+   if (! req.backend.healthy) {
+      set req.grace = 5m;
+   } else {
+      set req.grace = 15s;
+   }
+
+So, to sum up, grace mode solves two problems:
+ * it serves stale content to avoid request pile-up.
+ * it serves stale content if the backend is not healthy.
+
diff --git a/doc/sphinx/users-guide/vcl-saint-and-grace.rst b/doc/sphinx/users-guide/vcl-saint-and-grace.rst
deleted file mode 100644
index f318aa3..0000000
--- a/doc/sphinx/users-guide/vcl-saint-and-grace.rst
+++ /dev/null
@@ -1,58 +0,0 @@
-.. _users-guide-handling_misbehaving_servers:
-
-Misbehaving servers
--------------------
-
-A key feature of Varnish is its ability to shield you from misbehaving
-web- and application servers.
-
-
-
-Grace mode
-~~~~~~~~~~
-
-When several clients are requesting the same page Varnish will send
-one request to the backend and place the others on hold while fetching
-one copy from the back end. In some products this is called request
-coalescing and Varnish does this automatically.
-
-If you are serving thousands of hits per second the queue of waiting
-requests can get huge. There are two potential problems - one is a
-thundering herd problem - suddenly releasing a thousand threads to
-serve content might send the load sky high. Secondly - nobody likes to
-wait. To deal with this we can instruct Varnish to keep
-the objects in cache beyond their TTL and to serve the waiting
-requests somewhat stale content.
-
-So, in order to serve stale content we must first have some content to
-serve. So to make Varnish keep all objects for 30 minutes beyond their
-TTL use the following VCL::
-
-  sub vcl_fetch {
-    set beresp.grace = 30m;
-  }
-
-Varnish still won't serve the stale objects. In order to enable
-Varnish to actually serve the stale object we must enable this on the
-request. Lets us say that we accept serving 15s old object.::
-
-  sub vcl_recv {
-    set req.grace = 15s;
-  }
-
-You might wonder why we should keep the objects in the cache for 30
-minutes if we are unable to serve them? Well, if you have enabled
-:ref:`users-guide-advanced_backend_servers-health` you can check if the
-backend is sick and if it is we can serve the stale content for a bit
-longer.::
-
-   if (! req.backend.healthy) {
-      set req.grace = 5m;
-   } else {
-      set req.grace = 15s;
-   }
-
-So, to sum up, grace mode solves two problems:
- * it serves stale content to avoid request pile-up.
- * it serves stale content if the backend is not healthy.
-
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index defe566..ad65403 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -39,7 +39,7 @@ code commented out in builtin.vcl that ships with Varnish Cache.
    vcl-actions
    vcl-backends
    vcl-hashing
-   vcl-saint-and-grace
+   vcl-grace
    vcl-inline-c
    vcl-examples
    websockets



From perbu at varnish-software.com  Thu Mar 20 11:26:09 2014
From: perbu at varnish-software.com (Per Buer)
Date: Thu, 20 Mar 2014 12:26:09 +0100
Subject: [master] 836d855 Remove references to vcl_error.
Message-ID: <E1WQb6v-0005No-AH@project.varnish-software.com>


commit 836d85599d38836df5d74c8245c240e4ad362e84
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 20 12:25:55 2014 +0100

    Remove references to vcl_error.

diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index e60b438..5750918 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -251,11 +251,11 @@ VCL::
         # call some detection engine
 
         if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
-            error 750 "Moved Temporarily";
+            return(synth(750, "Moved Temporarily"));
         }
     }
      
-    sub vcl_error {
+    sub vcl_synth {
         if (obj.status == 750) {
             set obj.http.Location = "http://m.example.com" + req.url;
             set obj.status = 302;
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 4aad344..058bd7d 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -99,14 +99,14 @@ You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
 	  if (req.method == "BAN") {
                   # Same ACL check as above:
 		  if (!client.ip ~ purge) {
-			  error 405 "Not allowed.";
+			  return(synth(403, "Not allowed."));
 		  }
 		  ban("req.http.host == " + req.http.host +
 		        "&& req.url == " + req.url);
 
 		  # Throw a synthetic page so the
                   # request won't go to the backend.
-		  return(synth(200m"Ban added"));
+		  return(synth(200, "Ban added"));
 	  }
   }
 
@@ -130,7 +130,7 @@ You can use the following template to write ban lurker friendly bans::
   sub vcl_recv {
     if (req.method == "PURGE") {
       if (client.ip !~ purge) {
-        error 401 "Not allowed";
+        return(synth(403, "Not allowed"));
       }
       ban("obj.http.x-url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple
     }
diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 73934c7..b30985c 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -22,8 +22,9 @@ yourself doing frequently.
 The vcl_recv subroutine may terminate with calling ``return()`` on one
 of the following keywords:
 
-  error 
-    Return the specified error code to the client and abandon the request.
+  synth 
+    Return a synthetic object with the specified error code to the
+    client and abandon the request.
 
   pass
     Switch to pass mode.  Control will eventually pass to vcl_pass.
@@ -51,7 +52,7 @@ shuffling bytes back and forth.
 The vcl_pipe subroutine may terminate with calling return() with one
 of the following keywords:
 
-  error code [reason]
+  synth(error code, reason)
     Return the specified error code to the client and abandon the request.
 
   pipe
@@ -68,7 +69,7 @@ submitted over the same client connection are handled normally.
 The vcl_pass subroutine may terminate with calling return() with one
 of the following keywords:
 
-  error [reason]
+  synth(error code, reason)
     Return the specified error code to the client and abandon the request.
 
   pass
@@ -157,13 +158,6 @@ keywords:
     error.
 
 
-.. XXX
-.. vcl_error
-.. ~~~~~~~~~
-
-.. Not sure if we're going to keep this around.
-
-
 vcl_backend_fetch
 ~~~~~~~~~~~~~~~~~
 



From lkarsten at varnish-software.com  Thu Mar 20 11:57:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 12:57:27 +0100
Subject: [master] 22dcc15 Updated to running.rst in users guide
Message-ID: <E1WQbbD-0001Zf-KH@project.varnish-software.com>


commit 22dcc15faf1aa41c90eeee64c06d25cb25a55cd6
Author: benc <benc at redpill-linpro.com>
Date:   Fri Mar 14 13:51:03 2014 +0100

    Updated to running.rst in users guide

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index dde63a6..830149a 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -18,19 +18,18 @@ Welcome to the Varnish documentation!
 Introduction
 ------------
 
-Varnish is a state of the art web accelerator. It has its mission in front of a
-web server and cache content.
+Varnish Cache is a web application accelerator also known as a caching HTTP reverse proxy. You install it in front of any server that speaks HTTP and configure it to cache the contents. Varnish Cache is really, really fast. It typically speeds up delivery with a factor of 300 - 1000x, depending on your architecture. 
+It has its mission in front of a
+web server and cache content and it makes your web site go faster.
 
-It makes your web site go faster.
-
-We suggest you start by reading the installation guide
-:ref:`install-index`. Once you have Varnish up and running go through
+To get started with Varnish-Cache we recommend that you read the installation guide
+:ref:`install-index`. Once you have Varnish up and running we recommend that you go through
 our tutorial - :ref:`tutorial-index`, and finally the :ref:`users_guide_index`.
 
-If you need to look up how a specific Varnish tool works, the
-:ref:`reference-index` can help you out. Changes from previous versions are in
-the :ref:`whats-new-index` chapter. In closing we have :ref:`phk`, a collection
-of blog posts related to Varnish and HTTP.
+If you need to find out how to use a specific Varnish tool, the
+:ref:`reference-index` contains detailed documentation over the tools. Changes from previous versions are located in
+the :ref:`whats-new-index` chapter. In closing, we have :ref:`phk`, a collection
+of blog posts from Poul-Henning Kamp related to Varnish and HTTP.
 
 
 
diff --git a/doc/sphinx/installation/bugs.rst b/doc/sphinx/installation/bugs.rst
index 98c0e19..dd71cc8 100644
--- a/doc/sphinx/installation/bugs.rst
+++ b/doc/sphinx/installation/bugs.rst
@@ -13,14 +13,17 @@ So if you run into a bug, it is important that you spend a little bit
 of time collecting the right information, to help us fix the bug.
 
 The most valuable information you can give us, is **always** how
-to trigger and reproduce the problem.  If you can tell us that, we
-rarely need anything else to solve it.  The caveat being, that we
+to trigger and reproduce the problem. If you can tell us that, we
+rarely need anything else to solve it.The caveat being, that we
 do not have a way to simulate high levels of real-life web-traffic,
 so telling us to "have 10.000 clients hit at once" does not really
 allow us to reproduce.
 
-Roughly we have three clases of bugs with Varnish, and the information
-we need to debug them depends on the kind of bug.
+To report a bug please follow the suggested procedure described in the "Trouble Tickets" 
+section of the documentation (above).
+
+Roughly we categorize bugs in to three kinds of bugs (described below) with Varnish. The information
+we need to debug them depends on what kind of bug we are facing.
 
 Varnish crashes
 ===============
@@ -32,11 +35,11 @@ does all the work, and the manager hangs around to resurrect it if it
 crashes.
 
 Therefore, the first thing to do if you see a Varnish crash, is to examine
-your syslogs to see if it has happened before.  (One site is rumoured
+your syslogs to see if it has happened before. (One site is rumoured
 to have had Varnish restarting every 10 minutes and *still* provide better
 service than their CMS system.)
 
-When it crashes, if at all possible, Varnish will spew out a crash dump
+When it crashes, which is highly unlikely to begin with, Varnish will spew out a crash dump
 that looks something like::
 
 	Child (32619) died signal=6 (core dumped)
@@ -62,7 +65,7 @@ If you can get that information to us, we are usually able to
 see exactly where things went haywire, and that speeds up bugfixing
 a lot.
 
-There will be a lot more information than this, and before sending
+There will be a lot more information in the crash dump besides this, and before sending
 it all to us, you should obscure any sensitive/secret
 data/cookies/passwords/ip# etc.  Please make sure to keep context
 when you do so, ie: do not change all the IP# to "X.X.X.X", but
@@ -73,7 +76,7 @@ The most important line is the "Panic Message", which comes in two
 general forms:
 
 "Missing errorhandling code in ..."
-	This is a place where we can conceive ending up, and have not
+	This is a situation where we can conceive Varnish ending up, which we have not
 	(yet) written the padded-box error handling code for.
 
 	The most likely cause here, is that you need a larger workspace
@@ -84,11 +87,11 @@ general forms:
 
 "Assert error in ..."
 	This is something bad that should never happen, and a bug
-	report is almost certainly in order.  As always, if in doubt
+	report is almost certainly in order. As always, if in doubt
 	ask us on IRC before opening the ticket.
 
 In your syslog it may all be joined into one single line, but if you
-can reproduce the crash, do so while running varnishd manually:
+can reproduce the crash, do so while running `varnishd` manually:
 
 	``varnishd -d <your other arguments> |& tee /tmp/_catch_bug``
 
@@ -105,7 +108,7 @@ kill the process and send us an email saying "Varnish hung, I
 restarted it" which gives us only about 1.01 bit of usable debug
 information to work with.
 
-What we need here is all the information can you squeeze out of
+What we need here is all the information you can squeeze out of
 your operating system **before** you kill the Varnish process.
 
 One of the most valuable bits of information, is if all Varnish'
@@ -115,18 +118,21 @@ furiously on some futile condition.
 Commands like ``top -H`` or ``ps -Haxlw`` or ``ps -efH`` should be
 able to figure that out.
 
+.. XXX:Maybe a short description of what valuable information the various commands above generates? /benc 
+
+
 If one or more threads are spinning, use ``strace`` or ``ktrace`` or ``truss``
 (or whatever else your OS provides) to get a trace of which system calls
-the Varnish process issues.  Be aware that this may generate a lot
-of very repetitive data, usually one second worth is more than enough.
+the Varnish process issues. Be aware that this may generate a lot
+of very repetitive data, usually one second worth of data is more than enough.
 
 Also, run ``varnishlog`` for a second, and collect the output
 for us, and if ``varnishstat`` shows any activity, capture that also.
 
 When you have done this, kill the Varnish *child* process, and let
 the *master* process restart it.  Remember to tell us if that does
-or does not work.  If it does not, kill all Varnish processes, and
-start from scratch.  If that does not work either, tell us, that
+or does not work. If it does not, kill all Varnish processes, and
+start from scratch. If that does not work either, tell us, that
 means that we have wedged your kernel.
 
 
@@ -139,10 +145,12 @@ what is wrong about what Varnish does.
 
 Be aware, that often Varnish does exactly what you asked it to, rather
 than what you intended it to do. If it sounds like a bug that would
-have tripped up everybody else, take a moment to read though your
-VCL and see if it really does what you think.
+have tripped up everybody else, take a moment to read through your
+VCL and see if it really does what you think it does.
 
 You can also try setting the ``vcl_trace`` parameter, that will generate log
 records with like and character number for each statement executed in your VCL
 program.
 
+.. XXX:Example of the command perhaps? benc
+
diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index aaea9ec..946e921 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -1,15 +1,14 @@
-%%%%%%%%%%%%%%%%%%
-Getting hold of us
-%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%
+Getting help
+%%%%%%%%%%%%
 
 Getting hold of the gang behind Varnish is pretty straight forward,
-we try to help as much as time permits and have tried to streamline
+we try to help out as much as time permits and have tried to streamline
 this process as much as possible.
 
 But before you grab hold of us, spend a moment composing your thoughts and
-formulate your question. There is nothing as pointless as simply telling
-us "Varnish does not work for me" with no further information to give
-any clue to why.
+formulate your question. From our perspective there is nothing as pointless as simply telling
+us "Varnish does not work for me" with no further information. This does not give us any relevant information to use when trying to figure out whats wrong. 
 
 And before you even do that, do a couple of searches to see if your
 question is already answered, if it has been, you will get your answer
@@ -20,7 +19,7 @@ IRC Channel
 
 The most immediate way to get hold of us is to join our IRC channel:
 
-	``#varnish on server irc.linpro.no``
+	`#varnish on server irc.linpro.no`
 
 The main timezone of the channel is Europe work hours.
 
@@ -32,27 +31,27 @@ If the channel is all quiet, try again some time later, we do have lives,
 families and jobs to deal with also.
 
 You are more than welcome to just hang out, and while we don't mind
-the occational intrusion of the real world into the flow, keep
-it mostly on topic, and don't paste random links unless they are
+the occasional intrusion from the real world into our flow, we try and keep
+it mostly on topic, and please don't paste random links unless they are
 *really* spectacular and intelligent.
 
 Mailing Lists
 =============
 
-Getting on or off our mailing lists happens through Mailman_.
+Subscribing or unsubscribing to our mailing lists is handled through our Mailman_.
 
-If you are going to use Varnish, subscribing to our ``varnish-announce``
-mailing list is probably a very good idea. The typical pattern is that
+If you are going to use Varnish, subscribing to our `varnish-announce`
+mailing list is a very good idea. The typical pattern is that
 people spend some time getting Varnish running, and then more or less
 forget about it. Therefore the announce list is a good way to be
-reminded about new releases, bad bugs or security holes.
+reminded about new releases, bugs or potential (security) vulnerabilities.
 
 
-The ``varnish-misc`` mailing list is for general banter, questions,
+The `varnish-misc` mailing list is for general banter, questions,
 suggestions, ideas and so on.  If you are new to Varnish it may pay
 off to subscribe to it, simply to have an ear to the telegraph-pole
-and learn some smart tricks.  This is a good place to ask for help
-with more complex issues, that require quoting of files and long
+and potentially learn some smart tricks.  This is also a good place to ask for help
+with more complex issues, that may require file-chunks, references to files and/or long
 explanations.
 
 Make sure to pick a good subject line, and if the subject of the
@@ -61,26 +60,29 @@ with hundreds of emails per day, after spam-filters, and we need all
 the help we can get to pick the interesting ones.
 
 
-The ``varnish-dev`` mailing list is used by the developers and is
-usually quite focused on source-code and such.  Everybody on
-the -dev list is also on -misc, so cross-posting only serves to annoy
+The `varnish-dev` mailing list is used by the developers and is
+usually quite focused on source-code and such. Everybody on
+the `-dev` list is also on `-misc`, so cross-posting only serves to annoy
 those people.
 
+We also maintain a community wiki_ for Varnish, there you will find information on planned events, meetings, current backlog, troube tickets ,  and links to resources and documentation. 
+
 .. XXX: we should introduce the wiki (if we care about it) before
 .. we start referring to it (below). Make a wiki chapter?
 
-
 Trouble Tickets
 ===============
 
 Please do not open a trouble ticket, unless you have spotted an actual
 bug in Varnish.  Ask on IRC first if you are in doubt.
 
-The reason for this policy, is to avoid the bugs being drowned in a
-pile of sensible suggestions for future enhancements and call for help
-from people who forget to check back if they get it and so on.
+The reason for this policy, is to avoid bugs being drowned in a
+pile of other `issues`, feature suggestions for future releases, and double postings of calls for help
+from people who forgot to check back on already opened Tickets. 
 
-We track suggestions and ideas in our `"Shopping-List" wiki page`_, and user
+.. XXX: Not sure what you want with the last sentence above. benc
+
+We instead track suggestions and feature ideas in our `"Shopping-List" wiki page`_, and through user
 support via email and IRC.
 
 Commercial Support
@@ -90,6 +92,8 @@ The following companies offer commercial Varnish support, and are listed
 here for your convenience.  If you want your company listed here, drop
 an email to phk at FreeBSD.org.
 
+.. XXX: Should we perhaps enhance this to explain Varnish Plus? benc
+
 	Varnish Software
 	sales at varnish-software.com
 
@@ -97,3 +101,4 @@ an email to phk at FreeBSD.org.
 .. _mailman: http://lists.varnish-cache.org/mailman/listinfo
 .. _pastebin: http://gist.github.com/
 .. _"Shopping-List" wiki page: http://varnish-cache.org/wiki/PostTwoShoppingList
+.. _wiki: https://www.varnish-cache.org/trac   
diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index 7952958..03e60b8 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -4,11 +4,8 @@
 Varnish Installation
 %%%%%%%%%%%%%%%%%%%%
 
-This document explains how to get Varnish onto your system, where
-to get help, how report bugs and so on.
-
-In other words, it is a manual about pretty much everything else than
-actually using Varnish to move traffic.
+This section covers installation prerequisites, a step-by-step installation procedure, how and where
+to get help, and how to report bugs. It also contains a set of platform specific notes to aid you when installing Varnish on certain platforms.
 
 .. XXX: rewrite the last paragraph.
 
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index a32eb68..348b6ef 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -6,15 +6,15 @@ Installing Varnish
 With open source software, you can choose to install binary packages
 or compile it yourself from source code. To install a package or compile
 from source is a matter of personal taste. If you don't know which
-method to choose, read the whole document and choose the method you
-are most comfortable with.
+method to choose, we recommend that you read this whole section and then choose the method you
+feel most comfortable with.
 
 
 Source or packages?
 -------------------
 
 Installing Varnish on most relevant operating systems can usually
-be done with with the systems package manager, typical examples
+be done with with the specific systems package manager, typical examples
 being:
 
 FreeBSD
@@ -38,13 +38,13 @@ Varnish is included in the `EPEL
 incompatible syntax changes in newer versions of Varnish, only older
 versions are available.
 
-We recommend that you install the latest version from our repository.
+We therefore recommend that you install the latest version directly from our repository, as described above.
 
 Debian/Ubuntu
 -------------
 
 Varnish is distributed with both Debian and Ubuntu. In order to get
-Varnish up and running type `sudo apt-get install varnish`. Please
+Varnish up and running type ``sudo apt-get install varnish``. Please
 note that this might not be the latest version of Varnish.  If you
 need a later version of Varnish, please follow the online installation
 instructions for `Debian
@@ -59,16 +59,16 @@ If there are no binary packages available for your system, or if you
 want to compile Varnish from source for other reasons, follow these
 steps:
 
-We recommend downloading a release tarball, which you can find on
+Download the appropriate release tarball, which you can find on
 `repo.varnish-cache.org/source <http://repo.varnish-cache.org/source/>`_.
 
 Alternatively, if you want to hack on Varnish, you should clone our
 git repository by doing.
 
-      git clone git://git.varnish-cache.org/varnish-cache
+      ``git clone git://git.varnish-cache.org/varnish-cache``
 
 Please note that a git checkout will need some more build-dependencies
-than listed below, in particular the Python Docutils and Sphinx.
+than listed below, in particular the `Python Docutis` and `Sphinx`.
 
 Build dependencies on Debian / Ubuntu
 --------------------------------------
@@ -76,21 +76,21 @@ Build dependencies on Debian / Ubuntu
 In order to build Varnish from source you need a number of packages
 installed. On a Debian or Ubuntu system these are:
 
-* autotools-dev
-* automake1.11
-* libtool
-* autoconf
-* libncurses-dev
-* groff-base
-* libpcre3-dev
-* pkg-config
-* make
-* libedit-dev
+* `autotools-dev`
+* `automake1.1`
+* `libtool`
+* `autoconf`
+* `libncurses-dev`
+* `groff-base`
+* `libpcre3-dev`
+* `pkg-config`
+* `make`
+* `libedit-dev`
 
 If you're building from git, you also need the following:
 
-* python-docutils
-* python-sphinx (optional, if you want to build the documentation)
+* `python-docutils`
+* `python-sphinx` (optional, if you want to build the documentation)
 
 Build dependencies on RedHat / CentOS
 --------------------------------------
@@ -98,19 +98,19 @@ Build dependencies on RedHat / CentOS
 To build Varnish on a RedHat or CentOS system you need the following
 packages installed:
 
-* automake
-* autoconf
-* libtool
-* ncurses-devel
-* groff
-* pcre-devel
-* pkgconfig
-* libedit-devel
+* `automake`
+* `autoconf`
+* `libtool`
+* `ncurses-devel`
+* `groff`
+* `pcre-devel`
+* `pkgconfig`
+* `libedit-devel`
 
 If you're building from git, you also need the following:
 
-* docutils
-* python-sphinx (optional, if you want to build the documentation)
+* `docutils`
+* `python-sphinx` (optional, if you want to build the documentation)
 
 Compiling Varnish
 -----------------
@@ -123,18 +123,17 @@ taken care of::
 	sh configure
 	make
 
-The ``configure`` script takes some arguments, but more likely than
-not you can forget about that for now, almost everything in Varnish
-are run time parameters.
+The `configure` script takes some arguments, but more likely than
+not you can forget about that for now, almost everything in Varnish can be tweaked with run time parameters.
 
 Before you install, you may want to run the test suite, make a cup of
-tea while it runs, it takes some minutes::
+tea while it runs, it usually takes a couple of minutes::
 
 	make check
 
-Don't worry if a single or two tests fail, some of the tests are a
-bit too timing sensitive (Please tell us which so we can fix it) but
-if a lot of them fails, and in particular if the ``b00000.vtc`` test
+Don't worry if one or two tests fail, some of the tests are a
+bit too timing sensitive (Please tell us which so we can fix them.) but
+if a lot of them fails, and in particular if the `b00000.vtc` test
 fails, something is horribly wrong, and you will get nowhere without
 figuring out what.
 
@@ -145,9 +144,9 @@ And finally, the true test of a brave heart::
 
 	sudo make install
 
-Varnish will now be installed in /usr/local. The varnishd binary is in
-/usr/local/sbin/varnishd and its default configuration will be
-/usr/local/etc/varnish/default.vcl.
+Varnish will now be installed in `/usr/loca``l. The ``varnishd` binary is in
+`/usr/local/sbin/varnishd` and its default configuration will be
+`/usr/local/etc/varnish/default.vcl`.
 
-You can now proceed to the :ref:`tutorial-index`.
+After succesful installation you are ready to proceed to the :ref:`tutorial-index`.
 
diff --git a/doc/sphinx/installation/platformnotes.rst b/doc/sphinx/installation/platformnotes.rst
index a7d3fb4..7f11eaa 100644
--- a/doc/sphinx/installation/platformnotes.rst
+++ b/doc/sphinx/installation/platformnotes.rst
@@ -31,7 +31,7 @@ performance.
 If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
 the maximum stack size before starting Varnish.
 
-The default allocates to much memory per thread, which will make Varnish fail
+The default allocates too much memory per thread, which will make Varnish fail
 as soon as the number of threads (traffic) increases.
 
 Reduce the maximum stack size by running::
@@ -44,13 +44,13 @@ TCP keep-alive configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 On some Solaris, FreeBSD and OS X systems, Varnish is not able to set the TCP
-keep-alive values per socket, and therefor the tcp_keepalive_* Varnish runtime
-parameters are not available. On these platforms it can be benefitial to tune
+keep-alive values per socket, and therefore the *tcp_keepalive_* Varnish runtime
+parameters are not available. On these platforms it can be beneficial to tune
 the system wide values for these in order to more reliably detect remote close
 for sessions spending long time on waitinglists. This will help free up
 resources faster.
 
-Systems to not support TCP keep-alive values per socket include:
+Systems that does not support TCP keep-alive values per socket include:
 
 - Solaris releases prior to version 11
 - FreeBSD releases prior to version 9.1
@@ -59,9 +59,11 @@ Systems to not support TCP keep-alive values per socket include:
 On platforms with the necessary socket options the defaults are set
 to:
 
-- tcp_keepalive_time = 600 seconds
-- tcp_keepalive_probes = 5
-- tcp_keepalive_intvl = 5 seconds
+- `tcp_keepalive_time` = 600 seconds
+- `tcp_keepalive_probes` = 5
+- `tcp_keepalive_intvl` = 5 seconds
 
 Note that Varnish will only apply these run-time parameters so long as
 they are less than the system default value.
+
+.. XXX:Maybe a sample-command of using/setting/changing these values? benc
diff --git a/doc/sphinx/installation/prerequisites.rst b/doc/sphinx/installation/prerequisites.rst
index 736bcd8..7ef0bd7 100644
--- a/doc/sphinx/installation/prerequisites.rst
+++ b/doc/sphinx/installation/prerequisites.rst
@@ -7,18 +7,17 @@ In order for you to install Varnish you must have the following:
   * A recent, preferably server grade, computer.
   * A fairly modern and 64 bit version of either
     - Linux
-    - FreeBSD
-    - Solaris (x86 only)
-  * root access to said system.
+    - FreeBSD, or
+    - Solaris (x86 only).
+  * Root access.
 
 
-Varnish can be installed on other UNIX systems as well, but it is not
-tested particularly well on these platforms. Varnish is, from time to
+Varnish can be installed on other UNIX systems as well, but it is not extensively or systematically tested by us on other systems than the above. Varnish is, from time to
 time, said to work on:
 
-  * 32 bit versions of the before-mentioned systems.
-  * OS X
-  * NetBSD
-  * OpenBSD
-  * Windows with Cygwin
+  * 32 bit versions of the before-mentioned systems,
+  * OS X,
+  * NetBSD,
+  * OpenBSD, and
+  * Windows with Cygwin.
 
diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
index 476d325..1c9ab65 100644
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -3,14 +3,14 @@
 Backend servers
 ---------------
 
-Varnish has a concept of "backend" or "origin" servers. A backend
-server is the server providing the content Varnish will accelerate.
+Varnish has a concept of `backend` or `origin` servers. A backend
+server is the server providing the content Varnish will accelerate via the cache.
 
 Our first task is to tell Varnish where it can find its content. Start
 your favorite text editor and open the Varnish default configuration
 file. If you installed from source this is
-/usr/local/etc/varnish/default.vcl, if you installed from a package it
-is probably /etc/varnish/default.vcl.
+`/usr/local/etc/varnish/default.vcl`, if you installed from a package it
+is probably `/etc/varnish/default.vcl`.
 
 If you've been following the tutorial there is probably a section of
 the configuration that looks like this:::
@@ -41,11 +41,16 @@ localhost, port 8080.::
   }
 
 
-Varnish can have several backends defined and can you can even join
+Varnish can have several backends defined and can even join
 several backends together into clusters of backends for load balancing
 purposes, having Varnish pick one backend based on different
 algorithms. 
 
 A lot of the power of Varnish Cache comes from it's design, which
-might not be what you are expecting. Next, let's have a look at some of
-them.
+might not be what you are expecting.
+
+.. XXX:What am I expecting? benc
+
+Next, let's have a look at some of what makes Varnish unique and what you can do with it.
+
+
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index c540f8e..44008e7 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -4,12 +4,12 @@
 The Varnish Tutorial
 %%%%%%%%%%%%%%%%%%%%
 
-This documents the basic concept. What is Varnish and how does it
-work? How can I get it up and running. Most likely you would want to
-continue with the users guide (:ref:`users-guide-index`) afterwards.
+This section covers the Varnish basics in a tutorial form. It will cover what Varnish is and how it
+works. It also covers how to get Varnish up and running. After this section you probably would want to
+continue with the users guide (:ref:`users-guide-index`).
 
 If you're reading this on the web note the "Next topic" and "Previous
-topic" on the right side.
+topic" links on the right side of each page.
 
 .. toctree:: :maxdepth: 1
 
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 9e6c541..98cfddc 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -3,25 +3,22 @@
 The fundamentals of web proxy caching with Varnish
 --------------------------------------------------
 
-Varnish is a caching HTTP reverse proxy. It receives requests from
-clients and tries to answer them from its cache. If it cannot answer
-the request from its cache it will forward the request to the backend,
-fetch the response, store it and deliver it to the client.
+Varnish is a caching HTTP reverse proxy. It recieves requests from
+clients and tries to answer them from the cache. If Varnish cannot answer
+the request from the cache it will forward the request to the backend,
+fetch the response, store it in the cache and deliver it to the client.
 
 When Varnish has a cached response ready it is typically delivered in
 a matter of microseconds, several orders of magnitude faster than your
-typical application server, so you want to make sure to have it answer
-as many of the requests as possible.
+typical backend server, so you want to make sure to have Varnish answer
+as many of the requests as possible directly from the cache.
 
 Varnish decides whether it can store the content or not based on the
 response it gets back from the backend. The backend can instruct
 Varnish to cache the content with the HTTP response header
-Cache-Control. There are a few other conditions where Varnish will not
-cache, the most common one being cookies. Since cookies is a good
-indication that a web object is personalised, Varnish will with its
-default configuration, not cache it.
-
-This behaviour and most other behaviour can be changed using policies
+`Cache-Control`. There are a few conditions where Varnish will not
+cache, the most common one being the use of cookies. Since cookies indicates a client-specific web object, Varnish will by default not cache it. 
+This behaviour as most of Varnish functionality can be changed using policies
 written in the Varnish Configuration Language. See the Users Guide
 for more information on how to do that.
 
@@ -34,6 +31,8 @@ turning performance into a non-issue. You get to focus on how your web
 application work and you can allow yourself, to some degree, to care
 less about performance and scalability.
 
+.. XXX:Not totally sure what the last sentence above means. benc
+
 Flexibility
 ~~~~~~~~~~~
 
@@ -77,3 +76,5 @@ information and signup.  There is also a web forum on the same site.
 
 Now that you have a vague idea on what Varnish Cache is, let see if we
 can get it up and running.
+
+.. XXX:The above three paragraphs are repetetive this is already handled in previos chapters. The only new information is Governing Board which could be moved to the introduction and the paragraphs scrapped. benc
diff --git a/doc/sphinx/tutorial/peculiarities.rst b/doc/sphinx/tutorial/peculiarities.rst
index af12eff..45f2f25 100644
--- a/doc/sphinx/tutorial/peculiarities.rst
+++ b/doc/sphinx/tutorial/peculiarities.rst
@@ -3,16 +3,19 @@ Peculiarities
 -------------
 
 There are a couple of things that are different with Varnish Cache, as
-opposed to other programs. One thing you've already seen - VCL. I'll
-just give you a very quick tour of the other pecularities.
+opposed to other programs. One thing you've already seen - VCL. In this section we provide a very quick tour of other pecularities you need to know about to get the most out of Varnish.
 
 Configuration
 ~~~~~~~~~~~~~
 
 The Varnish Configuration is written in VCL. When Varnish is ran this
 configuration is transformed into C code and then fed into a C
-compiler, loaded and run. So, as opposed to declaring various
-settings, you write polices on how the incoming traffic should be
+compiler, loaded and executed.
+
+.. XXX:Ran sounds strange above, maybe "is running" "is started" "executes"? benc
+
+So, as opposed to switching various
+settings on or off, you write polices on how the incoming traffic should be
 handled.
 
 
@@ -20,8 +23,8 @@ varnishadm
 ~~~~~~~~~~
 
 Varnish Cache has an admin console. You can connect it it through the
-"varnishadm" command. In order to connect the user needs to be able to
-read /etc/varnish/secret in order to authenticate.
+``varnishadm`` command. In order to connect the user needs to be able to
+read `/etc/varnish/secret` in order to authenticate.
 
 Once you've started the console you can do quite a few operations on
 Varnish, like stopping and starting the cache process, load VCL,
@@ -30,11 +33,13 @@ adjust the built in load balancer and invalidate cached content.
 It has a built in command "help" which will give you some hints on
 what it does.
 
+.. XXX:sample of the command here. benc
+
 varnishlog
 ~~~~~~~~~~
 
-Varnish does not log to disk. Instead it logs to a bit of memory. It
-is like a stream of logs. At any time you'll be able to connect to the
+Varnish does not log to disk. Instead it logs to a chunk of memory. It
+is actually streaming the logs. At any time you'll be able to connect to the
 stream and see what is going on. Varnish logs quite a bit of
 information. You can have a look at the logstream with the command
 ``varnishlog``.
diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
index acef0c6..0fc2ddc 100644
--- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
+++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
@@ -9,13 +9,15 @@ First we stop varnish::
 
      # service varnish stop
 
+.. XXX:This renders to a different font than other commands. it should be the double backtick format for the command. benc
+
 Now we need to edit the configuration file that starts Varnish. 
 
 
 Debian/Ubuntu
 ~~~~~~~~~~~~~
 
-On Debian/Ubuntu this is /etc/default/varnish. In the file you'll find
+On Debian/Ubuntu this is `/etc/default/varnish`. In the file you'll find
 some text that looks like this::
 
   DAEMON_OPTS="-a :6081 \
@@ -36,7 +38,7 @@ Red Hat EL / Centos
 ~~~~~~~~~~~~~~~~~~~
 
 On Red Hat EL / Centos
-On Red Hat/Centos it is /etc/sysconfig/varnish
+On Red Hat/Centos it is `/etc/sysconfig/varnish`
 
 
 Restarting Varnish
diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
index 58bf574..2896394 100644
--- a/doc/sphinx/tutorial/starting_varnish.rst
+++ b/doc/sphinx/tutorial/starting_varnish.rst
@@ -4,36 +4,35 @@
 Starting Varnish
 ----------------
 
-This tutorial will assume that you are either running Ubuntu, Debian,
+This tutorial will assume that you are running Varnish on Ubuntu, Debian,
 Red Hat Enterprise Linux or Centos. Those of you running on other
 platforms might have to do some mental translation exercises in order
-to follow me. Since you're on a "weird" platform you're probably used
+to follow this. Since you're on a "weird" platform you're probably used
 to it. :-)
 
-I assume you have Varnish Cache installed.
+Make sure you have Varnish succesfully installed (following one of the procedures described in "Installing Varnish" above.
 
-You start Varnish with ``service varnish start``.  This will start
+When properly installed you start Varnish with ``service varnish start``.  This will start
 Varnish if it isn't already running.
 
+.. XXX:What does it do if it is already running? benc
+
 Now you have Varnish running. Let us make sure that it works
 properly. Use your browser to go to http://127.0.0.1:6081/
-(obviously, you should replace the IP address with one on your own
-system). 
-
+(obviously, you should replace the IP address with the IP for the machine that currently runs Varnish. 
 The default configuration will try to forward requests to a web
-service running on the same computer as Varnish was installed at,
-port 8080.
+application running on the same machine as Varnish was installed on. Varnish expects the web application to be exposed over http on port 8080.
 
-If there is no web application being served up there Varnish will
+If there is no web application being served up on that location Varnish will
 issue an error. Varnish Cache is very conservative about telling the
 world what is wrong so whenever something is amiss it will issue the
 same generic "Error 503 Service Unavilable".
 
 You might have a web application running on some other port or some
-other computer. Let's edit the configuration and make it point to
+other machine. Let's edit the configuration and make it point to
 something that actually works.
 
-Fire up your favorite editor and edit /etc/varnish/default.vcl. Most
+Fire up your favorite editor and edit `/etc/varnish/default.vcl`. Most
 of it is commented out but there is some text that is not. It will
 probably look like this::
 
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 7cd0c25..3d4d831 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -15,7 +15,7 @@ The Varnish documentation consists of three main documents:
   looking up specific questions.
 
 After :ref:`users_intro`, this Users Guide is organized in sections
-along the major interfaces to Varnish as a service:
+following the major interfaces to Varnish as a service:
 
 :ref:`users_running` is about getting Varnish configured, with
 respect to storage, sockets, security and how you can control and
@@ -25,12 +25,12 @@ communicate with Varnish once it is running.
 HTTP requests the way you want, what to cache, how to cache it,
 modifying HTTP headers etc. etc.
 
-:ref:`users_report` explains how you can see and monitor what Varnish does,
-from transaction level to aggregate statistics.
+:ref:`users_report` explains how you can monitor what Varnish does,
+from a transactional level to aggregating statistics.
 
 :ref:`users_performance` is about tuning your website with Varnish.
 
-:ref:`users_trouble` is for locating and fixing trouble with Varnish.
+:ref:`users_trouble` is for locating and fixing common issues with Varnish.
 
 .. toctree::
    :maxdepth: 2
diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index 6a67fc5..8b31b32 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -3,10 +3,12 @@
 The Big Varnish Picture
 =======================
 
-What is in this package called "Varnish", what are all the different
-bits and pieces named and will you need a hex-wrench for assembly?
+In this section we will cover the questions:
+- What is in this package called "Varnish"?
+- what are all the different bits and pieces named? 
+- Will you need a hex-wrench for assembly?
 
-The two main parts of Varnish are the two processes in the varnishd
+The two main parts of Varnish are the two processes in the `varnishd`
 program. The first process is called "the manager", and its job is to
 talk to you, the administrator, and make the things you ask for
 happen.
@@ -15,12 +17,12 @@ The second process is called "the worker" or just "the child" and
 this is the process which does all the actual work with your HTTP
 traffic.
 
-When you start varnishd, you start the manager process, and once it is
+When you start `varnishd`, you start the manager process, and once it is
 done handling all the command line flags, it will start the child
 process for you. Should the child process die, the manager will start
 it again for you, automatically and right away.
 
-The main reason for this division of labor is security:  The manager
+The main reason for this division of labor is security: The manager
 process will typically run with "root" permissions, in order to
 open TCP socket port 80, but it starts the child process with minimal
 permissions, as a defensive measure.
@@ -39,11 +41,15 @@ or tie it to your CMS.
 
 All this is covered in :ref:`users_running`.
 
-How the child process should deal with the HTTP requests, what to
-cache, which headers to remove etc, is al specified in a small
+Things like, how the child process should deal with the HTTP requests, what to
+cache, which headers to remove etc, is all specified using a small
 programming language called VCL -- Varnish Configuration Language.
 The manager process will compile the VCL program and check it for
-errors, but it is the child process which runs the VCL program, for
+errors,
+
+.. XXX:What does manager do after compile and error-check? Maybe a short description of further handling when no errors as well as when errors? benc
+
+but it is the child process which runs the VCL program, for
 each and every HTTP request which comes in.
 
 Because the VCL is compiled to C code, and the C code is compiled
@@ -69,9 +75,9 @@ can do for your HTTP traffic, there really is no limit.
 
 :ref:`users_vcl` describes VCL and what it can do in great detail.
 
-Varnish uses a piece of shared memory to report its activity and
+Varnish uses a segment of shared memory to report and log its activities and
 status. For each HTTP request, a number of very detailed records will
-be appended to the log segment in this shared memory.  Other processes
+be appended to the log memory segment. Other processes
 can subscribe to log-records, filter them, and format them, for
 instance as Apache/NCSA style log records.
 
@@ -81,24 +87,25 @@ of cache hit-rate, resource usage and specific performance indicating
 metrics.
 
 Varnish comes with a number of tools which reports from shared
-memory, varnishlog, varnishstats, varnishncsa etc, and with a API
+memory, `varnishlog`, `varnishstats`, `varnishncsa` etc, and with an API
 library so you can write your own tools, should you need that.
-writing
 
 :ref:`users_report` explains how all that work.
 
-Presumably the reason why you are interested in Varnish, is that you
+Presumably the reason for your interest in Varnish, is that you
 want your website to work better. There are many aspects of
 performance tuning a website, from relatively simple policy decisions
 about what to cache, to designing a geographically diverse multilevel
 CDNs using ESI and automatic failover.
 
+.. XXX:CDNs or CDN? benc
+
 :ref:`users_performance` will take you through the possibilities
 and facilities Varnish offers.
 
-Finally, Murphys Law must be contended with: Things will go wrong, and
+Finally, Murphys Law must be referenced here: Things will go wrong, and
 more likely than not, they will do so at zero-zero-dark O'clock. Most
-likely during a hurricane, when your phones battery is flat and your
+likely during a hurricane, when your phone battery is flat and your
 wife had prepared a intimate evening to celebrate your anniversary.
 
 Yes, we've all been there, haven't we?



From lkarsten at varnish-software.com  Thu Mar 20 11:57:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 12:57:27 +0100
Subject: [master] 45637bb Done. Important changes are consequently commented
 and all comments are marked with the string 'benc' for easy identification.
 There are some heavy refactorings done,
 that fundamentally changes content meaing.
Message-ID: <E1WQbbD-0001Zm-MO@project.varnish-software.com>


commit 45637bb9988ef7f20e1fab51ca8744c0a62a6651
Author: benc <benc at redpill-linpro.com>
Date:   Sun Mar 16 12:50:38 2014 +0100

    Done. Important changes are consequently commented and all comments are marked with the string 'benc' for easy identification. There are some heavy refactorings done, that fundamentally changes content meaing.

diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst
index dc25087..9016d51 100644
--- a/doc/sphinx/users-guide/command-line.rst
+++ b/doc/sphinx/users-guide/command-line.rst
@@ -3,22 +3,22 @@
 Important command line arguments
 --------------------------------
 
-There a two command line arguments you will simply have choose
-values for, what TCP port serve HTTP from and where the backend
-server can be contacted.
+There a two command line arguments you have to set when starting Varnish, these are: 
+* what TCP port to serve HTTP from, and 
+* where the backend server can be contacted.
 
-If you run Varnish from a package for your operating system,
+If you have installed Varnish through using a provided operating system bound package,
 you will find the startup options here:
 
-* Debian, Ubuntu: /etc/default/varnish
-* Red Hat, Centos: /etc/sysconfig/varnish
-* FreeBSD: /etc/rc.conf (See also: /usr/local/etc/rc.d/varnishd)
+* Debian, Ubuntu: `/etc/default/varnish`
+* Red Hat, Centos: `/etc/sysconfig/varnish`
+* FreeBSD: `/etc/rc.conf` (See also: /usr/local/etc/rc.d/varnishd)
 
 
--a *listen_address*
+'-a' *listen_address*
 ^^^^^^^^^^^^^^^^^^^
 
-What address should Varnish listen to, and service HTTP requests from.
+The '-a' argument defines what address Varnish should listen to, and service HTTP requests from.
 
 You will most likely want to set this to ":80" which is the Well
 Known Port for HTTP.
@@ -26,7 +26,7 @@ Known Port for HTTP.
 You can specify multiple addresses separated by a comma, and you
 can use numeric or host/service names if you like, Varnish will try
 to open and service as many of them as possible, but if none of them
-can be opened, varnishd will not start.
+can be opened, `varnishd` will not start.
 
 Here are some examples::
 
@@ -36,16 +36,19 @@ Here are some examples::
 	-a '[fe80::1]:80'
 	-a '0.0.0.0:8080,[::]:8081'
 
-If your webserver runs on the same computer, you will have to move
+.. XXX:brief explanation of some of the more comples examples perhaps? benc
+
+If your webserver runs on the same machine, you will have to move
 it to another port number first.
 
--f *VCL-file* or -b *backend*
+'-f' *VCL-file* or '-b' *backend*
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+
 Varnish needs to know where to find the HTTP server it is caching for.
-You can either specify it with -b, or you can put it in your own VCL file.
+You can either specify it with the '-b' argument, or you can put it in your own VCL file, specified with the '-f' argument.
 
-Using -b is a quick way to get started::
+Using '-b' is a quick way to get started::
 
 	-b localhost:81
 	-b thatotherserver.example.com:80
@@ -54,31 +57,34 @@ Using -b is a quick way to get started::
 Notice that if you specify a name, it can at most resolve to one IPv4
 *and* one IPv6 address.
 
-If you go with -f, you can start with a VCL file containing just::
+If you go with '-f', you can start with a VCL file containing just::
 
 	backend default {
 		.host = "localhost:81";
 	}
 
-which is exactly what -b does.
+which is exactly what '-b' does.
+
+.. XXX:What happens if I start with -b and then have the backend defined in my VCL? benc
 
 In both cases the built-in VCL code is appended.
 
 Other options
 ^^^^^^^^^^^^^
 
-Varnish has more command line arguments you can and maybe want
-to tweak, but to get started, the above will be sufficient.
+Varnish comes with an abundance of useful command line arguments. We recommend that you study them but not necessary use them all, but to get started, the above will be sufficient.
 
 By default Varnish will use 100 megabytes of malloc(3) storage
 for caching objects, if you want to cache more than that, you
 should look at the '-s' argument.
 
+.. XXX: 3? benc
+
 If you run a really big site, you may want to tune the number of
 worker threads and other parameters with the '-p' argument,
 but we generally advice not to do that unless you need to.
 
-Before you go into production, you may also want to re-visit the
+Before you go into production, you may also want to revisit the
 chapter
 :ref:`run_security` to see if you need to partition administrative
 privileges.
diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
index d611807..fc84f9f 100644
--- a/doc/sphinx/users-guide/compression.rst
+++ b/doc/sphinx/users-guide/compression.rst
@@ -3,31 +3,33 @@
 Compression
 -----------
 
-New in Varnish 3.0 was native support for compression, using gzip
+In Varnish 3.0 we introduced native support for compression, using gzip
 encoding. *Before* 3.0, Varnish would never compress objects. 
 
-In Varnish 3.0 compression defaults to "on", meaning that it tries to
+In Varnish 4.0 compression defaults to "on", meaning that it tries to
 be smart and do the sensible thing.
 
+.. XXX:Heavy refactoring to VArnish 4 above. benc
+
 If you don't want Varnish tampering with the encoding you can disable
-compression all together by setting the parameter http_gzip_support to
-*false*. Please see man :ref:`ref-varnishd` for details.
+compression all together by setting the parameter 'http_gzip_support' to
+false. Please see man :ref:`ref-varnishd` for details.
 
 
 Default behaviour
 ~~~~~~~~~~~~~~~~~
 
-The default for Varnish is to check if the client supports our
+The default behaviour for Varnish is to check if the client supports our
 compression scheme (gzip) and if it does it will override the
-Accept-Encoding header and set it to "gzip".
+'Accept-Encoding' header and set it to "gzip".
 
-When Varnish then issues a backend request the Accept-Encoding will
+When Varnish then issues a backend request the 'Accept-Encoding' will
 then only consist of "gzip". If the server responds with gzip'ed
 content it will be stored in memory in its compressed form. If the
-backend sends content in clear text it will be stored like that.
+backend sends content in clear text it will be stored in clear text.
 
 You can make Varnish compress content before storing it in cache in
-vcl_fetch by setting do_gzip to true, like this::
+`vcl_fetch` by setting 'do_gzip' to true, like this::
 
    sub vcl_backend_response {
         if (beresp.http.content-type ~ "text") {
@@ -38,9 +40,7 @@ vcl_fetch by setting do_gzip to true, like this::
 Please make sure that you don't try to compress content that is
 uncompressable, like jpgs, gifs and mp3. You'll only waste CPU
 cycles. You can also uncompress objects before storing it in memory by
-setting do_gunzip to *true* but I have no idea why anybody would want
-to do that.
-
+setting 'do_gunzip' to true but that will ususally not be the most sensible thing to do.
 Generally, Varnish doesn't use much CPU so it might make more sense to
 have Varnish spend CPU cycles compressing content than doing it in
 your web- or application servers, which are more likely to be
@@ -49,7 +49,7 @@ CPU-bound.
 GZIP and ESI
 ~~~~~~~~~~~~
 
-If you are using Edge Side Includes you'll be happy to note that ESI
+If you are using Edge Side Includes (ESIs) you'll be happy to note that ESI
 and GZIP work together really well. Varnish will magically decompress
 the content to do the ESI-processing, then recompress it for efficient
 storage and delivery. 
@@ -58,10 +58,10 @@ storage and delivery.
 Clients that don't support gzip
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If the client does not support gzip the Accept-Encoding header is left
-alone and we'll end up serving whatever we get from the backend
-server. Remember that the Backend might tell Varnish to *Vary* on the
-Accept-Encoding.
+If the client does not support gzip the 'Accept-Encoding' header is left
+alone then we'll end up serving whatever we get from the backend
+server. Remember that the backend might tell Varnish to *Vary* on the
+'Accept-Encoding'.
 
 If the client does not support gzip but we've already got a compressed
 version of the page in memory Varnish will automatically decompress
@@ -71,5 +71,5 @@ the page while delivering it.
 A random outburst
 ~~~~~~~~~~~~~~~~~
 
-Poul has written :ref:`phk_gzip` which talks abit more about how the
+Poul-Henning Kamp has written :ref:`phk_gzip` which talks abit more about how the
 implementation works. 
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index 5750918..5067c4c 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -10,14 +10,13 @@ Use cases for this are for example to send size reduced files to mobile
 clients with small screens and on high latency networks, or to 
 provide a streaming video codec that the client understands.
 
-There are a couple of strategies on what to do with such clients:
-1) Redirect them to another URL.
-2) Use a different backend for the special clients.
-3) Change the backend requests so the usual backend sends tailored content.
+There are a couple of typical strategies to use for this type of scenario:
+1) Redirect to another URL.
+2) Use a different backend for the special client.
+3) Change the backend request so that the backend sends tailored content.
 
-To make the examples easier to understand, it is assumed in this text 
-that all the req.http.X-UA-Device header is present and unique per client class
-that content is to be served to. 
+To perhaps make the strategies easier to understand, we, in this context, assume
+that the `req.http.X-UA-Device` header is present and unique per client class. 
 
 Setting this header can be as simple as::
 
@@ -28,34 +27,34 @@ Setting this header can be as simple as::
    }
 
 There are different commercial and free offerings in doing grouping and
-identifying clients in further detail than this. For a basic and community
+identifying clients in further detail. For a basic and community
 based regular expression set, see
-https://github.com/varnish/varnish-devicedetect/ .
+https://github.com/varnish/varnish-devicedetect/.
 
 
 Serve the different content on the same URL
 -------------------------------------------
 
 The tricks involved are: 
-1. Detect the client (pretty simple, just include devicedetect.vcl and call
-it)
-2. Figure out how to signal the backend what client class this is. This
+1. Detect the client (pretty simple, just include `devicedetect.vcl` and call
+it).
+2. Figure out how to signal the backend the client class. This
 includes for example setting a header, changing a header or even changing the
 backend request URL.
-3. Modify any response from the backend to add missing Vary headers, so
+3. Modify any response from the backend to add missing 'Vary' headers, so
 Varnish' internal handling of this kicks in.
 4. Modify output sent to the client so any caches outside our control don't
 serve the wrong content.
 
-All this while still making sure that we only get 1 cached object per URL per
+All this needs to be done while still making sure that we only get one cached object per URL per
 device class.
 
 
 Example 1: Send HTTP header to backend
 ''''''''''''''''''''''''''''''''''''''
 
-The basic case is that Varnish adds the X-UA-Device HTTP header on the backend
-requests, and the backend mentions in the response Vary header that the content
+The basic case is that Varnish adds the 'X-UA-Device' HTTP header on the backend
+requests, and the backend mentions in the response 'Vary' header that the content
 is dependant on this header. 
 
 Everything works out of the box from Varnish' perspective.
@@ -103,13 +102,13 @@ Example 2: Normalize the User-Agent string
 ''''''''''''''''''''''''''''''''''''''''''
 
 Another way of signaling the device type is to override or normalize the
-User-Agent header sent to the backend.
+'User-Agent' header sent to the backend.
 
-For example
+For example::
 
     User-Agent: Mozilla/5.0 (Linux; U; Android 2.2; nb-no; HTC Desire Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1
 
-becomes:
+becomes::
 
     User-Agent: mobile-android
 
@@ -218,7 +217,7 @@ Different backend for mobile clients
 ------------------------------------
 
 If you have a different backend that serves pages for mobile clients, or any
-special needs in VCL, you can use the X-UA-Device header like this::
+special needs in VCL, you can use the 'X-UA-Device' header like this::
 
     backend mobile {
         .host = "10.0.0.1";
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 9035224..45f8814 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -3,19 +3,23 @@
 Content composition with Edge Side Includes
 -------------------------------------------
 
-Varnish can cache create web pages by putting different pages
-together. These *fragments* can have individual cache policies. If you
-have a web site with a list showing the 5 most popular articles on
-your site, this list can probably be cached as a fragment and included
-in all the other pages. Used properly it can dramatically increase
+Varnish can cache create web pages by assembling different pages, called `fragments`,
+together into one page. These `fragments` can have individual cache policies. If you
+have a web site with a list showing the five most popular articles on
+your site, this list can probably be cached as a `fragment` and included
+in all the other pages.
+
+.. XXX:What other pages? benc
+
+Used properly this strategy can dramatically increase
 your hit rate and reduce the load on your servers. 
 
-In Varnish we've only implemented a small subset of ESI. As of 2.1 we
-have three ESI statements:
+In Varnish we've only so far implemented a small subset of ESI. As of version 2.1 we
+have three ESI statements::
 
- * esi:include 
- * esi:remove
- * <!--esi ...-->
+ esi:include 
+ esi:remove
+ <!--esi ...-->
 
 Content substitution based on variables and cookies is not implemented
 but is on the roadmap. At least if you look at the roadmap from a
@@ -58,13 +62,13 @@ For ESI to work you need to activate ESI processing in VCL, like this::
 
 Example: esi:remove and <!--esi ... -->
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The <esi:remove> and <!--esi ... --> constructs can be used to present
+The `<esi:remove>` and `<!--esi ... -->` constructs can be used to present
 appropriate content whether or not ESI is available, for example you can
 include content when ESI is available or link to it when it is not.
-ESI processors will remove the start ("<!--esi") and end ("-->") when
+ESI processors will remove the start ("<!--esi") and the end ("-->") when
 the page is processed, while still processing the contents. If the page
-is not processed, it will remain, becoming an HTML/XML comment tag.
-ESI processors will remove <esi:remove> tags and all content contained
+is not processed, it will remain intact, becoming a HTML/XML comment tag.
+ESI processors will remove `<esi:remove>` tags and all content contained
 in them, allowing you to only render the content when the page is not
 being ESI-processed.
 For example::
@@ -77,10 +81,10 @@ For example::
   <esi:include src="http://example.com/LICENSE" />
   -->
 
-Doing ESI on JSON and other non-XMLish content
+Doing ESI on JSON and other non-XML'ish content
 ----------------------------------------------
 
 Please note that Varnish will peek at the included content. If it
 doesn't start with a "<" Varnish assumes you didn't really mean to
 include it and disregard it. You can alter this behaviour by setting
-the esi_syntax parameter (see ref:`ref-varnishd`).
+the 'esi_syntax' parameter (see ref:`ref-varnishd`).
diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 3e1adc5..2f22e3c 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -7,37 +7,37 @@ Now that Varnish is up and running, and you can access your web
 application through Varnish. Unless your application is specifically
 written to work behind a web accelerator you'll probably need to do
 some changes to either the configuration or the application in order
-to get a high hit rate in Varnish.
+to get a high hitrate in Varnish.
 
 Varnish will not cache your data unless it's absolutely sure it is
 safe to do so. So, for you to understand how Varnish decides if and
-how to cache a page, I'll guide you through a couple of tools that you
-will find useful.
+how to cache a page, We'll guide you through a couple of tools that you
+should find useful to understand what is happening in your Varnish setup.
 
-Note that you need a tool to see what HTTP headers fly between you and
-the web server. On the Varnish server, the easiest is to use
-varnishlog and varnishtop but sometimes a client-side tool makes
-sense. Here are the ones I use.
+Note that you need a tool to see the HTTP headers that fly between Varnish and
+the backend. On the Varnish server, the easiest way to do this is to use
+`varnishlog` and `varnishtop` but sometimes a client-side tool makes
+sense. Here are the ones we commonly use.
 
 Tool: varnishtop
 ~~~~~~~~~~~~~~~~
 
 You can use varnishtop to identify what URLs are hitting the backend
 the most. ``varnishtop -i txurl`` is an essential command, showing you
-the top txurl requests Varnish is sending towards the backend. You can
-see some other examples of varnishtop usage in
+the top `txurl` requests Varnish is sending to the backend. You can
+see some other examples of `varnishtop` usage in
 :ref:`users-guide-statistics`.
 
 
 Tool: varnishlog
 ~~~~~~~~~~~~~~~~
 
-When you have identified the an URL which is frequently sent to the
-backend you can use varnishlog to have a look at the request.
+When you have identified an URL which is frequently sent to the
+backend you can use `varnishlog` to have a look at the request.
 ``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
-coming from the client (-c) matching /foo/bar.
+coming from the client ('-c') matching `/foo/bar`.
 
-For more information on how varnishlog works please see
+For more information on how `varnishlog` works please see
 :ref:`users-guide-logging` or man :ref:`ref-varnishlog`.
 
 For extended diagnostics headers, see
@@ -47,9 +47,9 @@ http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
 Tool: lwp-request
 ~~~~~~~~~~~~~~~~~
 
-lwp-request is part of The World-Wide Web library for Perl. It's a
-couple of really basic programs that can execute an HTTP request and
-give you the result. I mostly use two programs, GET and HEAD.
+`lwp-request` is tool that is a part of The World-Wide Web library for Perl. It's a
+couple of really basic programs that can execute a HTTP request and
+show you the result. We mostly use the two programs, ``GET`` and ``HEAD``.
 
 vg.no was the first site to use Varnish and the people running Varnish
 there are quite clueful. So it's interesting to look at their HTTP
@@ -72,26 +72,28 @@ Headers. Let's send a GET request for their home page::
   X-VG-WebCache: joanie
   X-VG-WebServer: leon
 
-OK. Let me explain what it does. GET usually sends off HTTP 0.9
-requests, which lack the Host header. So I add a Host header with the
--H option. -U print request headers, -s prints response status, -e
-prints response headers and -d discards the actual content. We don't
+OK. Lets look at what ``GET`` does. ``GET`` usually sends off HTTP 0.9
+requests, which lack the 'Host' header. So we add a 'Host' header with the
+'-H' option. '-U' print request headers, '-s' prints response status, '-e'
+prints response headers and '-d' discards the actual content. We don't
 really care about the content, only the headers.
 
 As you can see, VG adds quite a bit of information in their
-headers. Some of the headers, like the X-Rick-Would-Never are specific
+headers. Some of the headers, like the 'X-Rick-Would-Never' are specific
 to vg.no and their somewhat odd sense of humour. Others, like the
-X-VG-Webcache are for debugging purposes. 
+'X-VG-Webcache' are for debugging purposes. 
 
 So, to check whether a site sets cookies for a specific URL, just do::
 
   GET -Used http://example.com/ |grep ^Set-Cookie
 
+.. XXX:Missing explanation and sample for HEAD here. benc
+
 Tool: Live HTTP Headers
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-There is also a plugin for Firefox. *Live HTTP Headers* can show you
-what headers are being sent and recieved. Live HTTP Headers can be
+There is also a plugin for Firefox called `Live HTTP Headers`. This plugin can show you
+what headers are being sent and recieved. `Live HTTP Headers` can be
 found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
 googling "Live HTTP Headers".
 
@@ -102,14 +104,14 @@ The role of HTTP Headers
 Along with each HTTP request and response comes a bunch of headers
 carrying metadata. Varnish will look at these headers to determine if
 it is appropriate to cache the contents and how long Varnish can keep
-the content.
+the content cached.
 
-Please note that when considering these headers Varnish actually
+Please note that when Varnish considers these headers Varnish actually
 considers itself *part of* the actual webserver. The rationale being
 that both are under your control. 
 
 The term *surrogate origin cache* is not really well defined by the
-IETF so RFC 2616 so the various ways Varnish works might differ from
+IETF or RFC 2616 so the various ways Varnish works might differ from
 your expectations.
 
 Let's take a look at the important headers you should be aware of:
@@ -119,8 +121,8 @@ Let's take a look at the important headers you should be aware of:
 Cookies
 -------
 
-Varnish will, in the default configuration, not cache a object coming
-from the backend with a Set-Cookie header present. Also, if the client
+Varnish will, in the default configuration, not cache an object coming
+from the backend with a 'Set-Cookie' header present. Also, if the client
 sends a Cookie header, Varnish will bypass the cache and go directly to
 the backend.
 
@@ -132,10 +134,10 @@ interest to the server.
 Cookies from the client
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-For a lot of web application it makes sense to completely disregard the
+For a lot of web applications it makes sense to completely disregard the
 cookies unless you are accessing a special part of the web site. This
-VCL snippet in vcl_recv will disregard cookies unless you are
-accessing /admin/::
+VCL snippet in `vcl_recv` will disregard cookies unless you are
+accessing `/admin/`::
 
   if ( !( req.url ~ ^/admin/) ) {
     unset req.http.Cookie;
@@ -146,15 +148,15 @@ like removing one out of several cookies, things get
 difficult. Unfortunately Varnish doesn't have good tools for
 manipulating the Cookies. We have to use regular expressions to do the
 work. If you are familiar with regular expressions you'll understand
-whats going on. If you don't I suggest you either pick up a book on
-the subject, read through the *pcrepattern* man page or read through
+whats going on. If you aren't we recommend that you either pick up a book on
+the subject, read through the *pcrepattern* man page, or read through
 one of many online guides.
 
-Let me show you what Varnish Software uses. We use some cookies for
+Lets use the Varnish Software (VS) web as an example here. Very simplified the setup VS uses can be described as a Drupal-based backend with a Varnish cache infront. VS uses some cookies for
 Google Analytics tracking and similar tools. The cookies are all set
 and used by Javascript. Varnish and Drupal doesn't need to see those
 cookies and since Varnish will cease caching of pages when the client
-sends cookies we will discard these unnecessary cookies in VCL. 
+sends cookies Varnish will discard these unnecessary cookies in VCL. 
 
 In the following VCL we discard all cookies that start with a
 underscore::
@@ -164,8 +166,8 @@ underscore::
   // Remove a ";" prefix, if present.
   set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
 
-Let me show you an example where we remove everything except the
-cookies named COOKIE1 and COOKIE2 and you can marvel at it::
+Lets look at an example where we remove everything except the
+cookies named "COOKIE1" and "COOKIE2" and you can marvel at the "beauty" of it::
 
   sub vcl_recv {
     if (req.http.Cookie) {
@@ -181,10 +183,12 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it::
     }
   }
 
-A somewhat simpler example that can accomplish almost the same can be
-found below. Instead of filtering out the other cookies it picks out
-the one cookie that is needed, copies it to another header and then
-copies it back, deleting the original cookie header.::
+A somewhat simpler example that can accomplish almost the same functionality can be
+found below. Instead of filtering out "other" cookies it instead picks out
+"the one" cookie that is needed, copies it to another header and then
+copies it back to the request, deleting the original cookie header.
+.. XXX:Verify correctness of request above! benc
+::
 
   sub vcl_recv {
          # save the original cookie header so we can mangle it
@@ -200,30 +204,31 @@ copies it back, deleting the original cookie header.::
 There are other scary examples of what can be done in VCL in the
 Varnish Cache Wiki.
 
+.. XXX:Missing link here.
+
 
 Cookies coming from the backend
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If your backend server sets a cookie using the Set-Cookie header
-Varnish will not cache the page in the default configuration.  A
-hit-for-pass object (see :ref:`user-guide-vcl_actions`) is created.
+If your backend server sets a cookie using the 'Set-Cookie' header
+Varnish will not cache the page when using the default configuration. A
+`hit-for-pass` object (see :ref:`user-guide-vcl_actions`) is created.
 So, if the backend server acts silly and sets unwanted cookies just unset
-the Set-Cookie header and all should be fine. 
-
+the 'Set-Cookie' header and all should be fine. 
 
 
 Cache-Control
 ~~~~~~~~~~~~~
 
-The Cache-Control instructs caches how to handle the content. Varnish
+The 'Cache-Control' header instructs caches how to handle the content. Varnish
 cares about the *max-age* parameter and uses it to calculate the TTL
 for an object. 
 
-"Cache-Control: nocache" is ignored but if you need this you can
+``Cache-Control: nocache`` is ignored but if you need this you can
 easily add support for it.
 
-So make sure you issue a Cache-Control header with a max-age
-header. You can have a look at what Varnish Software's drupal server
+So make sure you issue a 'Cache-Control' header with a max-age
+header. You can have a look at what Varnish Software's Drupal server
 issues::
 
   $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control
@@ -232,18 +237,18 @@ issues::
 Age
 ~~~
 
-Varnish adds an Age header to indicate how long the object has been
-kept inside Varnish. You can grep out Age from varnishlog like this::
+Varnish adds an 'Age' header to indicate how long the object has been
+kept inside Varnish. You can grep out 'Age' from `varnishlog` like this::
 
   varnishlog -i TxHeader -I ^Age
 
 Pragma
 ~~~~~~
 
-An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
+An HTTP 1.0 server might send the header ``Pragma: nocache``. Varnish ignores this
 header. You could easily add support for this header in VCL.
 
-In vcl_backend_response::
+In `vcl_backend_response`::
 
   if (beresp.http.Pragma ~ "nocache") {
         set beresp.uncacheable = true;
@@ -253,7 +258,7 @@ In vcl_backend_response::
 Authorization
 ~~~~~~~~~~~~~
 
-If Varnish sees an Authorization header it will pass the request. If
+If Varnish sees an 'Authorization' header it will pass the request. If
 this is not what you want you can unset the header.
 
 Overriding the time-to-live (ttl)
@@ -264,7 +269,7 @@ setup, be easier to override the ttl in Varnish than to fix your
 somewhat cumbersome backend. 
 
 You need VCL to identify the objects you want and then you set the
-beresp.ttl to whatever you want::
+'beresp.ttl' to whatever you want::
 
   sub vcl_backend_response {
       if (req.url ~ "^/legacy_broken_cms/") {
@@ -272,13 +277,13 @@ beresp.ttl to whatever you want::
       }
   }
 
-The example will set the TTL to 5 days for the old legacy stuff on
+This example will set the ttl to 5 days for the old legacy stuff on
 your site.
 
 Forcing caching for certain requests and certain responses
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Since you still have this cumbersome backend that isn't very friendly
+Since you still might have this cumbersome backend that isn't very friendly
 to work with you might want to override more stuff in Varnish. We
 recommend that you rely as much as you can on the default caching
 rules. It is perfectly easy to force Varnish to lookup an object in
@@ -291,7 +296,8 @@ Normalizing your namespace
 Some sites are accessed via lots of
 hostnames. http://www.varnish-software.com/,
 http://varnish-software.com/ and http://varnishsoftware.com/ all point
-at the same site. Since Varnish doesn't know they are different,
+at the same site. Since Varnish doesn't know they are the same,
+.. XXX: heavy meaning change above. benc
 Varnish will cache different versions of every page for every
 hostname. You can mitigate this in your web server configuration by
 setting up redirects or by using the following VCL::
@@ -310,33 +316,33 @@ HTTP Vary
 misunderstood HTTP header.*
 
 A lot of the response headers tell the client something about the HTTP
-object being delivered. Clients can request different variants a an
+object being delivered. Clients can request different variants of a
 HTTP object, based on their preference. Their preferences might cover
 stuff like encoding or language. When a client prefers UK English this
-is indicated through "Accept-Language: en-uk". Caches need to keep
+is indicated through ``Accept-Language: en-uk``. Caches need to keep
 these different variants apart and this is done through the HTTP
-response header "Vary".
+response header 'Vary'.
 
-When a backend server issues a "Vary: Accept-Language" it tells
+When a backend server issues a ``Vary: Accept-Language`` it tells
 Varnish that its needs to cache a separate version for every different
 Accept-Language that is coming from the clients.
 
 If two clients say they accept the languages "en-us, en-uk" and "da,
 de" respectively, Varnish will cache and serve two different versions
 of the page if the backend indicated that Varnish needs to vary on the
-Accept-Language header.
+'Accept-Language' header.
 
-Please note that the headers that Vary refer to need to match
+Please note that the headers that 'Vary' refer to need to match
 *exactly* for there to be a match. So Varnish will keep two copies of
 a page if one of them was created for "en-us, en-uk" and the other for
-"en-us,en-uk". Just the lack of space will force Varnish to cache
+"en-us,en-uk". Just the lack of a whitespace will force Varnish to cache
 another version.
 
-To achieve a high hitrate whilst using Vary is there therefor crucial
+To achieve a high hitrate whilst using Vary is there therefore crucial
 to normalize the headers the backends varies on. Remember, just a
-difference in case can force different cache entries.
+difference in casing can force different cache entries.
 
-The following VCL code will normalize the Accept-Language headers, to
+The following VCL code will normalize the 'Accept-Language' headers, to
 one of either "en","de" or "fr"::
 
     if (req.http.Accept-Language) {
@@ -353,28 +359,28 @@ one of either "en","de" or "fr"::
         }
     }
 
-The code sets the Accept-Encoding header from the client to either
+The code sets the 'Accept-Encoding' header from the client to either
 gzip, deflate with a preference for gzip.
 
 Vary parse errors
 ~~~~~~~~~~~~~~~~~
 
-Varnish will return a 503 internal server error page when it fails to
-parse the Vary server header, or if any of the client headers listed
-in the Vary header exceeds the limit of 65k characters. An SLT_Error
+Varnish will return a "503 internal server error" page when it fails to
+parse the 'Vary' header, or if any of the client headers listed
+in the Vary header exceeds the limit of 65k characters. An 'SLT_Error'
 log entry is added in these cases.
 
 Pitfall - Vary: User-Agent
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Some applications or application servers send *Vary: User-Agent* along
+Some applications or application servers send ``Vary: User-Agent`` along
 with their content. This instructs Varnish to cache a separate copy
-for every variation of User-Agent there is. There are plenty. Even a
+for every variation of 'User-Agent' there is and there are plenty. Even a
 single patchlevel of the same browser will generate at least 10
-different User-Agent headers based just on what operating system they
+different 'User-Agent' headers based just on what operating system they
 are running. 
 
-So if you *really* need to Vary based on User-Agent be sure to
+So if you *really* need to vary based on 'User-Agent' be sure to
 normalize the header or your hit rate will suffer badly. Use the above
 code as a template.
 
diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index 8b31b32..f9c5587 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -3,7 +3,7 @@
 The Big Varnish Picture
 =======================
 
-In this section we will cover the questions:
+In this section we will cover answers to the questions:
 - What is in this package called "Varnish"?
 - what are all the different bits and pieces named? 
 - Will you need a hex-wrench for assembly?
diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
index 547a987..4bfdca0 100644
--- a/doc/sphinx/users-guide/operation-logging.rst
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -3,8 +3,8 @@
 Logging in Varnish
 ------------------
 
-One of the really nice features in Varnish is how logging
-works. Instead of logging to normal log file Varnish logs to a shared
+One of the really nice features in Varnish is the way logging
+works. Instead of logging to a normal log file Varnish logs to a shared
 memory segment, called the VSL - the Varnish Shared Log. When the end
 of the segment is reached we start over, overwriting old data. 
 
@@ -13,14 +13,14 @@ require disk space. Besides it gives you much, much more information
 when you need it.
 
 The flip side is that if you forget to have a program actually write the
-logs to disk they will disappear.
+logs to disk they will be overwritten.
 
-varnishlog is one of the programs you can use to look at what Varnish
-is logging. Varnishlog gives you the raw logs, everything that is
-written to the logs. There are other clients as well, we'll show you
+`varnishlog` is one of the programs you can use to look at what Varnish
+is logging. `varnishlog` gives you the raw logs, everything that is
+written to the logs. There are other clients that can access the logs as well, we'll show you
 these later.
 
-In the terminal window you started Varnish now type *varnishlog* and
+In the terminal window you started Varnish now type ``varnishlog`` and
 press enter.
 
 You'll see lines like these scrolling slowly by.::
@@ -32,7 +32,10 @@ These is the Varnish master process checking up on the caching process
 to see that everything is OK.
 
 Now go to the browser and reload the page displaying your web
-app. You'll see lines like these.::
+app. 
+.. XXX:Doesn't this require a setup of a running varnishd and a web application being cached? benc
+
+You'll see lines like these.::
 
    11 SessionOpen  c 127.0.0.1 58912 0.0.0.0:8080
    11 ReqStart     c 127.0.0.1 58912 595005213
@@ -42,29 +45,32 @@ app. You'll see lines like these.::
    11 RxHeader     c Host: localhost:8080
    11 RxHeader     c Connection: keep-alive
 
+
 The first column is an arbitrary number, it identifies the
 session. Lines with the same number are coming from the same session
 and are being handled by the same thread. The second column is the
 *tag* of the log message. All log entries are tagged with a tag
 indicating what sort of activity is being logged. Tags starting with
-Rx indicate Varnish is recieving data and Tx indicates sending data.
+'Rx' indicate Varnish is recieving data and 'Tx' indicates sending data.
 
 The third column tell us whether this is is data coming or going to
-the client (c) or to/from the backend (b). The forth column is the
+the client ('c') or to/from the backend ('b'). The forth column is the
 data being logged.
 
-Now, you can filter quite a bit with varnishlog. The basic option you
+Now, you can filter quite a bit with `varnishlog`. The basic options we think you
 want to know are:
 
--b
+'-b'
  Only show log lines from traffic going between Varnish and the backend
  servers. This will be useful when we want to optimize cache hit rates.
 
--c
- Same as -b but for client side traffic.
+'-c'
+ Same as '-b' but for client side traffic.
 
--m tag:regex
+'-m tag:regex'
  Only list transactions where the tag matches a regular expression. If
  it matches you will get the whole transaction.
 
+.. XXX:Maybe a couple of sample commands here? benc
+
 For more information on this topic please see :ref:`ref-varnishlog`.
diff --git a/doc/sphinx/users-guide/operation-statistics.rst b/doc/sphinx/users-guide/operation-statistics.rst
index c891342..788e600 100644
--- a/doc/sphinx/users-guide/operation-statistics.rst
+++ b/doc/sphinx/users-guide/operation-statistics.rst
@@ -4,13 +4,14 @@
 Statistics
 ----------
 
-Now that your Varnish is up and running let's have a look at how it is
-doing. There are several tools that can help.
+Varnish comes with a couple of nifty and very useful statistics generating tools that generates statistics in real time by constantly updating and presenting a specific dataset by aggregating and analyzing logdata from the shared memory logs.
+
+.. XXX:Heavy rewrite above. benc
 
 varnishtop
 ~~~~~~~~~~
 
-The varnishtop utility reads the shared memory logs and presents a
+The `varnishtop` utility reads the shared memory logs and presents a
 continuously updated list of the most commonly occurring log entries.
 
 With suitable filtering using the -I, -i, -X and -x options, it can be
@@ -28,9 +29,11 @@ For more information please see :ref:`ref-varnishtop`.
 varnishhist
 ~~~~~~~~~~~
 
-The varnishhist utility reads varnishd(1) shared memory logs and
+The `varnishhist` utility reads `varnishd(1)` shared memory logs and
 presents a continuously updated histogram showing the distribution of
-the last N requests by their processing.  The value of N and the
+the last N requests by their processing.  
+.. XXX:1? benc
+The value of N and the
 vertical scale are displayed in the top left corner.  The horizontal
 scale is logarithmic.  Hits are marked with a pipe character ("|"),
 and misses are marked with a hash character ("#").
@@ -43,10 +46,10 @@ varnishstat
 
 Varnish has lots of counters. We count misses, hits, information about
 the storage, threads created, deleted objects. Just about
-everything. varnishstat will dump these counters. This is useful when
+everything. `varnishstat` will dump these counters. This is useful when
 tuning Varnish.
 
-There are programs that can poll varnishstat regularly and make nice
+There are programs that can poll `varnishstat` regularly and make nice
 graphs of these counters. One such program is Munin. Munin can be
 found at http://munin-monitoring.org/ . There is a plugin for munin in
 the Varnish source code.
diff --git a/doc/sphinx/users-guide/params.rst b/doc/sphinx/users-guide/params.rst
index a283d51..df4bfdf 100644
--- a/doc/sphinx/users-guide/params.rst
+++ b/doc/sphinx/users-guide/params.rst
@@ -3,21 +3,21 @@
 Parameters
 ----------
 
-Varnish Cache has a set of parameters that affect its behaviour and
+Varnish Cache comes with a set of parameters that affects behaviour and
 performance. Most of these parameters can be set on the Varnish
-command line (through varnishadm) using the param.set keyword.
+command line (through `varnishadm`) using the ``param.set`` keyword.
 
-Some parameters can, for security purposes be read only using the "-r"
-command line switch to varnishd.
+Some parameters can, for security purposes be read only using the '-r'
+command line switch to `varnishd`.
 
-I don't recommend tweaking the parameters unless you're sure of what
+We don't recommend that you tweak parameters unless you're sure of what
 you're doing. We've worked hard to make the defaults sane and Varnish
 should be able to handle most workloads with the default settings.
 
 For a complete listing of all the parameters and a short descriptions
 type ``param.show`` in the CLI. To inspect a certain parameter and get
 a somewhat longer description on what it does and what the default is
-type param.show and the name of the parameter, like this::
+type ``param.show`` and the name of the parameter, like this::
 
   varnish> param.show shortlived
   200        
diff --git a/doc/sphinx/users-guide/performance.rst b/doc/sphinx/users-guide/performance.rst
index baf9c75..cd00ff5 100644
--- a/doc/sphinx/users-guide/performance.rst
+++ b/doc/sphinx/users-guide/performance.rst
@@ -3,19 +3,18 @@
 Varnish and Website Performance
 ===============================
 
-This section is about tuning the performance of your Varnish server,
-and about tuning the performance of your website using Varnish.
+This section focuses on how to tune the performance of your Varnish server,
+and how to tune the performance of your website using Varnish.
 
-The section is split in three sections. One deals with the various tools and
-functions of Varnish that you should be aware of and the other focuses
+The section is split in three subsections. The first subsection deals with the various tools and
+functions of Varnish that you should be aware of. The next subsection focuses
 on the how to purge content out of your cache. Purging of content is
 essential in a performance context because it allows you to extend the
 *time-to-live* (TTL) of your cached objects. Having a long TTL allows
-Varnish to keep the content in cache longer, meaning Varnish will make
-send fewer requests to your relativly slow backend.
+Varnish to keep the content in cache longer, meaning Varnish will make fewer requests to your relativly slower backend.
 
-The final section deals with compression of web content. Varnish can
-gzip content when fetching it from the backend and then deliver
+The final subsection deals with compression of web content. Varnish can
+gzip content when fetching it from the backend and then deliver it
 compressed. This will reduce the time it takes to download the content
 thereby increasing the performance of your website.
 
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 058bd7d..2be32e3 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -6,24 +6,24 @@ Purging and banning
 
 One of the most effective ways of increasing your hit ratio is to
 increase the time-to-live (ttl) of your objects. But, as you're aware
-of, in this twitterific day of age serving content that is outdated is
+of, in this twitterific day of age, serving content that is outdated is
 bad for business.
 
 The solution is to notify Varnish when there is fresh content
 available. This can be done through three mechanisms. HTTP purging,
-banning and forced cache misses. First, let me explain the HTTP purges.
+banning and forced cache misses. First, lets look at HTTP purging.
 
 
-HTTP Purges
-~~~~~~~~~~~
+HTTP Purging
+~~~~~~~~~~~~
 
 A *purge* is what happens when you pick out an object from the cache
 and discard it along with its variants. Usually a purge is invoked
-through HTTP with the method PURGE.
+through HTTP with the method `PURGE`.
 
-An HTTP purge is similar to an HTTP GET request, except that the
-*method* is PURGE. Actually you can call the method whatever you'd
-like, but most people refer to this as purging. Squid supports the
+An HTTP purge is similar to a HTTP GET request, except that the
+*method* is `PURGE`. Actually you can call the method whatever you'd
+like, but most people refer to this as purging. Squid, for example, supports the
 same mechanism. In order to support purging in Varnish you need the
 following VCL in place::
 
@@ -64,7 +64,7 @@ Bans
 ~~~~
 
 There is another way to invalidate content: Bans. You can think of
-bans as a sort of a filter on objects already in the cache. You *ban*
+bans as a sort of a filter on objects already in the cache. You ``ban``
 certain content from being served from your cache. You can ban
 content based on any metadata we have.
 A ban will only work on objects already in the cache, it does not
@@ -81,16 +81,18 @@ Quite powerful, really.
 Bans are checked when we hit an object in the cache, but before we
 deliver it. *An object is only checked against newer bans*.
 
-Bans that only match against obj.* are also processed by a background
-worker threads called the *ban lurker*. The ban lurker will walk the
+Bans that only match against `obj.*` are also processed by a background
+worker threads called the `ban lurker`. The `ban lurker` will walk the
 heap and try to match objects and will evict the matching objects. How
-aggressive the ban lurker is can be controlled by the parameter
-ban_lurker_sleep. The ban lurker can be disabled by setting
-ban_lurker_sleep to 0.
+aggressive the `ban lurker` is can be controlled by the parameter
+'ban_lurker_sleep'. The `ban lurker` can be disabled by setting
+'ban_lurker_sleep' to 0.
+
+.. XXX: sample here? benc
 
 Bans that are older than the oldest objects in the cache are discarded
-without evaluation.  If you have a lot of objects with long TTL, that
-are seldom accessed you might accumulate a lot of bans. This might
+without evaluation. If you have a lot of objects with long TTL, that
+are seldom accessed, you might accumulate a lot of bans. This might
 impact CPU usage and thereby performance.
 
 You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
@@ -110,14 +112,14 @@ You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
 	  }
   }
 
-This VCL sniplet enables Varnish to handle an HTTP BAN method, adding a
+This VCL stanza enables Varnish to handle a `HTTP BAN` method, adding a
 ban on the URL, including the host part.
 
-The ban lurker can help you keep the ban list at a manageable size, so
-we recommend that you avoid using req.* in your bans, as the request
-object is not available in the ban lurker thread.
+The `ban lurker` can help you keep the ban list at a manageable size, so
+we recommend that you avoid using `req.*` in your bans, as the request
+object is not available in the `ban lurker` thread.
 
-You can use the following template to write ban lurker friendly bans::
+You can use the following template to write `ban lurker` friendly bans::
 
   sub vcl_backend_response {
     set beresp.http.x-url = req.url;
@@ -136,7 +138,7 @@ You can use the following template to write ban lurker friendly bans::
     }
   }
 
-To inspect the current ban list, issue the ban.list command in CLI. This
+To inspect the current ban list, issue the ``ban.list`` command in the CLI. This
 will produce a status of all current bans::
 
   0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
@@ -146,15 +148,15 @@ The ban list contains the ID of the ban, the timestamp when the ban
 entered the ban list. A count of the objects that has reached this point
 in the ban list, optionally postfixed with a 'G' for "Gone", if the ban
 is no longer valid.  Finally, the ban expression is listed. The ban can
-be marked as Gone if it is a duplicate ban, but is still kept in the list
+be marked as "Gone" if it is a duplicate ban, but is still kept in the list
 for optimization purposes.
 
 Forcing a cache miss
 ~~~~~~~~~~~~~~~~~~~~
 
 The final way to invalidate an object is a method that allows you to
-refresh an object by forcing a hash miss for a single request. If you set
-req.hash_always_miss to true, Varnish will miss the current object in the
+refresh an object by forcing a `hash miss` for a single request. If you set
+'req.hash_always_miss' to true, Varnish will miss the current object in the
 cache, thus forcing a fetch from the backend. This can in turn add the
 freshly fetched object to the cache, thus overriding the current one. The
 old object will stay in the cache until ttl expires or it is evicted by
diff --git a/doc/sphinx/users-guide/report.rst b/doc/sphinx/users-guide/report.rst
index 5a444ab..13daf3e 100644
--- a/doc/sphinx/users-guide/report.rst
+++ b/doc/sphinx/users-guide/report.rst
@@ -3,7 +3,7 @@
 Reporting and statistics
 ========================
 
-This section is about how to find out what Varnish is doing, from
+This section covers how to find out what Varnish is doing, from
 the detailed per HTTP request blow-by-blow logrecords to the global
 summary statistics counters.
 
diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst
index b534971..3e75ac5 100644
--- a/doc/sphinx/users-guide/run_cli.rst
+++ b/doc/sphinx/users-guide/run_cli.rst
@@ -3,46 +3,46 @@
 CLI - bossing Varnish around
 ============================
 
-Once varnishd is started, you can control it using the command line
+Once `varnishd` is started, you can control it using the command line
 interface.
 
-The easiest way to do this, is using the varnishadm program on the
-same machine as varnishd is running::
+The easiest way to do this, is using `varnishadm` on the
+same machine as `varnishd` is running::
 
 	varnishadm help
 
-If you want to run varnishadm from a remote system, you can do it
+If you want to run `varnishadm` from a remote system, you can do it
 two ways.
 
-You can SSH into the varnishd computer and run varnishadm::
+You can SSH into the `varnishd` computer and run `varnishadm`::
 
 	ssh $http_front_end varnishadm help
 
-But you can also configure varnishd to accept remote CLI connections
-(using the -T and -S arguments)::
+But you can also configure `varnishd` to accept remote CLI connections
+(using the '-T' and '-S' arguments)::
 
 	varnishd -T :6082 -S /etc/varnish_secret
 
-And then on the remote system run varnishadm::
+And then on the remote system run `varnishadm`::
 
 	varnishadm -T $http_front_end -S /etc/copy_of_varnish_secret help
 
 but as you can see, SSH is much more convenient.
 
-If you run varnishadm without arguments, it will read CLI commands from
-stdin, if you give it arguments, it will treat those as the single
+If you run `varnishadm` without arguments, it will read CLI commands from
+`stdin`, if you give it arguments, it will treat those as the single
 CLI command to execute.
 
 The CLI always returns a status code to tell how it went:  '200'
 means OK, anything else means there were some kind of trouble.
 
-varnishadm will exit with status 1 and print the status code on
+`varnishadm` will exit with status 1 and print the status code on
 standard error if it is not 200.
 
 What can you do with the CLI
 ----------------------------
 
-The CLI gives you almost total control over varnishd:
+The CLI gives you almost total control over `varnishd` some of the more important tasks you can perform are:
 
 * load/use/discard VCL programs
 * ban (invalidate) cache content
@@ -64,7 +64,7 @@ To load new VCL program::
 
 	varnish> vcl.load some_name some_filename
 
-Loading will read the VCL program from the file, and compile it.  If
+Loading will read the VCL program from the file, and compile it. If
 the compilation fails, you will get an error messages::
 
 	.../mask is not numeric.
@@ -93,6 +93,8 @@ It is good idea to design an emergency-VCL before you need it,
 and always have it loaded, so you can switch to it with a single
 vcl.use command.
 
+.. XXX:Should above have a clearer admonition like a NOTE:? benc
+
 Ban cache content
 ^^^^^^^^^^^^^^^^^
 
@@ -103,7 +105,7 @@ But sometimes it is useful to be able to throw things out of cache
 without having an exact list of what to throw out.
 
 Imagine for instance that the company logo changed and now you need
-to get all versions of it out of the cache::
+Varnish to stop serving the old logo out of the cache::
 
 	varnish> ban req.url ~ "logo.*[.]png"
 
@@ -119,7 +121,7 @@ a HTTP request asks for it.
 Banning stuff is much cheaper than restarting Varnish to get rid
 of wronly cached content.
 
-.. In addition to handling such special occations, banning can be used
+.. In addition to handling such special occasions, banning can be used
 .. in many creative ways to keep the cache up to date, more about
 .. that in: (TODO: xref)
 
@@ -130,7 +132,7 @@ Change parameters
 Parameters can be set on the command line with the '-p' argument,
 but they can also be examined and changed on the fly from the CLI::
 
-	varnish> param.show perfer_ipv6
+	varnish> param.show prefer_ipv6
 	200
 	prefer_ipv6         off [bool]
                             Default is off
@@ -144,8 +146,12 @@ In general it is not a good idea to modify parameters unless you
 have a good reason, such as performance tuning or security configuration.
 
 Most parameters will take effect instantly, or with a natural delay
-of some duration, but a few of them requires you to restart the
-child process before they take effect.  This is always noted in the
+of some duration,
+
+.. XXX: Natural delay of some duration sounds vague. benc
+
+but a few of them requires you to restart the
+child process before they take effect. This is always noted in the
 description of the parameter.
 
 Starting and stopping the worker process
@@ -160,7 +166,7 @@ and::
 
 	varnish> start
 
-If you start varnishd with the '-d' (debugging) argument, you will
+If you start `varnishd` with the '-d' (debugging) argument, you will
 always need to start the child process explicitly.
 
 Should the child process die, the master process will automatically
diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index dfcfc6d..d7340a0 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -5,8 +5,8 @@ Security first
 
 If you are the only person involved in running Varnish, or if all
 the people involved are trusted to the same degree, you can skip
-this chapter:  We have protected Varnish as well as we can from
-anything which can come in through HTTP socket.
+this chapter. We have protected Varnish as well as we can from
+anything which can come in through a HTTP socket.
 
 If parts of your web infrastructure are outsourced or otherwise
 partitioned along administrative lines, you need to think about
@@ -15,19 +15,18 @@ security.
 Varnish provides four levels of authority, roughly related to
 how and where the command comes into Varnish:
 
-  * The command line arguments
+  * the command line arguments,
 
-  * The CLI interface
+  * the CLI interface,
 
-  * VCL programs
+  * VCL programs, and
 
-  * HTTP requests
+  * HTTP requests.
 
 Command line arguments
 ----------------------
 
-The top level security decisions is taken on and from the command
-line, in order to make them invulnerable to subsequent manipulation.
+The top level security decisions is decided and defined when starting Varnish in the form of command line arguments, we use this strategy in order to make them invulnerable to subsequent manipulation.
 
 The important decisions to make are:
 
@@ -44,8 +43,8 @@ CLI interface access
 
 The command line interface can be accessed three ways.
 
-Varnishd can be told til listen and offer CLI connections
-on a TCP socket.  You can bind the socket to pretty
+`Varnishd` can be told til listen and offer CLI connections
+on a TCP socket. You can bind the socket to pretty
 much anything the kernel will accept::
 
 	-T 127.0.0.1:631
@@ -53,15 +52,17 @@ much anything the kernel will accept::
 	-T 192.168.1.1:34
 	-T '[fe80::1]:8082'
 
-The default is '-T localhost:0' which will pick a random
-port number, which varnishadm(8) can learn in the shared
+The default is ``-T localhost:0`` which will pick a random
+port number, which `varnishadm(8)` can learn in the shared
 memory.
 
+.. XXX:Me no understand sentence above, (8)? and learn in the shared memory? Stored and retrieved by varnishadm from th e shared memory? benc 
+
 By using a "localhost" address, you restrict CLI access
 to the local machine.
 
 You can also bind the CLI port to an IP number reachable across
-the net, and let other computers connect directly.
+the net, and let other machines connect directly.
 
 This gives you no secrecy, ie, the CLI commands will
 go across the network as ASCII text with no encryption, but
@@ -72,42 +73,52 @@ Alternatively you can bind the CLI port to a 'localhost' address,
 and give remote users access via a secure connection to the local
 machine, using ssh/VPN or similar.
 
-If you use ssh you can restrict which commands each user can execute to
-just varnishadm, or even to wrapper scripts around varnishadm, which
+If you use `ssh` you can restrict which commands each user can execute to
+just `varnishadm`, or even to wrapper scripts around `varnishadm`, which
 only allow specific CLI commands.
 
-It is also possible to configure varnishd for "reverse mode", using
-the '-M' argument.  In that case varnishd will attempt to open a
+It is also possible to configure `varnishd` for "reverse mode", using
+the '-M' argument.  In that case `varnishd` will attempt to open a
 TCP connection to the specified address, and initiate a CLI connection
 to your central Varnish management facility.
 
+.. XXX:Maybe a sample command here with a brief explanation? benc
+
 The connection is also in this case without secrecy, but
 the remote end must still satisfy -S/PSK authentication.
 
+.. XXX:Without encryption instead of secrecy? benc
+
 Finally, if you run varnishd with the '-d' option, you get a CLI
 command on stdin/stdout, but since you started the process, it
 would be hard to prevent you getting CLI access, wouldn't it ?
 
+
 CLI interface authentication
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-By default the CLI interface is protected with a simple,  yet
+By default the CLI interface is protected with a simple, yet
 strong "Pre Shared Key" authentication method, which do not provide
 secrecy (ie: The CLI commands and responses are not encrypted).
 
+.. XXX:Encryption instead of secrecy? benc
+
 The way -S/PSK works is really simple:  During startup a file is
 created with a random content and the file is only accessible to
-the user who started varnishd (or the superuser).
+the user who started `varnishd` (or the superuser).
 
 To authenticate and use a CLI connection, you need to know the
 contents of that file, in order to answer the cryptographic
-challenge varnishd issues. (XXX: xref to algo in refman)
+challenge `varnishd` issues. 
+
 
-The varnishadm program knows all about this, it will just work,
+(XXX: xref to algo in refman)
+.. XXX:Dunno what this is? benc
+
+`varnishadm` uses all of this to restrict access, it will only function,
 provided it can read the secret file.
 
-If you want to allow other users on the local system or remote
-users, to be able to access CLI connections, you must create your
+If you want to allow other users, local or remote, to be able to access CLI connections, you must create your
 own secret file and make it possible for (only!) these users to
 read it.
 
@@ -115,18 +126,18 @@ A good way to create the secret file is::
 
 	dd if=/dev/random of=/etc/varnish_secret count=1
 
-When you start varnishd, you specify the filename with -S, and
-it goes without saying that the varnishd master process needs
+When you start `varnishd`, you specify the filename with '-S', and
+it goes without saying that the `varnishd` master process needs
 to be able to read the file too.
 
-You can change the contents of the secret file while varnishd
+You can change the contents of the secret file while `varnishd`
 runs, it is read every time a CLI connection is authenticated.
 
-On the local system, varnishadm can find the filename from
-shared memory, but on remote systems, you need to give it
+On the local system, `varnishadm` can retrieve the filename from
+shared memory, but on remote systems, you need to give `varnishadm`
 a copy of the secret file, with the -S argument.
 
-If you want to disable -S/PSK authentication, specify -S with
+If you want to disable -S/PSK authentication, specify '-S' with
 an empty argument to varnishd::
 
 	varnishd [...] -S "" [...]
@@ -135,17 +146,17 @@ Parameters
 ^^^^^^^^^^
 
 Parameters can be set from the command line, and made "read-only"
-(using -r) so they cannot subsequently be modified from the CLI
+(using '-r') so they cannot subsequently be modified from the CLI
 interface.
 
 Pretty much any parameter can be used to totally mess up your
-HTTP service, but a few can do more damage than that:
+HTTP service, but a few can do more damage than others:
 
 :ref:`ref_param_user` and :ref:`ref_param_group`
 	Access to local system via VCL
 
 :ref:`ref_param_listen_address`
-	Trojan other TCP sockets, like ssh
+	Trojan other TCP sockets, like `ssh`
 
 :ref:`ref_param_cc_command`
 	Execute arbitrary programs
@@ -156,10 +167,12 @@ HTTP service, but a few can do more damage than that:
 Furthermore you may want to look at and lock down:
 
 :ref:`ref_param_syslog_cli_traffic`
-	Log all CLI commands to syslog(8), so you know what goes on.
+	Log all CLI commands to `syslog(8)`, so you know what goes on.
+.. XXX: syslog(8)? benc
+
 
 :ref:`ref_param_vcc_unsafe_path`
-	Retrict VCL/VMODS to :ref:`ref_param_vcl_dir` and :ref:`ref_param_vmod_dir`
+	Restrict VCL/VMODS to :ref:`ref_param_vcl_dir` and :ref:`ref_param_vmod_dir`
 
 :ref:`ref_param_vmod_dir` 
         The directory where Varnish will will look
@@ -178,11 +191,16 @@ certain parameters, but that will only protect the local filesystem,
 and operating system, it will not protect your HTTP service.
 
 We do not currently have a way to restrict specific CLI commands
-to specific CLI connections.   One way to get such an effect is to
-"wrap" all CLI access in pre-approved scripts which use varnishadm(1)
+to specific CLI connections. One way to get such an effect is to
+"wrap" all CLI access in pre-approved scripts which use `varnishadm(1)`
+
+.. XXX:what does the 1 stand for? benc
+
 to submit the sanitized CLI commands, and restrict a remote user
 to only those scripts, for instance using sshd(8)'s configuration.
 
+.. XXX:what does the 8 stand for? benc
+
 VCL programs
 ------------
 
@@ -190,35 +208,35 @@ There are two "dangerous" mechanisms available in VCL code:  VMODs
 and inline-C.
 
 Both of these mechanisms allow execution of arbitrary code and will
-therefore allow a person to get access on the computer, with the
+thus allow a person to get access to the machine, with the
 privileges of the child process.
 
-If varnishd is started as root/superuser, we sandbox the child
+If `varnishd` is started as root/superuser, we sandbox the child
 process, using whatever facilities are available on the operating
-system, but if varnishd is not started as root/superuser, this is
-not possible.  No, don't ask me why you have to be superuser to
+system, but if `varnishd` is not started as root/superuser, this is
+not possible. No, don't ask me why you have to be superuser to
 lower the privilege of a child process...
 
-Inline-C is disabled by default starting with Varnish 4, so unless
+Inline-C is disabled by default starting with Varnish version 4, so unless
 you enable it, you don't have to worry about it.
 
-The parameters mentioned above can restrict VMOD, so they can only
-be imported from a designated directory, restricting VCL wranglers
+The parameters mentioned above can restrict the loading of VMODs to only 
+be loaded from a designated directory, restricting VCL wranglers
 to a pre-approved subset of VMODs.
 
-If you do that, we believe that your local system cannot be compromised
+If you do that, we are confident that your local system cannot be compromised
 from VCL code.
 
 HTTP requests
 -------------
 
 We have gone to great lengths to make Varnish resistant to anything
-coming in throught he socket where HTTP requests are received, and
+coming in throught the socket where HTTP requests are received, and
 you should, generally speaking, not need to protect it any further.
 
 The caveat is that since VCL is a programming language which lets you
-decide exactly what to do about HTTP requests, you can also decide
-to do exactly stupid things to them, including opening youself up
+decide exactly what to do with HTTP requests, you can also decide
+to do stupid and potentially dangerous things with them, including opening youself up
 to various kinds of attacks and subversive activities.
 
 If you have "administrative" HTTP requests, for instance PURGE
diff --git a/doc/sphinx/users-guide/running.rst b/doc/sphinx/users-guide/running.rst
index 2b49147..5cfd539 100644
--- a/doc/sphinx/users-guide/running.rst
+++ b/doc/sphinx/users-guide/running.rst
@@ -3,8 +3,8 @@
 Starting and running Varnish
 ============================
 
-This section is about starting, running, and stopping Varnish, about
-command line flags and options, communicating with the running
+This section covers starting, running, and stopping Varnish,
+command line flags and options, and communicating with the running
 Varnish processes, configuring storage and sockets and, and about
 securing and protecting Varnish against attacks.
 
diff --git a/doc/sphinx/users-guide/sizing-your-cache.rst b/doc/sphinx/users-guide/sizing-your-cache.rst
index 8f2dfba..497ce0e 100644
--- a/doc/sphinx/users-guide/sizing-your-cache.rst
+++ b/doc/sphinx/users-guide/sizing-your-cache.rst
@@ -2,7 +2,7 @@
 Sizing your cache
 -----------------
 
-Picking how much memory you should give Varnish can be a tricky
+Deciding on cache size can be a tricky
 task. A few things to consider:
 
  * How big is your *hot* data set. For a portal or news site that
@@ -12,14 +12,16 @@ task. A few things to consider:
    to only cache images a little while or not to cache them at all if
    they are cheap to serve from the backend and you have a limited
    amount of memory.
- * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or
+ * Watch the `n_lru_nuked` counter with :ref:`reference-varnishstat` or
    some other tool. If you have a lot of LRU activity then your cache
    is evicting objects due to space constraints and you should
    consider increasing the size of the cache.
 
 Be aware that every object that is stored also carries overhead that
-is kept outside the actually storage area. So, even if you specify -s
-malloc,16G Varnish might actually use **double** that. Varnish has a
+is kept outside the actually storage area. So, even if you specify '-s
+malloc,16G' Varnish might actually use **double** that. Varnish has a
 overhead of about 1k per object. So, if you have lots of small objects
 in your cache the overhead might be significant.
 
+.. XXX:This seems to contradict the last paragraph in "storage-backends". benc
+
diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index b12dba0..6664d8d 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -8,9 +8,9 @@ Intro
 ~~~~~
 
 Varnish has pluggable storage backends. It can store data in various
-backends which have different performance characteristics. The default
+backends which can have different performance characteristics. The default
 configuration is to use the malloc backend with a limited size. For a
-serious Varnish deployment you probably need to adjust the storage
+serious Varnish deployment you probably would want to adjust the storage
 settings.
 
 malloc
@@ -21,11 +21,13 @@ syntax: malloc[,size]
 Malloc is a memory based backend. Each object will be allocated from
 memory. If your system runs low on memory swap will be used.
 
-Be aware that the size limitation only limits the actual storage and that
-approximately 1k of memory per object will be used for various internal
-structures.
+Be aware that the size limitation only limits the actual storage and that the
+approximately 1k of memory per object, used for various internal
+structures, is included in the actual storage as well.
 
-The size parameter specifies the maximum amount of memory varnishd
+.. XXX:This seems to contradict the last paragraph in "sizing-your-cache". benc
+
+The size parameter specifies the maximum amount of memory `varnishd`
 will allocate.  The size is assumed to be in bytes, unless followed by
 one of the following suffixes:
 
@@ -39,9 +41,9 @@ one of the following suffixes:
 
 The default size is unlimited.
 
-malloc's performance is bound by memory speed so it is very fast. If
-the dataset is bigger than what can fit in memory performance will
-depend on the operating system and how well it does paging.
+malloc's performance is bound to memory speed so it is very fast. If
+the dataset is bigger than available memory performance will
+depend on the operating systems ability to page effectively.
 
 file
 ~~~~
@@ -49,13 +51,13 @@ file
 syntax: file[,path[,size[,granularity]]]
 
 The file backend stores objects in memory backed by an unlinked file on disk
-with mmap.
+with `mmap`.
 
-The path parameter specifies either the path to the backing file or
-the path to a directory in which varnishd will create the backing
-file.  The default is /tmp.
+The 'path' parameter specifies either the path to the backing file or
+the path to a directory in which `varnishd` will create the backing
+file. The default is `/tmp`.
 
-The size parameter specifies the size of the backing file.  The size
+The size parameter specifies the size of the backing file. The size
 is assumed to be in bytes, unless followed by one of the following
 suffixes:
 
@@ -75,20 +77,22 @@ The default size is to use 50% of the space available on the device.
 If the backing file already exists, it will be truncated or expanded
 to the specified size.
 
-Note that if varnishd has to create or expand the file, it will not
+Note that if `varnishd` has to create or expand the file, it will not
 pre-allocate the added space, leading to fragmentation, which may
 adversely impact performance on rotating hard drives.  Pre-creating
-the storage file using dd(1) will reduce fragmentation to a minimum.
+the storage file using `dd(1)` will reduce fragmentation to a minimum.
+
+.. XXX:1? benc
 
-The granularity parameter specifies the granularity of
-allocation.  All allocations are rounded up to this size.  The
-is assumed to be in bytes, unless followed by one of the
+The 'granularity' parameter specifies the granularity of
+allocation. All allocations are rounded up to this size. The granularity is
+is assumed to be expressed in bytes, unless followed by one of the
 suffixes described for size except for %.
 
-The default granularity is the VM page size.  The size should be reduced if you
+The default granularity is the VM page size. The size should be reduced if you
 have many small objects.
 
-File performance is typically limited by the write speed of the
+File performance is typically limited to the write speed of the
 device, and depending on use, the seek time.
 
 persistent (experimental)
@@ -100,11 +104,11 @@ Persistent storage. Varnish will store objects in a file in a manner
 that will secure the survival of *most* of the objects in the event of
 a planned or unplanned shutdown of Varnish.
 
-The path parameter specifies the path to the backing file. If
+The 'path' parameter specifies the path to the backing file. If
 the file doesn't exist Varnish will create it.
 
-The size parameter specifies the size of the backing file.  The
-size is assumed to be in bytes, unless followed by one of the
+The 'size' parameter specifies the size of the backing file. The
+size is expressed in bytes, unless followed by one of the
 following suffixes:
 
       K, k    The size is expressed in kibibytes.
@@ -122,9 +126,9 @@ starts after a shutdown it will discard the content of any silo that
 isn't sealed.
 
 Note that taking persistent silos offline and at the same time using
-bans can cause problems. This because bans added while the silo was
-offline will not be applied to the silo when it reenters the cache,
-and can make previously banned objects reappear.
+bans can cause problems. This is due to the fact that bans added while the silo was
+offline will not be applied to the silo when it reenters the cache. Consequently enabling
+previously banned objects to reappear.
 
 Transient Storage
 -----------------
@@ -133,6 +137,10 @@ If you name any of your storage backend "Transient" it will be
 used for transient (short lived) objects. By default Varnish
 would use an unlimited malloc backend for this.
 
+.. XXX: Is this another paramater? In that case handled in the same manner as above? benc
+
 Varnish will consider an object short lived if the TTL is below the
-parameter "shortlived".
+parameter 'shortlived'.
+
 
+.. XXX: I am generally missing samples of setting all of these parameters, maybe one sample per section or a couple of examples here with a brief explanation to also work as a summary? benc
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index fa01890..9d57744 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -3,9 +3,9 @@
 Troubleshooting Varnish
 =======================
 
-Sometimes Varnish misbehaves. In order for you to understand whats
-going on there are a couple of places you can check. varnishlog,
-/var/log/syslog, /var/log/messages are all places where Varnish might
+Sometimes Varnish misbehaves or rather behaves the way you told it to behave but not necessarily the way you want it to behave. In order for you to understand whats
+going on there are a couple of places you can check. `varnishlog`,
+`/var/log/syslog`, `/var/log/messages` are all good places where Varnish might
 leave clues of whats going on. This section will guide you through
 basic troubleshooting in Varnish.
 
@@ -13,9 +13,9 @@ basic troubleshooting in Varnish.
 When Varnish won't start
 ------------------------
 
-Sometimes Varnish wont start. There is a plethora of reasons why
+Sometimes Varnish wont start. There is a plethora of possible reasons why
 Varnish wont start on your machine. We've seen everything from wrong
-permissions on /dev/null to other processes blocking the ports.
+permissions on `/dev/null` to other processes blocking the ports.
 
 Starting Varnish in debug mode to see what is going on.
 
@@ -23,8 +23,8 @@ Try to start Varnish by::
 
     # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1: 2000  -a 0.0.0.0:8080 -d
 
-Notice the -d option. It will give you some more information on what
-is going on. Let us see how Varnish will react to something else
+Notice the '-d' parameter. It will give you some more information on what
+is going on. Let us see how Varnish will react when something else is
 listening on its port.::
 
     # varnishd -n foo -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000  -a 0.0.0.0:8080 -d
@@ -39,7 +39,7 @@ listening on its port.::
     Type 'quit' to close CLI session.
     Type 'start' to launch worker process.
 
-Now Varnish is running. Only the master process is running, in debug
+Now Varnish is running but only the master process is running, in debug
 mode the cache does not start. Now you're on the console. You can
 instruct the master process to start the cache by issuing "start".::
 
@@ -49,7 +49,7 @@ instruct the master process to start the cache by issuing "start".::
 	 Could not open sockets
 
 And here we have our problem. Something else is bound to the HTTP port
-of Varnish. If this doesn't help try strace or truss or come find us
+of Varnish. If this doesn't help try ``strace`` or ``truss`` or come find us
 on IRC.
 
 
@@ -57,34 +57,33 @@ Varnish is crashing - panics
 ----------------------------
 
 When Varnish goes bust the child processes crashes. Most of the
-crashes are caught by one of the many consistency checks spread around
-the Varnish source code. When Varnish hits one of these the caching
-process it will crash itself in a controlled manner, leaving a nice
+crashes are caught by one of the many consistency checks we have included in the Varnish source code. When Varnish hits one of these the caching
+process will crash itself in a controlled manner, leaving a nice
 stack trace with the mother process.
 
-You can inspect any panic messages by typing panic.show in the CLI.
-
-| panic.show
-| Last panic at: Tue, 15 Mar 2011 13:09:05 GMT
-| Assert error in ESI_Deliver(), cache_esi_deliver.c line 354:
-|   Condition(i == Z_OK || i == Z_STREAM_END) not true.
-| thread = (cache-worker)
-| ident = Linux,2.6.32-28-generic,x86_64,-sfile,-smalloc,-hcritbit,epoll
-| Backtrace:
-|   0x42cbe8: pan_ic+b8
-|   0x41f778: ESI_Deliver+438
-|   0x42f838: RES_WriteObj+248
-|   0x416a70: cnt_deliver+230
-|   0x4178fd: CNT_Session+31d
-|   (..)
+You can inspect any panic messages by typing ``panic.show`` in the CLI.::
+
+ panic.show
+ Last panic at: Tue, 15 Mar 2011 13:09:05 GMT
+ Assert error in ESI_Deliver(), cache_esi_deliver.c line 354:
+   Condition(i == Z_OK || i == Z_STREAM_END) not true.
+ thread = (cache-worker)
+ ident = Linux,2.6.32-28-generic,x86_64,-sfile,-smalloc,-hcritbit,epoll
+ Backtrace:
+   0x42cbe8: pan_ic+b8
+   0x41f778: ESI_Deliver+438
+   0x42f838: RES_WriteObj+248
+   0x416a70: cnt_deliver+230
+   0x4178fd: CNT_Session+31d
+   (..)
 
 The crash might be due to misconfiguration or a bug. If you suspect it
-is a bug you can use the output in a bug report.
+is a bug you can use the output in a bug report, see the "Trouble Tickets" section in the Introduction chapter above.
 
 Varnish is crashing - segfaults
 -------------------------------
 
-Sometimes the bug escapes the consistency checks and Varnish get hit
+Sometimes a bug escapes the consistency checks and Varnish gets hit
 with a segmentation error. When this happens with the child process it
 is logged, the core is dumped and the child process starts up again.
 
@@ -93,29 +92,31 @@ debug a segfault the developers need you to provide a fair bit of
 data.
 
  * Make sure you have Varnish installed with symbols
+.. XXX:Symbols? benc
  * Make sure core dumps are enabled (ulimit)
+.. XXX:ulimit? benc
 
-Once you have the core you open it with gdb and issue the command "bt"
+Once you have the core you open it with `gdb` and issue the command ``bt``
 to get a stack trace of the thread that caused the segfault.
 
 
 Varnish gives me Guru meditation
 --------------------------------
 
-First find the relevant log entries in varnishlog. That will probably
-give you a clue. Since varnishlog logs so much data it might be hard
-to track the entries down. You can set varnishlog to log all your 503
+First find the relevant log entries in `varnishlog`. That will probably
+give you a clue. Since `varnishlog` logs a lot of data it might be hard
+to track the entries down. You can set `varnishlog` to log all your 503
 errors by issuing the following command::
 
    $ varnishlog -c -m TxStatus:503
 
 If the error happened just a short time ago the transaction might still
-be in the shared memory log segment. To get varnishlog to process the
-whole shared memory log just add the -d option::
+be in the shared memory log segment. To get `varnishlog` to process the
+whole shared memory log just add the '-d' parameter::
 
    $ varnishlog -d -c -m TxStatus:503
 
-Please see the varnishlog man page for elaborations on further
+Please see the `varnishlog` man page for elaborations on further
 filtering capabilities and explanation of the various options.
 
 
diff --git a/doc/sphinx/users-guide/vcl-actions.rst b/doc/sphinx/users-guide/vcl-actions.rst
index 691baa4..0775afe 100644
--- a/doc/sphinx/users-guide/vcl-actions.rst
+++ b/doc/sphinx/users-guide/vcl-actions.rst
@@ -5,18 +5,22 @@ actions
 
 The most common actions to return are these:
 
+.. XXX:Maybe a bit more explanation here what is an action and how it is returned? benc
+
 *pass*
  When you return pass the request and subsequent response will be passed to
- and from the backend server. It won't be cached. pass can be returned from
- vcl_recv
+ and from the backend server. It won't be cached. `pass` can be returned from
+ `vcl_recv`
 
 *lookup*
-  When you return lookup from vcl_recv you tell Varnish to deliver content 
+  When you return lookup from `vcl_recv` you tell Varnish to deliver content 
   from cache even if the request othervise indicates that the request 
   should be passed. 
 
 *pipe*
-  Pipe can be returned from vcl_recv as well. Pipe short circuits the
+.. XXX:What is pipe? benc
+
+  Pipe can be returned from `vcl_recv` as well. Pipe short circuits the
   client and the backend connections and Varnish will just sit there
   and shuffle bytes back and forth. Varnish will not look at the data being 
   send back and forth - so your logs will be incomplete. 
@@ -25,13 +29,13 @@ The most common actions to return are these:
   header before actually returning pipe. 
 
 *deliver*
- Deliver the object to the client.  Usually returned from vcl_backend_response. 
+ Deliver the object to the client. Usually returned from `vcl_backend_response`. 
 
 *restart*
  Restart processing of the request. You can restart the processing of
- the whole transaction. Changes to the req object are retained.
+ the whole transaction. Changes to the `req` object are retained.
 
 *retry*
  Retry the request against the backend. This can be called from
- vcl_backend_response or vcl_backend_error if you don't like the response 
+ `vcl_backend_response` or `vcl_backend_error` if you don't like the response 
  that the backend delivered.
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 099e2a6..c54d951 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -6,7 +6,7 @@ Backend servers
 Varnish has a concept of "backend" or "origin" servers. A backend
 server is the server providing the content Varnish will accelerate.
 
-Our first task is to tell Varnish where it can find its content. Start
+Our first task is to tell Varnish where it can find its backends. Start
 your favorite text editor and open the relevant VCL file.
 
 Somewhere in the top there will be a section that looks a bit like this.::
@@ -16,7 +16,7 @@ Somewhere in the top there will be a section that looks a bit like this.::
     #     .port = "8080";
     # }
 
-We comment in this bit of text making the text look like.::
+We remove the comment markings in this text stanza making the it look like.::
 
     backend default {
         .host = "127.0.0.1";
@@ -27,7 +27,7 @@ Now, this piece of configuration defines a backend in Varnish called
 *default*. When Varnish needs to get content from this backend it will
 connect to port 8080 on localhost (127.0.0.1).
 
-Varnish can have several backends defined and can you can even join
+Varnish can have several backends defined you can even join
 several backends together into clusters of backends for load balancing
 purposes.
 
@@ -41,10 +41,10 @@ host or not. There are lot of options.
 
 Lets say we need to introduce a Java application into out PHP web
 site. Lets say our Java application should handle URL beginning with
-/java/.
+`/java/`.
 
 We manage to get the thing up and running on port 8000. Now, lets have
-a look at the default.vcl.::
+a look at the `default.vcl`.::
 
     backend default {
         .host = "127.0.0.1";
@@ -58,7 +58,7 @@ We add a new backend.::
         .port = "8000";
     }
 
-Now we need tell where to send the difference URL. Lets look at vcl_recv.::
+Now we need tell Varnish where to send the difference URL. Lets look at `vcl_recv`.::
 
     sub vcl_recv {
         if (req.url ~ "^/java/") {
@@ -71,18 +71,18 @@ Now we need tell where to send the difference URL. Lets look at vcl_recv.::
 It's quite simple, really. Lets stop and think about this for a
 moment. As you can see you can define how you choose backends based on
 really arbitrary data. You want to send mobile devices to a different
-backend? No problem. if (req.User-agent ~ /mobile/) .. should do the
+backend? No problem. ``if (req.User-agent ~ /mobile/) ..`` should do the
 trick.
 
 
 Backends and virtual hosts in Varnish
 -------------------------------------
 
-Varnish fully supports virtual hosts. They might work in a somewhat
+Varnish fully supports virtual hosts. They might however work in a somewhat
 counter intuitive fashion since they are never declared
 explicitly. You set up the routing of incoming HTTP requests in
-vcl_recv. If you want this routing to be done on the basis of virtual
-hosts you just need to inspect req.http.host.
+`vcl_recv`. If you want this routing to be done on the basis of virtual
+hosts you just need to inspect `req.http.host`.
 
 You can have something like this:::
 
@@ -94,10 +94,10 @@ You can have something like this:::
         }
     }
 
-Note that the first regular expressions will match foo.com,
-www.foo.com, zoop.foo.com and any other host ending in foo.com. In
+Note that the first regular expressions will match "foo.com",
+"www.foo.com", "zoop.foo.com" and any other host ending in "foo.com". In
 this example this is intentional but you might want it to be a bit
-more tight, maybe relying on the == operator in stead, like this:::
+more tight, maybe relying on the ``==`` operator in stead, like this:::
 
     sub vcl_recv {
         if (req.http.host == "foo.com" or req.http.host == "www.foo.com") {
@@ -118,7 +118,7 @@ and resilience.
 
 You can define several backends and group them together in a
 director. This requires you to load a VMOD, a Varnish module, and then to
-call certain actions in vcl_init.::
+call certain actions in `vcl_init`.::
 
 
     import directors;    # load the directors
@@ -180,11 +180,11 @@ define the backends.::
         }
     }
 
-Whats new here is the probe. Varnish will check the health of each
+Whats new here is the ``probe``. Varnish will check the health of each
 backend with a probe. The options are:
 
 url
-    What URL should Varnish request.
+    The URL Varnish will use to send a probe request.
 
 interval
     How often should we poll
@@ -197,13 +197,17 @@ window
     window has five checks.
 
 threshold
-    How many of the .window last polls must be good for the backend to be declared healthy.
+    How many of the '.window' last polls must be good for the backend to be declared healthy.
+
+.. XXX: .window probably means something but not to me :) benc
 
 initial
-    How many of the of the probes a good when Varnish starts - defaults
+    How many of the probes that needs to be succesful when Varnish starts - defaults
     to the same amount as the threshold.
 
-Now we define the director.::
+Now we define the 'director'.::
+
+.. XXX: Where and why? benc
 
     import directors;
 
diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index b30985c..708c05e 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -1,7 +1,7 @@
 
 .. _vcl-built-in-subs:
 
-.. XXX This document needs substational review.
+.. XXX:This document needs substantional review.
 
 
 Built in subroutines
@@ -12,14 +12,14 @@ vcl_recv
 ~~~~~~~~
 
 Called at the beginning of a request, after the complete request has
-been received and parsed.  Its purpose is to decide whether or not to
+been received and parsed. Its purpose is to decide whether or not to
 serve the request, how to do it, and, if applicable, which backend to
 use.
 
 It is also used to modify the request, something you'll probably find
 yourself doing frequently. 
 
-The vcl_recv subroutine may terminate with calling ``return()`` on one
+The `vcl_recv` subroutine may terminate with calling ``return()`` on one
 of the following keywords:
 
   synth 
@@ -27,10 +27,10 @@ of the following keywords:
     client and abandon the request.
 
   pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
+    Switch to pass mode. Control will eventually pass to vcl_pass.
 
   pipe
-    Switch to pipe mode.  Control will eventually pass to vcl_pipe.
+    Switch to pipe mode. Control will eventually pass to vcl_pipe.
 
   hash
     Continue processing the object as a potential candidate for
@@ -43,13 +43,13 @@ of the following keywords:
 vcl_pipe
 ~~~~~~~~
 
-Called upon entering pipe mode.  In this mode, the request is passed
+Called upon entering pipe mode. In this mode, the request is passed
 on to the backend, and any further data from either client or backend
 is passed on unaltered until either end closes the
 connection. Basically, Varnish will degrade into a simple TCP proxy,
 shuffling bytes back and forth.
 
-The vcl_pipe subroutine may terminate with calling return() with one
+The `vcl_pipe` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
   synth(error code, reason)
@@ -61,12 +61,12 @@ of the following keywords:
 vcl_pass
 ~~~~~~~~
 
-Called upon entering pass mode.  In this mode, the request is passed
+Called upon entering pass mode. In this mode, the request is passed
 on to the backend, and the backend's response is passed on to the
-client, but is not entered into the cache.  Subsequent requests
+client, but is not entered into the cache. Subsequent requests
 submitted over the same client connection are handled normally.
 
-The vcl_pass subroutine may terminate with calling return() with one
+The `vcl_pass` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
   synth(error code, reason)
@@ -84,7 +84,10 @@ of the following keywords:
 vcl_hit
 ~~~~~~~
 
-Called is a cache lookup is successful. 
+Called when a cache lookup is successful. 
+
+.. XXX: missing the "The `vcl_hit` subroutine may terminate with calling ``return()`` with one of the following keywords:" thing. benc
+
 
   restart
     Restart the transaction. Increases the restart counter. If the number
@@ -92,7 +95,7 @@ Called is a cache lookup is successful.
     error.
 
   deliver
-    Deliver the object. Control passes to vcl_deliver.
+    Deliver the object. Control passes to `vcl_deliver`.
 
   synth(error code, reason)
     Return the specified error code to the client and abandon the request.
@@ -102,26 +105,26 @@ vcl_miss
 ~~~~~~~~
 
 Called after a cache lookup if the requested document was not found in
-the cache.  Its purpose is to decide whether or not to attempt to
+the cache. Its purpose is to decide whether or not to attempt to
 retrieve the document from the backend, and which backend to use.
 
-The vcl_miss subroutine may terminate with calling return() with one
+The `vcl_miss` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
   synth(error code, reason)
     Return the specified error code to the client and abandon the request.
 
   pass
-    Switch to pass mode.  Control will eventually pass to vcl_pass.
+    Switch to pass mode. Control will eventually pass to `vcl_pass`.
 
   fetch
-    Retrieve the requested object from the backend.  Control will
-    eventually pass to vcl_fetch.
+    Retrieve the requested object from the backend. Control will
+    eventually pass to `vcl_fetch`.
 
 vcl_hash
 ~~~~~~~~
 
-Called after vcl_recv to create a hash value for the request. This is
+Called after `vcl_recv` to create a hash value for the request. This is
 used as a key to look up the object in Varnish.
 
   lookup
@@ -134,20 +137,21 @@ used as a key to look up the object in Varnish.
 vcl_purge
 ~~~~~~~~~
 
-Called after the purge has been executed and all it's variant have been evited. 
+Called after the purge has been executed and all its variants have been evited.
 
   synth
     Produce a response.
 
 
-
 vcl_deliver
 ~~~~~~~~~~~
 
 Called before a cached object is delivered to the client.
 
-The vcl_deliver subroutine may terminate with one of the following
-keywords:
+The ``vcl_deliver`` subroutine may terminate calling ``return()`` with one
+of the following keywords:
+
+.. XXX: Should perhaps be return as above? benc
 
   deliver
     Deliver the object to the client.
@@ -164,35 +168,46 @@ vcl_backend_fetch
 Called before sending the backend request. In this subroutine you
 typically alter the request before it gets to the backend.
 
+.. XXX: Missing terminate..keywords sentence? benc
+
   fetch
     Fetch the object from the backend.
 
   abandon
     Abandon the backend request and generates an error.
-  
+
 
 vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~
 
-Called after an response has been successfully retrieved from the
-backend. The response is availble as beresp. Note that Varnish might
+Called after a response has been successfully retrieved from the
+backend. The response is available as `beresp`. 
+
+.. XXX: beresp comes out of the blue here. maybe a short description? benc
+
+Note that Varnish might
 not be talking to an actual client, so operations that require a
-client to be present are not allowed. Specifically there is no req
-object and restarts are not allowed.
+client to be present are not allowed. Specifically there is no `req
+object` and restarts are not allowed.
+
+.. XXX: I do not follow sentence above. benc
 
-The vcl_backend_response subroutine may terminate with calling return() with one
+The `vcl_backend_response` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
   deliver
     Possibly insert the object into the cache, then deliver it to the
-    Control will eventually pass to vcl_deliver. Caching is dependant
-    on beresp.cacheable.
+    Control will eventually pass to `vcl_deliver`. Caching is dependant
+    on 'beresp.cacheable'.
+
+.. XXX:A parameter? that is set how? benc
+    
 
   error(error code, reason)
     Return the specified error code to the client and abandon the request.
 
   retry
-    Retry the backend transaction. Increases the retries counter. If the number
+    Retry the backend transaction. Increases the `retries` counter. If the number
     of retries is higher than *max_retries* Varnish emits a guru meditation
     error.
 
@@ -201,11 +216,13 @@ vcl_backend_error
 
 This subroutine is called if we fail the backend fetch. 
 
+.. XXX:Missing the terminate return structure? benc
+
   deliver
     Deliver the error.
 
   retry
-    Retry the backend transaction. Increases the retries counter. If the number
+    Retry the backend transaction. Increases the `retries` counter. If the number
     of retries is higher than *max_retries* Varnish emits a guru meditation
     error.
 
@@ -213,10 +230,12 @@ This subroutine is called if we fail the backend fetch.
 vcl_backend_error
 ~~~~~~~~~~~~~~~~~
 
+.. XXX: Same name as section above? benc
+
 Called when we hit an error, either explicitly or implicitly due to
 backend or internal errors.
 
-The vcl_backend_error subroutine may terminate by calling return with one of
+The `vcl_backend_error` subroutine may terminate by calling ``return()`` with one of
 the following keywords:
 
   deliver
@@ -234,7 +253,9 @@ vcl_init
 Called when VCL is loaded, before any requests pass through it.
 Typically used to initialize VMODs.
 
-  return() values:
+.. XXX: Missing the terminate return structure? benc
+
+  ``return()`` values:
 
   ok
     Normal return, VCL continues loading.
@@ -246,7 +267,13 @@ vcl_fini
 Called when VCL is discarded only after all requests have exited the VCL.
 Typically used to clean up VMODs.
 
-  return() values:
+
+.. XXX: Missing the terminate return structure? benc
+
+  ``return()`` values:
 
   ok
     Normal return, VCL will be discarded.
+
+
+.. XXX: Maybe end here with the detailed flowchart from the book together with a reference to the book? benc
diff --git a/doc/sphinx/users-guide/vcl-example-acls.rst b/doc/sphinx/users-guide/vcl-example-acls.rst
index b460bbe..45afaa2 100644
--- a/doc/sphinx/users-guide/vcl-example-acls.rst
+++ b/doc/sphinx/users-guide/vcl-example-acls.rst
@@ -2,7 +2,7 @@
 ACLs
 ~~~~
 
-You create a named access control list with the *acl* keyword. You can match
+You create a named access control list with the ``acl`` keyword. You can match
 the IP address of the client against an ACL with the match operator.::
 
   # Who is allowed to purge....
@@ -21,4 +21,4 @@ the IP address of the client against an ACL with the match operator.::
       }
     } 
   }
-  
+ 
diff --git a/doc/sphinx/users-guide/vcl-example-manipulating-headers.rst b/doc/sphinx/users-guide/vcl-example-manipulating-headers.rst
index 35dcb1e..7e65663 100644
--- a/doc/sphinx/users-guide/vcl-example-manipulating-headers.rst
+++ b/doc/sphinx/users-guide/vcl-example-manipulating-headers.rst
@@ -4,7 +4,7 @@
 Manipulating request headers in VCL
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Lets say we want to remove the cookie for all objects in the /images
+Lets say we want to remove the cookie for all objects in the `/images`
 directory of our web server::
 
   sub vcl_recv {
diff --git a/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst b/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
index b5724b4..6719524 100644
--- a/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
+++ b/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
@@ -15,5 +15,5 @@ matches certain criteria::
 
 
 
-We also remove any Set-Cookie headers in order to avoid a hit-for-pass
+We also remove any Set-Cookie headers in order to avoid a `hit-for-pass`
 object to be created. See :ref:`user-guide-vcl_actions`.
diff --git a/doc/sphinx/users-guide/vcl-example-websockets.rst b/doc/sphinx/users-guide/vcl-example-websockets.rst
index 7217b88..609fe61 100644
--- a/doc/sphinx/users-guide/vcl-example-websockets.rst
+++ b/doc/sphinx/users-guide/vcl-example-websockets.rst
@@ -18,3 +18,4 @@ VCL config to do so::
          }
     }
 
+.. XXX: Pipe it? maybe a bit more explanation here? benc
diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
index f318aa3..afc92bd 100644
--- a/doc/sphinx/users-guide/vcl-grace.rst
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -7,13 +7,12 @@ A key feature of Varnish is its ability to shield you from misbehaving
 web- and application servers.
 
 
-
 Grace mode
 ~~~~~~~~~~
 
 When several clients are requesting the same page Varnish will send
 one request to the backend and place the others on hold while fetching
-one copy from the back end. In some products this is called request
+one copy from the backend. In some products this is called request
 coalescing and Varnish does this automatically.
 
 If you are serving thousands of hits per second the queue of waiting
diff --git a/doc/sphinx/users-guide/vcl-hashing.rst b/doc/sphinx/users-guide/vcl-hashing.rst
index fdbe37f..83758ad 100644
--- a/doc/sphinx/users-guide/vcl-hashing.rst
+++ b/doc/sphinx/users-guide/vcl-hashing.rst
@@ -1,12 +1,12 @@
 Hashing
 -------
 
-Internally, when Varnish stores content in it's store it uses a hash
+Internally, when Varnish stores content in the cache it stores the object together with a hash
 key to find the object again. In the default setup this key is
 calculated based on the content of the *Host* header or the IP address
 of the server and the URL.
 
-Behold the default vcl::
+Behold the `default vcl`::
 
  sub vcl_hash {
      hash_data(req.url);
@@ -18,10 +18,10 @@ Behold the default vcl::
      return (hash);
  }
 
-As you can see it first chucks in req.url then req.http.host if it
+As you can see it first checks in `req.url` then `req.http.host` if it
 exists. It is worth pointing out that Varnish doesn't lowercase the
-hostname or the URL before hashing it so in theory having Varnish.org/
-and varnish.org/ would result in different cache entries. Browers
+hostname or the URL before hashing it so in theory having "Varnish.org/"
+and "varnish.org/" would result in different cache entries. Browsers
 however, tend to lowercase hostnames.
 
 You can change what goes into the hash. This way you can make Varnish
@@ -33,11 +33,11 @@ based on where their IP address is located. You would need some Vmod
 to get a country code and then put it into the hash. It might look
 like this.
 
-In vcl_recv::
+In `vcl_recv`::
 
   set req.http.X-Country-Code = geoip.lookup(client.ip);
 
-And then add a vcl_hash::
+And then add a `vcl_hash`::
 
  sub vcl_hash {
    hash_data(req.http.X-Country-Code);
@@ -45,6 +45,6 @@ And then add a vcl_hash::
 
 As the default VCL will take care of adding the host and URL to the
 hash we don't have to do anything else. Be careful calling
-return(hash) as this will abort the execution of the default VCL and
-thereby you can end up with a Varnish that will return data based on
+``return(hash)`` as this will abort the execution of the default VCL and
+Varnish can end up returning data based on
 more or less random inputs.
diff --git a/doc/sphinx/users-guide/vcl-inline-c.rst b/doc/sphinx/users-guide/vcl-inline-c.rst
index 7c88cf9..5cc0ead 100644
--- a/doc/sphinx/users-guide/vcl-inline-c.rst
+++ b/doc/sphinx/users-guide/vcl-inline-c.rst
@@ -10,7 +10,7 @@ You can use *in-line C* to extend Varnish. Please note that you can
 seriously mess up Varnish this way. The C code runs within the Varnish
 Cache process so if your code generates a segfault the cache will crash.
 
-One of the first uses I saw of In-line C was logging to syslog.::
+One of the first uses of In-line C was logging to `syslog`.::
 
         # The include statements must be outside the subroutines.
         C{
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 889ea97..4615291 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -9,7 +9,7 @@ preferences.
 
 Note that VCL doesn't contain any loops or jump statements.
 
-This document gives an outline of the most important parts of the
+This section provides an outline of the more important parts of the
 syntax. For a full documentation of VCL syntax please see
 :ref:`reference-vcl` in the reference.
 
@@ -18,7 +18,7 @@ Strings
 
 Basic strings are enclosed in " ... ", and may not contain newlines.
 
-Backslash is not special, so for instance in regsub() you do not need
+Backslash is not special, so for instance in `regsub()` you do not need
 to do the "count-the-backslashes" polka:::
 
   regsub("barf", "(b)(a)(r)(f)", "\4\3\2p") -> "frap"
@@ -43,9 +43,9 @@ which can later be used to match client addresses::
        }
 
 If an ACL entry specifies a host name which Varnish is unable to
-resolve, it will match any address it is compared to.  Consequently,
+resolve, it will match any address it is compared to. Consequently,
 if it is preceded by a negation mark, it will reject any address it is
-compared to, which may not be what you intended.  If the entry is
+compared to, which may not be what you intended. If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
 To match an IP address against an ACL, simply use the match operator::
@@ -93,10 +93,13 @@ A subroutine is used to group code for legibility or reusability:
 
 Subroutines in VCL do not take arguments, nor do they return values.
 
-To call a subroutine, use the call keyword followed by the subroutine's name:
+To call a subroutine, use the call keyword followed by the subroutine's name::
 
 call pipe_if_local;
 
 Varnish has quite a few built in subroutines that are called for each
 transaction as it flows through Varnish. These builtin subroutines are all named vcl_*. Your own subroutines cannot start their name with vcl_.
+
+.. XXX:looks as bit funky as red text? benc
+
 See :ref:`vcl-built-in-subs`.
diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst
index 20fcd4e..88ab8f4 100644
--- a/doc/sphinx/users-guide/vcl-variables.rst
+++ b/doc/sphinx/users-guide/vcl-variables.rst
@@ -1,27 +1,33 @@
 
-Requests, responses and objects
+Requests and responses as objects
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-In VCL, there several important objects.
+.. XXX: refactored headline. benc
+
+In VCL, there several important objects that you need to be aware of. These objects can be accessed and manipulated using VCL.
 
 
 *req*
- The request object. When Varnish has received the request the req object is 
- created and populated. Most of the work you do in vcl_recv you 
- do on or with the req object.
+ The request object. When Varnish has received the request the `req` object is 
+ created and populated. Most of the work you do in `vcl_recv` you 
+ do on or with the `req` object.
 
 *bereq*
  The backend request object. Varnish contructs this before sending it to the 
- backend. It is based on the req object.
+ backend. It is based on the `req` object.
+
+.. XXX:in what way? benc
 
 *beresp*
  The backend response object. It contains the headers of the object 
  coming from the backend. If you want to modify the reponse coming from the 
- server you modify this object in vcl_backend_reponse. 
+ server you modify this object in `vcl_backend_reponse`. 
 
 *resp*
  The HTTP response right before it is delivered to the client. It is
- typically modified in vcl_deliver.
+ typically modified in `vcl_deliver`.
 
 *obj* 
  The object as it is stored in cache. Mostly read only.
+.. XXX:What object? the current request? benc
+
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index ad65403..8585241 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -3,12 +3,12 @@
 VCL - Varnish Configuration Language
 ------------------------------------
 
-This section is about getting Varnish to do what you want to
+This section covers how to tell Varnish how to handle
 your HTTP traffic, using the Varnish Configuration Language (VCL).
 
 Varnish has a great configuration system. Most other systems use
 configuration directives, where you basically turn on and off lots of
-switches. Varnish uses a domain specific language called VCL for this.
+switches. We have instead chosen to use a domain specific language called VCL for this.
 
 Every inbound request flows through Varnish and you can influence how
 the request is being handled by altering the VCL code. You can direct
@@ -26,7 +26,7 @@ request, another when files are fetched from the backend server.
 
 If you don't call an action in your subroutine and it reaches the end
 Varnish will execute some built-in VCL code. You will see this VCL
-code commented out in builtin.vcl that ships with Varnish Cache.
+code commented out in the file `builtin.vcl` that ships with Varnish Cache.
 
 .. _users-guide-vcl_fetch_actions:
 
@@ -43,5 +43,7 @@ code commented out in builtin.vcl that ships with Varnish Cache.
    vcl-inline-c
    vcl-examples
    websockets
+.. XXX: websockets seems to be missing? does it refer to the last sample in the vcl index if so already included. benc
+
    devicedetection
 
diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst
index cf027c8..274c9f5 100644
--- a/doc/sphinx/whats-new/changes.rst
+++ b/doc/sphinx/whats-new/changes.rst
@@ -3,14 +3,15 @@
 Changes in Varnish 4
 ====================
 
-Varnish 4 is quite an extensive update over Varnish 3, with some very big improvements to central parts of varnish.
+Varnish 4 is quite an extensive update to Varnish 3, with some very big improvements to central parts of varnish.
 
 Client/backend split
 --------------------
 In the past, Varnish has fetched the content from the backend in the same
-thread as the client request. The client and backend code has now been split,
-allowing for some much requested improvements.
-This split allows varnish to refresh content in the background while serving
+thread as the client request.In Varnish 4 we have  split the client and backend code into separate trheads allowing for some much requested improvements.
+This split allows Varnish to refresh content in the background while serving
 stale content quickly to the client.
 
-This split has also necessitated a change of the VCL-functions, in particular functionality has moved from the old vcl_fetch method to the two new methods vcl_backend_fetch and vcl_backend_response.
+This split has also necessitated a change of the VCL-functions, in particular functionality has moved from the old `vcl_fetch` method to the two new methods `vcl_backend_fetch` and `vcl_backend_response`.
+
+.. XXX:Here would an updated flow-diagram over functions be great. benc
diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst
index a1d1d73..c9d9130 100644
--- a/doc/sphinx/whats-new/index.rst
+++ b/doc/sphinx/whats-new/index.rst
@@ -1,14 +1,16 @@
 .. _whats-new-index:
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%
-What's new for Varnish 4.0
+What's new in Varnish 4.0
 %%%%%%%%%%%%%%%%%%%%%%%%%%
 
-This document describes the changes that have been made for Varnish 4. The
-first section will describe the overarching changes that have gone into
-Varnish, while the second section describes what changes you need to make to
-your configuration as well as any changes in behaviour that you need to take
-into consideration while upgrading.
+This section describes the changes that have been made for Varnish 4. The
+first subsection describes overarching changes that have gone into
+Varnish 4.0, while the second subsection describes changes you need to make to
+your current configuration (assuming you are on Varnish 3.x) as well as any changes in behaviour that you need to be aware of and take
+into consideration when upgrading.
+
+.. XXX:Heavy change of meaning above! benc
 
 .. toctree::
    :maxdepth: 2
diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 5f52318..2921835 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -37,7 +37,7 @@ Since the client director was already a special case of the hash director, it ha
 
 error() is now a return value
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-You must now explicitly return an error::
+You must explicitly return an error::
 
 	return(error(999, "Response));
 
@@ -76,11 +76,11 @@ vcl_recv should return(hash) instead of lookup now
 
 req.* not available in vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-req.* used to be available in vcl_fetch, but after the split of functionality, you only have bereq.* in vcl_backend_response.
+req.* used to be available in `vcl_fetch`, but after the split of functionality, you only have 'bereq.*' in `vcl_backend_response`.
 
 vcl_* reserved
 ~~~~~~~~~~~~~~
-Your own subs cannot be named vcl_* anymore. That is reserved for builtin subs.
+Any custom-made subs cannot be named 'vcl_*' anymore. This namespace is reserved for builtin subs.
 
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~



From lkarsten at varnish-software.com  Thu Mar 20 11:57:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 12:57:27 +0100
Subject: [master] b4068d3 minor update
Message-ID: <E1WQbbD-0001Zw-Pc@project.varnish-software.com>


commit b4068d3ead5974626bfae87a01bda521a5af0fa3
Author: benc <benc at redpill-linpro.com>
Date:   Sun Mar 16 12:55:16 2014 +0100

    minor update

diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 2be32e3..214fe81 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -21,7 +21,7 @@ A *purge* is what happens when you pick out an object from the cache
 and discard it along with its variants. Usually a purge is invoked
 through HTTP with the method `PURGE`.
 
-An HTTP purge is similar to a HTTP GET request, except that the
+A HTTP purge is similar to a HTTP GET request, except that the
 *method* is `PURGE`. Actually you can call the method whatever you'd
 like, but most people refer to this as purging. Squid, for example, supports the
 same mechanism. In order to support purging in Varnish you need the



From lkarsten at varnish-software.com  Thu Mar 20 11:57:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 12:57:27 +0100
Subject: [master] 64438cd Minor cleanups
Message-ID: <E1WQbbD-0001a5-SL@project.varnish-software.com>


commit 64438cd0e89378430e55f86b349159e4564aa207
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 12:35:38 2014 +0100

    Minor cleanups

diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 946e921..66433de 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -38,7 +38,7 @@ it mostly on topic, and please don't paste random links unless they are
 Mailing Lists
 =============
 
-Subscribing or unsubscribing to our mailing lists is handled through our Mailman_.
+Subscribing or unsubscribing to our mailing lists is handled through mailman_.
 
 If you are going to use Varnish, subscribing to our `varnish-announce`
 mailing list is a very good idea. The typical pattern is that
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 348b6ef..2cddd59 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -106,11 +106,8 @@ packages installed:
 * `pcre-devel`
 * `pkgconfig`
 * `libedit-devel`
-
-If you're building from git, you also need the following:
-
 * `docutils`
-* `python-sphinx` (optional, if you want to build the documentation)
+* `python-sphinx`
 
 Compiling Varnish
 -----------------
@@ -144,9 +141,9 @@ And finally, the true test of a brave heart::
 
 	sudo make install
 
-Varnish will now be installed in `/usr/loca``l. The ``varnishd` binary is in
+Varnish will now be installed in `/usr/local`. The ``varnishd` binary is in
 `/usr/local/sbin/varnishd` and its default configuration will be
 `/usr/local/etc/varnish/default.vcl`.
 
-After succesful installation you are ready to proceed to the :ref:`tutorial-index`.
+After successful installation you are ready to proceed to the :ref:`tutorial-index`.
 
diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
index 1c9ab65..cb5f087 100644
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -3,7 +3,7 @@
 Backend servers
 ---------------
 
-Varnish has a concept of `backend` or `origin` servers. A backend
+Varnish has a concept of `backend` or origin servers. A backend
 server is the server providing the content Varnish will accelerate via the cache.
 
 Our first task is to tell Varnish where it can find its content. Start
@@ -41,15 +41,9 @@ localhost, port 8080.::
   }
 
 
-Varnish can have several backends defined and can even join
-several backends together into clusters of backends for load balancing
-purposes, having Varnish pick one backend based on different
-algorithms. 
-
-A lot of the power of Varnish Cache comes from it's design, which
-might not be what you are expecting.
-
-.. XXX:What am I expecting? benc
+Varnish can have several backends defined and can even join several backends
+together into clusters of backends for load balancing purposes, having Varnish
+pick one backend based on different algorithms. 
 
 Next, let's have a look at some of what makes Varnish unique and what you can do with it.
 



From lkarsten at varnish-software.com  Thu Mar 20 11:57:27 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 12:57:27 +0100
Subject: [master] a077da7 Use pipe for chunked POST/PUT requests.
Message-ID: <E1WQbbD-0001aE-Tt@project.varnish-software.com>


commit a077da7c9561299a4727387785cbe8f3211d6722
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 12:40:40 2014 +0100

    Use pipe for chunked POST/PUT requests.
    
    We don't know how to handle incoming chunked bodies currently, but the backend
    might.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 3949689..71e01f7 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -55,6 +55,13 @@ sub vcl_recv {
         /* Non-RFC2616 or CONNECT which is weird. */
         return (pipe);
     }
+
+    /* We don't support chunked uploads, except when piping. */
+    if ((req.request == "POST" || req.request == "PUT") &&
+      req.http.transfer-encoding ~ "chunked") {
+        return(pipe);
+    }
+
     if (req.method != "GET" && req.method != "HEAD") {
         /* We only deal with GET and HEAD by default */
         return (pass);



From lkarsten at varnish-software.com  Thu Mar 20 12:07:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 13:07:15 +0100
Subject: [master] b966907 Squash some RST warnings
Message-ID: <E1WQbkh-0004q5-JG@project.varnish-software.com>


commit b966907afd8a2790b66a68e1d74b0e0e200bb959
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 13:06:55 2014 +0100

    Squash some RST warnings

diff --git a/doc/sphinx/installation/bugs.rst b/doc/sphinx/installation/bugs.rst
index dd71cc8..de433a1 100644
--- a/doc/sphinx/installation/bugs.rst
+++ b/doc/sphinx/installation/bugs.rst
@@ -83,13 +83,14 @@ general forms:
 	for HTTP headers and Cookies.
 
 	Please try that before reporting a bug.
-..  (TODO: which params to tweak)
 
 "Assert error in ..."
 	This is something bad that should never happen, and a bug
 	report is almost certainly in order. As always, if in doubt
 	ask us on IRC before opening the ticket.
 
+..  (TODO: in the ws-size note above, mention which params to tweak)
+
 In your syslog it may all be joined into one single line, but if you
 can reproduce the crash, do so while running `varnishd` manually:
 
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 2cddd59..a407624 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -141,7 +141,7 @@ And finally, the true test of a brave heart::
 
 	sudo make install
 
-Varnish will now be installed in `/usr/local`. The ``varnishd` binary is in
+Varnish will now be installed in `/usr/local`. The ``varnishd`` binary is in
 `/usr/local/sbin/varnishd` and its default configuration will be
 `/usr/local/etc/varnish/default.vcl`.
 
diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst
index 9016d51..aa8c5be 100644
--- a/doc/sphinx/users-guide/command-line.rst
+++ b/doc/sphinx/users-guide/command-line.rst
@@ -16,7 +16,7 @@ you will find the startup options here:
 
 
 '-a' *listen_address*
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^
 
 The '-a' argument defines what address Varnish should listen to, and service HTTP requests from.
 
@@ -42,7 +42,7 @@ If your webserver runs on the same machine, you will have to move
 it to another port number first.
 
 '-f' *VCL-file* or '-b' *backend*
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 
 Varnish needs to know where to find the HTTP server it is caching for.
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 45f8814..5c7ce72 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -82,7 +82,7 @@ For example::
   -->
 
 Doing ESI on JSON and other non-XML'ish content
-----------------------------------------------
+-----------------------------------------------
 
 Please note that Varnish will peek at the included content. If it
 doesn't start with a "<" Varnish assumes you didn't really mean to
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 3d4d831..bbc7d6a 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -40,9 +40,8 @@ from a transactional level to aggregating statistics.
    vcl
    report
    performance
-   esi   
+   esi
    troubleshooting
-   orphans
 
 .. customizing (which is a non ideal title)
 
diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index d7340a0..7607cfc 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -168,8 +168,6 @@ Furthermore you may want to look at and lock down:
 
 :ref:`ref_param_syslog_cli_traffic`
 	Log all CLI commands to `syslog(8)`, so you know what goes on.
-.. XXX: syslog(8)? benc
-
 
 :ref:`ref_param_vcc_unsafe_path`
 	Restrict VCL/VMODS to :ref:`ref_param_vcl_dir` and :ref:`ref_param_vmod_dir`
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index 9d57744..d05b9d6 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -91,10 +91,8 @@ A core dumped is usually due to a bug in Varnish. However, in order to
 debug a segfault the developers need you to provide a fair bit of
 data.
 
- * Make sure you have Varnish installed with symbols
-.. XXX:Symbols? benc
- * Make sure core dumps are enabled (ulimit)
-.. XXX:ulimit? benc
+ * Make sure you have Varnish installed with debugging symbols.
+ * Make sure core dumps are allowed in the parent shell. (``ulimit -c unlimited``)
 
 Once you have the core you open it with `gdb` and issue the command ``bt``
 to get a stack trace of the thread that caused the segfault.
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 4615291..f0b427f 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -82,20 +82,20 @@ down for, uhm, examples.
 Subroutines
 ~~~~~~~~~~~
 
-A subroutine is used to group code for legibility or reusability:
-::
-  
+A subroutine is used to group code for legibility or reusability::
+
   sub pipe_if_local {
     if (client.ip ~ local) {
       return (pipe);
     }
   }
 
+
 Subroutines in VCL do not take arguments, nor do they return values.
 
 To call a subroutine, use the call keyword followed by the subroutine's name::
 
-call pipe_if_local;
+  call pipe_if_local;
 
 Varnish has quite a few built in subroutines that are called for each
 transaction as it flows through Varnish. These builtin subroutines are all named vcl_*. Your own subroutines cannot start their name with vcl_.
diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst
index 88ab8f4..644c046 100644
--- a/doc/sphinx/users-guide/vcl-variables.rst
+++ b/doc/sphinx/users-guide/vcl-variables.rst
@@ -1,33 +1,35 @@
 
-Requests and responses as objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Request and response VCL objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. XXX: refactored headline. benc
 
-In VCL, there several important objects that you need to be aware of. These objects can be accessed and manipulated using VCL.
+In VCL, there several important objects that you need to be aware of. These
+objects can be accessed and manipulated using VCL.
 
 
 *req*
- The request object. When Varnish has received the request the `req` object is 
- created and populated. Most of the work you do in `vcl_recv` you 
+ The request object. When Varnish has received the request the `req` object is
+ created and populated. Most of the work you do in `vcl_recv` you
  do on or with the `req` object.
 
 *bereq*
- The backend request object. Varnish contructs this before sending it to the 
+ The backend request object. Varnish contructs this before sending it to the
  backend. It is based on the `req` object.
 
 .. XXX:in what way? benc
 
 *beresp*
- The backend response object. It contains the headers of the object 
- coming from the backend. If you want to modify the reponse coming from the 
- server you modify this object in `vcl_backend_reponse`. 
+ The backend response object. It contains the headers of the object
+ coming from the backend. If you want to modify the response coming from the
+ server you modify this object in `vcl_backend_reponse`.
 
 *resp*
  The HTTP response right before it is delivered to the client. It is
  typically modified in `vcl_deliver`.
 
 *obj* 
- The object as it is stored in cache. Mostly read only.
+ The object as it is stored in cache. Read only.
+
 .. XXX:What object? the current request? benc
 



From lkarsten at varnish-software.com  Thu Mar 20 12:13:42 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 13:13:42 +0100
Subject: [master] 8c56d06 Old habits die hard.
Message-ID: <E1WQbqw-0007Pt-N5@project.varnish-software.com>


commit 8c56d063c1dfd5dbdafdeae26416f0bb4bce5288
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 13:11:55 2014 +0100

    Old habits die hard.
    
    req.request is now called req.method.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 71e01f7..f9d3527 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -57,7 +57,7 @@ sub vcl_recv {
     }
 
     /* We don't support chunked uploads, except when piping. */
-    if ((req.request == "POST" || req.request == "PUT") &&
+    if ((req.method == "POST" || req.method == "PUT") &&
       req.http.transfer-encoding ~ "chunked") {
         return(pipe);
     }



From lkarsten at varnish-software.com  Thu Mar 20 12:30:25 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 13:30:25 +0100
Subject: [master] 5e3e98f Correct article
Message-ID: <E1WQc77-0000E3-CR@project.varnish-software.com>


commit 5e3e98f054309e34ccbef67bd030acf22369f388
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 13:30:22 2014 +0100

    Correct article

diff --git a/doc/sphinx/glossary/index.rst b/doc/sphinx/glossary/index.rst
index 3da5af3..574156b 100644
--- a/doc/sphinx/glossary/index.rst
+++ b/doc/sphinx/glossary/index.rst
@@ -34,11 +34,11 @@ Varnish Glossary
    backend
 	The HTTP server varnishd is caching for.  This can be
 	any sort of device that handles HTTP requests, including, but
-	not limited to: a webserver, a CMS, a load-balancer 
+	not limited to: a webserver, a CMS, a load-balancer
 	another varnishd, etc.
 
    client
-	The program which sends varnishd a HTTP request, typically
+	The program which sends varnishd an HTTP request, typically
 	a browser, but do not forget to think about spiders, robots
 	script-kiddies and criminals.
 
@@ -65,13 +65,13 @@ Varnish Glossary
    .. comment: "components of traffic ---------------------------------"
 
    header
-	A HTTP protocol header, like "Accept-Encoding:".
+	An HTTP protocol header, like "Accept-Encoding:".
 
    request
 	What the client sends to varnishd and varnishd sends to the backend.
 
    response
-	What the backend returns to varnishd and varnishd returns to 
+	What the backend returns to varnishd and varnishd returns to
 	the client.  When the response is stored in varnishd's cache,
 	we call it an object.
 
diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 2f22e3c..4be147f 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -48,7 +48,7 @@ Tool: lwp-request
 ~~~~~~~~~~~~~~~~~
 
 `lwp-request` is tool that is a part of The World-Wide Web library for Perl. It's a
-couple of really basic programs that can execute a HTTP request and
+couple of really basic programs that can execute an HTTP request and
 show you the result. We mostly use the two programs, ``GET`` and ``HEAD``.
 
 vg.no was the first site to use Varnish and the people running Varnish
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 214fe81..c0e8381 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -21,7 +21,7 @@ A *purge* is what happens when you pick out an object from the cache
 and discard it along with its variants. Usually a purge is invoked
 through HTTP with the method `PURGE`.
 
-A HTTP purge is similar to a HTTP GET request, except that the
+An HTTP purge is similar to an HTTP GET request, except that the
 *method* is `PURGE`. Actually you can call the method whatever you'd
 like, but most people refer to this as purging. Squid, for example, supports the
 same mechanism. In order to support purging in Varnish you need the
diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst
index 3e75ac5..40c6b4e 100644
--- a/doc/sphinx/users-guide/run_cli.rst
+++ b/doc/sphinx/users-guide/run_cli.rst
@@ -116,7 +116,7 @@ but they are banned from delivery.
 
 Instead of checking each and every cached object right away, we
 test each object against the regular expression only if and when
-a HTTP request asks for it.
+an HTTP request asks for it.
 
 Banning stuff is much cheaper than restarting Varnish to get rid
 of wronly cached content.
diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index 7607cfc..5fe3890 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -6,7 +6,7 @@ Security first
 If you are the only person involved in running Varnish, or if all
 the people involved are trusted to the same degree, you can skip
 this chapter. We have protected Varnish as well as we can from
-anything which can come in through a HTTP socket.
+anything which can come in through an HTTP socket.
 
 If parts of your web infrastructure are outsourced or otherwise
 partitioned along administrative lines, you need to think about



From lkarsten at varnish-software.com  Thu Mar 20 14:02:35 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 15:02:35 +0100
Subject: [master] bb8e863 Reindent for spaces, explain obj changes.
Message-ID: <E1WQdYJ-0001Ib-UM@project.varnish-software.com>


commit bb8e863d89f0a8c19e245f41143f187cb91bd921
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 15:02:08 2014 +0100

    Reindent for spaces, explain obj changes.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 2921835..02e6f02 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -25,15 +25,14 @@ Use the hash director as a client director
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Since the client director was already a special case of the hash director, it has been removed, and you should use the hash director directly::
 
-	sub vcl_init {
-        	new h = directors.hash();
-        	h.add_backend(b1, 1);
-        	h.add_backend(b2, 1);
-	}
-
-	sub vcl_recv {
-		set req.backend_hint = h.backend(client.ip);
-	}
+    sub vcl_init {
+        new h = directors.hash();
+        h.add_backend(b1, 1);
+        h.add_backend(b2, 1);
+    }
+    sub vcl_recv {
+        set req.backend_hint = h.backend(client.ip);
+    }
 
 error() is now a return value
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -85,6 +84,12 @@ Any custom-made subs cannot be named 'vcl_*' anymore. This namespace is reserved
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+obj is now read-only
+~~~~~~~~~~~~~~~~~~~~
+
+`obj` is now read-only. `obj.hits`, if enabled in VCL, now counts per objecthead,
+not per object. `obj.last_use` has been retired.
+
 Changes to parameters
 =====================
 



From lkarsten at varnish-software.com  Thu Mar 20 14:34:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 15:34:15 +0100
Subject: [master] c8add6f Add a conventions section.
Message-ID: <E1WQe2x-0005vg-N7@project.varnish-software.com>


commit c8add6f7d95d49846c23d5c21c31c6504fcbb092
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 15:29:52 2014 +0100

    Add a conventions section.
    
    Try to document our conventions when it comes to commands, paths, tool names
    and similar.
    
    Also write some formatting rules for maintainers, like wrapping at 80
    characters and avoiding tabs.
    
    I'm happy to discuss changing these if someone feels strongly about anything,
    the main point is just to make it easier to read though being consequent about
    it throughout the manual.

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 830149a..11b4cce 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -1,36 +1,75 @@
+Welcome to the Varnish documentation!
+=====================================
+
+Introduction
+------------
+
+Varnish Cache is a web application accelerator also known as a caching HTTP
+reverse proxy. You install it in front of any server that speaks HTTP and
+configure it to cache the contents. Varnish Cache is really, really fast. It
+typically speeds up delivery with a factor of 300 - 1000x, depending on your
+architecture.
+
+
+To get started with Varnish-Cache we recommend that you read the installation
+guide :ref:`install-index`. Once you have Varnish up and running we recommend
+that you go through our tutorial - :ref:`tutorial-index`, and finally the
+:ref:`users_guide_index`.
+
+If you need to find out how to use a specific Varnish tool, the
+:ref:`reference-index` contains detailed documentation over the tools. Changes
+from previous versions are located in the :ref:`whats-new-index` chapter. In
+closing, we have :ref:`phk`, a collection of blog posts from Poul-Henning Kamp
+related to Varnish and HTTP.
+
+
+Conventions used
+----------------
+
+Conventions used in this manual include:
+
+  ``service varnish restart``
+    A command you can run, or a shortkey you can press. Used either in the
+    terminal or after starting one of the tools.
+
+  `/usr/local/`, `varnishadm`, `sess_timeout`
+    A utility, Varnish configurable parameter or path.
+
+  http://www.varnish-cache.org/
+    A hyperlink.
+
+Longer listings like example command output and VCL will look like this::
+
+    $ /opt/varnish/sbin/varnishd -V
+    varnishd (varnish-4.0.0-tp1 revision ddd00e1)
+    Copyright (c) 2006 Verdens Gang AS
+    Copyright (c) 2006-2011 Varnish Software AS
+
+
+.. For maintainers:
+.. * always write Varnish with a capital V: Varnish, Varnish Cache.
+.. * Write Varnish tools as their executable name: `varnishd`, `varnishadm`.
+.. * if part of a command actually runable by the reader, use double backticks:
+..   ``varnishd -f foo.c``
+.. * wrap lines at 80 characters, ident with 4 spaces. No tabs, please.
 .. We use the following header indicators
 .. For titles:
 
 .. H1
 .. %%%%%
+
 .. Title
 .. %%%%%
 
 .. H2 - H5
-.. ====================== 
+.. ======================
 .. ----------------------
 .. ~~~~~~~~~~~~~~~~~~~~~~
-.. ...................... 
+.. ......................
 
-Welcome to the Varnish documentation!
-=====================================
-
-Introduction
-------------
-
-Varnish Cache is a web application accelerator also known as a caching HTTP reverse proxy. You install it in front of any server that speaks HTTP and configure it to cache the contents. Varnish Cache is really, really fast. It typically speeds up delivery with a factor of 300 - 1000x, depending on your architecture. 
-It has its mission in front of a
-web server and cache content and it makes your web site go faster.
-
-To get started with Varnish-Cache we recommend that you read the installation guide
-:ref:`install-index`. Once you have Varnish up and running we recommend that you go through
-our tutorial - :ref:`tutorial-index`, and finally the :ref:`users_guide_index`.
-
-If you need to find out how to use a specific Varnish tool, the
-:ref:`reference-index` contains detailed documentation over the tools. Changes from previous versions are located in
-the :ref:`whats-new-index` chapter. In closing, we have :ref:`phk`, a collection
-of blog posts from Poul-Henning Kamp related to Varnish and HTTP.
 
+Table of contents
+-----------------
 
 
 .. toctree::



From lkarsten at varnish-software.com  Thu Mar 20 15:06:49 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 16:06:49 +0100
Subject: [master] b8f32b9 Squash more warnings
Message-ID: <E1WQeYT-0002Bc-2N@project.varnish-software.com>


commit b8f32b911ef11e626ceb116851e66ea8a76a2367
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 15:51:24 2014 +0100

    Squash more warnings

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 11b4cce..db8b5a5 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -14,7 +14,7 @@ architecture.
 To get started with Varnish-Cache we recommend that you read the installation
 guide :ref:`install-index`. Once you have Varnish up and running we recommend
 that you go through our tutorial - :ref:`tutorial-index`, and finally the
-:ref:`users_guide_index`.
+:ref:`users-guide-index`.
 
 If you need to find out how to use a specific Varnish tool, the
 :ref:`reference-index` contains detailed documentation over the tools. Changes
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index 5bb9f6e..8143abd 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishadm:
+
 ==========
 varnishadm
 ==========
diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst
index 8071384..685aebe 100644
--- a/doc/sphinx/reference/varnishhist.rst
+++ b/doc/sphinx/reference/varnishhist.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishhist:
+
 ===========
 varnishhist
 ===========
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 2902f6b..77086ac 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishncsa:
+
 ===========
 varnishncsa
 ===========
diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index 3505526..4178778 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -1,4 +1,4 @@
-.. _reference-varnishstat:
+.. _ref-varnishstat:
 
 ===========
 varnishstat
@@ -13,6 +13,8 @@ SYNOPSIS
 
 varnishstat [-1] [-x] [-j] [-f field_list] [-l] [-n varnish_name] [-N filename] [-V] [-w delay]
 
+.. TODO: autogenerate this synopsis like the others.
+
 DESCRIPTION
 ===========
 
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index a3a0eec..bfae5a5 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishtest:
+
 ===========
 varnishtest
 ===========
diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 4692143..e1abbcb 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishtop:
+
 ==========
 varnishtop
 ==========
diff --git a/doc/sphinx/users-guide/sizing-your-cache.rst b/doc/sphinx/users-guide/sizing-your-cache.rst
index 497ce0e..90621f8 100644
--- a/doc/sphinx/users-guide/sizing-your-cache.rst
+++ b/doc/sphinx/users-guide/sizing-your-cache.rst
@@ -2,26 +2,26 @@
 Sizing your cache
 -----------------
 
-Deciding on cache size can be a tricky
-task. A few things to consider:
+Deciding on cache size can be a tricky task. A few things to consider:
 
  * How big is your *hot* data set. For a portal or news site that
    would be the size of the front page with all the stuff on it, and
-   the size of all the pages and objects linked from the first page. 
+   the size of all the pages and objects linked from the first page.
+
  * How expensive is it to generate an object? Sometimes it makes sense
    to only cache images a little while or not to cache them at all if
    they are cheap to serve from the backend and you have a limited
    amount of memory.
- * Watch the `n_lru_nuked` counter with :ref:`reference-varnishstat` or
+
+ * Watch the `n_lru_nuked` counter with :ref:`ref-varnishstat` or
    some other tool. If you have a lot of LRU activity then your cache
    is evicting objects due to space constraints and you should
    consider increasing the size of the cache.
 
 Be aware that every object that is stored also carries overhead that
-is kept outside the actually storage area. So, even if you specify '-s
-malloc,16G' Varnish might actually use **double** that. Varnish has a
-overhead of about 1k per object. So, if you have lots of small objects
+is kept outside the actually storage area. So, even if you specify ``-s
+malloc,16G`` Varnish might actually use **double** that. Varnish has a
+overhead of about 1KB per object. So, if you have lots of small objects
 in your cache the overhead might be significant.
 
 .. XXX:This seems to contradict the last paragraph in "storage-backends". benc
-
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index c54d951..77347c4 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -180,34 +180,32 @@ define the backends.::
         }
     }
 
-Whats new here is the ``probe``. Varnish will check the health of each
+What is new here is the ``probe``. Varnish will check the health of each
 backend with a probe. The options are:
 
 url
     The URL Varnish will use to send a probe request.
 
 interval
-    How often should we poll
+    How often should we poll.
 
 timeout
-    What is the timeout of the probe
+    What is the timeout of the probe.
 
 window
-    Varnish will maintain a *sliding window* of the results. Here the
-    window has five checks.
+    Varnish will retain up to this many probes when considering backend health.
 
 threshold
-    How many of the '.window' last polls must be good for the backend to be declared healthy.
-
-.. XXX: .window probably means something but not to me :) benc
+    How many of the `.window` last polls must be good for the backend to be
+    declared healthy.
 
 initial
-    How many of the probes that needs to be succesful when Varnish starts - defaults
-    to the same amount as the threshold.
+    How many of the probes that needs to be succesful when Varnish starts.
+    Defaults to the same amount as the threshold.
 
-Now we define the 'director'.::
 
 .. XXX: Where and why? benc
+Now we define the 'director'::
 
     import directors;
 
@@ -222,10 +220,10 @@ unhealthy. Varnish can also serve stale content if all the backends are
 down. See :ref:`users-guide-handling_misbehaving_servers` for more
 information on how to enable this.
 
-Please note that Varnish will keep probes active for all loaded
-VCLs. Varnish will coalesce probes that seem identical - so be careful
-not to change the probe config if you do a lot of VCL
-loading. Unloading the VCL will discard the probes.
+Please note that Varnish will keep probes active for all loaded VCLs. Varnish
+will coalesce probes that seem identical - so be careful not to change the
+probe config if you do a lot of VCL loading. Unloading the VCL will discard the
+probes.
 
 For more information on how to do this please see
 ref:`reference-vcl-director`.
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index 8585241..86ed8cf 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -42,8 +42,5 @@ code commented out in the file `builtin.vcl` that ships with Varnish Cache.
    vcl-grace
    vcl-inline-c
    vcl-examples
-   websockets
-.. XXX: websockets seems to be missing? does it refer to the last sample in the vcl index if so already included. benc
-
    devicedetection
 



From lkarsten at varnish-software.com  Thu Mar 20 15:06:49 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 16:06:49 +0100
Subject: [master] 1309012 Skip generated files, they are included manually.
Message-ID: <E1WQeYT-0002Bj-3Z@project.varnish-software.com>


commit 1309012f56d31b29b40961e7183cb4916001cfc2
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 15:52:56 2014 +0100

    Skip generated files, they are included manually.
    
    We include these autogenerated files directly where they are needed.
    
    Let the main sphinx process ignore them, so it doesn't complain that they're
    not referenced anywhere.

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index f68a20f..8104bb3 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_patterns = ['build','reference/params.rst']
+exclude_patterns = ['build','include/*.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None



From lkarsten at varnish-software.com  Thu Mar 20 15:06:49 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 16:06:49 +0100
Subject: [master] 9f9d069 Squash more sphinx warnings
Message-ID: <E1WQeYT-0002Bs-4Y@project.varnish-software.com>


commit 9f9d0690b37962aa9188f23ef7204568a034b5f3
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 16:06:45 2014 +0100

    Squash more sphinx warnings

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 77347c4..a51e4f6 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -205,6 +205,7 @@ initial
 
 
 .. XXX: Where and why? benc
+
 Now we define the 'director'::
 
     import directors;
@@ -214,6 +215,7 @@ Now we define the 'director'::
         { .backend = server2; }
     }
 
+
 You use this director just as you would use any other director or
 backend. Varnish will not send traffic to hosts that are marked as
 unhealthy. Varnish can also serve stale content if all the backends are
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index f0b427f..d4acdd7 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -98,8 +98,7 @@ To call a subroutine, use the call keyword followed by the subroutine's name::
   call pipe_if_local;
 
 Varnish has quite a few built in subroutines that are called for each
-transaction as it flows through Varnish. These builtin subroutines are all named vcl_*. Your own subroutines cannot start their name with vcl_.
+transaction as it flows through Varnish. These builtin subroutines are all
+named `vcl_*`. Your own subroutines cannot start their name with `vcl\_`.
 
 .. XXX:looks as bit funky as red text? benc
-
-See :ref:`vcl-built-in-subs`.



From lkarsten at varnish-software.com  Thu Mar 20 15:35:53 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 16:35:53 +0100
Subject: [master] b109eed Formatting, typo
Message-ID: <E1WQf0b-0006gk-2E@project.varnish-software.com>


commit b109eed58c17ee0dadea7d82847945bf1137eb2f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 16:22:23 2014 +0100

    Formatting, typo

diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
index 2896394..969e07e 100644
--- a/doc/sphinx/tutorial/starting_varnish.rst
+++ b/doc/sphinx/tutorial/starting_varnish.rst
@@ -5,12 +5,13 @@ Starting Varnish
 ----------------
 
 This tutorial will assume that you are running Varnish on Ubuntu, Debian,
-Red Hat Enterprise Linux or Centos. Those of you running on other
+Red Hat Enterprise Linux or CentOS. Those of you running on other
 platforms might have to do some mental translation exercises in order
 to follow this. Since you're on a "weird" platform you're probably used
 to it. :-)
 
-Make sure you have Varnish succesfully installed (following one of the procedures described in "Installing Varnish" above.
+Make sure you have Varnish successfully installed (following one of the
+procedures described in "Installing Varnish" above.
 
 When properly installed you start Varnish with ``service varnish start``.  This will start
 Varnish if it isn't already running.
@@ -18,15 +19,16 @@ Varnish if it isn't already running.
 .. XXX:What does it do if it is already running? benc
 
 Now you have Varnish running. Let us make sure that it works
-properly. Use your browser to go to http://127.0.0.1:6081/
-(obviously, you should replace the IP address with the IP for the machine that currently runs Varnish. 
-The default configuration will try to forward requests to a web
-application running on the same machine as Varnish was installed on. Varnish expects the web application to be exposed over http on port 8080.
+properly. Use your browser to go to http://127.0.0.1:6081/ (Replace the IP
+address with the IP for the machine that runs Varnish) The default
+configuration will try to forward requests to a web application running on the
+same machine as Varnish was installed on. Varnish expects the web application
+to be exposed over http on port 8080.
 
 If there is no web application being served up on that location Varnish will
 issue an error. Varnish Cache is very conservative about telling the
 world what is wrong so whenever something is amiss it will issue the
-same generic "Error 503 Service Unavilable".
+same generic "Error 503 Service Unavailable".
 
 You might have a web application running on some other port or some
 other machine. Let's edit the configuration and make it point to
@@ -44,7 +46,7 @@ probably look like this::
   }
 
 We'll change it and make it point to something that works. Hopefully
-www.varnish-cache.org is up. Let's use that. Replace the text with::
+http://www.varnish-cache.org/ is up. Let's use that. Replace the text with::
 
   vcl 4.0;
   
@@ -58,6 +60,6 @@ Now issue ``service varnish reload`` to make Varnish reload it's
 configuration. If that succeeded visit http://127.0.0.1:6081/ in your
 browser and you should see some directory listing. It works! The
 reason you're not seeing the Varnish official website is because your
-client isn't sending the appropriate Host: header in the request and
+client isn't sending the appropriate `Host` header in the request and
 it ends up showing a listing of the default webfolder on the machine
-usually serving up varnish-cache.org.
+usually serving up http://www.varnish-cache.org/ .



From lkarsten at varnish-software.com  Thu Mar 20 15:35:53 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 16:35:53 +0100
Subject: [master] 31a4724 Formatting cleanup
Message-ID: <E1WQf0b-0006gq-3Y@project.varnish-software.com>


commit 31a472467f331836d82b60e3fd32a7bc6f3ac330
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 16:29:34 2014 +0100

    Formatting cleanup

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index a407624..f95f4b4 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -25,12 +25,12 @@ Binary package:
 From source:
 		``cd /usr/ports/varnish && make install clean``
 
-CentOS/RedHat
--------------
+Red Hat / CentOS
+----------------
 
 We try to keep the latest version available as prebuilt RPMs (el5 and el6)
 on `repo.varnish-cache.org <http://repo.varnish-cache.org/>`_.  See the online
-`RedHat installation instructions
+`Red Hat installation instructions
 <http://www.varnish-cache.org/installation/redhat>`_ for more information.
 
 Varnish is included in the `EPEL
@@ -92,10 +92,10 @@ If you're building from git, you also need the following:
 * `python-docutils`
 * `python-sphinx` (optional, if you want to build the documentation)
 
-Build dependencies on RedHat / CentOS
+Build dependencies on Red Hat / CentOS
 --------------------------------------
 
-To build Varnish on a RedHat or CentOS system you need the following
+To build Varnish on a Red Hat or CentOS system you need the following
 packages installed:
 
 * `automake`
diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
index 0fc2ddc..3e57917 100644
--- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
+++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
@@ -2,17 +2,12 @@
 Put Varnish on port 80
 ----------------------
 
-Until now we've been running with Varnish on a high port, which is
-great for testing purposes. Lets put Varnish on the default HTTP port.
+Until now we've been running with Varnish on a high port which is great for
+testing purposes. Let's now put Varnish on the default HTTP port 80.
 
-First we stop varnish::
-
-     # service varnish stop
-
-.. XXX:This renders to a different font than other commands. it should be the double backtick format for the command. benc
-
-Now we need to edit the configuration file that starts Varnish. 
+First we stop varnish: ``service varnish stop``
 
+Now we need to edit the configuration file that starts Varnish.
 
 Debian/Ubuntu
 ~~~~~~~~~~~~~
@@ -34,17 +29,17 @@ Change it to::
                -S /etc/varnish/secret \
                -s malloc,256m"
 
-Red Hat EL / Centos
-~~~~~~~~~~~~~~~~~~~
+Red Hat Enterprise Linux / CentOS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On Red Hat/CentOS you can find a similar configuration file in
+`/etc/sysconfig/varnish`.
 
-On Red Hat EL / Centos
-On Red Hat/Centos it is `/etc/sysconfig/varnish`
 
+Restarting Varnish again
+------------------------
 
-Restarting Varnish
-------------------
+Once the change is done, restart Varnish: ``service varnish start``.
 
-Once the change is done. Restart varnish: ``service varnish
-restart``. Now everyone accessing your site will be accessing through
-Varnish.
+Now everyone accessing your site will be accessing through Varnish.
 



From lkarsten at varnish-software.com  Thu Mar 20 15:35:53 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 16:35:53 +0100
Subject: [master] c510ea2 Endless reformatting
Message-ID: <E1WQf0b-0006gz-4u@project.varnish-software.com>


commit c510ea2194d136c3f8bed3d9b359d9707af0f7d3
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 16:35:27 2014 +0100

    Endless reformatting

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 98cfddc..2fbc43a 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -9,18 +9,20 @@ the request from the cache it will forward the request to the backend,
 fetch the response, store it in the cache and deliver it to the client.
 
 When Varnish has a cached response ready it is typically delivered in
-a matter of microseconds, several orders of magnitude faster than your
+a matter of microseconds, two orders of magnitude faster than your
 typical backend server, so you want to make sure to have Varnish answer
 as many of the requests as possible directly from the cache.
 
-Varnish decides whether it can store the content or not based on the
-response it gets back from the backend. The backend can instruct
-Varnish to cache the content with the HTTP response header
-`Cache-Control`. There are a few conditions where Varnish will not
-cache, the most common one being the use of cookies. Since cookies indicates a client-specific web object, Varnish will by default not cache it. 
+Varnish decides whether it can store the content or not based on the response
+it gets back from the backend. The backend can instruct Varnish to cache the
+content with the HTTP response header `Cache-Control`. There are a few
+conditions where Varnish will not cache, the most common one being the use of
+cookies. Since cookies indicates a client-specific web object, Varnish will by
+default not cache it.
+
 This behaviour as most of Varnish functionality can be changed using policies
-written in the Varnish Configuration Language. See the Users Guide
-for more information on how to do that.
+written in the Varnish Configuration Language (VCL). See
+:ref:`users-guide-index` for more information on how to do that.
 
 Performance
 ~~~~~~~~~~~
diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
index 969e07e..8d9b64c 100644
--- a/doc/sphinx/tutorial/starting_varnish.rst
+++ b/doc/sphinx/tutorial/starting_varnish.rst
@@ -13,8 +13,8 @@ to it. :-)
 Make sure you have Varnish successfully installed (following one of the
 procedures described in "Installing Varnish" above.
 
-When properly installed you start Varnish with ``service varnish start``.  This will start
-Varnish if it isn't already running.
+When properly installed you start Varnish with ``service varnish start``.  This
+will start Varnish if it isn't already running.
 
 .. XXX:What does it do if it is already running? benc
 



From lkarsten at varnish-software.com  Thu Mar 20 16:37:14 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 17:37:14 +0100
Subject: [master] af74f47 WIP, redoing structure
Message-ID: <E1WQfxy-0000A4-VM@project.varnish-software.com>


commit af74f47154126408ad463ed9b004ab9302566b7f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 17:18:32 2014 +0100

    WIP, redoing structure

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index f95f4b4..a8ef31c 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -3,11 +3,13 @@
 Installing Varnish
 ==================
 
-With open source software, you can choose to install binary packages
-or compile it yourself from source code. To install a package or compile
-from source is a matter of personal taste. If you don't know which
-method to choose, we recommend that you read this whole section and then choose the method you
-feel most comfortable with.
+.. no section heading here.
+
+With open source software, you can choose to install binary packages or compile
+it yourself from source code. To install a package or compile from source is a
+matter of personal taste. If you don't know which method to choose, we
+recommend that you read this whole section and then choose the method you feel
+most comfortable with.
 
 
 Source or packages?
@@ -60,15 +62,13 @@ want to compile Varnish from source for other reasons, follow these
 steps:
 
 Download the appropriate release tarball, which you can find on
-`repo.varnish-cache.org/source <http://repo.varnish-cache.org/source/>`_.
+http://repo.varnish-cache.org/source/ .
 
 Alternatively, if you want to hack on Varnish, you should clone our
 git repository by doing.
 
       ``git clone git://git.varnish-cache.org/varnish-cache``
 
-Please note that a git checkout will need some more build-dependencies
-than listed below, in particular the `Python Docutis` and `Sphinx`.
 
 Build dependencies on Debian / Ubuntu
 --------------------------------------
@@ -120,8 +120,9 @@ taken care of::
 	sh configure
 	make
 
-The `configure` script takes some arguments, but more likely than
-not you can forget about that for now, almost everything in Varnish can be tweaked with run time parameters.
+The `configure` script takes some arguments, but more likely than not you can
+forget about that for now, almost everything in Varnish can be tweaked with run
+time parameters.
 
 Before you install, you may want to run the test suite, make a cup of
 tea while it runs, it usually takes a couple of minutes::
@@ -139,9 +140,9 @@ Installing
 
 And finally, the true test of a brave heart::
 
-	sudo make install
+And finally, the true test of a brave heart: ``sudo make install``
 
-Varnish will now be installed in `/usr/local`. The ``varnishd`` binary is in
+Varnish will now be installed in `/usr/local`. The `varnishd` binary is in
 `/usr/local/sbin/varnishd` and its default configuration will be
 `/usr/local/etc/varnish/default.vcl`.
 



From lkarsten at varnish-software.com  Thu Mar 20 16:37:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 17:37:15 +0100
Subject: [master] cfb7fd2 C-c: nocache is honored in 4.0
Message-ID: <E1WQfxz-0000AA-0U@project.varnish-software.com>


commit cfb7fd2a30a8b1bb7931ec25ab213d8179ec00d0
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 17:35:02 2014 +0100

    C-c: nocache is honored in 4.0

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 4be147f..886659e 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -222,10 +222,7 @@ Cache-Control
 
 The 'Cache-Control' header instructs caches how to handle the content. Varnish
 cares about the *max-age* parameter and uses it to calculate the TTL
-for an object. 
-
-``Cache-Control: nocache`` is ignored but if you need this you can
-easily add support for it.
+for an object.
 
 So make sure you issue a 'Cache-Control' header with a max-age
 header. You can have a look at what Varnish Software's Drupal server



From lkarsten at varnish-software.com  Thu Mar 20 16:37:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 17:37:15 +0100
Subject: [master] 9d5e807 Formatting
Message-ID: <E1WQfxz-0000AJ-1r@project.varnish-software.com>


commit 9d5e80703499aa0d175b937983ba0a3fa9fa1cd1
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 17:35:13 2014 +0100

    Formatting

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 886659e..9d8e0af 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -235,9 +235,7 @@ Age
 ~~~
 
 Varnish adds an 'Age' header to indicate how long the object has been
-kept inside Varnish. You can grep out 'Age' from `varnishlog` like this::
-
-  varnishlog -i TxHeader -I ^Age
+kept inside Varnish. You can grep out 'Age' from `varnishlog` with ``varnishlog -i TxHeader -I ^Age``.
 
 Pragma
 ~~~~~~
@@ -247,10 +245,10 @@ header. You could easily add support for this header in VCL.
 
 In `vcl_backend_response`::
 
-  if (beresp.http.Pragma ~ "nocache") {
+    if (beresp.http.Pragma ~ "nocache") {
         set beresp.uncacheable = true;
-	set beresp.ttl = 120s; // how long not to cache this url.
-  }
+        set beresp.ttl = 120s; // how long not to cache this url.
+    }
 
 Authorization
 ~~~~~~~~~~~~~
@@ -258,12 +256,12 @@ Authorization
 If Varnish sees an 'Authorization' header it will pass the request. If
 this is not what you want you can unset the header.
 
-Overriding the time-to-live (ttl)
+Overriding the time-to-live (TTL)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Sometimes your backend will misbehave. It might, depending on your
-setup, be easier to override the ttl in Varnish than to fix your
-somewhat cumbersome backend. 
+setup, be easier to override the TTL in Varnish then to fix your
+somewhat cumbersome backend.
 
 You need VCL to identify the objects you want and then you set the
 'beresp.ttl' to whatever you want::
@@ -274,7 +272,7 @@ You need VCL to identify the objects you want and then you set the
       }
   }
 
-This example will set the ttl to 5 days for the old legacy stuff on
+This example will set the TTL to 5 days for the old legacy stuff on
 your site.
 
 Forcing caching for certain requests and certain responses



From lkarsten at varnish-software.com  Thu Mar 20 16:37:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 17:37:15 +0100
Subject: [master] 0ab1908 Redo heading structure a bit
Message-ID: <E1WQfxz-0000AU-36@project.varnish-software.com>


commit 0ab19082d27e4ce559c4d17c8daf85dc0b2658c2
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 17:36:00 2014 +0100

    Redo heading structure a bit

diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst
index 03e60b8..f758d84 100644
--- a/doc/sphinx/installation/index.rst
+++ b/doc/sphinx/installation/index.rst
@@ -1,8 +1,7 @@
 .. _install-index:
 
-%%%%%%%%%%%%%%%%%%%%
 Varnish Installation
-%%%%%%%%%%%%%%%%%%%%
+====================
 
 This section covers installation prerequisites, a step-by-step installation procedure, how and where
 to get help, and how to report bugs. It also contains a set of platform specific notes to aid you when installing Varnish on certain platforms.
diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 58da708..7eac4e5 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -1,8 +1,7 @@
 .. _phk:
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 Poul-Hennings random outbursts
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+==============================
 
 You may or may not want to know what Poul-Henning thinks.
 
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 44008e7..edac3c9 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -1,8 +1,7 @@
 .. _tutorial-index:
 
-%%%%%%%%%%%%%%%%%%%%
 The Varnish Tutorial
-%%%%%%%%%%%%%%%%%%%%
+====================
 
 This section covers the Varnish basics in a tutorial form. It will cover what Varnish is and how it
 works. It also covers how to get Varnish up and running. After this section you probably would want to
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index bbc7d6a..e1c3e83 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -1,8 +1,7 @@
 .. _users-guide-index:
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 The Varnish Users Guide
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+=======================
 
 The Varnish documentation consists of three main documents:
 



From lkarsten at varnish-software.com  Thu Mar 20 16:37:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 17:37:15 +0100
Subject: [master] 830151a Adhere to style manual
Message-ID: <E1WQfxz-0000Ad-4H@project.varnish-software.com>


commit 830151a85ef4a65b9278c6005c6834257151aa21
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 17:36:28 2014 +0100

    Adhere to style manual

diff --git a/doc/sphinx/installation/platformnotes.rst b/doc/sphinx/installation/platformnotes.rst
index 7f11eaa..0eca96f 100644
--- a/doc/sphinx/installation/platformnotes.rst
+++ b/doc/sphinx/installation/platformnotes.rst
@@ -12,9 +12,10 @@ Transparent hugepages on Redhat Enterprise Linux 6
 On RHEL6 Transparent Hugepage kernel support is enabled by default.
 This is known to cause sporadic crashes of Varnish.
 
-It is recommended to disable transparent hugepages on affected systems::
-
-    $ echo "never" > /sys/kernel/mm/redhat_transparent_hugepage/enabled
+It is recommended to disable transparent hugepages on affected systems. This
+can be done with 
+``echo "never" > /sys/kernel/mm/redhat_transparent_hugepage/enabled`` (runtime) and changes to
+`/etc/sysctl.conf` (persisted.)
 
 On Debian/Ubuntu systems running 3.2 kernels the default value is "madvise" and
 does not need to be changed.
@@ -34,11 +35,8 @@ the maximum stack size before starting Varnish.
 The default allocates too much memory per thread, which will make Varnish fail
 as soon as the number of threads (traffic) increases.
 
-Reduce the maximum stack size by running::
-
-    ulimit -s 256
-
-in the Varnish startup script.
+Reduce the maximum stack size by adding ``ulimit -s 256`` before starting
+Varnish in the init script.
 
 TCP keep-alive configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/sphinx/installation/prerequisites.rst b/doc/sphinx/installation/prerequisites.rst
index 7ef0bd7..05aea31 100644
--- a/doc/sphinx/installation/prerequisites.rst
+++ b/doc/sphinx/installation/prerequisites.rst
@@ -1,7 +1,6 @@
 Prerequisites
 =============
 
-
 In order for you to install Varnish you must have the following:
 
   * A recent, preferably server grade, computer.



From lkarsten at varnish-software.com  Thu Mar 20 16:37:15 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 20 Mar 2014 17:37:15 +0100
Subject: [master] c4ee24f TOC looks OK in HTML and latex now
Message-ID: <E1WQfxz-0000Am-5k@project.varnish-software.com>


commit c4ee24f8b09750cad8975af67ee04b695ae420c5
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 20 17:37:10 2014 +0100

    TOC looks OK in HTML and latex now

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index db8b5a5..b87a1a6 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -1,8 +1,6 @@
-Welcome to the Varnish documentation!
-=====================================
 
-Introduction
-------------
+Varnish Administrator Documentation
+===================================
 
 Varnish Cache is a web application accelerator also known as a caching HTTP
 reverse proxy. You install it in front of any server that speaks HTTP and
@@ -10,7 +8,6 @@ configure it to cache the contents. Varnish Cache is really, really fast. It
 typically speeds up delivery with a factor of 300 - 1000x, depending on your
 architecture.
 
-
 To get started with Varnish-Cache we recommend that you read the installation
 guide :ref:`install-index`. Once you have Varnish up and running we recommend
 that you go through our tutorial - :ref:`tutorial-index`, and finally the
@@ -23,9 +20,6 @@ closing, we have :ref:`phk`, a collection of blog posts from Poul-Henning Kamp
 related to Varnish and HTTP.
 
 
-Conventions used
-----------------
-
 Conventions used in this manual include:
 
   ``service varnish restart``
@@ -68,23 +62,21 @@ Longer listings like example command output and VCL will look like this::
 .. ......................
 
 
-Table of contents
------------------
+.. toctree::
+    :maxdepth: 1
 
+    installation/index.rst
+    tutorial/index.rst
+    users-guide/index.rst
+    reference/index.rst
+    whats-new/index.rst
+    phk/index.rst
+    glossary/index.rst
 
-.. toctree::
-   :maxdepth: 1
 
-   installation/index.rst
-   tutorial/index.rst
-   users-guide/index.rst
-   reference/index.rst
-   whats-new/index.rst
-   phk/index.rst
-   glossary/index.rst
 
 Indices and tables
-==================
+------------------
 
 * :ref:`genindex`
 * :ref:`search`



From tfheen at err.no  Thu Mar 20 19:09:59 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 20 Mar 2014 20:09:59 +0100
Subject: [master] 02e38f7 Fix two small resource leaks to make coverity happy
Message-ID: <E1WQiLn-0007Zo-M0@project.varnish-software.com>


commit 02e38f7f26e35a23faf4356ab6b7d3bae264cffa
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Thu Mar 20 20:05:41 2014 +0100

    Fix two small resource leaks to make coverity happy

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index e1686ad..90016a9 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -128,6 +128,7 @@ cli_sock(const char *T_arg, const char *S_arg)
 	if (status != CLIS_OK) {
 		fprintf(stderr, "Rejected %u\n%s\n", status, answer);
 		AZ(close(sock));
+		free(answer);
 		return (-1);
 	}
 	free(answer);
@@ -137,7 +138,8 @@ cli_sock(const char *T_arg, const char *S_arg)
 	if (status != CLIS_OK || strstr(answer, "PONG") == NULL) {
 		fprintf(stderr, "No pong received from server\n");
 		AZ(close(sock));
-		return(-1);
+		free(answer);
+		return (-1);
 	}
 	free(answer);
 



From tfheen at err.no  Thu Mar 20 19:09:59 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 20 Mar 2014 20:09:59 +0100
Subject: [master] 597f4e9 Free topbuild, makes coverity happy
Message-ID: <E1WQiLn-0007Zu-N5@project.varnish-software.com>


commit 597f4e909b85a0ff7a988f6646fc3afe76d0e603
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Thu Mar 20 20:09:26 2014 +0100

    Free topbuild, makes coverity happy

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 3fc2b43..fc3b588 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -378,7 +378,7 @@ i_mode(void)
 	}
 	AN(topbuild);
 	extmacro_def("topbuild", "%s", topbuild);
-
+	free(topbuild);
 	/*
 	 * Build $PATH which can find all programs in the build tree
 	 */



From tfheen at err.no  Fri Mar 21 07:20:35 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Fri, 21 Mar 2014 08:20:35 +0100
Subject: [master] 7b48fae Move freeing of topbuild later to prevent
 use-after-free
Message-ID: <E1WQtkp-0006b1-KS@project.varnish-software.com>


commit 7b48faecf3c71d1ed8f630878425f0e1f1d4db92
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Fri Mar 21 08:12:25 2014 +0100

    Move freeing of topbuild later to prevent use-after-free

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index fc3b588..de08003 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -378,7 +378,6 @@ i_mode(void)
 	}
 	AN(topbuild);
 	extmacro_def("topbuild", "%s", topbuild);
-	free(topbuild);
 	/*
 	 * Build $PATH which can find all programs in the build tree
 	 */
@@ -411,6 +410,7 @@ i_mode(void)
 	} while (0);
 #include "vmods.h"
 #undef VTC_VMOD
+	free(topbuild);
 	VSB_delete(vsb);
 }
 



From lkarsten at varnish-software.com  Fri Mar 21 13:57:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 21 Mar 2014 14:57:31 +0100
Subject: [master] 8f176df Use system Python3 interpreter
Message-ID: <E1WQzwx-0002sL-T4@project.varnish-software.com>


commit 8f176df35efa4f1e04f4866ec0543719c4aeb429
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 21 14:18:34 2014 +0100

    Use system Python3 interpreter
    
    I don't have Python 3.1, but I do have Python 3.3.5 which should suffice.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 7c7e9db..84e363d 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3.1
+#!/usr/bin/env python3
 #-
 # Copyright (c) 2006 Verdens Gang AS
 # Copyright (c) 2006-2014 Varnish Software AS



From lkarsten at varnish-software.com  Fri Mar 21 13:57:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 21 Mar 2014 14:57:31 +0100
Subject: [master] 2ebf017 Clean up for 4.0 syntax.
Message-ID: <E1WQzwx-0002sR-UA@project.varnish-software.com>


commit 2ebf017d5af8104e59b8876f3b3bf720c44da4bb
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 21 14:43:48 2014 +0100

    Clean up for 4.0 syntax.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 942d1ac..c71577b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -12,36 +12,19 @@ DESCRIPTION
 ===========
 
 The VCL language is a small domain-specific language designed to be
-used to define request handling and document caching policies for
+used to describe request handling and document caching policies for
 Varnish Cache.
 
 When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
-is then dynamically linked into the server process.
+is then loaded into the server process.
 
 This document focuses on the syntax of the VCL language. For a full
 description of syntax and semantics, with ample examples, please see
-the users guide at https://www.varnish-cache.org/docs/
-
-VCL consists of the following elements:
- * Operators
- * Conditionals
- * Strings, booleans, time, duration, ints
- * Regular expressions
-
-In addition VCL has the following constructs:
- * Include
- * Backend definitions
- * Probes
- * Access control lists - ACLs
- * Import statement
- * Functions
- * Subroutines
+the online documentation at https://www.varnish-cache.org/docs/ .
 
-Note that are no loops or iterators of any kind in VCL.
-
-Each VCL file must start by declaring its version. For Varnish 4.0 you
-must include a "vcl 4.0;" at the top of the file.
+Starting with Varnish 4.0, each VCL file must start by declaring its version
+with a special "vcl 4.0;" marker at the top of the file.
 
 
 Operators
@@ -62,35 +45,39 @@ The following operators are available in VCL:
     Negation.
 
   &&
-    Logical and
+    Logical and.
 
   ||
-    Logical or
+    Logical or.
 
 
 Conditionals
 ------------
 
-VCL has *if* statements.
+VCL has *if* and *else* statements. Nested logic can be implemented
+with the *elseif* statement. (*elsif*/*elif*/*else if* is equivalent.)
 
+Note that are no loops or iterators of any kind in VCL.
 
-Strings, booleans, time, duration and ints
-------------------------------------------
+
+Strings, booleans, time, duration and integers
+----------------------------------------------
 
 These are the data types in Varnish. You can *set* or *unset* these.
 
 Example::
 
-  set req.http.user-agent = "unknown";
+  set req.http.User-Agent = "unknown";
+  unset req.http.Range;
 
 
 Strings
 ~~~~~~~
 
-Basic strings are enclosed in " ... ", and may not contain
+Basic strings are enclosed in double quotes (" ... "), and may not contain
 newlines. Long strings are enclosed in {" ... "}. They may contain any
-character including ", newline and other control characters except for
-the NUL (0x00) character
+character including single double quotes ("), newline and other control
+characters except for the NUL (0x00) character.
 
 Booleans
 ~~~~~~~~
@@ -107,7 +94,8 @@ formatted string.
 Durations
 ---------
 
-Durations are defined by a number and a designation. The number can be a real so 1.5w is allowed.
+Durations are defined by a number and a designation. The number can be a real
+so 1.5w is allowed.
 
   ms
     milliseconds
@@ -128,14 +116,14 @@ Durations are defined by a number and a designation. The number can be a real so
     weeks
 
 
-Ints
-----
+Integers
+--------
 
 Certain fields are integers, used as expected. In string context they
 return a string.
 
-Reals
------
+Real numbers
+------------
 
 VCL understands real numbers. As with integers, when used in a string
 context they will return a string.
@@ -144,12 +132,11 @@ context they will return a string.
 Regular Expressions
 -------------------
 
-Varnish uses PCRE - Perl-compatible regular expressions. For a
-complete description of PCRE please see the pcre(3) man page.
+Varnish uses Perl-compatible regular expressions (PCRE). For a
+complete description please see the pcre(3) man page.
 
-To send flags to the PCRE engine, such as to turn on *case insensitivity* 
-add the flag within parens following a question mark,
-like this::
+To send flags to the PCRE engine, such as to do case insensitive matching, add
+the flag within parens following a question mark, like this::
 
     # If host is NOT example dot com..
     if (req.http.host !~ "(?i)example.com$") {
@@ -162,22 +149,35 @@ Include statement
 
 To include a VCL file in another file use the include keyword::
 
-  include "foo.vcl";
+    include "foo.vcl";
+
+
+Import statement
+----------------
+
+The *import* statement is used to load Varnish Modules (VMODs.)
+
+Example::
+
+    import std;
+    sub vcl_recv {
+        std.log("foo");
+    }
+
 
 
 Backend definition
 ------------------
 
-A backend declaration creates and initializes a named backend
-object. A declaration start with the keyword *backend* followed by the
-name of the backend. The actual declaration is in curly brackets, in a
-key/value fashion.::
+A backend declaration creates and initialises a named backend object. A
+declaration start with the keyword *backend* followed by the name of the
+backend. The actual declaration is in curly brackets, in a key/value fashion.::
 
     backend name {
         .attribute = "value";
     }
 
-The only mandatory attribute is host. The attributes will inherit
+The only mandatory attribute is *host*. The attributes will inherit
 their defaults from the global parameters. The following attributes
 are available:
 
@@ -233,9 +233,9 @@ There are no mandatory options. These are the options you can set:
 
   expected_response
     The expected HTTP response code. Defaults to 200.
-            
+
   timeout
-    The timeout for the probe. Default it 2s.
+    The timeout for the probe. Default is 2s.
 
   interval
     How often the probe is run. Default is 5s.
@@ -245,10 +245,11 @@ There are no mandatory options. These are the options you can set:
     starts. Defaults to the value of threshold - 1. In this case, the
     backend starts as sick and requires one single poll to be
     considered healthy.
-            
+
   window
-    How many of the latest polls we examine to determine backend health. Defaults to 8.
-            
+    How many of the latest polls we examine to determine backend health.
+    Defaults to 8.
+
   threshold
     How many of the polls in .window must have succeeded for us to
     consider the backend healthy. If this is set to more than or equal
@@ -258,68 +259,82 @@ There are no mandatory options. These are the options you can set:
     threshold - 1.
 
 
-ACLs
-----
+Access Control List (ACL)
+-------------------------
 
-An ACL declaration creates and initializes a named access control list
-which can later be used to match client addresses::
+An Access Control List (ACL) declaration creates and initialises a named access
+control list which can later be used to match client addresses::
 
-    acl local {
+    acl localnetwork {
         "localhost";    # myself
         "192.0.2.0"/24; # and everyone on the local network
-        ! "192.0.2.23"; # except for the dialin router
+        ! "192.0.2.23"; # except for the dial-in router
     }
 
 If an ACL entry specifies a host name which Varnish is unable to
-resolve, it will match any address it is compared to.  Consequently,
+resolve, it will match any address it is compared to. Consequently,
 if it is preceded by a negation mark, it will reject any address it is
-compared to, which may not be what you intended.  If the entry is
+compared to, which may not be what you intended. If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
 To match an IP address against an ACL, simply use the match operator::
 
-    if (client.ip ~ local) {
+    if (client.ip ~ localnetwork) {
         return (pipe);
     }
 
 
+VCL objects
+-----------
+
+A VCL object can be made with the *new* keyword.
+
+Example::
+
+    sub vcl_init {
+        new b = directors.round_robin()
+        b.add_backend(node1);
+    }
+
+
 Subroutines
 -----------
 
 A subroutine is used to group code for legibility or reusability::
 
     sub pipe_if_local {
-        if (client.ip ~ local) {
+        if (client.ip ~ localnetwork) {
             return (pipe);
         }
     }
 
 Subroutines in VCL do not take arguments, nor do they return
-values. The built in subroutines all have names beginning with vcl_,
+values. The built in subroutines all have names beginning with vcl\_,
 which is reserved.
 
 To call a subroutine, use the call keyword followed by the subroutine's name::
 
-    call pipe_if_local;
+    sub vcl_recv {
+        call pipe_if_local;
+    }
 
 Return statements
 ~~~~~~~~~~~~~~~~~
 
-The subroutine executions ends when a return(*action*) statement is
-made. The *action* specifies how execution should proceed. The context
-defines which actions are available. See the user guide for information
-on what actions are available where.
+The ongoing vcl\_* subroutine execution ends when a return(*action*) statement
+is made.
+
+The *action* specifies how execution should proceed. The context defines
+which actions are available.
 
 Multiple subroutines
 ~~~~~~~~~~~~~~~~~~~~
 
-If multiple subroutines with the name of one of the builtin
-ones are defined, they are concatenated in the order in which they
-appear in the source.
-
-The default versions distributed with Varnish will be implicitly
-concatenated.
+If multiple subroutines with the name of one of the built-in ones are defined,
+they are concatenated in the order in which they appear in the source.
 
+The built-in VCL distributed with Varnish will be implicitly concatenated
+when the VCL is compiled.
 
 
 Variables
@@ -338,11 +353,35 @@ Functions
 The following built-in functions are available:
 
 ban(expression)
-  Bans all objects in cache that match the expression.
+  Invalidates all objects in cache that match the expression with the
+  ban mechanism.
+
+call(subroutine)
+  Run a VCL subroutine within the current scope.
+
+hash_data(input)
+  Adds an input to the hash input. In the built-in VCL hash_data()
+  is called on the host and URL of the *request*. Available in vcl_hash.
+
+new()
+  Instanciate a new VCL object. Available in vcl_init.
+
+purge()
+  Invalidate all variants of the current object using purge. Available in 
+  vcl_miss and vcl_hit.
 
-hash_data(str)
-  Adds a string to the hash input. In the built-in VCL hash_data()
-  is called on the host and URL of the *request*.
+return()
+  End execution of the current VCL subroutine, and continue to the next step
+  in the request handling state machine.
+
+rollback()
+  Restore request HTTP headers to their original state.
+
+synthetic(STRING)
+  Prepare a synthetic response body containing the STRING. Available in
+  vcl_synth and vcl_backend_error.
+
+.. list above comes from struct action_table[] in vcc_action.c.
 
 regsub(str, regex, sub)
   Returns a copy of str with the first occurrence of the regular
@@ -354,11 +393,13 @@ regsub(str, regex, sub)
 regsuball(str, regex, sub)
   As regsub() but this replaces all occurrences.
 
+.. regsub* is in vcc_expr.c
+
 
 EXAMPLES
 ========
 
-For examples, please see the guide guide.
+For examples, please see the online documentation.
 
 SEE ALSO
 ========
@@ -373,7 +414,7 @@ HISTORY
 VCL was developed by Poul-Henning Kamp in cooperation with Verdens
 Gang AS, Redpill Linpro and Varnish Software.  This manual page is
 written by Per Buer, Poul-Henning Kamp, Martin Blix Grydeland,
-Kristian Lyngst?l and possibly others.
+Kristian Lyngst?l, Lasse Karstensen and possibly others.
 
 COPYRIGHT
 =========



From lkarsten at varnish-software.com  Fri Mar 21 13:57:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 21 Mar 2014 14:57:31 +0100
Subject: [master] 9556bb4 HTTP is an abbreviation
Message-ID: <E1WQzwx-0002sb-W0@project.varnish-software.com>


commit 9556bb4c23b9e8617760756d6d670831660bbe1f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 21 14:57:16 2014 +0100

    HTTP is an abbreviation

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 602beff..84724c1 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -135,7 +135,7 @@ parse_unset(struct vcc *tl)
 	assert(vp != NULL);
 	if (vp->fmt != HEADER) {
 		VSB_printf(tl->sb,
-		    "Only http header variables can be unset.\n");
+		    "Only HTTP header variables can be unset.\n");
 		vcc_ErrWhere(tl, tl->t);
 		return;
 	}



From lkarsten at varnish-software.com  Fri Mar 21 13:57:32 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 21 Mar 2014 14:57:32 +0100
Subject: [master] cca0de0 Update copyright
Message-ID: <E1WQzwy-0002sk-0n@project.varnish-software.com>


commit cca0de063ee656dcac52f63c0a05ed7172eca2c4
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 21 14:57:25 2014 +0100

    Update copyright

diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c
index 5f9712a..0ab3f64 100644
--- a/lib/libvcc/vcc_storage.c
+++ b/lib/libvcc/vcc_storage.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2010 Varnish Software AS
+ * Copyright (c) 2010-2014 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>



From lkarsten at varnish-software.com  Fri Mar 21 14:09:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 21 Mar 2014 15:09:48 +0100
Subject: [master] 54e060c Move vcl_var.rst, use sane variable name for fp.
Message-ID: <E1WR08q-00061O-RP@project.varnish-software.com>


commit 54e060c6542fe1109b7f893cc259065f7c21e596
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 21 15:03:02 2014 +0100

    Move vcl_var.rst, use sane variable name for fp.
    
    Generated RST file should live in include/ like the others, not in reference/.
    
    Try to make the code slightly more readable through a descriptive variable name
    for the file handle.

diff --git a/.gitignore b/.gitignore
index a37080a..ba64e39 100644
--- a/.gitignore
+++ b/.gitignore
@@ -95,7 +95,7 @@ cscope.*out
 /doc/*.html
 /doc/sphinx/build/
 /doc/sphinx/conf.py
-/doc/sphinx/reference/vcl_var.rst
+/doc/sphinx/include/vcl_var.rst
 
 # NetBeans insists on this
 /nbproject/private/
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index c71577b..1268920 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -344,7 +344,7 @@ In VCL you have access to certain variable objects. These contain
 requests and responses currently being worked on. What variables are
 available depends on context.
 
-.. include:: vcl_var.rst
+.. include:: ../include/vcl_var.rst
 
 
 Functions
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 84e363d..95c32eb 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1131,7 +1131,7 @@ fo.close
 
 #######################################################################
 
-fo = open(buildroot + "/doc/sphinx/reference/vcl_var.rst", "w")
+fp_vclvar = open(buildroot + "/doc/sphinx/include/vcl_var.rst", "w")
 
 l = list()
 for i in sp_variables:
@@ -1164,14 +1164,14 @@ hdr=""
 for i in l:
 	j = i[0].split(".")
 	if j[0] != hdr:
-		fo.write("\n" + j[0] + "\n")
-		fo.write("~" * len(j[0]) + "\n")
+		fp_vclvar.write("\n" + j[0] + "\n")
+		fp_vclvar.write("~" * len(j[0]) + "\n")
 		hdr = j[0]
-	fo.write("\n" + i[0] + "\n\n")
-	fo.write("\tType: " + i[1] + "\n\n")
-	rst_where(fo, "Readable from: ", i[2])
-	rst_where(fo, "Writable from: ", i[3])
+	fp_vclvar.write("\n" + i[0] + "\n\n")
+	fp_vclvar.write("\tType: " + i[1] + "\n\n")
+	rst_where(fp_vclvar, "Readable from: ", i[2])
+	rst_where(fp_vclvar, "Writable from: ", i[3])
 	for j in i[4].split("\n"):
-		fo.write("\t" + j.strip() + "\n")
+		fp_vclvar.write("\t%s\n" % j.strip())
 
-fo.close()
+fp_vclvar.close()



From phk at FreeBSD.org  Fri Mar 21 14:16:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 21 Mar 2014 15:16:49 +0100
Subject: [master] ccc2005 Discover failure to execute varnishd faster
Message-ID: <E1WR0Fe-0000De-01@project.varnish-software.com>


commit ccc2005dac37c2a7d24fdba734d89e09d15310a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 21 14:07:59 2014 +0000

    Discover failure to execute varnishd faster

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 0b71923..19f51dd 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -339,8 +339,10 @@ varnish_thread(void *priv)
 		i = poll(fds, 1, 1000);
 		if (i == 0)
 			continue;
-		if (fds->revents & (POLLERR|POLLHUP))
+		if (fds->revents & (POLLERR|POLLHUP)) {
+			vtc_log(v->vl, 4, "STDOUT poll 0x%x", fds->revents);
 			break;
+		}
 		i = read(v->fds[0], buf, sizeof buf - 1);
 		if (i <= 0)
 			break;
@@ -426,8 +428,8 @@ varnish_launch(struct varnish *v)
 	memset(fd, 0, sizeof fd);
 	fd[0].fd = v->cli_fd;
 	fd[0].events = POLLIN;
-	fd[1].fd = v->fds[0];
-	fd[1].events = 0; /* Only care about POLLHUP, which is output-only */
+	fd[1].fd = v->fds[1];
+	fd[1].events = POLLIN;
 	i = poll(fd, 2, 10000);
 	vtc_log(v->vl, 4, "CLIPOLL %d 0x%x 0x%x",
 	    i, fd[0].revents, fd[1].revents);



From phk at FreeBSD.org  Fri Mar 21 14:32:10 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 21 Mar 2014 15:32:10 +0100
Subject: [master] 755a6c5 Track source changes
Message-ID: <E1WR0UU-00013s-Hj@project.varnish-software.com>


commit 755a6c533d22a07fdfa40b00fcb020c16c7860a3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 21 14:31:36 2014 +0000

    Track source changes

diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 004ec15..aae3723 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -73,7 +73,7 @@ varnish v1 -errvcl {Symbol not found: 'mu' (expected type BOOL):} {
 	sub vcl_backend_response { set beresp.do_gzip = mu; }
 }
 
-varnish v1 -errvcl {Only http header variables can be unset.} {
+varnish v1 -errvcl {Only HTTP header variables can be unset.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { unset beresp.do_gzip; }
 }



From lkarsten at varnish-software.com  Sat Mar 22 08:58:10 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Sat, 22 Mar 2014 09:58:10 +0100
Subject: [master] d6a7f31 Cache-Control: no-cache and friends is honored.
Message-ID: <E1WRHko-0004jI-Mk@project.varnish-software.com>


commit d6a7f3172cb882f70a17c95c550f0cb6cde48d20
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Sat Mar 22 09:56:43 2014 +0100

    Cache-Control: no-cache and friends is honored.
    
    The builtin VCL supports this now. Since so many noticed this missing
    in 3.0, I think it should be mentioned.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 02e6f02..402b693 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -79,7 +79,8 @@ req.* used to be available in `vcl_fetch`, but after the split of functionality,
 
 vcl_* reserved
 ~~~~~~~~~~~~~~
-Any custom-made subs cannot be named 'vcl_*' anymore. This namespace is reserved for builtin subs.
+Any custom-made subs cannot be named 'vcl_*' anymore. This namespace is
+reserved for builtin subs.
 
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -87,8 +88,19 @@ req.backend.healthy replaced by std.healthy(req.backend)
 obj is now read-only
 ~~~~~~~~~~~~~~~~~~~~
 
-`obj` is now read-only. `obj.hits`, if enabled in VCL, now counts per objecthead,
-not per object. `obj.last_use` has been retired.
+`obj` is now read-only. `obj.hits`, if enabled in VCL, now counts per
+objecthead, not per object. `obj.last_use` has been retired.
+
+
+default/builtin VCL changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The VCL code that is appended to user-configured VCL automatically is now
+called the builtin VCL. (previously default.vcl)
+
+The builtin VCL now honors Cache-Control: no-cache (and friends) to indicate
+uncacheable content from the backend.
+
 
 Changes to parameters
 =====================



From phk at FreeBSD.org  Mon Mar 24 08:01:05 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 09:01:05 +0100
Subject: [master] 71a357a Make objects read-only, once they have been stored.
Message-ID: <E1WRzof-0005tV-D0@project.varnish-software.com>


commit 71a357ab29bb4ec4f8e671665395e101ad885d2e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 07:59:58 2014 +0000

    Make objects read-only, once they have been stored.
    
    Now that we have a proper purge-mechanism, all the magic to
    set obj.{ttl,grace,keep} in vcl_hit{} can go away.

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b51d2de..c48dc37 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -436,23 +436,21 @@ VRT_r_bereq_retries(const struct vrt_ctx *ctx)
  * keep are relative to ttl.
  */
 
-#define VRT_DO_EXP(which, sexp, fld, offset, now, extra)	\
+#define VRT_DO_EXP_L(which, sexp, fld)				\
 								\
 void								\
 VRT_l_##which##_##fld(const struct vrt_ctx *ctx, double a)	\
 {								\
-	double dt;						\
 								\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
-	if (a > 0.0)						\
-		sexp.fld = a + offset;				\
-	else							\
-		sexp.fld = 0;					\
-	extra;							\
-	dt = now - sexp.t_origin;				\
-	VSLb(ctx->vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %.0f",	\
-	    sexp.ttl - dt, sexp.grace, sexp.keep, now, dt);	\
-}								\
+	if (a < 0.0)						\
+		a = 0.0;					\
+	sexp.fld = a;						\
+	VSLb(ctx->vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f",	\
+	    sexp.ttl, sexp.grace, sexp.keep, sexp.t_origin);	\
+}
+
+#define VRT_DO_EXP_R(which, sexp, fld, offset)			\
 								\
 double								\
 VRT_r_##which##_##fld(const struct vrt_ctx *ctx)		\
@@ -464,17 +462,17 @@ VRT_r_##which##_##fld(const struct vrt_ctx *ctx)		\
 	return(0.0);						\
 }
 
-VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0, ctx->req->t_req,
-   EXP_Rearm(ctx->req->obj, ctx->req->t_req, NAN, NAN, NAN);)
-VRT_DO_EXP(obj, ctx->req->obj->exp, ttl,
-   (ctx->req->t_req - ctx->req->obj->exp.t_origin), ctx->req->t_req,
-   EXP_Rearm(ctx->req->obj, ctx->req->t_req, NAN, NAN, NAN);)
-VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0, ctx->req->t_req,
-   EXP_Rearm(ctx->req->obj, ctx->req->t_req, NAN, NAN, NAN);)
+VRT_DO_EXP_R(obj, ctx->req->obj->exp, ttl,
+   (ctx->req->t_req - ctx->req->obj->exp.t_origin))
+VRT_DO_EXP_R(obj, ctx->req->obj->exp, grace, 0)
+VRT_DO_EXP_R(obj, ctx->req->obj->exp, keep, 0)
 
-VRT_DO_EXP(beresp, ctx->bo->exp, grace, 0, ctx->bo->exp.t_origin,)
-VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0, ctx->bo->exp.t_origin,)
-VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0, ctx->bo->exp.t_origin,)
+VRT_DO_EXP_L(beresp, ctx->bo->exp, ttl)
+VRT_DO_EXP_R(beresp, ctx->bo->exp, ttl, 0)
+VRT_DO_EXP_L(beresp, ctx->bo->exp, grace)
+VRT_DO_EXP_R(beresp, ctx->bo->exp, grace, 0)
+VRT_DO_EXP_L(beresp, ctx->bo->exp, keep)
+VRT_DO_EXP_R(beresp, ctx->bo->exp, keep, 0)
 
 /*--------------------------------------------------------------------
  * [be]req.xid
diff --git a/bin/varnishtest/tests/p00001.vtc b/bin/varnishtest/tests/p00001.vtc
deleted file mode 100644
index 8c38715..0000000
--- a/bin/varnishtest/tests/p00001.vtc
+++ /dev/null
@@ -1,70 +0,0 @@
-varnishtest "Change TTL on persistent object"
-
-shell "rm -f ${tmpdir}/_.per"
-
-server s1 {
-	rxreq
-	txresp -hdr "Foo: foo"
-} -start
-
-varnish v1 \
-	-arg "-pfeature=+wait_silo" \
-	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
-	-vcl+backend { } -start
-
-varnish v1 -cliok "param.set default_grace 0"
-varnish v1 -cliok "param.set default_keep 0"
-
-client c1 {
-	txreq -url "/"
-	rxresp
-	expect resp.status == 200
-	expect resp.http.X-Varnish == "1001"
-	expect resp.http.foo == "foo"
-} -run
-
-varnish v1 -stop
-
-varnish v1 -vcl+backend {
-	sub vcl_hit {
-		set obj.ttl = 1 s;
-	}
-}
-
-server s1 -wait
-
-varnish v1 -start
-varnish v1 -cliok "debug.xid 1999"
-
-client c1 {
-	txreq -url "/"
-	rxresp
-	expect resp.status == 200
-	expect resp.http.X-Varnish == "2001 1002"
-	expect resp.http.foo == "foo"
-} -run
-
-varnish v1 -stop
-
-server s1 {
-	rxreq
-	txresp -hdr "Foo: bar"
-} -start
-
-varnish v1 -vcl+backend { }
-
-delay 2
-
-varnish v1 -start
-varnish v1 -cliok "debug.xid 2999"
-
-client c1 {
-	txreq -url "/"
-	rxresp
-	expect resp.status == 200
-	expect resp.http.X-Varnish == "3001"
-	expect resp.http.foo == "bar"
-} -run
-
-# shell "rm -f /tmp/__v1/_.per"
diff --git a/bin/varnishtest/tests/r00365.vtc b/bin/varnishtest/tests/r00365.vtc
deleted file mode 100644
index 43c76b3..0000000
--- a/bin/varnishtest/tests/r00365.vtc
+++ /dev/null
@@ -1,29 +0,0 @@
-varnishtest "Test restarts in vcl_hit"
-
-server s1 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -status 200 -body "1"
-	rxreq
-	expect req.url == "/foo"
-	txresp -status 200 -body "22"
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_hit {
-		set obj.ttl = 0s;
-		return (restart);
-	}
-} -start
-
-client c1 {
-	txreq -url "/foo"
-	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 1
-
-	txreq -url "/foo"
-	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 2
-} -run
diff --git a/bin/varnishtest/tests/r00444.vtc b/bin/varnishtest/tests/r00444.vtc
deleted file mode 100644
index 899ab57..0000000
--- a/bin/varnishtest/tests/r00444.vtc
+++ /dev/null
@@ -1,59 +0,0 @@
-varnishtest "purging on POST"
-
-server s1 {
-	rxreq
-	expect req.method == "GET"
-	txresp -body "1"
-
-	rxreq
-	expect req.method == "POST"
-	txresp -body "22"
-
-	rxreq
-	expect req.method == "POST"
-	txresp -body "333"
-
-	rxreq
-	expect req.method == "GET"
-	txresp -body "4444"
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_recv {
-		if (req.method == "POST") {
-			/* Lookup so we find any cached object */
-			return (hash);
-		}
-	}
-	sub vcl_hit {
-		if (req.method == "POST") {
-			/* Get rid of this object */
-			set obj.ttl = 0s;
-			return (pass);
-		}
-	}
-	sub vcl_miss {
-		if (req.method == "POST") {
-			/* Make sure we don't cache the POST result */
-			return (pass);
-		}
-	}
-} -start
-
-client c1 {
-	txreq
-	rxresp
-	expect resp.bodylen == 1
-
-	txreq -req POST
-	rxresp
-	expect resp.bodylen == 2
-
-	txreq -req POST
-	rxresp
-	expect resp.bodylen == 3
-
-	txreq
-	rxresp
-	expect resp.bodylen == 4
-} -run
diff --git a/bin/varnishtest/tests/r00956.vtc b/bin/varnishtest/tests/r00956.vtc
index 2744a44..85cca43 100644
--- a/bin/varnishtest/tests/r00956.vtc
+++ b/bin/varnishtest/tests/r00956.vtc
@@ -9,18 +9,12 @@ varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.ttl = 10s;
 		set beresp.http.fooB = beresp.ttl;
-		set beresp.http.barB = "xxx";
 	}
 	sub vcl_hit {
 		set req.http.foo = obj.ttl;
-		set obj.ttl = 7s;
-		set obj.grace = 120s;
-		set obj.keep = 1h;
-		set req.http.bar = obj.ttl;
 	}
 	sub vcl_deliver {
 		set resp.http.foo = req.http.foo;
-		set resp.http.bar = req.http.bar;
 	}
 } -start
 
@@ -29,7 +23,6 @@ client c1 {
 	rxresp
 	expect resp.bodylen == 40
 	expect resp.http.fooB == 10.000
-	expect resp.http.barB == "xxx"
 
 	delay 2
 	txreq
@@ -37,19 +30,13 @@ client c1 {
 	expect resp.bodylen == 40
 	# XXX: should be: < 8
 	expect resp.http.fooB == 10.000
-	expect resp.http.barB == "xxx"
 	expect resp.http.foo != 10.000
 	expect resp.http.foo != 9.000
-	expect resp.http.bar == 7.000
 
 	delay 2
 	txreq
 	rxresp
 	expect resp.bodylen == 40
-	# XXX: should be: < 5
 	expect resp.http.fooB == 10.000
-	expect resp.http.barB == "xxx"
-	expect resp.http.foo != 7.000
-	expect resp.http.foo != 6.000
-	expect resp.http.bar == 7.000
+	expect resp.http.foo <= 5.000
 } -run
diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index b42106d..1107f86 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -75,12 +75,12 @@ varnish v1 -vcl {
 
 varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_backend_response { set obj.ttl = 1. k; }
+	sub vcl_backend_response { set beresp.ttl = 1. k; }
 }
 
 varnish v1 -errvcl {Expression has type DURATION, expected BOOL} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_backend_response { if (obj.ttl *= 2) { } }
+	sub vcl_backend_response { if (beresp.ttl *= 2) { } }
 }
 
 varnish v1 -errvcl {Operator > not possible on BACKEND} {
diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc
index 636207e..301384b 100644
--- a/bin/varnishtest/tests/v00021.vtc
+++ b/bin/varnishtest/tests/v00021.vtc
@@ -1,11 +1,11 @@
 varnishtest "VCL compiler coverage test: vcc_xref.c"
 
-varnish v1 -errvcl {'obj.ttl': cannot be set in method 'vcl_recv'.} {
+varnish v1 -errvcl {Variable 'obj.ttl' is read only.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { set obj.ttl = 1 w; }
 }
 
-varnish v1 -errvcl {'obj.ttl': cannot be set from method 'foo'.} {
+varnish v1 -errvcl {Variable 'obj.ttl' is read only.} {
 	backend b { .host = "127.0.0.1"; }
 
 	sub foo { set obj.ttl = 1 w; }
diff --git a/bin/varnishtest/tests/v00023.vtc b/bin/varnishtest/tests/v00023.vtc
deleted file mode 100644
index e96585a..0000000
--- a/bin/varnishtest/tests/v00023.vtc
+++ /dev/null
@@ -1,29 +0,0 @@
-varnishtest "Test that obj.ttl = 0s prevents subsequent hits"
-
-server s1 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -status 200 -body "1"
-	rxreq
-	expect req.url == "/foo"
-	txresp -status 200 -body "22"
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_hit {
-		set obj.ttl = 0s;
-		return (restart);
-	}
-} -start
-
-client c1 {
-	txreq -url "/foo"
-	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 1
-
-	txreq -url "/foo"
-	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 2
-} -run
diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc
index b090502..1be49af 100644
--- a/bin/varnishtest/tests/v00025.vtc
+++ b/bin/varnishtest/tests/v00025.vtc
@@ -24,14 +24,10 @@ varnish v1 -vcl+backend {
 
 	sub vcl_hit {
 		if (obj.grace < 3m) {
-			set obj.grace = 1m;
-		} else {
-			set obj.grace = 2m;
+			return (deliver);
 		}
 		if (obj.ttl < 3m) {
-			set obj.ttl = 2m;
-		} else {
-			set obj.ttl = 3m;
+			return (deliver);
 		}
 	}
 
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 9217582..5f768ec 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -230,27 +230,26 @@ SLTM(TTL, 0, "TTL set on object",
 	"A TTL record is emitted whenever the ttl, grace or keep"
 	" values for an object is set.\n\n"
 	"The format is::\n\n"
-	"\t%u %s %d %d %d %d %d [ %d %u %u ]\n"
-	"\t|  |  |  |  |  |  |    |  |  |\n"
-	"\t|  |  |  |  |  |  |    |  |  +- Max-Age from Cache-Control header\n"
-	"\t|  |  |  |  |  |  |    |  +---- Expires header\n"
-	"\t|  |  |  |  |  |  |    +------- Date header\n"
-	"\t|  |  |  |  |  |  +------------ Age (incl Age: header value)\n"
-	"\t|  |  |  |  |  +--------------- Reference time for TTL\n"
-	"\t|  |  |  |  +------------------ Keep\n"
-	"\t|  |  |  +--------------------- Grace\n"
-	"\t|  |  +------------------------ TTL\n"
-	"\t|  +--------------------------- \"RFC\" or \"VCL\"\n"
-	"\t+------------------------------ object XID\n"
+	"\t%s %d %d %d %d %d [ %d %u %u ]\n"
+	"\t|  |  |  |  |  |    |  |  |\n"
+	"\t|  |  |  |  |  |    |  |  +- Max-Age from Cache-Control header\n"
+	"\t|  |  |  |  |  |    |  +---- Expires header\n"
+	"\t|  |  |  |  |  |    +------- Date header\n"
+	"\t|  |  |  |  |  +------------ Age (incl Age: header value)\n"
+	"\t|  |  |  |  +--------------- Reference time for TTL\n"
+	"\t|  |  |  +------------------ Keep\n"
+	"\t|  |  +--------------------- Grace\n"
+	"\t|  +------------------------ TTL\n"
+	"\t+--------------------------- \"RFC\" or \"VCL\"\n"
 	"\n"
-	"The last three fields are only present in \"RFC\" headers.\n\n"
+	"The last four fields are only present in \"RFC\" headers.\n\n"
 	"Examples::\n\n"
-	"\t1001 RFC 19 -1 -1 1312966109 4 0 0 23\n"
-	"\t1001 VCL 10 -1 -1 1312966109 4\n"
-	"\t1001 VCL 7 -1 -1 1312966111 6\n"
-	"\t1001 VCL 7 120 -1 1312966111 6\n"
-	"\t1001 VCL 7 120 3600 1312966111 6\n"
-	"\t1001 VCL 12 120 3600 1312966113 8\n"
+	"\tRFC 19 -1 -1 1312966109 4 0 0 23\n"
+	"\tVCL 10 -1 -1 1312966109 4\n"
+	"\tVCL 7 -1 -1 1312966111 6\n"
+	"\tVCL 7 120 -1 1312966111 6\n"
+	"\tVCL 7 120 3600 1312966111 6\n"
+	"\tVCL 12 120 3600 1312966113 8\n"
 	"\n"
 )
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 95c32eb..b6ccfe2 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -528,7 +528,7 @@ sp_variables = [
 	('obj.ttl',
 		'DURATION',
 		( 'hit', ),
-		( 'hit', ), """
+		( ), """
 		The object's remaining time to live, in seconds.
 		obj.ttl is writable.
 		"""
@@ -536,14 +536,14 @@ sp_variables = [
 	('obj.grace',
 		'DURATION',
 		( 'hit', ),
-		( 'hit', ), """
+		( ), """
 		The object's grace period in seconds. obj.grace is writable.
 		"""
 	),
 	('obj.keep',
 		'DURATION',
 		( 'hit', ),
-		( 'hit', ), """
+		( ), """
 		"""
 	),
 	('obj.uncacheable',



From phk at FreeBSD.org  Mon Mar 24 09:33:33 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 10:33:33 +0100
Subject: [master] 67538af Move retries behind refcount so it is cleared when
 busyobj is recycled.
Message-ID: <E1WS1G9-0008E5-1N@project.varnish-software.com>


commit 67538af2af3f4285a12b1e7d9a0044a44c614ec3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 09:33:06 2014 +0000

    Move retries behind refcount so it is cleared when busyobj is recycled.
    
    Spotted by:	martin

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ac615a4..01ec775 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -523,8 +523,8 @@ struct busyobj {
 	 * All fields from refcount and down are zeroed when the busyobj
 	 * is recycled.
 	 */
-	int			retries;
 	unsigned		refcount;
+	int			retries;
 	double			t_fetch;
 	struct req		*req;
 



From phk at FreeBSD.org  Mon Mar 24 10:07:34 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 11:07:34 +0100
Subject: [master] e557692 If vcl_synth{} or vcl_deliver{} sets a "Connection:
 close" header, we close from our side with the new SC_RESP_CLOSE status.
Message-ID: <E1WS1n4-0003rA-Rw@project.varnish-software.com>


commit e55769294b592eb3cf7a0cf9ea6109ad08e22466
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 10:06:48 2014 +0000

    If vcl_synth{} or vcl_deliver{} sets a "Connection: close" header,
    we close from our side with the new SC_RESP_CLOSE status.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 6650c31..d3d70d6 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -153,6 +153,9 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	V1D_Deliver(req);
 
+	if (http_HdrIs(req->resp, H_Connection, "close"))
+		req->doclose = SC_RESP_CLOSE;
+
 	if (req->obj->objcore->flags & OC_F_PASS) {
 		/*
 		 * No point in saving the body if it is hit-for-pass,
@@ -185,7 +188,7 @@ DOT }
  */
 
 static enum req_fsm_nxt
-cnt_error(struct worker *wrk, struct req *req)
+cnt_synth(struct worker *wrk, struct req *req)
 {
 	char date[40];
 	struct http *h;
@@ -234,6 +237,9 @@ cnt_error(struct worker *wrk, struct req *req)
 	}
 	assert(wrk->handling == VCL_RET_DELIVER);
 
+	if (http_HdrIs(req->resp, H_Connection, "close"))
+		req->doclose = SC_RESP_CLOSE;
+
 	V1D_Deliver_Synth(req);
 
 	VSB_delete(req->synth_body);
@@ -272,7 +278,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	if (req->objcore->flags & OC_F_FAILED) {
 		req->err_code = 503;
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 		(void)HSH_DerefObjCore(&wrk->stats, &req->objcore);
 		req->objcore = NULL;
 		return (REQ_FSM_MORE);
@@ -422,7 +428,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_RESTART;
 		break;
 	case VCL_RET_SYNTH:
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 		break;
 	case VCL_RET_PASS:
 		wrk->stats.cache_hit++;
@@ -480,7 +486,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_FETCH;
 		return (REQ_FSM_MORE);
 	case VCL_RET_SYNTH:
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 		break;
 	case VCL_RET_RESTART:
 		req->req_step = R_STP_RESTART;
@@ -528,7 +534,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 	VCL_pass_method(req->vcl, wrk, req, NULL, req->http->ws);
 	switch (wrk->handling) {
 	case VCL_RET_SYNTH:
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 		break;
 	case VCL_RET_RESTART:
 		req->req_step = R_STP_RESTART;
@@ -620,7 +626,7 @@ cnt_restart(struct worker *wrk, struct req *req)
 	if (++req->restarts >= cache_param->max_restarts) {
 		VSLb(req->vsl, SLT_VCL_Error, "Too many restarts");
 		req->err_code = 503;
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 	} else {
 		wid = VXID_Get(&wrk->vxid_pool);
 		// XXX: ReqEnd + ReqAcct ?
@@ -676,7 +682,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 	http_VSL_log(req->http);
 
 	if (req->err_code) {
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 		return (REQ_FSM_MORE);
 	}
 
@@ -755,7 +761,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_PASS;
 		return (REQ_FSM_MORE);
 	case VCL_RET_SYNTH:
-		req->req_step = R_STP_ERROR;
+		req->req_step = R_STP_SYNTH;
 		return (REQ_FSM_MORE);
 	default:
 		WRONG("Illegal return from vcl_recv{}");
@@ -795,7 +801,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	AZ(HSH_DerefObjCore(&wrk->stats, &boc));
 
 	VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws);
-	req->req_step = R_STP_ERROR;
+	req->req_step = R_STP_SYNTH;
 	return (REQ_FSM_MORE);
 }
 
diff --git a/include/tbl/sess_close.h b/include/tbl/sess_close.h
index 522eb02..5b77eba 100644
--- a/include/tbl/sess_close.h
+++ b/include/tbl/sess_close.h
@@ -39,6 +39,7 @@ SESS_CLOSE(RX_TIMEOUT,	"Receive timeout")
 SESS_CLOSE(TX_PIPE,	"Piped transaction")
 SESS_CLOSE(TX_ERROR,	"Error transaction")
 SESS_CLOSE(TX_EOF,	"EOF transmission")
+SESS_CLOSE(RESP_CLOSE,	"Backend/VCL requested close")
 SESS_CLOSE(OVERLOAD,	"Out of some resource")
 SESS_CLOSE(SESS_PIPE_OVERFLOW,	"Session pipe overflow")
 
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index d0efae7..bb02fb4 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -45,7 +45,7 @@ REQ_STEP(purge,		PURGE,		(wrk, req))
 REQ_STEP(miss,		MISS,		(wrk, req))
 REQ_STEP(fetch,		FETCH,		(wrk, req))
 REQ_STEP(deliver,	DELIVER,	(wrk, req))
-REQ_STEP(error,		ERROR,		(wrk, req))
+REQ_STEP(synth,		SYNTH,		(wrk, req))
 #endif
 
 #ifdef FETCH_STEP



From phk at FreeBSD.org  Mon Mar 24 10:23:10 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 11:23:10 +0100
Subject: [master] d850882 Add tests for outgoing Connection: close handling
Message-ID: <E1WS22A-0001Wf-Nf@project.varnish-software.com>


commit d850882688678e514bc01766390b1f93731057ba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 10:22:54 2014 +0000

    Add tests for outgoing Connection: close handling

diff --git a/bin/varnishtest/tests/c00064.vtc b/bin/varnishtest/tests/c00064.vtc
new file mode 100644
index 0000000..4791597
--- /dev/null
+++ b/bin/varnishtest/tests/c00064.vtc
@@ -0,0 +1,35 @@
+varnishtest "Connection: close in vcl_synth{}"
+
+server s1 {
+	rxreq
+	txresp -status 400
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_miss {
+		if (req.url == "/333") {
+			return (synth(333, "FOO"));
+		} else {
+			return (synth(334, "BAR"));
+		}
+	}
+
+	sub vcl_synth {
+		if (resp.status == 333) {
+			set resp.http.connection = "close";
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url /334
+	rxresp
+	expect resp.status == 334
+	txreq -url /334
+	rxresp
+	expect resp.status == 334
+	txreq -url /333
+	rxresp
+	expect resp.status == 333
+	expect_close
+} -run
diff --git a/bin/varnishtest/tests/c00065.vtc b/bin/varnishtest/tests/c00065.vtc
new file mode 100644
index 0000000..aed48cd
--- /dev/null
+++ b/bin/varnishtest/tests/c00065.vtc
@@ -0,0 +1,24 @@
+varnishtest "Connection: close in vcl_deliver{}"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		set resp.http.Connection  = "close";
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect_close
+} -run
+
+client c1 {
+	txreq
+	rxresp
+	expect_close
+} -run



From phk at FreeBSD.org  Mon Mar 24 10:57:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 11:57:24 +0100
Subject: [master] 6abce40 Reflect change of obj size
Message-ID: <E1WS2ZI-0006zf-PA@project.varnish-software.com>


commit 6abce4069f3715e8f1f0812051b43c3a0b310cd3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 10:56:57 2014 +0000

    Reflect change of obj size

diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc
index 9bd7950..2d1d3cc 100644
--- a/bin/varnishtest/tests/c00044.vtc
+++ b/bin/varnishtest/tests/c00044.vtc
@@ -40,7 +40,7 @@ varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes == 0
 varnish v1 -expect SMA.s0.g_space > 1000000
 varnish v1 -expect SMA.s1.g_bytes > 1000000
-varnish v1 -expect SMA.s1.g_space < 170
+varnish v1 -expect SMA.s1.g_space < 180
 varnish v1 -expect SMA.s2.g_bytes == 0
 varnish v1 -expect SMA.s2.g_space > 1000000
 
@@ -55,9 +55,9 @@ varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes == 0
 varnish v1 -expect SMA.s0.g_space > 1000000
 varnish v1 -expect SMA.s1.g_bytes > 1000000
-varnish v1 -expect SMA.s1.g_space < 170
+varnish v1 -expect SMA.s1.g_space < 180
 varnish v1 -expect SMA.s2.g_bytes > 1000000
-varnish v1 -expect SMA.s2.g_space < 170
+varnish v1 -expect SMA.s2.g_space < 180
 
 client c1 {
 	txreq -url /burp
@@ -68,11 +68,11 @@ client c1 {
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 170
+varnish v1 -expect SMA.s0.g_space < 180
 varnish v1 -expect SMA.s1.g_bytes > 1000000
-varnish v1 -expect SMA.s1.g_space < 170
+varnish v1 -expect SMA.s1.g_space < 180
 varnish v1 -expect SMA.s2.g_bytes > 1000000
-varnish v1 -expect SMA.s2.g_space < 170
+varnish v1 -expect SMA.s2.g_space < 180
 
 client c1 {
 	txreq -url /foo1
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index c653cf0..3105041 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -30,7 +30,7 @@ client c1 {
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 170
+varnish v1 -expect SMA.s0.g_space < 180
 varnish v1 -expect SMA.s1.g_bytes == 0
 varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0



From phk at FreeBSD.org  Mon Mar 24 11:29:39 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 12:29:39 +0100
Subject: [master] d5a6e32 Bump 10 bytes further to catch all platforms
Message-ID: <E1WS34V-0003JS-VB@project.varnish-software.com>


commit d5a6e32141034ae08ed2e29a5c916f20b46441d9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 11:29:21 2014 +0000

    Bump 10 bytes further to catch all platforms

diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 3105041..024a3dd 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -30,7 +30,7 @@ client c1 {
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 180
+varnish v1 -expect SMA.s0.g_space < 190
 varnish v1 -expect SMA.s1.g_bytes == 0
 varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0



From phk at FreeBSD.org  Mon Mar 24 12:06:48 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 13:06:48 +0100
Subject: [master] e135798 more tweaking
Message-ID: <E1WS3eS-0008NB-N2@project.varnish-software.com>


commit e1357988dcb4959930eb1695f877f11bda56501b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 12:06:39 2014 +0000

    more tweaking

diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 024a3dd..16f26f7 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -46,7 +46,7 @@ client c1 {
 varnish v1 -expect n_lru_nuked == 1
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 1171
+varnish v1 -expect SMA.s0.g_space < 1180
 varnish v1 -expect SMA.s1.g_bytes == 0
 varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0



From phk at FreeBSD.org  Mon Mar 24 12:22:04 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 13:22:04 +0100
Subject: [master] fe32532 more tweaking
Message-ID: <E1WS3tE-00027m-I4@project.varnish-software.com>


commit fe325325aabca22700048a3815fa8d9e23ebd115
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 12:21:52 2014 +0000

    more tweaking

diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 16f26f7..126f1a9 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -62,7 +62,7 @@ client c1 {
 varnish v1 -expect n_lru_nuked == 2
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 1172
+varnish v1 -expect SMA.s0.g_space < 1182
 varnish v1 -expect SMA.s1.g_bytes == 0
 varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0



From phk at FreeBSD.org  Mon Mar 24 12:51:47 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 13:51:47 +0100
Subject: [master] 836f128 Add a couple of tiny delays to stabilize this test.
Message-ID: <E1WS4Lz-0006FM-Ma@project.varnish-software.com>


commit 836f1283eff963c3a73d4fe7b4c0cf9b9bb50286
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 12:51:25 2014 +0000

    Add a couple of tiny delays to stabilize this test.

diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index 4ecbe35..e02bc47 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -31,11 +31,15 @@ client c1 {
 	expect resp.bodylen == 2
 } -run
 
+delay .1
+
 client c1 {
 	txreq -req POST -nolen -hdr "Content-Length: 52"
 	delay .3
 } -run
 
+delay .1
+
 server s1 {
 	rxreq
 	txresp



From phk at FreeBSD.org  Mon Mar 24 13:13:16 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 14:13:16 +0100
Subject: [master] 7a14478 Stabilize this test with a short sleep
Message-ID: <E1WS4gm-00012b-LQ@project.varnish-software.com>


commit 7a144781c3d2a926e84bfa1cc32f44b11a4621b8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 13:13:01 2014 +0000

    Stabilize this test with a short sleep

diff --git a/bin/varnishtest/tests/c00041.vtc b/bin/varnishtest/tests/c00041.vtc
index 24a7871..2b7efc2 100644
--- a/bin/varnishtest/tests/c00041.vtc
+++ b/bin/varnishtest/tests/c00041.vtc
@@ -47,21 +47,26 @@ client c1 {
 	txreq -url "/1" -hdr "foo: foo1"
 	rxresp
 	expect resp.status == 200
+	expect resp.http.x-varnish == 1001
 	expect resp.bodylen == 1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
 	expect resp.status == 200
+	expect resp.http.x-varnish == 1003
 	expect resp.bodylen == 2
+	delay .1
 
 	txreq -url "/1" -hdr "foo: foo1"
 	rxresp
 	expect resp.status == 200
+	expect resp.http.x-varnish == "1005 1002"
 	expect resp.bodylen == 1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
 	expect resp.status == 200
+	expect resp.http.x-varnish == "1006 1004"
 	expect resp.bodylen == 2
 
 	# Purge on hit



From daghf at varnish-software.com  Mon Mar 24 14:12:44 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 24 Mar 2014 15:12:44 +0100
Subject: [master] 32c8acb Handle 'return (abandon);' in vcl_backend_response.
Message-ID: <E1WS5cK-0008Cx-8g@project.varnish-software.com>


commit 32c8acbf89a0ce4f3a6fe1ed16b08ae7e9af7564
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Mar 24 14:54:30 2014 +0100

    Handle 'return (abandon);' in vcl_backend_response.
    
    Test case by: Federico G. Schwindt
    
    Fixes: #1417

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5b5be58..93be439 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -341,6 +341,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
 
+	if (wrk->handling == VCL_RET_ABANDON)
+		return (F_STP_FAIL);
+
 	if (wrk->handling == VCL_RET_RETRY) {
 		AN (bo->vbc);
 		VDI_CloseFd(&bo->vbc);
diff --git a/bin/varnishtest/tests/r01417.vtc b/bin/varnishtest/tests/r01417.vtc
new file mode 100644
index 0000000..60263c7
--- /dev/null
+++ b/bin/varnishtest/tests/r01417.vtc
@@ -0,0 +1,18 @@
+varnishtest "vcl_backend_response abandon"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		return (abandon);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run



From phk at FreeBSD.org  Mon Mar 24 16:33:56 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Mar 2014 17:33:56 +0100
Subject: [master] 5a6cf6c Add more panic detail on busyobjs.
Message-ID: <E1WS7oy-0005jn-19@project.varnish-software.com>


commit 5a6cf6c74a71ccdd3bcb5257bb1d81ff432a684d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 24 16:33:30 2014 +0000

    Add more panic detail on busyobjs.
    
    Remove a debugging message.

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index a01e771..575d049 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -283,6 +283,10 @@ pan_busyobj(const struct busyobj *bo)
 
 	VSB_printf(pan_vsp, "  busyobj = %p {\n", bo);
 	pan_ws(bo->ws, 4);
+	VSB_printf(pan_vsp, "  refcnt = %u\n", bo->refcount);
+	VSB_printf(pan_vsp, "  retries = %d\n", bo->retries);
+	VSB_printf(pan_vsp, "  failed = %d\n", bo->failed);
+	VSB_printf(pan_vsp, "  state = %d\n", (int)bo->state);
 #define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, "    is_" #l "\n");
 #include "tbl/bo_flags.h"
 #undef BO_FLAG
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index c48dc37..b68031c 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -192,7 +192,6 @@ VRT_l_beresp_uncacheable(const struct vrt_ctx *ctx, unsigned a)
 		VSLb(ctx->vsl, SLT_VCL_Error,
 		    "Ignoring attempt to reset beresp.uncacheable");
 	} else if (a) {
-		VSLb(ctx->vsl, SLT_Debug, "XXXX: UNCACHEABLE %u" , a);
 		ctx->bo->uncacheable = 1;
 	}
 }



From phk at FreeBSD.org  Tue Mar 25 07:51:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 25 Mar 2014 08:51:53 +0100
Subject: [master] d94db39 Remove a debugging print
Message-ID: <E1WSM9J-0006g4-No@project.varnish-software.com>


commit d94db39f17311760bb95e166e68a9a109fafdc78
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 25 07:51:44 2014 +0000

    Remove a debugging print

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index b6ccfe2..a118513 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1002,7 +1002,6 @@ def one_var(nm, spec):
 		if nm == i[0]:
 			fh.write("VCL_" + typ +
 			    " VRT_r_%s(const struct vrt_ctx *);\n" % cnam )
-	print(spec)
 	restrict(fo, spec[2])
 
 	if len(spec[3]) == 0:



From phk at FreeBSD.org  Tue Mar 25 08:29:33 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 25 Mar 2014 09:29:33 +0100
Subject: [master] 405aad9 Split the oc->flags into those locked by oh->mtx,
 and those locked by lru->mtx.
Message-ID: <E1WSMjl-000105-Pc@project.varnish-software.com>


commit 405aad97af2482ec28df3e248055d0cd512a7fb0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 25 08:29:05 2014 +0000

    Split the oc->flags into those locked by oh->mtx, and those locked
    by lru->mtx.
    
    Spotted by: Martin

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 01ec775..68c74ff 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -422,17 +422,21 @@ struct objcore {
 	struct objhead		*objhead;
 	struct busyobj		*busyobj;
 	double			timer_when;
-	unsigned		flags;
+
+	uint16_t		flags;
 #define OC_F_BUSY		(1<<1)
 #define OC_F_PASS		(1<<2)
-#define OC_F_OFFLRU		(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
-#define OC_F_DYING		(1<<7)
 #define OC_F_PRIVATE		(1<<8)
 #define OC_F_FAILED		(1<<9)
-#define OC_F_MOVE		(1<<10)
-#define OC_F_INSERT		(1<<11)
-#define OC_F_EXP		(1<<12)
+
+	uint16_t		exp_flags;
+#define OC_EF_OFFLRU		(1<<4)
+#define OC_EF_MOVE		(1<<10)
+#define OC_EF_INSERT		(1<<11)
+#define OC_EF_EXP		(1<<12)
+#define OC_EF_DYING		(1<<7)
+
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 4f21152..8bd0707 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -111,9 +111,9 @@ exp_mail_it(struct objcore *oc)
 {
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	AN(oc->flags & OC_F_OFFLRU);
+	AN(oc->exp_flags & OC_EF_OFFLRU);
 	Lck_Lock(&exphdl->mtx);
-	if (oc->flags & OC_F_DYING)
+	if (oc->exp_flags & OC_EF_DYING)
 		VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
 	else
 		VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
@@ -135,8 +135,8 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	AZ(oc->flags & (OC_F_OFFLRU | OC_F_INSERT | OC_F_MOVE));
-	AZ(oc->flags & OC_F_DYING);
+	AZ(oc->exp_flags & (OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_MOVE));
+	AZ(oc->exp_flags & OC_EF_DYING);
 	// AN(oc->flags & OC_F_BUSY);
 
 	if (lru == NULL)
@@ -145,9 +145,9 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 
 	Lck_Lock(&lru->mtx);
 	lru->n_objcore++;
-	oc->flags |= OC_F_OFFLRU | OC_F_INSERT | OC_F_EXP;
+	oc->exp_flags |= OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_EXP;
 	if (when < 0)
-		oc->flags |= OC_F_MOVE;
+		oc->exp_flags |= OC_EF_MOVE;
 	else
 		oc->timer_when = when;
 	Lck_Unlock(&lru->mtx);
@@ -205,9 +205,9 @@ EXP_Touch(struct objcore *oc, double now)
 	if (Lck_Trylock(&lru->mtx))
 		return;
 
-	AN(oc->flags & OC_F_EXP);
+	AN(oc->exp_flags & OC_EF_EXP);
 
-	if (!(oc->flags & OC_F_OFFLRU)) {
+	if (!(oc->exp_flags & OC_EF_OFFLRU)) {
 		/* Can only touch it while it's actually on the LRU list */
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
@@ -234,7 +234,7 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
-	AN(oc->flags & OC_F_EXP);
+	AN(oc->exp_flags & OC_EF_EXP);
 
 	if (!isnan(ttl))
 		o->exp.ttl = now + ttl - o->exp.t_origin;
@@ -257,14 +257,14 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 	Lck_Lock(&lru->mtx);
 
 	if (!isnan(now) && when <= now)
-		oc->flags |= OC_F_DYING;
+		oc->exp_flags |= OC_EF_DYING;
 	else
-		oc->flags |= OC_F_MOVE;
+		oc->exp_flags |= OC_EF_MOVE;
 
-	if (oc->flags & OC_F_OFFLRU) {
+	if (oc->exp_flags & OC_EF_OFFLRU) {
 		oc = NULL;
 	} else {
-		oc->flags |= OC_F_OFFLRU;
+		oc->exp_flags |= OC_EF_OFFLRU;
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 	}
 	Lck_Unlock(&lru->mtx);
@@ -294,8 +294,8 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 		VSLb(bo->vsl, SLT_ExpKill, "LRU_Cand p=%p f=0x%x r=%d",
 		    oc, oc->flags, oc->refcnt);
 
-		AZ(oc->flags & OC_F_OFFLRU);
-		AZ(oc->flags & OC_F_DYING);
+		AZ(oc->exp_flags & OC_EF_OFFLRU);
+		AZ(oc->exp_flags & OC_EF_DYING);
 
 		/*
 		 * It wont release any space if we cannot release the last
@@ -309,7 +309,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 		if (Lck_Trylock(&oh->mtx))
 			continue;
 		if (oc->refcnt == 1) {
-			oc->flags |= OC_F_DYING | OC_F_OFFLRU;
+			oc->exp_flags |= OC_EF_DYING | OC_EF_OFFLRU;
 			oc->refcnt++;
 			VSC_C_main->n_lru_nuked++; // XXX per lru ?
 			VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
@@ -434,20 +434,20 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 
 	/* Evacuate our action-flags, and put it back on the LRU list */
 	Lck_Lock(&lru->mtx);
-	flags = oc->flags;
-	AN(flags & OC_F_OFFLRU);
-	oc->flags &= ~(OC_F_INSERT | OC_F_MOVE);
+	flags = oc->exp_flags;
+	AN(flags & OC_EF_OFFLRU);
+	oc->exp_flags &= ~(OC_EF_INSERT | OC_EF_MOVE);
 	oc->last_lru = now;
-	if (!(flags & OC_F_DYING)) {
+	if (!(flags & OC_EF_DYING)) {
 		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
-		oc->flags &= ~OC_F_OFFLRU;
+		oc->exp_flags &= ~OC_EF_OFFLRU;
 	}
 	Lck_Unlock(&lru->mtx);
 
-	if (flags & OC_F_DYING) {
+	if (flags & OC_EF_DYING) {
 		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Kill p=%p e=%.9f f=0x%x", oc,
 		    oc->timer_when, oc->flags);
-		if (!(flags & OC_F_INSERT)) {
+		if (!(flags & OC_EF_INSERT)) {
 			assert(oc->timer_idx != BINHEAP_NOIDX);
 			binheap_delete(ep->heap, oc->timer_idx);
 		}
@@ -456,7 +456,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 		return;
 	}
 
-	if (flags & OC_F_MOVE) {
+	if (flags & OC_EF_MOVE) {
 		o = oc_getobj(&ep->wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc->timer_when = exp_when(o);
@@ -472,11 +472,11 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	 * XXX: the next moment and rip them out again.
 	 */
 
-	if (flags & OC_F_INSERT) {
+	if (flags & OC_EF_INSERT) {
 		assert(oc->timer_idx == BINHEAP_NOIDX);
 		binheap_insert(exphdl->heap, oc);
 		assert(oc->timer_idx != BINHEAP_NOIDX);
-	} else if (flags & OC_F_MOVE) {
+	} else if (flags & OC_EF_MOVE) {
 		assert(oc->timer_idx != BINHEAP_NOIDX);
 		binheap_reorder(exphdl->heap, oc->timer_idx);
 		assert(oc->timer_idx != BINHEAP_NOIDX);
@@ -514,11 +514,11 @@ exp_expire(struct exp_priv *ep, double now)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	Lck_Lock(&lru->mtx);
 	// AZ(oc->flags & OC_F_BUSY);
-	oc->flags |= OC_F_DYING;
-	if (oc->flags & OC_F_OFFLRU)
+	oc->exp_flags |= OC_EF_DYING;
+	if (oc->exp_flags & OC_EF_OFFLRU)
 		oc = NULL;
 	else {
-		oc->flags |= OC_F_OFFLRU;
+		oc->exp_flags |= OC_EF_OFFLRU;
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 	}
 	Lck_Unlock(&lru->mtx);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index d7263e1..6cb2de8 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -401,7 +401,9 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
 
-		if (oc->flags & (OC_F_FAILED | OC_F_DYING))
+		if (oc->exp_flags & OC_EF_DYING)
+			continue;
+		if (oc->flags & OC_F_FAILED)
 			continue;
 
 		if (oc->flags & OC_F_BUSY) {
@@ -572,7 +574,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
-		if (oc->flags & (OC_F_BUSY|OC_F_DYING)) {
+		if (oc->flags & OC_F_BUSY) {
 			/*
 			 * We cannot purge busy objects here, because their
 			 * owners have special rights to them, and may nuke
@@ -581,6 +583,8 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 			 */
 			continue;
 		}
+		if (oc->exp_flags & OC_EF_DYING)
+			continue;
 		xxxassert(spc >= sizeof *ocp);
 		oc->refcnt++;
 		spc -= sizeof *ocp;
@@ -680,7 +684,7 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	if (!(oc->flags & OC_F_PRIVATE)) {
 		BAN_NewObjCore(oc);
 		EXP_Insert(oc);
-		AN(oc->flags & OC_F_EXP);
+		AN(oc->exp_flags & OC_EF_EXP);
 		AN(oc->ban);
 	}
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d3d70d6..fd9783c 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -100,7 +100,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	assert(req->obj->objcore->refcnt > 0);
 
 	req->t_resp = W_TIM_real(wrk);
-	if (req->obj->objcore->flags & OC_F_EXP)
+	if (req->obj->objcore->exp_flags & OC_EF_EXP)
 		EXP_Touch(req->obj->objcore, req->t_resp);
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
@@ -399,7 +399,8 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	switch (wrk->handling) {
 	case VCL_RET_DELIVER:
 		if (boc != NULL) {
-			AZ(oc->flags & (OC_F_FAILED|OC_F_DYING|OC_F_PASS));
+			AZ(oc->flags & (OC_F_FAILED|OC_F_PASS));
+			AZ(oc->exp_flags & OC_EF_DYING);
 			AZ(oc->busyobj);
 			VBF_Fetch(wrk, req, boc, o, VBF_BACKGROUND);
 		} else {



From phk at FreeBSD.org  Tue Mar 25 10:47:34 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 25 Mar 2014 11:47:34 +0100
Subject: [master] ac21ca0 If the backend sends "Content-Length: 0" and
 "Content_Encoding: gzip" it is buggy,
 work around it. If the backend sends chunked encoding we don't catch it and
 fail the transfer.
Message-ID: <E1WSOtK-0004rj-9J@project.varnish-software.com>


commit ac21ca073d7a93268e9f7340519e2e9cc8db1d00
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 25 10:46:35 2014 +0000

    If the backend sends "Content-Length: 0" and "Content_Encoding: gzip"
    it is buggy, work around it.  If the backend sends chunked encoding
    we don't catch it and fail the transfer.
    
    Fixes: #1320

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 93be439..ecbcf2e 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -416,6 +416,20 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->do_gzip && !bo->is_gunzip)
 		bo->do_gzip = 0;
 
+	AN(bo->vbc);
+	est = V1F_Setup_Fetch(bo);
+
+	if (est == 0) {
+		/*
+		 * If the length is known to be zero, it's not gziped.
+		 * A similar issue exists for chunked encoding but we
+		 * don't handle that.  See #1320.
+		 */
+		http_Unset(bo->beresp, H_Content_Encoding);
+		bo->is_gzip = 0;
+		bo->is_gunzip = 1;
+	}
+
 	/* But we can't do both at the same time */
 	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
 
@@ -425,9 +439,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	else if (bo->do_gunzip)
 		http_Unset(bo->beresp, H_Content_Encoding);
 
-	AN(bo->vbc);
-	est = V1F_Setup_Fetch(bo);
-
 	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
 		RFC2616_Weaken_Etag(bo->beresp);
 		VFP_Push(bo, vfp_gunzip_pull, 0);
diff --git a/bin/varnishtest/tests/r01320.vtc b/bin/varnishtest/tests/r01320.vtc
new file mode 100644
index 0000000..568699d
--- /dev/null
+++ b/bin/varnishtest/tests/r01320.vtc
@@ -0,0 +1,15 @@
+varnishtest "#1320 zero-length gzip"
+
+server s1 {
+	rxreq
+	txresp -hdr "Content-Encoding: gzip"
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+
+client c1 {
+	txreq -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.status == 200
+} -run



From perbu at varnish-software.com  Tue Mar 25 11:17:01 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 25 Mar 2014 12:17:01 +0100
Subject: [master] c17fc43 Reorder command line options with the binary and add
 the missing ones. Reformat for easier editing.
Message-ID: <E1WSPLp-0007ia-8Y@project.varnish-software.com>


commit c17fc433fa65c4294814fe810efc7f413038075d
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 25 12:16:52 2014 +0100

    Reorder command line options with the binary and add the missing ones. Reformat for easier editing.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index a93ecac..506e2e2 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -12,17 +12,19 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config]
-	 [-g group] [-h type[,options]] [-i identity]
-	 [-l shmlogsize] [-n name] [-P file] [-p param=value]
-	 [-s type[,options]] [-T address[:port]] [-t ttl]
-	 [-r param[,param...]] [-u user] [-V]
+varnishd [-a address[:port]] [-b host[:port]] [-C] [-d] [-f config]
+	 [-F] [-g group] [-h type[,options]] [-i identity]
+	 [-l shl[,free[,fill]]] [-M address:port] [-n name] 
+         [-P file] [-p param=value] [-r param[,param...]
+	 [-s [name=]kind[,options]] [-S secret-file] [-T address[:port]] 
+         [-t ttl] [-u user] [-V]
 
 DESCRIPTION
 ===========
 
-The varnishd daemon accepts HTTP requests from clients, passes them on to a backend server and caches the
-returned documents to better satisfy future requests for the same document.
+The varnishd daemon accepts HTTP requests from clients, passes them on
+to a backend server and caches the returned documents to better
+satisfy future requests for the same document.
 
 .. _ref-varnishd-options:
 
@@ -30,33 +32,39 @@ OPTIONS
 =======
 
 -a address[:port][,address[:port][...]
-            Listen for client requests on the specified address and port.  The address can be a host
-            name (?localhost?), an IPv4 dotted-quad (?127.0.0.1?), or an IPv6 address enclosed in
-            square brackets (?[::1]?).  If address is not specified, varnishd will listen on all
-            available IPv4 and IPv6 interfaces.  If port is not specified, the default HTTP port as
-            listed in /etc/services is used.  Multiple listening addresses and ports can be
-            specified as a whitespace or comma -separated list.
+            Listen for client requests on the specified address and
+            port.  The address can be a host name (?localhost?), an
+            IPv4 dotted-quad (?127.0.0.1?), or an IPv6 address
+            enclosed in square brackets (?[::1]?).  If address is not
+            specified, varnishd will listen on all available IPv4 and
+            IPv6 interfaces.  If port is not specified, the default
+            HTTP port as listed in /etc/services is used.  Multiple
+            listening addresses and ports can be specified as a
+            whitespace or comma -separated list.
 
 -b host[:port]
-            Use the specified host as backend server.  If port is not specified,
-	        the default is 8080.
+            Use the specified host as backend server.  If port is not
+	    specified, the default is 8080.
 
--C	        Print VCL code compiled to C language and exit. Specify the VCL file
-	        to compile with the -f option.
+-C	    Print VCL code compiled to C language and exit. Specify the VCL file
+	    to compile with the -f option.
 
--d          Enables debugging mode: The parent process runs in the foreground with a CLI connection
-            on stdin/stdout, and the child process must be started explicitly with a CLI command.
-            Terminating the parent process will also terminate the child.
-
--F          Run in the foreground.
+-d          Enables debugging mode: The parent process runs in the foreground
+            with a CLI connection on stdin/stdout, and the child
+            process must be started explicitly with a CLI command.
+            Terminating the parent process will also terminate the
+            child.
 
 -f config   Use the specified VCL configuration file instead of the
             builtin default.  See vcl(7) for details on VCL
             syntax. When no configuration is supplied varnishd will
             not start the cache process.
 
--g group    Specifies the name of an unprivileged group to which the child process should switch
-            before it starts accepting connections.  This is a shortcut for specifying the group
+-F          Run in the foreground.
+
+-g group    Specifies the name of an unprivileged group to which the
+            child process should switch before it starts accepting
+            connections.  This is a shortcut for specifying the group
             run-time parameter.
 
 -h type[,options]
@@ -66,14 +74,25 @@ OPTIONS
             Specify the identity of the Varnish server.  This can be accessed using server.identity
             from VCL
 
--l shmlogsize
-            Specify size of shmlog file.  Scaling suffixes like 'k', 'm' can be used up to
-            (e)tabytes.  Default is 80 Megabytes.  Specifying less than 8 Megabytes is unwise.
+-l shl[,free[,fill]]
+            Specifies size of shmlog file. shl is the store for the
+            shared memory log records [80m], free is the store for other
+            allocations [1m] and fill determines how the log is [+].
+            Scaling suffixes like 'k', 'm' can be used up to
+            (e)tabytes.  Default is 80 Megabytes.
 
--n name     Specify a name for this instance.  Amonst other things, this name is used to construct
-            the name of the directory in which varnishd keeps temporary files and persistent state.
-            If the specified name begins with a forward slash, it is interpreted as the absolute
-            path to the directory which should be used for this purpose.
+-M address:port
+            Connect to this port and offer the command line interface.
+            Think of it as a reverse shell. When running with -M and there is
+            no backend defined the child process (the cache) will not start
+            initially.
+
+-n name     Specify the name for this instance.  Amonst other things, this
+            name is used to construct the name of the directory in
+            which varnishd keeps temporary files and persistent state.
+            If the specified name begins with a forward slash, it is
+            interpreted as the absolute path to the directory which
+            should be used for this purpose.
 
 -P file     Write the process's PID to the specified file.
 
@@ -82,7 +101,13 @@ OPTIONS
             Run-Time Parameters for a list of parameters. This option can be
             used multiple times to specify multiple parameters.
 
--S file     Path to a file containing a secret used for authorizing access to the management port.
+-r param[,param...]
+            Make the listed parameters read only. This gives the
+            system administrator a way to limit what the Varnish CLI can do.
+            Consider making parameters such as *user*, *group*, *cc_command*,
+            *vcc_allow_inline_c* read only as these can potentially be used
+            to escalate privileges from the CLI.
+            Protecting *listen_address* may also be a good idea.
 
 -s [name=]type[,options]
             Use the specified storage backend. The storage backends can be one of the following:
@@ -95,27 +120,15 @@ OPTIONS
             multiple times to specify multiple storage files. Names
             are referenced in logs, vcl, statistics, etc.
 
+-S file     Path to a file containing a secret used for authorizing access to the management port.
+
 -T address[:port]
             Offer a management interface on the specified address and port.  See Management
             Interface for a list of management commands.
 
--M address:port
-            Connect to this port and offer the command line interface.
-            Think of it as a reverse shell. When running with -M and there is
-            no backend defined the child process (the cache) will not start
-            initially.
-
 -t ttl      Specifies a hard minimum time to live for cached documents. This
             is a shortcut for specifying the default_ttl run-time parameter.
 
--r param[,param...]
-            Make the listed parameters read only. This gives the
-            system administrator a way to limit what the Varnish CLI can do.
-            Consider making parameters such as *user*, *group*, *cc_command*,
-            *vcc_allow_inline_c* read only as these can potentially be used
-            to escalate privileges from the CLI.
-            Protecting *listen_address* may also be a good idea.
-
 -u user     Specifies the name of an unprivileged user to which the child
             process should switch before it starts accepting
             connections. This is a shortcut for specifying the user



From perbu at varnish-software.com  Tue Mar 25 11:24:50 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 25 Mar 2014 12:24:50 +0100
Subject: [master] 1e08789 zap a warning. Formatting error
Message-ID: <E1WSPTO-00022F-9J@project.varnish-software.com>


commit 1e08789d6c99866250dc2fcab98d72eb76981152
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 25 12:24:47 2014 +0100

    zap a warning. Formatting error

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index a8ef31c..9b61d35 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -138,8 +138,6 @@ figuring out what.
 Installing
 ----------
 
-And finally, the true test of a brave heart::
-
 And finally, the true test of a brave heart: ``sudo make install``
 
 Varnish will now be installed in `/usr/local`. The `varnishd` binary is in



From perbu at varnish-software.com  Tue Mar 25 11:57:08 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 25 Mar 2014 12:57:08 +0100
Subject: [master] a1a10e0 Clear up hash algorithms. Critbit is teh shit.
Message-ID: <E1WSPye-00087B-N9@project.varnish-software.com>


commit a1a10e0790f9ffb2aa859e76f6f4d1c7670ef33c
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 25 12:56:21 2014 +0100

    Clear up hash algorithms.
    Critbit is teh shit.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 506e2e2..975fa14 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -145,20 +145,22 @@ Hash Algorithms
 
 The following hash algorithms are available:
 
+critbit 
+   A self-scaling tree structure. The default hash algorithm in
+    Varnish Cache 2.1 and onwards. In comparison to a more traditional
+    B tree the critbit tree is almost completely lockless. Do not 
+    change this unless you are certain what you're doing.
+
 simple_list
     A simple doubly-linked list.  Not recommended for production use.
 
 classic[,buckets]
-    A standard hash table.  This is the default.  The hash key is the
-    CRC32 of the object's URL modulo the size of the hash table.  Each
-    table entry points to a list of elements which share the same hash
-    key. The buckets parameter specifies the number of entries in the
-    hash table.  The default is 16383.
-
-critbit
-    A self-scaling tree structure. The default hash algorithm in 2.1. In
-    comparison to a more traditional B tree the critbit tree is almost
-    completely lockless.
+    A standard hash table. The hash key is the CRC32 of the object's
+    URL modulo the size of the hash table.  Each table entry points to
+    a list of elements which share the same hash key. The buckets
+    parameter specifies the number of entries in the hash table.  The
+    default is 16383.
+
 
 Storage Types
 -------------



From perbu at varnish-software.com  Tue Mar 25 11:57:08 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 25 Mar 2014 12:57:08 +0100
Subject: [master] a343a8e Discourage use of persistent. You need to know what
 you are doing.
Message-ID: <E1WSPye-00087H-OF@project.varnish-software.com>


commit a343a8ede5db704fb43da48f5047e682c93c2410
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 25 12:56:45 2014 +0100

    Discourage use of persistent.
    You need to know what you are doing.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 975fa14..3e30d19 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -188,7 +188,9 @@ syntax: persistent,path,size
 
 Persistent storage. Varnish will store objects in a file in a manner
 that will secure the survival of *most* of the objects in the event of
-a planned or unplanned shutdown of Varnish.
+a planned or unplanned shutdown of Varnish. The persistent storage
+backend has multiple issues with it and will likely be removed from a
+future version of Varnish.
 
 
 Management Interface



From perbu at varnish-software.com  Tue Mar 25 11:57:08 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 25 Mar 2014 12:57:08 +0100
Subject: [master] 2d38d19 Too much Varnish Software here.
Message-ID: <E1WSPye-00087Q-PP@project.varnish-software.com>


commit 2d38d194dc6f630a0f51f7b8855d539e22d7a20e
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Mar 25 12:57:02 2014 +0100

    Too much Varnish Software here.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 3e30d19..633ea43 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -255,7 +255,7 @@ HISTORY
 =======
 
 The varnishd daemon was developed by Poul-Henning Kamp in cooperation
-with Verdens Gang AS, Varnish Software AS and Varnish Software.
+with Verdens Gang AS and Varnish Software.
 
 This manual page was written by Dag-Erling Sm?rgrav with updates by
 Stig Sandbeck Mathisen <ssm at debian.org>.



From daghf at varnish-software.com  Tue Mar 25 14:28:31 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 25 Mar 2014 15:28:31 +0100
Subject: [master] 8a986ce Fix 'varnishstat -l' segmentation fault.
Message-ID: <E1WSSL9-00011I-Ix@project.varnish-software.com>


commit 8a986ce96e1396edb80fdd538d2045ca2dd2d115
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Mar 25 15:08:29 2014 +0100

    Fix 'varnishstat -l' segmentation fault.
    
    Fixes: #1450

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 10f6d51..31c0a1b 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -210,6 +210,10 @@ do_list_cb(void *priv, const struct VSC_point * const pt)
 	const struct VSC_section * sec;
 
 	(void)priv;
+
+	if (pt == NULL)
+		return (0);
+
 	sec = pt->section;
 	i = 0;
 	if (strcmp(sec->fantom->type, ""))



From daghf at varnish-software.com  Tue Mar 25 14:28:31 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 25 Mar 2014 15:28:31 +0100
Subject: [master] 26bc79e Fix 'varnishstat -l' lack of output.
Message-ID: <E1WSSL9-00011R-K5@project.varnish-software.com>


commit 26bc79ea9b1e743cf224883c45fd2b3fdbf7ebfb
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Mar 25 15:20:44 2014 +0100

    Fix 'varnishstat -l' lack of output.
    
    Postpone invoking list_fields() until after VSM_Open().

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 31c0a1b..7b37f0b 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -273,7 +273,7 @@ main(int argc, char * const *argv)
 {
 	int c;
 	struct VSM_data *vd;
-	int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0;
+	int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0, f_list = 0;
 
 	vd = VSM_New();
 
@@ -283,8 +283,8 @@ main(int argc, char * const *argv)
 			once = 1;
 			break;
 		case 'l':
-			list_fields(vd);
-			exit(0);
+			f_list = 1;
+			break;
 		case 'V':
 			VCS_Message("varnishstat");
 			exit(0);
@@ -310,7 +310,7 @@ main(int argc, char * const *argv)
 		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
 	}
-	if (!(xml || json || once)) {
+	if (!(xml || json || once || f_list)) {
 		do_curses(vd, delay);
 		exit(0);
 	}
@@ -322,6 +322,8 @@ main(int argc, char * const *argv)
 			do_json(vd);
 		else if (once)
 			do_once(vd, VSC_Main(vd, NULL));
+		else if (f_list)
+			list_fields(vd);
 		else {
 			assert(0);
 		}



From lkarsten at varnish-software.com  Tue Mar 25 16:54:36 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 25 Mar 2014 17:54:36 +0100
Subject: [master] f8c6b70 Document weight on the hash director
Message-ID: <E1WSUcW-00069C-2i@project.varnish-software.com>


commit f8c6b70e4edeed99d25f8c17521ebbb1981746af
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Mar 25 17:54:09 2014 +0100

    Document weight on the hash director

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 24e8839..3b21dc5 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -141,9 +141,13 @@ Example
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
-       Adds a backend to the director.
+	Adds a backend to the director with a certain weight.
+
+	Weight is used as in the round_robin director. Recommended value is
+	1.0 unless you have special needs.
+
 Example
-       hdir.add_backend(backend1);
+       hdir.add_backend(backend1, 1.0);
 
 $Method BACKEND .backend(STRING_LIST)
 



From lkarsten at varnish-software.com  Tue Mar 25 16:54:36 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 25 Mar 2014 17:54:36 +0100
Subject: [master] b112e9a Remove incomplete sentence and visible XXX
Message-ID: <E1WSUcW-00069I-4M@project.varnish-software.com>


commit b112e9a9e83ada9ff1f508d628e61a594cb09f41
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Mar 25 17:54:31 2014 +0100

    Remove incomplete sentence and visible XXX

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 3b21dc5..230bf7f 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -81,9 +81,9 @@ Example
 $Object fallback()
 
 Description
-        Creates a fallback director. This director will
+        Creates a fallback director.
 
-	XXX: Add description of fallback director.
+#	XXX: Add description of fallback director.
 
 Example
         new foo = directors.fallback();



From lkarsten at varnish-software.com  Tue Mar 25 17:32:45 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 25 Mar 2014 18:32:45 +0100
Subject: [master] efc1871 Improve backend_hint + director example
Message-ID: <E1WSVDR-0001CR-7s@project.varnish-software.com>


commit efc187150ab61681113f5ca54825670931724c40
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Mar 25 18:32:40 2014 +0100

    Improve backend_hint + director example

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 402b693..3634a33 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -21,9 +21,26 @@ vcl_fetch is now vcl_backend_response
 Directors have been moved to the vmod_directors
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+To make directors (backend selection logic) easier to extend, the directors
+are now defined in loadable VMODs.
+
+Setting a backend for future fetches in `vcl_recv` is now done as follows::
+
+    sub vcl_init {
+        new cluster1 = directors.round_robin();
+        cluster1.add_backend(b1, 1.0);
+        cluster1.add_backend(b2, 1.0);
+    }
+    sub vcl_recv {
+        set req.backend_hint = cluster1.backend();
+    }
+
+Note the extra `.backend()` needed after the director name.
+
 Use the hash director as a client director
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since the client director was already a special case of the hash director, it has been removed, and you should use the hash director directly::
+Since the client director was already a special case of the hash director, it
+has been removed, and you should use the hash director directly::
 
     sub vcl_init {
         new h = directors.hash();
@@ -102,6 +119,11 @@ The builtin VCL now honors Cache-Control: no-cache (and friends) to indicate
 uncacheable content from the backend.
 
 
+The `remove` keyword is gone
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Replaced by `unset`.
+
 Changes to parameters
 =====================
 



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] c204366 Introduce SLT_Timestamp mechanism for request
 handling timing information.
Message-ID: <E1WSmDu-0004ob-DI@project.varnish-software.com>


commit c2043664d86377909145e1e54f84c58fc40858ad
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 18 17:47:14 2014 +0100

    Introduce SLT_Timestamp mechanism for request handling timing information.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 68c74ff..ecc2185 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -48,6 +48,7 @@
 #include <string.h>
 #include <limits.h>
 #include <unistd.h>
+#include <math.h>
 
 #if defined(HAVE_EPOLL_CTL)
 #include <sys/epoll.h>
@@ -674,8 +675,10 @@ struct req {
 
 	char			*ws_req;	/* WS above request data */
 
-	double			t_req;
-	double			t_resp;
+	/* Timestamps */
+	double			t_first;	/* First timestamp logged */
+	double			t_prev;		/* Previous timestamp logged */
+	double			t_req;		/* Headers complete */
 
 	struct http_conn	htc[1];
 	const char		*client_identity;
@@ -1138,6 +1141,17 @@ void VSLbv(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, va_list va);
 void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...)
     __printflike(3, 4);
 void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t);
+void VSLb_ts(struct vsl_log *, const char *event, double first, double *pprev,
+    double now);
+static inline void
+VSLb_ts_req(struct req *req, const char *event, double now)
+{
+
+	if (isnan(req->t_first) || req->t_first == 0.)
+		req->t_first = req->t_prev = now;
+	VSLb_ts(req->vsl, event, req->t_first, &req->t_prev, now);
+}
+
 
 void VSL_Flush(struct vsl_log *, int overflow);
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 65ad51d..b1c3362 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -35,6 +35,7 @@
 
 #include "cache.h"
 
+#include "vtim.h"
 #include "cache_esi.h"
 #include "vend.h"
 #include "vgz.h"
@@ -98,12 +99,16 @@ ved_include(struct req *preq, const char *src, const char *host)
 	 */
 	req->req_step = R_STP_RECV;
 	req->t_req = preq->t_req;
+	assert(isnan(req->t_first));
+	assert(isnan(req->t_prev));
 	req->gzip_resp = preq->gzip_resp;
 	req->crc = preq->crc;
 	req->l_crc = preq->l_crc;
 
 	THR_SetRequest(req);
 
+	VSLb_ts_req(req, "Start", W_TIM_real(wrk));
+
 	while (1) {
 		req->wrk = wrk;
 		s = CNT_Request(wrk, req);
@@ -122,7 +127,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 	/* Reset the workspace */
 	WS_Reset(wrk->aws, wrk_ws_wm);	/* XXX ? */
 
-	WRW_Reserve(preq->wrk, &preq->sp->fd, preq->vsl, preq->t_resp);
+	WRW_Reserve(preq->wrk, &preq->sp->fd, preq->vsl, preq->t_prev);
 	if (preq->res_mode & RES_CHUNKED)
 		WRW_Chunked(preq->wrk);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ecbcf2e..e689913 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -877,6 +877,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 			AZ(bo->fetch_objcore->flags & OC_F_BUSY);
 		}
 	}
+	VSLb_ts_req(req, "Fetch", W_TIM_real(wrk));
 	THR_SetBusyobj(NULL);
 	VBO_DerefBusyObj(wrk, &bo);
 }
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 231df94..e82ed31 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -282,7 +282,7 @@ V1D_Deliver(struct req *req)
 	if (req->res_mode & RES_ESI)
 		RFC2616_Weaken_Etag(req->resp);
 
-	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp);
+	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_prev);
 
 	/*
 	 * Send HTTP protocol header, unless interior ESI object
@@ -389,7 +389,7 @@ V1D_Deliver_Synth(struct req *req)
 			v1d_dorange(req, r);
 	}
 
-	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp);
+	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_prev);
 
 	/*
 	 * Send HTTP protocol header, unless interior ESI object
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 5d0f286..37ed35a 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -103,8 +103,10 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->vcl);
 	AZ(req->obj);
 	AZ(req->esi_level);
+	assert(!isnan(sp->t_idle));
+	assert(isnan(req->t_first));
+	assert(isnan(req->t_prev));
 	assert(isnan(req->t_req));
-	assert(isnan(req->t_resp));
 
 	tmo = (int)(1e3 * cache_param->timeout_linger);
 	while (1) {
@@ -120,7 +122,10 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			hs = HTTP1_Complete(req->htc);
 		if (hs == HTTP1_COMPLETE) {
 			/* Got it, run with it */
-			req->t_req = now;
+			if (isnan(req->t_first))
+				VSLb_ts_req(req, "Start", now);
+			VSLb_ts_req(req, "Req", now);
+			req->t_req = req->t_prev;
 			return (REQ_FSM_MORE);
 		} else if (hs == HTTP1_ERROR_EOF) {
 			why = SC_REM_CLOSE;
@@ -138,7 +143,6 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			when = sp->t_idle + cache_param->timeout_linger;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
-				req->t_req = NAN;
 				wrk->stats.sess_herd++;
 				SES_ReleaseReq(req);
 				WAIT_Enter(sp);
@@ -146,9 +150,10 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			}
 		} else {
 			/* Working on it */
-			if (isnan(req->t_req))
-				req->t_req = now;
-			when = req->t_req + cache_param->timeout_req;
+			if (isnan(req->t_first))
+				/* Timestamp Start on first byte received */
+				VSLb_ts_req(req, "Start", now);
+			when = sp->t_idle + cache_param->timeout_req;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
 				why = SC_RX_TIMEOUT;
@@ -195,11 +200,15 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 		req->vcl = NULL;
 	}
 
-	sp->t_idle = W_TIM_real(wrk);
+	if (!isnan(req->t_prev) && req->t_prev > 0.)
+		sp->t_idle = req->t_prev;
+	else
+		sp->t_idle = W_TIM_real(wrk);
 	VSL_Flush(req->vsl, 0);
 
+	req->t_first = NAN;
+	req->t_prev = NAN;
 	req->t_req = NAN;
-	req->t_resp = NAN;
 	req->req_body_status = REQ_BODY_INIT;
 
 	// req->req_bodybytes = 0;
@@ -226,7 +235,9 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->aws, NULL);
 
 	if (HTTP1_Reinit(req->htc) == HTTP1_COMPLETE) {
-		req->t_req = sp->t_idle;
+		VSLb_ts_req(req, "Start", sp->t_idle);
+		VSLb_ts_req(req, "Req", sp->t_idle);
+		req->t_req = req->t_prev;
 		wrk->stats.sess_pipeline++;
 		return (SESS_DONE_RET_START);
 	} else {
@@ -522,16 +533,19 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 		l = http1_iter_req_body(req, buf, sizeof buf);
 		if (l < 0) {
 			req->doclose = SC_RX_BODY;
-			return (l);
+			break;
 		}
 		if (l > 0) {
 			i = func(req, priv, buf, l);
 			if (i) {
-				return (i);
+				l = i;
+				break;
 			}
 		}
 	} while (l > 0);
-	return(0);
+	VSLb_ts_req(req, "ReqBody", VTIM_real());
+
+	return (l);
 }
 
 /*----------------------------------------------------------------------
@@ -605,7 +619,8 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 			    req->h1.bytes_yet : cache_param->fetch_chunksize);
 			if (st == NULL) {
 				req->req_body_status = REQ_BODY_FAIL;
-				return (-1);
+				l = -1;
+				break;
 			} else {
 				VTAILQ_INSERT_TAIL(&req->body, st, list);
 			}
@@ -615,11 +630,12 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		l = http1_iter_req_body(req, st->ptr + st->len, l);
 		if (l < 0) {
 			req->doclose = SC_RX_BODY;
-			return (l);
+			break;
 		}
 		if (req->req_bodybytes > maxsize) {
 			req->req_body_status = REQ_BODY_FAIL;
-			return (-1);
+			l = -1;
+			break;
 		}
 		if (l > 0) {
 			st->len += l;
@@ -627,6 +643,8 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 				st = NULL;
 		}
 	} while (l > 0);
-	req->req_body_status = REQ_BODY_CACHED;
-	return(0);
+	if (l == 0)
+		req->req_body_status = REQ_BODY_CACHED;
+	VSLb_ts_req(req, "ReqBody", VTIM_real());
+	return (l);
 }
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index b4403bb..1a0a44f 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -89,14 +89,14 @@ PipeRequest(struct req *req, struct busyobj *bo)
 
 	i = WRW_FlushRelease(wrk);
 
+	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
+
 	if (i) {
 		SES_Close(req->sp, SC_TX_PIPE);
 		VDI_CloseFd(&vc);
 		return;
 	}
 
-	req->t_resp = VTIM_real();
-
 	memset(fds, 0, sizeof fds);
 
 	// XXX: not yet (void)VTCP_linger(vc->fd, 0);
@@ -130,6 +130,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 			fds[1].fd = -1;
 		}
 	}
+	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
 	SES_Close(req->sp, SC_TX_PIPE);
 	VDI_CloseFd(&vc);
 	bo->vbc = NULL;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index fd9783c..9826ae7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -88,6 +88,7 @@ static enum req_fsm_nxt
 cnt_deliver(struct worker *wrk, struct req *req)
 {
 	char time_str[30];
+	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -99,9 +100,10 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	assert(req->obj->objcore->refcnt > 0);
 
-	req->t_resp = W_TIM_real(wrk);
+	now = W_TIM_real(wrk);
+	VSLb_ts_req(req, "Process", now);
 	if (req->obj->objcore->exp_flags & OC_EF_EXP)
-		EXP_Touch(req->obj->objcore, req->t_resp);
+		EXP_Touch(req->obj->objcore, now);
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 
@@ -109,7 +111,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	http_FilterResp(req->obj->http, req->resp, 0);
 
 	http_Unset(req->resp, H_Date);
-	VTIM_format(req->t_resp, time_str);
+	VTIM_format(now, time_str);
 	http_PrintfHeader(req->resp, "Date: %s", time_str);
 
 	if (req->wrk->stats.cache_hit)
@@ -121,7 +123,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
 	http_PrintfHeader(req->resp, "Age: %.0f",
-	    req->t_resp - req->obj->exp.t_origin);
+	    now - req->obj->exp.t_origin);
 
 	http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
 
@@ -152,6 +154,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
 
 	V1D_Deliver(req);
+	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 
 	if (http_HdrIs(req->resp, H_Connection, "close"))
 		req->doclose = SC_RESP_CLOSE;
@@ -192,6 +195,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 {
 	char date[40];
 	struct http *h;
+	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -200,7 +204,9 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	h = req->resp;
-	req->t_resp = VTIM_real();
+
+	now = W_TIM_real(wrk);
+	VSLb_ts_req(req, "Process", now);
 
 	if (req->err_code < 100 || req->err_code > 999)
 		req->err_code = 501;
@@ -208,7 +214,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 	http_ClrHeader(h);
 	http_PutProtocol(h, "HTTP/1.1");
 	http_PutStatus(h, req->err_code);
-	VTIM_format(req->t_resp, date);
+	VTIM_format(now, date);
 	http_PrintfHeader(h, "Date: %s", date);
 	http_SetHeader(h, "Server: Varnish");
 	http_PrintfHeader(req->resp,
@@ -242,6 +248,8 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	V1D_Deliver_Synth(req);
 
+	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
+
 	VSB_delete(req->synth_body);
 	req->synth_body = NULL;
 
@@ -326,6 +334,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	struct object *o;
 	struct objhead *oh;
 	enum lookup_e lr;
+	int had_objhead = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -336,6 +345,8 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	VRY_Prep(req);
 
 	AZ(req->objcore);
+	if (req->hash_objhead)
+		had_objhead = 1;
 	lr = HSH_Lookup(req, &oc, &boc,
 	    req->esi_level == 0 ? 1 : 0,
 	    req->hash_always_miss ? 1 : 0
@@ -349,6 +360,8 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		 */
 		return (REQ_FSM_DISEMBARK);
 	}
+	if (had_objhead)
+		VSLb_ts_req(req, "Waitinglist", W_TIM_real(wrk));
 
 	if (boc == NULL) {
 		VRY_Finish(req, DISCARD);
@@ -631,12 +644,14 @@ cnt_restart(struct worker *wrk, struct req *req)
 	} else {
 		wid = VXID_Get(&wrk->vxid_pool);
 		// XXX: ReqEnd + ReqAcct ?
+		VSLb_ts_req(req, "Restart", W_TIM_real(wrk));
 		VSLb(req->vsl, SLT_Link, "req %u restart", wid);
 		VSLb(req->vsl, SLT_End, "%s", "");
 		VSL_Flush(req->vsl, 0);
 		owid = req->vsl->wid & VSL_IDENTMASK;
 		req->vsl->wid = wid | VSL_CLIENTMARKER;
 		VSLb(req->vsl, SLT_Begin, "req %u restart", owid);
+		VSLb_ts_req(req, "Start", req->t_prev);
 		req->err_code = 0;
 		req->req_step = R_STP_RECV;
 	}
@@ -677,6 +692,10 @@ cnt_recv(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->objcore);
 
+	assert(!isnan(req->t_first));
+	assert(!isnan(req->t_prev));
+	assert(!isnan(req->t_req));
+
 	VSLb(req->vsl, SLT_ReqStart, "%s %s",
 	    req->sp->client_addr_str, req->sp->client_port_str);
 
@@ -882,12 +901,6 @@ CNT_Request(struct worker *wrk, struct req *req)
 			VSLb(req->vsl, SLT_Length, "%ju",
 			    (uintmax_t)req->resp_bodybytes);
 		}
-		VSLb(req->vsl, SLT_ReqEnd, "%.9f %.9f %.9f %.9f %.9f",
-		    req->t_req,
-		    req->sp->t_idle,
-		    req->sp->t_idle - req->t_resp,
-		    req->t_resp - req->t_req,
-		    req->sp->t_idle - req->t_resp);
 
 		while (!VTAILQ_EMPTY(&req->body)) {
 			st = VTAILQ_FIRST(&req->body);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 96a2507..73ed5c7 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -399,8 +399,9 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 
 	WS_Init(req->ws, "req", p, e - p);
 
+	req->t_first = NAN;
+	req->t_prev = NAN;
 	req->t_req = NAN;
-	req->t_resp = NAN;
 
 	VTAILQ_INIT(&req->body);
 
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 81f5a03..19048c2 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -350,6 +350,17 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
 	va_end(ap);
 }
 
+void
+VSLb_ts(struct vsl_log *vsl, const char *event, double first, double *pprev,
+    double now)
+{
+
+	assert(!isnan(now) && now != 0.);
+	VSLb(vsl, SLT_Timestamp, "%s: %.9f %.9f %.9f",
+	    event, now, now - first, now - *pprev);
+	*pprev = now;
+}
+
 /*--------------------------------------------------------------------
  * Setup a VSL buffer, allocate space if none provided.
  */
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 5f768ec..34c5c13 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -462,4 +462,21 @@ SLTM(Storage, 0, "Where object is stored",
 	"\t+---- Type (\"malloc\", \"file\", \"persistent\" etc.)\n"
 	"\n"
 )
+
+SLTM(Timestamp, 0, "Timing information",
+	"Contains timing information for the Varnish worker threads.\n\n"
+	"Time stamps are issued by Varnish on certain events,"
+	" and show the absolute time of the event, the time spent since the"
+	" start of the work unit, and the time spent since the last timestamp"
+	" was logged.\n\n"
+	"The format is::\n\n"
+	"\t%s: %f %f %f\n"
+	"\t|   |  |  |\n"
+	"\t|   |  |  +- Time since last timestamp\n"
+	"\t|   |  +---- Time since start of work unit\n"
+	"\t|   +------- Absolute time of event\n"
+	"\t+----------- Event label\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] dc66d1c Reduce logged timer precision to microseconds
Message-ID: <E1WSmDu-0004oi-EX@project.varnish-software.com>


commit dc66d1cbdcba1e5ac2ccf428ac147d2ed6788bb2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 24 14:34:15 2014 +0100

    Reduce logged timer precision to microseconds

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 19048c2..900ecda 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -356,7 +356,7 @@ VSLb_ts(struct vsl_log *vsl, const char *event, double first, double *pprev,
 {
 
 	assert(!isnan(now) && now != 0.);
-	VSLb(vsl, SLT_Timestamp, "%s: %.9f %.9f %.9f",
+	VSLb(vsl, SLT_Timestamp, "%s: %.6f %.6f %.6f",
 	    event, now, now - first, now - *pprev);
 	*pprev = now;
 }



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] e15f957 Backend fetch timestamps
Message-ID: <E1WSmDu-0004or-Fs@project.varnish-software.com>


commit e15f9573f1536497716e4f6d08b58b427e4db36c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 25 16:28:31 2014 +0100

    Backend fetch timestamps

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ecc2185..ea5d990 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -569,11 +569,8 @@ struct busyobj {
 	double			between_bytes_timeout;
 
 	/* Timers */
-	double			t_start;
-	double			t_send;
-	double			t_sent;
-	double			t_hdr;
-	double			t_body;
+	double			t_first;	/* First timestamp logged */
+	double			t_prev;		/* Previous timestamp logged */
 
 	const char		*storage_hint;
 	struct director		*director;
@@ -1143,6 +1140,7 @@ void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...)
 void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t);
 void VSLb_ts(struct vsl_log *, const char *event, double first, double *pprev,
     double now);
+
 static inline void
 VSLb_ts_req(struct req *req, const char *event, double now)
 {
@@ -1152,6 +1150,15 @@ VSLb_ts_req(struct req *req, const char *event, double now)
 	VSLb_ts(req->vsl, event, req->t_first, &req->t_prev, now);
 }
 
+static inline void
+VSLb_ts_busyobj(struct busyobj *bo, const char *event, double now)
+{
+
+	if (isnan(bo->t_first) || bo->t_first == 0.)
+		bo->t_first = bo->t_prev = now;
+	VSLb_ts(bo->vsl, event, bo->t_first, &bo->t_prev, now);
+}
+
 
 void VSL_Flush(struct vsl_log *, int overflow);
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index db1c7a1..69f849b 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -149,6 +149,8 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	VCL_Ref(bo->vcl);
 
 	bo->t_fetch = req->t_req;
+	assert(!isnan(bo->t_fetch) && bo->t_fetch != 0.);
+	bo->t_first = bo->t_prev = NAN;
 
 	return (bo);
 }
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e689913..cd627cd 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -226,6 +226,8 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk));
+
 	// XXX: BereqEnd + BereqAcct ?
 	wid = VXID_Get(&wrk->vxid_pool);
 	VSLb(bo->vsl, SLT_Link, "bereq %u retry", wid);
@@ -234,6 +236,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 	owid = bo->vsl->wid & VSL_IDENTMASK;
 	bo->vsl->wid = wid | VSL_BACKENDMARKER;
 	VSLb(bo->vsl, SLT_Begin, "bereq %u retry", owid);
+	VSLb_ts_busyobj(bo, "Start", bo->t_prev);
 
 	return (F_STP_STARTFETCH);
 }
@@ -285,9 +288,11 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	 * Do a single retry in that case.
 	 */
 	if (i == 1) {
+		VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk));
 		VSC_C_main->backend_retry++;
 		i = V1F_fetch_hdr(wrk, bo, bo->req);
 	}
+	VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk));
 
 	if (i) {
 		AZ(bo->vbc);
@@ -364,8 +369,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	bo->t_body = VTIM_mono();
-
 	return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
 }
 
@@ -525,6 +528,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		assert(bo->state == BOS_REQ_DONE);
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
+	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
@@ -617,6 +621,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	assert(obj->len == al);
 	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
 	VBO_setstate(bo, BOS_FINISHED);
+	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	return (F_STP_DONE);
 }
 
@@ -633,6 +638,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	VSLb_ts_busyobj(bo, "Error", W_TIM_real(wrk));
+
 	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
 	AZ(bo->synth_body);
@@ -720,7 +727,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 {
 	struct busyobj *bo;
 	enum fetch_step stp;
-	double t_hdr, t_body;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
@@ -729,11 +735,9 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	THR_SetBusyobj(bo);
 	stp = F_STP_MKBEREQ;
-	bo->t_start = VTIM_real();
-	bo->t_send = NAN;
-	bo->t_sent = NAN;
-	bo->t_hdr = NAN;
-	bo->t_body = NAN;
+	assert(isnan(bo->t_first));
+	assert(isnan(bo->t_prev));
+	VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk));
 
 	bo->stats = &wrk->stats;
 
@@ -790,14 +794,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	if (bo->ims_obj != NULL)
 		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
 
-	t_hdr = bo->t_hdr - bo->t_sent;
-	t_body = bo->t_body - bo->t_hdr;
-	VSLb(bo->vsl, SLT_BereqEnd, "%.9f %.9f %.9f %.9f %.9f %.9f",
-	     bo->t_start,
-	     VTIM_real(),
-	     bo->t_sent - bo->t_send,
-	     t_hdr, t_body, t_hdr + t_body);
-
 	VBO_DerefBusyObj(wrk, &bo);
 	THR_SetBusyobj(NULL);
 }
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index e25ced9..317bfcb 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -311,8 +311,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 		VDI_AddHostHeader(bo->bereq, vc);
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
-	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_fetch);
-	bo->t_send = VTIM_mono();
+	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
 	(void)HTTP1_Write(wrk, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might (still) have */
@@ -334,13 +333,14 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	if (WRW_FlushRelease(wrk) || i != 0) {
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
+		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 		VDI_CloseFd(&bo->vbc);
 		/* XXX: other cleanup ? */
 		return (retry);
 	}
+	VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 
 	VSC_C_main->backend_req++;
-	bo->t_sent = VTIM_mono();
 
 	/* Receive response */
 
@@ -377,7 +377,6 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 			    vc->between_bytes_timeout);
 		}
 	} while (hs != HTTP1_COMPLETE);
-	bo->t_hdr = VTIM_mono();
 
 	hp = bo->beresp;
 



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] 7b5ee1a Timestamp documentation
Message-ID: <E1WSmDu-0004p1-HH@project.varnish-software.com>


commit 7b5ee1a64c41dbe0044a74a6a9ea562309a28f04
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 25 16:28:58 2014 +0100

    Timestamp documentation

diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst
index 78c3456..36d51bd 100644
--- a/doc/sphinx/reference/vsl.rst
+++ b/doc/sphinx/reference/vsl.rst
@@ -20,6 +20,83 @@ VSL tags
 
 .. include:: ../../../lib/libvarnishapi/vsl-tags.rst
 
+TIMESTAMPS
+==========
+
+Timestamps are inserted in the log on completing certain events during
+the worker thread's task handling. The timestamps has a label showing
+which event was completed. The reported fields show the absolute time
+of the event, the time spent since the start of the task and the time
+spent since the last timestamp was logged.
+
+The timestamps logged automatically by Varnish are inserted after
+completing events that are expected to have delays (e.g. network IO or
+spending time on a waitinglist). Timestamps can also be inserted from
+VCL using the std.timestamp() method. If one is doing time consuming
+tasks in the VCL configuration, it's a good idea to log a timestamp
+after completing that task. This keeps the timing information in
+subsequent timestamps from including the time spent on the VCL event.
+
+Request handling timestamps
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Start
+	The start of request processing (first byte received or
+	restart).
+
+Req
+	Complete client request received.
+
+ReqBody
+	Client request body processed (discarded, cached or passed to
+	the backend).
+
+Waitinglist
+	Came off waitinglist.
+
+Fetch
+	Fetch processing finished (completely fetched or ready for
+	streaming).
+
+Process
+	Processing finished, ready to deliver the client response.
+
+Resp
+	Delivery of response to the client finished.
+
+Restart
+	Client request is being restarted.
+
+Pipe handling timestamps
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Pipe
+	Opened a pipe to the backend and forwarded the request.
+
+PipeSess
+	The pipe session has finished.
+
+Backend fetch timestamps
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Start
+	Start of the backend fetch processing.
+
+Bereq
+	Backend request sent.
+
+Beresp
+	Backend response headers received.
+
+BerespBody
+	Backend response body received.
+
+Retry
+	Backend request is being retried.
+
+Error
+	Backend request failed to vcl_error.
+
 
 HISTORY
 =======
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 34c5c13..0f50a48 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -468,7 +468,8 @@ SLTM(Timestamp, 0, "Timing information",
 	"Time stamps are issued by Varnish on certain events,"
 	" and show the absolute time of the event, the time spent since the"
 	" start of the work unit, and the time spent since the last timestamp"
-	" was logged.\n\n"
+	" was logged. See vsl(7) for information about the individual"
+	" timestamps.\n\n"
 	"The format is::\n\n"
 	"\t%s: %f %f %f\n"
 	"\t|   |  |  |\n"



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] 7e2ac56 Add an XXX comment about turning off unnecessary
 timestamps
Message-ID: <E1WSmDu-0004pA-Ip@project.varnish-software.com>


commit 7e2ac56eee82982ea7d0e65c4bd9b12f23fea543
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 25 17:16:15 2014 +0100

    Add an XXX comment about turning off unnecessary timestamps

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 900ecda..af78bfa 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -355,6 +355,10 @@ VSLb_ts(struct vsl_log *vsl, const char *event, double first, double *pprev,
     double now)
 {
 
+	/* XXX: Make an option to turn off some unnecessary timestamp
+	   logging. This must be done carefully because some functions
+	   (e.g. WRW_Reserve) takes the last timestamp as it's inital
+	   value for timeout calculation. */
 	assert(!isnan(now) && now != 0.);
 	VSLb(vsl, SLT_Timestamp, "%s: %.6f %.6f %.6f",
 	    event, now, now - first, now - *pprev);



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] acc49fd Fix test cases to pass with timestamp records present
Message-ID: <E1WSmDu-0004pJ-K7@project.varnish-software.com>


commit acc49fdda0cbefa8722a4b9ce4975ee516ae7d0b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 25 17:43:48 2014 +0100

    Fix test cases to pass with timestamp records present

diff --git a/bin/varnishtest/tests/l00000.vtc b/bin/varnishtest/tests/l00000.vtc
index 09423cc..1c732fc 100644
--- a/bin/varnishtest/tests/l00000.vtc
+++ b/bin/varnishtest/tests/l00000.vtc
@@ -16,14 +16,13 @@ logexpect l1 -v v1 -g session {
 	expect 0 =	End
 
 	expect 0 *	Begin		"req 1000"
-	expect 0 =	ReqStart
+	expect * =	ReqStart
 	expect 0 =	ReqMethod	GET
 	expect 0 =	ReqURL		/
 	expect 0 =	ReqProtocol	HTTP/1.1
 	expect * =	ReqHeader	"Foo: bar"
 	expect * =	Link		bereq
-	expect * =	ReqEnd
-	expect 0 =	End
+	expect * =	End
 
 	expect 0 1002	Begin		"bereq 1001"
 	expect * =	End
diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index a161c7b..b1ec63c 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -22,175 +22,150 @@ client c1 {
 # Test 'eq' operator
 logexpect l1 -d 1 -g vxid -q "Begin eq 'req 1000 rxreq'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test 'ne' operator
 logexpect l1 -d 1 -g vxid -q "ReqProtocol ne 'HTTP/1.0'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '==' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '==' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200." {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '!=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus != 503" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '!=' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus != 503." {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus < 201" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus < 201." {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus > 199" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus > 199." {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus <= 200" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<=' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus <= 200." {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus >= 200" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>=' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus >= 200." {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '~' operator
 logexpect l1 -d 1 -g vxid -q "RespStatus ~ '^200$'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '!~' operator
 logexpect l1 -d 1 -g vxid -q "RespStatus !~ '^404$'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test boolean and
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200 and RespStatus ~ '^200$'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test boolean or
 logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or RespStatus ~ '^200$'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test boolean not
 logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or not RespStatus ~ '^404$'" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test grouping
 logexpect l1 -d 1 -g vxid -q "(RespStatus == 200 or RespStatus == 404) and RespStatus == 200" {
 	expect 0 *	Begin	req
-	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test and/or precedence
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200 or RespStatus == 503 and RespStatus == 404" {
 	  expect 0 *	Begin	req
-	  expect * =	ReqEnd
 	  expect * =	End
 } -run
 
 # Test field
 logexpect l1 -d 1 -g vxid -q "RespHeader[2] == 123" {
 	  expect 0 *	Begin	req
-	  expect * =	ReqEnd
 	  expect * =	End
 } -run
 
 # Test taglists
 logexpect l1 -d 1 -g vxid -q "Debug,Resp* == 200" {
 	  expect 0 *	Begin	req
-	  expect * =	ReqEnd
 	  expect * =	End
 } -run
 
 # Test record prefix
 logexpect l1 -d 1 -g vxid -q "Resp*:x-test eq '123 321'" {
 	  expect 0 *	Begin	req
-	  expect * =	ReqEnd
 	  expect * =	End
 } -run
 
 # Test tag presence (no operator)
 logexpect l1 -d 1 -g vxid -q "RespStatus" {
 	  expect 0 *	Begin	req
-	  expect * =	ReqEnd
 	  expect * =	End
 } -run
 



From martin at varnish-software.com  Wed Mar 26 11:42:22 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 12:42:22 +0100
Subject: [master] d8c8c66 Timestamp test case
Message-ID: <E1WSmDu-0004pT-LU@project.varnish-software.com>


commit d8c8c66d4c4d474af66997ece6491483f8a17cbb
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Mar 26 12:27:19 2014 +0100

    Timestamp test case

diff --git a/bin/varnishtest/tests/s00004.vtc b/bin/varnishtest/tests/s00004.vtc
new file mode 100644
index 0000000..169d6ab
--- /dev/null
+++ b/bin/varnishtest/tests/s00004.vtc
@@ -0,0 +1,65 @@
+varnishtest "Timestamps"
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	delay 1
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	delay 1
+	chunkedlen 1000
+	chunkedlen 0
+
+	rxreq
+	expect req.url == "/2"
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+	sub vcl_deliver {
+		if (req.url == "/1" && req.restarts == 0) {
+			return (restart);
+		}
+	}
+} -start
+
+logexpect l1 -v v1 -g request {
+	expect 0 1001	Begin		req
+	expect * =	Timestamp	{Start: \S+ 0\.000000 0\.000000}
+	expect * =	Timestamp	{Req: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Fetch: \S+ 2\.\d+ 2\.\d+}
+	expect * =	Timestamp	{Process: \S+ 2\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Restart: \S+ 2\.\d+ 0\.\d+}
+	expect * =	End
+	expect 0 1002	Begin		bereq
+	expect * =	Timestamp	{Start: \S+ 0\.000000 0\.000000}
+	expect * =	Timestamp	{Bereq: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Beresp: \S+ 1\.\d+ 1\.\d+}
+	expect * =	Timestamp	{BerespBody: \S+ 2\.\d+ 1\.\d+}
+	expect * =	End
+	expect 0 1003	Begin		req 1001 restart
+	expect * =	Timestamp	{Start: \S+ 2\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Process: \S+ 2\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Resp: \S+ 2\.\d+ 0\.\d+}
+	expect * =	End
+
+	expect 0 1004	Begin		req
+	expect * =	Timestamp	{Start: \S+ 0\.000000 0\.000000}
+	expect * =	Timestamp	{Req: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	{ReqBody: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Fetch: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	{Resp: \S+ 0\.\d+ 0\.\d+}
+	expect * =	End
+} -start
+
+client c1 {
+	txreq -url "/1"
+	rxresp
+	delay 1
+	txreq -req "POST" -url "/2" -body "asdf"
+	rxresp
+} -run
+
+logexpect l1 -wait



From martin at varnish-software.com  Wed Mar 26 12:24:40 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 26 Mar 2014 13:24:40 +0100
Subject: [master] 064f034 Add std.timestamp() method
Message-ID: <E1WSmsq-0002oS-HD@project.varnish-software.com>


commit 064f03498b6d9f11156a1f5ab0648b992495a08b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Mar 26 13:22:01 2014 +0100

    Add std.timestamp() method

diff --git a/bin/varnishtest/tests/m00013.vtc b/bin/varnishtest/tests/m00013.vtc
new file mode 100644
index 0000000..c3a6295
--- /dev/null
+++ b/bin/varnishtest/tests/m00013.vtc
@@ -0,0 +1,44 @@
+varnishtest "Timestamps from std vmod"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+
+	sub vcl_init {
+		# Make sure that this doesn't cause ill effects
+		std.timestamp("bogus");
+	}
+
+	sub vcl_recv {
+		std.timestamp("vcl_recv");
+	}
+
+	sub vcl_backend_fetch {
+		std.timestamp("vcl_backend_fetch");
+	}
+} -start
+
+logexpect l1 -v v1 -g request {
+	expect 0 1001	Begin		req
+	expect * =	Timestamp	Start:
+	expect * =	Timestamp	{vcl_recv: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	Resp:
+	expect * =	End
+	expect 0 1002	Begin		bereq
+	expect * =	Timestamp	Start:
+	expect * =	Timestamp	{vcl_backend_fetch: \S+ 0\.\d+ 0\.\d+}
+	expect * =	Timestamp	BerespBody:
+	expect * =	End
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+logexpect l1 -wait
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index dcb12ec..b1b7010 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -145,6 +145,15 @@ $Function INT port(IP)
 Description
 	Returns the port number of an IP address.
 
+$Function VOID timestamp(STRING)
+
+Description
+	Introduces a timestamp in the log with the current time. Uses
+	the argument as the timespamp label. This is useful to time
+	the execution of lengthy VCL procedures, and makes the
+	timestamps inserted automatically by Varnish more accurate.
+Example
+	std.timestamp("curl-request");
 
 SEE ALSO
 ========
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 7301559..233c2da 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -41,6 +41,7 @@
 #include "vrt.h"
 #include "vtcp.h"
 #include "vsa.h"
+#include "vtim.h"
 
 #include "cache/cache.h"
 #include "cache/cache_backend.h"
@@ -204,3 +205,23 @@ vmod_port(const struct vrt_ctx *ctx, VCL_IP ip)
 		return (0);
 	return (VSA_Port(ip));
 }
+
+VCL_VOID __match_proto__(td_std_timestamp)
+vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	if (label == NULL)
+		return;
+	if (*label == '\0')
+		return;
+	if (ctx->bo != NULL && ctx->req == NULL) {
+		/* Called from backend vcl methods */
+		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+		VSLb_ts_busyobj(ctx->bo, label, VTIM_real());
+	} else if (ctx->req != NULL) {
+		/* Called from request vcl methdos */
+		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+		VSLb_ts_req(ctx->req, label, VTIM_real());
+	}
+}



From daghf at varnish-software.com  Wed Mar 26 13:26:57 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Wed, 26 Mar 2014 14:26:57 +0100
Subject: [master] 59c5e3d Update varnishncsa to read timing data from
 SLT_Timestamp records.
Message-ID: <E1WSnr7-00074g-8x@project.varnish-software.com>


commit 59c5e3d38ebc117b8799cd1818f7cdc634df77ce
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Mar 26 14:26:43 2014 +0100

    Update varnishncsa to read timing data from SLT_Timestamp records.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 7434070..d03f91c 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -777,10 +777,16 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			case SLT_Length:
 				frag_line(b, e, &CTX.frag[F_b]);
 				break;
-			case SLT_ReqEnd:
-				frag_fields(b, e, 1, &CTX.frag[F_tstart],
-				    2, &CTX.frag[F_tend], 3, &CTX.frag[F_ttfb],
-				    0, NULL);
+			case SLT_Timestamp:
+				if (isprefix(b, "Start:", e, &p)) {
+					frag_fields(p, e, 1, &CTX.frag[F_tstart], 0, NULL);
+
+				} else if (isprefix(b, "Resp:", e, &p)) {
+					frag_fields(p, e, 1, &CTX.frag[F_tend], 0, NULL);
+
+				} else if (isprefix(b, "Process:", e, &p)) {
+					frag_fields(p, e, 2, &CTX.frag[F_ttfb], 0, NULL);
+				}
 				break;
 			case SLT_ReqHeader:
 				if (isprefix(b, "Host:", e, &p))



From daghf at varnish-software.com  Wed Mar 26 14:51:50 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Wed, 26 Mar 2014 15:51:50 +0100
Subject: [master] b40bd3d Update varnishhist to read ttfb from VSL_Timestamp
 record.
Message-ID: <E1WSpBG-0004Xl-9a@project.varnish-software.com>


commit b40bd3dc159b3cf1efbb8eaa0521abd0645800a8
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Mar 26 15:49:45 2014 +0100

    Update varnishhist to read ttfb from VSL_Timestamp record.
    
    Fixes: #1460

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index c7406da..91078d8 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -105,6 +105,7 @@ static int scales[] = {
 struct profile {
 	const char *name;
 	enum VSL_tag_e tag;
+	const char *prefix;
 	int field;
 	int hist_low;
 	int hist_high;
@@ -112,13 +113,15 @@ struct profile {
 profiles[] = {
 	{
 		.name = "responsetime",
-		.tag = SLT_ReqEnd,
-		.field = 5,
+		.tag = SLT_Timestamp,
+		.prefix = "Process:",
+		.field = 3,
 		.hist_low = -6,
 		.hist_high = 3
 	}, {
 		.name = "size",
 		.tag = SLT_Length,
+		.prefix = NULL,
 		.field = 1,
 		.hist_low = 1,
 		.hist_high = 8
@@ -224,6 +227,13 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			default:
 				if (tag != match_tag)
 					break;
+
+				if (active_profile->prefix &&
+				    strncmp(VSL_CDATA(tr->c->rec.ptr),
+					active_profile->prefix,
+					strlen(active_profile->prefix)) != 0)
+					break;
+
 				i = sscanf(VSL_CDATA(tr->c->rec.ptr), format,
 				    &value);
 				if (i != 1)
@@ -362,7 +372,7 @@ main(int argc, char **argv)
 	const char *profile = "responsetime";
 	pthread_t thr;
 	int fnum = -1;
-	struct profile cli_p;
+	struct profile cli_p = {0};
 	cli_p.name = 0;
 
 	VUT_Init(progname);



From lkarsten at varnish-software.com  Wed Mar 26 16:04:02 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 26 Mar 2014 17:04:02 +0100
Subject: [master] d6298a7 Restore default_ttl to 120 seconds.
Message-ID: <E1WSqJ8-0003cB-IR@project.varnish-software.com>


commit d6298a76f621b1083bb645ec82f1401f2963aa52
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Mar 26 17:01:19 2014 +0100

    Restore default_ttl to 120 seconds.
    
    This was reduced to 20 seconds in 6d0c94c34c0e471b12982b207857934ec4e6ea57.
    
    The git commit message does not mention default_ttl, and since this was a
    fairly big commit, I don't believe the change was intentional.

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index b6ae75c..be862e2 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -64,7 +64,7 @@ struct parspec mgt_parspec[] = {
 		"The TTL assigned to objects if neither the backend nor "
 		"the VCL code assigns one.",
 		OBJ_STICKY,
-		"20", "seconds" },
+		"120", "seconds" },
 	{ "default_grace", tweak_timeout, &mgt_param.default_grace,
 		"0", NULL,
 		"Default grace period.  We will deliver an object "



From daghf at varnish-software.com  Wed Mar 26 16:27:04 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Wed, 26 Mar 2014 17:27:04 +0100
Subject: [master] 1a6bb1d Reformat to <=80 cols.
Message-ID: <E1WSqfQ-0005M3-1q@project.varnish-software.com>


commit 1a6bb1d6d39d9685c9e347d3f8dc15cc3e298d70
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Mar 26 17:26:02 2014 +0100

    Reformat to <=80 cols.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index d03f91c..75a0452 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -779,13 +779,16 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				break;
 			case SLT_Timestamp:
 				if (isprefix(b, "Start:", e, &p)) {
-					frag_fields(p, e, 1, &CTX.frag[F_tstart], 0, NULL);
+					frag_fields(p, e, 1,
+					    &CTX.frag[F_tstart], 0, NULL);
 
 				} else if (isprefix(b, "Resp:", e, &p)) {
-					frag_fields(p, e, 1, &CTX.frag[F_tend], 0, NULL);
+					frag_fields(p, e, 1,
+					    &CTX.frag[F_tend], 0, NULL);
 
 				} else if (isprefix(b, "Process:", e, &p)) {
-					frag_fields(p, e, 2, &CTX.frag[F_ttfb], 0, NULL);
+					frag_fields(p, e, 2,
+					    &CTX.frag[F_ttfb], 0, NULL);
 				}
 				break;
 			case SLT_ReqHeader:



From daghf at varnish-software.com  Thu Mar 27 10:35:50 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 27 Mar 2014 11:35:50 +0100
Subject: [master] 75eaba8 Varnishncsa %D format option is now in microseconds.
Message-ID: <E1WT7f4-0003fH-6e@project.varnish-software.com>


commit 75eaba88157ddbdb3d69d699f8e5feb3d7065ae0
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Mar 27 11:32:52 2014 +0100

    Varnishncsa %D format option is now in microseconds.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 75a0452..a38264b 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -276,7 +276,7 @@ format_time(const struct format *format)
 
 	switch (format->time_type) {
 	case 'D':
-		AZ(VSB_printf(CTX.vsb, "%f", t_end - t_start));
+		AZ(VSB_printf(CTX.vsb, "%f", (t_end - t_start) * 1e6));
 		break;
 	case 't':
 		AN(format->time_fmt);



From lkarsten at varnish-software.com  Thu Mar 27 11:14:57 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 27 Mar 2014 12:14:57 +0100
Subject: [master] 6dc740f Add manual Provides for libvarnishapi.so.1.
Message-ID: <E1WT8Gv-0006oC-Bd@project.varnish-software.com>


commit 6dc740f00a7d079d4464b0716e4f899632e712b8
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 27 12:11:10 2014 +0100

    Add manual Provides for libvarnishapi.so.1.
    
    find-provides did not pick up on libvarnishapi.so.1 for varnish-libs
    automatically, and find-requires for the main package did. This meant
    that package pair was uninstallable.
    
    I don't quite know why this is happening, so I'm working around it
    with a hardcoded Provides for now. This is on the critical path for
    4.0 release.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 6f35ecc..0d0537c 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -49,6 +49,7 @@ available on the following web site: http://www.varnish-cache.org/
 Summary: Libraries for %{name}
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
+Provides: libvarnishapi.so.1
 #Obsoletes: libvarnish1
 
 %description libs



From lkarsten at varnish-software.com  Thu Mar 27 15:35:58 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 27 Mar 2014 16:35:58 +0100
Subject: [master] 6f4a02d Amending 6dc740f with additional Provides.
Message-ID: <E1WTCLW-0003Iz-6m@project.varnish-software.com>


commit 6f4a02ddda2d5bd232316c3aff445b331c6b877c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 27 16:34:47 2014 +0100

    Amending 6dc740f with additional Provides.
    
    As Ingar mentions in #1445, this is a workaround for the
    dependency issue.
    
    Originally attempt at a shorter fix in 6dc740f didn't work out.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0d0537c..9b08aa2 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -50,6 +50,10 @@ Summary: Libraries for %{name}
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
 Provides: libvarnishapi.so.1
+Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.0)(64bit)
+Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.1)(64bit)
+Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.2)(64bit)
+Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.3)(64bit)
 #Obsoletes: libvarnish1
 
 %description libs



From fgsch at lodoss.net  Thu Mar 27 16:07:49 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 27 Mar 2014 17:07:49 +0100
Subject: [master] f021385 Use -p to pass parameters, -w is long gone
Message-ID: <E1WTCqL-0005u4-OW@project.varnish-software.com>


commit f021385325303bca545699df961571a26374de08
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 27 16:05:06 2014 +0000

    Use -p to pass parameters, -w is long gone

diff --git a/redhat/varnish.sysconfig b/redhat/varnish.sysconfig
index f5846c9..aa0f8ba 100644
--- a/redhat/varnish.sysconfig
+++ b/redhat/varnish.sysconfig
@@ -100,7 +100,9 @@ DAEMON_OPTS="-a ${VARNISH_LISTEN_ADDRESS}:${VARNISH_LISTEN_PORT} \
              -f ${VARNISH_VCL_CONF} \
              -T ${VARNISH_ADMIN_LISTEN_ADDRESS}:${VARNISH_ADMIN_LISTEN_PORT} \
              -t ${VARNISH_TTL} \
-             -w ${VARNISH_MIN_THREADS},${VARNISH_MAX_THREADS},${VARNISH_THREAD_TIMEOUT} \
+             -p thread_pool_min=${VARNISH_MIN_THREADS} \
+             -p thread_pool_max=${VARNISH_MAX_THREADS} \
+             -p thread_pool_timeout=${VARNISH_THREAD_TIMEOUT} \
              -u varnish -g varnish \
              -S ${VARNISH_SECRET_FILE} \
              -s ${VARNISH_STORAGE}"



From lkarsten at varnish-software.com  Thu Mar 27 16:09:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 27 Mar 2014 17:09:47 +0100
Subject: [master] 648951e Use jemalloc again.
Message-ID: <E1WTCsF-00068K-1c@project.varnish-software.com>


commit 648951e42ec5154c975b230d1c5e6b0977938575
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 27 17:08:18 2014 +0100

    Use jemalloc again.
    
    Since we removed the bundled jemalloc, our packages have been
    using system malloc on RHEL Linux.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 9b08aa2..9adcaa2 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -16,11 +16,12 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
-BuildRequires: ncurses-devel groff pcre-devel pkgconfig python-docutils libedit-devel
+BuildRequires: ncurses-devel groff pcre-devel pkgconfig python-docutils libedit-devel jemalloc-devel
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses
 Requires: pcre
+Requires: jemalloc
 Requires(pre): shadow-utils
 Requires(post): /sbin/chkconfig, /usr/bin/uuidgen
 Requires(preun): /sbin/chkconfig



From lkarsten at varnish-software.com  Thu Mar 27 16:16:06 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 27 Mar 2014 17:16:06 +0100
Subject: [master] aa526ab Harmonise default storage across distributions.
Message-ID: <E1WTCyM-0006hS-8o@project.varnish-software.com>


commit aa526abd8f81b7ec9a8bb8803fdbb6a36fbff1ce
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 27 17:13:40 2014 +0100

    Harmonise default storage across distributions.
    
    The standard storage in our other Linux packages is 256 MB malloc.
    
    Change the default on Redhat systems to match this.

diff --git a/redhat/varnish.sysconfig b/redhat/varnish.sysconfig
index aa0f8ba..6aa2354 100644
--- a/redhat/varnish.sysconfig
+++ b/redhat/varnish.sysconfig
@@ -81,15 +81,12 @@ VARNISH_MAX_THREADS=1000
 # # Idle timeout for worker threads
 VARNISH_THREAD_TIMEOUT=120
 #
-# # Cache file location
-VARNISH_STORAGE_FILE=/var/lib/varnish/varnish_storage.bin
-#
 # # Cache file size: in bytes, optionally using k / M / G / T suffix,
 # # or in percentage of available disk space using the % suffix.
-VARNISH_STORAGE_SIZE=1G
+VARNISH_STORAGE_SIZE=256M
 #
 # # Backend storage specification
-VARNISH_STORAGE="file,${VARNISH_STORAGE_FILE},${VARNISH_STORAGE_SIZE}"
+VARNISH_STORAGE="malloc,${VARNISH_STORAGE_SIZE}"
 #
 # # Default TTL used when the backend does not specify one
 VARNISH_TTL=120



From daghf at varnish-software.com  Thu Mar 27 16:43:20 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 27 Mar 2014 17:43:20 +0100
Subject: [master] 723da93 Fix duplicate wording in example.vcl
Message-ID: <E1WTDOi-0001HR-Pb@project.varnish-software.com>


commit 723da932036b8e0046b408ffc6c8c0c0fd531e9c
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Mar 27 17:42:13 2014 +0100

    Fix duplicate wording in example.vcl

diff --git a/etc/example.vcl b/etc/example.vcl
index b99ec90..a2987bb 100644
--- a/etc/example.vcl
+++ b/etc/example.vcl
@@ -1,9 +1,9 @@
 #
 # This is an example VCL file for Varnish.
 #
-# It does not do anything by default, delegating control to the builtin vcl.
-# The builtin VCL is called when there is no explicit explicit return
-# statement.
+# It does not do anything by default, delegating control to the
+# builtin VCL. The builtin VCL is called when there is no explicit
+# return statement.
 #
 # See the VCL chapters in the Users Guide at https://www.varnish-cache.org/docs/
 # and http://varnish-cache.org/trac/wiki/VCLExamples for more examples.



From fgsch at lodoss.net  Thu Mar 27 16:48:19 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 27 Mar 2014 17:48:19 +0100
Subject: [master] 1a1f1d3 Update dependencies and sort entries
Message-ID: <E1WTDTX-0001p6-9E@project.varnish-software.com>


commit 1a1f1d30d917b50073ba1458960da833b2ddb829
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 27 16:47:29 2014 +0000

    Update dependencies and sort entries

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 9b61d35..883cbf9 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -76,20 +76,20 @@ Build dependencies on Debian / Ubuntu
 In order to build Varnish from source you need a number of packages
 installed. On a Debian or Ubuntu system these are:
 
-* `autotools-dev`
-* `automake1.1`
-* `libtool`
 * `autoconf`
-* `libncurses-dev`
+* `automake1.1`
+* `autotools-dev`
 * `groff-base`
-* `libpcre3-dev`
-* `pkg-config`
 * `make`
 * `libedit-dev`
+* `libncurses-dev`
+* `libpcre3-dev`
+* `libtool`
+* `pkg-config`
+* `python-docutils`
 
 If you're building from git, you also need the following:
 
-* `python-docutils`
 * `python-sphinx` (optional, if you want to build the documentation)
 
 Build dependencies on Red Hat / CentOS
@@ -100,14 +100,17 @@ packages installed:
 
 * `automake`
 * `autoconf`
+* `groff`
+* `libedit-devel`
 * `libtool`
 * `ncurses-devel`
-* `groff`
 * `pcre-devel`
 * `pkgconfig`
-* `libedit-devel`
-* `docutils`
-* `python-sphinx`
+* `python-docutils`
+
+If you're building from git, you also need the following:
+
+* `python-sphinx` (optional, if you want to build the documentation)
 
 Compiling Varnish
 -----------------



From lkarsten at varnish-software.com  Thu Mar 27 17:00:54 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 27 Mar 2014 18:00:54 +0100
Subject: [master] d8292c2 Track upstream packaging for default.vcl
Message-ID: <E1WTDfi-0004uT-Pn@project.varnish-software.com>


commit d8292c24284a3d8b6ba091991d5252221c4d0375
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 27 17:59:43 2014 +0100

    Track upstream packaging for default.vcl
    
    New example.vcl should be used as a template from now on.

diff --git a/rules b/rules
index 91ca406..7609b79 100755
--- a/rules
+++ b/rules
@@ -33,9 +33,9 @@ override_dh_auto_configure:
 
 override_dh_auto_install:
 	dh_auto_install -a
-	@ # Activate the default backend
-	sed -i '/backend default {/,/}/ s/^#[[:space:]]//'  \
-		$(CURDIR)/debian/tmp/etc/varnish/default.vcl
+	@ # Install the example as default.
+	install -o root -g root -m 644 $(CURDIR)/etc/example.vlc \
+		$(CURDIR)/debian/varnish/etc/default.vcl
 	@ # Remove .la files
 	@ # (See http://wiki.debian.org/ReleaseGoals/LAFileRemoval)
 	find $(CURDIR)/debian/ -name \*.la -delete



From fgsch at lodoss.net  Thu Mar 27 17:57:09 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 27 Mar 2014 18:57:09 +0100
Subject: [master] fa54ec5 Check if the compiler supports -Wno-address
Message-ID: <E1WTEY9-0006Dv-7I@project.varnish-software.com>


commit fa54ec5f5ce2aed29dbbce749ac637ae6180c454
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 27 17:54:22 2014 +0000

    Check if the compiler supports -Wno-address
    
    gcc 4.1.2 (and previous version I assume) doesn't support it.

diff --git a/configure.ac b/configure.ac
index e2a0b98..d1e203e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -483,8 +483,7 @@ DEVELOPER_CFLAGS=" \
 	-Wchar-subscripts \
 	-Wnested-externs \
 	-Wextra \
-	-Wno-sign-compare \
-	-Wno-address \
+	-Wno-sign-compare
 "
 
 # These are not compliable yet
@@ -519,6 +518,10 @@ if test "x$enable_developer_warnings" != "xno"; then
 	# compiler flags not available on gcc3
 	AX_CHECK_COMPILE_FLAG([-Wno-pointer-sign],
 		[DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -Wno-pointer-sign"], [], [])
+	# Not available in gcc 4.1.2
+	AX_CHECK_COMPILE_FLAG([-Wno-address],
+		[DEVELOPER_CFLAGS="${DEVELOPER_CFLAGS} -Wno-address"])
+
 	# no known way to specifically disabling missing-field-initializers warnings
 	# keeping the rest of Wextra
 	AX_CHECK_COMPILE_FLAG([-Wno-missing-field-initializers],



From fgsch at lodoss.net  Thu Mar 27 18:07:00 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 27 Mar 2014 19:07:00 +0100
Subject: [master] 7bee7ba Fix previous commit
Message-ID: <E1WTEhg-0007OQ-LJ@project.varnish-software.com>


commit 7bee7ba694f0ed9cbd38c479be721fd2e672ff25
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 27 18:06:40 2014 +0000

    Fix previous commit

diff --git a/configure.ac b/configure.ac
index d1e203e..6612215 100644
--- a/configure.ac
+++ b/configure.ac
@@ -483,7 +483,7 @@ DEVELOPER_CFLAGS=" \
 	-Wchar-subscripts \
 	-Wnested-externs \
 	-Wextra \
-	-Wno-sign-compare
+	-Wno-sign-compare \
 "
 
 # These are not compliable yet



From phk at FreeBSD.org  Fri Mar 28 07:31:01 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 28 Mar 2014 08:31:01 +0100
Subject: [master] dc8b9fc Restrict client.ip and client.identity to req
 processing to avoid the predictable panic if they are accessed from the
 backend side.
Message-ID: <E1WTRFl-0002i1-Oq@project.varnish-software.com>


commit dc8b9fc9b309106321f2cdd1793a68f36f5f0287
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 28 07:30:17 2014 +0000

    Restrict client.ip and client.identity to req processing to avoid
    the predictable panic if they are accessed from the backend side.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index a118513..06ed058 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -159,15 +159,15 @@ returns =(
 sp_variables = [
 	('client.ip',
 		'IP',
-		( 'both',),
+		( 'client',),
 		( ), """
 		The client's IP address.
 		"""
 	),
 	('client.identity',
 		'STRING',
-		( 'both',),
-		( 'both',), """
+		( 'client',),
+		( 'client',), """
 		Identification of the client, used to load balance
 		in the client director.
 		"""



From fgsch at lodoss.net  Fri Mar 28 08:46:50 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Fri, 28 Mar 2014 09:46:50 +0100
Subject: [master] c17a6d2 Spelling, formatting and various other fixes
Message-ID: <E1WTSR8-00013B-DM@project.varnish-software.com>


commit c17a6d23c0255d0ae11c5612be5e313078273db8
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Mar 28 08:44:29 2014 +0000

    Spelling, formatting and various other fixes

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 580caff..033e04b 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -158,7 +158,7 @@ STRING_LIST
 
 	A multi-component text-string.  We try very hard to avoid
 	doing text-processing in Varnish, and this is one way we
-	to avoid that, by not editing separate pieces of a sting
+	to avoid that, by not editing separate pieces of a string
 	together to one string, unless we have to.
 
 	Consider this contrived example::
diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index 9d8e0af..a413b71 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -3,29 +3,30 @@
 Achieving a high hitrate
 ------------------------
 
-Now that Varnish is up and running, and you can access your web
-application through Varnish. Unless your application is specifically
-written to work behind a web accelerator you'll probably need to do
-some changes to either the configuration or the application in order
-to get a high hitrate in Varnish.
+Now that Varnish is up and running you can access your web application
+through Varnish. Unless your application is specifically written to
+work behind a web accelerator you'll probably need to do some
+changes to either the configuration or the application in order to
+get a high hitrate in Varnish.
 
 Varnish will not cache your data unless it's absolutely sure it is
 safe to do so. So, for you to understand how Varnish decides if and
-how to cache a page, We'll guide you through a couple of tools that you
-should find useful to understand what is happening in your Varnish setup.
+how to cache a page, we'll guide you through a couple of tools that
+you should find useful to understand what is happening in your
+Varnish setup.
 
-Note that you need a tool to see the HTTP headers that fly between Varnish and
-the backend. On the Varnish server, the easiest way to do this is to use
-`varnishlog` and `varnishtop` but sometimes a client-side tool makes
-sense. Here are the ones we commonly use.
+Note that you need a tool to see the HTTP headers that fly between
+Varnish and the backend. On the Varnish server, the easiest way to
+do this is to use `varnishlog` and `varnishtop` but sometimes a
+client-side tool makes sense. Here are the ones we commonly use.
 
 Tool: varnishtop
 ~~~~~~~~~~~~~~~~
 
 You can use varnishtop to identify what URLs are hitting the backend
-the most. ``varnishtop -i txurl`` is an essential command, showing you
-the top `txurl` requests Varnish is sending to the backend. You can
-see some other examples of `varnishtop` usage in
+the most. ``varnishtop -i txurl`` is an essential command, showing
+you the top `txurl` requests Varnish is sending to the backend. You
+can see some other examples of `varnishtop` usage in
 :ref:`users-guide-statistics`.
 
 
@@ -47,9 +48,10 @@ http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
 Tool: lwp-request
 ~~~~~~~~~~~~~~~~~
 
-`lwp-request` is tool that is a part of The World-Wide Web library for Perl. It's a
-couple of really basic programs that can execute an HTTP request and
-show you the result. We mostly use the two programs, ``GET`` and ``HEAD``.
+`lwp-request` is tool that is a part of The World-Wide Web library
+for Perl. It's a couple of really basic programs that can execute
+an HTTP request and show you the result. We mostly use the two
+programs, ``GET`` and ``HEAD``.
 
 vg.no was the first site to use Varnish and the people running Varnish
 there are quite clueful. So it's interesting to look at their HTTP
@@ -59,7 +61,7 @@ Headers. Let's send a GET request for their home page::
   GET http://vg.no/
   Host: www.vg.no
   User-Agent: lwp-request/5.834 libwww-perl/5.834
-  
+
   200 OK
   Cache-Control: must-revalidate
   Refresh: 600
@@ -81,7 +83,7 @@ really care about the content, only the headers.
 As you can see, VG adds quite a bit of information in their
 headers. Some of the headers, like the 'X-Rick-Would-Never' are specific
 to vg.no and their somewhat odd sense of humour. Others, like the
-'X-VG-Webcache' are for debugging purposes. 
+'X-VG-Webcache' are for debugging purposes.
 
 So, to check whether a site sets cookies for a specific URL, just do::
 
@@ -92,10 +94,11 @@ So, to check whether a site sets cookies for a specific URL, just do::
 Tool: Live HTTP Headers
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-There is also a plugin for Firefox called `Live HTTP Headers`. This plugin can show you
-what headers are being sent and recieved. `Live HTTP Headers` can be
-found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
-googling "Live HTTP Headers".
+There is also a plugin for Firefox called `Live HTTP Headers`. This
+plugin can show you what headers are being sent and received.
+`Live HTTP Headers` can be found at
+https://addons.mozilla.org/en-US/firefox/addon/3829/ or by googling
+"Live HTTP Headers".
 
 
 The role of HTTP Headers
@@ -108,7 +111,7 @@ the content cached.
 
 Please note that when Varnish considers these headers Varnish actually
 considers itself *part of* the actual webserver. The rationale being
-that both are under your control. 
+that both are under your control.
 
 The term *surrogate origin cache* is not really well defined by the
 IETF or RFC 2616 so the various ways Varnish works might differ from
@@ -129,7 +132,7 @@ the backend.
 This can be overly conservative. A lot of sites use Google Analytics
 (GA) to analyze their traffic. GA sets a cookie to track you. This
 cookie is used by the client side javascript and is therefore of no
-interest to the server. 
+interest to the server.
 
 Cookies from the client
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -139,9 +142,9 @@ cookies unless you are accessing a special part of the web site. This
 VCL snippet in `vcl_recv` will disregard cookies unless you are
 accessing `/admin/`::
 
-  if ( !( req.url ~ ^/admin/) ) {
-    unset req.http.Cookie;
-  }
+    if (!(req.url ~ "^/admin/")) {
+        unset req.http.Cookie;
+    }
 
 Quite simple. If, however, you need to do something more complicated,
 like removing one out of several cookies, things get
@@ -152,36 +155,39 @@ whats going on. If you aren't we recommend that you either pick up a book on
 the subject, read through the *pcrepattern* man page, or read through
 one of many online guides.
 
-Lets use the Varnish Software (VS) web as an example here. Very simplified the setup VS uses can be described as a Drupal-based backend with a Varnish cache infront. VS uses some cookies for
-Google Analytics tracking and similar tools. The cookies are all set
-and used by Javascript. Varnish and Drupal doesn't need to see those
-cookies and since Varnish will cease caching of pages when the client
-sends cookies Varnish will discard these unnecessary cookies in VCL. 
+Lets use the Varnish Software (VS) web as an example here. Very
+simplified the setup VS uses can be described as a Drupal-based
+backend with a Varnish cache in front. VS uses some cookies for
+Google Analytics tracking and similar tools. The cookies are all
+set and used by Javascript. Varnish and Drupal doesn't need to see
+those cookies and since Varnish will cease caching of pages when
+the client sends cookies Varnish will discard these unnecessary
+cookies in VCL.
 
-In the following VCL we discard all cookies that start with a
+In the following VCL we discard all cookies that start with an
 underscore::
 
-  // Remove has_js and Google Analytics __* cookies.
-  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
-  // Remove a ";" prefix, if present.
-  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
+    # Remove has_js and Google Analytics __* cookies.
+    set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
+    # Remove a ";" prefix, if present.
+    set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
 
 Lets look at an example where we remove everything except the
 cookies named "COOKIE1" and "COOKIE2" and you can marvel at the "beauty" of it::
 
-  sub vcl_recv {
-    if (req.http.Cookie) {
-      set req.http.Cookie = ";" + req.http.Cookie;
-      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
-      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
-
-      if (req.http.Cookie == "") {
-          remove req.http.Cookie;
-      }
+    sub vcl_recv {
+        if (req.http.Cookie) {
+            set req.http.Cookie = ";" + req.http.Cookie;
+            set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
+            set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
+            set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
+            set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
+
+            if (req.http.Cookie == "") {
+                unset req.http.Cookie;
+            }
+        }
     }
-  }
 
 A somewhat simpler example that can accomplish almost the same functionality can be
 found below. Instead of filtering out "other" cookies it instead picks out
@@ -190,16 +196,16 @@ copies it back to the request, deleting the original cookie header.
 .. XXX:Verify correctness of request above! benc
 ::
 
-  sub vcl_recv {
-         # save the original cookie header so we can mangle it
+    sub vcl_recv {
+        # save the original cookie header so we can mangle it
         set req.http.X-Varnish-PHP_SID = req.http.Cookie;
-        # using a capturing sub pattern, extract the continuous string of 
+        # using a capturing sub pattern, extract the continuous string of
         # alphanumerics that immediately follows "PHPSESSID="
-        set req.http.X-Varnish-PHP_SID = 
+        set req.http.X-Varnish-PHP_SID =
            regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
         set req.http.Cookie = req.X-Varnish-PHP_SID;
-        remove req.X-Varnish-PHP_SID;
-   }   
+        unset req.X-Varnish-PHP_SID;
+    }
 
 There are other scary examples of what can be done in VCL in the
 Varnish Cache Wiki.
@@ -211,10 +217,10 @@ Cookies coming from the backend
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If your backend server sets a cookie using the 'Set-Cookie' header
-Varnish will not cache the page when using the default configuration. A
-`hit-for-pass` object (see :ref:`user-guide-vcl_actions`) is created.
-So, if the backend server acts silly and sets unwanted cookies just unset
-the 'Set-Cookie' header and all should be fine. 
+Varnish will not cache the page when using the default configuration.
+A `hit-for-pass` object (see :ref:`user-guide-vcl_actions`) is
+created.  So, if the backend server acts silly and sets unwanted
+cookies just unset the 'Set-Cookie' header and all should be fine.
 
 
 Cache-Control
@@ -247,7 +253,7 @@ In `vcl_backend_response`::
 
     if (beresp.http.Pragma ~ "nocache") {
         set beresp.uncacheable = true;
-        set beresp.ttl = 120s; // how long not to cache this url.
+        set beresp.ttl = 120s; # how long not to cache this url.
     }
 
 Authorization
@@ -266,11 +272,11 @@ somewhat cumbersome backend.
 You need VCL to identify the objects you want and then you set the
 'beresp.ttl' to whatever you want::
 
-  sub vcl_backend_response {
-      if (req.url ~ "^/legacy_broken_cms/") {
-          set beresp.ttl = 5d;
-      }
-  }
+    sub vcl_backend_response {
+        if (req.url ~ "^/legacy_broken_cms/") {
+            set beresp.ttl = 5d;
+        }
+    }
 
 This example will set the TTL to 5 days for the old legacy stuff on
 your site.
@@ -288,18 +294,18 @@ the cache but it isn't really recommended.
 Normalizing your namespace
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Some sites are accessed via lots of
-hostnames. http://www.varnish-software.com/,
-http://varnish-software.com/ and http://varnishsoftware.com/ all point
-at the same site. Since Varnish doesn't know they are the same,
+Some sites are accessed via lots of hostnames.
+http://www.varnish-software.com/, http://varnish-software.com/ and
+http://varnishsoftware.com/ all point at the same site. Since Varnish
+doesn't know they are the same,
 .. XXX: heavy meaning change above. benc
 Varnish will cache different versions of every page for every
-hostname. You can mitigate this in your web server configuration by
-setting up redirects or by using the following VCL::
+hostname. You can mitigate this in your web server configuration
+by setting up redirects or by using the following VCL::
 
-  if (req.http.host ~ "(?i)^(www.)?varnish-?software.com") {
-    set req.http.host = "varnish-software.com";
-  }
+    if (req.http.host ~ "(?i)^(www.)?varnish-?software.com") {
+        set req.http.host = "varnish-software.com";
+    }
 
 
 .. _users-guide-vary:
@@ -307,38 +313,38 @@ setting up redirects or by using the following VCL::
 HTTP Vary
 ---------
 
-*HTTP Vary is not a trivial concept. It is by far the most
-misunderstood HTTP header.*
+*HTTP Vary is not a trivial concept. It is by far the most misunderstood
+HTTP header.*
 
-A lot of the response headers tell the client something about the HTTP
-object being delivered. Clients can request different variants of a
-HTTP object, based on their preference. Their preferences might cover
-stuff like encoding or language. When a client prefers UK English this
-is indicated through ``Accept-Language: en-uk``. Caches need to keep
-these different variants apart and this is done through the HTTP
-response header 'Vary'.
+A lot of the response headers tell the client something about the
+HTTP object being delivered. Clients can request different variants
+of a HTTP object, based on their preference. Their preferences might
+cover stuff like encoding or language. When a client prefers UK
+English this is indicated through ``Accept-Language: en-uk``. Caches
+need to keep these different variants apart and this is done through
+the HTTP response header 'Vary'.
 
 When a backend server issues a ``Vary: Accept-Language`` it tells
 Varnish that its needs to cache a separate version for every different
 Accept-Language that is coming from the clients.
 
-If two clients say they accept the languages "en-us, en-uk" and "da,
-de" respectively, Varnish will cache and serve two different versions
-of the page if the backend indicated that Varnish needs to vary on the
-'Accept-Language' header.
+If two clients say they accept the languages "en-us, en-uk" and
+"da, de" respectively, Varnish will cache and serve two different
+versions of the page if the backend indicated that Varnish needs
+to vary on the 'Accept-Language' header.
 
 Please note that the headers that 'Vary' refer to need to match
-*exactly* for there to be a match. So Varnish will keep two copies of
-a page if one of them was created for "en-us, en-uk" and the other for
-"en-us,en-uk". Just the lack of a whitespace will force Varnish to cache
-another version.
+*exactly* for there to be a match. So Varnish will keep two copies
+of a page if one of them was created for "en-us, en-uk" and the
+other for "en-us,en-uk". Just the lack of a whitespace will force
+Varnish to cache another version.
 
-To achieve a high hitrate whilst using Vary is there therefore crucial
-to normalize the headers the backends varies on. Remember, just a
-difference in casing can force different cache entries.
+To achieve a high hitrate whilst using Vary is there therefore
+crucial to normalize the headers the backends varies on. Remember,
+just a difference in casing can force different cache entries.
 
-The following VCL code will normalize the 'Accept-Language' headers, to
-one of either "en","de" or "fr"::
+The following VCL code will normalize the 'Accept-Language' headers,
+to one of either "en", "de" or "fr"::
 
     if (req.http.Accept-Language) {
         if (req.http.Accept-Language ~ "en") {
@@ -348,9 +354,9 @@ one of either "en","de" or "fr"::
         } elsif (req.http.Accept-Language ~ "fr") {
             set req.http.Accept-Language = "fr";
         } else {
-            # unknown language. Remove the accept-language header and 
-	    # use the backend default.
-            remove req.http.Accept-Language
+            # unknown language. Remove the accept-language header and
+            # use the backend default.
+            unset req.http.Accept-Language
         }
     }
 
@@ -360,22 +366,22 @@ gzip, deflate with a preference for gzip.
 Vary parse errors
 ~~~~~~~~~~~~~~~~~
 
-Varnish will return a "503 internal server error" page when it fails to
-parse the 'Vary' header, or if any of the client headers listed
+Varnish will return a "503 internal server error" page when it fails
+to parse the 'Vary' header, or if any of the client headers listed
 in the Vary header exceeds the limit of 65k characters. An 'SLT_Error'
 log entry is added in these cases.
 
 Pitfall - Vary: User-Agent
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Some applications or application servers send ``Vary: User-Agent`` along
-with their content. This instructs Varnish to cache a separate copy
-for every variation of 'User-Agent' there is and there are plenty. Even a
-single patchlevel of the same browser will generate at least 10
-different 'User-Agent' headers based just on what operating system they
-are running. 
+Some applications or application servers send ``Vary: User-Agent``
+along with their content. This instructs Varnish to cache a separate
+copy for every variation of 'User-Agent' there is and there are
+plenty. Even a single patchlevel of the same browser will generate
+at least 10 different 'User-Agent' headers based just on what
+operating system they are running.
 
 So if you *really* need to vary based on 'User-Agent' be sure to
-normalize the header or your hit rate will suffer badly. Use the above
-code as a template.
+normalize the header or your hit rate will suffer badly. Use the
+above code as a template.
 
diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 3634a33..d64772d 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -7,13 +7,17 @@ Upgrading to Varnish 4
 Changes to VCL
 ==============
 
-Much of the VCL syntax has changed in Varnish 4. We've tried to compile a list of changes needed to upgrade here.
+Much of the VCL syntax has changed in Varnish 4. We've tried to
+compile a list of changes needed to upgrade here.
 
 Version statement
 ~~~~~~~~~~~~~~~~~
-To make sure that people have upgraded their VCL to the current version, Varnish now requires the first line of VCL to indicate the VCL version number::
 
-	vcl 4.0;
+To make sure that people have upgraded their VCL to the current
+version, Varnish now requires the first line of VCL to indicate the
+VCL version number::
+
+    vcl 4.0;
 
 vcl_fetch is now vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -21,8 +25,8 @@ vcl_fetch is now vcl_backend_response
 Directors have been moved to the vmod_directors
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To make directors (backend selection logic) easier to extend, the directors
-are now defined in loadable VMODs.
+To make directors (backend selection logic) easier to extend, the
+directors are now defined in loadable VMODs.
 
 Setting a backend for future fetches in `vcl_recv` is now done as follows::
 
@@ -31,6 +35,7 @@ Setting a backend for future fetches in `vcl_recv` is now done as follows::
         cluster1.add_backend(b1, 1.0);
         cluster1.add_backend(b2, 1.0);
     }
+
     sub vcl_recv {
         set req.backend_hint = cluster1.backend();
     }
@@ -47,21 +52,22 @@ has been removed, and you should use the hash director directly::
         h.add_backend(b1, 1);
         h.add_backend(b2, 1);
     }
+
     sub vcl_recv {
         set req.backend_hint = h.backend(client.ip);
     }
 
-error() is now a return value
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-You must explicitly return an error::
+error() is now synth()
+~~~~~~~~~~~~~~~~~~~~~~
 
-	return(error(999, "Response));
+And you must explicitly return it::
 
+    return (synth(999, "Response"));
 
-Synthetic responses in vcl_error
+Synthetic responses in vcl_synth
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Setting headers on synthetic response bodies made in vcl_error are now done on
+Setting headers on synthetic response bodies made in vcl_synth are now done on
 resp.http instead of obj.http.
 
 The synthetic keyword is now a function::
@@ -73,31 +79,33 @@ The synthetic keyword is now a function::
         return (deliver);
     }
 
-
-
 hit_for_pass objects are created using beresp.uncacheable
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Example::
 
-	sub vcl_backend_response {
-		if(beresp.http.X-No-Cache) {
-			set beresp.uncacheable = true;
-			set beresp.ttl = 120s;
-			return(deliver);
-		}
-	}
+    sub vcl_backend_response {
+        if (beresp.http.X-No-Cache) {
+            set beresp.uncacheable = true;
+            set beresp.ttl = 120s;
+            return (deliver);
+        }
+    }
 
 vcl_recv should return(hash) instead of lookup now
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 req.* not available in vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-req.* used to be available in `vcl_fetch`, but after the split of functionality, you only have 'bereq.*' in `vcl_backend_response`.
+
+req.* used to be available in `vcl_fetch`, but after the split of
+functionality, you only have 'bereq.*' in `vcl_backend_response`.
 
 vcl_* reserved
 ~~~~~~~~~~~~~~
-Any custom-made subs cannot be named 'vcl_*' anymore. This namespace is
-reserved for builtin subs.
+
+Any custom-made subs cannot be named 'vcl_*' anymore. This namespace
+is reserved for builtin subs.
 
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -112,15 +120,15 @@ objecthead, not per object. `obj.last_use` has been retired.
 default/builtin VCL changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The VCL code that is appended to user-configured VCL automatically is now
-called the builtin VCL. (previously default.vcl)
+The VCL code that is appended to user-configured VCL automatically
+is now called the builtin VCL. (previously default.vcl)
 
-The builtin VCL now honors Cache-Control: no-cache (and friends) to indicate
-uncacheable content from the backend.
+The builtin VCL now honors Cache-Control: no-cache (and friends)
+to indicate uncacheable content from the backend.
 
 
 The `remove` keyword is gone
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Replaced by `unset`.
 



From phk at FreeBSD.org  Fri Mar 28 09:03:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 28 Mar 2014 10:03:25 +0100
Subject: [master] 78cbdea Fix test-case
Message-ID: <E1WTShB-0004ie-Fn@project.varnish-software.com>


commit 78cbdeac8765f1f6e82345bf58d25fd93b1ce918
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 28 08:57:42 2014 +0000

    Fix test-case

diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc
index 49eec50..b0d6c20 100644
--- a/bin/varnishtest/tests/v00017.vtc
+++ b/bin/varnishtest/tests/v00017.vtc
@@ -64,7 +64,7 @@ varnish v1 -vcl {
 	sub vcl_recv { if (client.ip ~ a) { return(pass); } }
 }
 
-varnish v1 -vcl {
+varnish v1 -errvcl {'client.ip': Not available in method 'vcl_backend_response'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { if (client.ip == "127.0.0.1") { return(pass); } }
 	sub vcl_backend_response { if (client.ip != "127.0.0.1") { return(retry); } }



From phk at FreeBSD.org  Fri Mar 28 09:10:32 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 28 Mar 2014 10:10:32 +0100
Subject: [master] db1d398 Fix an assert to look at the right objcore
Message-ID: <E1WTSo4-0005cu-6G@project.varnish-software.com>


commit db1d3987b8f5113495905a953f68b5ccf61ba28d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 28 09:10:21 2014 +0000

    Fix an assert to look at the right objcore

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9826ae7..090e0b6 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -414,7 +414,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		if (boc != NULL) {
 			AZ(oc->flags & (OC_F_FAILED|OC_F_PASS));
 			AZ(oc->exp_flags & OC_EF_DYING);
-			AZ(oc->busyobj);
+			AZ(boc->busyobj);
 			VBF_Fetch(wrk, req, boc, o, VBF_BACKGROUND);
 		} else {
 			(void)HTTP1_DiscardReqBody(req);// XXX: handle err



From lkarsten at varnish-software.com  Fri Mar 28 11:49:52 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 28 Mar 2014 12:49:52 +0100
Subject: [master] 15b4a7e s/.vlc/.vcl.
Message-ID: <E1WTVIG-0001lP-LU@project.varnish-software.com>


commit 15b4a7e21329bedccee67528dc66d501a03e0033
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 28 12:49:27 2014 +0100

    s/.vlc/.vcl.

diff --git a/rules b/rules
index 7609b79..6521ede 100755
--- a/rules
+++ b/rules
@@ -34,7 +34,7 @@ override_dh_auto_configure:
 override_dh_auto_install:
 	dh_auto_install -a
 	@ # Install the example as default.
-	install -o root -g root -m 644 $(CURDIR)/etc/example.vlc \
+	install -o root -g root -m 644 $(CURDIR)/etc/example.vcl \
 		$(CURDIR)/debian/varnish/etc/default.vcl
 	@ # Remove .la files
 	@ # (See http://wiki.debian.org/ReleaseGoals/LAFileRemoval)



From lkarsten at varnish-software.com  Fri Mar 28 11:58:05 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 28 Mar 2014 12:58:05 +0100
Subject: [master] ba5e3c9 Create the /etc/varnish directory.
Message-ID: <E1WTVQD-0002bT-Md@project.varnish-software.com>


commit ba5e3c9247c32784c7e15d1b6ccbf34e1080583e
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Mar 28 12:57:36 2014 +0100

    Create the /etc/varnish directory.

diff --git a/rules b/rules
index 6521ede..41bc703 100755
--- a/rules
+++ b/rules
@@ -34,6 +34,7 @@ override_dh_auto_configure:
 override_dh_auto_install:
 	dh_auto_install -a
 	@ # Install the example as default.
+	mkdir -p $(CURDIR)/debian/varnish/etc/
 	install -o root -g root -m 644 $(CURDIR)/etc/example.vcl \
 		$(CURDIR)/debian/varnish/etc/default.vcl
 	@ # Remove .la files



From daghf at varnish-software.com  Fri Mar 28 12:03:24 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 28 Mar 2014 13:03:24 +0100
Subject: [master] 8d3e492 VCC default vcl -> builtin vcl renames.
Message-ID: <E1WTVVM-00035z-68@project.varnish-software.com>


commit 8d3e4924058aa4f5ca90ea7c917fb183bd42a6a9
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Fri Mar 28 13:01:33 2014 +0100

    VCC default vcl -> builtin vcl renames.

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 09e4eb4..2c2ffd9 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -488,7 +488,7 @@ mgt_vcc_init(void)
 
 	vcc = VCC_New();
 	AN(vcc);
-	VCC_Default_VCL(vcc, builtin_vcl);
+	VCC_Builtin_VCL(vcc, builtin_vcl);
 	AZ(atexit(mgt_vcc_atexit));
 }
 
diff --git a/include/libvcc.h b/include/libvcc.h
index 5c50c34..6b16252 100644
--- a/include/libvcc.h
+++ b/include/libvcc.h
@@ -31,7 +31,7 @@
 struct vcc;
 
 struct vcc *VCC_New(void);
-void VCC_Default_VCL(struct vcc *, const char *str);
+void VCC_Builtin_VCL(struct vcc *, const char *str);
 void VCC_VCL_dir(struct vcc *, const char *str);
 void VCC_VMOD_dir(struct vcc *, const char *str);
 void VCC_Err_Unref(struct vcc *tl, unsigned u);
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 91ea4c9..81a7620 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -489,7 +489,7 @@ vcc_NewVcc(const struct vcc *tl0)
 	ALLOC_OBJ(tl, VCC_MAGIC);
 	AN(tl);
 	if (tl0 != NULL) {
-		REPLACE(tl->default_vcl, tl0->default_vcl);
+		REPLACE(tl->builtin_vcl, tl0->builtin_vcl);
 		REPLACE(tl->vcl_dir, tl0->vcl_dir);
 		REPLACE(tl->vmod_dir, tl0->vmod_dir);
 		tl->vars = tl0->vars;
@@ -612,8 +612,8 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	if (tl->err)
 		return (vcc_DestroyTokenList(tl, NULL));
 
-	/* Register and lex the default VCL */
-	sp = vcc_new_source(tl->default_vcl, NULL, "Default");
+	/* Register and lex the builtin VCL */
+	sp = vcc_new_source(tl->builtin_vcl, NULL, "Builtin");
 	assert(sp != NULL);
 	VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
 	sp->idx = tl->nsources++;
@@ -736,15 +736,15 @@ VCC_New(void)
 }
 
 /*--------------------------------------------------------------------
- * Configure default VCL source code
+ * Configure builtin VCL source code
  */
 
 void
-VCC_Default_VCL(struct vcc *tl, const char *str)
+VCC_Builtin_VCL(struct vcc *tl, const char *str)
 {
 
 	CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
-	REPLACE(tl->default_vcl, str);
+	REPLACE(tl->builtin_vcl, str);
 }
 
 /*--------------------------------------------------------------------
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index bdfda05..128916b 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -160,7 +160,7 @@ struct vcc {
 #define VCC_MAGIC		0x24ad719d
 
 	/* Parameter/Template section */
-	char			*default_vcl;
+	char			*builtin_vcl;
 	char			*vcl_dir;
 	char			*vmod_dir;
 



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 631f23e Fix the meaning of '=' in logexpect
Message-ID: <E1WUYVc-0004aP-6R@project.varnish-software.com>


commit 631f23e6f043b5a6c67de2ce69dd231c2b77c4a7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 03:36:25 2014 +0200

    Fix the meaning of '=' in logexpect
    
    Fix the meaning of '=' in logexpect to mean same value as that of the
    last successfully matched log record.

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index 5b18f0b..92fa5ae 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -42,7 +42,7 @@
  * tag:  [tagname|*|=]		Tag to match against
  * regex:			regular expression to match against (optional)
  * *:				Match anything
- * =:				Match value of last examined log record
+ * =:				Match value of last successfully matched record
  */
 
 #include "config.h"
@@ -245,6 +245,8 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			    legend, vxid, VSL_tags[tag], type, len, data);
 
 			if (ok) {
+				le->vxid_last = vxid;
+				le->tag_last = tag;
 				le->skip_cnt = 0;
 				logexp_next(le);
 				if (le->test == NULL)
@@ -253,9 +255,6 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			}
 			if (skip)
 				le->skip_cnt++;
-
-			le->vxid_last = vxid;
-			le->tag_last = tag;
 		}
 	}
 



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 5b17fb2 Fix logexp record length to account for terminating
 zero.
Message-ID: <E1WUYVc-0004aV-7T@project.varnish-software.com>


commit 5b17fb212b68dcbae35a6e693bb80d5949730313
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 12:31:30 2014 +0200

    Fix logexp record length to account for terminating zero.
    
    This makes it possible to anchor regex's at the end again.

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index 92fa5ae..e6adae7 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -196,7 +196,7 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			vxid = VSL_ID(t->c->rec.ptr);
 			tag = VSL_TAG(t->c->rec.ptr);
 			data = VSL_CDATA(t->c->rec.ptr);
-			len = VSL_LEN(t->c->rec.ptr);
+			len = VSL_LEN(t->c->rec.ptr) - 1;
 
 			if (tag == SLT__Batch)
 				continue;



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 34f4b96 Remove struct sess acct_bit and bit fields from
 SessClose log record
Message-ID: <E1WUYVc-0004ae-8f@project.varnish-software.com>


commit 34f4b96a53356f49e782d7a31957704f2129d7b2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Mar 28 11:04:12 2014 +0100

    Remove struct sess acct_bit and bit fields from SessClose log record
    
    We want to remove the req->acct_req dup counters that was feeding this
    data to reduce the size of struct req. The statistics value of the
    boolean flags was also small. So it's going away.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ea5d990..b793467 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -234,12 +234,6 @@ struct acct {
 #undef ACCT
 };
 
-struct acct_bit {
-#define ACCT(foo)	unsigned int	foo : 1;
-#include "tbl/acct_fields.h"
-#undef ACCT
-};
-
 /*--------------------------------------------------------------------*/
 
 #define L0(t, n)
@@ -748,8 +742,6 @@ struct sess {
 	enum sess_close		reason;
 	uint32_t		vxid;
 
-	struct acct_bit		acct_bit;
-
 	/* Cross references ------------------------------------------*/
 
 	struct sesspool		*sesspool;
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 73ed5c7..8059c83 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -81,7 +81,6 @@ SES_Charge(struct worker *wrk, struct req *req)
 
 #define ACCT(foo)				\
 	wrk->stats.s_##foo += a->foo;		\
-	if (a->foo) sp->acct_bit.foo =1;	\
 	a->foo = 0;
 #include "tbl/acct_fields.h"
 #undef ACCT
@@ -316,7 +315,6 @@ SES_Close(struct sess *sp, enum sess_close reason)
 void
 SES_Delete(struct sess *sp, enum sess_close reason, double now)
 {
-	struct acct_bit *b;
 	struct sesspool *pp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -332,10 +330,8 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 		now = VTIM_real();
 	assert(!isnan(sp->t_open));
 
-	b = &sp->acct_bit;
-	VSL(SLT_SessClose, sp->vxid, "%s %.3f %u %u %u %u %u %u",
-	    sess_close_2str(sp->reason, 0), now - sp->t_open, b->req,
-	    b->pipe, b->pass, b->fetch, b->hdrbytes, b->bodybytes);
+	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
+	    sess_close_2str(sp->reason, 0), now - sp->t_open);
 	VSL(SLT_End, sp->vxid, "%s", "");
 
 	Lck_Delete(&sp->mtx);
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 0f50a48..10a3a15 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -102,16 +102,10 @@ SLTM(SessOpen, 0, "Client connection opened",
 SLTM(SessClose, 0, "Client connection closed",
 	"SessionClose is the last record for any client connection.\n\n"
 	"The format is::\n\n"
-	"\t%s %f %u %u %u %u %u %u\n"
-	"\t|  |  |  |  |  |  |  |\n"
-	"\t|  |  |  |  |  |  |  +- Bool: Body bytes were sent\n"
-	"\t|  |  |  |  |  |  +---- Bool: Resp.Header bytes sent\n"
-	"\t|  |  |  |  |  +------- Bool: Backend fetches initiated\n"
-	"\t|  |  |  |  +---------- Bool: 'pass' used\n"
-	"\t|  |  |  +------------- Bool: 'pipe' used\n"
-	"\t|  |  +---------------- Bool: good requests completed\n"
-	"\t|  +------------------- How long the session was open\n"
-	"\t+---------------------- Why the connection closed\n"
+	"\t%s %f\n"
+	"\t|  |\n"
+	"\t|  +- How long the session was open\n"
+	"\t+---- Why the connection closed\n"
 	"\n"
 )
 



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 6f2e1bc Request handling byte accounting
Message-ID: <E1WUYVc-0004an-9s@project.varnish-software.com>


commit 6f2e1bcd343a7a745ac01aa61e8ce0e66b3bb6cc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Mar 28 11:25:38 2014 +0100

    Request handling byte accounting
    
    Counters show bytes received/transmitted, split between header and
    body bytes.
    
    SLT_ReqAcct logs per request counter values.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b793467..de7d895 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -228,9 +228,9 @@ struct http_conn {
 
 /*--------------------------------------------------------------------*/
 
-struct acct {
+struct acct_req {
 #define ACCT(foo)	uint64_t	foo;
-#include "tbl/acct_fields.h"
+#include "tbl/acct_fields_req.h"
 #undef ACCT
 };
 
@@ -654,9 +654,9 @@ struct req {
 	double			d_ttl;
 
 	unsigned char		wantbody;
-	uint64_t		req_bodybytes;
+	uint64_t		req_bodybytes;	/* Parsed req bodybytes */
 
-	uint64_t		resp_bodybytes;
+	uint64_t		resp_hdrbytes;	/* Scheduled resp hdrbytes */
 
 	uint16_t		err_code;
 	const char		*err_reason;
@@ -716,7 +716,7 @@ struct req {
 	struct vsl_log		vsl[1];
 
 	/* Temporary accounting */
-	struct acct		acct_req;
+	struct acct_req		acct;
 
 	/* Synth content in vcl_error */
 	struct vsb		*synth_body;
@@ -847,6 +847,7 @@ int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize);
 int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
 
 /* cache_http1_deliver.c */
+unsigned V1D_FlushReleaseAcct(struct req *req);
 void V1D_Deliver(struct req *);
 void V1D_Deliver_Synth(struct req *req);
 
@@ -892,6 +893,7 @@ VDP_pop(struct req *req, vdp_bytes *func)
 
 /* cache_req_fsm.c [CNT] */
 enum req_fsm_nxt CNT_Request(struct worker *, struct req *);
+void CNT_AcctLogCharge(struct dstat *, struct req *);
 
 /* cache_cli.c [CLI] */
 void CLI_Init(void);
@@ -1099,14 +1101,13 @@ void WRW_Chunked(const struct worker *w);
 void WRW_EndChunk(const struct worker *w);
 void WRW_Reserve(struct worker *w, int *fd, struct vsl_log *, double t0);
 unsigned WRW_Flush(const struct worker *w);
-unsigned WRW_FlushRelease(struct worker *w);
+unsigned WRW_FlushRelease(struct worker *w, ssize_t *pacc);
 unsigned WRW_Write(const struct worker *w, const void *ptr, int len);
 unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf);
 
 /* cache_session.c [SES] */
 void SES_Close(struct sess *sp, enum sess_close reason);
 void SES_Delete(struct sess *sp, enum sess_close reason, double now);
-void SES_Charge(struct worker *, struct req *);
 struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
 void SES_DeletePool(struct sesspool *sp);
 int SES_ScheduleReq(struct req *);
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index 8aa022b..ca9da16 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -126,7 +126,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct ws);
 	SZOF(struct http);
 	SZOF(struct http_conn);
-	SZOF(struct acct);
+	SZOF(struct acct_req);
 	SZOF(struct worker);
 	SZOF(struct wrk_accept);
 	SZOF(struct storage);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index b1c3362..dbbe7b1 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -55,7 +55,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 	if (preq->esi_level >= cache_param->max_esi_depth)
 		return;
 
-	(void)WRW_FlushRelease(wrk);
+	(void)V1D_FlushReleaseAcct(preq);
 
 	/* Take a workspace snapshot */
 	wrk_ws_wm = WS_Snapshot(wrk->aws); /* XXX ? */
@@ -120,9 +120,17 @@ ved_include(struct req *preq, const char *src, const char *host)
 		AZ(req->wrk);
 		(void)usleep(10000);
 	}
+	AN(WRW_IsReleased(wrk));
 
-	/* Make sure the VSL id has been released */
-	AZ(req->vsl->wid);
+	/* Flush and release the log */
+	AN(req->vsl->wid);
+	VSLb(req->vsl, SLT_End, "%s", "");
+	VSL_Flush(req->vsl, 0);
+	req->vsl->wid = 0;
+
+	/* Charge the transmitted body byte counts to the parent request */
+	preq->acct.resp_bodybytes += req->acct.resp_bodybytes;
+	req->acct.resp_bodybytes = 0;
 
 	/* Reset the workspace */
 	WS_Reset(wrk->aws, wrk_ws_wm);	/* XXX ? */
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 6cb2de8..1530937 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -501,7 +501,6 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	}
 
 	wrk->stats.busy_sleep++;
-	SES_Charge(wrk, req);
 	/*
 	 * The objhead reference transfers to the sess, we get it
 	 * back when the sess comes off the waiting list and
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index e82ed31..5e99668 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -46,8 +46,6 @@ v1d_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len)
 
 	if (len > 0)
 		wl = WRW_Write(req->wrk, ptr, len);
-	if (wl > 0)
-		req->acct_req.bodybytes += wl;
 	if (act > VDP_NULL && WRW_Flush(req->wrk))
 		return (-1);
 	if (len != wl)
@@ -193,6 +191,33 @@ v1d_WriteDirObj(struct req *req)
 	return (ois);
 }
 
+/*--------------------------------------------------------------------
+ * V1D_FlushReleaseAcct()
+ * Call WRW_FlushRelease on the worker and update the requests
+ * byte accounting with the number of bytes transmitted
+ *
+ * Returns the return value from WRW_FlushRelease()
+ */
+unsigned
+V1D_FlushReleaseAcct(struct req *req)
+{
+	unsigned u;
+	ssize_t txcnt = 0, hdrbytes;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->wrk, WORKER_MAGIC);
+	u = WRW_FlushRelease(req->wrk, &txcnt);
+	if (req->acct.resp_hdrbytes < req->resp_hdrbytes) {
+		hdrbytes = req->resp_hdrbytes - req->acct.resp_hdrbytes;
+		if (hdrbytes > txcnt)
+			hdrbytes = txcnt;
+	} else
+		hdrbytes = 0;
+	req->acct.resp_hdrbytes += hdrbytes;
+	req->acct.resp_bodybytes += txcnt - hdrbytes;
+	return (u);
+}
+
 void
 V1D_Deliver(struct req *req)
 {
@@ -288,7 +313,7 @@ V1D_Deliver(struct req *req)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(req->res_mode & RES_ESI_CHILD))
-		req->acct_req.hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+		req->resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
 
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
@@ -320,7 +345,7 @@ V1D_Deliver(struct req *req)
 	    !(req->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(req->wrk);
 
-	if ((WRW_FlushRelease(req->wrk) || ois != OIS_DONE) && req->sp->fd >= 0)
+	if ((V1D_FlushReleaseAcct(req) || ois != OIS_DONE) && req->sp->fd >= 0)
 		SES_Close(req->sp, SC_REM_CLOSE);
 }
 
@@ -395,7 +420,7 @@ V1D_Deliver_Synth(struct req *req)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(req->res_mode & RES_ESI_CHILD))
-		req->acct_req.hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+		req->resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
 
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
@@ -418,6 +443,6 @@ V1D_Deliver_Synth(struct req *req)
 	if (req->res_mode & RES_CHUNKED && !(req->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(req->wrk);
 
-	if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0)
+	if (V1D_FlushReleaseAcct(req) && req->sp->fd >= 0)
 		SES_Close(req->sp, SC_REM_CLOSE);
 }
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 317bfcb..82dcab6 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -330,7 +330,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 		}
 	}
 
-	if (WRW_FlushRelease(wrk) || i != 0) {
+	if (WRW_FlushRelease(wrk, NULL) || i != 0) {
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 37ed35a..a46534c 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -126,6 +126,7 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 				VSLb_ts_req(req, "Start", now);
 			VSLb_ts_req(req, "Req", now);
 			req->t_req = req->t_prev;
+			req->acct.req_hdrbytes += Tlen(req->htc->rxbuf);
 			return (REQ_FSM_MORE);
 		} else if (hs == HTTP1_ERROR_EOF) {
 			why = SC_REM_CLOSE;
@@ -161,6 +162,8 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			}
 		}
 	}
+	req->acct.req_hdrbytes += Tlen(req->htc->rxbuf);
+	CNT_AcctLogCharge(&wrk->stats, req);
 	SES_ReleaseReq(req);
 	assert(why != SC_NULL);
 	SES_Delete(sp, why, now);
@@ -200,20 +203,28 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 		req->vcl = NULL;
 	}
 
+	/* Charge and log byte counters */
+	AN(req->vsl->wid);
+	CNT_AcctLogCharge(&wrk->stats, req);
+	req->req_bodybytes = 0;
+	req->resp_hdrbytes = 0;
+
+	/* Nuke the VXID. http1_dissect() will allocate a new one when
+	   necessary */
+	VSLb(req->vsl, SLT_End, "%s", "");
+	VSL_Flush(req->vsl, 0);
+	req->vsl->wid = 0;
+
 	if (!isnan(req->t_prev) && req->t_prev > 0.)
 		sp->t_idle = req->t_prev;
 	else
 		sp->t_idle = W_TIM_real(wrk);
-	VSL_Flush(req->vsl, 0);
 
 	req->t_first = NAN;
 	req->t_prev = NAN;
 	req->t_req = NAN;
 	req->req_body_status = REQ_BODY_INIT;
 
-	// req->req_bodybytes = 0;
-	req->resp_bodybytes = 0;
-
 	req->hash_always_miss = 0;
 	req->hash_ignore_busy = 0;
 
@@ -235,10 +246,17 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->aws, NULL);
 
 	if (HTTP1_Reinit(req->htc) == HTTP1_COMPLETE) {
+		AZ(req->vsl->wid);
+		req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER;
+		VSLb(req->vsl, SLT_Begin, "req %u rxreq",
+		    req->sp->vxid & VSL_IDENTMASK);
+		VSL(SLT_Link, req->sp->vxid, "req %u rxreq",
+		    req->vsl->wid & VSL_IDENTMASK);
 		VSLb_ts_req(req, "Start", sp->t_idle);
 		VSLb_ts_req(req, "Req", sp->t_idle);
 		req->t_req = req->t_prev;
 		wrk->stats.sess_pipeline++;
+		req->acct.req_hdrbytes += Tlen(req->htc->rxbuf);
 		return (SESS_DONE_RET_START);
 	} else {
 		if (Tlen(req->htc->rxbuf))
@@ -285,6 +303,7 @@ http1_dissect(struct worker *wrk, struct req *req)
 	const char *r_411 = "HTTP/1.1 411 Length Required\r\n\r\n";
 	const char *r_417 = "HTTP/1.1 417 Expectation Failed\r\n\r\n";
 	char *p;
+	ssize_t r;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -314,7 +333,9 @@ http1_dissect(struct worker *wrk, struct req *req)
 	/* If we could not even parse the request, just close */
 	if (req->err_code != 0) {
 		wrk->stats.client_req_400++;
-		(void)write(req->sp->fd, r_400, strlen(r_400));
+		r = write(req->sp->fd, r_400, strlen(r_400));
+		if (r > 0)
+			req->acct.resp_hdrbytes += r;
 		SES_Close(req->sp, SC_RX_JUNK);
 		return (REQ_FSM_DONE);
 	}
@@ -326,7 +347,9 @@ http1_dissect(struct worker *wrk, struct req *req)
 
 	if (req->req_body_status == REQ_BODY_FAIL) {
 		wrk->stats.client_req_411++;
-		(void)write(req->sp->fd, r_411, strlen(r_411));
+		r = write(req->sp->fd, r_411, strlen(r_411));
+		if (r > 0)
+			req->acct.resp_hdrbytes += r;
 		SES_Close(req->sp, SC_RX_JUNK);
 		return (REQ_FSM_DONE);
 	}
@@ -335,21 +358,25 @@ http1_dissect(struct worker *wrk, struct req *req)
 		if (strcasecmp(p, "100-continue")) {
 			wrk->stats.client_req_417++;
 			req->err_code = 417;
-			(void)write(req->sp->fd, r_417, strlen(r_417));
+			r = write(req->sp->fd, r_417, strlen(r_417));
+			if (r > 0)
+				req->resp_hdrbytes += r;
 			SES_Close(req->sp, SC_RX_JUNK);
 			return (REQ_FSM_DONE);
 		}
-		if (strlen(r_100) != write(req->sp->fd, r_100, strlen(r_100))) {
-			// XXX: stats counter ?
+		r = write(req->sp->fd, r_100, strlen(r_100));
+		if (r > 0)
+			req->acct.resp_hdrbytes += r;
+		if (r != strlen(r_100)) {
 			SES_Close(req->sp, SC_REM_CLOSE);
 			return (REQ_FSM_DONE);
 		}
 	}
 
 	wrk->stats.client_req++;
+	wrk->stats.s_req++;
 
 	AZ(req->err_code);
-	req->acct_req.req++;
 	req->ws_req = WS_Snapshot(req->ws);
 	req->doclose = http_DoConnection(req->http);
 
@@ -475,6 +502,7 @@ http1_iter_req_body(struct req *req, void *buf, ssize_t len)
 	}
 	req->h1.bytes_done += len;
 	req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
+	req->acct.req_bodybytes += len;
 	return (len);
 }
 
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 1a0a44f..7d77096 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -66,6 +66,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	struct worker *wrk;
 	struct pollfd fds[2];
 	int i;
+	ssize_t txcnt;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
@@ -80,14 +81,13 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req);
-	req->acct_req.hdrbytes += HTTP1_Write(wrk, bo->bereq, 0);
+	(void)HTTP1_Write(wrk, bo->bereq, 0);
 
 	if (req->htc->pipeline.b != NULL)
-		req->acct_req.bodybytes +=
-		    WRW_Write(wrk, req->htc->pipeline.b,
+		(void)WRW_Write(wrk, req->htc->pipeline.b,
 		    Tlen(req->htc->pipeline));
 
-	i = WRW_FlushRelease(wrk);
+	i = WRW_FlushRelease(wrk, &txcnt);
 
 	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 090e0b6..8888226 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -200,7 +200,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	req->acct_req.synth++;
+	wrk->stats.s_synth++;
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	h = req->resp;
@@ -281,7 +281,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	AZ(req->obj);
 
-	req->acct_req.fetch++;
+	wrk->stats.s_fetch++;
 	(void)HTTP1_DiscardReqBody(req);
 
 	if (req->objcore->flags & OC_F_FAILED) {
@@ -554,7 +554,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_RESTART;
 		break;
 	case VCL_RET_FETCH:
-		req->acct_req.pass++;
+		wrk->stats.s_pass++;
 		req->objcore = HSH_Private(wrk);
 		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 		VBF_Fetch(wrk, req, req->objcore, NULL, VBF_PASS);
@@ -593,7 +593,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
-	req->acct_req.pipe++;
+	wrk->stats.s_pipe++;
 	bo = VBO_GetBusyObj(wrk, req);
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
 	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
@@ -897,33 +897,43 @@ CNT_Request(struct worker *wrk, struct req *req)
 	}
 	if (nxt == REQ_FSM_DONE) {
 		/* XXX: Workaround for pipe */
-		if (req->sp->fd >= 0) {
-			VSLb(req->vsl, SLT_Length, "%ju",
-			    (uintmax_t)req->resp_bodybytes);
-		}
 
 		while (!VTAILQ_EMPTY(&req->body)) {
 			st = VTAILQ_FIRST(&req->body);
 			VTAILQ_REMOVE(&req->body, st, list);
 			STV_free(st);
 		}
-
-		/* done == 2 was charged by cache_hash.c */
-		SES_Charge(wrk, req);
-
-		/*
-		 * Nuke the VXID, cache_http1_fsm.c::http1_dissect() will
-		 * allocate a new one when necessary.
-		 */
-		VSLb(req->vsl, SLT_End, "%s", "");
-		req->vsl->wid = 0;
-		req->wrk = NULL;
 	}
-
+	req->wrk = NULL;
 	assert(WRW_IsReleased(wrk));
 	return (nxt);
 }
 
+void
+CNT_AcctLogCharge(struct dstat *ds, struct req *req)
+{
+	struct acct_req *a;
+
+	AN(ds);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	a = &req->acct;
+
+	VSLb(req->vsl, SLT_ReqAcct, "%ju %ju %ju %ju %ju %ju",
+	    (uintmax_t)a->req_hdrbytes,
+	    (uintmax_t)a->req_bodybytes,
+	    (uintmax_t)(a->req_hdrbytes + a->req_bodybytes),
+	    (uintmax_t)a->resp_hdrbytes,
+	    (uintmax_t)a->resp_bodybytes,
+	    (uintmax_t)(a->resp_hdrbytes + a->resp_bodybytes));
+
+#define ACCT(foo)			\
+	ds->s_##foo += a->foo;		\
+	a->foo = 0;
+#include "tbl/acct_fields_req.h"
+#undef ACCT
+}
+
 /*
 DOT }
 */
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 8059c83..0c5d2f9 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -62,31 +62,6 @@ struct sesspool {
 };
 
 /*--------------------------------------------------------------------
- * Charge statistics from worker to request and session.
- */
-
-void
-SES_Charge(struct worker *wrk, struct req *req)
-{
-	struct sess *sp;
-	struct acct *a;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	a = &req->acct_req;
-	req->resp_bodybytes += a->bodybytes;
-
-#define ACCT(foo)				\
-	wrk->stats.s_##foo += a->foo;		\
-	a->foo = 0;
-#include "tbl/acct_fields.h"
-#undef ACCT
-}
-
-/*--------------------------------------------------------------------
  * Get a new session, preferably by recycling an already ready one
  *
  * Layout is:
@@ -395,6 +370,9 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 
 	WS_Init(req->ws, "req", p, e - p);
 
+	req->req_bodybytes = 0;
+	req->resp_hdrbytes = 0;
+
 	req->t_first = NAN;
 	req->t_prev = NAN;
 	req->t_req = NAN;
@@ -411,10 +389,14 @@ SES_ReleaseReq(struct req *req)
 	struct sesspool *pp;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AZ(req->vcl);
-#define ACCT(foo)	AZ(req->acct_req.foo);
-#include "tbl/acct_fields.h"
+
+	/* Make sure the request counters have all been zeroed */
+#define ACCT(foo) \
+	AZ(req->acct.foo);
+#include "tbl/acct_fields_req.h"
 #undef ACCT
+
+	AZ(req->vcl);
 	sp = req->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	pp = sp->sesspool;
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index adf2f11..a418ea0 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -58,6 +58,7 @@ struct wrw {
 	unsigned		ciov;	/* Chunked header marker */
 	double			t0;
 	struct vsl_log		*vsl;
+	ssize_t			cnt;	/* Flushed byte count */
 };
 
 /*--------------------------------------------------------------------
@@ -101,7 +102,7 @@ WRW_Reserve(struct worker *wrk, int *fd, struct vsl_log *vsl, double t0)
 }
 
 static void
-WRW_Release(struct worker *wrk)
+wrw_release(struct worker *wrk, ssize_t *pacc)
 {
 	struct wrw *wrw;
 
@@ -109,6 +110,8 @@ WRW_Release(struct worker *wrk)
 	wrw = wrk->wrw;
 	wrk->wrw = NULL;
 	CHECK_OBJ_NOTNULL(wrw, WRW_MAGIC);
+	if (pacc != NULL)
+		*pacc += wrw->cnt;
 	WS_Release(wrk->aws, 0);
 	WS_Reset(wrk->aws, NULL);
 }
@@ -171,6 +174,8 @@ WRW_Flush(const struct worker *wrk)
 		}
 
 		i = writev(*wrw->wfd, wrw->iov, wrw->niov);
+		if (i > 0)
+			wrw->cnt += i;
 		while (i != wrw->liov && i > 0) {
 			/* Remove sent data from start of I/O vector,
 			 * then retry; we hit a timeout, but some data
@@ -195,6 +200,8 @@ WRW_Flush(const struct worker *wrk)
 
 			wrw_prune(wrw, i);
 			i = writev(*wrw->wfd, wrw->iov, wrw->niov);
+			if (i > 0)
+				wrw->cnt += i;
 		}
 		if (i <= 0) {
 			wrw->werr++;
@@ -212,14 +219,14 @@ WRW_Flush(const struct worker *wrk)
 }
 
 unsigned
-WRW_FlushRelease(struct worker *wrk)
+WRW_FlushRelease(struct worker *wrk, ssize_t *pacc)
 {
 	unsigned u;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(wrk->wrw->wfd);
 	u = WRW_Flush(wrk);
-	WRW_Release(wrk);
+	wrw_release(wrk, pacc);
 	return (u);
 }
 
diff --git a/bin/varnishtest/tests/l00002.vtc b/bin/varnishtest/tests/l00002.vtc
new file mode 100644
index 0000000..0106324
--- /dev/null
+++ b/bin/varnishtest/tests/l00002.vtc
@@ -0,0 +1,108 @@
+varnishtest "Test request byte counters"
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp -bodylen 1000
+
+	rxreq
+	expect req.url == "/2"
+	txresp -bodylen 2000
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		unset resp.http.date;
+		unset resp.http.age;
+		unset resp.http.via;
+		unset resp.http.x-varnish;
+	}
+} -start
+
+# Request (1001):
+# GET /1 HTTP/1.1\r\n		 17 bytes
+# Content-Length: 4\r\n		 19 bytes
+# \r\n				  2 bytes
+# Total:			 38 bytes
+# Response:
+# HTTP/1.1 200 Ok\r\n		 17 bytes
+# Content-Length: 1000\r\n	 22 bytes
+# Connection: keep-alive\r\n	 24 bytes
+# Accept-Ranges: bytes\r\n	 22 bytes
+# \r\n				  2 bytes
+# Total:			 87 bytes
+
+# Request (1003):
+# GET /2 HTTP/1.1\r\n		 17 bytes
+# \r\n				  2 bytes
+# Total:			 19 bytes
+# Response:
+# HTTP/1.1 200 Ok\r\n		 17 bytes
+# Content-Length: 2000\r\n	 22 bytes
+# Connection: keep-alive\r\n	 24 bytes
+# Accept-Ranges: bytes\r\n	 22 bytes
+# \r\n				  2 bytes
+# Total:			 87 bytes
+
+# Request (1005):
+# GET /2 HTTP/1.1\r\n		 17 bytes
+# \r\n				  2 bytes
+# Total:			 19 bytes
+# Response:
+# HTTP/1.1 200 Ok\r\n		 17 bytes
+# Content-Length: 2000\r\n	 22 bytes
+# Connection: keep-alive\r\n	 24 bytes
+# Accept-Ranges: bytes\r\n	 22 bytes
+# \r\n				  2 bytes
+# Total:			 87 bytes
+
+# Request (1006):
+# GET\r\n			  5 bytes
+# \r\n				  2 bytes
+# Total:			  7 bytes
+# Response:
+# HTTP/1.1 400 Bad Request\r\n	 26 bytes
+# \r\n				  2 bytes
+# Total:			 28 bytes
+logexpect l1 -v v1 -g request {
+	expect 0 1001	Begin	"^req .* rxreq"
+	expect * =	ReqAcct	"^38 4 42 87 1000 1087$"
+	expect 0 =	End
+	expect 0 1002	Begin	"^bereq "
+	expect * =	End
+	expect 0 1003	Begin	"^req .* rxreq"
+	expect * =	ReqAcct "^19 0 19 87 2000 2087$"
+	expect 0 =	End
+	expect 0 1004	Begin	"^bereq"
+	expect * =	End
+	expect 0 1005	Begin	"^req .* rxreq"
+	expect * =	ReqAcct "^19 0 19 87 2000 2087$"
+	expect 0 =	End
+	expect 0 1006	Begin	"^req .* rxreq"
+	expect * =	ReqAcct	"^7 0 7 28 0 28$"
+	expect 0 =	End
+} -start
+
+# Request 1001
+client c1 {
+	txreq -url "/1" -body "asdf"
+	rxresp
+	expect resp.status == 200
+
+	send "GET /2 HTTP/1.1\r\n\r\nGET /2 HTTP/1.1\r\n\r\n"
+	rxresp
+	expect resp.status == 200
+	rxresp
+	expect resp.status == 200
+
+	send "GET\r\n\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -expect s_req_hdrbytes == 83
+varnish v1 -expect s_req_bodybytes == 4
+varnish v1 -expect s_resp_hdrbytes == 289
+varnish v1 -expect s_resp_bodybytes == 5000
diff --git a/include/Makefile.am b/include/Makefile.am
index 3813836..e996fc0 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -2,7 +2,7 @@
 
 # API headers
 nobase_pkginclude_HEADERS = \
-	tbl/acct_fields.h \
+	tbl/acct_fields_req.h \
 	tbl/backend_poll.h \
 	tbl/ban_vars.h \
 	tbl/bo_flags.h \
diff --git a/include/tbl/acct_fields.h b/include/tbl/acct_fields.h
deleted file mode 100644
index ac205a7..0000000
--- a/include/tbl/acct_fields.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*-
- * Copyright (c) 2008 Verdens Gang AS
- * Copyright (c) 2008-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * These are the stats we keep track of per session.
- * SES_Charge() sums them into wrk->stats
- * NB: Remember to mark those in vsc_fields.h to be included in struct dstat.
- */
-
-ACCT(req)
-ACCT(pipe)
-ACCT(pass)
-ACCT(fetch)
-ACCT(synth)
-ACCT(hdrbytes)
-ACCT(bodybytes)
diff --git a/include/tbl/acct_fields_req.h b/include/tbl/acct_fields_req.h
new file mode 100644
index 0000000..a1759f5
--- /dev/null
+++ b/include/tbl/acct_fields_req.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2008 Verdens Gang AS
+ * Copyright (c) 2008-2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * These are the stats we keep track of per request.
+ * NB: Remember to mark those in vsc_fields.h to be included in struct dstat.
+ */
+
+ACCT(req_hdrbytes)
+ACCT(req_bodybytes)
+ACCT(resp_hdrbytes)
+ACCT(resp_bodybytes)
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 3874ec1..859a2eb 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -384,13 +384,21 @@ VSC_F(s_synth,			uint64_t, 1, 'a', info,
     "Total synth",
 	""
 )
-VSC_F(s_hdrbytes,		uint64_t, 1, 'a', info,
-    "Total header bytes",
-	""
+VSC_F(s_req_hdrbytes,		uint64_t, 1, 'a', info,
+    "Request header bytes",
+	"Total request header bytes received"
 )
-VSC_F(s_bodybytes,		uint64_t, 1, 'a', info,
-    "Total body bytes",
-	""
+VSC_F(s_req_bodybytes,		uint64_t, 1, 'a', info,
+    "Request body bytes",
+	"Total request body bytes received"
+)
+VSC_F(s_resp_hdrbytes,		uint64_t, 1, 'a', info,
+    "Response header bytes",
+	"Total response header bytes transmitted"
+)
+VSC_F(s_resp_bodybytes,		uint64_t, 1, 'a', info,
+    "Reponse body bytes",
+	"Total response body bytes transmitted"
 )
 
 VSC_F(sess_closed,		uint64_t, 1, 'a', info,
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 10a3a15..252b859 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -474,4 +474,18 @@ SLTM(Timestamp, 0, "Timing information",
 	"\n"
 )
 
+SLTM(ReqAcct, 0, "Request handling byte counts",
+	"Contains byte counts for the request handling.\n\n"
+	"The format is::\n\n"
+	"\t%d %d %d %d %d %d\n"
+	"\t|  |  |  |  |  |\n"
+	"\t|  |  |  |  |  +- Total bytes transmitted\n"
+	"\t|  |  |  |  +---- Body bytes transmitted\n"
+	"\t|  |  |  +------- Header bytes transmitted\n"
+	"\t|  |  +---------- Total bytes received\n"
+	"\t|  +------------- Body bytes received\n"
+	"\t+---------------- Header bytes received\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] c654dc5 ESI byte accounting
Message-ID: <E1WUYVc-0004ax-BD@project.varnish-software.com>


commit c654dc51e6bbb5b58ab8f9cc80b2cd6b12c85e1e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sat Mar 29 23:04:07 2014 +0100

    ESI byte accounting
    
    Charge ESI bodybytes to the parent request.
    
    Log SLT_ESI_BodyBytes in each ESI-involved log transaction. This shows
    how many bytes each request adds to the total byte count.
    
    SLT_ReqAcct is not logged for ESI sub-requests.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index de7d895..2e41e24 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -657,6 +657,7 @@ struct req {
 	uint64_t		req_bodybytes;	/* Parsed req bodybytes */
 
 	uint64_t		resp_hdrbytes;	/* Scheduled resp hdrbytes */
+	uint64_t		resp_bodybytes; /* Scheduled resp bodybytes */
 
 	uint16_t		err_code;
 	const char		*err_reason;
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index dbbe7b1..5aa85ee 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -218,15 +218,17 @@ ved_pretend_gzip(struct req *req, const uint8_t *p, ssize_t l)
 	while (l > 0) {
 		if (l >= 65535) {
 			lx = 65535;
-			(void)WRW_Write(req->wrk, buf1, sizeof buf1);
+			req->resp_bodybytes +=
+			    WRW_Write(req->wrk, buf1, sizeof buf1);
 		} else {
 			lx = (uint16_t)l;
 			buf2[0] = 0;
 			vle16enc(buf2 + 1, lx);
 			vle16enc(buf2 + 3, ~lx);
-			(void)WRW_Write(req->wrk, buf2, sizeof buf2);
+			req->resp_bodybytes +=
+			    WRW_Write(req->wrk, buf2, sizeof buf2);
 		}
-		(void)WRW_Write(req->wrk, p, lx);
+		req->resp_bodybytes += WRW_Write(req->wrk, p, lx);
 		req->crc = crc32(req->crc, p, lx);
 		req->l_crc += lx;
 		l -= lx;
@@ -283,7 +285,8 @@ ESI_Deliver(struct req *req)
 		if (isgzip && !(req->res_mode & RES_GUNZIP)) {
 			assert(sizeof gzip_hdr == 10);
 			/* Send out the gzip header */
-			(void)WRW_Write(req->wrk, gzip_hdr, 10);
+			req->resp_bodybytes +=
+			    WRW_Write(req->wrk, gzip_hdr, 10);
 			req->l_crc = 0;
 			req->gzip_resp = 1;
 			req->crc = crc32(0L, Z_NULL, 0);
@@ -339,8 +342,9 @@ ESI_Deliver(struct req *req)
 					 * We have a gzip'ed VEC and delivers
 					 * a gzip'ed ESI response.
 					 */
-					(void)WRW_Write(req->wrk,
-					    st->ptr + off, l2);
+					req->resp_bodybytes +=
+					    WRW_Write(req->wrk,
+						st->ptr + off, l2);
 				} else if (req->gzip_resp) {
 					/*
 					 * A gzip'ed ESI response, but the VEC
@@ -367,8 +371,9 @@ ESI_Deliver(struct req *req)
 					/*
 					 * Ungzip'ed VEC, ungzip'ed ESI response
 					 */
-					(void)WRW_Write(req->wrk,
-					    st->ptr + off, l2);
+					req->resp_bodybytes +=
+					    WRW_Write(req->wrk,
+						st->ptr + off, l2);
 				}
 				off += l2;
 				if (off == st->len) {
@@ -441,7 +446,7 @@ ESI_Deliver(struct req *req)
 		/* MOD(2^32) length */
 		vle32enc(tailbuf + 9, req->l_crc);
 
-		(void)WRW_Write(req->wrk, tailbuf, 13);
+		req->resp_bodybytes += WRW_Write(req->wrk, tailbuf, 13);
 	}
 	(void)WRW_Flush(req->wrk);
 }
@@ -451,7 +456,7 @@ ESI_Deliver(struct req *req)
  */
 
 static uint8_t
-ved_deliver_byterange(const struct req *req, ssize_t low, ssize_t high)
+ved_deliver_byterange(struct req *req, ssize_t low, ssize_t high)
 {
 	struct storage *st;
 	ssize_t l, lx;
@@ -477,7 +482,7 @@ ved_deliver_byterange(const struct req *req, ssize_t low, ssize_t high)
 			l = high - lx;
 		assert(lx >= low && lx + l <= high);
 		if (l != 0)
-			(void)WRW_Write(req->wrk, p, l);
+			req->resp_bodybytes += WRW_Write(req->wrk, p, l);
 		if (p + l < st->ptr + st->len)
 			return(p[l]);
 		lx += l;
@@ -532,7 +537,7 @@ ESI_DeliverChild(struct req *req)
 	 */
 	*dbits = ved_deliver_byterange(req, start/8, last/8);
 	*dbits &= ~(1U << (last & 7));
-	(void)WRW_Write(req->wrk, dbits, 1);
+	req->resp_bodybytes += WRW_Write(req->wrk, dbits, 1);
 	cc = ved_deliver_byterange(req, 1 + last/8, stop/8);
 	switch((int)(stop & 7)) {
 	case 0: /* xxxxxxxx */
@@ -576,7 +581,7 @@ ESI_DeliverChild(struct req *req)
 		INCOMPL();
 	}
 	if (lpad > 0)
-		(void)WRW_Write(req->wrk, dbits + 1, lpad);
+		req->resp_bodybytes += WRW_Write(req->wrk, dbits + 1, lpad);
 
 	/* We need the entire tail, but it may not be in one storage segment */
 	st = VTAILQ_LAST(&req->obj->store, storagehead);
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 5e99668..765fd83 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -46,6 +46,7 @@ v1d_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len)
 
 	if (len > 0)
 		wl = WRW_Write(req->wrk, ptr, len);
+	req->resp_bodybytes += wl;
 	if (act > VDP_NULL && WRW_Flush(req->wrk))
 		return (-1);
 	if (len != wl)
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index a46534c..9bb5e79 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -208,6 +208,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	CNT_AcctLogCharge(&wrk->stats, req);
 	req->req_bodybytes = 0;
 	req->resp_hdrbytes = 0;
+	req->resp_bodybytes = 0;
 
 	/* Nuke the VXID. http1_dissect() will allocate a new one when
 	   necessary */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8888226..2236553 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -897,6 +897,10 @@ CNT_Request(struct worker *wrk, struct req *req)
 	}
 	if (nxt == REQ_FSM_DONE) {
 		/* XXX: Workaround for pipe */
+		AN(req->vsl->wid);
+		if (req->res_mode & (RES_ESI|RES_ESI_CHILD))
+			VSLb(req->vsl, SLT_ESI_BodyBytes, "%ju",
+			    (uintmax_t)req->resp_bodybytes);
 
 		while (!VTAILQ_EMPTY(&req->body)) {
 			st = VTAILQ_FIRST(&req->body);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 0c5d2f9..0c010c1 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -372,6 +372,7 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 
 	req->req_bodybytes = 0;
 	req->resp_hdrbytes = 0;
+	req->resp_bodybytes = 0;
 
 	req->t_first = NAN;
 	req->t_prev = NAN;
diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc
new file mode 100644
index 0000000..cb03a18
--- /dev/null
+++ b/bin/varnishtest/tests/l00003.vtc
@@ -0,0 +1,84 @@
+varnishtest "Test request byte counters with ESI"
+
+server s1 {
+	rxreq
+	expect req.url == "/"
+	txresp -body {<esi:include src="/1"/>ghi}
+
+	rxreq
+	expect req.url == "/1"
+	txresp -body {<esi:include src="/2"/>abc<esi:include src="/2"/>def}
+
+	rxreq
+	expect req.url == "/2"
+	txresp -body {123}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+	sub vcl_deliver {
+		unset resp.http.date;
+		unset resp.http.age;
+		unset resp.http.via;
+		unset resp.http.x-varnish;
+	}
+} -start
+
+# Request (1001):
+# GET / HTTP/1.1\r\n			16 bytes
+# \r\n					 2 bytes
+# Total:				18 bytes
+
+# Reponse:
+# HTTP/1.1 200 Ok\r\n			17 bytes
+# Transfer-Encoding: chunked\r\n	28 bytes
+# Connection: keep-alive\r\n		24 bytes
+# \r\n					 2 bytes
+# Total:				71 bytes
+
+# Response body:
+# 003\r\n				 5 bytes
+# 123\r\n				 5 bytes
+# 003\r\n				 5 bytes
+# abc\r\n				 5 bytes
+# 003\r\n				 5 bytes
+# 123\r\n				 5 bytes
+# 003\r\n				 5 bytes
+# def\r\n				 5 bytes
+# 003\r\n				 5 bytes
+# ghi\r\n				 5 bytes
+# 0\r\n					 3 bytes
+# \r\n					 2 bytes
+# Total:				55 bytes
+
+logexpect l1 -v v1 -g request {
+	expect 0 1001	Begin	"^req .* rxreq"
+	expect * =	ESI_BodyBytes	"^3"
+	expect * =	ReqAcct		"^18 0 18 71 55 126$"
+	expect 0 =	End
+	expect * 1003	Begin		"^req .* esi"
+	expect * =	ESI_BodyBytes	"^6$"
+	expect 0 =	End
+	expect * 1005	Begin		"^req .* esi"
+	expect * =	ESI_BodyBytes	"^3$"
+	expect 0 =	End
+	expect * 1007	Begin		"^req .* esi"
+	expect * =	ESI_BodyBytes	"^3$"
+	expect 0 =	End
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "123abc123defghi"
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -expect s_req_hdrbytes == 18
+varnish v1 -expect s_req_bodybytes == 0
+varnish v1 -expect s_resp_hdrbytes == 71
+varnish v1 -expect s_resp_bodybytes == 55
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 252b859..0213a61 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -475,7 +475,9 @@ SLTM(Timestamp, 0, "Timing information",
 )
 
 SLTM(ReqAcct, 0, "Request handling byte counts",
-	"Contains byte counts for the request handling.\n\n"
+	"Contains byte counts for the request handling. This record is not"
+	" logged for ESI sub-requests, but the sub-requests' response"
+	" body count is added to the main request.\n\n"
 	"The format is::\n\n"
 	"\t%d %d %d %d %d %d\n"
 	"\t|  |  |  |  |  |\n"
@@ -488,4 +490,14 @@ SLTM(ReqAcct, 0, "Request handling byte counts",
 	"\n"
 )
 
+SLTM(ESI_BodyBytes, 0, "ESI body fragment byte counter",
+	"Contains the body byte count for this ESI body fragment."
+	" This number does not include any transfer encoding overhead.\n\n"
+	"The format is::\n\n"
+	"\t%d\n"
+	"\t|\n"
+	"\t+- Body bytes\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 6dfcd17 Pipe byte accounting
Message-ID: <E1WUYVc-0004b7-CH@project.varnish-software.com>


commit 6dfcd17ba234e18314d5f9d739c93aca215a4160
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 01:52:17 2014 +0100

    Pipe byte accounting
    
    Add VSC counters for pipe traffic. Traffic is split between header
    bytes, and input and output piped bytes.
    
    Log SLT_PipeAcct records showing per pipe session byte counts.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2e41e24..c58a1fc 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -699,6 +699,7 @@ struct req {
 #define RES_ESI			(1<<4)
 #define RES_ESI_CHILD		(1<<5)
 #define RES_GUNZIP		(1<<6)
+#define RES_PIPE		(1<<7)
 
 	/* Deliver pipeline */
 #define	N_VDPS			5
@@ -1088,6 +1089,7 @@ const char *reqbody_status_2str(enum req_body_state_e e);
 const char *sess_close_2str(enum sess_close sc, int want_desc);
 
 /* cache_pipe.c */
+void Pipe_Init(void);
 void PipeRequest(struct req *req, struct busyobj *bo);
 
 /* cache_pool.c */
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index a5674c0..c6a2599 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -219,6 +219,7 @@ child_main(void)
 	VBP_Init();
 	WRK_Init();
 	Pool_Init();
+	Pipe_Init();
 
 	EXP_Init();
 	HSH_Init(heritage.hash);
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 7d77096..05b8d0a 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -40,8 +40,17 @@
 #include "vtcp.h"
 #include "vtim.h"
 
+static struct lock pipestat_mtx;
+
+struct acct_pipe {
+	ssize_t		req;
+	ssize_t		bereq;
+	ssize_t		in;
+	ssize_t		out;
+};
+
 static int
-rdf(int fd0, int fd1)
+rdf(int fd0, int fd1, ssize_t *pcnt)
 {
 	int i, j;
 	char buf[BUFSIZ], *p;
@@ -53,12 +62,30 @@ rdf(int fd0, int fd1)
 		j = write(fd1, p, i);
 		if (j <= 0)
 			return (1);
+		*pcnt += j;
 		if (i != j)
 			(void)usleep(100000);		/* XXX hack */
 	}
 	return (0);
 }
 
+static void
+pipecharge(struct req *req, const struct acct_pipe *a)
+{
+
+	VSLb(req->vsl, SLT_PipeAcct, "%ju %ju %ju %ju",
+	    (uintmax_t)a->req,
+	    (uintmax_t)a->bereq,
+	    (uintmax_t)a->in,
+	    (uintmax_t)a->out);
+
+	Lck_Lock(&pipestat_mtx);
+	VSC_C_main->s_pipe_hdrbytes += a->req;
+	VSC_C_main->s_pipe_in += a->in;
+	VSC_C_main->s_pipe_out += a->out;
+	Lck_Unlock(&pipestat_mtx);
+}
+
 void
 PipeRequest(struct req *req, struct busyobj *bo)
 {
@@ -66,7 +93,8 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	struct worker *wrk;
 	struct pollfd fds[2];
 	int i;
-	ssize_t txcnt;
+	struct acct_pipe acct;
+	ssize_t hdrbytes;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
@@ -74,24 +102,38 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	req->res_mode = RES_PIPE;
+
+	memset(&acct, 0, sizeof acct);
+	acct.req = req->acct.req_hdrbytes;
+	req->acct.req_hdrbytes = 0;
+
 	vc = VDI_GetFd(bo);
-	if (vc == NULL)
+	if (vc == NULL) {
+		pipecharge(req, &acct);
+		SES_Close(req->sp, SC_OVERLOAD);
 		return;
+	}
 	bo->vbc = vc;		/* For panic dumping */
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req);
-	(void)HTTP1_Write(wrk, bo->bereq, 0);
+	hdrbytes = HTTP1_Write(wrk, bo->bereq, 0);
 
 	if (req->htc->pipeline.b != NULL)
 		(void)WRW_Write(wrk, req->htc->pipeline.b,
 		    Tlen(req->htc->pipeline));
 
-	i = WRW_FlushRelease(wrk, &txcnt);
+	i = WRW_FlushRelease(wrk, &acct.bereq);
+	if (acct.bereq > hdrbytes) {
+		acct.in = acct.bereq - hdrbytes;
+		acct.bereq = hdrbytes;
+	}
 
 	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
 
 	if (i) {
+		pipecharge(req, &acct);
 		SES_Close(req->sp, SC_TX_PIPE);
 		VDI_CloseFd(&vc);
 		return;
@@ -113,7 +155,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 		i = poll(fds, 2, (int)(cache_param->pipe_timeout * 1e3));
 		if (i < 1)
 			break;
-		if (fds[0].revents && rdf(vc->fd, req->sp->fd)) {
+		if (fds[0].revents && rdf(vc->fd, req->sp->fd, &acct.out)) {
 			if (fds[1].fd == -1)
 				break;
 			(void)shutdown(vc->fd, SHUT_RD);
@@ -121,7 +163,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 			fds[0].events = 0;
 			fds[0].fd = -1;
 		}
-		if (fds[1].revents && rdf(req->sp->fd, vc->fd)) {
+		if (fds[1].revents && rdf(req->sp->fd, vc->fd, &acct.in)) {
 			if (fds[0].fd == -1)
 				break;
 			(void)shutdown(req->sp->fd, SHUT_RD);
@@ -131,7 +173,17 @@ PipeRequest(struct req *req, struct busyobj *bo)
 		}
 	}
 	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
+	pipecharge(req, &acct);
 	SES_Close(req->sp, SC_TX_PIPE);
 	VDI_CloseFd(&vc);
 	bo->vbc = NULL;
 }
+
+/*--------------------------------------------------------------------*/
+
+void
+Pipe_Init(void)
+{
+
+	Lck_New(&pipestat_mtx, lck_pipestat);
+}
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 2236553..1e9ecd5 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -896,7 +896,6 @@ CNT_Request(struct worker *wrk, struct req *req)
 		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
 	}
 	if (nxt == REQ_FSM_DONE) {
-		/* XXX: Workaround for pipe */
 		AN(req->vsl->wid);
 		if (req->res_mode & (RES_ESI|RES_ESI_CHILD))
 			VSLb(req->vsl, SLT_ESI_BodyBytes, "%ju",
@@ -923,13 +922,15 @@ CNT_AcctLogCharge(struct dstat *ds, struct req *req)
 
 	a = &req->acct;
 
-	VSLb(req->vsl, SLT_ReqAcct, "%ju %ju %ju %ju %ju %ju",
-	    (uintmax_t)a->req_hdrbytes,
-	    (uintmax_t)a->req_bodybytes,
-	    (uintmax_t)(a->req_hdrbytes + a->req_bodybytes),
-	    (uintmax_t)a->resp_hdrbytes,
-	    (uintmax_t)a->resp_bodybytes,
-	    (uintmax_t)(a->resp_hdrbytes + a->resp_bodybytes));
+	if (!(req->res_mode & RES_PIPE)) {
+		VSLb(req->vsl, SLT_ReqAcct, "%ju %ju %ju %ju %ju %ju",
+		    (uintmax_t)a->req_hdrbytes,
+		    (uintmax_t)a->req_bodybytes,
+		    (uintmax_t)(a->req_hdrbytes + a->req_bodybytes),
+		    (uintmax_t)a->resp_hdrbytes,
+		    (uintmax_t)a->resp_bodybytes,
+		    (uintmax_t)(a->resp_hdrbytes + a->resp_bodybytes));
+	}
 
 #define ACCT(foo)			\
 	ds->s_##foo += a->foo;		\
diff --git a/bin/varnishtest/tests/l00004.vtc b/bin/varnishtest/tests/l00004.vtc
new file mode 100644
index 0000000..3b07e36
--- /dev/null
+++ b/bin/varnishtest/tests/l00004.vtc
@@ -0,0 +1,63 @@
+varnishtest "Test request byte counters on pipe"
+
+server s1 {
+	rxreq
+	expect req.url == "/"
+	expect req.http.test == "yes"
+	txresp -body "fdsa"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return (pipe);
+	}
+
+	sub vcl_pipe {
+		set bereq.http.test = "yes";
+		unset bereq.http.x-forwarded-for;
+		unset bereq.http.x-varnish;
+		unset bereq.http.connection;
+	}
+} -start
+
+# req:
+# POST / HTTP/1.1\r\n			17 bytes
+# Content-Length: 4\r\n			19 bytes
+# \r\n					 2 bytes
+# Total:				38 bytes
+
+# bereq:
+# POST / HTTP/1.1\r\n			17 bytes
+# Content-Length: 4\r\n			19 bytes
+# test: yes\r\n				11 bytes
+# \r\n					 2 bytes
+# Total:				49 bytes
+
+# reqbody
+# asdf					 4 bytes
+
+# resp:
+# HTTP/1.1 200 Ok\r\n			17 bytes
+# Content-Length: 4\r\n			19 bytes
+# \r\n					 2 bytes
+# fdsa					 4 bytes
+# Total:				42 bytes
+
+logexpect l1 -v v1 -g request {
+	expect 0 1001	Begin		"^req .* rxreq"
+	expect * =	PipeAcct	"^38 49 4 42$"
+	expect 0 =	End
+} -start
+
+client c1 {
+	txreq -req "POST" -url "/" -hdr "Content-Length: 4"
+	send "asdf"
+	rxresp
+	expect resp.status == 200
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -expect s_pipe_hdrbytes == 38
+varnish v1 -expect s_pipe_in == 4
+varnish v1 -expect s_pipe_out == 42
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index 8329bab..1a01782 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -53,4 +53,5 @@ LOCK(nbusyobj)
 LOCK(busyobj)
 LOCK(mempool)
 LOCK(vxid)
+LOCK(pipestat)
 /*lint -restore */
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 859a2eb..4040b0d 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -400,6 +400,20 @@ VSC_F(s_resp_bodybytes,		uint64_t, 1, 'a', info,
     "Reponse body bytes",
 	"Total response body bytes transmitted"
 )
+VSC_F(s_pipe_hdrbytes,		uint64_t, 0, 'a', info,
+    "Pipe request header bytes",
+	"Total request bytes received for piped sessions"
+)
+VSC_F(s_pipe_in,		uint64_t, 0, 'a', info,
+    "Piped bytes from client",
+	"Total number of bytes forwarded from clients in"
+	" pipe sessions"
+)
+VSC_F(s_pipe_out,		uint64_t, 0, 'a', info,
+    "Piped bytes to client",
+	"Total number of bytes forwarded to clients in"
+	" pipe sessions"
+)
 
 VSC_F(sess_closed,		uint64_t, 1, 'a', info,
     "Session Closed",
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 0213a61..99894cc 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -500,4 +500,16 @@ SLTM(ESI_BodyBytes, 0, "ESI body fragment byte counter",
 	"\n"
 )
 
+SLTM(PipeAcct, 0, "Pipe byte counts",
+	"Contains byte counters for pipe sessions.\n\n"
+	"The format is::\n\n"
+	"\t%d %d %d %d\n"
+	"\t|  |  |  |\n"
+	"\t|  |  |  +------- Piped bytes to client\n"
+	"\t|  |  +---------- Piped bytes from client\n"
+	"\t|  +------------- Backend request headers\n"
+	"\t+---------------- Client request headers\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 3398877 Backend fetch byte accounting
Message-ID: <E1WUYVc-0004bH-DI@project.varnish-software.com>


commit 3398877276d228cae9cdcb3c2f5f9a5e7d8178a9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 15:17:18 2014 +0200

    Backend fetch byte accounting
    
    Add SLT_BereqAcct log records showing headers and bodybytes
    received/transmitted when doing backend fetch operations.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c58a1fc..bf28983 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -236,6 +236,14 @@ struct acct_req {
 
 /*--------------------------------------------------------------------*/
 
+struct acct_bereq {
+#define ACCT(foo)	ssize_t		foo;
+#include "tbl/acct_fields_bereq.h"
+#undef ACCT
+};
+
+/*--------------------------------------------------------------------*/
+
 #define L0(t, n)
 #define L1(t, n)		t n;
 #define VSC_F(n,t,l,f,v,e,d)	L##l(t, n)
@@ -566,6 +574,9 @@ struct busyobj {
 	double			t_first;	/* First timestamp logged */
 	double			t_prev;		/* Previous timestamp logged */
 
+	/* Acct */
+	struct acct_bereq	acct;
+
 	const char		*storage_hint;
 	struct director		*director;
 	struct VCL_conf		*vcl;
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 69f849b..22f6441 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -187,6 +187,14 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	if (r)
 		return;
 
+	VSLb(bo->vsl, SLT_BereqAcct, "%ju %ju %ju %ju %ju %ju",
+	    (uintmax_t)bo->acct.bereq_hdrbytes,
+	    (uintmax_t)bo->acct.bereq_bodybytes,
+	    (uintmax_t)(bo->acct.bereq_hdrbytes + bo->acct.bereq_bodybytes),
+	    (uintmax_t)bo->acct.beresp_hdrbytes,
+	    (uintmax_t)bo->acct.beresp_bodybytes,
+	    (uintmax_t)(bo->acct.beresp_hdrbytes + bo->acct.beresp_bodybytes));
+
 	VSLb(bo->vsl, SLT_End, "%s", "");
 	VSL_Flush(bo->vsl, 0);
 
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 82dcab6..b6fac38 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -91,6 +91,7 @@ v1f_pull_straight(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 	if (*priv < l)
 		l = *priv;
 	lr = HTTP1_Read(&bo->htc, p, l);
+	bo->acct.beresp_bodybytes += lr;
 	if (lr <= 0)
 		return (VFP_Error(bo, "straight insufficient bytes"));
 	*lp = lr;
@@ -127,8 +128,10 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 	if (*priv == -1) {
 		/* Skip leading whitespace */
 		do {
-			if (HTTP1_Read(&bo->htc, buf, 1) <= 0)
+			lr = HTTP1_Read(&bo->htc, buf, 1);
+			if (lr <= 0)
 				return (VFP_Error(bo, "chunked read err"));
+			bo->acct.beresp_bodybytes += lr;
 		} while (vct_islws(buf[0]));
 
 		if (!vct_ishex(buf[0]))
@@ -137,9 +140,11 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				if (HTTP1_Read(&bo->htc, buf + u, 1) <= 0)
+				lr = HTTP1_Read(&bo->htc, buf + u, 1);
+				if (lr <= 0)
 					return (VFP_Error(bo,
 					    "chunked read err"));
+				bo->acct.beresp_bodybytes += lr;
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
 				break;
@@ -149,9 +154,12 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 			return (VFP_Error(bo,"chunked header too long"));
 
 		/* Skip trailing white space */
-		while(vct_islws(buf[u]) && buf[u] != '\n')
-			if (HTTP1_Read(&bo->htc, buf + u, 1) <= 0)
+		while(vct_islws(buf[u]) && buf[u] != '\n') {
+			lr = HTTP1_Read(&bo->htc, buf + u, 1);
+			if (lr <= 0)
 				return (VFP_Error(bo, "chunked read err"));
+			bo->acct.beresp_bodybytes += lr;
+		}
 
 		if (buf[u] != '\n')
 			return (VFP_Error(bo,"chunked header no NL"));
@@ -169,6 +177,7 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 		lr = HTTP1_Read(&bo->htc, p, l);
 		if (lr <= 0)
 			return (VFP_Error(bo, "straight insufficient bytes"));
+		bo->acct.beresp_bodybytes += lr;
 		*lp = lr;
 		*priv -= lr;
 		if (*priv == 0)
@@ -179,6 +188,7 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 	i = HTTP1_Read(&bo->htc, buf, 1);
 	if (i <= 0)
 		return (VFP_Error(bo, "chunked read err"));
+	bo->acct.beresp_bodybytes += i;
 	if (buf[0] == '\r' && HTTP1_Read(&bo->htc, buf, 1) <= 0)
 		return (VFP_Error(bo, "chunked read err"));
 	if (buf[0] != '\n')
@@ -281,8 +291,9 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	struct http *hp;
 	enum htc_status_e hs;
 	int retry = -1;
-	int i, first;
+	int i, j, first;
 	struct http_conn *htc;
+	ssize_t hdrbytes;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -312,7 +323,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
-	(void)HTTP1_Write(wrk, hp, 0);	/* XXX: stats ? */
+	hdrbytes = HTTP1_Write(wrk, hp, 0);
 
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
@@ -330,7 +341,12 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 		}
 	}
 
-	if (WRW_FlushRelease(wrk, NULL) || i != 0) {
+	j = WRW_FlushRelease(wrk, &bo->acct.bereq_hdrbytes);
+	if (bo->acct.bereq_hdrbytes > hdrbytes) {
+		bo->acct.bereq_bodybytes = bo->acct.bereq_hdrbytes - hdrbytes;
+		bo->acct.bereq_hdrbytes = hdrbytes;
+	}
+	if (j != 0 || i != 0) {
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
@@ -356,6 +372,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	do {
 		hs = HTTP1_Rx(htc);
 		if (hs == HTTP1_OVERFLOW) {
+			bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf);
 			VSLb(bo->vsl, SLT_FetchError,
 			    "http %sread error: overflow",
 			    first ? "first " : "");
@@ -364,6 +381,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 			return (-1);
 		}
 		if (hs == HTTP1_ERROR_EOF) {
+			bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf);
 			VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
 			    first ? "first " : "");
 			VDI_CloseFd(&bo->vbc);
@@ -377,6 +395,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 			    vc->between_bytes_timeout);
 		}
 	} while (hs != HTTP1_COMPLETE);
+	bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf);
 
 	hp = bo->beresp;
 
diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc
new file mode 100644
index 0000000..51d0d7e
--- /dev/null
+++ b/bin/varnishtest/tests/l00005.vtc
@@ -0,0 +1,68 @@
+varnishtest "Test backend fetch byte counters"
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp -bodylen 1000
+
+	rxreq
+	expect req.url == "/2"
+	send "HTTP/1.1\r\n\r\n"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_fetch {
+		unset bereq.http.x-forwarded-for;
+		unset bereq.http.x-varnish;
+		set bereq.http.Host = "foo.bar";
+	}
+} -start
+
+# Request (1002):
+# POST /1 HTTP/1.1\r\n		 18 bytes
+# Content-Length: 4\r\n		 19 bytes
+# Host: foo.bar\r\n		 15 bytes
+# \r\n				  2 bytes
+# Total:			 54 bytes
+# Response:
+# HTTP/1.1 200 Ok\r\n		 17 bytes
+# Content-Length: 1000\r\n	 22 bytes
+# \r\n				  2 bytes
+# Total:			 41 bytes
+
+# Request (1004):
+# POST /2 HTTP/1.1\r\n		 18 bytes
+# Content-Length: 4\r\n		 19 bytes
+# Host: foo.bar\r\n		 15 bytes
+# \r\n				  2 bytes
+# Total:			 54 bytes
+# Reponse:
+# HTTP/1.1\r\n			 10 bytes
+# \r\n				  2 bytes
+# Total:			 12 bytes
+
+logexpect l1 -v v1 -g request {
+	expect 0 1001	Begin		"^req .* rxreq"
+	expect * =	End
+	expect 0 1002	Begin		"^bereq "
+	expect * =	BereqAcct	"^54 4 58 41 1000 1041$"
+	expect 0 =	End
+	expect 0 1003	Begin		"^req .* rxreq"
+	expect * =	End
+	expect 0 1004	Begin		"^bereq"
+	expect * =	BereqAcct	"^54 4 58 12 0 12$"
+	expect * =	End
+} -start
+
+# Request 1001
+client c1 {
+	txreq -req "POST" -url "/1" -body "asdf"
+	rxresp
+	expect resp.status == 200
+
+	txreq -req "POST" -url "/2" -body "asdf"
+	rxresp
+	expect resp.status == 503
+} -run
+
+logexpect l1 -wait
diff --git a/include/Makefile.am b/include/Makefile.am
index e996fc0..8291a50 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -3,6 +3,7 @@
 # API headers
 nobase_pkginclude_HEADERS = \
 	tbl/acct_fields_req.h \
+	tbl/acct_fields_bereq.h \
 	tbl/backend_poll.h \
 	tbl/ban_vars.h \
 	tbl/bo_flags.h \
diff --git a/include/tbl/acct_fields_bereq.h b/include/tbl/acct_fields_bereq.h
new file mode 100644
index 0000000..6454bad
--- /dev/null
+++ b/include/tbl/acct_fields_bereq.h
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2008 Verdens Gang AS
+ * Copyright (c) 2008-2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * These are the stats we keep track of per busyobj.
+ */
+
+ACCT(bereq_hdrbytes)
+ACCT(bereq_bodybytes)
+ACCT(beresp_hdrbytes)
+ACCT(beresp_bodybytes)
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 99894cc..af15d07 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -512,4 +512,18 @@ SLTM(PipeAcct, 0, "Pipe byte counts",
 	"\n"
 )
 
+SLTM(BereqAcct, 0, "Backend request accounting",
+	"Contains byte counters from backend request processing.\n\n"
+	"The format is::\n\n"
+	"\t%d %d %d %d %d %d\n"
+	"\t|  |  |  |  |  |\n"
+	"\t|  |  |  |  |  +- Total bytes received\n"
+	"\t|  |  |  |  +---- Body bytes received\n"
+	"\t|  |  |  +------- Header bytes received\n"
+	"\t|  |  +---------- Total bytes transmitted\n"
+	"\t|  +------------- Body bytes transmitted\n"
+	"\t+---------------- Header bytes transmitted\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 7de67d2 Backend fetch counters
Message-ID: <E1WUYVc-0004bR-Ef@project.varnish-software.com>


commit 7de67d2d327c8e255e1900f62ee784b7dfdc4e66
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 16:00:03 2014 +0200

    Backend fetch counters
    
    Add per-backend VSC counters for traffic going in/out of this backend.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bf28983..c8d8d89 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -806,8 +806,8 @@ void VBE_DiscardHealth(const struct director *vdi);
 
 struct vbc *VDI_GetFd(struct busyobj *);
 int VDI_Healthy(const struct director *);
-void VDI_CloseFd(struct vbc **vbp);
-void VDI_RecycleFd(struct vbc **vbp);
+void VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *);
+void VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *);
 void VDI_AddHostHeader(struct http *to, const struct vbc *vbc);
 void VBE_Poll(void);
 void VDI_Init(void);
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 3725e89..6622f4b 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -288,7 +288,7 @@ vbe_GetVbe(struct busyobj *bo, struct vdi_simple *vs)
 		VSL_Flush(bo->vsl, 0);
 
 		VTCP_close(&vc->fd);
-		VBE_DropRefConn(bp);
+		VBE_DropRefConn(bp, NULL);
 		vc->backend = NULL;
 		VBE_ReleaseConn(vc);
 	}
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index c339563..c9ce112 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -152,9 +152,9 @@ struct vbc {
 void VBE_ReleaseConn(struct vbc *vc);
 
 /* cache_backend_cfg.c */
-void VBE_DropRefConn(struct backend *);
+void VBE_DropRefConn(struct backend *, const struct acct_bereq *);
 void VBE_DropRefVcl(struct backend *);
-void VBE_DropRefLocked(struct backend *b);
+void VBE_DropRefLocked(struct backend *b, const struct acct_bereq *);
 unsigned VBE_Healthy(const struct backend *b, double *changed);
 
 /* cache_backend_poll.c */
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 5a43c2e..9adb616 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -97,7 +97,7 @@ VBE_Poll(void)
  */
 
 void
-VBE_DropRefLocked(struct backend *b)
+VBE_DropRefLocked(struct backend *b, const struct acct_bereq *acct)
 {
 	int i;
 	struct vbc *vbe, *vbe2;
@@ -105,6 +105,13 @@ VBE_DropRefLocked(struct backend *b)
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 	assert(b->refcount > 0);
 
+	if (acct != NULL) {
+#define ACCT(foo) \
+		b->vsc->foo += acct->foo;
+#include "tbl/acct_fields_bereq.h"
+#undef ACCT
+	}
+
 	i = --b->refcount;
 	Lck_Unlock(&b->mtx);
 	if (i > 0)
@@ -131,11 +138,11 @@ VBE_DropRefVcl(struct backend *b)
 
 	Lck_Lock(&b->mtx);
 	b->vsc->vcls--;
-	VBE_DropRefLocked(b);
+	VBE_DropRefLocked(b, NULL);
 }
 
 void
-VBE_DropRefConn(struct backend *b)
+VBE_DropRefConn(struct backend *b, const struct acct_bereq *acct)
 {
 
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
@@ -143,7 +150,7 @@ VBE_DropRefConn(struct backend *b)
 	Lck_Lock(&b->mtx);
 	assert(b->n_conn > 0);
 	b->n_conn--;
-	VBE_DropRefLocked(b);
+	VBE_DropRefLocked(b, acct);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index 94ef47d..7491d44 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -40,7 +40,7 @@
 /* Close a connection ------------------------------------------------*/
 
 void
-VDI_CloseFd(struct vbc **vbp)
+VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *acct)
 {
 	struct backend *bp;
 	struct vbc *vc;
@@ -64,7 +64,7 @@ VDI_CloseFd(struct vbc **vbp)
 	vc->vsl = NULL;
 
 	VTCP_close(&vc->fd);
-	VBE_DropRefConn(bp);
+	VBE_DropRefConn(bp, acct);
 	vc->backend = NULL;
 	VBE_ReleaseConn(vc);
 }
@@ -72,7 +72,7 @@ VDI_CloseFd(struct vbc **vbp)
 /* Recycle a connection ----------------------------------------------*/
 
 void
-VDI_RecycleFd(struct vbc **vbp)
+VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *acct)
 {
 	struct backend *bp;
 	struct vbc *vc;
@@ -95,7 +95,7 @@ VDI_RecycleFd(struct vbc **vbp)
 	Lck_Lock(&bp->mtx);
 	VSC_C_main->backend_recycle++;
 	VTAILQ_INSERT_HEAD(&bp->connlist, vc, list);
-	VBE_DropRefLocked(bp);
+	VBE_DropRefLocked(bp, acct);
 }
 
 /* Get a connection --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cd627cd..775d66c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -319,7 +319,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->htc.body_status == BS_ERROR) {
 		AN (bo->vbc);
-		VDI_CloseFd(&bo->vbc);
+		VDI_CloseFd(&bo->vbc, &bo->acct);
 		VSLb(bo->vsl, SLT_VCL_Error, "Body cannot be fetched");
 		return (F_STP_ERROR);
 	}
@@ -351,7 +351,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (wrk->handling == VCL_RET_RETRY) {
 		AN (bo->vbc);
-		VDI_CloseFd(&bo->vbc);
+		VDI_CloseFd(&bo->vbc, &bo->acct);
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries)
 			return (F_STP_RETRY);
@@ -471,7 +471,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo, "Could not get storage");
-		VDI_CloseFd(&bo->vbc);
+		VDI_CloseFd(&bo->vbc, &bo->acct);
 		return (F_STP_ERROR);
 	}
 
@@ -761,9 +761,9 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	if (bo->vbc != NULL) {
 		if (bo->should_close)
-			VDI_CloseFd(&bo->vbc);
+			VDI_CloseFd(&bo->vbc, &bo->acct);
 		else
-			VDI_RecycleFd(&bo->vbc);
+			VDI_RecycleFd(&bo->vbc, &bo->acct);
 		AZ(bo->vbc);
 	}
 
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index b6fac38..4ddacb1 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -350,7 +350,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
-		VDI_CloseFd(&bo->vbc);
+		VDI_CloseFd(&bo->vbc, &bo->acct);
 		/* XXX: other cleanup ? */
 		return (retry);
 	}
@@ -376,7 +376,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 			VSLb(bo->vsl, SLT_FetchError,
 			    "http %sread error: overflow",
 			    first ? "first " : "");
-			VDI_CloseFd(&bo->vbc);
+			VDI_CloseFd(&bo->vbc, &bo->acct);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
@@ -384,7 +384,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 			bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf);
 			VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
 			    first ? "first " : "");
-			VDI_CloseFd(&bo->vbc);
+			VDI_CloseFd(&bo->vbc, &bo->acct);
 			/* XXX: other cleanup ? */
 			return (retry);
 		}
@@ -401,7 +401,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 
 	if (HTTP1_DissectResponse(hp, htc)) {
 		VSLb(bo->vsl, SLT_FetchError, "http format error");
-		VDI_CloseFd(&bo->vbc);
+		VDI_CloseFd(&bo->vbc, &bo->acct);
 		/* XXX: other cleanup ? */
 		return (-1);
 	}
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 05b8d0a..02d6c21 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -70,7 +70,7 @@ rdf(int fd0, int fd1, ssize_t *pcnt)
 }
 
 static void
-pipecharge(struct req *req, const struct acct_pipe *a)
+pipecharge(struct req *req, const struct acct_pipe *a, struct VSC_C_vbe *b)
 {
 
 	VSLb(req->vsl, SLT_PipeAcct, "%ju %ju %ju %ju",
@@ -83,6 +83,11 @@ pipecharge(struct req *req, const struct acct_pipe *a)
 	VSC_C_main->s_pipe_hdrbytes += a->req;
 	VSC_C_main->s_pipe_in += a->in;
 	VSC_C_main->s_pipe_out += a->out;
+	if (b != NULL) {
+		b->pipe_hdrbytes += a->bereq;
+		b->pipe_out += a->in;
+		b->pipe_in += a->out;
+	}
 	Lck_Unlock(&pipestat_mtx);
 }
 
@@ -110,7 +115,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 
 	vc = VDI_GetFd(bo);
 	if (vc == NULL) {
-		pipecharge(req, &acct);
+		pipecharge(req, &acct, NULL);
 		SES_Close(req->sp, SC_OVERLOAD);
 		return;
 	}
@@ -133,9 +138,9 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
 
 	if (i) {
-		pipecharge(req, &acct);
+		pipecharge(req, &acct, vc->backend->vsc);
 		SES_Close(req->sp, SC_TX_PIPE);
-		VDI_CloseFd(&vc);
+		VDI_CloseFd(&vc, NULL);
 		return;
 	}
 
@@ -173,9 +178,9 @@ PipeRequest(struct req *req, struct busyobj *bo)
 		}
 	}
 	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
-	pipecharge(req, &acct);
+	pipecharge(req, &acct, vc->backend->vsc);
 	SES_Close(req->sp, SC_TX_PIPE);
-	VDI_CloseFd(&vc);
+	VDI_CloseFd(&vc, NULL);
 	bo->vbc = NULL;
 }
 
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index c05d428..64e63a3 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -184,6 +184,36 @@ VSC_F(happy,			uint64_t, 0, 'b', info,
     "Happy health probes",
 	""
 )
+VSC_F(bereq_hdrbytes,		uint64_t, 0, 'a', info,
+    "Request header bytes",
+	"Total backend request header bytes sent"
+)
+VSC_F(bereq_bodybytes,		uint64_t, 0, 'a', info,
+    "Request body bytes",
+	"Total backend request body bytes sent"
+)
+VSC_F(beresp_hdrbytes,		uint64_t, 0, 'a', info,
+    "Response header bytes",
+	"Total backend response header bytes received"
+)
+VSC_F(beresp_bodybytes,		uint64_t, 0, 'a', info,
+    "Response body bytes",
+	"Total backend response body bytes received"
+)
+VSC_F(pipe_hdrbytes,		uint64_t, 0, 'a', info,
+    "Pipe request header bytes",
+	"Total request bytes sent for piped sessions"
+)
+VSC_F(pipe_out,			uint64_t, 0, 'a', info,
+    "Piped bytes to backend",
+	"Total number of bytes forwarded to backend in"
+	" pipe sessions"
+)
+VSC_F(pipe_in,			uint64_t, 0, 'a', info,
+    "Piped bytes from backend",
+	"Total number of bytes forwarded from backend in"
+	" pipe sessions"
+)
 
 #endif
 



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 621ac54 Use SLT_ReqAcct for length in varnishncsa
Message-ID: <E1WUYVc-0004bb-Fb@project.varnish-software.com>


commit 621ac54f87220c8c7413f13fe81ada8f71ce2c25
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 20:57:49 2014 +0200

    Use SLT_ReqAcct for length in varnishncsa

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index a38264b..7a47b1b 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -774,8 +774,8 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			case SLT_RespStatus:
 				frag_line(b, e, &CTX.frag[F_s]);
 				break;
-			case SLT_Length:
-				frag_line(b, e, &CTX.frag[F_b]);
+			case SLT_ReqAcct:
+				frag_fields(b, e, 5, &CTX.frag[F_b], 0, NULL);
 				break;
 			case SLT_Timestamp:
 				if (isprefix(b, "Start:", e, &p)) {



From martin at varnish-software.com  Mon Mar 31 09:28:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 11:28:00 +0200
Subject: [master] 2e75328 Use SLT_ReqAcct for length in varnishhist
Message-ID: <E1WUYVc-0004bk-GX@project.varnish-software.com>


commit 2e7532887c6304608d671095b3acc474751a7ab3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Mar 30 20:59:26 2014 +0200

    Use SLT_ReqAcct for length in varnishhist

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 91078d8..e8eb1ff 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -120,9 +120,9 @@ profiles[] = {
 		.hist_high = 3
 	}, {
 		.name = "size",
-		.tag = SLT_Length,
+		.tag = SLT_ReqAcct,
 		.prefix = NULL,
-		.field = 1,
+		.field = 5,
 		.hist_low = 1,
 		.hist_high = 8
 	}, {



From martin at varnish-software.com  Mon Mar 31 10:02:00 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 12:02:00 +0200
Subject: [master] d6d97bb Disable streaming on byte counter test cases
Message-ID: <E1WUZ2W-0007z3-E6@project.varnish-software.com>


commit d6d97bb34290da61ba350d44309adedc07c06824
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 31 12:00:47 2014 +0200

    Disable streaming on byte counter test cases
    
    This makes sure no extra 'transfer-encoding' headers should pop up
    changing the byte counts.

diff --git a/bin/varnishtest/tests/l00002.vtc b/bin/varnishtest/tests/l00002.vtc
index 0106324..8c7bc35 100644
--- a/bin/varnishtest/tests/l00002.vtc
+++ b/bin/varnishtest/tests/l00002.vtc
@@ -11,6 +11,9 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
 	sub vcl_deliver {
 		unset resp.http.date;
 		unset resp.http.age;
diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc
index 51d0d7e..dbbea1b 100644
--- a/bin/varnishtest/tests/l00005.vtc
+++ b/bin/varnishtest/tests/l00005.vtc
@@ -16,6 +16,9 @@ varnish v1 -vcl+backend {
 		unset bereq.http.x-varnish;
 		set bereq.http.Host = "foo.bar";
 	}
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
 } -start
 
 # Request (1002):



From martin at varnish-software.com  Mon Mar 31 10:37:16 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 12:37:16 +0200
Subject: [master] 6d468fe Don't use acct as variable name
Message-ID: <E1WUZae-00038b-Bm@project.varnish-software.com>


commit 6d468fe20062add9164d6070652385a5d4ac0557
Author: Martin Blix Grydeland <martin at friday>
Date:   Mon Mar 31 11:41:01 2014 +0200

    Don't use acct as variable name
    
    This triggers 'shadows global' build errors.

diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 02d6c21..2e83655 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -98,7 +98,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	struct worker *wrk;
 	struct pollfd fds[2];
 	int i;
-	struct acct_pipe acct;
+	struct acct_pipe acct_pipe;
 	ssize_t hdrbytes;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -109,13 +109,13 @@ PipeRequest(struct req *req, struct busyobj *bo)
 
 	req->res_mode = RES_PIPE;
 
-	memset(&acct, 0, sizeof acct);
-	acct.req = req->acct.req_hdrbytes;
+	memset(&acct_pipe, 0, sizeof acct_pipe);
+	acct_pipe.req = req->acct.req_hdrbytes;
 	req->acct.req_hdrbytes = 0;
 
 	vc = VDI_GetFd(bo);
 	if (vc == NULL) {
-		pipecharge(req, &acct, NULL);
+		pipecharge(req, &acct_pipe, NULL);
 		SES_Close(req->sp, SC_OVERLOAD);
 		return;
 	}
@@ -129,16 +129,16 @@ PipeRequest(struct req *req, struct busyobj *bo)
 		(void)WRW_Write(wrk, req->htc->pipeline.b,
 		    Tlen(req->htc->pipeline));
 
-	i = WRW_FlushRelease(wrk, &acct.bereq);
-	if (acct.bereq > hdrbytes) {
-		acct.in = acct.bereq - hdrbytes;
-		acct.bereq = hdrbytes;
+	i = WRW_FlushRelease(wrk, &acct_pipe.bereq);
+	if (acct_pipe.bereq > hdrbytes) {
+		acct_pipe.in = acct_pipe.bereq - hdrbytes;
+		acct_pipe.bereq = hdrbytes;
 	}
 
 	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
 
 	if (i) {
-		pipecharge(req, &acct, vc->backend->vsc);
+		pipecharge(req, &acct_pipe, vc->backend->vsc);
 		SES_Close(req->sp, SC_TX_PIPE);
 		VDI_CloseFd(&vc, NULL);
 		return;
@@ -160,7 +160,8 @@ PipeRequest(struct req *req, struct busyobj *bo)
 		i = poll(fds, 2, (int)(cache_param->pipe_timeout * 1e3));
 		if (i < 1)
 			break;
-		if (fds[0].revents && rdf(vc->fd, req->sp->fd, &acct.out)) {
+		if (fds[0].revents &&
+		    rdf(vc->fd, req->sp->fd, &acct_pipe.out)) {
 			if (fds[1].fd == -1)
 				break;
 			(void)shutdown(vc->fd, SHUT_RD);
@@ -168,7 +169,8 @@ PipeRequest(struct req *req, struct busyobj *bo)
 			fds[0].events = 0;
 			fds[0].fd = -1;
 		}
-		if (fds[1].revents && rdf(req->sp->fd, vc->fd, &acct.in)) {
+		if (fds[1].revents &&
+		    rdf(req->sp->fd, vc->fd, &acct_pipe.in)) {
 			if (fds[0].fd == -1)
 				break;
 			(void)shutdown(req->sp->fd, SHUT_RD);
@@ -178,7 +180,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 		}
 	}
 	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
-	pipecharge(req, &acct, vc->backend->vsc);
+	pipecharge(req, &acct_pipe, vc->backend->vsc);
 	SES_Close(req->sp, SC_TX_PIPE);
 	VDI_CloseFd(&vc, NULL);
 	bo->vbc = NULL;



From phk at FreeBSD.org  Mon Mar 31 10:40:30 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 31 Mar 2014 12:40:30 +0200
Subject: [master] e471a84 Silence Flexelint
Message-ID: <E1WUZdm-0003ba-V4@project.varnish-software.com>


commit e471a8469d02d53a8e7679b11961f179cfb70196
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 31 10:39:58 2014 +0000

    Silence Flexelint

diff --git a/include/tbl/acct_fields_bereq.h b/include/tbl/acct_fields_bereq.h
index 6454bad..7b28879 100644
--- a/include/tbl/acct_fields_bereq.h
+++ b/include/tbl/acct_fields_bereq.h
@@ -29,7 +29,9 @@
  * These are the stats we keep track of per busyobj.
  */
 
+/*lint -save -e525 -e539 */
 ACCT(bereq_hdrbytes)
 ACCT(bereq_bodybytes)
 ACCT(beresp_hdrbytes)
 ACCT(beresp_bodybytes)
+/*lint -restore */



From apj at mutt.dk  Mon Mar 31 10:41:30 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Mon, 31 Mar 2014 12:41:30 +0200
Subject: [master] d861c18 This file was named wrong,
 and the test does not make sense anymore,
 since the backend health information is in the fetch thread
Message-ID: <E1WUZek-0003mO-CJ@project.varnish-software.com>


commit d861c18fc97d74068e58337be86b306e0920b075
Author: Andreas Plesner <apj at mutt.dk>
Date:   Mon Mar 31 12:39:39 2014 +0200

    This file was named wrong, and the test does not make sense anymore, since the backend health information is in the fetch thread

diff --git a/bin/varnishtest/tests/r01228.vcl b/bin/varnishtest/tests/r01228.vcl
deleted file mode 100644
index 1cb7e01..0000000
--- a/bin/varnishtest/tests/r01228.vcl
+++ /dev/null
@@ -1,21 +0,0 @@
-varnishtest "req.backend.healthy in vcl_deliver"
-
-server s1 {
-	rxreq
-	txresp
-} -start
-
-varnish v1 -vcl+backend {
-
-	import ${vmod_std};
-
-	sub vcl_deliver {
-	    set resp.http.x-foo = std.healthy(req.backend);
-	}
-} -start
-
-client c1 {
-	txreq
-	rxresp
-	expect resp.status == 200
-} -run



From lkarsten at varnish-software.com  Mon Mar 31 11:28:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 31 Mar 2014 13:28:47 +0200
Subject: [master] d1e3ff9 Prepare for 4.0.0 beta1 release.
Message-ID: <E1WUaOV-0008Dj-OA@project.varnish-software.com>


commit d1e3ff9ab604a22bdc0aec23ebefcae7d527a8a6
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Mar 27 12:10:59 2014 +0100

    Prepare for 4.0.0 beta1 release.

diff --git a/configure.ac b/configure.ac
index 6612215..18e4ed0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [4.0.0-tp2], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [4.0.0-beta1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index 25d655a..28d64da 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,3 +1,63 @@
+
+=====================================
+Changes from 4.0.0 TP2 to 4.0.0 beta1
+=====================================
+
+New since TP2:
+
+- Previous always-appended code called default.vcl is now called builtin.vcl.
+  The new example.vcl is recommended as a starting point for new users.
+- vcl_error is now called vcl_synth, and does not any more mandate closing the
+  client connection.
+- New VCL function vcl_backend_error, where you can change the 503 prepared if
+  all your backends are failing. This can then be cached as a regular object.
+- Keyword "remove" in VCL is replaced by "unset".
+- new timestamp and accounting records in varnishlog.
+- std.timestamp() is introduced.
+- stored objects are now read only, meaning obj.hits now counts per objecthead
+  instead. obj.lastuse saw little use and has been removed.
+- builtin VCL now does return(pipe) for chunked POST and PUT requests.
+- python-docutils and rst2man are now build requirements.
+- cli_timeout is now 60 seconds to avoid slaughtering the child process in
+  times of high IO load/scheduling latency.
+- return(purge) from vcl_recv is now valid.
+- return(hash) is now the default return action from vcl_recv.
+- req.backend is now req.backend_hint. beresp.storage is beresp.storage_hint.
+
+
+Bugs fixed
+----------
+
+* 1460_ - tools now use the new timestamp format.
+* 1450_ - varnishstat -l segmentation fault.
+* 1320_ - Work around Content-Length: 0 and Content-Encoding: gzip gracefully.
+* 1458_ - Panic on busy object.
+* 1417_ - Handle return(abandon) in vcl_backend_response.
+* 1455_ - vcl_pipe now sets Connection: close by default on backend requests.
+* 1454_ - X-Forwarded-For is now done in C, before vcl_recv is run.
+* 1436_ - Better explanation when missing an import in VCL.
+* 1440_ - Serve ESI-includes from a different backend.
+* 1441_ - Incorrect grouping when logging ESI subrequests.
+* 1434_ - std.duration can now do ms/milliseconds.
+* 1419_ - Don't put objcores on the ban list until they go non-BUSY.
+* 1405_ - Ban lurker does not always evict all objects.
+
+.. _1460: https://www.varnish-cache.org/trac/ticket/1460
+.. _1450: https://www.varnish-cache.org/trac/ticket/1450
+.. _1320: https://www.varnish-cache.org/trac/ticket/1320
+.. _1458: https://www.varnish-cache.org/trac/ticket/1458
+.. _1417: https://www.varnish-cache.org/trac/ticket/1417
+.. _1455: https://www.varnish-cache.org/trac/ticket/1455
+.. _1454: https://www.varnish-cache.org/trac/ticket/1454
+.. _1436: https://www.varnish-cache.org/trac/ticket/1436
+.. _1440: https://www.varnish-cache.org/trac/ticket/1440
+.. _1441: https://www.varnish-cache.org/trac/ticket/1441
+.. _1434: https://www.varnish-cache.org/trac/ticket/1434
+.. _1419: https://www.varnish-cache.org/trac/ticket/1419
+.. _1405: https://www.varnish-cache.org/trac/ticket/1405
+
+
+
 ===================================
 Changes from 4.0.0 TP1 to 4.0.0 TP2
 ===================================
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 9adcaa2..c1b5f8d 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,11 +1,11 @@
-%define v_rc tp2
+%define v_rc beta1
 %define vd_rc %{?v_rc:-%{?v_rc}}
 %define    _use_internal_dependency_generator 0
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
 Summary: High-performance HTTP accelerator
 Name: varnish
 Version: 4.0.0
-Release: 0.20131129%{?v_rc}%{?dist}
+Release: 0.20140328%{?v_rc}%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/



From martin at varnish-software.com  Mon Mar 31 12:58:05 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 31 Mar 2014 14:58:05 +0200
Subject: [master] 20319dd Fix missing clearing of oc->busyobj on HSH_Fail.
Message-ID: <E1WUbmv-00038B-VE@project.varnish-software.com>


commit 20319dd6e73e2979948468f49ed11cb7a00a6156
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 31 14:51:19 2014 +0200

    Fix missing clearing of oc->busyobj on HSH_Fail.
    
    The oc's busyobj isn't cleared in HSH_Fail (like it is in HSH_Complete).
    This allows a thread to get a pointer to an already free'd busyobj.
    
    Fixes: #1467

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 775d66c..54a042d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -790,6 +790,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 				assert(uu == bo->fetch_obj->len);
 		}
 	}
+	AZ(bo->fetch_objcore->busyobj);
 
 	if (bo->ims_obj != NULL)
 		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 1530937..f4bbae0 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -642,6 +642,7 @@ HSH_Fail(struct objcore *oc)
 
 	Lck_Lock(&oh->mtx);
 	oc->flags |= OC_F_FAILED;
+	oc->busyobj = NULL;
 	Lck_Unlock(&oh->mtx);
 }