From phk at varnish-cache.org  Mon Sep  3 07:39:28 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 03 Sep 2012 09:39:28 +0200
Subject: [master] 1f836b1 Polish the dot-graph
Message-ID: <E1T8RFo-0006qm-Hs@project.varnish-software.com>

commit 1f836b1fa6c8ba51a1b788addfca210a0d817c72
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 07:39:15 2012 +0000

    Polish the dot-graph

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index e12da55..9ff1308 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -26,11 +26,8 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * This file contains the two central state machine for pushing HTTP
- * requests through their paces.
- *
- * The second part of the file, entrypoint CNT_Request() and below the
- * ==== separator, is intended to (over time) be(ome) protocol agnostic.
+ * This file contains the request-handling state engine, which is intended to
+ * (over time) be(ome) protocol agnostic.
  * We already use this now with ESI:includes, which are for all relevant
  * purposes a different "protocol"
  *
@@ -54,9 +51,11 @@ DOT acceptor [
 DOT	shape=hexagon
 DOT	label="Request received"
 DOT ]
+DOT ESI_REQ [ shape=hexagon ]
+DOT ESI_REQ -> recv
 DOT ERROR [shape=plaintext]
 DOT RESTART [shape=plaintext]
-DOT acceptor -> start [style=bold,color=green]
+DOT acceptor -> recv [style=bold,color=green]
  */
 
 #include "config.h"
@@ -1039,6 +1038,8 @@ DOT	]
 DOT }
 DOT RESTART -> restart [color=purple]
 DOT restart -> recv [color=purple]
+DOT restart -> err_restart
+DOT err_restart [label="ERROR",shape=plaintext]
  */
 
 static int
@@ -1077,8 +1078,6 @@ DOT		shape=record
 DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
-DOT ESI_REQ [ shape=hexagon ]
-DOT ESI_REQ -> recv
 DOT recv:pipe -> pipe [style=bold,color=orange]
 DOT recv:pass -> pass [style=bold,color=red]
 #DOT recv:error -> err_recv



From phk at varnish-cache.org  Mon Sep  3 08:39:12 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 03 Sep 2012 10:39:12 +0200
Subject: [master] adbff32 Fix an off by quite a lot error which neutered most
 of the good predictive vary could do.
Message-ID: <E1T8SBc-0003jE-CE@project.varnish-software.com>

commit adbff32298bb8992e62eeebe8dfaa984c061294d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 08:38:52 2012 +0000

    Fix an off by quite a lot error which neutered most of the good
    predictive vary could do.

diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index bf6537f..5f508d2 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -261,7 +261,7 @@ VRY_Match(struct req *req, const uint8_t *vary)
 			vsp[ln + 1] = 0xff;
 			vsp[ln + 2] = 0;
 			VRY_Validate(vsp);
-			req->vary_l = vsp + 3;
+			req->vary_l = vsp + ln + 3;
 
 			i = vry_cmp(vary, vsp);
 			assert(i == 0 || i == 2);



From phk at varnish-cache.org  Mon Sep  3 09:03:52 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 03 Sep 2012 11:03:52 +0200
Subject: [master] 5922eb7 Add a WS_Copy() function and use it a couple of
 places.
Message-ID: <E1T8SZU-00076Q-Ap@project.varnish-software.com>

commit 5922eb7519b734b45c1d2086b415ab432d53bfce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 08:57:00 2012 +0000

    Add a WS_Copy() function and use it a couple of places.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 94db4b6..c92a2d0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1034,6 +1034,7 @@ void WS_ReleaseP(struct ws *ws, char *ptr);
 void WS_Assert(const struct ws *ws);
 void WS_Reset(struct ws *ws, char *p);
 char *WS_Alloc(struct ws *ws, unsigned bytes);
+char *WS_Copy(struct ws *ws, const char *str, int len);
 char *WS_Snapshot(struct ws *ws);
 
 /* rfc2616.c */
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ad29899..73fb7fc 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -888,10 +888,9 @@ http_CopyHome(const struct http *hp)
 			continue;
 		}
 		l = Tlen(hp->hd[u]);
-		p = WS_Alloc(hp->ws, l + 1);
+		p = WS_Copy(hp->ws, hp->hd[u].b, l + 1L);
 		if (p != NULL) {
 			http_VSLH(hp, u);
-			memcpy(p, hp->hd[u].b, l + 1L);
 			hp->hd[u].b = p;
 			hp->hd[u].e = p + l;
 		} else {
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9ff1308..18ca2e3 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -617,9 +617,9 @@ cnt_fetchbody(struct worker *wrk, struct req *req)
 		req->obj->gziped = 1;
 
 	if (vary != NULL) {
-		req->obj->vary = (void *)WS_Alloc(req->obj->http->ws, varyl);
+		req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
+		    VSB_data(vary), varyl);
 		AN(req->obj->vary);
-		memcpy(req->obj->vary, VSB_data(vary), varyl);
 		VRY_Validate(req->obj->vary);
 		VSB_delete(vary);
 	}
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index a5251ff..6c08422 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -117,6 +117,33 @@ WS_Alloc(struct ws *ws, unsigned bytes)
 }
 
 char *
+WS_Copy(struct ws *ws, const char *str, int len)
+{
+	char *r;
+	unsigned bytes;
+
+	WS_Assert(ws);
+	assert(ws->r == NULL);
+
+	if (len == -1)
+		len = strlen(str) + 1;
+	assert(len >= 0);
+
+	bytes = PRNDUP((unsigned)len);
+	if (ws->f + bytes > ws->e) {
+		ws->overflow++;
+		WS_Assert(ws);
+		return(NULL);
+	}
+	r = ws->f;
+	ws->f += bytes;
+	memcpy(r, str, len);
+	DSL(DBG_WORKSPACE, 0, "WS_Copy(%p, %d) = %p", ws, len, r);
+	WS_Assert(ws);
+	return (r);
+}
+
+char *
 WS_Snapshot(struct ws *ws)
 {
 
@@ -174,15 +201,3 @@ WS_ReleaseP(struct ws *ws, char *ptr)
 	ws->r = NULL;
 	WS_Assert(ws);
 }
-
-#if 0
-/* XXX: not used anywhere (yet) */
-void
-WS_Return(struct ws *ws, char *s, char *e)
-{
-
-	WS_Assert(ws);
-	if (e == ws->f)
-		ws->f = s;
-}
-#endif



From phk at varnish-cache.org  Mon Sep  3 09:03:52 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 03 Sep 2012 11:03:52 +0200
Subject: [master] ed47f83 Move the busyobj allocation from cache_hash to
 cache_req_fsm Move the predictive vary from the req->ws to busyobj->ws.
Message-ID: <E1T8SZU-00076V-Ck@project.varnish-software.com>

commit ed47f83d7eec2db21abbf59f8314be6c47ff3ea6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 09:03:14 2012 +0000

    Move the busyobj allocation from cache_hash to cache_req_fsm
    Move the predictive vary from the req->ws to busyobj->ws.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 932cb67..d0d987e 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -60,7 +60,6 @@
 
 
 #include "hash/hash_slinger.h"
-#include "vmb.h"
 #include "vsha256.h"
 
 static const struct hash_slinger *hash;
@@ -445,19 +444,6 @@ HSH_Lookup(struct req *req)
 	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
 	/* NB: do not deref objhead the new object inherits our reference */
 	Lck_Unlock(&oh->mtx);
-
-	AZ(req->busyobj);
-	req->busyobj = VBO_GetBusyObj(wrk);
-	req->busyobj->refcount = 2;	/* One for req, one for FetchBody */
-
-	VRY_Validate(req->vary_b);
-	if (req->vary_l != NULL)
-		req->busyobj->vary = req->vary_b;
-	else
-		req->busyobj->vary = NULL;
-
-	VMB();
-	oc->busyobj = req->busyobj;
 	return (oc);
 }
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 18ca2e3..045dc6b 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -776,6 +776,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	struct objcore *oc;
 	struct object *o;
 	struct objhead *oh;
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -792,8 +793,8 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		/*
 		 * We lost the session to a busy object, disembark the
 		 * worker thread.   We return to STP_LOOKUP when the busy
-		 * object has been unbusied, and still have the hash digest
-		 * around to do the lookup with.
+		 * object has been unbusied, and still have the objhead
+		 * around to restart the lookup with.
 		 */
 		return (2);
 	}
@@ -805,22 +806,28 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	/* If we inserted a new object it's a miss */
 	if (oc->flags & OC_F_BUSY) {
-		CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
-		assert(oc->busyobj == req->busyobj);
-		wrk->stats.cache_miss++;
-
+		AZ(req->busyobj);
+		bo = VBO_GetBusyObj(wrk);
+		req->busyobj = bo;
+		/* One ref for req, one for FetchBody */
+		bo->refcount = 2;
+		VRY_Validate(req->vary_b);
 		if (req->vary_l != NULL) {
-			assert(oc->busyobj->vary == req->vary_b);
-			VRY_Validate(oc->busyobj->vary);
-			WS_ReleaseP(req->ws, (void*)req->vary_l);
-		} else {
-			AZ(oc->busyobj->vary);
-			WS_Release(req->ws, 0);
-		}
+			bo->vary = (void*)WS_Copy(bo->ws,
+			    (void*)req->vary_b, req->vary_l - req->vary_b);
+			AN(bo->vary);
+			VRY_Validate(bo->vary);
+		} else
+			bo->vary = NULL;
+
+		WS_Release(req->ws, 0);
 		req->vary_b = NULL;
 		req->vary_l = NULL;
 		req->vary_e = NULL;
 
+		oc->busyobj = bo;
+		wrk->stats.cache_miss++;
+
 		req->objcore = oc;
 		req->req_step = R_STP_MISS;
 		return (0);



From phk at varnish-cache.org  Mon Sep  3 09:43:17 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 03 Sep 2012 11:43:17 +0200
Subject: [master] c4b7322 Properly book-end the predictive vary code.
Message-ID: <E1T8TBd-0001nb-SB@project.varnish-software.com>

commit c4b7322c92c95b662f9853d4a31a389a790dd743
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 09:42:54 2012 +0000

    Properly book-end the predictive vary code.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c92a2d0..15db84c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -987,6 +987,7 @@ struct vsb *VRY_Create(struct req *sp, const struct http *hp);
 int VRY_Match(struct req *, const uint8_t *vary);
 void VRY_Validate(const uint8_t *vary);
 void VRY_Prep(struct req *);
+void VRY_Finish(struct req *req, struct busyobj *bo);
 
 /* cache_vcl.c */
 void VCL_Init(void);
@@ -1034,7 +1035,7 @@ void WS_ReleaseP(struct ws *ws, char *ptr);
 void WS_Assert(const struct ws *ws);
 void WS_Reset(struct ws *ws, char *p);
 char *WS_Alloc(struct ws *ws, unsigned bytes);
-char *WS_Copy(struct ws *ws, const char *str, int len);
+void *WS_Copy(struct ws *ws, const void *str, int len);
 char *WS_Snapshot(struct ws *ws);
 
 /* rfc2616.c */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 045dc6b..9f996c3 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -811,19 +811,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		req->busyobj = bo;
 		/* One ref for req, one for FetchBody */
 		bo->refcount = 2;
-		VRY_Validate(req->vary_b);
-		if (req->vary_l != NULL) {
-			bo->vary = (void*)WS_Copy(bo->ws,
-			    (void*)req->vary_b, req->vary_l - req->vary_b);
-			AN(bo->vary);
-			VRY_Validate(bo->vary);
-		} else
-			bo->vary = NULL;
-
-		WS_Release(req->ws, 0);
-		req->vary_b = NULL;
-		req->vary_l = NULL;
-		req->vary_e = NULL;
+		VRY_Finish(req, bo);
 
 		oc->busyobj = bo;
 		wrk->stats.cache_miss++;
@@ -840,10 +828,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
 
-	WS_Release(req->ws, 0);
-	req->vary_b = NULL;
-	req->vary_l = NULL;
-	req->vary_e = NULL;
+	VRY_Finish(req, NULL);
 
 	if (oc->flags & OC_F_PASS) {
 		wrk->stats.cache_hitpass++;
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 5f508d2..d4319b2 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -199,6 +199,31 @@ VRY_Prep(struct req *req)
 }
 
 /**********************************************************************
+ * Finish predictive vary procssing
+ */
+
+void
+VRY_Finish(struct req *req, struct busyobj *bo)
+{
+
+	if (bo != NULL) {
+		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+		VRY_Validate(req->vary_b);
+		if (req->vary_l != NULL) {
+			bo->vary = WS_Copy(bo->ws,
+			    req->vary_b, req->vary_l - req->vary_b);
+			AN(bo->vary);
+			VRY_Validate(bo->vary);
+		} else
+			bo->vary = NULL;
+	}
+	WS_Release(req->ws, 0);
+	req->vary_b = NULL;
+	req->vary_l = NULL;
+	req->vary_e = NULL;
+}
+
+/**********************************************************************
  * Match vary strings, and build a new cached string if possible.
  *
  * Return zero if there is certainly no match.
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 6c08422..31738e3 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -116,8 +116,8 @@ WS_Alloc(struct ws *ws, unsigned bytes)
 	return (r);
 }
 
-char *
-WS_Copy(struct ws *ws, const char *str, int len)
+void *
+WS_Copy(struct ws *ws, const void *str, int len)
 {
 	char *r;
 	unsigned bytes;



From perbu at varnish-cache.org  Mon Sep  3 19:41:42 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 03 Sep 2012 21:41:42 +0200
Subject: [master] ac082ae add tutorial
Message-ID: <E1T8cWk-0000p2-3M@project.varnish-software.com>

commit ac082ae952be44ec7ba56ab755a08639c3888a92
Author: Per Buer <per.buer at gmail.com>
Date:   Mon Sep 3 21:41:36 2012 +0200

    add tutorial

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
new file mode 100644
index 0000000..311a322
--- /dev/null
+++ b/doc/sphinx/tutorial/index.rst
@@ -0,0 +1,8 @@
+.. _tutorial-index:
+
+%%%%%%%%%%%%%%%%%%%%
+The Varnish Tutorial
+%%%%%%%%%%%%%%%%%%%%
+
+What is a tutorial, anyway?
+



From perbu at varnish-cache.org  Mon Sep  3 19:41:42 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 03 Sep 2012 21:41:42 +0200
Subject: [master] c1799a7 rename tutorial -> users guide
Message-ID: <E1T8cWk-0000ov-1i@project.varnish-software.com>

commit c1799a76c6bad27c3a2e5b9fe9338c86faf7cdb5
Author: Per Buer <per.buer at gmail.com>
Date:   Mon Sep 3 21:40:56 2012 +0200

    rename tutorial -> users guide

diff --git a/doc/sphinx/tutorial/advanced_backend_servers.rst b/doc/sphinx/tutorial/advanced_backend_servers.rst
deleted file mode 100644
index b4206d9..0000000
--- a/doc/sphinx/tutorial/advanced_backend_servers.rst
+++ /dev/null
@@ -1,157 +0,0 @@
-Advanced Backend configuration
-------------------------------
-
-At some point you might need Varnish to cache content from several
-servers. You might want Varnish to map all the URL into one single
-host or not. There are lot of options.
-
-Lets say we need to introduce a Java application into out PHP web
-site. Lets say our Java application should handle URL beginning with
-/java/.
-
-We manage to get the thing up and running on port 8000. Now, lets have
-a look a default.vcl.::
-
-  backend default {
-      .host = "127.0.0.1";
-      .port = "8080";
-  }
-
-We add a new backend.::
-
-  backend java {
-      .host = "127.0.0.1";
-      .port = "8000";
-  }
-
-Now we need tell where to send the difference URL. Lets look at vcl_recv.::
-
-  sub vcl_recv {
-      if (req.url ~ "^/java/") {
-          set req.backend = java;
-      } else {
-          set req.backend = default.
-      }
-  }
-
-It's quite simple, really. Lets stop and think about this for a
-moment. As you can see you can define how you choose backends based on
-really arbitrary data. You want to send mobile devices to a different
-backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
-trick. 
-
-.. _tutorial-advanced_backend_servers-directors:
-
-Directors
----------
-
-You can also group several backend into a group of backends. These
-groups are called directors. This will give you increased performance
-and resilience. You can define several backends and group them
-together in a director.::
-
-	 backend server1 {
-	     .host = "192.168.0.10";
-	 }
-	 backend server2{
-	     .host = "192.168.0.10";
-	 }
-
-Now we create the director.::
-
-       	director example_director round-robin {
-        {
-                .backend = server1;
-        }
-	# server2
-        {
-                .backend = server2;
-        }
-	# foo
-	}
-
-
-This director is a round-robin director. This means the director will
-distribute the incoming requests on a round-robin basis. There is
-also a *random* director which distributes requests in a, you guessed
-it, random fashion.
-
-But what if one of your servers goes down? Can Varnish direct all the
-requests to the healthy server? Sure it can. This is where the Health
-Checks come into play.
-
-.. _tutorial-advanced_backend_servers-health:
-
-Health checks
--------------
-
-Lets set up a director with two backends and health checks. First lets
-define the backends.::
-
-       backend server1 {
-         .host = "server1.example.com";
-	 .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	   }
-         }
-       backend server2 {
-  	  .host = "server2.example.com";
-  	  .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	  }
-        }
-
-Whats new here is the probe. Varnish will check the health of each
-backend with a probe. The options are
-
-url
- What URL should varnish request.
-
-interval
- How often should we poll
-
-timeout
- What is the timeout of the probe
-
-window
- Varnish will maintain a *sliding window* of the results. Here the
- window has five checks.
-
-threshold 
- How many of the .window last polls must be good for the backend to be declared healthy.
-
-initial 
- How many of the of the probes a good when Varnish starts - defaults
- to the same amount as the threshold.
-
-Now we define the director.::
-
-  director example_director round-robin {
-        {
-                .backend = server1;
-        }
-        # server2 
-        {
-                .backend = server2;
-        }
-	
-        }
-
-You use this director just as you would use any other director or
-backend. Varnish will not send traffic to hosts that are marked as
-unhealthy. Varnish can also serve stale content if all the backends are
-down. See :ref:`tutorial-handling_misbehaving_servers` for more
-information on how to enable this.
-
-Please note that Varnish will keep probes active for all loaded
-VCLs. Varnish will coalesce probes that seem identical - so be careful
-not to change the probe config if you do a lot of VCL
-loading. Unloading the VCL will discard the probes.
diff --git a/doc/sphinx/tutorial/advanced_topics.rst b/doc/sphinx/tutorial/advanced_topics.rst
deleted file mode 100644
index 1045de9..0000000
--- a/doc/sphinx/tutorial/advanced_topics.rst
+++ /dev/null
@@ -1,63 +0,0 @@
-.. _tutorial-advanced_topics:
-
-Advanced topics
----------------
-
-This tutorial has covered the basics in Varnish. If you read through
-it all you should now have the skills to run Varnish.
-
-Here is a short overview of topics that we haven't covered in the tutorial. 
-
-More VCL
-~~~~~~~~
-
-VCL is a bit more complex then what we've covered so far. There are a
-few more subroutines available and there a few actions that we haven't
-discussed. For a complete(ish) guide to VCL have a look at the VCL man
-page - ref:`reference-vcl`.
-
-Using In-line C to extend Varnish
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use *in-line C* to extend Varnish. Please note that you can
-seriously mess up Varnish this way. The C code runs within the Varnish
-Cache process so if your code generates a segfault the cache will crash.
-
-One of the first uses I saw of In-line C was logging to syslog.::
-
-	# The include statements must be outside the subroutines.
-	C{
-		#include <syslog.h>
-        }C
-	
-        sub vcl_something {
-                C{
-		        syslog(LOG_INFO, "Something happened at VCL line XX.");
-	        }C
-        }
-
-
-Edge Side Includes
-~~~~~~~~~~~~~~~~~~
-
-Varnish can cache create web pages by putting different pages
-together. These *fragments* can have individual cache policies. If you
-have a web site with a list showing the 5 most popular articles on
-your site, this list can probably be cached as a fragment and included
-in all the other pages. Used properly it can dramatically increase
-your hit rate and reduce the load on your servers. ESI looks like this::
-
-  <HTML>
-  <BODY>
-  The time is: <esi:include src="/cgi-bin/date.cgi"/>
-  at this very moment.
-  </BODY>
-  </HTML>
-
-ESI is processed in vcl_fetch by setting *do_esi* to true.::
-
-  sub vcl_fetch {
-      if (req.url == "/test.html") {
-	  set beresp.do_esi = true;  /* Do ESI processing */
-      }
-  }
diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
deleted file mode 100644
index 1b1aaf2..0000000
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ /dev/null
@@ -1,39 +0,0 @@
-.. _tutorial-backend_servers:
-
-Backend servers
----------------
-
-Varnish has a concept of "backend" or "origin" servers. A backend
-server is the server providing the content Varnish will accelerate.
-
-Our first task is to tell Varnish where it can find its content. Start
-your favorite text editor and open the varnish default configuration
-file. If you installed from source this is
-/usr/local/etc/varnish/default.vcl, if you installed from a package it
-is probably /etc/varnish/default.vcl.
-
-Somewhere in the top there will be a section that looks a bit like this.::
-
-	  # backend default {
-	  #     .host = "127.0.0.1";
-	  #     .port = "8080";
-	  # }
-
-We comment in this bit of text and change the port setting from 8080
-to 80, making the text look like.::
-
-          backend default {
-                .host = "127.0.0.1";
-    		.port = "80";
-	  }
-
-Now, this piece of configuration defines a backend in Varnish called
-*default*. When Varnish needs to get content from this backend it will
-connect to port 80 on localhost (127.0.0.1).
-
-Varnish can have several backends defined and can you can even join
-several backends together into clusters of backends for load balancing
-purposes. 
-
-Now that we have the basic Varnish configuration done, let us start up
-Varnish on port 8080 so we can do some fundamental testing on it.
diff --git a/doc/sphinx/tutorial/compression.rst b/doc/sphinx/tutorial/compression.rst
deleted file mode 100644
index 0b8d1e8..0000000
--- a/doc/sphinx/tutorial/compression.rst
+++ /dev/null
@@ -1,75 +0,0 @@
-.. _tutorial-compression:
-
-Compression
-~~~~~~~~~~~
-
-New in Varnish 3.0 was native support for compression, using gzip
-encoding. *Before* 3.0, Varnish would never compress objects. 
-
-In Varnish 3.0 compression defaults to "on", meaning that it tries to
-be smart and do the sensible thing.
-
-If you don't want Varnish tampering with the encoding you can disable
-compression all together by setting the parameter http_gzip_support to
-*false*. Please see man :ref:`ref-varnishd` for details.
-
-
-Default behaviour
-~~~~~~~~~~~~~~~~~
-
-The default for Varnish is to check if the client supports our
-compression scheme (gzip) and if it does it will override the
-Accept-Encoding header and set it to "gzip".
-
-When Varnish then issues a backend request the Accept-Encoding will
-then only consist of "gzip". If the server responds with gzip'ed
-content it will be stored in memory in its compressed form. If the
-backend sends content in clear text it will be stored like that.
-
-You can make Varnish compress content before storing it in cache in
-vcl_fetch by setting do_gzip to true, like this::
-
-   sub vcl_fetch {
-        if (beresp.http.content-type ~ "text") {
-                set beresp.do_gzip = true;
-        }
-  }
-
-Please make sure that you don't try to compress content that is
-incompressable, like jpgs, gifs and mp3. You'll only waste CPU
-cycles. You can also uncompress objects before storing it in memory by
-setting do_gunzip to *true* but I have no idea why anybody would want
-to do that.
-
-Generally, Varnish doesn't use much CPU so it might make more sense to
-have Varnish spend CPU cycles compressing content than doing it in
-your web- or application servers, which are more likely to be
-CPU-bound.
-
-GZIP and ESI
-~~~~~~~~~~~~
-
-If you are using Edge Side Includes you'll be happy to note that ESI
-and GZIP work together really well. Varnish will magically decompress
-the content to do the ESI-processing, then recompress it for efficient
-storage and delivery. 
-
-
-Clients that don't support gzip
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If the client does not support gzip the Accept-Encoding header is left
-alone and we'll end up serving whatever we get from the backend
-server. Remember that the Backend might tell Varnish to *Vary* on the
-Accept-Encoding.
-
-If the client does not support gzip but we've already got a compressed
-version of the page in memory Varnish will automatically decompress
-the page while delivering it.
-
-
-A random outburst
-~~~~~~~~~~~~~~~~~
-
-Poul has written :ref:`phk_gzip` which talks abit more about how the
-implementation works. 
diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
deleted file mode 100644
index c75171f..0000000
--- a/doc/sphinx/tutorial/cookies.rst
+++ /dev/null
@@ -1,95 +0,0 @@
-.. _tutorial-cookies:
-
-Cookies
--------
-
-Varnish will, in the default configuration, not cache a object coming
-from the backend with a Set-Cookie header present. Also, if the client
-sends a Cookie header, Varnish will bypass the cache and go directly to
-the backend.
-
-This can be overly conservative. A lot of sites use Google Analytics
-(GA) to analyze their traffic. GA sets a cookie to track you. This
-cookie is used by the client side javascript and is therefore of no
-interest to the server. 
-
-Cookies from the client
-~~~~~~~~~~~~~~~~~~~~~~~
-
-For a lot of web application it makes sense to completely disregard the
-cookies unless you are accessing a special part of the web site. This
-VCL snippet in vcl_recv will disregard cookies unless you are
-accessing /admin/::
-
-  if ( !( req.url ~ ^/admin/) ) {
-    unset req.http.Cookie;
-  }
-
-Quite simple. If, however, you need to do something more complicated,
-like removing one out of several cookies, things get
-difficult. Unfortunately Varnish doesn't have good tools for
-manipulating the Cookies. We have to use regular expressions to do the
-work. If you are familiar with regular expressions you'll understand
-whats going on. If you don't I suggest you either pick up a book on
-the subject, read through the *pcrepattern* man page or read through
-one of many online guides.
-
-Let me show you what Varnish Software uses. We use some cookies for
-Google Analytics tracking and similar tools. The cookies are all set
-and used by Javascript. Varnish and Drupal doesn't need to see those
-cookies and since Varnish will cease caching of pages when the client
-sends cookies we will discard these unnecessary cookies in VCL. 
-
-In the following VCL we discard all cookies that start with a
-underscore::
-
-  // Remove has_js and Google Analytics __* cookies.
-  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
-  // Remove a ";" prefix, if present.
-  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
-
-Let me show you an example where we remove everything except the
-cookies named COOKIE1 and COOKIE2 and you can marvel at it::
-
-  sub vcl_recv {
-    if (req.http.Cookie) {
-      set req.http.Cookie = ";" + req.http.Cookie;
-      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
-      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
-      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
-
-      if (req.http.Cookie == "") {
-          remove req.http.Cookie;
-      }
-    }
-  }
-
-A somewhat simpler example that can accomplish almost the same can be
-found below. Instead of filtering out the other cookies it picks out
-the one cookie that is needed, copies it to another header and then
-copies it back, deleting the original cookie header.::
-
-  sub vcl_recv {
-         # save the original cookie header so we can mangle it
-        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
-        # using a capturing sub pattern, extract the continuous string of 
-        # alphanumerics that immediately follows "PHPSESSID="
-        set req.http.X-Varnish-PHP_SID = 
-           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
-        set req.http.Cookie = req.X-Varnish-PHP_SID;
-        remove req.X-Varnish-PHP_SID;
-   }   
-
-There are other scary examples of what can be done in VCL in the
-Varnish Cache Wiki.
-
-
-Cookies coming from the backend
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If your backend server sets a cookie using the Set-Cookie header
-Varnish will not cache the page in the default configuration.  A
-hit-for-pass object (see :ref:`tutorial-vcl_fetch_actions`) is created.
-So, if the backend server acts silly and sets unwanted cookies just unset
-the Set-Cookie header and all should be fine. 
diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst
deleted file mode 100644
index 6bfc4c9..0000000
--- a/doc/sphinx/tutorial/devicedetection.rst
+++ /dev/null
@@ -1,268 +0,0 @@
-.. _tutorial-devicedetect:
-
-Device detection
-~~~~~~~~~~~~~~~~
-
-Device detection is figuring out what kind of content to serve to a
-client based on the User-Agent string supplied in a request.
-
-Use cases for this are for example to send size reduced files to mobile
-clients with small screens and on high latency networks, or to 
-provide a streaming video codec that the client understands.
-
-There are a couple of strategies on what to do with such clients:
-1) Redirect them to another URL.
-2) Use a different backend for the special clients.
-3) Change the backend requests so the usual backend sends tailored content.
-
-To make the examples easier to understand, it is assumed in this text 
-that all the req.http.X-UA-Device header is present and unique per client class
-that content is to be served to. 
-
-Setting this header can be as simple as::
-
-   sub vcl_recv {?
-       if (req.http.User-Agent ~ "(?i)iphone"?{
-           set req.http.X-UA-Device = "mobile-iphone";
-       }
-   }
-
-There are different commercial and free offerings in doing grouping and
-identifiying clients in further detail than this. For a basic and community
-based regular expression set, see
-https://github.com/varnish/varnish-devicedetect/ .
-
-
-Serve the different content on the same URL
--------------------------------------------
-
-The tricks involved are: 
-1. Detect the client (pretty simple, just include devicedetect.vcl and call
-it)
-2. Figure out how to signal the backend what client class this is. This
-includes for example setting a header, changing a header or even changing the
-backend request URL.
-3. Modify any response from the backend to add missing Vary headers, so
-Varnish' internal handling of this kicks in.
-4. Modify output sent to the client so any caches outside our control don't
-serve the wrong content.
-
-All this while still making sure that we only get 1 cached object per URL per
-device class.
-
-
-Example 1: Send HTTP header to backend
-''''''''''''''''''''''''''''''''''''''
-
-The basic case is that Varnish adds the X-UA-Device HTTP header on the backend
-requests, and the backend mentions in the response Vary header that the content
-is dependant on this header. 
-
-Everything works out of the box from Varnish' perspective.
-
-.. 071-example1-start
-
-VCL::
-
-    sub vcl_recv { 
-        # call some detection engine that set req.http.X-UA-Device
-    }
-    # req.http.X-UA-Device is copied by Varnish into bereq.http.X-UA-Device
-
-    # so, this is a bit conterintuitive. The backend creates content based on
-    # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will
-    # use the same cached object for all U-As that map to the same X-UA-Device.
-    #
-    # If the backend does not mention in Vary that it has crafted special
-    # content based on the User-Agent (==X-UA-Device), add it. 
-    # If your backend does set Vary: User-Agent, you may have to remove that here.
-    sub vcl_fetch {
-        if (req.http.X-UA-Device) {
-            if (!beresp.http.Vary) { # no Vary at all
-                set beresp.http.Vary = "X-UA-Device"; 
-            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
-                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; 
-            } 
-        }
-        # comment this out if you don't want the client to know your
-        # classification
-        set beresp.http.X-UA-Device = req.http.X-UA-Device;
-    }
-
-    # to keep any caches in the wild from serving wrong content to client #2
-    # behind them, we need to transform the Vary on the way out.
-    sub vcl_deliver {
-        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
-            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
-        }
-    }
-
-.. 071-example1-end
-
-Example 2: Normalize the User-Agent string
-''''''''''''''''''''''''''''''''''''''''''
-
-Another way of signaling the device type is to override or normalize the
-User-Agent header sent to the backend.
-
-For example
-
-    User-Agent: Mozilla/5.0 (Linux; U; Android 2.2; nb-no; HTC Desire Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1
-
-becomes:
-
-    User-Agent: mobile-android
-
-when seen by the backend.
-
-This works if you don't need the original header for anything on the backend.
-A possible use for this is for CGI scripts where only a small set of predefined
-headers are (by default) available for the script.
-
-.. 072-example2-start
-
-VCL::
-
-    sub vcl_recv { 
-        # call some detection engine that set req.http.X-UA-Device
-    }
-
-    # override the header before it is sent to the backend
-    sub vcl_miss { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
-    sub vcl_pass { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
-
-    # standard Vary handling code from previous examples.
-    sub vcl_fetch {
-        if (req.http.X-UA-Device) {
-            if (!beresp.http.Vary) { # no Vary at all
-                set beresp.http.Vary = "X-UA-Device";
-            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
-                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
-            }
-        }
-        set beresp.http.X-UA-Device = req.http.X-UA-Device;
-    }
-    sub vcl_deliver {
-        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
-            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
-        }
-    }
-
-.. 072-example2-end
-
-Example 3: Add the device class as a GET query parameter
-''''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
-If everything else fails, you can add the device type as a GET argument.
-
-    http://example.com/article/1234.html --> http://example.com/article/1234.html?devicetype=mobile-iphone
-
-The client itself does not see this classification, only the backend request
-is changed.
-
-.. 073-example3-start
-
-VCL::
-
-    sub vcl_recv { 
-        # call some detection engine that set req.http.X-UA-Device
-    }
-
-    sub append_ua {
-        if ((req.http.X-UA-Device) && (req.request == "GET")) {
-            # if there are existing GET arguments;
-            if (req.url ~ "\?") {
-                set req.http.X-get-devicetype = "&devicetype=" + req.http.X-UA-Device;
-            } else { 
-                set req.http.X-get-devicetype = "?devicetype=" + req.http.X-UA-Device;
-            }
-            set req.url = req.url + req.http.X-get-devicetype;
-            unset req.http.X-get-devicetype;
-        }
-    }
-
-    # do this after vcl_hash, so all Vary-ants can be purged in one go. (avoid ban()ing)
-    sub vcl_miss { call append_ua; }
-    sub vcl_pass { call append_ua; }
-
-    # Handle redirects, otherwise standard Vary handling code from previous
-    # examples.
-    sub vcl_fetch {
-        if (req.http.X-UA-Device) {
-            if (!beresp.http.Vary) { # no Vary at all
-                set beresp.http.Vary = "X-UA-Device";
-            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
-                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
-            }
-
-            # if the backend returns a redirect (think missing trailing slash),
-            # we will potentially show the extra address to the client. we
-            # don't want that.  if the backend reorders the get parameters, you
-            # may need to be smarter here. (? and & ordering)
-
-            if (beresp.status == 301 || beresp.status == 302 || beresp.status == 303) {
-                set beresp.http.location = regsub(beresp.http.location, "[?&]devicetype=.*$", "");
-            }
-        }
-        set beresp.http.X-UA-Device = req.http.X-UA-Device;
-    }
-    sub vcl_deliver {
-        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
-            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
-        }
-    }
-
-.. 073-example3-end
-
-Different backend for mobile clients
-------------------------------------
-
-If you have a different backend that serves pages for mobile clients, or any
-special needs in VCL, you can use the X-UA-Device header like this::
-
-    backend mobile {
-        .host = "10.0.0.1";
-        .port = "80";
-    }
-
-    sub vcl_recv {
-        # call some detection engine
-
-        if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
-            set req.backend = mobile;
-        }
-    }
-    sub vcl_hash {
-        if (req.http.X-UA-Device) {
-            hash_data(req.http.X-UA-Device);
-        }
-    }
-
-Redirecting mobile clients
---------------------------
-
-If you want to redirect mobile clients you can use the following snippet.
-
-.. 065-redir-mobile-start
-
-VCL::
-
-    sub vcl_recv {
-        # call some detection engine
-
-        if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
-            error 750 "Moved Temporarily";
-        }
-    }
-     
-    sub vcl_error {
-        if (obj.status == 750) {
-            set obj.http.Location = "http://m.example.com" + req.url;
-            set obj.status = 302;
-            return(deliver);
-        }
-    }
-
-.. 065-redir-mobile-end
-
-
diff --git a/doc/sphinx/tutorial/esi.rst b/doc/sphinx/tutorial/esi.rst
deleted file mode 100644
index 720b790..0000000
--- a/doc/sphinx/tutorial/esi.rst
+++ /dev/null
@@ -1,79 +0,0 @@
-.. _tutorial-esi:
-
-Edge Side Includes
-------------------
-
-*Edge Side Includes* is a language to include *fragments* of web pages
-in other web pages. Think of it as HTML include statement that works
-over HTTP. 
-
-On most web sites a lot of content is shared between
-pages. Regenerating this content for every page view is wasteful and
-ESI tries to address that letting you decide the cache policy for
-each fragment individually.
-
-In Varnish we've only implemented a small subset of ESI. As of 2.1 we
-have three ESI statements:
-
- * esi:include 
- * esi:remove
- * <!--esi ...-->
-
-Content substitution based on variables and cookies is not implemented
-but is on the roadmap. 
-
-Varnish will not process ESI instructions in HTML comments.
-
-Example: esi:include
-~~~~~~~~~~~~~~~~~~~~
-
-Lets see an example how this could be used. This simple cgi script
-outputs the date::
-
-     #!/bin/sh
-     
-     echo 'Content-type: text/html'
-     echo ''
-     date "+%Y-%m-%d %H:%M"
-
-Now, lets have an HTML file that has an ESI include statement::
-
-     <HTML>
-     <BODY>
-     The time is: <esi:include src="/cgi-bin/date.cgi"/>
-     at this very moment.
-     </BODY>
-     </HTML>
-
-For ESI to work you need to activate ESI processing in VCL, like this::
-
-    sub vcl_fetch {
-    	if (req.url == "/test.html") {
-           set beresp.do_esi = true; /* Do ESI processing		*/
-           set beresp.ttl = 24 h;    /* Sets the TTL on the HTML above  */
-    	} elseif (req.url == "/cgi-bin/date.cgi") {
-           set beresp.ttl = 1m;      /* Sets a one minute TTL on	*/
-	       	       	 	     /*  the included object		*/
-        }
-    }
-
-Example: esi:remove and <!--esi ... -->
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The <esi:remove> and <!--esi ... --> constructs can be used to present
-appropriate content whether or not ESI is available, for example you can
-include content when ESI is available or link to it when it is not.
-ESI processors will remove the start ("<!--esi") and end ("-->") when
-the page is processed, while still processing the contents. If the page
-is not processed, it will remain, becoming an HTML/XML comment tag.
-ESI processors will remove <esi:remove> tags and all content contained
-in them, allowing you to only render the content when the page is not
-being ESI-processed.
-For example::
-
-  <esi:remove> 
-    <a href="http://www.example.com/LICENSE">The license</a>
-  </esi:remove>
-  <!--esi  
-  <p>The full text of the license:</p>
-  <esi:include src="http://example.com/LICENSE" />
-  -->
diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
deleted file mode 100644
index 406b4b3..0000000
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ /dev/null
@@ -1,103 +0,0 @@
-.. _tutorial-handling_misbehaving_servers:
-
-Misbehaving servers
--------------------
-
-A key feature of Varnish is its ability to shield you from misbehaving
-web- and application servers.
-
-
-
-Grace mode
-~~~~~~~~~~
-
-When several clients are requesting the same page Varnish will send
-one request to the backend and place the others on hold while fetching
-one copy from the back end. In some products this is called request
-coalescing and Varnish does this automatically.
-
-If you are serving thousands of hits per second the queue of waiting
-requests can get huge. There are two potential problems - one is a
-thundering herd problem - suddenly releasing a thousand threads to
-serve content might send the load sky high. Secondly - nobody likes to
-wait. To deal with this we can instruct Varnish to keep
-the objects in cache beyond their TTL and to serve the waiting
-requests somewhat stale content.
-
-So, in order to serve stale content we must first have some content to
-serve. So to make Varnish keep all objects for 30 minutes beyond their
-TTL use the following VCL::
-
-  sub vcl_fetch {
-    set beresp.grace = 30m;
-  }
-
-Varnish still won't serve the stale objects. In order to enable
-Varnish to actually serve the stale object we must enable this on the
-request. Lets us say that we accept serving 15s old object.::
-
-  sub vcl_recv {
-    set req.grace = 15s;
-  }
-
-You might wonder why we should keep the objects in the cache for 30
-minutes if we are unable to serve them? Well, if you have enabled
-:ref:`tutorial-advanced_backend_servers-health` you can check if the
-backend is sick and if it is we can serve the stale content for a bit
-longer.::
-
-   if (! req.backend.healthy) {
-      set req.grace = 5m;
-   } else {
-      set req.grace = 15s;
-   }
-
-So, to sum up, grace mode solves two problems:
- * it serves stale content to avoid request pile-up.
- * it serves stale content if the backend is not healthy.
-
-Saint mode
-~~~~~~~~~~
-
-Sometimes servers get flaky. They start throwing out random
-errors. You can instruct Varnish to try to handle this in a
-more-than-graceful way - enter *Saint mode*. Saint mode enables you to
-discard a certain page from one backend server and either try another
-server or serve stale content from cache. Lets have a look at how this
-can be enabled in VCL::
-
-  sub vcl_fetch {
-    if (beresp.status == 500) { 
-      set beresp.saintmode = 10s;
-      return(restart);
-    }
-    set beresp.grace = 5m;
-  } 
-
-When we set beresp.saintmode to 10 seconds Varnish will not ask *that*
-server for URL for 10 seconds. A blacklist, more or less. Also a
-restart is performed so if you have other backends capable of serving
-that content Varnish will try those. When you are out of backends
-Varnish will serve the content from its stale cache.
-
-This can really be a life saver.
-
-Known limitations on grace- and saint mode
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If your request fails while it is being fetched you're thrown into
-vcl_error. vcl_error has access to a rather limited set of data so you
-can't enable saint mode or grace mode here. This will be addressed in a
-future release but a work-around available.
-
-* Declare a backend that is always sick.
-* Set a magic marker in vcl_error
-* Restart the transaction
-* Note the magic marker in vcl_recv and set the backend to the one mentioned
-* Varnish will now serve stale data is any is available
-
-
-God mode
-~~~~~~~~
-Not implemented yet. :-)
-
diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
deleted file mode 100644
index b9fa7e6..0000000
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ /dev/null
@@ -1,213 +0,0 @@
-.. _tutorial-increasing_your_hitrate:
-
-Achieving a high hitrate
-------------------------
-
-Now that Varnish is up and running, and you can access your web
-application through Varnish. Unless your application is specifically
-written to work behind a web accelerator you'll probably need to do
-some changes to either the configuration or the application in order
-to get a high hit rate in Varnish.
-
-Varnish will not cache your data unless it's absolutely sure it is
-safe to do so. So, for you to understand how Varnish decides if and
-how to cache a page, I'll guide you through a couple of tools that you
-will find useful.
-
-Note that you need a tool to see what HTTP headers fly between you and
-the web server. On the Varnish server, the easiest is to use
-varnishlog and varnishtop but sometimes a client-side tool makes
-sense. Here are the ones I use.
-
-Tool: varnishtop
-~~~~~~~~~~~~~~~~
-
-You can use varnishtop to identify what URLs are hitting the backend
-the most. ``varnishtop -i txurl`` is an essential command. You can see
-some other examples of varnishtop usage in :ref:`tutorial-statistics`.
-
-
-Tool: varnishlog
-~~~~~~~~~~~~~~~~
-
-When you have identified the an URL which is frequently sent to the
-backend you can use varnishlog to have a look at the request.
-``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
-coming from the client (-c) matching /foo/bar.
-
-For more information on how varnishlog works please see
-:ref:`tutorial-logging` or man :ref:`ref-varnishlog`.
-
-For extended diagnostics headers, see
-http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
-
-
-Tool: lwp-request
-~~~~~~~~~~~~~~~~~
-
-lwp-request is part of The World-Wide Web library for Perl. It's a
-couple of really basic programs that can execute an HTTP request and
-give you the result. I mostly use two programs, GET and HEAD.
-
-vg.no was the first site to use Varnish and the people running Varnish
-there are quite clueful. So it's interesting to look at their HTTP
-Headers. Let's send a GET request for their home page::
-
-  $ GET -H 'Host: www.vg.no' -Used http://vg.no/
-  GET http://vg.no/
-  Host: www.vg.no
-  User-Agent: lwp-request/5.834 libwww-perl/5.834
-  
-  200 OK
-  Cache-Control: must-revalidate
-  Refresh: 600
-  Title: VG Nett - Forsiden - VG Nett
-  X-Age: 463
-  X-Cache: HIT
-  X-Rick-Would-Never: Let you down
-  X-VG-Jobb: http://www.finn.no/finn/job/fulltime/result?keyword=vg+multimedia Merk:HeaderNinja
-  X-VG-Korken: http://www.youtube.com/watch?v=Fcj8CnD5188
-  X-VG-WebCache: joanie
-  X-VG-WebServer: leon
-
-OK. Let me explain what it does. GET usually sends off HTTP 0.9
-requests, which lack the Host header. So I add a Host header with the
--H option. -U print request headers, -s prints response status, -e
-prints response headers and -d discards the actual content. We don't
-really care about the content, only the headers.
-
-As you can see, VG adds quite a bit of information in their
-headers. Some of the headers, like the X-Rick-Would-Never are specific
-to vg.no and their somewhat odd sense of humour. Others, like the
-X-VG-Webcache are for debugging purposes. 
-
-So, to check whether a site sets cookies for a specific URL, just do::
-
-  GET -Used http://example.com/ |grep ^Set-Cookie
-
-Tool: Live HTTP Headers
-~~~~~~~~~~~~~~~~~~~~~~~
-
-There is also a plugin for Firefox. *Live HTTP Headers* can show you
-what headers are being sent and recieved. Live HTTP Headers can be
-found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
-googling "Live HTTP Headers".
-
-
-The role of HTTP Headers
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Along with each HTTP request and response comes a bunch of headers
-carrying metadata. Varnish will look at these headers to determine if
-it is appropriate to cache the contents and how long Varnish can keep
-the content.
-
-Please note that when considering these headers Varnish actually
-considers itself *part of* the actual webserver. The rationale being
-that both are under your control. 
-
-The term *surrogate origin cache* is not really well defined by the
-IETF so RFC 2616 so the various ways Varnish works might differ from
-your expectations.
-
-Let's take a look at the important headers you should be aware of:
-
-Cache-Control
-~~~~~~~~~~~~~
-
-The Cache-Control instructs caches how to handle the content. Varnish
-cares about the *max-age* parameter and uses it to calculate the TTL
-for an object. 
-
-"Cache-Control: nocache" is ignored but if you need this you can
-easily add support for it.
-
-So make sure you issue a Cache-Control header with a max-age
-header. You can have a look at what Varnish Software's drupal server
-issues::
-
-  $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control
-  Cache-Control: public, max-age=600
-
-Age
-~~~
-
-Varnish adds an Age header to indicate how long the object has been
-kept inside Varnish. You can grep out Age from varnishlog like this::
-
-  varnishlog -i TxHeader -I ^Age
-
-Pragma
-~~~~~~
-
-An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
-header. You could easily add support for this header in VCL.
-
-In vcl_fetch::
-
-  if (beresp.http.Pragma ~ "nocache") {
-     return(hit_for_pass);
-  }
-
-Authorization
-~~~~~~~~~~~~~
-
-If Varnish sees an Authorization header it will pass the request. If
-this is not what you want you can unset the header.
-
-Overriding the time-to-live (ttl)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Sometimes your backend will misbehave. It might, depending on your
-setup, be easier to override the ttl in Varnish than to fix your
-somewhat cumbersome backend. 
-
-You need VCL to identify the objects you want and then you set the
-beresp.ttl to whatever you want::
-
-  sub vcl_fetch {
-      if (req.url ~ "^/legacy_broken_cms/") {
-          set beresp.ttl = 5d;
-      }
-  }
-
-The example will set the TTL to 5 days for the old legacy stuff on
-your site.
-
-Forcing caching for certain requests and certain responses
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Since you still have this cumbersome backend that isn't very friendly
-to work with you might want to override more stuff in Varnish. We
-recommend that you rely as much as you can on the default caching
-rules. It is perfectly easy to force Varnish to lookup an object in
-the cache but it isn't really recommended.
-
-
-Normalizing your namespace
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Some sites are accessed via lots of
-hostnames. http://www.varnish-software.com/,
-http://varnish-software.com/ and http://varnishsoftware.com/ all point
-at the same site. Since Varnish doesn't know they are different,
-Varnish will cache different versions of every page for every
-hostname. You can mitigate this in your web server configuration by
-setting up redirects or by using the following VCL::
-
-  if (req.http.host ~ "(?i)^(www.)?varnish-?software.com") {
-    set req.http.host = "varnish-software.com";
-  }
-
-
-Ways of increasing your hitrate even more
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following chapters should give your ways of further increasing
-your hitrate, especially the chapter on Cookies.
-
- * :ref:`tutorial-cookies`
- * :ref:`tutorial-vary`
- * :ref:`tutorial-purging`
- * :ref:`tutorial-esi`
-
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
deleted file mode 100644
index 91fdb17..0000000
--- a/doc/sphinx/tutorial/index.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-.. _tutorial-index:
-
-%%%%%%%%%%%%%
-Using Varnish
-%%%%%%%%%%%%%
-
-This tutorial is intended for system administrators managing Varnish
-cache. The reader should know how to configure her web- or application
-server and have basic knowledge of the HTTP protocol. The reader
-should have Varnish up and running with the default configuration. 
-
-The tutorial is split into short chapters, each chapter taking on a
-separate topic. Good luck.
-
-.. toctree:: :maxdepth: 1
-
-        introduction
-	backend_servers
-        starting_varnish
-	logging
-        sizing_your_cache
-        putting_varnish_on_port_80
-	vcl
-        statistics
-        increasing_your_hitrate
-	cookies
-	vary
-	purging
-	compression
-	esi
-	virtualized
-	websockets
-	devicedetection
-	advanced_backend_servers
-        handling_misbehaving_servers
-        advanced_topics
-	troubleshooting
-
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
deleted file mode 100644
index 0d43623..0000000
--- a/doc/sphinx/tutorial/introduction.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-.. _tutorial-intro:
-
-What is Varnish?
-----------------
-
-Varnish Cache is a web application accelerator also known as a caching
-HTTP reverse proxy. You install it in front of any server that speaks
-HTTP and configure it to cache the contents. Varnish Cache is really,
-really fast. It typically speeds up delivery with a factor of 300 -
-1000x, depending on your architecture.
-
-
-Performance
-~~~~~~~~~~~
-
-Varnish performs really, really well. It is usually bound by the speed
-of the network, effectivly turning performance into a non-issue. We've
-seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
-
-Flexibility
-~~~~~~~~~~~
-
-One of the key features of Varnish Cache, in addition to it's
-performance, is the flexibility of it's configuration language,
-VCL. VCL enables you to write policies on how incoming requests should
-be handled. In such a policy you can decide what content you want to
-serve, from where you want to get the content and how the request or
-response should be altered. You can read more about this in our
-tutorial.
-
-
-Supported plattforms
-~~~~~~~~~~~~~~~~~~~~
-
-Varnish is written to run on modern versions of Linux and FreeBSD and
-the best experience is had on those plattforms. Thanks to our
-contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst
deleted file mode 100644
index 1f0bc18..0000000
--- a/doc/sphinx/tutorial/logging.rst
+++ /dev/null
@@ -1,68 +0,0 @@
-.. _tutorial-logging:
-
-Logging in Varnish
-------------------
-
-One of the really nice features in Varnish is how logging
-works. Instead of logging to normal log file Varnish logs to a shared
-memory segment. When the end of the segment is reached we start over,
-overwriting old data. This is much, much faster then logging to a file
-and it doesn't require disk space.
-
-The flip side is that if you forget to have a program actually write the
-logs to disk they will disappear.
-
-varnishlog is one of the programs you can use to look at what Varnish
-is logging. Varnishlog gives you the raw logs, everything that is
-written to the logs. There are other clients as well, we'll show you
-these later.
-
-In the terminal window you started varnish now type *varnishlog* and
-press enter.
-
-You'll see lines like these scrolling slowly by.::
-
-    0 CLI          - Rd ping
-    0 CLI          - Wr 200 PONG 1273698726 1.0
-
-These is the Varnish master process checking up on the caching process
-to see that everything is OK. 
-
-Now go to the browser and reload the page displaying your web
-app. You'll see lines like these.::
-
-   11 SessionOpen  c 127.0.0.1 58912 0.0.0.0:8080
-   11 ReqStart     c 127.0.0.1 58912 595005213
-   11 RxRequest    c GET
-   11 RxURL        c /
-   11 RxProtocol   c HTTP/1.1
-   11 RxHeader     c Host: localhost:8080
-   11 RxHeader     c Connection: keep-alive
-
-The first column is an arbitrary number, it defines the request. Lines
-with the same number are part of the same HTTP transaction. The second
-column is the *tag* of the log message. All log entries are tagged
-with a tag indicating what sort of activity is being logged. Tags
-starting with Rx indicate Varnish is recieving data and Tx indicates
-sending data.
-
-The third column tell us whether this is is data coming or going to
-the client (c) or to/from the backend (b). The forth column is the
-data being logged.
-
-Now, you can filter quite a bit with varnishlog. The basic option you
-want to know are:
-
--b
- Only show log lines from traffic going between Varnish and the backend 
- servers. This will be useful when we want to optimize cache hit rates.
-
--c 
- Same as -b but for client side traffic.
-
--m tag:regex
- Only list transactions where the tag matches a regular expression. If
- it matches you will get the whole transaction.
-
-Now that Varnish seem to work OK it's time to put Varnish on port 80
-while we tune it.
diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
deleted file mode 100644
index 422f9f4..0000000
--- a/doc/sphinx/tutorial/purging.rst
+++ /dev/null
@@ -1,175 +0,0 @@
-.. _tutorial-purging:
-
-=====================
- Purging and banning
-=====================
-
-One of the most effective ways of increasing your hit ratio is to
-increase the time-to-live (ttl) of your objects. But, as you're aware
-of, in this twitterific day of age serving content that is outdated is
-bad for business.
-
-The solution is to notify Varnish when there is fresh content
-available. This can be done through three mechanisms. HTTP purging,
-banning and forced cache misses. First, let me explain the HTTP purges. 
-
-
-HTTP Purges
-===========
-
-A *purge* is what happens when you pick out an object from the cache
-and discard it along with its variants. Usually a purge is invoked
-through HTTP with the method PURGE.
-
-An HTTP purge is similar to an HTTP GET request, except that the
-*method* is PURGE. Actually you can call the method whatever you'd
-like, but most people refer to this as purging. Squid supports the
-same mechanism. In order to support purging in Varnish you need the
-following VCL in place::
-
-  acl purge {
-	  "localhost";
-	  "192.168.55.0"/24;
-  }
-  
-  sub vcl_recv {
-      	  # allow PURGE from localhost and 192.168.55...
-
-	  if (req.request == "PURGE") {
-		  if (!client.ip ~ purge) {
-			  error 405 "Not allowed.";
-		  }
-		  return (lookup);
-	  }
-  }
-  
-  sub vcl_hit {
-	  if (req.request == "PURGE") {
-	          purge;
-		  error 200 "Purged.";
-	  }
-  }
-  
-  sub vcl_miss {
-	  if (req.request == "PURGE") {
-	          purge;
-		  error 200 "Purged.";
-	  }
-  }
-
-As you can see we have used to new VCL subroutines, vcl_hit and
-vcl_miss. When we call lookup Varnish will try to lookup the object in
-its cache. It will either hit an object or miss it and so the
-corresponding subroutine is called. In vcl_hit the object that is
-stored in cache is available and we can set the TTL. The purge in
-vcl_miss is necessary to purge all variants in the cases where you hit an
-object, but miss a particular variant.
-
-So for example.com to invalidate their front page they would call out
-to Varnish like this::
-
-  PURGE / HTTP/1.0
-  Host: example.com
-
-And Varnish would then discard the front page. This will remove all
-variants as defined by Vary.
-
-Bans
-====
-
-There is another way to invalidate content: Bans. You can think of
-bans as a sort of a filter on objects already in the cache. You *ban*
-certain content from being served from your cache. You can ban
-content based on any metadata we have.
-A ban will only work on objects already in the cache, it does not
-prevent new content from entering the cache or being served.
-
-Support for bans is built into Varnish and available in the CLI
-interface. To ban every png object belonging on example.com, issue
-the following command::
-
-  ban req.http.host == "example.com" && req.url ~ "\.png$"
-
-Quite powerful, really.
-
-Bans are checked when we hit an object in the cache, but before we
-deliver it. *An object is only checked against newer bans*. 
-
-Bans that only match against obj.* are also processed by a background
-worker threads called the *ban lurker*. The ban lurker will walk the
-heap and try to match objects and will evict the matching objects. How
-aggressive the ban lurker is can be controlled by the parameter
-ban_lurker_sleep. The ban lurker can be disabled by setting
-ban_lurker_sleep to 0.
-
-Bans that are older than the oldest objects in the cache are discarded
-without evaluation.  If you have a lot of objects with long TTL, that
-are seldom accessed you might accumulate a lot of bans. This might
-impact CPU usage and thereby performance.
-
-You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
-
-  sub vcl_recv {
-	  if (req.request == "BAN") {
-                  # Same ACL check as above:
-		  if (!client.ip ~ purge) {
-			  error 405 "Not allowed.";
-		  }
-		  ban("req.http.host == " + req.http.host +
-		        "&& req.url == " + req.url);
-
-		  # Throw a synthetic page so the
-                  # request won't go to the backend.
-		  error 200 "Ban added";
-	  }
-  }
-
-This VCL sniplet enables Varnish to handle an HTTP BAN method, adding a
-ban on the URL, including the host part.
-
-The ban lurker can help you keep the ban list at a manageable size, so
-we recommend that you avoid using req.* in your bans, as the request
-object is not available in the ban lurker thread.
-
-You can use the following template to write ban lurker friendly bans::
-
-  sub vcl_fetch {
-    set beresp.http.x-url = req.url;
-  }
-
-  sub vcl_deliver {
-    unset resp.http.x-url; # Optional
-  }
-
-  sub vcl_recv {
-    if (req.request == "PURGE") {
-      if (client.ip !~ purge) {
-        error 401 "Not allowed";
-      }
-      ban("obj.http.x-url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple
-    }
-  }
-
-To inspect the current ban list, issue the ban.list command in CLI. This
-will produce a status of all current bans::
-
-  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
-  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
-
-The ban list contains the ID of the ban, the timestamp when the ban
-entered the ban list. A count of the objects that has reached this point
-in the ban list, optionally postfixed with a 'G' for "Gone", if the ban
-is no longer valid.  Finally, the ban expression is listed. The ban can
-be marked as Gone if it is a duplicate ban, but is still kept in the list
-for optimization purposes.
-
-Forcing a cache miss
-====================
-
-The final way to invalidate an object is a method that allows you to
-refresh an object by forcing a hash miss for a single request. If you set
-req.hash_always_miss to true, varnish will miss the current object in the
-cache, thus forcing a fetch from the backend. This can in turn add the
-freshly fetched object to the cache, thus overriding the current one. The
-old object will stay in the cache until ttl expires or it is evicted by
-some other means.
diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
deleted file mode 100644
index 73a80ff..0000000
--- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
-Put Varnish on port 80
-----------------------
-
-Until now we've been running with Varnish on a high port, for testing
-purposes. You should test your application and if it works OK we can
-switch, so Varnish will be running on port 80 and your web server on a
-high port.
-
-First we kill off varnishd::
-
-     # pkill varnishd
-
-and stop your web server. Edit the configuration for your web server
-and make it bind to port 8080 instead of 80. Now open the Varnish
-default.vcl and change the port of the *default* backend to 8080.
-
-Start up your web server and then start varnish::
-
-      # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000
-
-Note that we've removed the -a option. Now Varnish, as its default
-setting dictates, will bind to the http port (80). Now everyone
-accessing your site will be accessing through Varnish.
-
diff --git a/doc/sphinx/tutorial/sizing_your_cache.rst b/doc/sphinx/tutorial/sizing_your_cache.rst
deleted file mode 100644
index c19647c..0000000
--- a/doc/sphinx/tutorial/sizing_your_cache.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
-Sizing your cache
------------------
-
-Picking how much memory you should give Varnish can be a tricky
-task. A few things to consider:
-
- * How big is your *hot* data set. For a portal or news site that
-   would be the size of the front page with all the stuff on it, and
-   the size of all the pages and objects linked from the first page. 
- * How expensive is it to generate an object? Sometimes it makes sense
-   to only cache images a little while or not to cache them at all if
-   they are cheap to serve from the backend and you have a limited
-   amount of memory.
- * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or some other
-   tool. If you have a lot of LRU activity then your cache is evicting
-   objects due to space constraints and you should consider increasing
-   the size of the cache.
-
-Be aware that every object that is stored also carries overhead that
-is kept outside the actually storage area. So, even if you specify -s
-malloc,16G varnish might actually use **double** that. Varnish has a
-overhead of about 1k per object. So, if you have lots of small objects
-in your cache the overhead might be significant.
-
diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
deleted file mode 100644
index 6c89f54..0000000
--- a/doc/sphinx/tutorial/starting_varnish.rst
+++ /dev/null
@@ -1,51 +0,0 @@
-.. _tutorial-starting_varnish:
-
-Starting Varnish
-----------------
-
-I assume varnishd is in your path. You might want to run ``pkill
-varnishd`` to make sure varnishd isn't running. Become root and type:
-
-``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
-
-I added a few options, lets go through them:
-
-``-f /usr/local/etc/varnish/default.vcl``
- The -f options specifies what configuration varnishd should use.
-
-``-s malloc,1G``
- The -s options chooses the storage type Varnish should use for
- storing its content. I used the type *malloc*, which just uses memory
- for storage. There are other backends as well, described in 
- :ref:tutorial-storage. 1G specifies how much memory should be allocated 
- - one gigabyte. 
-
-``-T 127.0.0.1:2000``
- Varnish has a built-in text-based administration
- interface. Activating the interface makes Varnish manageble without
- stopping it. You can specify what interface the management interface
- should listen to. Make sure you don't expose the management interface
- to the world as you can easily gain root access to a system via the
- Varnish management interface. I recommend tieing it to localhost. If
- you have users on your system that you don't fully trust, use firewall
- rules to restrict access to the interface to root only.
-
-``-a 0.0.0.0:8080``
- I specify that I want Varnish to listen on port 8080 for incomming
- HTTP requests. For a production environment you would probably make
- Varnish listen on port 80, which is the default.
-
-Now you have Varnish running. Let us make sure that it works
-properly. Use your browser to go to http://192.168.2.2:8080/
-(obviously, you should replace the IP address with one on your own
-system) - you should now see your web application running there.
-
-Whether or not the application actually goes faster when run through
-Varnish depends on a few factors. If you application uses cookies for
-every session (a lot of PHP and Java applications seem to send a
-session cookie if it is needed or not) or if it uses authentication
-chances are Varnish won't do much caching. Ignore that for the moment,
-we come back to that in :ref:`tutorial-increasing_your_hitrate`.
-
-Lets make sure that Varnish really does do something to your web
-site. To do that we'll take a look at the logs.
diff --git a/doc/sphinx/tutorial/statistics.rst b/doc/sphinx/tutorial/statistics.rst
deleted file mode 100644
index 4386111..0000000
--- a/doc/sphinx/tutorial/statistics.rst
+++ /dev/null
@@ -1,57 +0,0 @@
-.. _tutorial-statistics:
-
-
-Statistics
-----------
-
-Now that your varnish is up and running let's have a look at how it is
-doing. There are several tools that can help.
-
-varnishtop
-~~~~~~~~~~
-
-The varnishtop utility reads the shared memory logs and presents a
-continuously updated list of the most commonly occurring log entries.
-
-With suitable filtering using the -I, -i, -X and -x options, it can be
-used to display a ranking of requested documents, clients, user
-agents, or any other information which is recorded in the log.
-
-``varnishtop -i rxurl`` will show you what URLs are being asked for
-by the client. ``varnishtop -i txurl`` will show you what your backend
-is being asked the most. ``varnishtop -i RxHeader -I
-Accept-Encoding`` will show the most popular Accept-Encoding header
-the client are sending you.
-
-varnishhist
-~~~~~~~~~~~
-
-The varnishhist utility reads varnishd(1) shared memory logs and
-presents a continuously updated histogram showing the distribution of
-the last N requests by their processing.  The value of N and the
-vertical scale are displayed in the top left corner.  The horizontal
-scale is logarithmic.  Hits are marked with a pipe character ("|"),
-and misses are marked with a hash character ("#").
-
-
-varnishsizes
-~~~~~~~~~~~~
-
-Varnishsizes does the same as varnishhist, except it shows the size of
-the objects and not the time take to complete the request. This gives
-you a good overview of how big the objects you are serving are.
-
-
-varnishstat
-~~~~~~~~~~~
-
-Varnish has lots of counters. We count misses, hits, information about
-the storage, threads created, deleted objects. Just about
-everything. varnishstat will dump these counters. This is useful when
-tuning varnish. 
-
-There are programs that can poll varnishstat regularly and make nice
-graphs of these counters. One such program is Munin. Munin can be
-found at http://munin-monitoring.org/ . There is a plugin for munin in
-the varnish source code.
-
diff --git a/doc/sphinx/tutorial/troubleshooting.rst b/doc/sphinx/tutorial/troubleshooting.rst
deleted file mode 100644
index 5bbcf6c..0000000
--- a/doc/sphinx/tutorial/troubleshooting.rst
+++ /dev/null
@@ -1,99 +0,0 @@
-Troubleshooting Varnish
------------------------
-
-Sometimes Varnish misbehaves. In order for you to understand whats
-going on there are a couple of places you can check. varnishlog,
-/var/log/syslog, /var/log/messages are all places where varnish might
-leave clues of whats going on.
-
-
-When Varnish won't start
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Sometimes Varnish wont start. There is a plethora of reasons why
-Varnish wont start on your machine. We've seen everything from wrong
-permissions on /dev/null to other processes blocking the ports.
-
-Starting Varnish in debug mode to see what is going on.
-
-Try to start varnish by::
-
-    # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000  -a 0.0.0.0:8080 -d
-
-Notice the -d option. It will give you some more information on what
-is going on. Let us see how Varnish will react to something else
-listening on its port.::
-
-    # varnishd -n foo -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000  -a 0.0.0.0:8080 -d
-    storage_malloc: max size 1024 MB.
-    Using old SHMFILE
-    Platform: Linux,2.6.32-21-generic,i686,-smalloc,-hcritbit
-    200 193     
-    -----------------------------
-    Varnish Cache CLI.
-    -----------------------------
-    Type 'help' for command list.
-    Type 'quit' to close CLI session.
-    Type 'start' to launch worker process.
-
-Now Varnish is running. Only the master process is running, in debug
-mode the cache does not start. Now you're on the console. You can
-instruct the master process to start the cache by issuing "start".::
-
-	 start
-	 bind(): Address already in use
-	 300 22      
-	 Could not open sockets
-
-And here we have our problem. Something else is bound to the HTTP port
-of Varnish. If this doesn't help try strace or truss or come find us
-on IRC.
-
-
-Varnish is crashing
-~~~~~~~~~~~~~~~~~~~
-
-When varnish goes bust the child processes crashes. Usually the mother
-process will manage this by restarting the child process again. Any
-errors will be logged in syslog. It might look like this::
-
-       Mar  8 13:23:38 smoke varnishd[15670]: Child (15671) not responding to CLI, killing it.
-       Mar  8 13:23:43 smoke varnishd[15670]: last message repeated 2 times
-       Mar  8 13:23:43 smoke varnishd[15670]: Child (15671) died signal=3
-       Mar  8 13:23:43 smoke varnishd[15670]: Child cleanup complete
-       Mar  8 13:23:43 smoke varnishd[15670]: child (15697) Started
-
-Specifically if you see the "Error in munmap" error on Linux you might
-want to increase the amount of maps available. Linux is limited to a
-maximum of 64k maps. Setting vm.max_max_count i sysctl.conf will
-enable you to increase this limit. You can inspect the number of maps
-your program is consuming by counting the lines in /proc/$PID/maps
-
-This is a rather odd thing to document here - but hopefully Google
-will serve you this page if you ever encounter this error. 
-
-Varnish gives me Guru meditation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-First find the relevant log entries in varnishlog. That will probably
-give you a clue. Since varnishlog logs so much data it might be hard
-to track the entries down. You can set varnishlog to log all your 503
-errors by issuing the following command::
-
-   $ varnishlog -c -m TxStatus:503
-
-If the error happened just a short time ago the transaction might still
-be in the shared memory log segment. To get varnishlog to process the
-whole shared memory log just add the -d option::
-
-   $ varnishlog -d -c -m TxStatus:503
-
-Please see the varnishlog man page for elaborations on further
-filtering capabilities and explanation of the various options.
-
-
-Varnish doesn't cache
-~~~~~~~~~~~~~~~~~~~~~
-
-See :ref:`tutorial-increasing_your_hitrate`.
-
diff --git a/doc/sphinx/tutorial/vary.rst b/doc/sphinx/tutorial/vary.rst
deleted file mode 100644
index ad7b48d..0000000
--- a/doc/sphinx/tutorial/vary.rst
+++ /dev/null
@@ -1,58 +0,0 @@
-.. _tutorial-vary:
-
-Vary
-~~~~
-
-The Vary header is sent by the web server to indicate what makes a
-HTTP object Vary. This makes a lot of sense with headers like
-Accept-Encoding. When a server issues a "Vary: Accept-Encoding" it
-tells Varnish that its needs to cache a separate version for every
-different Accept-Encoding that is coming from the clients. So, if a
-clients only accepts gzip encoding Varnish won't serve the version of
-the page encoded with the deflate encoding.
-
-The problem is that the Accept-Encoding field contains a lot of
-different encodings. If one browser sends::
-
-  Accept-Encoding: gzip,deflate
-
-And another one sends::
-
-  Accept-Encoding: deflate,gzip
-
-Varnish will keep two variants of the page requested due to the
-different Accept-Encoding headers. Normalizing the accept-encoding
-header will sure that you have as few variants as possible. The
-following VCL code will normalize the Accept-Encoding headers::
-
-    if (req.http.Accept-Encoding) {
-        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
-            # No point in compressing these
-            remove req.http.Accept-Encoding;
-        } elsif (req.http.Accept-Encoding ~ "gzip") {
-            set req.http.Accept-Encoding = "gzip";
-        } elsif (req.http.Accept-Encoding ~ "deflate") {
-            set req.http.Accept-Encoding = "deflate";
-        } else {
-            # unknown algorithm
-            remove req.http.Accept-Encoding;
-        }
-    }
-
-The code sets the Accept-Encoding header from the client to either
-gzip, deflate with a preference for gzip.
-
-Pitfall - Vary: User-Agent
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Some applications or application servers send *Vary: User-Agent* along
-with their content. This instructs Varnish to cache a separate copy
-for every variation of User-Agent there is. There are plenty. Even a
-single patchlevel of the same browser will generate at least 10
-different User-Agent headers based just on what operating system they
-are running. 
-
-So if you *really* need to Vary based on User-Agent be sure to
-normalize the header or your hit rate will suffer badly. Use the above
-code as a template.
-
diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
deleted file mode 100644
index 0601468..0000000
--- a/doc/sphinx/tutorial/vcl.rst
+++ /dev/null
@@ -1,200 +0,0 @@
-Varnish Configuration Language - VCL
--------------------------------------
-
-Varnish has a great configuration system. Most other systems use
-configuration directives, where you basically turn on and off lots of
-switches. Varnish uses a domain specific language called Varnish
-Configuration Language, or VCL for short. Varnish translates this
-configuration into binary code which is then executed when requests
-arrive.
-
-The VCL files are divided into subroutines. The different subroutines
-are executed at different times. One is executed when we get the
-request, another when files are fetched from the backend server.
-
-Varnish will execute these subroutines of code at different stages of
-its work. Because it is code it is execute line by line precedence
-isn't a problem. At some point you call an action in this subroutine
-and then the execution of the subroutine stops.
-
-If you don't call an action in your subroutine and it reaches the end
-Varnish will execute some built in VCL code. You will see this VCL
-code commented out in default.vcl.
-
-99% of all the changes you'll need to do will be done in two of these
-subroutines. *vcl_recv* and *vcl_fetch*.
-
-vcl_recv
-~~~~~~~~
-
-vcl_recv (yes, we're skimpy with characters, it's Unix) is called at
-the beginning of a request, after the complete request has been
-received and parsed.  Its purpose is to decide whether or not to serve
-the request, how to do it, and, if applicable, which backend to use.
-
-In vcl_recv you can also alter the request. Typically you can alter
-the cookies and add and remove request headers.
-
-Note that in vcl_recv only the request object, req is available.
-
-vcl_fetch
-~~~~~~~~~
-
-vcl_fetch is called *after* a document has been successfully retrieved
-from the backend. Normal tasks her are to alter the response headers,
-trigger ESI processing, try alternate backend servers in case the
-request failed.
-
-In vcl_fetch you still have the request object, req, available. There
-is also a *backend response*, beresp. beresp will contain the HTTP
-headers from the backend.
-
-.. _tutorial-vcl_fetch_actions:
-
-actions
-~~~~~~~
-
-The most common actions to return are these:
-
-*pass*
- When you return pass the request and subsequent response will be passed to
- and from the backend server. It won't be cached. pass can be returned from
- vcl_recv
-
-*hit_for_pass*
-  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
-  will create a hitforpass object in the cache. This has the side-effect of
-  caching the decision not to cache. This is to allow would-be uncachable
-  requests to be passed to the backend at the same time. The same logic is
-  not necessary in vcl_recv because this happens before any potential
-  queueing for an object takes place.
-
-*lookup*
-  When you return lookup from vcl_recv you tell Varnish to deliver content 
-  from cache even if the request othervise indicates that the request 
-  should be passed. You can't return lookup from vcl_fetch.
-
-*pipe*
-  Pipe can be returned from vcl_recv as well. Pipe short circuits the
-  client and the backend connections and Varnish will just sit there
-  and shuffle bytes back and forth. Varnish will not look at the data being 
-  send back and forth - so your logs will be incomplete. 
-  Beware that with HTTP 1.1 a client can send several requests on the same 
-  connection and so you should instruct Varnish to add a "Connection: close"
-  header before actually returning pipe. 
-
-*deliver*
- Deliver the cached object to the client.  Usually returned from vcl_fetch. 
-
-Requests, responses and objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-In VCL, there are three important data structures. The request, coming
-from the client, the response coming from the backend server and the
-object, stored in cache.
-
-In VCL you should know the following structures.
-
-*req*
- The request object. When Varnish has received the request the req object is 
- created and populated. Most of the work you do in vcl_recv you 
- do on or with the req object.
-
-*beresp*
- The backend respons object. It contains the headers of the object 
- comming from the backend. Most of the work you do in vcl_fetch you 
- do on the beresp object.
-
-*obj*
- The cached object. Mostly a read only object that resides in memory. 
- obj.ttl is writable, the rest is read only.
-
-Operators
-~~~~~~~~~
-
-The following operators are available in VCL. See the examples further
-down for, uhm, examples.
-
-= 
- Assignment operator.
-
-== 
- Comparison.
-
-~
- Match. Can either be used with regular expressions or ACLs.
-
-!
- Negation.
-
-&&
- Logical *and*
-
-||
- Logical *or*
-
-Example 1 - manipulating headers
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Lets say we want to remove the cookie for all objects in the /images
-directory of our web server::
-
-  sub vcl_recv {
-    if (req.url ~ "^/images") {
-      unset req.http.cookie;
-    }
-  }
-
-Now, when the request is handled to the backend server there will be
-no cookie header. The interesting line is the one with the
-if-statement. It matches the URL, taken from the request object, and
-matches it against the regular expression. Note the match operator. If
-it matches the Cookie: header of the request is unset (deleted). 
-
-Example 2 - manipulating beresp
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Here we override the TTL of a object comming from the backend if it
-matches certain criteria::
-
-  sub vcl_fetch {
-     if (req.url ~ "\.(png|gif|jpg)$") {
-       unset beresp.http.set-cookie;
-       set beresp.ttl = 1h;
-    }
-  }
-
-Example 3 - ACLs
-~~~~~~~~~~~~~~~~
-
-You create a named access control list with the *acl* keyword. You can match
-the IP address of the client against an ACL with the match operator.::
-
-  # Who is allowed to purge....
-  acl local {
-      "localhost";
-      "192.168.1.0"/24; /* and everyone on the local network */
-      ! "192.168.1.23"; /* except for the dialin router */
-  }
-  
-  sub vcl_recv {
-    if (req.request == "PURGE") {
-      if (client.ip ~ local) {
-         return(lookup);
-      }
-    } 
-  }
-  
-  sub vcl_hit {
-     if (req.request == "PURGE") {
-       set obj.ttl = 0s;
-       error 200 "Purged.";
-      }
-  }
-
-  sub vcl_miss {
-    if (req.request == "PURGE") {
-      error 404 "Not in cache.";
-    }
-  }
-
diff --git a/doc/sphinx/tutorial/virtualized.rst b/doc/sphinx/tutorial/virtualized.rst
deleted file mode 100644
index 317d3e2..0000000
--- a/doc/sphinx/tutorial/virtualized.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-
-Running Varnish in a virtualized environment
---------------------------------------------
-
-It is possible, but not recommended for high performance, to run
-Varnish on virtualized hardware. Reduced disk- and network performance
-will reduce the performance a bit so make sure your system has good IO
-performance.
-
-OpenVZ
-~~~~~~
-
-If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
-the maximum stack size before starting Varnish. The default allocates
-to much memory per thread, which will make varnish fail as soon as the
-number of threads (==traffic) increases.
-
-Reduce the maximum stack size by running::
-
-    ulimit -s 256
-
-in the startup script.
-
diff --git a/doc/sphinx/tutorial/websockets.rst b/doc/sphinx/tutorial/websockets.rst
deleted file mode 100644
index a74353e..0000000
--- a/doc/sphinx/tutorial/websockets.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-
-Using Websockets 
-----------------
-
-Websockets is a technology for creating a bidirectional stream-based channel over HTTP.
-
-To run websockets through Varnish you need to pipe it, and copy the Upgrade header. Use the following
-VCL config to do so::
-
-    sub vcl_pipe {
-         if (req.http.upgrade) {
-             set bereq.http.upgrade = req.http.upgrade;
-         }
-    }
-    sub vcl_recv {
-         if (req.http.Upgrade ~ "(?i)websocket") {
-             return (pipe);
-         }
-    }
-
diff --git a/doc/sphinx/users-guide/advanced_backend_servers.rst b/doc/sphinx/users-guide/advanced_backend_servers.rst
new file mode 100644
index 0000000..b4206d9
--- /dev/null
+++ b/doc/sphinx/users-guide/advanced_backend_servers.rst
@@ -0,0 +1,157 @@
+Advanced Backend configuration
+------------------------------
+
+At some point you might need Varnish to cache content from several
+servers. You might want Varnish to map all the URL into one single
+host or not. There are lot of options.
+
+Lets say we need to introduce a Java application into out PHP web
+site. Lets say our Java application should handle URL beginning with
+/java/.
+
+We manage to get the thing up and running on port 8000. Now, lets have
+a look a default.vcl.::
+
+  backend default {
+      .host = "127.0.0.1";
+      .port = "8080";
+  }
+
+We add a new backend.::
+
+  backend java {
+      .host = "127.0.0.1";
+      .port = "8000";
+  }
+
+Now we need tell where to send the difference URL. Lets look at vcl_recv.::
+
+  sub vcl_recv {
+      if (req.url ~ "^/java/") {
+          set req.backend = java;
+      } else {
+          set req.backend = default.
+      }
+  }
+
+It's quite simple, really. Lets stop and think about this for a
+moment. As you can see you can define how you choose backends based on
+really arbitrary data. You want to send mobile devices to a different
+backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
+trick. 
+
+.. _tutorial-advanced_backend_servers-directors:
+
+Directors
+---------
+
+You can also group several backend into a group of backends. These
+groups are called directors. This will give you increased performance
+and resilience. You can define several backends and group them
+together in a director.::
+
+	 backend server1 {
+	     .host = "192.168.0.10";
+	 }
+	 backend server2{
+	     .host = "192.168.0.10";
+	 }
+
+Now we create the director.::
+
+       	director example_director round-robin {
+        {
+                .backend = server1;
+        }
+	# server2
+        {
+                .backend = server2;
+        }
+	# foo
+	}
+
+
+This director is a round-robin director. This means the director will
+distribute the incoming requests on a round-robin basis. There is
+also a *random* director which distributes requests in a, you guessed
+it, random fashion.
+
+But what if one of your servers goes down? Can Varnish direct all the
+requests to the healthy server? Sure it can. This is where the Health
+Checks come into play.
+
+.. _tutorial-advanced_backend_servers-health:
+
+Health checks
+-------------
+
+Lets set up a director with two backends and health checks. First lets
+define the backends.::
+
+       backend server1 {
+         .host = "server1.example.com";
+	 .probe = {
+                .url = "/";
+                .interval = 5s;
+                .timeout = 1 s;
+                .window = 5;
+                .threshold = 3;
+	   }
+         }
+       backend server2 {
+  	  .host = "server2.example.com";
+  	  .probe = {
+                .url = "/";
+                .interval = 5s;
+                .timeout = 1 s;
+                .window = 5;
+                .threshold = 3;
+	  }
+        }
+
+Whats new here is the probe. Varnish will check the health of each
+backend with a probe. The options are
+
+url
+ What URL should varnish request.
+
+interval
+ How often should we poll
+
+timeout
+ What is the timeout of the probe
+
+window
+ Varnish will maintain a *sliding window* of the results. Here the
+ window has five checks.
+
+threshold 
+ How many of the .window last polls must be good for the backend to be declared healthy.
+
+initial 
+ How many of the of the probes a good when Varnish starts - defaults
+ to the same amount as the threshold.
+
+Now we define the director.::
+
+  director example_director round-robin {
+        {
+                .backend = server1;
+        }
+        # server2 
+        {
+                .backend = server2;
+        }
+	
+        }
+
+You use this director just as you would use any other director or
+backend. Varnish will not send traffic to hosts that are marked as
+unhealthy. Varnish can also serve stale content if all the backends are
+down. See :ref:`tutorial-handling_misbehaving_servers` for more
+information on how to enable this.
+
+Please note that Varnish will keep probes active for all loaded
+VCLs. Varnish will coalesce probes that seem identical - so be careful
+not to change the probe config if you do a lot of VCL
+loading. Unloading the VCL will discard the probes.
diff --git a/doc/sphinx/users-guide/advanced_topics.rst b/doc/sphinx/users-guide/advanced_topics.rst
new file mode 100644
index 0000000..1045de9
--- /dev/null
+++ b/doc/sphinx/users-guide/advanced_topics.rst
@@ -0,0 +1,63 @@
+.. _tutorial-advanced_topics:
+
+Advanced topics
+---------------
+
+This tutorial has covered the basics in Varnish. If you read through
+it all you should now have the skills to run Varnish.
+
+Here is a short overview of topics that we haven't covered in the tutorial. 
+
+More VCL
+~~~~~~~~
+
+VCL is a bit more complex then what we've covered so far. There are a
+few more subroutines available and there a few actions that we haven't
+discussed. For a complete(ish) guide to VCL have a look at the VCL man
+page - ref:`reference-vcl`.
+
+Using In-line C to extend Varnish
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can use *in-line C* to extend Varnish. Please note that you can
+seriously mess up Varnish this way. The C code runs within the Varnish
+Cache process so if your code generates a segfault the cache will crash.
+
+One of the first uses I saw of In-line C was logging to syslog.::
+
+	# The include statements must be outside the subroutines.
+	C{
+		#include <syslog.h>
+        }C
+	
+        sub vcl_something {
+                C{
+		        syslog(LOG_INFO, "Something happened at VCL line XX.");
+	        }C
+        }
+
+
+Edge Side Includes
+~~~~~~~~~~~~~~~~~~
+
+Varnish can cache create web pages by putting different pages
+together. These *fragments* can have individual cache policies. If you
+have a web site with a list showing the 5 most popular articles on
+your site, this list can probably be cached as a fragment and included
+in all the other pages. Used properly it can dramatically increase
+your hit rate and reduce the load on your servers. ESI looks like this::
+
+  <HTML>
+  <BODY>
+  The time is: <esi:include src="/cgi-bin/date.cgi"/>
+  at this very moment.
+  </BODY>
+  </HTML>
+
+ESI is processed in vcl_fetch by setting *do_esi* to true.::
+
+  sub vcl_fetch {
+      if (req.url == "/test.html") {
+	  set beresp.do_esi = true;  /* Do ESI processing */
+      }
+  }
diff --git a/doc/sphinx/users-guide/backend_servers.rst b/doc/sphinx/users-guide/backend_servers.rst
new file mode 100644
index 0000000..1b1aaf2
--- /dev/null
+++ b/doc/sphinx/users-guide/backend_servers.rst
@@ -0,0 +1,39 @@
+.. _tutorial-backend_servers:
+
+Backend servers
+---------------
+
+Varnish has a concept of "backend" or "origin" servers. A backend
+server is the server providing the content Varnish will accelerate.
+
+Our first task is to tell Varnish where it can find its content. Start
+your favorite text editor and open the varnish default configuration
+file. If you installed from source this is
+/usr/local/etc/varnish/default.vcl, if you installed from a package it
+is probably /etc/varnish/default.vcl.
+
+Somewhere in the top there will be a section that looks a bit like this.::
+
+	  # backend default {
+	  #     .host = "127.0.0.1";
+	  #     .port = "8080";
+	  # }
+
+We comment in this bit of text and change the port setting from 8080
+to 80, making the text look like.::
+
+          backend default {
+                .host = "127.0.0.1";
+    		.port = "80";
+	  }
+
+Now, this piece of configuration defines a backend in Varnish called
+*default*. When Varnish needs to get content from this backend it will
+connect to port 80 on localhost (127.0.0.1).
+
+Varnish can have several backends defined and can you can even join
+several backends together into clusters of backends for load balancing
+purposes. 
+
+Now that we have the basic Varnish configuration done, let us start up
+Varnish on port 8080 so we can do some fundamental testing on it.
diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
new file mode 100644
index 0000000..0b8d1e8
--- /dev/null
+++ b/doc/sphinx/users-guide/compression.rst
@@ -0,0 +1,75 @@
+.. _tutorial-compression:
+
+Compression
+~~~~~~~~~~~
+
+New in Varnish 3.0 was native support for compression, using gzip
+encoding. *Before* 3.0, Varnish would never compress objects. 
+
+In Varnish 3.0 compression defaults to "on", meaning that it tries to
+be smart and do the sensible thing.
+
+If you don't want Varnish tampering with the encoding you can disable
+compression all together by setting the parameter http_gzip_support to
+*false*. Please see man :ref:`ref-varnishd` for details.
+
+
+Default behaviour
+~~~~~~~~~~~~~~~~~
+
+The default for Varnish is to check if the client supports our
+compression scheme (gzip) and if it does it will override the
+Accept-Encoding header and set it to "gzip".
+
+When Varnish then issues a backend request the Accept-Encoding will
+then only consist of "gzip". If the server responds with gzip'ed
+content it will be stored in memory in its compressed form. If the
+backend sends content in clear text it will be stored like that.
+
+You can make Varnish compress content before storing it in cache in
+vcl_fetch by setting do_gzip to true, like this::
+
+   sub vcl_fetch {
+        if (beresp.http.content-type ~ "text") {
+                set beresp.do_gzip = true;
+        }
+  }
+
+Please make sure that you don't try to compress content that is
+incompressable, like jpgs, gifs and mp3. You'll only waste CPU
+cycles. You can also uncompress objects before storing it in memory by
+setting do_gunzip to *true* but I have no idea why anybody would want
+to do that.
+
+Generally, Varnish doesn't use much CPU so it might make more sense to
+have Varnish spend CPU cycles compressing content than doing it in
+your web- or application servers, which are more likely to be
+CPU-bound.
+
+GZIP and ESI
+~~~~~~~~~~~~
+
+If you are using Edge Side Includes you'll be happy to note that ESI
+and GZIP work together really well. Varnish will magically decompress
+the content to do the ESI-processing, then recompress it for efficient
+storage and delivery. 
+
+
+Clients that don't support gzip
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the client does not support gzip the Accept-Encoding header is left
+alone and we'll end up serving whatever we get from the backend
+server. Remember that the Backend might tell Varnish to *Vary* on the
+Accept-Encoding.
+
+If the client does not support gzip but we've already got a compressed
+version of the page in memory Varnish will automatically decompress
+the page while delivering it.
+
+
+A random outburst
+~~~~~~~~~~~~~~~~~
+
+Poul has written :ref:`phk_gzip` which talks abit more about how the
+implementation works. 
diff --git a/doc/sphinx/users-guide/cookies.rst b/doc/sphinx/users-guide/cookies.rst
new file mode 100644
index 0000000..c75171f
--- /dev/null
+++ b/doc/sphinx/users-guide/cookies.rst
@@ -0,0 +1,95 @@
+.. _tutorial-cookies:
+
+Cookies
+-------
+
+Varnish will, in the default configuration, not cache a object coming
+from the backend with a Set-Cookie header present. Also, if the client
+sends a Cookie header, Varnish will bypass the cache and go directly to
+the backend.
+
+This can be overly conservative. A lot of sites use Google Analytics
+(GA) to analyze their traffic. GA sets a cookie to track you. This
+cookie is used by the client side javascript and is therefore of no
+interest to the server. 
+
+Cookies from the client
+~~~~~~~~~~~~~~~~~~~~~~~
+
+For a lot of web application it makes sense to completely disregard the
+cookies unless you are accessing a special part of the web site. This
+VCL snippet in vcl_recv will disregard cookies unless you are
+accessing /admin/::
+
+  if ( !( req.url ~ ^/admin/) ) {
+    unset req.http.Cookie;
+  }
+
+Quite simple. If, however, you need to do something more complicated,
+like removing one out of several cookies, things get
+difficult. Unfortunately Varnish doesn't have good tools for
+manipulating the Cookies. We have to use regular expressions to do the
+work. If you are familiar with regular expressions you'll understand
+whats going on. If you don't I suggest you either pick up a book on
+the subject, read through the *pcrepattern* man page or read through
+one of many online guides.
+
+Let me show you what Varnish Software uses. We use some cookies for
+Google Analytics tracking and similar tools. The cookies are all set
+and used by Javascript. Varnish and Drupal doesn't need to see those
+cookies and since Varnish will cease caching of pages when the client
+sends cookies we will discard these unnecessary cookies in VCL. 
+
+In the following VCL we discard all cookies that start with a
+underscore::
+
+  // Remove has_js and Google Analytics __* cookies.
+  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
+  // Remove a ";" prefix, if present.
+  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
+
+Let me show you an example where we remove everything except the
+cookies named COOKIE1 and COOKIE2 and you can marvel at it::
+
+  sub vcl_recv {
+    if (req.http.Cookie) {
+      set req.http.Cookie = ";" + req.http.Cookie;
+      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
+      set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
+      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
+      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
+
+      if (req.http.Cookie == "") {
+          remove req.http.Cookie;
+      }
+    }
+  }
+
+A somewhat simpler example that can accomplish almost the same can be
+found below. Instead of filtering out the other cookies it picks out
+the one cookie that is needed, copies it to another header and then
+copies it back, deleting the original cookie header.::
+
+  sub vcl_recv {
+         # save the original cookie header so we can mangle it
+        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
+        # using a capturing sub pattern, extract the continuous string of 
+        # alphanumerics that immediately follows "PHPSESSID="
+        set req.http.X-Varnish-PHP_SID = 
+           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
+        set req.http.Cookie = req.X-Varnish-PHP_SID;
+        remove req.X-Varnish-PHP_SID;
+   }   
+
+There are other scary examples of what can be done in VCL in the
+Varnish Cache Wiki.
+
+
+Cookies coming from the backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your backend server sets a cookie using the Set-Cookie header
+Varnish will not cache the page in the default configuration.  A
+hit-for-pass object (see :ref:`tutorial-vcl_fetch_actions`) is created.
+So, if the backend server acts silly and sets unwanted cookies just unset
+the Set-Cookie header and all should be fine. 
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
new file mode 100644
index 0000000..6bfc4c9
--- /dev/null
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -0,0 +1,268 @@
+.. _tutorial-devicedetect:
+
+Device detection
+~~~~~~~~~~~~~~~~
+
+Device detection is figuring out what kind of content to serve to a
+client based on the User-Agent string supplied in a request.
+
+Use cases for this are for example to send size reduced files to mobile
+clients with small screens and on high latency networks, or to 
+provide a streaming video codec that the client understands.
+
+There are a couple of strategies on what to do with such clients:
+1) Redirect them to another URL.
+2) Use a different backend for the special clients.
+3) Change the backend requests so the usual backend sends tailored content.
+
+To make the examples easier to understand, it is assumed in this text 
+that all the req.http.X-UA-Device header is present and unique per client class
+that content is to be served to. 
+
+Setting this header can be as simple as::
+
+   sub vcl_recv {?
+       if (req.http.User-Agent ~ "(?i)iphone"?{
+           set req.http.X-UA-Device = "mobile-iphone";
+       }
+   }
+
+There are different commercial and free offerings in doing grouping and
+identifiying clients in further detail than this. For a basic and community
+based regular expression set, see
+https://github.com/varnish/varnish-devicedetect/ .
+
+
+Serve the different content on the same URL
+-------------------------------------------
+
+The tricks involved are: 
+1. Detect the client (pretty simple, just include devicedetect.vcl and call
+it)
+2. Figure out how to signal the backend what client class this is. This
+includes for example setting a header, changing a header or even changing the
+backend request URL.
+3. Modify any response from the backend to add missing Vary headers, so
+Varnish' internal handling of this kicks in.
+4. Modify output sent to the client so any caches outside our control don't
+serve the wrong content.
+
+All this while still making sure that we only get 1 cached object per URL per
+device class.
+
+
+Example 1: Send HTTP header to backend
+''''''''''''''''''''''''''''''''''''''
+
+The basic case is that Varnish adds the X-UA-Device HTTP header on the backend
+requests, and the backend mentions in the response Vary header that the content
+is dependant on this header. 
+
+Everything works out of the box from Varnish' perspective.
+
+.. 071-example1-start
+
+VCL::
+
+    sub vcl_recv { 
+        # call some detection engine that set req.http.X-UA-Device
+    }
+    # req.http.X-UA-Device is copied by Varnish into bereq.http.X-UA-Device
+
+    # so, this is a bit conterintuitive. The backend creates content based on
+    # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will
+    # use the same cached object for all U-As that map to the same X-UA-Device.
+    #
+    # If the backend does not mention in Vary that it has crafted special
+    # content based on the User-Agent (==X-UA-Device), add it. 
+    # If your backend does set Vary: User-Agent, you may have to remove that here.
+    sub vcl_fetch {
+        if (req.http.X-UA-Device) {
+            if (!beresp.http.Vary) { # no Vary at all
+                set beresp.http.Vary = "X-UA-Device"; 
+            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
+                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; 
+            } 
+        }
+        # comment this out if you don't want the client to know your
+        # classification
+        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+    }
+
+    # to keep any caches in the wild from serving wrong content to client #2
+    # behind them, we need to transform the Vary on the way out.
+    sub vcl_deliver {
+        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
+            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
+        }
+    }
+
+.. 071-example1-end
+
+Example 2: Normalize the User-Agent string
+''''''''''''''''''''''''''''''''''''''''''
+
+Another way of signaling the device type is to override or normalize the
+User-Agent header sent to the backend.
+
+For example
+
+    User-Agent: Mozilla/5.0 (Linux; U; Android 2.2; nb-no; HTC Desire Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1
+
+becomes:
+
+    User-Agent: mobile-android
+
+when seen by the backend.
+
+This works if you don't need the original header for anything on the backend.
+A possible use for this is for CGI scripts where only a small set of predefined
+headers are (by default) available for the script.
+
+.. 072-example2-start
+
+VCL::
+
+    sub vcl_recv { 
+        # call some detection engine that set req.http.X-UA-Device
+    }
+
+    # override the header before it is sent to the backend
+    sub vcl_miss { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
+    sub vcl_pass { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
+
+    # standard Vary handling code from previous examples.
+    sub vcl_fetch {
+        if (req.http.X-UA-Device) {
+            if (!beresp.http.Vary) { # no Vary at all
+                set beresp.http.Vary = "X-UA-Device";
+            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
+                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
+            }
+        }
+        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+    }
+    sub vcl_deliver {
+        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
+            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
+        }
+    }
+
+.. 072-example2-end
+
+Example 3: Add the device class as a GET query parameter
+''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+If everything else fails, you can add the device type as a GET argument.
+
+    http://example.com/article/1234.html --> http://example.com/article/1234.html?devicetype=mobile-iphone
+
+The client itself does not see this classification, only the backend request
+is changed.
+
+.. 073-example3-start
+
+VCL::
+
+    sub vcl_recv { 
+        # call some detection engine that set req.http.X-UA-Device
+    }
+
+    sub append_ua {
+        if ((req.http.X-UA-Device) && (req.request == "GET")) {
+            # if there are existing GET arguments;
+            if (req.url ~ "\?") {
+                set req.http.X-get-devicetype = "&devicetype=" + req.http.X-UA-Device;
+            } else { 
+                set req.http.X-get-devicetype = "?devicetype=" + req.http.X-UA-Device;
+            }
+            set req.url = req.url + req.http.X-get-devicetype;
+            unset req.http.X-get-devicetype;
+        }
+    }
+
+    # do this after vcl_hash, so all Vary-ants can be purged in one go. (avoid ban()ing)
+    sub vcl_miss { call append_ua; }
+    sub vcl_pass { call append_ua; }
+
+    # Handle redirects, otherwise standard Vary handling code from previous
+    # examples.
+    sub vcl_fetch {
+        if (req.http.X-UA-Device) {
+            if (!beresp.http.Vary) { # no Vary at all
+                set beresp.http.Vary = "X-UA-Device";
+            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
+                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
+            }
+
+            # if the backend returns a redirect (think missing trailing slash),
+            # we will potentially show the extra address to the client. we
+            # don't want that.  if the backend reorders the get parameters, you
+            # may need to be smarter here. (? and & ordering)
+
+            if (beresp.status == 301 || beresp.status == 302 || beresp.status == 303) {
+                set beresp.http.location = regsub(beresp.http.location, "[?&]devicetype=.*$", "");
+            }
+        }
+        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+    }
+    sub vcl_deliver {
+        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
+            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
+        }
+    }
+
+.. 073-example3-end
+
+Different backend for mobile clients
+------------------------------------
+
+If you have a different backend that serves pages for mobile clients, or any
+special needs in VCL, you can use the X-UA-Device header like this::
+
+    backend mobile {
+        .host = "10.0.0.1";
+        .port = "80";
+    }
+
+    sub vcl_recv {
+        # call some detection engine
+
+        if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
+            set req.backend = mobile;
+        }
+    }
+    sub vcl_hash {
+        if (req.http.X-UA-Device) {
+            hash_data(req.http.X-UA-Device);
+        }
+    }
+
+Redirecting mobile clients
+--------------------------
+
+If you want to redirect mobile clients you can use the following snippet.
+
+.. 065-redir-mobile-start
+
+VCL::
+
+    sub vcl_recv {
+        # call some detection engine
+
+        if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
+            error 750 "Moved Temporarily";
+        }
+    }
+     
+    sub vcl_error {
+        if (obj.status == 750) {
+            set obj.http.Location = "http://m.example.com" + req.url;
+            set obj.status = 302;
+            return(deliver);
+        }
+    }
+
+.. 065-redir-mobile-end
+
+
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
new file mode 100644
index 0000000..720b790
--- /dev/null
+++ b/doc/sphinx/users-guide/esi.rst
@@ -0,0 +1,79 @@
+.. _tutorial-esi:
+
+Edge Side Includes
+------------------
+
+*Edge Side Includes* is a language to include *fragments* of web pages
+in other web pages. Think of it as HTML include statement that works
+over HTTP. 
+
+On most web sites a lot of content is shared between
+pages. Regenerating this content for every page view is wasteful and
+ESI tries to address that letting you decide the cache policy for
+each fragment individually.
+
+In Varnish we've only implemented a small subset of ESI. As of 2.1 we
+have three ESI statements:
+
+ * esi:include 
+ * esi:remove
+ * <!--esi ...-->
+
+Content substitution based on variables and cookies is not implemented
+but is on the roadmap. 
+
+Varnish will not process ESI instructions in HTML comments.
+
+Example: esi:include
+~~~~~~~~~~~~~~~~~~~~
+
+Lets see an example how this could be used. This simple cgi script
+outputs the date::
+
+     #!/bin/sh
+     
+     echo 'Content-type: text/html'
+     echo ''
+     date "+%Y-%m-%d %H:%M"
+
+Now, lets have an HTML file that has an ESI include statement::
+
+     <HTML>
+     <BODY>
+     The time is: <esi:include src="/cgi-bin/date.cgi"/>
+     at this very moment.
+     </BODY>
+     </HTML>
+
+For ESI to work you need to activate ESI processing in VCL, like this::
+
+    sub vcl_fetch {
+    	if (req.url == "/test.html") {
+           set beresp.do_esi = true; /* Do ESI processing		*/
+           set beresp.ttl = 24 h;    /* Sets the TTL on the HTML above  */
+    	} elseif (req.url == "/cgi-bin/date.cgi") {
+           set beresp.ttl = 1m;      /* Sets a one minute TTL on	*/
+	       	       	 	     /*  the included object		*/
+        }
+    }
+
+Example: esi:remove and <!--esi ... -->
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The <esi:remove> and <!--esi ... --> constructs can be used to present
+appropriate content whether or not ESI is available, for example you can
+include content when ESI is available or link to it when it is not.
+ESI processors will remove the start ("<!--esi") and end ("-->") when
+the page is processed, while still processing the contents. If the page
+is not processed, it will remain, becoming an HTML/XML comment tag.
+ESI processors will remove <esi:remove> tags and all content contained
+in them, allowing you to only render the content when the page is not
+being ESI-processed.
+For example::
+
+  <esi:remove> 
+    <a href="http://www.example.com/LICENSE">The license</a>
+  </esi:remove>
+  <!--esi  
+  <p>The full text of the license:</p>
+  <esi:include src="http://example.com/LICENSE" />
+  -->
diff --git a/doc/sphinx/users-guide/handling_misbehaving_servers.rst b/doc/sphinx/users-guide/handling_misbehaving_servers.rst
new file mode 100644
index 0000000..406b4b3
--- /dev/null
+++ b/doc/sphinx/users-guide/handling_misbehaving_servers.rst
@@ -0,0 +1,103 @@
+.. _tutorial-handling_misbehaving_servers:
+
+Misbehaving servers
+-------------------
+
+A key feature of Varnish is its ability to shield you from misbehaving
+web- and application servers.
+
+
+
+Grace mode
+~~~~~~~~~~
+
+When several clients are requesting the same page Varnish will send
+one request to the backend and place the others on hold while fetching
+one copy from the back end. In some products this is called request
+coalescing and Varnish does this automatically.
+
+If you are serving thousands of hits per second the queue of waiting
+requests can get huge. There are two potential problems - one is a
+thundering herd problem - suddenly releasing a thousand threads to
+serve content might send the load sky high. Secondly - nobody likes to
+wait. To deal with this we can instruct Varnish to keep
+the objects in cache beyond their TTL and to serve the waiting
+requests somewhat stale content.
+
+So, in order to serve stale content we must first have some content to
+serve. So to make Varnish keep all objects for 30 minutes beyond their
+TTL use the following VCL::
+
+  sub vcl_fetch {
+    set beresp.grace = 30m;
+  }
+
+Varnish still won't serve the stale objects. In order to enable
+Varnish to actually serve the stale object we must enable this on the
+request. Lets us say that we accept serving 15s old object.::
+
+  sub vcl_recv {
+    set req.grace = 15s;
+  }
+
+You might wonder why we should keep the objects in the cache for 30
+minutes if we are unable to serve them? Well, if you have enabled
+:ref:`tutorial-advanced_backend_servers-health` you can check if the
+backend is sick and if it is we can serve the stale content for a bit
+longer.::
+
+   if (! req.backend.healthy) {
+      set req.grace = 5m;
+   } else {
+      set req.grace = 15s;
+   }
+
+So, to sum up, grace mode solves two problems:
+ * it serves stale content to avoid request pile-up.
+ * it serves stale content if the backend is not healthy.
+
+Saint mode
+~~~~~~~~~~
+
+Sometimes servers get flaky. They start throwing out random
+errors. You can instruct Varnish to try to handle this in a
+more-than-graceful way - enter *Saint mode*. Saint mode enables you to
+discard a certain page from one backend server and either try another
+server or serve stale content from cache. Lets have a look at how this
+can be enabled in VCL::
+
+  sub vcl_fetch {
+    if (beresp.status == 500) { 
+      set beresp.saintmode = 10s;
+      return(restart);
+    }
+    set beresp.grace = 5m;
+  } 
+
+When we set beresp.saintmode to 10 seconds Varnish will not ask *that*
+server for URL for 10 seconds. A blacklist, more or less. Also a
+restart is performed so if you have other backends capable of serving
+that content Varnish will try those. When you are out of backends
+Varnish will serve the content from its stale cache.
+
+This can really be a life saver.
+
+Known limitations on grace- and saint mode
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your request fails while it is being fetched you're thrown into
+vcl_error. vcl_error has access to a rather limited set of data so you
+can't enable saint mode or grace mode here. This will be addressed in a
+future release but a work-around available.
+
+* Declare a backend that is always sick.
+* Set a magic marker in vcl_error
+* Restart the transaction
+* Note the magic marker in vcl_recv and set the backend to the one mentioned
+* Varnish will now serve stale data is any is available
+
+
+God mode
+~~~~~~~~
+Not implemented yet. :-)
+
diff --git a/doc/sphinx/users-guide/increasing_your_hitrate.rst b/doc/sphinx/users-guide/increasing_your_hitrate.rst
new file mode 100644
index 0000000..b9fa7e6
--- /dev/null
+++ b/doc/sphinx/users-guide/increasing_your_hitrate.rst
@@ -0,0 +1,213 @@
+.. _tutorial-increasing_your_hitrate:
+
+Achieving a high hitrate
+------------------------
+
+Now that Varnish is up and running, and you can access your web
+application through Varnish. Unless your application is specifically
+written to work behind a web accelerator you'll probably need to do
+some changes to either the configuration or the application in order
+to get a high hit rate in Varnish.
+
+Varnish will not cache your data unless it's absolutely sure it is
+safe to do so. So, for you to understand how Varnish decides if and
+how to cache a page, I'll guide you through a couple of tools that you
+will find useful.
+
+Note that you need a tool to see what HTTP headers fly between you and
+the web server. On the Varnish server, the easiest is to use
+varnishlog and varnishtop but sometimes a client-side tool makes
+sense. Here are the ones I use.
+
+Tool: varnishtop
+~~~~~~~~~~~~~~~~
+
+You can use varnishtop to identify what URLs are hitting the backend
+the most. ``varnishtop -i txurl`` is an essential command. You can see
+some other examples of varnishtop usage in :ref:`tutorial-statistics`.
+
+
+Tool: varnishlog
+~~~~~~~~~~~~~~~~
+
+When you have identified the an URL which is frequently sent to the
+backend you can use varnishlog to have a look at the request.
+``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
+coming from the client (-c) matching /foo/bar.
+
+For more information on how varnishlog works please see
+:ref:`tutorial-logging` or man :ref:`ref-varnishlog`.
+
+For extended diagnostics headers, see
+http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
+
+
+Tool: lwp-request
+~~~~~~~~~~~~~~~~~
+
+lwp-request is part of The World-Wide Web library for Perl. It's a
+couple of really basic programs that can execute an HTTP request and
+give you the result. I mostly use two programs, GET and HEAD.
+
+vg.no was the first site to use Varnish and the people running Varnish
+there are quite clueful. So it's interesting to look at their HTTP
+Headers. Let's send a GET request for their home page::
+
+  $ GET -H 'Host: www.vg.no' -Used http://vg.no/
+  GET http://vg.no/
+  Host: www.vg.no
+  User-Agent: lwp-request/5.834 libwww-perl/5.834
+  
+  200 OK
+  Cache-Control: must-revalidate
+  Refresh: 600
+  Title: VG Nett - Forsiden - VG Nett
+  X-Age: 463
+  X-Cache: HIT
+  X-Rick-Would-Never: Let you down
+  X-VG-Jobb: http://www.finn.no/finn/job/fulltime/result?keyword=vg+multimedia Merk:HeaderNinja
+  X-VG-Korken: http://www.youtube.com/watch?v=Fcj8CnD5188
+  X-VG-WebCache: joanie
+  X-VG-WebServer: leon
+
+OK. Let me explain what it does. GET usually sends off HTTP 0.9
+requests, which lack the Host header. So I add a Host header with the
+-H option. -U print request headers, -s prints response status, -e
+prints response headers and -d discards the actual content. We don't
+really care about the content, only the headers.
+
+As you can see, VG adds quite a bit of information in their
+headers. Some of the headers, like the X-Rick-Would-Never are specific
+to vg.no and their somewhat odd sense of humour. Others, like the
+X-VG-Webcache are for debugging purposes. 
+
+So, to check whether a site sets cookies for a specific URL, just do::
+
+  GET -Used http://example.com/ |grep ^Set-Cookie
+
+Tool: Live HTTP Headers
+~~~~~~~~~~~~~~~~~~~~~~~
+
+There is also a plugin for Firefox. *Live HTTP Headers* can show you
+what headers are being sent and recieved. Live HTTP Headers can be
+found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
+googling "Live HTTP Headers".
+
+
+The role of HTTP Headers
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Along with each HTTP request and response comes a bunch of headers
+carrying metadata. Varnish will look at these headers to determine if
+it is appropriate to cache the contents and how long Varnish can keep
+the content.
+
+Please note that when considering these headers Varnish actually
+considers itself *part of* the actual webserver. The rationale being
+that both are under your control. 
+
+The term *surrogate origin cache* is not really well defined by the
+IETF so RFC 2616 so the various ways Varnish works might differ from
+your expectations.
+
+Let's take a look at the important headers you should be aware of:
+
+Cache-Control
+~~~~~~~~~~~~~
+
+The Cache-Control instructs caches how to handle the content. Varnish
+cares about the *max-age* parameter and uses it to calculate the TTL
+for an object. 
+
+"Cache-Control: nocache" is ignored but if you need this you can
+easily add support for it.
+
+So make sure you issue a Cache-Control header with a max-age
+header. You can have a look at what Varnish Software's drupal server
+issues::
+
+  $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control
+  Cache-Control: public, max-age=600
+
+Age
+~~~
+
+Varnish adds an Age header to indicate how long the object has been
+kept inside Varnish. You can grep out Age from varnishlog like this::
+
+  varnishlog -i TxHeader -I ^Age
+
+Pragma
+~~~~~~
+
+An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
+header. You could easily add support for this header in VCL.
+
+In vcl_fetch::
+
+  if (beresp.http.Pragma ~ "nocache") {
+     return(hit_for_pass);
+  }
+
+Authorization
+~~~~~~~~~~~~~
+
+If Varnish sees an Authorization header it will pass the request. If
+this is not what you want you can unset the header.
+
+Overriding the time-to-live (ttl)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes your backend will misbehave. It might, depending on your
+setup, be easier to override the ttl in Varnish than to fix your
+somewhat cumbersome backend. 
+
+You need VCL to identify the objects you want and then you set the
+beresp.ttl to whatever you want::
+
+  sub vcl_fetch {
+      if (req.url ~ "^/legacy_broken_cms/") {
+          set beresp.ttl = 5d;
+      }
+  }
+
+The example will set the TTL to 5 days for the old legacy stuff on
+your site.
+
+Forcing caching for certain requests and certain responses
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since you still have this cumbersome backend that isn't very friendly
+to work with you might want to override more stuff in Varnish. We
+recommend that you rely as much as you can on the default caching
+rules. It is perfectly easy to force Varnish to lookup an object in
+the cache but it isn't really recommended.
+
+
+Normalizing your namespace
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some sites are accessed via lots of
+hostnames. http://www.varnish-software.com/,
+http://varnish-software.com/ and http://varnishsoftware.com/ all point
+at the same site. Since Varnish doesn't know they are different,
+Varnish will cache different versions of every page for every
+hostname. You can mitigate this in your web server configuration by
+setting up redirects or by using the following VCL::
+
+  if (req.http.host ~ "(?i)^(www.)?varnish-?software.com") {
+    set req.http.host = "varnish-software.com";
+  }
+
+
+Ways of increasing your hitrate even more
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following chapters should give your ways of further increasing
+your hitrate, especially the chapter on Cookies.
+
+ * :ref:`tutorial-cookies`
+ * :ref:`tutorial-vary`
+ * :ref:`tutorial-purging`
+ * :ref:`tutorial-esi`
+
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
new file mode 100644
index 0000000..6cf20a2
--- /dev/null
+++ b/doc/sphinx/users-guide/index.rst
@@ -0,0 +1,39 @@
+.. _users-guide-index:
+
+%%%%%%%%%%%%%
+Using Varnish
+%%%%%%%%%%%%%
+
+This guide is intended for system administrators managing Varnish
+Cache. The reader should know how to configure her web- or application
+server and have basic knowledge of the HTTP protocol. The reader
+should have Varnish Cache up and running with the default
+configuration.
+
+The tutorial is split into short chapters, each chapter taking on a
+separate topic. Good luck.
+
+.. toctree:: :maxdepth: 1
+
+        introduction
+	backend_servers
+        starting_varnish
+	logging
+        sizing_your_cache
+        putting_varnish_on_port_80
+	vcl
+        statistics
+        increasing_your_hitrate
+	cookies
+	vary
+	purging
+	compression
+	esi
+	virtualized
+	websockets
+	devicedetection
+	advanced_backend_servers
+        handling_misbehaving_servers
+        advanced_topics
+	troubleshooting
+
diff --git a/doc/sphinx/users-guide/introduction.rst b/doc/sphinx/users-guide/introduction.rst
new file mode 100644
index 0000000..0d43623
--- /dev/null
+++ b/doc/sphinx/users-guide/introduction.rst
@@ -0,0 +1,37 @@
+.. _tutorial-intro:
+
+What is Varnish?
+----------------
+
+Varnish Cache is a web application accelerator also known as a caching
+HTTP reverse proxy. You install it in front of any server that speaks
+HTTP and configure it to cache the contents. Varnish Cache is really,
+really fast. It typically speeds up delivery with a factor of 300 -
+1000x, depending on your architecture.
+
+
+Performance
+~~~~~~~~~~~
+
+Varnish performs really, really well. It is usually bound by the speed
+of the network, effectivly turning performance into a non-issue. We've
+seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
+
+Flexibility
+~~~~~~~~~~~
+
+One of the key features of Varnish Cache, in addition to it's
+performance, is the flexibility of it's configuration language,
+VCL. VCL enables you to write policies on how incoming requests should
+be handled. In such a policy you can decide what content you want to
+serve, from where you want to get the content and how the request or
+response should be altered. You can read more about this in our
+tutorial.
+
+
+Supported plattforms
+~~~~~~~~~~~~~~~~~~~~
+
+Varnish is written to run on modern versions of Linux and FreeBSD and
+the best experience is had on those plattforms. Thanks to our
+contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/users-guide/logging.rst b/doc/sphinx/users-guide/logging.rst
new file mode 100644
index 0000000..1f0bc18
--- /dev/null
+++ b/doc/sphinx/users-guide/logging.rst
@@ -0,0 +1,68 @@
+.. _tutorial-logging:
+
+Logging in Varnish
+------------------
+
+One of the really nice features in Varnish is how logging
+works. Instead of logging to normal log file Varnish logs to a shared
+memory segment. When the end of the segment is reached we start over,
+overwriting old data. This is much, much faster then logging to a file
+and it doesn't require disk space.
+
+The flip side is that if you forget to have a program actually write the
+logs to disk they will disappear.
+
+varnishlog is one of the programs you can use to look at what Varnish
+is logging. Varnishlog gives you the raw logs, everything that is
+written to the logs. There are other clients as well, we'll show you
+these later.
+
+In the terminal window you started varnish now type *varnishlog* and
+press enter.
+
+You'll see lines like these scrolling slowly by.::
+
+    0 CLI          - Rd ping
+    0 CLI          - Wr 200 PONG 1273698726 1.0
+
+These is the Varnish master process checking up on the caching process
+to see that everything is OK. 
+
+Now go to the browser and reload the page displaying your web
+app. You'll see lines like these.::
+
+   11 SessionOpen  c 127.0.0.1 58912 0.0.0.0:8080
+   11 ReqStart     c 127.0.0.1 58912 595005213
+   11 RxRequest    c GET
+   11 RxURL        c /
+   11 RxProtocol   c HTTP/1.1
+   11 RxHeader     c Host: localhost:8080
+   11 RxHeader     c Connection: keep-alive
+
+The first column is an arbitrary number, it defines the request. Lines
+with the same number are part of the same HTTP transaction. The second
+column is the *tag* of the log message. All log entries are tagged
+with a tag indicating what sort of activity is being logged. Tags
+starting with Rx indicate Varnish is recieving data and Tx indicates
+sending data.
+
+The third column tell us whether this is is data coming or going to
+the client (c) or to/from the backend (b). The forth column is the
+data being logged.
+
+Now, you can filter quite a bit with varnishlog. The basic option you
+want to know are:
+
+-b
+ Only show log lines from traffic going between Varnish and the backend 
+ servers. This will be useful when we want to optimize cache hit rates.
+
+-c 
+ Same as -b but for client side traffic.
+
+-m tag:regex
+ Only list transactions where the tag matches a regular expression. If
+ it matches you will get the whole transaction.
+
+Now that Varnish seem to work OK it's time to put Varnish on port 80
+while we tune it.
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
new file mode 100644
index 0000000..422f9f4
--- /dev/null
+++ b/doc/sphinx/users-guide/purging.rst
@@ -0,0 +1,175 @@
+.. _tutorial-purging:
+
+=====================
+ Purging and banning
+=====================
+
+One of the most effective ways of increasing your hit ratio is to
+increase the time-to-live (ttl) of your objects. But, as you're aware
+of, in this twitterific day of age serving content that is outdated is
+bad for business.
+
+The solution is to notify Varnish when there is fresh content
+available. This can be done through three mechanisms. HTTP purging,
+banning and forced cache misses. First, let me explain the HTTP purges. 
+
+
+HTTP Purges
+===========
+
+A *purge* is what happens when you pick out an object from the cache
+and discard it along with its variants. Usually a purge is invoked
+through HTTP with the method PURGE.
+
+An HTTP purge is similar to an HTTP GET request, except that the
+*method* is PURGE. Actually you can call the method whatever you'd
+like, but most people refer to this as purging. Squid supports the
+same mechanism. In order to support purging in Varnish you need the
+following VCL in place::
+
+  acl purge {
+	  "localhost";
+	  "192.168.55.0"/24;
+  }
+  
+  sub vcl_recv {
+      	  # allow PURGE from localhost and 192.168.55...
+
+	  if (req.request == "PURGE") {
+		  if (!client.ip ~ purge) {
+			  error 405 "Not allowed.";
+		  }
+		  return (lookup);
+	  }
+  }
+  
+  sub vcl_hit {
+	  if (req.request == "PURGE") {
+	          purge;
+		  error 200 "Purged.";
+	  }
+  }
+  
+  sub vcl_miss {
+	  if (req.request == "PURGE") {
+	          purge;
+		  error 200 "Purged.";
+	  }
+  }
+
+As you can see we have used to new VCL subroutines, vcl_hit and
+vcl_miss. When we call lookup Varnish will try to lookup the object in
+its cache. It will either hit an object or miss it and so the
+corresponding subroutine is called. In vcl_hit the object that is
+stored in cache is available and we can set the TTL. The purge in
+vcl_miss is necessary to purge all variants in the cases where you hit an
+object, but miss a particular variant.
+
+So for example.com to invalidate their front page they would call out
+to Varnish like this::
+
+  PURGE / HTTP/1.0
+  Host: example.com
+
+And Varnish would then discard the front page. This will remove all
+variants as defined by Vary.
+
+Bans
+====
+
+There is another way to invalidate content: Bans. You can think of
+bans as a sort of a filter on objects already in the cache. You *ban*
+certain content from being served from your cache. You can ban
+content based on any metadata we have.
+A ban will only work on objects already in the cache, it does not
+prevent new content from entering the cache or being served.
+
+Support for bans is built into Varnish and available in the CLI
+interface. To ban every png object belonging on example.com, issue
+the following command::
+
+  ban req.http.host == "example.com" && req.url ~ "\.png$"
+
+Quite powerful, really.
+
+Bans are checked when we hit an object in the cache, but before we
+deliver it. *An object is only checked against newer bans*. 
+
+Bans that only match against obj.* are also processed by a background
+worker threads called the *ban lurker*. The ban lurker will walk the
+heap and try to match objects and will evict the matching objects. How
+aggressive the ban lurker is can be controlled by the parameter
+ban_lurker_sleep. The ban lurker can be disabled by setting
+ban_lurker_sleep to 0.
+
+Bans that are older than the oldest objects in the cache are discarded
+without evaluation.  If you have a lot of objects with long TTL, that
+are seldom accessed you might accumulate a lot of bans. This might
+impact CPU usage and thereby performance.
+
+You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
+
+  sub vcl_recv {
+	  if (req.request == "BAN") {
+                  # Same ACL check as above:
+		  if (!client.ip ~ purge) {
+			  error 405 "Not allowed.";
+		  }
+		  ban("req.http.host == " + req.http.host +
+		        "&& req.url == " + req.url);
+
+		  # Throw a synthetic page so the
+                  # request won't go to the backend.
+		  error 200 "Ban added";
+	  }
+  }
+
+This VCL sniplet enables Varnish to handle an HTTP BAN method, adding a
+ban on the URL, including the host part.
+
+The ban lurker can help you keep the ban list at a manageable size, so
+we recommend that you avoid using req.* in your bans, as the request
+object is not available in the ban lurker thread.
+
+You can use the following template to write ban lurker friendly bans::
+
+  sub vcl_fetch {
+    set beresp.http.x-url = req.url;
+  }
+
+  sub vcl_deliver {
+    unset resp.http.x-url; # Optional
+  }
+
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (client.ip !~ purge) {
+        error 401 "Not allowed";
+      }
+      ban("obj.http.x-url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple
+    }
+  }
+
+To inspect the current ban list, issue the ban.list command in CLI. This
+will produce a status of all current bans::
+
+  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
+  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
+
+The ban list contains the ID of the ban, the timestamp when the ban
+entered the ban list. A count of the objects that has reached this point
+in the ban list, optionally postfixed with a 'G' for "Gone", if the ban
+is no longer valid.  Finally, the ban expression is listed. The ban can
+be marked as Gone if it is a duplicate ban, but is still kept in the list
+for optimization purposes.
+
+Forcing a cache miss
+====================
+
+The final way to invalidate an object is a method that allows you to
+refresh an object by forcing a hash miss for a single request. If you set
+req.hash_always_miss to true, varnish will miss the current object in the
+cache, thus forcing a fetch from the backend. This can in turn add the
+freshly fetched object to the cache, thus overriding the current one. The
+old object will stay in the cache until ttl expires or it is evicted by
+some other means.
diff --git a/doc/sphinx/users-guide/putting_varnish_on_port_80.rst b/doc/sphinx/users-guide/putting_varnish_on_port_80.rst
new file mode 100644
index 0000000..73a80ff
--- /dev/null
+++ b/doc/sphinx/users-guide/putting_varnish_on_port_80.rst
@@ -0,0 +1,25 @@
+
+Put Varnish on port 80
+----------------------
+
+Until now we've been running with Varnish on a high port, for testing
+purposes. You should test your application and if it works OK we can
+switch, so Varnish will be running on port 80 and your web server on a
+high port.
+
+First we kill off varnishd::
+
+     # pkill varnishd
+
+and stop your web server. Edit the configuration for your web server
+and make it bind to port 8080 instead of 80. Now open the Varnish
+default.vcl and change the port of the *default* backend to 8080.
+
+Start up your web server and then start varnish::
+
+      # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000
+
+Note that we've removed the -a option. Now Varnish, as its default
+setting dictates, will bind to the http port (80). Now everyone
+accessing your site will be accessing through Varnish.
+
diff --git a/doc/sphinx/users-guide/sizing_your_cache.rst b/doc/sphinx/users-guide/sizing_your_cache.rst
new file mode 100644
index 0000000..c19647c
--- /dev/null
+++ b/doc/sphinx/users-guide/sizing_your_cache.rst
@@ -0,0 +1,25 @@
+
+Sizing your cache
+-----------------
+
+Picking how much memory you should give Varnish can be a tricky
+task. A few things to consider:
+
+ * How big is your *hot* data set. For a portal or news site that
+   would be the size of the front page with all the stuff on it, and
+   the size of all the pages and objects linked from the first page. 
+ * How expensive is it to generate an object? Sometimes it makes sense
+   to only cache images a little while or not to cache them at all if
+   they are cheap to serve from the backend and you have a limited
+   amount of memory.
+ * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or some other
+   tool. If you have a lot of LRU activity then your cache is evicting
+   objects due to space constraints and you should consider increasing
+   the size of the cache.
+
+Be aware that every object that is stored also carries overhead that
+is kept outside the actually storage area. So, even if you specify -s
+malloc,16G varnish might actually use **double** that. Varnish has a
+overhead of about 1k per object. So, if you have lots of small objects
+in your cache the overhead might be significant.
+
diff --git a/doc/sphinx/users-guide/starting_varnish.rst b/doc/sphinx/users-guide/starting_varnish.rst
new file mode 100644
index 0000000..6c89f54
--- /dev/null
+++ b/doc/sphinx/users-guide/starting_varnish.rst
@@ -0,0 +1,51 @@
+.. _tutorial-starting_varnish:
+
+Starting Varnish
+----------------
+
+I assume varnishd is in your path. You might want to run ``pkill
+varnishd`` to make sure varnishd isn't running. Become root and type:
+
+``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
+
+I added a few options, lets go through them:
+
+``-f /usr/local/etc/varnish/default.vcl``
+ The -f options specifies what configuration varnishd should use.
+
+``-s malloc,1G``
+ The -s options chooses the storage type Varnish should use for
+ storing its content. I used the type *malloc*, which just uses memory
+ for storage. There are other backends as well, described in 
+ :ref:tutorial-storage. 1G specifies how much memory should be allocated 
+ - one gigabyte. 
+
+``-T 127.0.0.1:2000``
+ Varnish has a built-in text-based administration
+ interface. Activating the interface makes Varnish manageble without
+ stopping it. You can specify what interface the management interface
+ should listen to. Make sure you don't expose the management interface
+ to the world as you can easily gain root access to a system via the
+ Varnish management interface. I recommend tieing it to localhost. If
+ you have users on your system that you don't fully trust, use firewall
+ rules to restrict access to the interface to root only.
+
+``-a 0.0.0.0:8080``
+ I specify that I want Varnish to listen on port 8080 for incomming
+ HTTP requests. For a production environment you would probably make
+ Varnish listen on port 80, which is the default.
+
+Now you have Varnish running. Let us make sure that it works
+properly. Use your browser to go to http://192.168.2.2:8080/
+(obviously, you should replace the IP address with one on your own
+system) - you should now see your web application running there.
+
+Whether or not the application actually goes faster when run through
+Varnish depends on a few factors. If you application uses cookies for
+every session (a lot of PHP and Java applications seem to send a
+session cookie if it is needed or not) or if it uses authentication
+chances are Varnish won't do much caching. Ignore that for the moment,
+we come back to that in :ref:`tutorial-increasing_your_hitrate`.
+
+Lets make sure that Varnish really does do something to your web
+site. To do that we'll take a look at the logs.
diff --git a/doc/sphinx/users-guide/statistics.rst b/doc/sphinx/users-guide/statistics.rst
new file mode 100644
index 0000000..4386111
--- /dev/null
+++ b/doc/sphinx/users-guide/statistics.rst
@@ -0,0 +1,57 @@
+.. _tutorial-statistics:
+
+
+Statistics
+----------
+
+Now that your varnish is up and running let's have a look at how it is
+doing. There are several tools that can help.
+
+varnishtop
+~~~~~~~~~~
+
+The varnishtop utility reads the shared memory logs and presents a
+continuously updated list of the most commonly occurring log entries.
+
+With suitable filtering using the -I, -i, -X and -x options, it can be
+used to display a ranking of requested documents, clients, user
+agents, or any other information which is recorded in the log.
+
+``varnishtop -i rxurl`` will show you what URLs are being asked for
+by the client. ``varnishtop -i txurl`` will show you what your backend
+is being asked the most. ``varnishtop -i RxHeader -I
+Accept-Encoding`` will show the most popular Accept-Encoding header
+the client are sending you.
+
+varnishhist
+~~~~~~~~~~~
+
+The varnishhist utility reads varnishd(1) shared memory logs and
+presents a continuously updated histogram showing the distribution of
+the last N requests by their processing.  The value of N and the
+vertical scale are displayed in the top left corner.  The horizontal
+scale is logarithmic.  Hits are marked with a pipe character ("|"),
+and misses are marked with a hash character ("#").
+
+
+varnishsizes
+~~~~~~~~~~~~
+
+Varnishsizes does the same as varnishhist, except it shows the size of
+the objects and not the time take to complete the request. This gives
+you a good overview of how big the objects you are serving are.
+
+
+varnishstat
+~~~~~~~~~~~
+
+Varnish has lots of counters. We count misses, hits, information about
+the storage, threads created, deleted objects. Just about
+everything. varnishstat will dump these counters. This is useful when
+tuning varnish. 
+
+There are programs that can poll varnishstat regularly and make nice
+graphs of these counters. One such program is Munin. Munin can be
+found at http://munin-monitoring.org/ . There is a plugin for munin in
+the varnish source code.
+
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
new file mode 100644
index 0000000..5bbcf6c
--- /dev/null
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -0,0 +1,99 @@
+Troubleshooting Varnish
+-----------------------
+
+Sometimes Varnish misbehaves. In order for you to understand whats
+going on there are a couple of places you can check. varnishlog,
+/var/log/syslog, /var/log/messages are all places where varnish might
+leave clues of whats going on.
+
+
+When Varnish won't start
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes Varnish wont start. There is a plethora of reasons why
+Varnish wont start on your machine. We've seen everything from wrong
+permissions on /dev/null to other processes blocking the ports.
+
+Starting Varnish in debug mode to see what is going on.
+
+Try to start varnish by::
+
+    # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000  -a 0.0.0.0:8080 -d
+
+Notice the -d option. It will give you some more information on what
+is going on. Let us see how Varnish will react to something else
+listening on its port.::
+
+    # varnishd -n foo -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000  -a 0.0.0.0:8080 -d
+    storage_malloc: max size 1024 MB.
+    Using old SHMFILE
+    Platform: Linux,2.6.32-21-generic,i686,-smalloc,-hcritbit
+    200 193     
+    -----------------------------
+    Varnish Cache CLI.
+    -----------------------------
+    Type 'help' for command list.
+    Type 'quit' to close CLI session.
+    Type 'start' to launch worker process.
+
+Now Varnish is running. Only the master process is running, in debug
+mode the cache does not start. Now you're on the console. You can
+instruct the master process to start the cache by issuing "start".::
+
+	 start
+	 bind(): Address already in use
+	 300 22      
+	 Could not open sockets
+
+And here we have our problem. Something else is bound to the HTTP port
+of Varnish. If this doesn't help try strace or truss or come find us
+on IRC.
+
+
+Varnish is crashing
+~~~~~~~~~~~~~~~~~~~
+
+When varnish goes bust the child processes crashes. Usually the mother
+process will manage this by restarting the child process again. Any
+errors will be logged in syslog. It might look like this::
+
+       Mar  8 13:23:38 smoke varnishd[15670]: Child (15671) not responding to CLI, killing it.
+       Mar  8 13:23:43 smoke varnishd[15670]: last message repeated 2 times
+       Mar  8 13:23:43 smoke varnishd[15670]: Child (15671) died signal=3
+       Mar  8 13:23:43 smoke varnishd[15670]: Child cleanup complete
+       Mar  8 13:23:43 smoke varnishd[15670]: child (15697) Started
+
+Specifically if you see the "Error in munmap" error on Linux you might
+want to increase the amount of maps available. Linux is limited to a
+maximum of 64k maps. Setting vm.max_max_count i sysctl.conf will
+enable you to increase this limit. You can inspect the number of maps
+your program is consuming by counting the lines in /proc/$PID/maps
+
+This is a rather odd thing to document here - but hopefully Google
+will serve you this page if you ever encounter this error. 
+
+Varnish gives me Guru meditation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+First find the relevant log entries in varnishlog. That will probably
+give you a clue. Since varnishlog logs so much data it might be hard
+to track the entries down. You can set varnishlog to log all your 503
+errors by issuing the following command::
+
+   $ varnishlog -c -m TxStatus:503
+
+If the error happened just a short time ago the transaction might still
+be in the shared memory log segment. To get varnishlog to process the
+whole shared memory log just add the -d option::
+
+   $ varnishlog -d -c -m TxStatus:503
+
+Please see the varnishlog man page for elaborations on further
+filtering capabilities and explanation of the various options.
+
+
+Varnish doesn't cache
+~~~~~~~~~~~~~~~~~~~~~
+
+See :ref:`tutorial-increasing_your_hitrate`.
+
diff --git a/doc/sphinx/users-guide/vary.rst b/doc/sphinx/users-guide/vary.rst
new file mode 100644
index 0000000..ad7b48d
--- /dev/null
+++ b/doc/sphinx/users-guide/vary.rst
@@ -0,0 +1,58 @@
+.. _tutorial-vary:
+
+Vary
+~~~~
+
+The Vary header is sent by the web server to indicate what makes a
+HTTP object Vary. This makes a lot of sense with headers like
+Accept-Encoding. When a server issues a "Vary: Accept-Encoding" it
+tells Varnish that its needs to cache a separate version for every
+different Accept-Encoding that is coming from the clients. So, if a
+clients only accepts gzip encoding Varnish won't serve the version of
+the page encoded with the deflate encoding.
+
+The problem is that the Accept-Encoding field contains a lot of
+different encodings. If one browser sends::
+
+  Accept-Encoding: gzip,deflate
+
+And another one sends::
+
+  Accept-Encoding: deflate,gzip
+
+Varnish will keep two variants of the page requested due to the
+different Accept-Encoding headers. Normalizing the accept-encoding
+header will sure that you have as few variants as possible. The
+following VCL code will normalize the Accept-Encoding headers::
+
+    if (req.http.Accept-Encoding) {
+        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
+            # No point in compressing these
+            remove req.http.Accept-Encoding;
+        } elsif (req.http.Accept-Encoding ~ "gzip") {
+            set req.http.Accept-Encoding = "gzip";
+        } elsif (req.http.Accept-Encoding ~ "deflate") {
+            set req.http.Accept-Encoding = "deflate";
+        } else {
+            # unknown algorithm
+            remove req.http.Accept-Encoding;
+        }
+    }
+
+The code sets the Accept-Encoding header from the client to either
+gzip, deflate with a preference for gzip.
+
+Pitfall - Vary: User-Agent
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some applications or application servers send *Vary: User-Agent* along
+with their content. This instructs Varnish to cache a separate copy
+for every variation of User-Agent there is. There are plenty. Even a
+single patchlevel of the same browser will generate at least 10
+different User-Agent headers based just on what operating system they
+are running. 
+
+So if you *really* need to Vary based on User-Agent be sure to
+normalize the header or your hit rate will suffer badly. Use the above
+code as a template.
+
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
new file mode 100644
index 0000000..0601468
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -0,0 +1,200 @@
+Varnish Configuration Language - VCL
+-------------------------------------
+
+Varnish has a great configuration system. Most other systems use
+configuration directives, where you basically turn on and off lots of
+switches. Varnish uses a domain specific language called Varnish
+Configuration Language, or VCL for short. Varnish translates this
+configuration into binary code which is then executed when requests
+arrive.
+
+The VCL files are divided into subroutines. The different subroutines
+are executed at different times. One is executed when we get the
+request, another when files are fetched from the backend server.
+
+Varnish will execute these subroutines of code at different stages of
+its work. Because it is code it is execute line by line precedence
+isn't a problem. At some point you call an action in this subroutine
+and then the execution of the subroutine stops.
+
+If you don't call an action in your subroutine and it reaches the end
+Varnish will execute some built in VCL code. You will see this VCL
+code commented out in default.vcl.
+
+99% of all the changes you'll need to do will be done in two of these
+subroutines. *vcl_recv* and *vcl_fetch*.
+
+vcl_recv
+~~~~~~~~
+
+vcl_recv (yes, we're skimpy with characters, it's Unix) is called at
+the beginning of a request, after the complete request has been
+received and parsed.  Its purpose is to decide whether or not to serve
+the request, how to do it, and, if applicable, which backend to use.
+
+In vcl_recv you can also alter the request. Typically you can alter
+the cookies and add and remove request headers.
+
+Note that in vcl_recv only the request object, req is available.
+
+vcl_fetch
+~~~~~~~~~
+
+vcl_fetch is called *after* a document has been successfully retrieved
+from the backend. Normal tasks her are to alter the response headers,
+trigger ESI processing, try alternate backend servers in case the
+request failed.
+
+In vcl_fetch you still have the request object, req, available. There
+is also a *backend response*, beresp. beresp will contain the HTTP
+headers from the backend.
+
+.. _tutorial-vcl_fetch_actions:
+
+actions
+~~~~~~~
+
+The most common actions to return are these:
+
+*pass*
+ When you return pass the request and subsequent response will be passed to
+ and from the backend server. It won't be cached. pass can be returned from
+ vcl_recv
+
+*hit_for_pass*
+  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
+  will create a hitforpass object in the cache. This has the side-effect of
+  caching the decision not to cache. This is to allow would-be uncachable
+  requests to be passed to the backend at the same time. The same logic is
+  not necessary in vcl_recv because this happens before any potential
+  queueing for an object takes place.
+
+*lookup*
+  When you return lookup from vcl_recv you tell Varnish to deliver content 
+  from cache even if the request othervise indicates that the request 
+  should be passed. You can't return lookup from vcl_fetch.
+
+*pipe*
+  Pipe can be returned from vcl_recv as well. Pipe short circuits the
+  client and the backend connections and Varnish will just sit there
+  and shuffle bytes back and forth. Varnish will not look at the data being 
+  send back and forth - so your logs will be incomplete. 
+  Beware that with HTTP 1.1 a client can send several requests on the same 
+  connection and so you should instruct Varnish to add a "Connection: close"
+  header before actually returning pipe. 
+
+*deliver*
+ Deliver the cached object to the client.  Usually returned from vcl_fetch. 
+
+Requests, responses and objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In VCL, there are three important data structures. The request, coming
+from the client, the response coming from the backend server and the
+object, stored in cache.
+
+In VCL you should know the following structures.
+
+*req*
+ The request object. When Varnish has received the request the req object is 
+ created and populated. Most of the work you do in vcl_recv you 
+ do on or with the req object.
+
+*beresp*
+ The backend respons object. It contains the headers of the object 
+ comming from the backend. Most of the work you do in vcl_fetch you 
+ do on the beresp object.
+
+*obj*
+ The cached object. Mostly a read only object that resides in memory. 
+ obj.ttl is writable, the rest is read only.
+
+Operators
+~~~~~~~~~
+
+The following operators are available in VCL. See the examples further
+down for, uhm, examples.
+
+= 
+ Assignment operator.
+
+== 
+ Comparison.
+
+~
+ Match. Can either be used with regular expressions or ACLs.
+
+!
+ Negation.
+
+&&
+ Logical *and*
+
+||
+ Logical *or*
+
+Example 1 - manipulating headers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Lets say we want to remove the cookie for all objects in the /images
+directory of our web server::
+
+  sub vcl_recv {
+    if (req.url ~ "^/images") {
+      unset req.http.cookie;
+    }
+  }
+
+Now, when the request is handled to the backend server there will be
+no cookie header. The interesting line is the one with the
+if-statement. It matches the URL, taken from the request object, and
+matches it against the regular expression. Note the match operator. If
+it matches the Cookie: header of the request is unset (deleted). 
+
+Example 2 - manipulating beresp
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Here we override the TTL of a object comming from the backend if it
+matches certain criteria::
+
+  sub vcl_fetch {
+     if (req.url ~ "\.(png|gif|jpg)$") {
+       unset beresp.http.set-cookie;
+       set beresp.ttl = 1h;
+    }
+  }
+
+Example 3 - ACLs
+~~~~~~~~~~~~~~~~
+
+You create a named access control list with the *acl* keyword. You can match
+the IP address of the client against an ACL with the match operator.::
+
+  # Who is allowed to purge....
+  acl local {
+      "localhost";
+      "192.168.1.0"/24; /* and everyone on the local network */
+      ! "192.168.1.23"; /* except for the dialin router */
+  }
+  
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (client.ip ~ local) {
+         return(lookup);
+      }
+    } 
+  }
+  
+  sub vcl_hit {
+     if (req.request == "PURGE") {
+       set obj.ttl = 0s;
+       error 200 "Purged.";
+      }
+  }
+
+  sub vcl_miss {
+    if (req.request == "PURGE") {
+      error 404 "Not in cache.";
+    }
+  }
+
diff --git a/doc/sphinx/users-guide/virtualized.rst b/doc/sphinx/users-guide/virtualized.rst
new file mode 100644
index 0000000..317d3e2
--- /dev/null
+++ b/doc/sphinx/users-guide/virtualized.rst
@@ -0,0 +1,23 @@
+
+Running Varnish in a virtualized environment
+--------------------------------------------
+
+It is possible, but not recommended for high performance, to run
+Varnish on virtualized hardware. Reduced disk- and network performance
+will reduce the performance a bit so make sure your system has good IO
+performance.
+
+OpenVZ
+~~~~~~
+
+If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
+the maximum stack size before starting Varnish. The default allocates
+to much memory per thread, which will make varnish fail as soon as the
+number of threads (==traffic) increases.
+
+Reduce the maximum stack size by running::
+
+    ulimit -s 256
+
+in the startup script.
+
diff --git a/doc/sphinx/users-guide/websockets.rst b/doc/sphinx/users-guide/websockets.rst
new file mode 100644
index 0000000..a74353e
--- /dev/null
+++ b/doc/sphinx/users-guide/websockets.rst
@@ -0,0 +1,20 @@
+
+Using Websockets 
+----------------
+
+Websockets is a technology for creating a bidirectional stream-based channel over HTTP.
+
+To run websockets through Varnish you need to pipe it, and copy the Upgrade header. Use the following
+VCL config to do so::
+
+    sub vcl_pipe {
+         if (req.http.upgrade) {
+             set bereq.http.upgrade = req.http.upgrade;
+         }
+    }
+    sub vcl_recv {
+         if (req.http.Upgrade ~ "(?i)websocket") {
+             return (pipe);
+         }
+    }
+



From tfheen at varnish-cache.org  Tue Sep  4 06:18:08 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 04 Sep 2012 08:18:08 +0200
Subject: [master] 791706d Disable the PCRE JIT compiler by default
Message-ID: <E1T8mSe-0004EE-4X@project.varnish-software.com>

commit 791706d8e535fe1573a6ea5cd515113ddacbac52
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 4 08:16:50 2012 +0200

    Disable the PCRE JIT compiler by default
    
    The JIT compiler is broken on some versions of PCRE, at least on i386,
    so disable it by default.  It can be enabled using --enable-pcre-jit
    to configure.
    
    Fixes #1191

diff --git a/configure.ac b/configure.ac
index e831b31..87fdaf6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -531,6 +531,17 @@ fi
 
 AC_DEFINE_UNQUOTED([VCC_CC],"$VCC_CC",[C compiler command line for VCL code])
 
+# --enable-pcre-jit
+AC_ARG_ENABLE(pcre-jit,
+    AS_HELP_STRING([--enable-pcre-jit],
+	[use the PCRE JIT compiler (default is NO)]),
+    ,
+    [enable_pcre_jit=no])
+
+if test "$enable_pcre_jit" = yes; then
+   AC_DEFINE([USE_PCRE_JIT],[1],[use the PCRE JIT compiler])
+fi
+
 # Stupid automake needs this
 VTC_TESTS="$(cd $srcdir/bin/varnishtest && echo tests/*.vtc)"
 AC_SUBST(VTC_TESTS)
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 2fce5c7..c7eccfa 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -37,8 +37,10 @@
 
 #include "vre.h"
 
-#ifndef PCRE_STUDY_JIT_COMPILE
-#define PCRE_STUDY_JIT_COMPILE 0
+#if USE_PCRE_JIT
+#define VRE_STUDY_JIT_COMPILE PCRE_STUDY_JIT_COMPILE
+#else
+#define VRE_STUDY_JIT_COMPILE 0
 #endif
 
 #if PCRE_MAJOR < 8 || (PCRE_MAJOR == 8 && PCRE_MINOR < 20)
@@ -78,7 +80,7 @@ VRE_compile(const char *pattern, int options,
 		VRE_free(&v);
 		return (NULL);
 	}
-	v->re_extra = pcre_study(v->re, PCRE_STUDY_JIT_COMPILE, errptr);
+	v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, errptr);
 	if (*errptr != NULL) {
 		VRE_free(&v);
 		return (NULL);



From martin at varnish-cache.org  Wed Sep  5 13:47:08 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 05 Sep 2012 15:47:08 +0200
Subject: [master] 68fa175 Include <stdint.h> in params.h for the uint8_t
 typedef, to fix build on Linux.
Message-ID: <E1T9Fwi-0004l4-66@project.varnish-software.com>

commit 68fa175ea80fbc1275d499915de9cfb93f1c9c03
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 5 15:45:40 2012 +0200

    Include <stdint.h> in params.h for the uint8_t typedef, to fix build on Linux.

diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 7c1ccd5..a6e881b 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -29,6 +29,8 @@
  * This file contains the heritage passed when mgt forks cache
  */
 
+#include <stdint.h>
+
 #include "vre.h"
 
 #define VSM_CLASS_PARAM		"Params"



From martin at varnish-cache.org  Thu Sep  6 08:03:27 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 06 Sep 2012 10:03:27 +0200
Subject: [master] 7d1c8cf Invoke libtoolize before aclocal.
Message-ID: <E1T9X3f-0003pP-NT@project.varnish-software.com>

commit 7d1c8cf43c6c39867c3faccdb67c151ca8d5ebe5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 6 09:59:00 2012 +0200

    Invoke libtoolize before aclocal.
    
    libtoolize was overwriting m4/libtool.m4, which aclocal.m4 depended on
    and needed aclocal to update. This caused the autotools files to be
    updated, triggering a second configure run on first build.

diff --git a/autogen.sh b/autogen.sh
index 6d28093..8a13e62 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -41,8 +41,8 @@ fi
 
 set -ex
 
-aclocal -I m4
 $LIBTOOLIZE --copy --force
+aclocal -I m4
 autoheader
 automake --add-missing --copy --foreign
 autoconf



From perbu at varnish-cache.org  Thu Sep  6 11:56:48 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 06 Sep 2012 13:56:48 +0200
Subject: [master] 060e644 More tutorial --> users guide stuff
Message-ID: <E1T9ahU-0007sl-Re@project.varnish-software.com>

commit 060e64492bded482613f64876bab0bc5bff08ec8
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Sep 6 13:56:43 2012 +0200

    More tutorial --> users guide stuff

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 06eb52a..9f367c3 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -20,6 +20,7 @@ Contents:
 
    installation/index.rst
    tutorial/index.rst
+   users-guide/index.rst
    reference/index.rst
    phk/index.rst
    glossary/index.rst
diff --git a/doc/sphinx/users-guide/advanced_backend_servers.rst b/doc/sphinx/users-guide/advanced_backend_servers.rst
index b4206d9..52af46e 100644
--- a/doc/sphinx/users-guide/advanced_backend_servers.rst
+++ b/doc/sphinx/users-guide/advanced_backend_servers.rst
@@ -40,7 +40,7 @@ really arbitrary data. You want to send mobile devices to a different
 backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
 trick. 
 
-.. _tutorial-advanced_backend_servers-directors:
+.. _users-guide-advanced_backend_servers-directors:
 
 Directors
 ---------
@@ -80,7 +80,7 @@ But what if one of your servers goes down? Can Varnish direct all the
 requests to the healthy server? Sure it can. This is where the Health
 Checks come into play.
 
-.. _tutorial-advanced_backend_servers-health:
+.. _users-guide-advanced_backend_servers-health:
 
 Health checks
 -------------
@@ -148,7 +148,7 @@ Now we define the director.::
 You use this director just as you would use any other director or
 backend. Varnish will not send traffic to hosts that are marked as
 unhealthy. Varnish can also serve stale content if all the backends are
-down. See :ref:`tutorial-handling_misbehaving_servers` for more
+down. See :ref:`users-guide-handling_misbehaving_servers` for more
 information on how to enable this.
 
 Please note that Varnish will keep probes active for all loaded
diff --git a/doc/sphinx/users-guide/advanced_topics.rst b/doc/sphinx/users-guide/advanced_topics.rst
index 1045de9..07e5237 100644
--- a/doc/sphinx/users-guide/advanced_topics.rst
+++ b/doc/sphinx/users-guide/advanced_topics.rst
@@ -1,12 +1,12 @@
-.. _tutorial-advanced_topics:
+.. _users-guide-advanced_topics:
 
 Advanced topics
 ---------------
 
-This tutorial has covered the basics in Varnish. If you read through
+This guide has covered the basics in Varnish. If you read through
 it all you should now have the skills to run Varnish.
 
-Here is a short overview of topics that we haven't covered in the tutorial. 
+Here is a short overview of topics that we haven't covered in the guide. 
 
 More VCL
 ~~~~~~~~
diff --git a/doc/sphinx/users-guide/backend_servers.rst b/doc/sphinx/users-guide/backend_servers.rst
index 1b1aaf2..695a21a 100644
--- a/doc/sphinx/users-guide/backend_servers.rst
+++ b/doc/sphinx/users-guide/backend_servers.rst
@@ -1,4 +1,4 @@
-.. _tutorial-backend_servers:
+.. _users-guide-backend_servers:
 
 Backend servers
 ---------------
@@ -19,8 +19,7 @@ Somewhere in the top there will be a section that looks a bit like this.::
 	  #     .port = "8080";
 	  # }
 
-We comment in this bit of text and change the port setting from 8080
-to 80, making the text look like.::
+We comment in this bit of text making the text look like.::
 
           backend default {
                 .host = "127.0.0.1";
@@ -29,11 +28,9 @@ to 80, making the text look like.::
 
 Now, this piece of configuration defines a backend in Varnish called
 *default*. When Varnish needs to get content from this backend it will
-connect to port 80 on localhost (127.0.0.1).
+connect to port 8080 on localhost (127.0.0.1).
 
 Varnish can have several backends defined and can you can even join
 several backends together into clusters of backends for load balancing
-purposes. 
+purposes.  XXX: ref
 
-Now that we have the basic Varnish configuration done, let us start up
-Varnish on port 8080 so we can do some fundamental testing on it.
diff --git a/doc/sphinx/users-guide/command_line.rst b/doc/sphinx/users-guide/command_line.rst
new file mode 100644
index 0000000..94d8f5c
--- /dev/null
+++ b/doc/sphinx/users-guide/command_line.rst
@@ -0,0 +1,53 @@
+.. _users-guide-command-line:
+
+Starting Varnish
+----------------
+
+I assume varnishd is in your path. You might want to run ``pkill
+varnishd`` to make sure varnishd isn't running. 
+
+Become root and type:
+
+``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
+
+I added a few options, lets go through them:
+
+``-f /usr/local/etc/varnish/default.vcl``
+ The -f options specifies what configuration varnishd should use.
+
+``-s malloc,1G``
+ The -s options chooses the storage type Varnish should use for
+ storing its content. I used the type *malloc*, which just uses memory
+ for storage. There are other backends as well, described in 
+ :ref:users-guide-storage. 1G specifies how much memory should be allocated 
+ - one gigabyte. 
+
+``-T 127.0.0.1:2000``
+ Varnish has a built-in text-based administration
+ interface. Activating the interface makes Varnish manageble without
+ stopping it. You can specify what interface the management interface
+ should listen to. Make sure you don't expose the management interface
+ to the world as you can easily gain root access to a system via the
+ Varnish management interface. I recommend tieing it to localhost. If
+ you have users on your system that you don't fully trust, use firewall
+ rules to restrict access to the interface to root only.
+
+``-a 0.0.0.0:8080``
+ I specify that I want Varnish to listen on port 8080 for incomming
+ HTTP requests. For a production environment you would probably make
+ Varnish listen on port 80, which is the default.
+
+Now you have Varnish running. Let us make sure that it works
+properly. Use your browser to go to http://192.168.2.2:8080/
+(obviously, you should replace the IP address with one on your own
+system) - you should now see your web application running there.
+
+Whether or not the application actually goes faster when run through
+Varnish depends on a few factors. If you application uses cookies for
+every session (a lot of PHP and Java applications seem to send a
+session cookie if it is needed or not) or if it uses authentication
+chances are Varnish won't do much caching. Ignore that for the moment,
+we come back to that in :ref:`users-guide-increasing_your_hitrate`.
+
+Lets make sure that Varnish really does do something to your web
+site. To do that we'll take a look at the logs.
diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
index 0b8d1e8..1e93384 100644
--- a/doc/sphinx/users-guide/compression.rst
+++ b/doc/sphinx/users-guide/compression.rst
@@ -1,4 +1,4 @@
-.. _tutorial-compression:
+.. _users-guide-compression:
 
 Compression
 ~~~~~~~~~~~
diff --git a/doc/sphinx/users-guide/cookies.rst b/doc/sphinx/users-guide/cookies.rst
index c75171f..1df1851 100644
--- a/doc/sphinx/users-guide/cookies.rst
+++ b/doc/sphinx/users-guide/cookies.rst
@@ -1,4 +1,4 @@
-.. _tutorial-cookies:
+.. _users-guide-cookies:
 
 Cookies
 -------
@@ -90,6 +90,6 @@ Cookies coming from the backend
 
 If your backend server sets a cookie using the Set-Cookie header
 Varnish will not cache the page in the default configuration.  A
-hit-for-pass object (see :ref:`tutorial-vcl_fetch_actions`) is created.
+hit-for-pass object (see :ref:`users-guide-vcl_fetch_actions`) is created.
 So, if the backend server acts silly and sets unwanted cookies just unset
 the Set-Cookie header and all should be fine. 
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index 6bfc4c9..13d501a 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -1,4 +1,4 @@
-.. _tutorial-devicedetect:
+.. _users-guide-devicedetect:
 
 Device detection
 ~~~~~~~~~~~~~~~~
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 720b790..afd6e14 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -1,4 +1,4 @@
-.. _tutorial-esi:
+.. _users-guide-esi:
 
 Edge Side Includes
 ------------------
diff --git a/doc/sphinx/users-guide/handling_misbehaving_servers.rst b/doc/sphinx/users-guide/handling_misbehaving_servers.rst
index 406b4b3..bb5fd35 100644
--- a/doc/sphinx/users-guide/handling_misbehaving_servers.rst
+++ b/doc/sphinx/users-guide/handling_misbehaving_servers.rst
@@ -1,4 +1,4 @@
-.. _tutorial-handling_misbehaving_servers:
+.. _users-guide-handling_misbehaving_servers:
 
 Misbehaving servers
 -------------------
@@ -42,7 +42,7 @@ request. Lets us say that we accept serving 15s old object.::
 
 You might wonder why we should keep the objects in the cache for 30
 minutes if we are unable to serve them? Well, if you have enabled
-:ref:`tutorial-advanced_backend_servers-health` you can check if the
+:ref:`users-guide-advanced_backend_servers-health` you can check if the
 backend is sick and if it is we can serve the stale content for a bit
 longer.::
 
diff --git a/doc/sphinx/users-guide/increasing_your_hitrate.rst b/doc/sphinx/users-guide/increasing_your_hitrate.rst
index b9fa7e6..462a781 100644
--- a/doc/sphinx/users-guide/increasing_your_hitrate.rst
+++ b/doc/sphinx/users-guide/increasing_your_hitrate.rst
@@ -1,4 +1,4 @@
-.. _tutorial-increasing_your_hitrate:
+.. _users-guide-increasing_your_hitrate:
 
 Achieving a high hitrate
 ------------------------
@@ -24,7 +24,7 @@ Tool: varnishtop
 
 You can use varnishtop to identify what URLs are hitting the backend
 the most. ``varnishtop -i txurl`` is an essential command. You can see
-some other examples of varnishtop usage in :ref:`tutorial-statistics`.
+some other examples of varnishtop usage in :ref:`users-guide-statistics`.
 
 
 Tool: varnishlog
@@ -36,7 +36,7 @@ backend you can use varnishlog to have a look at the request.
 coming from the client (-c) matching /foo/bar.
 
 For more information on how varnishlog works please see
-:ref:`tutorial-logging` or man :ref:`ref-varnishlog`.
+:ref:`users-guide-logging` or man :ref:`ref-varnishlog`.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
@@ -206,8 +206,8 @@ Ways of increasing your hitrate even more
 The following chapters should give your ways of further increasing
 your hitrate, especially the chapter on Cookies.
 
- * :ref:`tutorial-cookies`
- * :ref:`tutorial-vary`
- * :ref:`tutorial-purging`
- * :ref:`tutorial-esi`
+ * :ref:`users-guide-cookies`
+ * :ref:`users-guide-vary`
+ * :ref:`users-guide-purging`
+ * :ref:`users-guide-esi`
 
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 6cf20a2..024738b 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -10,17 +10,15 @@ server and have basic knowledge of the HTTP protocol. The reader
 should have Varnish Cache up and running with the default
 configuration.
 
-The tutorial is split into short chapters, each chapter taking on a
+The guide is split into short chapters, each chapter taking on a
 separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
-        introduction
+        command_line
 	backend_servers
-        starting_varnish
 	logging
         sizing_your_cache
-        putting_varnish_on_port_80
 	vcl
         statistics
         increasing_your_hitrate
diff --git a/doc/sphinx/users-guide/logging.rst b/doc/sphinx/users-guide/logging.rst
index 1f0bc18..f2e36fd 100644
--- a/doc/sphinx/users-guide/logging.rst
+++ b/doc/sphinx/users-guide/logging.rst
@@ -1,4 +1,4 @@
-.. _tutorial-logging:
+.. _users-guide-logging:
 
 Logging in Varnish
 ------------------
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 422f9f4..15b85d2 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -1,4 +1,4 @@
-.. _tutorial-purging:
+.. _users-guide-purging:
 
 =====================
  Purging and banning
diff --git a/doc/sphinx/users-guide/statistics.rst b/doc/sphinx/users-guide/statistics.rst
index 4386111..ad57037 100644
--- a/doc/sphinx/users-guide/statistics.rst
+++ b/doc/sphinx/users-guide/statistics.rst
@@ -1,4 +1,4 @@
-.. _tutorial-statistics:
+.. _users-guide-statistics:
 
 
 Statistics
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index 5bbcf6c..e1df5f2 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -95,5 +95,5 @@ filtering capabilities and explanation of the various options.
 Varnish doesn't cache
 ~~~~~~~~~~~~~~~~~~~~~
 
-See :ref:`tutorial-increasing_your_hitrate`.
+See :ref:`users-guide-increasing_your_hitrate`.
 
diff --git a/doc/sphinx/users-guide/vary.rst b/doc/sphinx/users-guide/vary.rst
index ad7b48d..8b6fa2b 100644
--- a/doc/sphinx/users-guide/vary.rst
+++ b/doc/sphinx/users-guide/vary.rst
@@ -1,4 +1,4 @@
-.. _tutorial-vary:
+.. _users-guide-vary:
 
 Vary
 ~~~~
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index 0601468..57fb7e0 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -49,7 +49,7 @@ In vcl_fetch you still have the request object, req, available. There
 is also a *backend response*, beresp. beresp will contain the HTTP
 headers from the backend.
 
-.. _tutorial-vcl_fetch_actions:
+.. _users-guide-vcl_fetch_actions:
 
 actions
 ~~~~~~~



From phk at varnish-cache.org  Thu Sep  6 12:06:00 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 06 Sep 2012 14:06:00 +0200
Subject: [master] 4371f7b Various code-polishings to make OmnitiOS happy.
Message-ID: <E1T9aqO-0001gl-DX@project.varnish-software.com>

commit 4371f7bd07e63ccb0338ed12c295bfb04968cb8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 6 12:05:04 2012 +0000

    Various code-polishings to make OmnitiOS happy.
    
    Some of them are rather pedantic:  "NULL" vs "(char*)0" -- Really ?

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 95d1794..e6f8700 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -476,7 +476,7 @@ mgt_sigchld(const struct vev *e, int what)
 	assert(r == child_pid);
 	vsb = VSB_new_auto();
 	XXXAN(vsb);
-	VSB_printf(vsb, "Child (%d) %s", r, status ? "died" : "ended");
+	VSB_printf(vsb, "Child (%ld) %s", (long)r, status ? "died" : "ended");
 	if (WIFEXITED(status) && WEXITSTATUS(status)) {
 		VSB_printf(vsb, " status=%d", WEXITSTATUS(status));
 		exit_status |= 0x20;
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index f7f199a..d2d077f 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -177,7 +177,7 @@ static void
 run_cc(void *priv)
 {
 	mgt_sandbox(SANDBOX_CC);
-	(void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL);
+	(void)execl("/bin/sh", "/bin/sh", "-c", priv, (char*)0);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index b4e8c8c..5d31caf 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -320,7 +320,7 @@ smp_open(const struct stevedore *st)
 	/* We trust the parent to give us a valid silo, for good measure: */
 	AZ(smp_valid_silo(sc));
 
-	AZ(mprotect(sc->base, 4096, PROT_READ));
+	AZ(mprotect((void*)sc->base, 4096, PROT_READ));
 
 	sc->ident = SIGN_DATA(&sc->idn);
 
diff --git a/bin/varnishd/storage/storage_persistent_mgt.c b/bin/varnishd/storage/storage_persistent_mgt.c
index bb0eb38..c5b1dfd 100644
--- a/bin/varnishd/storage/storage_persistent_mgt.c
+++ b/bin/varnishd/storage/storage_persistent_mgt.c
@@ -178,7 +178,7 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
 	else
 		target = NULL;
 
-	sc->base = mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
+	sc->base = (void*)mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
 	    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0);
 
 	if (sc->base == MAP_FAILED)
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 0ddf1d0..0036720 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -377,10 +377,10 @@ varnish_launch(struct varnish *v)
 		AZ(close(v->fds[3]));
 		for (i = 3; i <getdtablesize(); i++)
 			(void)close(i);
-		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), NULL));
+		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), (char*)0));
 		exit(1);
 	} else {
-		vtc_log(v->vl, 3, "PID: %d", v->pid);
+		vtc_log(v->vl, 3, "PID: %ld", (long)v->pid);
 	}
 	AZ(close(v->fds[0]));
 	AZ(close(v->fds[3]));



From perbu at varnish-cache.org  Thu Sep  6 12:11:48 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 06 Sep 2012 14:11:48 +0200
Subject: [master] 6924b79 The tutorial is reappearing
Message-ID: <E1T9aw0-0002Pq-H4@project.varnish-software.com>

commit 6924b79f84c64fd5d1c731a252f9b5d4acbb7ae2
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Sep 6 14:11:00 2012 +0200

    The tutorial is reappearing

diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
new file mode 100644
index 0000000..1b1aaf2
--- /dev/null
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -0,0 +1,39 @@
+.. _tutorial-backend_servers:
+
+Backend servers
+---------------
+
+Varnish has a concept of "backend" or "origin" servers. A backend
+server is the server providing the content Varnish will accelerate.
+
+Our first task is to tell Varnish where it can find its content. Start
+your favorite text editor and open the varnish default configuration
+file. If you installed from source this is
+/usr/local/etc/varnish/default.vcl, if you installed from a package it
+is probably /etc/varnish/default.vcl.
+
+Somewhere in the top there will be a section that looks a bit like this.::
+
+	  # backend default {
+	  #     .host = "127.0.0.1";
+	  #     .port = "8080";
+	  # }
+
+We comment in this bit of text and change the port setting from 8080
+to 80, making the text look like.::
+
+          backend default {
+                .host = "127.0.0.1";
+    		.port = "80";
+	  }
+
+Now, this piece of configuration defines a backend in Varnish called
+*default*. When Varnish needs to get content from this backend it will
+connect to port 80 on localhost (127.0.0.1).
+
+Varnish can have several backends defined and can you can even join
+several backends together into clusters of backends for load balancing
+purposes. 
+
+Now that we have the basic Varnish configuration done, let us start up
+Varnish on port 8080 so we can do some fundamental testing on it.
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 311a322..15c8fdc 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -4,5 +4,13 @@
 The Varnish Tutorial
 %%%%%%%%%%%%%%%%%%%%
 
-What is a tutorial, anyway?
+What is a tutorial, anyway? Let's start off by including some of the stuff we chucked out of the previous tutorial - now "users guide"
+
+.. toctree:: :maxdepth: 1
+
+   introduction
+   starting_varnish
+   putting_varnish_on_port_80
+   backend_servers
+   now_what
 
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
new file mode 100644
index 0000000..0d43623
--- /dev/null
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -0,0 +1,37 @@
+.. _tutorial-intro:
+
+What is Varnish?
+----------------
+
+Varnish Cache is a web application accelerator also known as a caching
+HTTP reverse proxy. You install it in front of any server that speaks
+HTTP and configure it to cache the contents. Varnish Cache is really,
+really fast. It typically speeds up delivery with a factor of 300 -
+1000x, depending on your architecture.
+
+
+Performance
+~~~~~~~~~~~
+
+Varnish performs really, really well. It is usually bound by the speed
+of the network, effectivly turning performance into a non-issue. We've
+seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
+
+Flexibility
+~~~~~~~~~~~
+
+One of the key features of Varnish Cache, in addition to it's
+performance, is the flexibility of it's configuration language,
+VCL. VCL enables you to write policies on how incoming requests should
+be handled. In such a policy you can decide what content you want to
+serve, from where you want to get the content and how the request or
+response should be altered. You can read more about this in our
+tutorial.
+
+
+Supported plattforms
+~~~~~~~~~~~~~~~~~~~~
+
+Varnish is written to run on modern versions of Linux and FreeBSD and
+the best experience is had on those plattforms. Thanks to our
+contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/tutorial/now_what.rst b/doc/sphinx/tutorial/now_what.rst
new file mode 100644
index 0000000..36b81b7
--- /dev/null
+++ b/doc/sphinx/tutorial/now_what.rst
@@ -0,0 +1,9 @@
+
+
+=========
+Now what?
+=========
+
+You've read through the tutorial. You should have Varnish up and
+running. You should know about the logs and you should have a rough
+idea of what VCL is.
\ No newline at end of file
diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
new file mode 100644
index 0000000..73a80ff
--- /dev/null
+++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
@@ -0,0 +1,25 @@
+
+Put Varnish on port 80
+----------------------
+
+Until now we've been running with Varnish on a high port, for testing
+purposes. You should test your application and if it works OK we can
+switch, so Varnish will be running on port 80 and your web server on a
+high port.
+
+First we kill off varnishd::
+
+     # pkill varnishd
+
+and stop your web server. Edit the configuration for your web server
+and make it bind to port 8080 instead of 80. Now open the Varnish
+default.vcl and change the port of the *default* backend to 8080.
+
+Start up your web server and then start varnish::
+
+      # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000
+
+Note that we've removed the -a option. Now Varnish, as its default
+setting dictates, will bind to the http port (80). Now everyone
+accessing your site will be accessing through Varnish.
+
diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
new file mode 100644
index 0000000..6c89f54
--- /dev/null
+++ b/doc/sphinx/tutorial/starting_varnish.rst
@@ -0,0 +1,51 @@
+.. _tutorial-starting_varnish:
+
+Starting Varnish
+----------------
+
+I assume varnishd is in your path. You might want to run ``pkill
+varnishd`` to make sure varnishd isn't running. Become root and type:
+
+``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
+
+I added a few options, lets go through them:
+
+``-f /usr/local/etc/varnish/default.vcl``
+ The -f options specifies what configuration varnishd should use.
+
+``-s malloc,1G``
+ The -s options chooses the storage type Varnish should use for
+ storing its content. I used the type *malloc*, which just uses memory
+ for storage. There are other backends as well, described in 
+ :ref:tutorial-storage. 1G specifies how much memory should be allocated 
+ - one gigabyte. 
+
+``-T 127.0.0.1:2000``
+ Varnish has a built-in text-based administration
+ interface. Activating the interface makes Varnish manageble without
+ stopping it. You can specify what interface the management interface
+ should listen to. Make sure you don't expose the management interface
+ to the world as you can easily gain root access to a system via the
+ Varnish management interface. I recommend tieing it to localhost. If
+ you have users on your system that you don't fully trust, use firewall
+ rules to restrict access to the interface to root only.
+
+``-a 0.0.0.0:8080``
+ I specify that I want Varnish to listen on port 8080 for incomming
+ HTTP requests. For a production environment you would probably make
+ Varnish listen on port 80, which is the default.
+
+Now you have Varnish running. Let us make sure that it works
+properly. Use your browser to go to http://192.168.2.2:8080/
+(obviously, you should replace the IP address with one on your own
+system) - you should now see your web application running there.
+
+Whether or not the application actually goes faster when run through
+Varnish depends on a few factors. If you application uses cookies for
+every session (a lot of PHP and Java applications seem to send a
+session cookie if it is needed or not) or if it uses authentication
+chances are Varnish won't do much caching. Ignore that for the moment,
+we come back to that in :ref:`tutorial-increasing_your_hitrate`.
+
+Lets make sure that Varnish really does do something to your web
+site. To do that we'll take a look at the logs.



From perbu at varnish-cache.org  Thu Sep  6 12:11:48 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 06 Sep 2012 14:11:48 +0200
Subject: [master] 12a475d Removed stuff from the guide that goes into the
 tutorial
Message-ID: <E1T9aw0-0002Pw-IY@project.varnish-software.com>

commit 12a475d9fcaebf720b7a6cf6f3cf5d12dfdc69bf
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Sep 6 14:11:40 2012 +0200

    Removed stuff from the guide that goes into the tutorial

diff --git a/doc/sphinx/users-guide/introduction.rst b/doc/sphinx/users-guide/introduction.rst
deleted file mode 100644
index 0d43623..0000000
--- a/doc/sphinx/users-guide/introduction.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-.. _tutorial-intro:
-
-What is Varnish?
-----------------
-
-Varnish Cache is a web application accelerator also known as a caching
-HTTP reverse proxy. You install it in front of any server that speaks
-HTTP and configure it to cache the contents. Varnish Cache is really,
-really fast. It typically speeds up delivery with a factor of 300 -
-1000x, depending on your architecture.
-
-
-Performance
-~~~~~~~~~~~
-
-Varnish performs really, really well. It is usually bound by the speed
-of the network, effectivly turning performance into a non-issue. We've
-seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
-
-Flexibility
-~~~~~~~~~~~
-
-One of the key features of Varnish Cache, in addition to it's
-performance, is the flexibility of it's configuration language,
-VCL. VCL enables you to write policies on how incoming requests should
-be handled. In such a policy you can decide what content you want to
-serve, from where you want to get the content and how the request or
-response should be altered. You can read more about this in our
-tutorial.
-
-
-Supported plattforms
-~~~~~~~~~~~~~~~~~~~~
-
-Varnish is written to run on modern versions of Linux and FreeBSD and
-the best experience is had on those plattforms. Thanks to our
-contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/users-guide/putting_varnish_on_port_80.rst b/doc/sphinx/users-guide/putting_varnish_on_port_80.rst
deleted file mode 100644
index 73a80ff..0000000
--- a/doc/sphinx/users-guide/putting_varnish_on_port_80.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
-Put Varnish on port 80
-----------------------
-
-Until now we've been running with Varnish on a high port, for testing
-purposes. You should test your application and if it works OK we can
-switch, so Varnish will be running on port 80 and your web server on a
-high port.
-
-First we kill off varnishd::
-
-     # pkill varnishd
-
-and stop your web server. Edit the configuration for your web server
-and make it bind to port 8080 instead of 80. Now open the Varnish
-default.vcl and change the port of the *default* backend to 8080.
-
-Start up your web server and then start varnish::
-
-      # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000
-
-Note that we've removed the -a option. Now Varnish, as its default
-setting dictates, will bind to the http port (80). Now everyone
-accessing your site will be accessing through Varnish.
-
diff --git a/doc/sphinx/users-guide/starting_varnish.rst b/doc/sphinx/users-guide/starting_varnish.rst
deleted file mode 100644
index 6c89f54..0000000
--- a/doc/sphinx/users-guide/starting_varnish.rst
+++ /dev/null
@@ -1,51 +0,0 @@
-.. _tutorial-starting_varnish:
-
-Starting Varnish
-----------------
-
-I assume varnishd is in your path. You might want to run ``pkill
-varnishd`` to make sure varnishd isn't running. Become root and type:
-
-``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
-
-I added a few options, lets go through them:
-
-``-f /usr/local/etc/varnish/default.vcl``
- The -f options specifies what configuration varnishd should use.
-
-``-s malloc,1G``
- The -s options chooses the storage type Varnish should use for
- storing its content. I used the type *malloc*, which just uses memory
- for storage. There are other backends as well, described in 
- :ref:tutorial-storage. 1G specifies how much memory should be allocated 
- - one gigabyte. 
-
-``-T 127.0.0.1:2000``
- Varnish has a built-in text-based administration
- interface. Activating the interface makes Varnish manageble without
- stopping it. You can specify what interface the management interface
- should listen to. Make sure you don't expose the management interface
- to the world as you can easily gain root access to a system via the
- Varnish management interface. I recommend tieing it to localhost. If
- you have users on your system that you don't fully trust, use firewall
- rules to restrict access to the interface to root only.
-
-``-a 0.0.0.0:8080``
- I specify that I want Varnish to listen on port 8080 for incomming
- HTTP requests. For a production environment you would probably make
- Varnish listen on port 80, which is the default.
-
-Now you have Varnish running. Let us make sure that it works
-properly. Use your browser to go to http://192.168.2.2:8080/
-(obviously, you should replace the IP address with one on your own
-system) - you should now see your web application running there.
-
-Whether or not the application actually goes faster when run through
-Varnish depends on a few factors. If you application uses cookies for
-every session (a lot of PHP and Java applications seem to send a
-session cookie if it is needed or not) or if it uses authentication
-chances are Varnish won't do much caching. Ignore that for the moment,
-we come back to that in :ref:`tutorial-increasing_your_hitrate`.
-
-Lets make sure that Varnish really does do something to your web
-site. To do that we'll take a look at the logs.



From phk at varnish-cache.org  Thu Sep  6 12:41:35 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 06 Sep 2012 14:41:35 +0200
Subject: [master] c613b13 Try to make the sandboxing work on omnitios
Message-ID: <E1T9bOp-0006CD-6x@project.varnish-software.com>

commit c613b135570f87535839e3a94630880d16910f4f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 6 12:40:58 2012 +0000

    Try to make the sandboxing work on omnitios

diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index d443cc0..728eca0 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -102,13 +102,15 @@ mgt_sandbox_solaris_add_inheritable(priv_set_t *pset, enum sandbox_e who)
 {
 	switch (who) {
 	case SANDBOX_VCC:
+		/* for /etc/resolv.conf and /etc/hosts */
+		AZ(priv_addset(pset, "file_read"));
 		break;
 	case SANDBOX_CC:
-		priv_addset(pset, "proc_exec");
-		priv_addset(pset, "proc_fork");
+		AZ(priv_addset(pset, "proc_exec"));
+		AZ(priv_addset(pset, "proc_fork"));
 		/* PSARC/2009/378 - 63678502e95e - onnv_140 */
-		priv_addset(pset, "file_read");
-		priv_addset(pset, "file_write");
+		AZ(priv_addset(pset, "file_read"));
+		AZ(priv_addset(pset, "file_write"));
 		break;
 	case SANDBOX_VCLLOAD:
 		break;
@@ -131,19 +133,19 @@ mgt_sandbox_solaris_add_effective(priv_set_t *pset, enum sandbox_e who)
 	switch (who) {
 	case SANDBOX_VCC:
 		/* PSARC/2009/378 - 63678502e95e - onnv_140 */
-		priv_addset(pset, "file_write");
+		AZ(priv_addset(pset, "file_write"));
 		break;
 	case SANDBOX_CC:
 		break;
 	case SANDBOX_VCLLOAD:
 		/* PSARC/2009/378 - 63678502e95e - onnv_140 */
-		priv_addset(pset, "file_read");
+		AZ(priv_addset(pset, "file_read"));
 	case SANDBOX_WORKER:
 		/* PSARC/2009/685 - 8eca52188202 - onnv_132 */
-		priv_addset(pset, "net_access");
+		AZ(priv_addset(pset, "net_access"));
 		/* PSARC/2009/378 - 63678502e95e - onnv_140 */
-		priv_addset(pset, "file_read");
-		priv_addset(pset, "file_write");
+		AZ(priv_addset(pset, "file_read"));
+		AZ(priv_addset(pset, "file_write"));
 		break;
 	default:
 		REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__);
@@ -166,7 +168,7 @@ mgt_sandbox_solaris_add_permitted(priv_set_t *pset, enum sandbox_e who)
 		break;
 	case SANDBOX_WORKER:
 		/* for raising limits in cache_waiter_ports.c */
-		priv_addset(pset, PRIV_SYS_RESOURCE);
+		AZ(priv_addset(pset, PRIV_SYS_RESOURCE));
 		break;
 	default:
 		REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__);
@@ -184,7 +186,7 @@ mgt_sandbox_solaris_add_initial(priv_set_t *pset, enum sandbox_e who)
 	(void)who;
 
 	/* for setgid/setuid */
-	priv_addset(pset, PRIV_PROC_SETID);
+	AZ(priv_addset(pset, PRIV_PROC_SETID));
 }
 
 /*
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index af5d965..aa3d766 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -154,7 +154,8 @@ vws_thread(void *priv)
 
 	while (1) {
 		port_event_t ev[MAX_EVENTS];
-		int nevents, ei, ret;
+		u_int nevents;
+		int ei, ret;
 		double now, deadline;
 
 		/*
@@ -239,6 +240,7 @@ vws_thread(void *priv)
 			timeout = &max_ts;
 		}
 	}
+	return(0);
 }
 
 /*--------------------------------------------------------------------*/



From perbu at varnish-cache.org  Fri Sep  7 13:23:19 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 07 Sep 2012 15:23:19 +0200
Subject: [master] d92673d Short introduction to hashing in Varnish
Message-ID: <E1T9yWl-0002IY-IZ@project.varnish-software.com>

commit d92673d4e48d50a0148a157152ed1d29ab00ec40
Author: Per Buer <per.buer at gmail.com>
Date:   Fri Sep 7 15:20:44 2012 +0200

    Short introduction to hashing in Varnish

diff --git a/doc/sphinx/users-guide/hashing.rst b/doc/sphinx/users-guide/hashing.rst
new file mode 100644
index 0000000..b7bbe62
--- /dev/null
+++ b/doc/sphinx/users-guide/hashing.rst
@@ -0,0 +1,51 @@
+
+Hashing
+-------
+
+Internally, when Varnish stores content in it's store it uses a hash
+key to find the object again. In the default setup this key is
+calculated based on the content of the *Host* header or the IP adress
+of the server and the URL.
+
+Behold the default vcl.::
+
+ sub vcl_hash {
+     hash_data(req.url);
+     if (req.http.host) {
+         hash_data(req.http.host);
+     } else {
+         hash_data(server.ip);
+     }
+     return (hash);
+ }
+
+As you can see it first chucks in req.url then req.http.host if it
+exsists. It is worth pointing out that Varnish doesn't lowercase the
+hostname or the URL before hashing it so in thery having Varnish.org/
+and varnish.org/ would result in different cache entries. Browers
+however, tend to lowercase hostnames.
+
+You can change what goes into the hash. This way you can make Varnish
+serve up different content to different clients based on arbitrary
+criteria.
+
+Let's say you want to serve pages in different languages to your users
+based on where their IP address is located. You would need some Vmod
+to get a country code and then put it into the hash. It might look
+like this.
+
+In vcl_recv::
+
+  set req.http.X-Country-Code = geoip.lookup(client.ip);
+
+And then add a vcl_hash:
+
+sub vcl_hash {
+  hash_data(req.http.X-Country-Code);
+}
+
+As the default VCL will take care of adding the host and URL to the
+hash we don't have to do anything else. Be careful calling
+return(hash) as this will abort the execution of the default VCL and
+thereby you can end up with a Varnish that will return data based on
+more or less random inputs.



From perbu at varnish-cache.org  Fri Sep  7 13:23:19 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 07 Sep 2012 15:23:19 +0200
Subject: [master] bc2083f Chapter on what a web accl is
Message-ID: <E1T9yWl-0002IT-H2@project.varnish-software.com>

commit bc2083f669f1c9f2decbcd440f9404c0bb54ebdf
Author: Per Buer <per.buer at gmail.com>
Date:   Fri Sep 7 15:20:26 2012 +0200

    Chapter on what a web accl is

diff --git a/doc/sphinx/tutorial/web_accelerator.rst b/doc/sphinx/tutorial/web_accelerator.rst
new file mode 100644
index 0000000..4d455ae
--- /dev/null
+++ b/doc/sphinx/tutorial/web_accelerator.rst
@@ -0,0 +1,21 @@
+.. _tutorial-web_accelerator:
+
+What is a web accelerator
+-------------------------
+
+
+The problem
+-----------
+
+Application servers  er slow. They do lots of different things and
+they can sometimes take seconds to complete a web page.
+
+The solution
+------------
+
+Enter Varnish. It keeps a copy of the web pages that pass through
+it. If it finds that it can reuse these later so the server doesn't
+have to regenerate these it speeds things up.
+
+
+



From perbu at varnish-cache.org  Fri Sep  7 13:23:19 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 07 Sep 2012 15:23:19 +0200
Subject: [master] 8a573e0 Add link for ref
Message-ID: <E1T9yWl-0002Ie-K0@project.varnish-software.com>

commit 8a573e050403960ff40468f79b57a4a61618a65f
Author: Per Buer <per.buer at gmail.com>
Date:   Fri Sep 7 15:21:07 2012 +0200

    Add link for ref

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index eee9c0b..8d6ed9c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -137,6 +137,8 @@ list size. Setting a value of 0 disables saint mode checking entirely
 for that backend.  The value in the backend declaration overrides the
 parameter.
 
+.. _reference-vcl-director:
+
 Directors
 ---------
 



From perbu at varnish-cache.org  Fri Sep  7 13:23:19 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 07 Sep 2012 15:23:19 +0200
Subject: [master] f53869e Rework tutorial
Message-ID: <E1T9yWl-0002Ik-Lb@project.varnish-software.com>

commit f53869e8ed9680247531a8d0eaa50907de721b27
Author: Per Buer <per.buer at gmail.com>
Date:   Fri Sep 7 15:21:42 2012 +0200

    Rework tutorial

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 15c8fdc..6444be9 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -4,11 +4,22 @@
 The Varnish Tutorial
 %%%%%%%%%%%%%%%%%%%%
 
-What is a tutorial, anyway? Let's start off by including some of the stuff we chucked out of the previous tutorial - now "users guide"
+What is a tutorial, anyway?
+
+Scope: Lets start off by explaining the basic concept. Cover the
+following points:
+- what is a cache. How does Varnish work?
+- how do I start it.  
+
+Then we hand things over to the user guide. 
+
+Let's start off by including some of the  stuff we chucked out of the
+previous tutorial - now "users guide"
 
 .. toctree:: :maxdepth: 1
 
    introduction
+   web_accelerator
    starting_varnish
    putting_varnish_on_port_80
    backend_servers
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 0d43623..cdbab1d 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -1,21 +1,18 @@
 .. _tutorial-intro:
 
 What is Varnish?
-----------------
-
-Varnish Cache is a web application accelerator also known as a caching
-HTTP reverse proxy. You install it in front of any server that speaks
-HTTP and configure it to cache the contents. Varnish Cache is really,
-really fast. It typically speeds up delivery with a factor of 300 -
-1000x, depending on your architecture.
+-------------
 
+Varnish Cache is a web application accelerator. It can also be called
+a HTTP reverse proxy. The next chapter :ref:`tutorial-web-accelerator`
+will go into detail on what Varnish is.
 
 Performance
 ~~~~~~~~~~~
 
-Varnish performs really, really well. It is usually bound by the speed
-of the network, effectivly turning performance into a non-issue. We've
-seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
+Varnish has a modern architecture and is written with performance in
+mind.  It is usually bound by the speed of the network, effectivly
+turning performance into a non-issue.
 
 Flexibility
 ~~~~~~~~~~~
@@ -23,15 +20,34 @@ Flexibility
 One of the key features of Varnish Cache, in addition to it's
 performance, is the flexibility of it's configuration language,
 VCL. VCL enables you to write policies on how incoming requests should
-be handled. In such a policy you can decide what content you want to
-serve, from where you want to get the content and how the request or
-response should be altered. You can read more about this in our
-tutorial.
+be handled. 
 
+In such a policy you can decide what content you want to serve, from
+where you want to get the content and how the request or response
+should be altered. 
 
 Supported plattforms
-~~~~~~~~~~~~~~~~~~~~
+-----------------
 
 Varnish is written to run on modern versions of Linux and FreeBSD and
 the best experience is had on those plattforms. Thanks to our
 contributors it also runs on NetBSD, OpenBSD and OS X.
+
+About the Varnish development process
+-------------------------------
+
+Varnish is a community driven project. The development is overseen by
+the Varnish Governing Board which currently consist of Poul-Henning
+Kamp (Architect), Rogier Mulhuijzen (Fastly) and Kristian Lyngst?l
+(Varnish Software).
+
+Getting in touch
+-------------
+
+You can get in touch with us trough many channels. For real time chat
+you can reach us on IRC trough the server irc.linpro.net on the
+#varnish and #varnish-hacking channels.
+The are two mailing lists available. One for user questions and one
+for development discussions. See varnish-cache.org/mailinglist for
+information and signup.  There is also a web forum on the same site.
+
diff --git a/doc/sphinx/tutorial/starting_varnish.rst b/doc/sphinx/tutorial/starting_varnish.rst
index 6c89f54..fcdc37c 100644
--- a/doc/sphinx/tutorial/starting_varnish.rst
+++ b/doc/sphinx/tutorial/starting_varnish.rst
@@ -3,49 +3,33 @@
 Starting Varnish
 ----------------
 
-I assume varnishd is in your path. You might want to run ``pkill
-varnishd`` to make sure varnishd isn't running. Become root and type:
+You might want to run ``pkill varnishd`` to make sure varnishd isn't
+already running. Become root and type:
 
-``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
+``# /usr/local/sbin/varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -a :80``
 
 I added a few options, lets go through them:
 
 ``-f /usr/local/etc/varnish/default.vcl``
- The -f options specifies what configuration varnishd should use.
+ The -f options specifies what configuration varnishd should use. If
+ you are on a Linux system and have installed Varnish through packages
+ the configuration files might reside in ``/etc/varnish``.
 
 ``-s malloc,1G``
  The -s options chooses the storage type Varnish should use for
- storing its content. I used the type *malloc*, which just uses memory
- for storage. There are other backends as well, described in 
- :ref:tutorial-storage. 1G specifies how much memory should be allocated 
- - one gigabyte. 
-
-``-T 127.0.0.1:2000``
- Varnish has a built-in text-based administration
- interface. Activating the interface makes Varnish manageble without
- stopping it. You can specify what interface the management interface
- should listen to. Make sure you don't expose the management interface
- to the world as you can easily gain root access to a system via the
- Varnish management interface. I recommend tieing it to localhost. If
- you have users on your system that you don't fully trust, use firewall
- rules to restrict access to the interface to root only.
-
-``-a 0.0.0.0:8080``
- I specify that I want Varnish to listen on port 8080 for incomming
- HTTP requests. For a production environment you would probably make
- Varnish listen on port 80, which is the default.
+ storing its content. I used the type *malloc*, which uses memory for
+ storage. There are other backends as well, described in
+ :ref:`user-guide-storage`. 1G specifies how much memory should be
+ allocated - one gigabyte. 
 
 Now you have Varnish running. Let us make sure that it works
-properly. Use your browser to go to http://192.168.2.2:8080/
+properly. Use your browser to go to http://192.168.2.2/
 (obviously, you should replace the IP address with one on your own
 system) - you should now see your web application running there.
 
-Whether or not the application actually goes faster when run through
-Varnish depends on a few factors. If you application uses cookies for
-every session (a lot of PHP and Java applications seem to send a
-session cookie if it is needed or not) or if it uses authentication
-chances are Varnish won't do much caching. Ignore that for the moment,
-we come back to that in :ref:`tutorial-increasing_your_hitrate`.
+There are many command line options available for Varnish. For a walk
+through the most important ones see :ref:`users-guide-command-line` or
+for a complete list see :ref:`ref-varnishd`. 
 
-Lets make sure that Varnish really does do something to your web
-site. To do that we'll take a look at the logs.
+Ignore that for the moment, we'll revisit that topic in the Users
+Guide :ref:`users-guide-increasing_your_hitrate`.



From perbu at varnish-cache.org  Fri Sep  7 13:23:19 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 07 Sep 2012 15:23:19 +0200
Subject: [master] b31835c Merge backend stuff into one chapter
Message-ID: <E1T9yWl-0002Iv-NG@project.varnish-software.com>

commit b31835c437da5cb97b6c3a6f62010def7f4e8c90
Author: Per Buer <per.buer at gmail.com>
Date:   Fri Sep 7 15:22:23 2012 +0200

    Merge backend stuff into one chapter

diff --git a/doc/sphinx/users-guide/advanced_backend_servers.rst b/doc/sphinx/users-guide/advanced_backend_servers.rst
deleted file mode 100644
index 52af46e..0000000
--- a/doc/sphinx/users-guide/advanced_backend_servers.rst
+++ /dev/null
@@ -1,157 +0,0 @@
-Advanced Backend configuration
-------------------------------
-
-At some point you might need Varnish to cache content from several
-servers. You might want Varnish to map all the URL into one single
-host or not. There are lot of options.
-
-Lets say we need to introduce a Java application into out PHP web
-site. Lets say our Java application should handle URL beginning with
-/java/.
-
-We manage to get the thing up and running on port 8000. Now, lets have
-a look a default.vcl.::
-
-  backend default {
-      .host = "127.0.0.1";
-      .port = "8080";
-  }
-
-We add a new backend.::
-
-  backend java {
-      .host = "127.0.0.1";
-      .port = "8000";
-  }
-
-Now we need tell where to send the difference URL. Lets look at vcl_recv.::
-
-  sub vcl_recv {
-      if (req.url ~ "^/java/") {
-          set req.backend = java;
-      } else {
-          set req.backend = default.
-      }
-  }
-
-It's quite simple, really. Lets stop and think about this for a
-moment. As you can see you can define how you choose backends based on
-really arbitrary data. You want to send mobile devices to a different
-backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
-trick. 
-
-.. _users-guide-advanced_backend_servers-directors:
-
-Directors
----------
-
-You can also group several backend into a group of backends. These
-groups are called directors. This will give you increased performance
-and resilience. You can define several backends and group them
-together in a director.::
-
-	 backend server1 {
-	     .host = "192.168.0.10";
-	 }
-	 backend server2{
-	     .host = "192.168.0.10";
-	 }
-
-Now we create the director.::
-
-       	director example_director round-robin {
-        {
-                .backend = server1;
-        }
-	# server2
-        {
-                .backend = server2;
-        }
-	# foo
-	}
-
-
-This director is a round-robin director. This means the director will
-distribute the incoming requests on a round-robin basis. There is
-also a *random* director which distributes requests in a, you guessed
-it, random fashion.
-
-But what if one of your servers goes down? Can Varnish direct all the
-requests to the healthy server? Sure it can. This is where the Health
-Checks come into play.
-
-.. _users-guide-advanced_backend_servers-health:
-
-Health checks
--------------
-
-Lets set up a director with two backends and health checks. First lets
-define the backends.::
-
-       backend server1 {
-         .host = "server1.example.com";
-	 .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	   }
-         }
-       backend server2 {
-  	  .host = "server2.example.com";
-  	  .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	  }
-        }
-
-Whats new here is the probe. Varnish will check the health of each
-backend with a probe. The options are
-
-url
- What URL should varnish request.
-
-interval
- How often should we poll
-
-timeout
- What is the timeout of the probe
-
-window
- Varnish will maintain a *sliding window* of the results. Here the
- window has five checks.
-
-threshold 
- How many of the .window last polls must be good for the backend to be declared healthy.
-
-initial 
- How many of the of the probes a good when Varnish starts - defaults
- to the same amount as the threshold.
-
-Now we define the director.::
-
-  director example_director round-robin {
-        {
-                .backend = server1;
-        }
-        # server2 
-        {
-                .backend = server2;
-        }
-	
-        }
-
-You use this director just as you would use any other director or
-backend. Varnish will not send traffic to hosts that are marked as
-unhealthy. Varnish can also serve stale content if all the backends are
-down. See :ref:`users-guide-handling_misbehaving_servers` for more
-information on how to enable this.
-
-Please note that Varnish will keep probes active for all loaded
-VCLs. Varnish will coalesce probes that seem identical - so be careful
-not to change the probe config if you do a lot of VCL
-loading. Unloading the VCL will discard the probes.
diff --git a/doc/sphinx/users-guide/backend_servers.rst b/doc/sphinx/users-guide/backend_servers.rst
index 695a21a..4f721aa 100644
--- a/doc/sphinx/users-guide/backend_servers.rst
+++ b/doc/sphinx/users-guide/backend_servers.rst
@@ -23,7 +23,7 @@ We comment in this bit of text making the text look like.::
 
           backend default {
                 .host = "127.0.0.1";
-    		.port = "80";
+    		.port = "8080";
 	  }
 
 Now, this piece of configuration defines a backend in Varnish called
@@ -32,5 +32,166 @@ connect to port 8080 on localhost (127.0.0.1).
 
 Varnish can have several backends defined and can you can even join
 several backends together into clusters of backends for load balancing
-purposes.  XXX: ref
+purposes. 
+
+Multiple backends
+-----------------
+
+At some point you might need Varnish to cache content from several
+servers. You might want Varnish to map all the URL into one single
+host or not. There are lot of options.
+
+Lets say we need to introduce a Java application into out PHP web
+site. Lets say our Java application should handle URL beginning with
+/java/.
+
+We manage to get the thing up and running on port 8000. Now, lets have
+a look at the default.vcl.::
+
+  backend default {
+      .host = "127.0.0.1";
+      .port = "8080";
+  }
+
+We add a new backend.::
+
+  backend java {
+      .host = "127.0.0.1";
+      .port = "8000";
+  }
+
+Now we need tell where to send the difference URL. Lets look at vcl_recv.::
+
+  sub vcl_recv {
+      if (req.url ~ "^/java/") {
+          set req.backend = java;
+      } else {
+          set req.backend = default.
+      }
+  }
+
+It's quite simple, really. Lets stop and think about this for a
+moment. As you can see you can define how you choose backends based on
+really arbitrary data. You want to send mobile devices to a different
+backend? No problem. if (req.User-agent ~ /mobile/) .. should do the
+trick. 
+
+.. _users-guide-advanced_backend_servers-directors:
+
+Directors
+---------
+
+You can also group several backend into a group of backends. These
+groups are called directors. This will give you increased performance
+and resilience. You can define several backends and group them
+together in a director.::
+
+	 backend server1 {
+	     .host = "192.168.0.10";
+	 }
+	 backend server2{
+	     .host = "192.168.0.10";
+	 }
+
+Now we create the director.::
+
+       	director example_director round-robin {
+        {
+                .backend = server1;
+        }
+	# server2
+        {
+                .backend = server2;
+        }
+	# foo
+	}
+
+
+This director is a round-robin director. This means the director will
+distribute the incoming requests on a round-robin basis. There is
+also a *random* director which distributes requests in a, you guessed
+it, random fashion.
+
+But what if one of your servers goes down? Can Varnish direct all the
+requests to the healthy server? Sure it can. This is where the Health
+Checks come into play.
+
+.. _users-guide-advanced_backend_servers-health:
+
+Health checks
+-------------
+
+Lets set up a director with two backends and health checks. First lets
+define the backends.::
+
+       backend server1 {
+         .host = "server1.example.com";
+	 .probe = {
+                .url = "/";
+                .interval = 5s;
+                .timeout = 1 s;
+                .window = 5;
+                .threshold = 3;
+	   }
+         }
+       backend server2 {
+  	  .host = "server2.example.com";
+  	  .probe = {
+                .url = "/";
+                .interval = 5s;
+                .timeout = 1 s;
+                .window = 5;
+                .threshold = 3;
+	  }
+        }
+
+Whats new here is the probe. Varnish will check the health of each
+backend with a probe. The options are
+
+url
+ What URL should varnish request.
+
+interval
+ How often should we poll
+
+timeout
+ What is the timeout of the probe
+
+window
+ Varnish will maintain a *sliding window* of the results. Here the
+ window has five checks.
+
+threshold 
+ How many of the .window last polls must be good for the backend to be declared healthy.
+
+initial 
+ How many of the of the probes a good when Varnish starts - defaults
+ to the same amount as the threshold.
+
+Now we define the director.::
+
+  director example_director round-robin {
+        {
+                .backend = server1;
+        }
+        # server2 
+        {
+                .backend = server2;
+        }
+	
+        }
+
+You use this director just as you would use any other director or
+backend. Varnish will not send traffic to hosts that are marked as
+unhealthy. Varnish can also serve stale content if all the backends are
+down. See :ref:`users-guide-handling_misbehaving_servers` for more
+information on how to enable this.
+
+Please note that Varnish will keep probes active for all loaded
+VCLs. Varnish will coalesce probes that seem identical - so be careful
+not to change the probe config if you do a lot of VCL
+loading. Unloading the VCL will discard the probes.
+
+For more information on how to do this please see
+ref:`reference-vcl-director`.
 



From perbu at varnish-cache.org  Fri Sep  7 13:23:19 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 07 Sep 2012 15:23:19 +0200
Subject: [master] f125f37 Making the user guide more user guidery
Message-ID: <E1T9yWl-0002JB-PJ@project.varnish-software.com>

commit f125f37903c4507973248406fd778de1ace870f9
Author: Per Buer <per.buer at gmail.com>
Date:   Fri Sep 7 15:23:13 2012 +0200

    Making the user guide more user guidery

diff --git a/doc/sphinx/users-guide/advanced_topics.rst b/doc/sphinx/users-guide/advanced_topics.rst
index 07e5237..6368d6e 100644
--- a/doc/sphinx/users-guide/advanced_topics.rst
+++ b/doc/sphinx/users-guide/advanced_topics.rst
@@ -19,6 +19,8 @@ page - ref:`reference-vcl`.
 Using In-line C to extend Varnish
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+(Here there be dragons)
+
 You can use *in-line C* to extend Varnish. Please note that you can
 seriously mess up Varnish this way. The C code runs within the Varnish
 Cache process so if your code generates a segfault the cache will crash.
@@ -37,27 +39,3 @@ One of the first uses I saw of In-line C was logging to syslog.::
         }
 
 
-Edge Side Includes
-~~~~~~~~~~~~~~~~~~
-
-Varnish can cache create web pages by putting different pages
-together. These *fragments* can have individual cache policies. If you
-have a web site with a list showing the 5 most popular articles on
-your site, this list can probably be cached as a fragment and included
-in all the other pages. Used properly it can dramatically increase
-your hit rate and reduce the load on your servers. ESI looks like this::
-
-  <HTML>
-  <BODY>
-  The time is: <esi:include src="/cgi-bin/date.cgi"/>
-  at this very moment.
-  </BODY>
-  </HTML>
-
-ESI is processed in vcl_fetch by setting *do_esi* to true.::
-
-  sub vcl_fetch {
-      if (req.url == "/test.html") {
-	  set beresp.do_esi = true;  /* Do ESI processing */
-      }
-  }
diff --git a/doc/sphinx/users-guide/command_line.rst b/doc/sphinx/users-guide/command_line.rst
index 94d8f5c..318868f 100644
--- a/doc/sphinx/users-guide/command_line.rst
+++ b/doc/sphinx/users-guide/command_line.rst
@@ -1,5 +1,7 @@
 .. _users-guide-command-line:
 
+XXX: Total rewrite of this
+
 Starting Varnish
 ----------------
 
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index afd6e14..8a12461 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -3,14 +3,12 @@
 Edge Side Includes
 ------------------
 
-*Edge Side Includes* is a language to include *fragments* of web pages
-in other web pages. Think of it as HTML include statement that works
-over HTTP. 
-
-On most web sites a lot of content is shared between
-pages. Regenerating this content for every page view is wasteful and
-ESI tries to address that letting you decide the cache policy for
-each fragment individually.
+Varnish can cache create web pages by putting different pages
+together. These *fragments* can have individual cache policies. If you
+have a web site with a list showing the 5 most popular articles on
+your site, this list can probably be cached as a fragment and included
+in all the other pages. Used properly it can dramatically increase
+your hit rate and reduce the load on your servers. 
 
 In Varnish we've only implemented a small subset of ESI. As of 2.1 we
 have three ESI statements:
@@ -20,7 +18,8 @@ have three ESI statements:
  * <!--esi ...-->
 
 Content substitution based on variables and cookies is not implemented
-but is on the roadmap. 
+but is on the roadmap. At least if you look at the roadmap from a
+certain angle. During a full moon.
 
 Varnish will not process ESI instructions in HTML comments.
 
@@ -77,3 +76,11 @@ For example::
   <p>The full text of the license:</p>
   <esi:include src="http://example.com/LICENSE" />
   -->
+
+Doing ESI on JSON and other non-XMLish content
+----------------------------------------------
+
+Please note that Varnish will peek at the included content. If it
+doesn't start with a "<" Varnish assumes you didn't really mean to
+include it and disregard it. You can alter this behaviour by setting
+the esi_syntax parameter (see ref:`ref-varnishd`).
\ No newline at end of file
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index 024738b..dacb13d 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -5,13 +5,10 @@ Using Varnish
 %%%%%%%%%%%%%
 
 This guide is intended for system administrators managing Varnish
-Cache. The reader should know how to configure her web- or application
-server and have basic knowledge of the HTTP protocol. The reader
-should have Varnish Cache up and running with the default
-configuration.
+Cache. 
 
-The guide is split into short chapters, each chapter taking on a
-separate topic. Good luck.
+The guide is split into short chapters, each chapter explaining a
+separate topic.
 
 .. toctree:: :maxdepth: 1
 
@@ -24,13 +21,13 @@ separate topic. Good luck.
         increasing_your_hitrate
 	cookies
 	vary
+        hashing
 	purging
 	compression
 	esi
 	virtualized
 	websockets
 	devicedetection
-	advanced_backend_servers
         handling_misbehaving_servers
         advanced_topics
 	troubleshooting
diff --git a/doc/sphinx/users-guide/logging.rst b/doc/sphinx/users-guide/logging.rst
index f2e36fd..01d8f36 100644
--- a/doc/sphinx/users-guide/logging.rst
+++ b/doc/sphinx/users-guide/logging.rst
@@ -7,7 +7,8 @@ One of the really nice features in Varnish is how logging
 works. Instead of logging to normal log file Varnish logs to a shared
 memory segment. When the end of the segment is reached we start over,
 overwriting old data. This is much, much faster then logging to a file
-and it doesn't require disk space.
+and it doesn't require disk space. Besides it gives you much, much
+more information when you need it.
 
 The flip side is that if you forget to have a program actually write the
 logs to disk they will disappear.
@@ -64,5 +65,4 @@ want to know are:
  Only list transactions where the tag matches a regular expression. If
  it matches you will get the whole transaction.
 
-Now that Varnish seem to work OK it's time to put Varnish on port 80
-while we tune it.
+For more information on this topic please see ref:`ref-varnishlog`.
diff --git a/doc/sphinx/users-guide/sizing_your_cache.rst b/doc/sphinx/users-guide/sizing_your_cache.rst
index c19647c..7d48200 100644
--- a/doc/sphinx/users-guide/sizing_your_cache.rst
+++ b/doc/sphinx/users-guide/sizing_your_cache.rst
@@ -12,10 +12,10 @@ task. A few things to consider:
    to only cache images a little while or not to cache them at all if
    they are cheap to serve from the backend and you have a limited
    amount of memory.
- * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or some other
-   tool. If you have a lot of LRU activity then your cache is evicting
-   objects due to space constraints and you should consider increasing
-   the size of the cache.
+ * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or
+   some other tool. If you have a lot of LRU activity then your cache
+   is evicting objects due to space constraints and you should
+   consider increasing the size of the cache.
 
 Be aware that every object that is stored also carries overhead that
 is kept outside the actually storage area. So, even if you specify -s
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index e1df5f2..be24e7a 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -63,14 +63,11 @@ errors will be logged in syslog. It might look like this::
        Mar  8 13:23:43 smoke varnishd[15670]: Child cleanup complete
        Mar  8 13:23:43 smoke varnishd[15670]: child (15697) Started
 
-Specifically if you see the "Error in munmap" error on Linux you might
-want to increase the amount of maps available. Linux is limited to a
-maximum of 64k maps. Setting vm.max_max_count i sysctl.conf will
-enable you to increase this limit. You can inspect the number of maps
-your program is consuming by counting the lines in /proc/$PID/maps
-
-This is a rather odd thing to document here - but hopefully Google
-will serve you this page if you ever encounter this error. 
+In this situation the mother process assumes that the cache died and
+killed it off.
+
+XXX: Describe crashing child process and crashing mother process here too.
+XXX: panic.show
 
 Varnish gives me Guru meditation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/sphinx/users-guide/virtualized.rst b/doc/sphinx/users-guide/virtualized.rst
index 317d3e2..5fc59b6 100644
--- a/doc/sphinx/users-guide/virtualized.rst
+++ b/doc/sphinx/users-guide/virtualized.rst
@@ -1,4 +1,6 @@
 
+XXX: What is this doing here?
+
 Running Varnish in a virtualized environment
 --------------------------------------------
 



From perbu at varnish-cache.org  Sat Sep  8 08:36:06 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Sat, 08 Sep 2012 10:36:06 +0200
Subject: [master] 88b33f6 New docs structure according to the scoof ToC
Message-ID: <E1TAGWM-000115-Hk@project.varnish-software.com>

commit 88b33f6987008506c583d8757764dc534a25c678
Author: Per Buer <per.buer at gmail.com>
Date:   Sat Sep 8 10:36:08 2012 +0200

    New docs structure according to the scoof ToC

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 9f367c3..a663170 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -16,7 +16,7 @@ our tutorial - :ref:`tutorial-index`.
 Contents:
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    installation/index.rst
    tutorial/index.rst
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index e6857a8..87141bf 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -31,6 +31,8 @@ DESCRIPTION
 The varnishd daemon accepts HTTP requests from clients, passes them on to a backend server and caches the
 returned documents to better satisfy future requests for the same document.
 
+.. _ref-varnishd-options:
+
 OPTIONS
 =======
 
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index cdbab1d..3968811 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -1,7 +1,7 @@
 .. _tutorial-intro:
 
 What is Varnish?
--------------
+----------------
 
 Varnish Cache is a web application accelerator. It can also be called
 a HTTP reverse proxy. The next chapter :ref:`tutorial-web-accelerator`
@@ -27,14 +27,14 @@ where you want to get the content and how the request or response
 should be altered. 
 
 Supported plattforms
------------------
+--------------------
 
 Varnish is written to run on modern versions of Linux and FreeBSD and
 the best experience is had on those plattforms. Thanks to our
 contributors it also runs on NetBSD, OpenBSD and OS X.
 
 About the Varnish development process
--------------------------------
+-------------------------------------
 
 Varnish is a community driven project. The development is overseen by
 the Varnish Governing Board which currently consist of Poul-Henning
@@ -42,7 +42,7 @@ Kamp (Architect), Rogier Mulhuijzen (Fastly) and Kristian Lyngst?l
 (Varnish Software).
 
 Getting in touch
--------------
+----------------
 
 You can get in touch with us trough many channels. For real time chat
 you can reach us on IRC trough the server irc.linpro.net on the
diff --git a/doc/sphinx/tutorial/web_accelerator.rst b/doc/sphinx/tutorial/web_accelerator.rst
index 4d455ae..28f8365 100644
--- a/doc/sphinx/tutorial/web_accelerator.rst
+++ b/doc/sphinx/tutorial/web_accelerator.rst
@@ -3,6 +3,8 @@
 What is a web accelerator
 -------------------------
 
+Really.XXX.
+
 
 The problem
 -----------
diff --git a/doc/sphinx/users-guide/advanced_topics.rst b/doc/sphinx/users-guide/advanced_topics.rst
deleted file mode 100644
index 6368d6e..0000000
--- a/doc/sphinx/users-guide/advanced_topics.rst
+++ /dev/null
@@ -1,41 +0,0 @@
-.. _users-guide-advanced_topics:
-
-Advanced topics
----------------
-
-This guide has covered the basics in Varnish. If you read through
-it all you should now have the skills to run Varnish.
-
-Here is a short overview of topics that we haven't covered in the guide. 
-
-More VCL
-~~~~~~~~
-
-VCL is a bit more complex then what we've covered so far. There are a
-few more subroutines available and there a few actions that we haven't
-discussed. For a complete(ish) guide to VCL have a look at the VCL man
-page - ref:`reference-vcl`.
-
-Using In-line C to extend Varnish
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-(Here there be dragons)
-
-You can use *in-line C* to extend Varnish. Please note that you can
-seriously mess up Varnish this way. The C code runs within the Varnish
-Cache process so if your code generates a segfault the cache will crash.
-
-One of the first uses I saw of In-line C was logging to syslog.::
-
-	# The include statements must be outside the subroutines.
-	C{
-		#include <syslog.h>
-        }C
-	
-        sub vcl_something {
-                C{
-		        syslog(LOG_INFO, "Something happened at VCL line XX.");
-	        }C
-        }
-
-
diff --git a/doc/sphinx/users-guide/backend_servers.rst b/doc/sphinx/users-guide/backend_servers.rst
deleted file mode 100644
index 4f721aa..0000000
--- a/doc/sphinx/users-guide/backend_servers.rst
+++ /dev/null
@@ -1,197 +0,0 @@
-.. _users-guide-backend_servers:
-
-Backend servers
----------------
-
-Varnish has a concept of "backend" or "origin" servers. A backend
-server is the server providing the content Varnish will accelerate.
-
-Our first task is to tell Varnish where it can find its content. Start
-your favorite text editor and open the varnish default configuration
-file. If you installed from source this is
-/usr/local/etc/varnish/default.vcl, if you installed from a package it
-is probably /etc/varnish/default.vcl.
-
-Somewhere in the top there will be a section that looks a bit like this.::
-
-	  # backend default {
-	  #     .host = "127.0.0.1";
-	  #     .port = "8080";
-	  # }
-
-We comment in this bit of text making the text look like.::
-
-          backend default {
-                .host = "127.0.0.1";
-    		.port = "8080";
-	  }
-
-Now, this piece of configuration defines a backend in Varnish called
-*default*. When Varnish needs to get content from this backend it will
-connect to port 8080 on localhost (127.0.0.1).
-
-Varnish can have several backends defined and can you can even join
-several backends together into clusters of backends for load balancing
-purposes. 
-
-Multiple backends
------------------
-
-At some point you might need Varnish to cache content from several
-servers. You might want Varnish to map all the URL into one single
-host or not. There are lot of options.
-
-Lets say we need to introduce a Java application into out PHP web
-site. Lets say our Java application should handle URL beginning with
-/java/.
-
-We manage to get the thing up and running on port 8000. Now, lets have
-a look at the default.vcl.::
-
-  backend default {
-      .host = "127.0.0.1";
-      .port = "8080";
-  }
-
-We add a new backend.::
-
-  backend java {
-      .host = "127.0.0.1";
-      .port = "8000";
-  }
-
-Now we need tell where to send the difference URL. Lets look at vcl_recv.::
-
-  sub vcl_recv {
-      if (req.url ~ "^/java/") {
-          set req.backend = java;
-      } else {
-          set req.backend = default.
-      }
-  }
-
-It's quite simple, really. Lets stop and think about this for a
-moment. As you can see you can define how you choose backends based on
-really arbitrary data. You want to send mobile devices to a different
-backend? No problem. if (req.User-agent ~ /mobile/) .. should do the
-trick. 
-
-.. _users-guide-advanced_backend_servers-directors:
-
-Directors
----------
-
-You can also group several backend into a group of backends. These
-groups are called directors. This will give you increased performance
-and resilience. You can define several backends and group them
-together in a director.::
-
-	 backend server1 {
-	     .host = "192.168.0.10";
-	 }
-	 backend server2{
-	     .host = "192.168.0.10";
-	 }
-
-Now we create the director.::
-
-       	director example_director round-robin {
-        {
-                .backend = server1;
-        }
-	# server2
-        {
-                .backend = server2;
-        }
-	# foo
-	}
-
-
-This director is a round-robin director. This means the director will
-distribute the incoming requests on a round-robin basis. There is
-also a *random* director which distributes requests in a, you guessed
-it, random fashion.
-
-But what if one of your servers goes down? Can Varnish direct all the
-requests to the healthy server? Sure it can. This is where the Health
-Checks come into play.
-
-.. _users-guide-advanced_backend_servers-health:
-
-Health checks
--------------
-
-Lets set up a director with two backends and health checks. First lets
-define the backends.::
-
-       backend server1 {
-         .host = "server1.example.com";
-	 .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	   }
-         }
-       backend server2 {
-  	  .host = "server2.example.com";
-  	  .probe = {
-                .url = "/";
-                .interval = 5s;
-                .timeout = 1 s;
-                .window = 5;
-                .threshold = 3;
-	  }
-        }
-
-Whats new here is the probe. Varnish will check the health of each
-backend with a probe. The options are
-
-url
- What URL should varnish request.
-
-interval
- How often should we poll
-
-timeout
- What is the timeout of the probe
-
-window
- Varnish will maintain a *sliding window* of the results. Here the
- window has five checks.
-
-threshold 
- How many of the .window last polls must be good for the backend to be declared healthy.
-
-initial 
- How many of the of the probes a good when Varnish starts - defaults
- to the same amount as the threshold.
-
-Now we define the director.::
-
-  director example_director round-robin {
-        {
-                .backend = server1;
-        }
-        # server2 
-        {
-                .backend = server2;
-        }
-	
-        }
-
-You use this director just as you would use any other director or
-backend. Varnish will not send traffic to hosts that are marked as
-unhealthy. Varnish can also serve stale content if all the backends are
-down. See :ref:`users-guide-handling_misbehaving_servers` for more
-information on how to enable this.
-
-Please note that Varnish will keep probes active for all loaded
-VCLs. Varnish will coalesce probes that seem identical - so be careful
-not to change the probe config if you do a lot of VCL
-loading. Unloading the VCL will discard the probes.
-
-For more information on how to do this please see
-ref:`reference-vcl-director`.
-
diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst
new file mode 100644
index 0000000..b91e4f0
--- /dev/null
+++ b/doc/sphinx/users-guide/command-line.rst
@@ -0,0 +1,44 @@
+.. _users-guide-command-line:
+
+XXX: Total rewrite of this
+
+Command Line options
+--------------------
+
+I assume varnishd is in your path. You might want to run ``pkill
+varnishd`` to make sure varnishd isn't running. 
+
+Become root and type:
+
+``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
+
+I added a few options, lets go through them:
+
+``-f /usr/local/etc/varnish/default.vcl``
+ The -f options specifies what configuration varnishd should use.
+
+``-s malloc,1G``
+ The -s options chooses the storage type Varnish should use for
+ storing its content. I used the type *malloc*, which just uses memory
+ for storage. There are other backends as well, described in 
+ :ref:users-guide-storage. 1G specifies how much memory should be allocated 
+ - one gigabyte. 
+
+``-T 127.0.0.1:2000``
+ Varnish has a built-in text-based administration
+ interface. Activating the interface makes Varnish manageble without
+ stopping it. You can specify what interface the management interface
+ should listen to. Make sure you don't expose the management interface
+ to the world as you can easily gain root access to a system via the
+ Varnish management interface. I recommend tieing it to localhost. If
+ you have users on your system that you don't fully trust, use firewall
+ rules to restrict access to the interface to root only.
+
+``-a 0.0.0.0:8080``
+ I specify that I want Varnish to listen on port 8080 for incomming
+ HTTP requests. For a production environment you would probably make
+ Varnish listen on port 80, which is the default.
+
+For a complete list of the command line parameters please see
+:ref:`ref-varnishd-options`.
+
diff --git a/doc/sphinx/users-guide/command_line.rst b/doc/sphinx/users-guide/command_line.rst
deleted file mode 100644
index 318868f..0000000
--- a/doc/sphinx/users-guide/command_line.rst
+++ /dev/null
@@ -1,55 +0,0 @@
-.. _users-guide-command-line:
-
-XXX: Total rewrite of this
-
-Starting Varnish
-----------------
-
-I assume varnishd is in your path. You might want to run ``pkill
-varnishd`` to make sure varnishd isn't running. 
-
-Become root and type:
-
-``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080``
-
-I added a few options, lets go through them:
-
-``-f /usr/local/etc/varnish/default.vcl``
- The -f options specifies what configuration varnishd should use.
-
-``-s malloc,1G``
- The -s options chooses the storage type Varnish should use for
- storing its content. I used the type *malloc*, which just uses memory
- for storage. There are other backends as well, described in 
- :ref:users-guide-storage. 1G specifies how much memory should be allocated 
- - one gigabyte. 
-
-``-T 127.0.0.1:2000``
- Varnish has a built-in text-based administration
- interface. Activating the interface makes Varnish manageble without
- stopping it. You can specify what interface the management interface
- should listen to. Make sure you don't expose the management interface
- to the world as you can easily gain root access to a system via the
- Varnish management interface. I recommend tieing it to localhost. If
- you have users on your system that you don't fully trust, use firewall
- rules to restrict access to the interface to root only.
-
-``-a 0.0.0.0:8080``
- I specify that I want Varnish to listen on port 8080 for incomming
- HTTP requests. For a production environment you would probably make
- Varnish listen on port 80, which is the default.
-
-Now you have Varnish running. Let us make sure that it works
-properly. Use your browser to go to http://192.168.2.2:8080/
-(obviously, you should replace the IP address with one on your own
-system) - you should now see your web application running there.
-
-Whether or not the application actually goes faster when run through
-Varnish depends on a few factors. If you application uses cookies for
-every session (a lot of PHP and Java applications seem to send a
-session cookie if it is needed or not) or if it uses authentication
-chances are Varnish won't do much caching. Ignore that for the moment,
-we come back to that in :ref:`users-guide-increasing_your_hitrate`.
-
-Lets make sure that Varnish really does do something to your web
-site. To do that we'll take a look at the logs.
diff --git a/doc/sphinx/users-guide/configuration.rst b/doc/sphinx/users-guide/configuration.rst
new file mode 100644
index 0000000..e4f3e00
--- /dev/null
+++ b/doc/sphinx/users-guide/configuration.rst
@@ -0,0 +1,12 @@
+Configuration
+=============
+
+This should deal with 
+
+.. toctree::
+   :maxdepth: 2
+
+   command-line
+   storage-backends
+   params
+
diff --git a/doc/sphinx/users-guide/handling_misbehaving_servers.rst b/doc/sphinx/users-guide/handling_misbehaving_servers.rst
deleted file mode 100644
index bb5fd35..0000000
--- a/doc/sphinx/users-guide/handling_misbehaving_servers.rst
+++ /dev/null
@@ -1,103 +0,0 @@
-.. _users-guide-handling_misbehaving_servers:
-
-Misbehaving servers
--------------------
-
-A key feature of Varnish is its ability to shield you from misbehaving
-web- and application servers.
-
-
-
-Grace mode
-~~~~~~~~~~
-
-When several clients are requesting the same page Varnish will send
-one request to the backend and place the others on hold while fetching
-one copy from the back end. In some products this is called request
-coalescing and Varnish does this automatically.
-
-If you are serving thousands of hits per second the queue of waiting
-requests can get huge. There are two potential problems - one is a
-thundering herd problem - suddenly releasing a thousand threads to
-serve content might send the load sky high. Secondly - nobody likes to
-wait. To deal with this we can instruct Varnish to keep
-the objects in cache beyond their TTL and to serve the waiting
-requests somewhat stale content.
-
-So, in order to serve stale content we must first have some content to
-serve. So to make Varnish keep all objects for 30 minutes beyond their
-TTL use the following VCL::
-
-  sub vcl_fetch {
-    set beresp.grace = 30m;
-  }
-
-Varnish still won't serve the stale objects. In order to enable
-Varnish to actually serve the stale object we must enable this on the
-request. Lets us say that we accept serving 15s old object.::
-
-  sub vcl_recv {
-    set req.grace = 15s;
-  }
-
-You might wonder why we should keep the objects in the cache for 30
-minutes if we are unable to serve them? Well, if you have enabled
-:ref:`users-guide-advanced_backend_servers-health` you can check if the
-backend is sick and if it is we can serve the stale content for a bit
-longer.::
-
-   if (! req.backend.healthy) {
-      set req.grace = 5m;
-   } else {
-      set req.grace = 15s;
-   }
-
-So, to sum up, grace mode solves two problems:
- * it serves stale content to avoid request pile-up.
- * it serves stale content if the backend is not healthy.
-
-Saint mode
-~~~~~~~~~~
-
-Sometimes servers get flaky. They start throwing out random
-errors. You can instruct Varnish to try to handle this in a
-more-than-graceful way - enter *Saint mode*. Saint mode enables you to
-discard a certain page from one backend server and either try another
-server or serve stale content from cache. Lets have a look at how this
-can be enabled in VCL::
-
-  sub vcl_fetch {
-    if (beresp.status == 500) { 
-      set beresp.saintmode = 10s;
-      return(restart);
-    }
-    set beresp.grace = 5m;
-  } 
-
-When we set beresp.saintmode to 10 seconds Varnish will not ask *that*
-server for URL for 10 seconds. A blacklist, more or less. Also a
-restart is performed so if you have other backends capable of serving
-that content Varnish will try those. When you are out of backends
-Varnish will serve the content from its stale cache.
-
-This can really be a life saver.
-
-Known limitations on grace- and saint mode
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If your request fails while it is being fetched you're thrown into
-vcl_error. vcl_error has access to a rather limited set of data so you
-can't enable saint mode or grace mode here. This will be addressed in a
-future release but a work-around available.
-
-* Declare a backend that is always sick.
-* Set a magic marker in vcl_error
-* Restart the transaction
-* Note the magic marker in vcl_recv and set the backend to the one mentioned
-* Varnish will now serve stale data is any is available
-
-
-God mode
-~~~~~~~~
-Not implemented yet. :-)
-
diff --git a/doc/sphinx/users-guide/hashing.rst b/doc/sphinx/users-guide/hashing.rst
deleted file mode 100644
index b7bbe62..0000000
--- a/doc/sphinx/users-guide/hashing.rst
+++ /dev/null
@@ -1,51 +0,0 @@
-
-Hashing
--------
-
-Internally, when Varnish stores content in it's store it uses a hash
-key to find the object again. In the default setup this key is
-calculated based on the content of the *Host* header or the IP adress
-of the server and the URL.
-
-Behold the default vcl.::
-
- sub vcl_hash {
-     hash_data(req.url);
-     if (req.http.host) {
-         hash_data(req.http.host);
-     } else {
-         hash_data(server.ip);
-     }
-     return (hash);
- }
-
-As you can see it first chucks in req.url then req.http.host if it
-exsists. It is worth pointing out that Varnish doesn't lowercase the
-hostname or the URL before hashing it so in thery having Varnish.org/
-and varnish.org/ would result in different cache entries. Browers
-however, tend to lowercase hostnames.
-
-You can change what goes into the hash. This way you can make Varnish
-serve up different content to different clients based on arbitrary
-criteria.
-
-Let's say you want to serve pages in different languages to your users
-based on where their IP address is located. You would need some Vmod
-to get a country code and then put it into the hash. It might look
-like this.
-
-In vcl_recv::
-
-  set req.http.X-Country-Code = geoip.lookup(client.ip);
-
-And then add a vcl_hash:
-
-sub vcl_hash {
-  hash_data(req.http.X-Country-Code);
-}
-
-As the default VCL will take care of adding the host and URL to the
-hash we don't have to do anything else. Be careful calling
-return(hash) as this will abort the execution of the default VCL and
-thereby you can end up with a Varnish that will return data based on
-more or less random inputs.
diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
new file mode 100644
index 0000000..462a781
--- /dev/null
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -0,0 +1,213 @@
+.. _users-guide-increasing_your_hitrate:
+
+Achieving a high hitrate
+------------------------
+
+Now that Varnish is up and running, and you can access your web
+application through Varnish. Unless your application is specifically
+written to work behind a web accelerator you'll probably need to do
+some changes to either the configuration or the application in order
+to get a high hit rate in Varnish.
+
+Varnish will not cache your data unless it's absolutely sure it is
+safe to do so. So, for you to understand how Varnish decides if and
+how to cache a page, I'll guide you through a couple of tools that you
+will find useful.
+
+Note that you need a tool to see what HTTP headers fly between you and
+the web server. On the Varnish server, the easiest is to use
+varnishlog and varnishtop but sometimes a client-side tool makes
+sense. Here are the ones I use.
+
+Tool: varnishtop
+~~~~~~~~~~~~~~~~
+
+You can use varnishtop to identify what URLs are hitting the backend
+the most. ``varnishtop -i txurl`` is an essential command. You can see
+some other examples of varnishtop usage in :ref:`users-guide-statistics`.
+
+
+Tool: varnishlog
+~~~~~~~~~~~~~~~~
+
+When you have identified the an URL which is frequently sent to the
+backend you can use varnishlog to have a look at the request.
+``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
+coming from the client (-c) matching /foo/bar.
+
+For more information on how varnishlog works please see
+:ref:`users-guide-logging` or man :ref:`ref-varnishlog`.
+
+For extended diagnostics headers, see
+http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
+
+
+Tool: lwp-request
+~~~~~~~~~~~~~~~~~
+
+lwp-request is part of The World-Wide Web library for Perl. It's a
+couple of really basic programs that can execute an HTTP request and
+give you the result. I mostly use two programs, GET and HEAD.
+
+vg.no was the first site to use Varnish and the people running Varnish
+there are quite clueful. So it's interesting to look at their HTTP
+Headers. Let's send a GET request for their home page::
+
+  $ GET -H 'Host: www.vg.no' -Used http://vg.no/
+  GET http://vg.no/
+  Host: www.vg.no
+  User-Agent: lwp-request/5.834 libwww-perl/5.834
+  
+  200 OK
+  Cache-Control: must-revalidate
+  Refresh: 600
+  Title: VG Nett - Forsiden - VG Nett
+  X-Age: 463
+  X-Cache: HIT
+  X-Rick-Would-Never: Let you down
+  X-VG-Jobb: http://www.finn.no/finn/job/fulltime/result?keyword=vg+multimedia Merk:HeaderNinja
+  X-VG-Korken: http://www.youtube.com/watch?v=Fcj8CnD5188
+  X-VG-WebCache: joanie
+  X-VG-WebServer: leon
+
+OK. Let me explain what it does. GET usually sends off HTTP 0.9
+requests, which lack the Host header. So I add a Host header with the
+-H option. -U print request headers, -s prints response status, -e
+prints response headers and -d discards the actual content. We don't
+really care about the content, only the headers.
+
+As you can see, VG adds quite a bit of information in their
+headers. Some of the headers, like the X-Rick-Would-Never are specific
+to vg.no and their somewhat odd sense of humour. Others, like the
+X-VG-Webcache are for debugging purposes. 
+
+So, to check whether a site sets cookies for a specific URL, just do::
+
+  GET -Used http://example.com/ |grep ^Set-Cookie
+
+Tool: Live HTTP Headers
+~~~~~~~~~~~~~~~~~~~~~~~
+
+There is also a plugin for Firefox. *Live HTTP Headers* can show you
+what headers are being sent and recieved. Live HTTP Headers can be
+found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
+googling "Live HTTP Headers".
+
+
+The role of HTTP Headers
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Along with each HTTP request and response comes a bunch of headers
+carrying metadata. Varnish will look at these headers to determine if
+it is appropriate to cache the contents and how long Varnish can keep
+the content.
+
+Please note that when considering these headers Varnish actually
+considers itself *part of* the actual webserver. The rationale being
+that both are under your control. 
+
+The term *surrogate origin cache* is not really well defined by the
+IETF so RFC 2616 so the various ways Varnish works might differ from
+your expectations.
+
+Let's take a look at the important headers you should be aware of:
+
+Cache-Control
+~~~~~~~~~~~~~
+
+The Cache-Control instructs caches how to handle the content. Varnish
+cares about the *max-age* parameter and uses it to calculate the TTL
+for an object. 
+
+"Cache-Control: nocache" is ignored but if you need this you can
+easily add support for it.
+
+So make sure you issue a Cache-Control header with a max-age
+header. You can have a look at what Varnish Software's drupal server
+issues::
+
+  $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control
+  Cache-Control: public, max-age=600
+
+Age
+~~~
+
+Varnish adds an Age header to indicate how long the object has been
+kept inside Varnish. You can grep out Age from varnishlog like this::
+
+  varnishlog -i TxHeader -I ^Age
+
+Pragma
+~~~~~~
+
+An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
+header. You could easily add support for this header in VCL.
+
+In vcl_fetch::
+
+  if (beresp.http.Pragma ~ "nocache") {
+     return(hit_for_pass);
+  }
+
+Authorization
+~~~~~~~~~~~~~
+
+If Varnish sees an Authorization header it will pass the request. If
+this is not what you want you can unset the header.
+
+Overriding the time-to-live (ttl)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes your backend will misbehave. It might, depending on your
+setup, be easier to override the ttl in Varnish than to fix your
+somewhat cumbersome backend. 
+
+You need VCL to identify the objects you want and then you set the
+beresp.ttl to whatever you want::
+
+  sub vcl_fetch {
+      if (req.url ~ "^/legacy_broken_cms/") {
+          set beresp.ttl = 5d;
+      }
+  }
+
+The example will set the TTL to 5 days for the old legacy stuff on
+your site.
+
+Forcing caching for certain requests and certain responses
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since you still have this cumbersome backend that isn't very friendly
+to work with you might want to override more stuff in Varnish. We
+recommend that you rely as much as you can on the default caching
+rules. It is perfectly easy to force Varnish to lookup an object in
+the cache but it isn't really recommended.
+
+
+Normalizing your namespace
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some sites are accessed via lots of
+hostnames. http://www.varnish-software.com/,
+http://varnish-software.com/ and http://varnishsoftware.com/ all point
+at the same site. Since Varnish doesn't know they are different,
+Varnish will cache different versions of every page for every
+hostname. You can mitigate this in your web server configuration by
+setting up redirects or by using the following VCL::
+
+  if (req.http.host ~ "(?i)^(www.)?varnish-?software.com") {
+    set req.http.host = "varnish-software.com";
+  }
+
+
+Ways of increasing your hitrate even more
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following chapters should give your ways of further increasing
+your hitrate, especially the chapter on Cookies.
+
+ * :ref:`users-guide-cookies`
+ * :ref:`users-guide-vary`
+ * :ref:`users-guide-purging`
+ * :ref:`users-guide-esi`
+
diff --git a/doc/sphinx/users-guide/increasing_your_hitrate.rst b/doc/sphinx/users-guide/increasing_your_hitrate.rst
deleted file mode 100644
index 462a781..0000000
--- a/doc/sphinx/users-guide/increasing_your_hitrate.rst
+++ /dev/null
@@ -1,213 +0,0 @@
-.. _users-guide-increasing_your_hitrate:
-
-Achieving a high hitrate
-------------------------
-
-Now that Varnish is up and running, and you can access your web
-application through Varnish. Unless your application is specifically
-written to work behind a web accelerator you'll probably need to do
-some changes to either the configuration or the application in order
-to get a high hit rate in Varnish.
-
-Varnish will not cache your data unless it's absolutely sure it is
-safe to do so. So, for you to understand how Varnish decides if and
-how to cache a page, I'll guide you through a couple of tools that you
-will find useful.
-
-Note that you need a tool to see what HTTP headers fly between you and
-the web server. On the Varnish server, the easiest is to use
-varnishlog and varnishtop but sometimes a client-side tool makes
-sense. Here are the ones I use.
-
-Tool: varnishtop
-~~~~~~~~~~~~~~~~
-
-You can use varnishtop to identify what URLs are hitting the backend
-the most. ``varnishtop -i txurl`` is an essential command. You can see
-some other examples of varnishtop usage in :ref:`users-guide-statistics`.
-
-
-Tool: varnishlog
-~~~~~~~~~~~~~~~~
-
-When you have identified the an URL which is frequently sent to the
-backend you can use varnishlog to have a look at the request.
-``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
-coming from the client (-c) matching /foo/bar.
-
-For more information on how varnishlog works please see
-:ref:`users-guide-logging` or man :ref:`ref-varnishlog`.
-
-For extended diagnostics headers, see
-http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader
-
-
-Tool: lwp-request
-~~~~~~~~~~~~~~~~~
-
-lwp-request is part of The World-Wide Web library for Perl. It's a
-couple of really basic programs that can execute an HTTP request and
-give you the result. I mostly use two programs, GET and HEAD.
-
-vg.no was the first site to use Varnish and the people running Varnish
-there are quite clueful. So it's interesting to look at their HTTP
-Headers. Let's send a GET request for their home page::
-
-  $ GET -H 'Host: www.vg.no' -Used http://vg.no/
-  GET http://vg.no/
-  Host: www.vg.no
-  User-Agent: lwp-request/5.834 libwww-perl/5.834
-  
-  200 OK
-  Cache-Control: must-revalidate
-  Refresh: 600
-  Title: VG Nett - Forsiden - VG Nett
-  X-Age: 463
-  X-Cache: HIT
-  X-Rick-Would-Never: Let you down
-  X-VG-Jobb: http://www.finn.no/finn/job/fulltime/result?keyword=vg+multimedia Merk:HeaderNinja
-  X-VG-Korken: http://www.youtube.com/watch?v=Fcj8CnD5188
-  X-VG-WebCache: joanie
-  X-VG-WebServer: leon
-
-OK. Let me explain what it does. GET usually sends off HTTP 0.9
-requests, which lack the Host header. So I add a Host header with the
--H option. -U print request headers, -s prints response status, -e
-prints response headers and -d discards the actual content. We don't
-really care about the content, only the headers.
-
-As you can see, VG adds quite a bit of information in their
-headers. Some of the headers, like the X-Rick-Would-Never are specific
-to vg.no and their somewhat odd sense of humour. Others, like the
-X-VG-Webcache are for debugging purposes. 
-
-So, to check whether a site sets cookies for a specific URL, just do::
-
-  GET -Used http://example.com/ |grep ^Set-Cookie
-
-Tool: Live HTTP Headers
-~~~~~~~~~~~~~~~~~~~~~~~
-
-There is also a plugin for Firefox. *Live HTTP Headers* can show you
-what headers are being sent and recieved. Live HTTP Headers can be
-found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
-googling "Live HTTP Headers".
-
-
-The role of HTTP Headers
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Along with each HTTP request and response comes a bunch of headers
-carrying metadata. Varnish will look at these headers to determine if
-it is appropriate to cache the contents and how long Varnish can keep
-the content.
-
-Please note that when considering these headers Varnish actually
-considers itself *part of* the actual webserver. The rationale being
-that both are under your control. 
-
-The term *surrogate origin cache* is not really well defined by the
-IETF so RFC 2616 so the various ways Varnish works might differ from
-your expectations.
-
-Let's take a look at the important headers you should be aware of:
-
-Cache-Control
-~~~~~~~~~~~~~
-
-The Cache-Control instructs caches how to handle the content. Varnish
-cares about the *max-age* parameter and uses it to calculate the TTL
-for an object. 
-
-"Cache-Control: nocache" is ignored but if you need this you can
-easily add support for it.
-
-So make sure you issue a Cache-Control header with a max-age
-header. You can have a look at what Varnish Software's drupal server
-issues::
-
-  $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control
-  Cache-Control: public, max-age=600
-
-Age
-~~~
-
-Varnish adds an Age header to indicate how long the object has been
-kept inside Varnish. You can grep out Age from varnishlog like this::
-
-  varnishlog -i TxHeader -I ^Age
-
-Pragma
-~~~~~~
-
-An HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
-header. You could easily add support for this header in VCL.
-
-In vcl_fetch::
-
-  if (beresp.http.Pragma ~ "nocache") {
-     return(hit_for_pass);
-  }
-
-Authorization
-~~~~~~~~~~~~~
-
-If Varnish sees an Authorization header it will pass the request. If
-this is not what you want you can unset the header.
-
-Overriding the time-to-live (ttl)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Sometimes your backend will misbehave. It might, depending on your
-setup, be easier to override the ttl in Varnish than to fix your
-somewhat cumbersome backend. 
-
-You need VCL to identify the objects you want and then you set the
-beresp.ttl to whatever you want::
-
-  sub vcl_fetch {
-      if (req.url ~ "^/legacy_broken_cms/") {
-          set beresp.ttl = 5d;
-      }
-  }
-
-The example will set the TTL to 5 days for the old legacy stuff on
-your site.
-
-Forcing caching for certain requests and certain responses
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Since you still have this cumbersome backend that isn't very friendly
-to work with you might want to override more stuff in Varnish. We
-recommend that you rely as much as you can on the default caching
-rules. It is perfectly easy to force Varnish to lookup an object in
-the cache but it isn't really recommended.
-
-
-Normalizing your namespace
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Some sites are accessed via lots of
-hostnames. http://www.varnish-software.com/,
-http://varnish-software.com/ and http://varnishsoftware.com/ all point
-at the same site. Since Varnish doesn't know they are different,
-Varnish will cache different versions of every page for every
-hostname. You can mitigate this in your web server configuration by
-setting up redirects or by using the following VCL::
-
-  if (req.http.host ~ "(?i)^(www.)?varnish-?software.com") {
-    set req.http.host = "varnish-software.com";
-  }
-
-
-Ways of increasing your hitrate even more
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following chapters should give your ways of further increasing
-your hitrate, especially the chapter on Cookies.
-
- * :ref:`users-guide-cookies`
- * :ref:`users-guide-vary`
- * :ref:`users-guide-purging`
- * :ref:`users-guide-esi`
-
diff --git a/doc/sphinx/users-guide/index.rst b/doc/sphinx/users-guide/index.rst
index dacb13d..f3f4a2c 100644
--- a/doc/sphinx/users-guide/index.rst
+++ b/doc/sphinx/users-guide/index.rst
@@ -1,8 +1,8 @@
 .. _users-guide-index:
 
-%%%%%%%%%%%%%
-Using Varnish
-%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+Using Varnish - A Users Guide
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 This guide is intended for system administrators managing Varnish
 Cache. 
@@ -10,13 +10,24 @@ Cache.
 The guide is split into short chapters, each chapter explaining a
 separate topic.
 
-.. toctree:: :maxdepth: 1
+.. toctree::
+   :maxdepth: 3
 
+   configuration
+   vcl
+   operation
+   troubleshooting
+
+.. customizing (which is a non ideal title)
+
+.. No longer used:
+
+        configuration
         command_line
+        VCL
 	backend_servers
 	logging
         sizing_your_cache
-	vcl
         statistics
         increasing_your_hitrate
 	cookies
diff --git a/doc/sphinx/users-guide/logging.rst b/doc/sphinx/users-guide/logging.rst
deleted file mode 100644
index 01d8f36..0000000
--- a/doc/sphinx/users-guide/logging.rst
+++ /dev/null
@@ -1,68 +0,0 @@
-.. _users-guide-logging:
-
-Logging in Varnish
-------------------
-
-One of the really nice features in Varnish is how logging
-works. Instead of logging to normal log file Varnish logs to a shared
-memory segment. When the end of the segment is reached we start over,
-overwriting old data. This is much, much faster then logging to a file
-and it doesn't require disk space. Besides it gives you much, much
-more information when you need it.
-
-The flip side is that if you forget to have a program actually write the
-logs to disk they will disappear.
-
-varnishlog is one of the programs you can use to look at what Varnish
-is logging. Varnishlog gives you the raw logs, everything that is
-written to the logs. There are other clients as well, we'll show you
-these later.
-
-In the terminal window you started varnish now type *varnishlog* and
-press enter.
-
-You'll see lines like these scrolling slowly by.::
-
-    0 CLI          - Rd ping
-    0 CLI          - Wr 200 PONG 1273698726 1.0
-
-These is the Varnish master process checking up on the caching process
-to see that everything is OK. 
-
-Now go to the browser and reload the page displaying your web
-app. You'll see lines like these.::
-
-   11 SessionOpen  c 127.0.0.1 58912 0.0.0.0:8080
-   11 ReqStart     c 127.0.0.1 58912 595005213
-   11 RxRequest    c GET
-   11 RxURL        c /
-   11 RxProtocol   c HTTP/1.1
-   11 RxHeader     c Host: localhost:8080
-   11 RxHeader     c Connection: keep-alive
-
-The first column is an arbitrary number, it defines the request. Lines
-with the same number are part of the same HTTP transaction. The second
-column is the *tag* of the log message. All log entries are tagged
-with a tag indicating what sort of activity is being logged. Tags
-starting with Rx indicate Varnish is recieving data and Tx indicates
-sending data.
-
-The third column tell us whether this is is data coming or going to
-the client (c) or to/from the backend (b). The forth column is the
-data being logged.
-
-Now, you can filter quite a bit with varnishlog. The basic option you
-want to know are:
-
--b
- Only show log lines from traffic going between Varnish and the backend 
- servers. This will be useful when we want to optimize cache hit rates.
-
--c 
- Same as -b but for client side traffic.
-
--m tag:regex
- Only list transactions where the tag matches a regular expression. If
- it matches you will get the whole transaction.
-
-For more information on this topic please see ref:`ref-varnishlog`.
diff --git a/doc/sphinx/users-guide/operation-cli.rst b/doc/sphinx/users-guide/operation-cli.rst
new file mode 100644
index 0000000..5cb40a5
--- /dev/null
+++ b/doc/sphinx/users-guide/operation-cli.rst
@@ -0,0 +1,6 @@
+
+
+Varnishadm
+----------
+
+You connect to it and everything becomes awesome.
diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
new file mode 100644
index 0000000..01d8f36
--- /dev/null
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -0,0 +1,68 @@
+.. _users-guide-logging:
+
+Logging in Varnish
+------------------
+
+One of the really nice features in Varnish is how logging
+works. Instead of logging to normal log file Varnish logs to a shared
+memory segment. When the end of the segment is reached we start over,
+overwriting old data. This is much, much faster then logging to a file
+and it doesn't require disk space. Besides it gives you much, much
+more information when you need it.
+
+The flip side is that if you forget to have a program actually write the
+logs to disk they will disappear.
+
+varnishlog is one of the programs you can use to look at what Varnish
+is logging. Varnishlog gives you the raw logs, everything that is
+written to the logs. There are other clients as well, we'll show you
+these later.
+
+In the terminal window you started varnish now type *varnishlog* and
+press enter.
+
+You'll see lines like these scrolling slowly by.::
+
+    0 CLI          - Rd ping
+    0 CLI          - Wr 200 PONG 1273698726 1.0
+
+These is the Varnish master process checking up on the caching process
+to see that everything is OK. 
+
+Now go to the browser and reload the page displaying your web
+app. You'll see lines like these.::
+
+   11 SessionOpen  c 127.0.0.1 58912 0.0.0.0:8080
+   11 ReqStart     c 127.0.0.1 58912 595005213
+   11 RxRequest    c GET
+   11 RxURL        c /
+   11 RxProtocol   c HTTP/1.1
+   11 RxHeader     c Host: localhost:8080
+   11 RxHeader     c Connection: keep-alive
+
+The first column is an arbitrary number, it defines the request. Lines
+with the same number are part of the same HTTP transaction. The second
+column is the *tag* of the log message. All log entries are tagged
+with a tag indicating what sort of activity is being logged. Tags
+starting with Rx indicate Varnish is recieving data and Tx indicates
+sending data.
+
+The third column tell us whether this is is data coming or going to
+the client (c) or to/from the backend (b). The forth column is the
+data being logged.
+
+Now, you can filter quite a bit with varnishlog. The basic option you
+want to know are:
+
+-b
+ Only show log lines from traffic going between Varnish and the backend 
+ servers. This will be useful when we want to optimize cache hit rates.
+
+-c 
+ Same as -b but for client side traffic.
+
+-m tag:regex
+ Only list transactions where the tag matches a regular expression. If
+ it matches you will get the whole transaction.
+
+For more information on this topic please see ref:`ref-varnishlog`.
diff --git a/doc/sphinx/users-guide/operation-statistics.rst b/doc/sphinx/users-guide/operation-statistics.rst
new file mode 100644
index 0000000..ad57037
--- /dev/null
+++ b/doc/sphinx/users-guide/operation-statistics.rst
@@ -0,0 +1,57 @@
+.. _users-guide-statistics:
+
+
+Statistics
+----------
+
+Now that your varnish is up and running let's have a look at how it is
+doing. There are several tools that can help.
+
+varnishtop
+~~~~~~~~~~
+
+The varnishtop utility reads the shared memory logs and presents a
+continuously updated list of the most commonly occurring log entries.
+
+With suitable filtering using the -I, -i, -X and -x options, it can be
+used to display a ranking of requested documents, clients, user
+agents, or any other information which is recorded in the log.
+
+``varnishtop -i rxurl`` will show you what URLs are being asked for
+by the client. ``varnishtop -i txurl`` will show you what your backend
+is being asked the most. ``varnishtop -i RxHeader -I
+Accept-Encoding`` will show the most popular Accept-Encoding header
+the client are sending you.
+
+varnishhist
+~~~~~~~~~~~
+
+The varnishhist utility reads varnishd(1) shared memory logs and
+presents a continuously updated histogram showing the distribution of
+the last N requests by their processing.  The value of N and the
+vertical scale are displayed in the top left corner.  The horizontal
+scale is logarithmic.  Hits are marked with a pipe character ("|"),
+and misses are marked with a hash character ("#").
+
+
+varnishsizes
+~~~~~~~~~~~~
+
+Varnishsizes does the same as varnishhist, except it shows the size of
+the objects and not the time take to complete the request. This gives
+you a good overview of how big the objects you are serving are.
+
+
+varnishstat
+~~~~~~~~~~~
+
+Varnish has lots of counters. We count misses, hits, information about
+the storage, threads created, deleted objects. Just about
+everything. varnishstat will dump these counters. This is useful when
+tuning varnish. 
+
+There are programs that can poll varnishstat regularly and make nice
+graphs of these counters. One such program is Munin. Munin can be
+found at http://munin-monitoring.org/ . There is a plugin for munin in
+the varnish source code.
+
diff --git a/doc/sphinx/users-guide/operation.rst b/doc/sphinx/users-guide/operation.rst
new file mode 100644
index 0000000..c7c74f9
--- /dev/null
+++ b/doc/sphinx/users-guide/operation.rst
@@ -0,0 +1,17 @@
+Operation
+=========
+
+.. toctree::
+   :maxdepth: 2
+
+   operation-logging
+   operation-statistics
+   operation-cli
+   purging
+   sizing-your-cache
+   increasing-your-hitrate
+   compression
+   esi
+   vary
+   cookies
+   virtualized
\ No newline at end of file
diff --git a/doc/sphinx/users-guide/params.rst b/doc/sphinx/users-guide/params.rst
new file mode 100644
index 0000000..a0123ec
--- /dev/null
+++ b/doc/sphinx/users-guide/params.rst
@@ -0,0 +1,4 @@
+
+
+Parameters
+----------
\ No newline at end of file
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 15b85d2..53facd3 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -1,8 +1,8 @@
 .. _users-guide-purging:
 
-=====================
- Purging and banning
-=====================
+
+Purging and banning
+-------------------
 
 One of the most effective ways of increasing your hit ratio is to
 increase the time-to-live (ttl) of your objects. But, as you're aware
@@ -15,7 +15,7 @@ banning and forced cache misses. First, let me explain the HTTP purges.
 
 
 HTTP Purges
-===========
+-----------
 
 A *purge* is what happens when you pick out an object from the cache
 and discard it along with its variants. Usually a purge is invoked
@@ -75,7 +75,7 @@ And Varnish would then discard the front page. This will remove all
 variants as defined by Vary.
 
 Bans
-====
+----
 
 There is another way to invalidate content: Bans. You can think of
 bans as a sort of a filter on objects already in the cache. You *ban*
@@ -164,7 +164,7 @@ be marked as Gone if it is a duplicate ban, but is still kept in the list
 for optimization purposes.
 
 Forcing a cache miss
-====================
+--------------------
 
 The final way to invalidate an object is a method that allows you to
 refresh an object by forcing a hash miss for a single request. If you set
diff --git a/doc/sphinx/users-guide/sizing-your-cache.rst b/doc/sphinx/users-guide/sizing-your-cache.rst
new file mode 100644
index 0000000..7d48200
--- /dev/null
+++ b/doc/sphinx/users-guide/sizing-your-cache.rst
@@ -0,0 +1,25 @@
+
+Sizing your cache
+-----------------
+
+Picking how much memory you should give Varnish can be a tricky
+task. A few things to consider:
+
+ * How big is your *hot* data set. For a portal or news site that
+   would be the size of the front page with all the stuff on it, and
+   the size of all the pages and objects linked from the first page. 
+ * How expensive is it to generate an object? Sometimes it makes sense
+   to only cache images a little while or not to cache them at all if
+   they are cheap to serve from the backend and you have a limited
+   amount of memory.
+ * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or
+   some other tool. If you have a lot of LRU activity then your cache
+   is evicting objects due to space constraints and you should
+   consider increasing the size of the cache.
+
+Be aware that every object that is stored also carries overhead that
+is kept outside the actually storage area. So, even if you specify -s
+malloc,16G varnish might actually use **double** that. Varnish has a
+overhead of about 1k per object. So, if you have lots of small objects
+in your cache the overhead might be significant.
+
diff --git a/doc/sphinx/users-guide/sizing_your_cache.rst b/doc/sphinx/users-guide/sizing_your_cache.rst
deleted file mode 100644
index 7d48200..0000000
--- a/doc/sphinx/users-guide/sizing_your_cache.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
-Sizing your cache
------------------
-
-Picking how much memory you should give Varnish can be a tricky
-task. A few things to consider:
-
- * How big is your *hot* data set. For a portal or news site that
-   would be the size of the front page with all the stuff on it, and
-   the size of all the pages and objects linked from the first page. 
- * How expensive is it to generate an object? Sometimes it makes sense
-   to only cache images a little while or not to cache them at all if
-   they are cheap to serve from the backend and you have a limited
-   amount of memory.
- * Watch the n_lru_nuked counter with :ref:`reference-varnishstat` or
-   some other tool. If you have a lot of LRU activity then your cache
-   is evicting objects due to space constraints and you should
-   consider increasing the size of the cache.
-
-Be aware that every object that is stored also carries overhead that
-is kept outside the actually storage area. So, even if you specify -s
-malloc,16G varnish might actually use **double** that. Varnish has a
-overhead of about 1k per object. So, if you have lots of small objects
-in your cache the overhead might be significant.
-
diff --git a/doc/sphinx/users-guide/statistics.rst b/doc/sphinx/users-guide/statistics.rst
deleted file mode 100644
index ad57037..0000000
--- a/doc/sphinx/users-guide/statistics.rst
+++ /dev/null
@@ -1,57 +0,0 @@
-.. _users-guide-statistics:
-
-
-Statistics
-----------
-
-Now that your varnish is up and running let's have a look at how it is
-doing. There are several tools that can help.
-
-varnishtop
-~~~~~~~~~~
-
-The varnishtop utility reads the shared memory logs and presents a
-continuously updated list of the most commonly occurring log entries.
-
-With suitable filtering using the -I, -i, -X and -x options, it can be
-used to display a ranking of requested documents, clients, user
-agents, or any other information which is recorded in the log.
-
-``varnishtop -i rxurl`` will show you what URLs are being asked for
-by the client. ``varnishtop -i txurl`` will show you what your backend
-is being asked the most. ``varnishtop -i RxHeader -I
-Accept-Encoding`` will show the most popular Accept-Encoding header
-the client are sending you.
-
-varnishhist
-~~~~~~~~~~~
-
-The varnishhist utility reads varnishd(1) shared memory logs and
-presents a continuously updated histogram showing the distribution of
-the last N requests by their processing.  The value of N and the
-vertical scale are displayed in the top left corner.  The horizontal
-scale is logarithmic.  Hits are marked with a pipe character ("|"),
-and misses are marked with a hash character ("#").
-
-
-varnishsizes
-~~~~~~~~~~~~
-
-Varnishsizes does the same as varnishhist, except it shows the size of
-the objects and not the time take to complete the request. This gives
-you a good overview of how big the objects you are serving are.
-
-
-varnishstat
-~~~~~~~~~~~
-
-Varnish has lots of counters. We count misses, hits, information about
-the storage, threads created, deleted objects. Just about
-everything. varnishstat will dump these counters. This is useful when
-tuning varnish. 
-
-There are programs that can poll varnishstat regularly and make nice
-graphs of these counters. One such program is Munin. Munin can be
-found at http://munin-monitoring.org/ . There is a plugin for munin in
-the varnish source code.
-
diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
new file mode 100644
index 0000000..5d576cf
--- /dev/null
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -0,0 +1,21 @@
+
+
+Storage backends
+----------------
+
+Intro
+~~~~~
+
+Malloc
+~~~~~~
+
+File
+~~~~
+
+Persistent
+~~~~~~~~~~
+
+Transient
+~~~~~~~~~
+
+
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index be24e7a..51797d8 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -1,14 +1,15 @@
 Troubleshooting Varnish
------------------------
+=======================
 
 Sometimes Varnish misbehaves. In order for you to understand whats
 going on there are a couple of places you can check. varnishlog,
 /var/log/syslog, /var/log/messages are all places where varnish might
-leave clues of whats going on.
+leave clues of whats going on. This chapter will guide you through
+basic troubleshooting in Varnish.
 
 
 When Varnish won't start
-~~~~~~~~~~~~~~~~~~~~~~~~
+------------------------
 
 Sometimes Varnish wont start. There is a plethora of reasons why
 Varnish wont start on your machine. We've seen everything from wrong
@@ -18,7 +19,7 @@ Starting Varnish in debug mode to see what is going on.
 
 Try to start varnish by::
 
-    # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000  -a 0.0.0.0:8080 -d
+    # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1: 2000  -a 0.0.0.0:8080 -d
 
 Notice the -d option. It will give you some more information on what
 is going on. Let us see how Varnish will react to something else
@@ -51,7 +52,7 @@ on IRC.
 
 
 Varnish is crashing
-~~~~~~~~~~~~~~~~~~~
+-------------------
 
 When varnish goes bust the child processes crashes. Usually the mother
 process will manage this by restarting the child process again. Any
@@ -70,7 +71,7 @@ XXX: Describe crashing child process and crashing mother process here too.
 XXX: panic.show
 
 Varnish gives me Guru meditation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+--------------------------------
 
 First find the relevant log entries in varnishlog. That will probably
 give you a clue. Since varnishlog logs so much data it might be hard
@@ -90,7 +91,7 @@ filtering capabilities and explanation of the various options.
 
 
 Varnish doesn't cache
-~~~~~~~~~~~~~~~~~~~~~
+---------------------
 
 See :ref:`users-guide-increasing_your_hitrate`.
 
diff --git a/doc/sphinx/users-guide/vcl-actions.rst b/doc/sphinx/users-guide/vcl-actions.rst
new file mode 100644
index 0000000..cfb5019
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-actions.rst
@@ -0,0 +1,34 @@
+actions
+~~~~~~~
+
+The most common actions to return are these:
+
+*pass*
+ When you return pass the request and subsequent response will be passed to
+ and from the backend server. It won't be cached. pass can be returned from
+ vcl_recv
+
+*hit_for_pass*
+  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
+  will create a hitforpass object in the cache. This has the side-effect of
+  caching the decision not to cache. This is to allow would-be uncachable
+  requests to be passed to the backend at the same time. The same logic is
+  not necessary in vcl_recv because this happens before any potential
+  queueing for an object takes place.
+
+*lookup*
+  When you return lookup from vcl_recv you tell Varnish to deliver content 
+  from cache even if the request othervise indicates that the request 
+  should be passed. You can't return lookup from vcl_fetch.
+
+*pipe*
+  Pipe can be returned from vcl_recv as well. Pipe short circuits the
+  client and the backend connections and Varnish will just sit there
+  and shuffle bytes back and forth. Varnish will not look at the data being 
+  send back and forth - so your logs will be incomplete. 
+  Beware that with HTTP 1.1 a client can send several requests on the same 
+  connection and so you should instruct Varnish to add a "Connection: close"
+  header before actually returning pipe. 
+
+*deliver*
+ Deliver the cached object to the client.  Usually returned from vcl_fetch. 
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
new file mode 100644
index 0000000..4f721aa
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -0,0 +1,197 @@
+.. _users-guide-backend_servers:
+
+Backend servers
+---------------
+
+Varnish has a concept of "backend" or "origin" servers. A backend
+server is the server providing the content Varnish will accelerate.
+
+Our first task is to tell Varnish where it can find its content. Start
+your favorite text editor and open the varnish default configuration
+file. If you installed from source this is
+/usr/local/etc/varnish/default.vcl, if you installed from a package it
+is probably /etc/varnish/default.vcl.
+
+Somewhere in the top there will be a section that looks a bit like this.::
+
+	  # backend default {
+	  #     .host = "127.0.0.1";
+	  #     .port = "8080";
+	  # }
+
+We comment in this bit of text making the text look like.::
+
+          backend default {
+                .host = "127.0.0.1";
+    		.port = "8080";
+	  }
+
+Now, this piece of configuration defines a backend in Varnish called
+*default*. When Varnish needs to get content from this backend it will
+connect to port 8080 on localhost (127.0.0.1).
+
+Varnish can have several backends defined and can you can even join
+several backends together into clusters of backends for load balancing
+purposes. 
+
+Multiple backends
+-----------------
+
+At some point you might need Varnish to cache content from several
+servers. You might want Varnish to map all the URL into one single
+host or not. There are lot of options.
+
+Lets say we need to introduce a Java application into out PHP web
+site. Lets say our Java application should handle URL beginning with
+/java/.
+
+We manage to get the thing up and running on port 8000. Now, lets have
+a look at the default.vcl.::
+
+  backend default {
+      .host = "127.0.0.1";
+      .port = "8080";
+  }
+
+We add a new backend.::
+
+  backend java {
+      .host = "127.0.0.1";
+      .port = "8000";
+  }
+
+Now we need tell where to send the difference URL. Lets look at vcl_recv.::
+
+  sub vcl_recv {
+      if (req.url ~ "^/java/") {
+          set req.backend = java;
+      } else {
+          set req.backend = default.
+      }
+  }
+
+It's quite simple, really. Lets stop and think about this for a
+moment. As you can see you can define how you choose backends based on
+really arbitrary data. You want to send mobile devices to a different
+backend? No problem. if (req.User-agent ~ /mobile/) .. should do the
+trick. 
+
+.. _users-guide-advanced_backend_servers-directors:
+
+Directors
+---------
+
+You can also group several backend into a group of backends. These
+groups are called directors. This will give you increased performance
+and resilience. You can define several backends and group them
+together in a director.::
+
+	 backend server1 {
+	     .host = "192.168.0.10";
+	 }
+	 backend server2{
+	     .host = "192.168.0.10";
+	 }
+
+Now we create the director.::
+
+       	director example_director round-robin {
+        {
+                .backend = server1;
+        }
+	# server2
+        {
+                .backend = server2;
+        }
+	# foo
+	}
+
+
+This director is a round-robin director. This means the director will
+distribute the incoming requests on a round-robin basis. There is
+also a *random* director which distributes requests in a, you guessed
+it, random fashion.
+
+But what if one of your servers goes down? Can Varnish direct all the
+requests to the healthy server? Sure it can. This is where the Health
+Checks come into play.
+
+.. _users-guide-advanced_backend_servers-health:
+
+Health checks
+-------------
+
+Lets set up a director with two backends and health checks. First lets
+define the backends.::
+
+       backend server1 {
+         .host = "server1.example.com";
+	 .probe = {
+                .url = "/";
+                .interval = 5s;
+                .timeout = 1 s;
+                .window = 5;
+                .threshold = 3;
+	   }
+         }
+       backend server2 {
+  	  .host = "server2.example.com";
+  	  .probe = {
+                .url = "/";
+                .interval = 5s;
+                .timeout = 1 s;
+                .window = 5;
+                .threshold = 3;
+	  }
+        }
+
+Whats new here is the probe. Varnish will check the health of each
+backend with a probe. The options are
+
+url
+ What URL should varnish request.
+
+interval
+ How often should we poll
+
+timeout
+ What is the timeout of the probe
+
+window
+ Varnish will maintain a *sliding window* of the results. Here the
+ window has five checks.
+
+threshold 
+ How many of the .window last polls must be good for the backend to be declared healthy.
+
+initial 
+ How many of the of the probes a good when Varnish starts - defaults
+ to the same amount as the threshold.
+
+Now we define the director.::
+
+  director example_director round-robin {
+        {
+                .backend = server1;
+        }
+        # server2 
+        {
+                .backend = server2;
+        }
+	
+        }
+
+You use this director just as you would use any other director or
+backend. Varnish will not send traffic to hosts that are marked as
+unhealthy. Varnish can also serve stale content if all the backends are
+down. See :ref:`users-guide-handling_misbehaving_servers` for more
+information on how to enable this.
+
+Please note that Varnish will keep probes active for all loaded
+VCLs. Varnish will coalesce probes that seem identical - so be careful
+not to change the probe config if you do a lot of VCL
+loading. Unloading the VCL will discard the probes.
+
+For more information on how to do this please see
+ref:`reference-vcl-director`.
+
diff --git a/doc/sphinx/users-guide/vcl-examples.rst b/doc/sphinx/users-guide/vcl-examples.rst
new file mode 100644
index 0000000..ffa730f
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-examples.rst
@@ -0,0 +1,65 @@
+Example 1 - manipulating headers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Lets say we want to remove the cookie for all objects in the /images
+directory of our web server::
+
+  sub vcl_recv {
+    if (req.url ~ "^/images") {
+      unset req.http.cookie;
+    }
+  }
+
+Now, when the request is handled to the backend server there will be
+no cookie header. The interesting line is the one with the
+if-statement. It matches the URL, taken from the request object, and
+matches it against the regular expression. Note the match operator. If
+it matches the Cookie: header of the request is unset (deleted). 
+
+Example 2 - manipulating beresp
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Here we override the TTL of a object comming from the backend if it
+matches certain criteria::
+
+  sub vcl_fetch {
+     if (req.url ~ "\.(png|gif|jpg)$") {
+       unset beresp.http.set-cookie;
+       set beresp.ttl = 1h;
+    }
+  }
+
+Example 3 - ACLs
+~~~~~~~~~~~~~~~~
+
+You create a named access control list with the *acl* keyword. You can match
+the IP address of the client against an ACL with the match operator.::
+
+  # Who is allowed to purge....
+  acl local {
+      "localhost";
+      "192.168.1.0"/24; /* and everyone on the local network */
+      ! "192.168.1.23"; /* except for the dialin router */
+  }
+  
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (client.ip ~ local) {
+         return(lookup);
+      }
+    } 
+  }
+  
+  sub vcl_hit {
+     if (req.request == "PURGE") {
+       set obj.ttl = 0s;
+       error 200 "Purged.";
+      }
+  }
+
+  sub vcl_miss {
+    if (req.request == "PURGE") {
+      error 404 "Not in cache.";
+    }
+  }
+
diff --git a/doc/sphinx/users-guide/vcl-hashing.rst b/doc/sphinx/users-guide/vcl-hashing.rst
new file mode 100644
index 0000000..10b2920
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-hashing.rst
@@ -0,0 +1,51 @@
+
+Hashing
+-------
+
+Internally, when Varnish stores content in it's store it uses a hash
+key to find the object again. In the default setup this key is
+calculated based on the content of the *Host* header or the IP adress
+of the server and the URL.
+
+Behold the default vcl.::
+
+ sub vcl_hash {
+     hash_data(req.url);
+     if (req.http.host) {
+         hash_data(req.http.host);
+     } else {
+         hash_data(server.ip);
+     }
+     return (hash);
+ }
+
+As you can see it first chucks in req.url then req.http.host if it
+exsists. It is worth pointing out that Varnish doesn't lowercase the
+hostname or the URL before hashing it so in thery having Varnish.org/
+and varnish.org/ would result in different cache entries. Browers
+however, tend to lowercase hostnames.
+
+You can change what goes into the hash. This way you can make Varnish
+serve up different content to different clients based on arbitrary
+criteria.
+
+Let's say you want to serve pages in different languages to your users
+based on where their IP address is located. You would need some Vmod
+to get a country code and then put it into the hash. It might look
+like this.
+
+In vcl_recv::
+
+  set req.http.X-Country-Code = geoip.lookup(client.ip);
+
+And then add a vcl_hash::
+
+ sub vcl_hash {
+   hash_data(req.http.X-Country-Code);
+ }
+
+As the default VCL will take care of adding the host and URL to the
+hash we don't have to do anything else. Be careful calling
+return(hash) as this will abort the execution of the default VCL and
+thereby you can end up with a Varnish that will return data based on
+more or less random inputs.
diff --git a/doc/sphinx/users-guide/vcl-inline-c.rst b/doc/sphinx/users-guide/vcl-inline-c.rst
new file mode 100644
index 0000000..7c88cf9
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-inline-c.rst
@@ -0,0 +1,24 @@
+
+
+
+Using In-line C to extend Varnish
+---------------------------------
+
+(Here there be dragons. Big and mean ones.)
+
+You can use *in-line C* to extend Varnish. Please note that you can
+seriously mess up Varnish this way. The C code runs within the Varnish
+Cache process so if your code generates a segfault the cache will crash.
+
+One of the first uses I saw of In-line C was logging to syslog.::
+
+        # The include statements must be outside the subroutines.
+        C{
+                #include <syslog.h>
+        }C
+        
+        sub vcl_something {
+                C{
+                        syslog(LOG_INFO, "Something happened at VCL line XX.");
+                }C
+        }
diff --git a/doc/sphinx/users-guide/vcl-intro.rst b/doc/sphinx/users-guide/vcl-intro.rst
new file mode 100644
index 0000000..922310a
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-intro.rst
@@ -0,0 +1,30 @@
+Varnish Configuration Language - VCL
+-------------------------------------
+
+Varnish has a great configuration system. Most other systems use
+configuration directives, where you basically turn on and off lots of
+switches. Varnish uses a domain specific language called Varnish
+Configuration Language, or VCL for short. Varnish translates this
+configuration into binary code which is then executed when requests
+arrive.
+
+The VCL files are divided into subroutines. The different subroutines
+are executed at different times. One is executed when we get the
+request, another when files are fetched from the backend server.
+
+Varnish will execute these subroutines of code at different stages of
+its work. Because it is code it is execute line by line precedence
+isn't a problem. At some point you call an action in this subroutine
+and then the execution of the subroutine stops.
+
+If you don't call an action in your subroutine and it reaches the end
+Varnish will execute some built in VCL code. You will see this VCL
+code commented out in default.vcl.
+
+99% of all the changes you'll need to do will be done in two of these
+subroutines. *vcl_recv* and *vcl_fetch*.
+
+
+.. _users-guide-vcl_fetch_actions:
+
+
diff --git a/doc/sphinx/users-guide/vcl-saint-and-grace.rst b/doc/sphinx/users-guide/vcl-saint-and-grace.rst
new file mode 100644
index 0000000..bb5fd35
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-saint-and-grace.rst
@@ -0,0 +1,103 @@
+.. _users-guide-handling_misbehaving_servers:
+
+Misbehaving servers
+-------------------
+
+A key feature of Varnish is its ability to shield you from misbehaving
+web- and application servers.
+
+
+
+Grace mode
+~~~~~~~~~~
+
+When several clients are requesting the same page Varnish will send
+one request to the backend and place the others on hold while fetching
+one copy from the back end. In some products this is called request
+coalescing and Varnish does this automatically.
+
+If you are serving thousands of hits per second the queue of waiting
+requests can get huge. There are two potential problems - one is a
+thundering herd problem - suddenly releasing a thousand threads to
+serve content might send the load sky high. Secondly - nobody likes to
+wait. To deal with this we can instruct Varnish to keep
+the objects in cache beyond their TTL and to serve the waiting
+requests somewhat stale content.
+
+So, in order to serve stale content we must first have some content to
+serve. So to make Varnish keep all objects for 30 minutes beyond their
+TTL use the following VCL::
+
+  sub vcl_fetch {
+    set beresp.grace = 30m;
+  }
+
+Varnish still won't serve the stale objects. In order to enable
+Varnish to actually serve the stale object we must enable this on the
+request. Lets us say that we accept serving 15s old object.::
+
+  sub vcl_recv {
+    set req.grace = 15s;
+  }
+
+You might wonder why we should keep the objects in the cache for 30
+minutes if we are unable to serve them? Well, if you have enabled
+:ref:`users-guide-advanced_backend_servers-health` you can check if the
+backend is sick and if it is we can serve the stale content for a bit
+longer.::
+
+   if (! req.backend.healthy) {
+      set req.grace = 5m;
+   } else {
+      set req.grace = 15s;
+   }
+
+So, to sum up, grace mode solves two problems:
+ * it serves stale content to avoid request pile-up.
+ * it serves stale content if the backend is not healthy.
+
+Saint mode
+~~~~~~~~~~
+
+Sometimes servers get flaky. They start throwing out random
+errors. You can instruct Varnish to try to handle this in a
+more-than-graceful way - enter *Saint mode*. Saint mode enables you to
+discard a certain page from one backend server and either try another
+server or serve stale content from cache. Lets have a look at how this
+can be enabled in VCL::
+
+  sub vcl_fetch {
+    if (beresp.status == 500) { 
+      set beresp.saintmode = 10s;
+      return(restart);
+    }
+    set beresp.grace = 5m;
+  } 
+
+When we set beresp.saintmode to 10 seconds Varnish will not ask *that*
+server for URL for 10 seconds. A blacklist, more or less. Also a
+restart is performed so if you have other backends capable of serving
+that content Varnish will try those. When you are out of backends
+Varnish will serve the content from its stale cache.
+
+This can really be a life saver.
+
+Known limitations on grace- and saint mode
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your request fails while it is being fetched you're thrown into
+vcl_error. vcl_error has access to a rather limited set of data so you
+can't enable saint mode or grace mode here. This will be addressed in a
+future release but a work-around available.
+
+* Declare a backend that is always sick.
+* Set a magic marker in vcl_error
+* Restart the transaction
+* Note the magic marker in vcl_recv and set the backend to the one mentioned
+* Varnish will now serve stale data is any is available
+
+
+God mode
+~~~~~~~~
+Not implemented yet. :-)
+
diff --git a/doc/sphinx/users-guide/vcl-subs.rst b/doc/sphinx/users-guide/vcl-subs.rst
new file mode 100644
index 0000000..d759b2a
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-subs.rst
@@ -0,0 +1,25 @@
+
+vcl_recv
+~~~~~~~~
+
+vcl_recv (yes, we're skimpy with characters, it's Unix) is called at
+the beginning of a request, after the complete request has been
+received and parsed.  Its purpose is to decide whether or not to serve
+the request, how to do it, and, if applicable, which backend to use.
+
+In vcl_recv you can also alter the request. Typically you can alter
+the cookies and add and remove request headers.
+
+Note that in vcl_recv only the request object, req is available.
+
+vcl_fetch
+~~~~~~~~~
+
+vcl_fetch is called *after* a document has been successfully retrieved
+from the backend. Normal tasks her are to alter the response headers,
+trigger ESI processing, try alternate backend servers in case the
+request failed.
+
+In vcl_fetch you still have the request object, req, available. There
+is also a *backend response*, beresp. beresp will contain the HTTP
+headers from the backend.
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
new file mode 100644
index 0000000..c49ef91
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -0,0 +1,45 @@
+VCL Syntax
+----------
+
+VCL has inherited a lot from C and it reads much like simple C or Perl.
+
+Blocks are delimited by curly braces, statements end with semicolons,
+and comments may be written as in C, C++ or Perl according to your own
+preferences.
+
+Note that VCL doesn't contain any loops or jump statements.
+
+
+Strings
+~~~~~~~
+
+
+
+Access control lists (ACLs)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+Operators
+~~~~~~~~~
+
+The following operators are available in VCL. See the examples further
+down for, uhm, examples.
+
+= 
+ Assignment operator.
+
+== 
+ Comparison.
+
+~
+ Match. Can either be used with regular expressions or ACLs.
+
+!
+ Negation.
+
+&&
+ Logical *and*
+
+||
+ Logical *or*
+
diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst
new file mode 100644
index 0000000..10af0ab
--- /dev/null
+++ b/doc/sphinx/users-guide/vcl-variables.rst
@@ -0,0 +1,23 @@
+
+Requests, responses and objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In VCL, there are three important data structures. The request, coming
+from the client, the response coming from the backend server and the
+object, stored in cache.
+
+In VCL you should know the following structures.
+
+*req*
+ The request object. When Varnish has received the request the req object is 
+ created and populated. Most of the work you do in vcl_recv you 
+ do on or with the req object.
+
+*beresp*
+ The backend respons object. It contains the headers of the object 
+ comming from the backend. Most of the work you do in vcl_fetch you 
+ do on the beresp object.
+
+*obj*
+ The cached object. Mostly a read only object that resides in memory. 
+ obj.ttl is writable, the rest is read only.
diff --git a/doc/sphinx/users-guide/vcl.rst b/doc/sphinx/users-guide/vcl.rst
index 57fb7e0..b36a43f 100644
--- a/doc/sphinx/users-guide/vcl.rst
+++ b/doc/sphinx/users-guide/vcl.rst
@@ -1,200 +1,23 @@
-Varnish Configuration Language - VCL
--------------------------------------
+VCL
+---
 
-Varnish has a great configuration system. Most other systems use
-configuration directives, where you basically turn on and off lots of
-switches. Varnish uses a domain specific language called Varnish
-Configuration Language, or VCL for short. Varnish translates this
-configuration into binary code which is then executed when requests
-arrive.
 
-The VCL files are divided into subroutines. The different subroutines
-are executed at different times. One is executed when we get the
-request, another when files are fetched from the backend server.
+Yes. Is great. Ja.
 
-Varnish will execute these subroutines of code at different stages of
-its work. Because it is code it is execute line by line precedence
-isn't a problem. At some point you call an action in this subroutine
-and then the execution of the subroutine stops.
 
-If you don't call an action in your subroutine and it reaches the end
-Varnish will execute some built in VCL code. You will see this VCL
-code commented out in default.vcl.
-
-99% of all the changes you'll need to do will be done in two of these
-subroutines. *vcl_recv* and *vcl_fetch*.
-
-vcl_recv
-~~~~~~~~
-
-vcl_recv (yes, we're skimpy with characters, it's Unix) is called at
-the beginning of a request, after the complete request has been
-received and parsed.  Its purpose is to decide whether or not to serve
-the request, how to do it, and, if applicable, which backend to use.
-
-In vcl_recv you can also alter the request. Typically you can alter
-the cookies and add and remove request headers.
-
-Note that in vcl_recv only the request object, req is available.
-
-vcl_fetch
-~~~~~~~~~
-
-vcl_fetch is called *after* a document has been successfully retrieved
-from the backend. Normal tasks her are to alter the response headers,
-trigger ESI processing, try alternate backend servers in case the
-request failed.
-
-In vcl_fetch you still have the request object, req, available. There
-is also a *backend response*, beresp. beresp will contain the HTTP
-headers from the backend.
-
-.. _users-guide-vcl_fetch_actions:
-
-actions
-~~~~~~~
-
-The most common actions to return are these:
-
-*pass*
- When you return pass the request and subsequent response will be passed to
- and from the backend server. It won't be cached. pass can be returned from
- vcl_recv
-
-*hit_for_pass*
-  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
-  will create a hitforpass object in the cache. This has the side-effect of
-  caching the decision not to cache. This is to allow would-be uncachable
-  requests to be passed to the backend at the same time. The same logic is
-  not necessary in vcl_recv because this happens before any potential
-  queueing for an object takes place.
-
-*lookup*
-  When you return lookup from vcl_recv you tell Varnish to deliver content 
-  from cache even if the request othervise indicates that the request 
-  should be passed. You can't return lookup from vcl_fetch.
-
-*pipe*
-  Pipe can be returned from vcl_recv as well. Pipe short circuits the
-  client and the backend connections and Varnish will just sit there
-  and shuffle bytes back and forth. Varnish will not look at the data being 
-  send back and forth - so your logs will be incomplete. 
-  Beware that with HTTP 1.1 a client can send several requests on the same 
-  connection and so you should instruct Varnish to add a "Connection: close"
-  header before actually returning pipe. 
-
-*deliver*
- Deliver the cached object to the client.  Usually returned from vcl_fetch. 
-
-Requests, responses and objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-In VCL, there are three important data structures. The request, coming
-from the client, the response coming from the backend server and the
-object, stored in cache.
-
-In VCL you should know the following structures.
-
-*req*
- The request object. When Varnish has received the request the req object is 
- created and populated. Most of the work you do in vcl_recv you 
- do on or with the req object.
-
-*beresp*
- The backend respons object. It contains the headers of the object 
- comming from the backend. Most of the work you do in vcl_fetch you 
- do on the beresp object.
-
-*obj*
- The cached object. Mostly a read only object that resides in memory. 
- obj.ttl is writable, the rest is read only.
-
-Operators
-~~~~~~~~~
-
-The following operators are available in VCL. See the examples further
-down for, uhm, examples.
-
-= 
- Assignment operator.
-
-== 
- Comparison.
-
-~
- Match. Can either be used with regular expressions or ACLs.
-
-!
- Negation.
-
-&&
- Logical *and*
-
-||
- Logical *or*
-
-Example 1 - manipulating headers
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Lets say we want to remove the cookie for all objects in the /images
-directory of our web server::
-
-  sub vcl_recv {
-    if (req.url ~ "^/images") {
-      unset req.http.cookie;
-    }
-  }
-
-Now, when the request is handled to the backend server there will be
-no cookie header. The interesting line is the one with the
-if-statement. It matches the URL, taken from the request object, and
-matches it against the regular expression. Note the match operator. If
-it matches the Cookie: header of the request is unset (deleted). 
-
-Example 2 - manipulating beresp
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Here we override the TTL of a object comming from the backend if it
-matches certain criteria::
-
-  sub vcl_fetch {
-     if (req.url ~ "\.(png|gif|jpg)$") {
-       unset beresp.http.set-cookie;
-       set beresp.ttl = 1h;
-    }
-  }
-
-Example 3 - ACLs
-~~~~~~~~~~~~~~~~
-
-You create a named access control list with the *acl* keyword. You can match
-the IP address of the client against an ACL with the match operator.::
-
-  # Who is allowed to purge....
-  acl local {
-      "localhost";
-      "192.168.1.0"/24; /* and everyone on the local network */
-      ! "192.168.1.23"; /* except for the dialin router */
-  }
-  
-  sub vcl_recv {
-    if (req.request == "PURGE") {
-      if (client.ip ~ local) {
-         return(lookup);
-      }
-    } 
-  }
-  
-  sub vcl_hit {
-     if (req.request == "PURGE") {
-       set obj.ttl = 0s;
-       error 200 "Purged.";
-      }
-  }
-
-  sub vcl_miss {
-    if (req.request == "PURGE") {
-      error 404 "Not in cache.";
-    }
-  }
+.. toctree::
+   :maxdepth: 2
 
+   vcl-intro
+   vcl-syntax
+   vcl-variables
+   vcl-actions
+   vcl-subs
+   vcl-backends
+   vcl-hashing
+   vcl-saint-and-grace
+   vcl-inline-c
+   vcl-examples
+   websockets
+   devicedetection
+   
\ No newline at end of file



From perbu at varnish-cache.org  Sat Sep  8 17:53:11 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Sat, 08 Sep 2012 19:53:11 +0200
Subject: [master] d6aca56 Separate out the text describing the storage
 backends so it can be resued in both the man page and users guide
Message-ID: <E1TAPDT-0001g7-QS@project.varnish-software.com>

commit d6aca56d15aa7ba737bb088bd12e622e19ab087b
Author: Per Buer <per.buer at gmail.com>
Date:   Sat Sep 8 19:50:53 2012 +0200

    Separate out the text describing the storage backends so it can be resued in both the man page and users guide

diff --git a/doc/sphinx/include/storage_backends.rst b/doc/sphinx/include/storage_backends.rst
new file mode 100644
index 0000000..ec80601
--- /dev/null
+++ b/doc/sphinx/include/storage_backends.rst
@@ -0,0 +1,113 @@
+
+malloc
+~~~~~~
+
+syntax: malloc[,size]
+
+Malloc is a memory based backend. Each object will be allocated from
+memory. If your system runs low on memory swap will be used. Be aware
+that the size limitation only limits the actual storage and that
+approximately 1k of memory per object will be used for various
+internal structures.
+
+The size parameter specifies the maximum amount of memory varnishd
+will allocate.  The size is assumed to be in bytes, unless followed by
+one of the following suffixes:
+
+      K, k    The size is expressed in kibibytes.
+
+      M, m    The size is expressed in mebibytes.
+
+      G, g    The size is expressed in gibibytes.
+
+      T, t    The size is expressed in tebibytes.
+
+The default size is unlimited. 
+
+Mallocs performance is bound by memory speed so it is very fast. 
+
+file
+~~~~
+
+syntax: file[,path[,size[,granularity]]]
+
+The file backend stores objects in memory backed by a file on disk
+with mmap. This is the default storage backend and unless you specify
+another storage this one will used along with Transient storage.
+
+The path parameter specifies either the path to the backing file or
+the path to a directory in which varnishd will create the backing
+file.  The default is /tmp.
+
+The size parameter specifies the size of the backing file.  The size
+is assumed to be in bytes, unless fol? lowed by one of the following
+suffixes:
+
+      K, k    The size is expressed in kibibytes.
+
+      M, m    The size is expressed in mebibytes.
+
+      G, g    The size is expressed in gibibytes.
+
+      T, t    The size is expressed in tebibytes.
+
+      %       The size is expressed as a percentage of the free space on the
+              file system where it resides.
+
+The default size is 50%.
+
+If the backing file already exists, it will be truncated or expanded
+to the specified size.
+
+Note that if varnishd has to create or expand the file, it will not
+pre-allocate the added space, leading to fragmentation, which may
+adversely impact performance.  Pre-creating the storage file using
+dd(1) will reduce fragmentation to a minimum.
+
+The granularity parameter specifies the granularity of
+allocation.  All allocations are rounded up to this size.  The
+is assumed to be in bytes, unless followed by one of the
+suffixes described for size except for %.
+
+The default size is the VM page size.  The size should be reduced if
+you have many small objects.
+
+File performance is typically limited by the write speed of the
+device, and depending on use, the seek time.
+
+persistent (experimental)
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+syntax: persistent,path,size {experimental}
+
+Persistent storage. Varnish will store objects in a file in a manner
+that will secure the survival of *most* of the objects in the event of
+a planned or unplanned shutdown of Varnish.
+
+The path parameter specifies the path to the backing file. If
+the file doesn't exist Varnish will create it.
+
+The size parameter specifies the size of the backing file.  The
+size is assumed to be in bytes, unless followed by one of the
+following suffixes:
+
+      K, k    The size is expressed in kibibytes.
+
+      M, m    The size is expressed in mebibytes.
+
+      G, g    The size is expressed in gibibytes.
+
+      T, t    The size is expressed in tebibytes.
+
+Varnish will split the file into logical *silos* and write to the
+silos in the manner of a circular buffer. Only one silo will be kept
+open at any given point in time. Full silos are *sealed*. When Varnish
+starts after a shutdown it will discard the content of any silo that
+isn't sealed.
+
+Transient Storage
+-----------------
+      
+If you name any of your storage backend "Transient" it will be
+used for transient (short lived) objects. By default Varnish
+would use an unlimited malloc backend for this.
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 87141bf..95b0097 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -157,110 +157,7 @@ Storage Types
 
 The following storage types are available:
 
-malloc[,size]
-
-      Malloc is a memory based backend. Each object will be allocated
-      from memory. If your system runs low on memory swap will be
-      used. Be aware that the size limitation only limits the actual
-      storage and that approximately 1k of memory per object will be
-      used for various internal structures.
-
-      The size parameter specifies the maximum amount of memory
-      varnishd will allocate.  The size is assumed to be in bytes,
-      unless followed by one of the following suffixes:
-
-      K, k    The size is expressed in kibibytes.
-
-      M, m    The size is expressed in mebibytes.
-
-      G, g    The size is expressed in gibibytes.
-
-      T, t    The size is expressed in tebibytes.
-
-      The default size is unlimited. 
-
-      Mallocs performance is bound by memory speed so it is very fast. 
-
-file[,path[,size[,granularity]]]
-
-      The file backend stores objects in memory backed by a file on
-      disk with mmap. This is the default storage backend and unless
-      you specify another storage this one will used along with
-      Transient storage.
-
-      The path parameter specifies either the path to the backing file
-      or the path to a directory in which varnishd will create the
-      backing file.  The default is /tmp.
-
-      The size parameter specifies the size of the backing file.  The
-      size is assumed to be in bytes, unless fol? lowed by one of the
-      following suffixes:
-
-      K, k    The size is expressed in kibibytes.
-
-      M, m    The size is expressed in mebibytes.
-
-      G, g    The size is expressed in gibibytes.
-
-      T, t    The size is expressed in tebibytes.
-
-      %       The size is expressed as a percentage of the free space on the
-              file system where it resides.
-
-      The default size is 50%.
-
-      If the backing file already exists, it will be truncated or
-      expanded to the specified size.
-
-      Note that if varnishd has to create or expand the file, it will
-      not pre-allocate the added space, leading to fragmentation,
-      which may adversely impact performance.  Pre-creating the
-      storage file using dd(1) will reduce fragmentation to a minimum.
-
-      The granularity parameter specifies the granularity of
-      allocation.  All allocations are rounded up to this size.  The
-      size is assumed to be in bytes, unless followed by one of the
-      suffixes described for size except for %.
-
-      The default size is the VM page size.  The size should be
-      reduced if you have many small objects.
-
-      File performance is typically limited by the write speed of the
-      device, and depending on use, the seek time.
-
-persistent,path,size {experimental}
-
-      Persistent storage. Varnish will store objects in a file in a
-      manner that will secure the survival of *most* of the objects in
-      the event of a planned or unplanned shutdown of Varnish.
-
-      The path parameter specifies the path to the backing file. If
-      the file doesn't exist Varnish will create it.
-
-      The size parameter specifies the size of the backing file.  The
-      size is assumed to be in bytes, unless followed by one of the
-      following suffixes:
-
-      K, k    The size is expressed in kibibytes.
-
-      M, m    The size is expressed in mebibytes.
-
-      G, g    The size is expressed in gibibytes.
-
-      T, t    The size is expressed in tebibytes.
-
-      Varnish will split the file into logical *silos* and write to
-      the silos in the manner of a circular buffer. Only one silo will
-      be kept open at any given point in time. Full silos are
-      *sealed*. When Varnish starts after a shutdown it will discard
-      the content of any silo that isn't sealed.
-
-Transient Storage
------------------
-      
-      If you name any of your storage backend "Transient" it will be
-      used for transient (short lived) objects. By default Varnish
-      would use an unlimited malloc backend for this.
+.. include:: ../include/storage_backends.rst
 
 Management Interface
 --------------------
diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 5d576cf..7436837 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -6,16 +6,8 @@ Storage backends
 Intro
 ~~~~~
 
-Malloc
-~~~~~~
-
-File
-~~~~
-
-Persistent
-~~~~~~~~~~
-
-Transient
-~~~~~~~~~
+Varnish has pluggable storage backends. It can store data in various
+backends which have different performance characteristics.
 
+.. include:: ../include/storage_backends.rst
 



From perbu at varnish-cache.org  Sun Sep  9 08:29:41 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Sun, 09 Sep 2012 10:29:41 +0200
Subject: [master] a057cfa Try to split out small chunks of man vcl into
 separate bitsize files for reuse in users guide
Message-ID: <E1TActh-0004WP-Si@project.varnish-software.com>

commit a057cfa7ce0a4199add3118fe2c8ed440753bb17
Author: Per Buer <per.buer at gmail.com>
Date:   Sun Sep 9 10:29:40 2012 +0200

    Try to split out small chunks of man vcl into separate bitsize files for reuse in users guide

diff --git a/doc/sphinx/include/vcl-backends.rst b/doc/sphinx/include/vcl-backends.rst
new file mode 100644
index 0000000..0290bf7
--- /dev/null
+++ b/doc/sphinx/include/vcl-backends.rst
@@ -0,0 +1,44 @@
+Backend declarations
+--------------------
+
+A backend declaration creates and initializes a named backend object:
+::
+
+  backend www {
+    .host = "www.example.com";
+    .port = "http";
+  }
+
+The backend object can later be used to select a backend at request time:
+::
+
+  if (req.http.host ~ "(?i)^(www.)?example.com$") {
+    set req.backend = www;
+  }
+
+To avoid overloading backend servers, .max_connections can be set to
+limit the maximum number of concurrent backend connections.
+
+The timeout parameters can be overridden in the backend declaration.
+The timeout parameters are .connect_timeout for the time to wait for a
+backend connection, .first_byte_timeout for the time to wait for the
+first byte from the backend and .between_bytes_timeout for time to
+wait between each received byte.
+
+These can be set in the declaration like this:
+::
+
+  backend www {
+    .host = "www.example.com";
+    .port = "http";
+    .connect_timeout = 1s;
+    .first_byte_timeout = 5s;
+    .between_bytes_timeout = 2s;
+  }
+
+To mark a backend as unhealthy after number of items have been added
+to its saintmode list ``.saintmode_threshold`` can be set to the maximum
+list size. Setting a value of 0 disables saint mode checking entirely
+for that backend.  The value in the backend declaration overrides the
+parameter.
+
diff --git a/doc/sphinx/include/vcl-syntax.rst b/doc/sphinx/include/vcl-syntax.rst
new file mode 100644
index 0000000..441aa06
--- /dev/null
+++ b/doc/sphinx/include/vcl-syntax.rst
@@ -0,0 +1,65 @@
+VCL SYNTAX
+==========
+
+The VCL syntax is very simple, and deliberately similar to C and Perl.
+Blocks are delimited by curly braces, statements end with semicolons,
+and comments may be written as in C, C++ or Perl according to your own
+preferences.
+
+In addition to the C-like assignment (=), comparison (==, !=) and
+boolean (!, && and \|\|) operators, VCL supports both regular
+expression and ACL matching using the ~ and the !~ operators.
+
+Basic strings are enclosed in " ... ", and may not contain newlines.
+
+Long strings are enclosed in {" ... "}. They may contain any
+character including ", newline and other control characters except
+for the NUL (0x00) character.
+
+Unlike C and Perl, the backslash (\) character has no special meaning
+in strings in VCL, so it can be freely used in regular expressions
+without doubling.
+
+Strings are concatenated using the '+' operator. 
+
+Assignments are introduced with the *set* keyword.  There are no
+user-defined variables; values can only be assigned to variables
+attached to backend, request or document objects.  Most of these are
+typed, and the values assigned to them must have a compatible unit
+suffix.
+
+You can use the *set* keyword to arbitrary HTTP headers. You can
+remove headers with the *remove* or *unset* keywords, which are
+synonym.
+
+You can use the *rollback* keyword to revert any changes to req at
+any time.
+
+The *synthetic* keyword is used to produce a synthetic response
+body in vcl_error. It takes a single string as argument.
+
+You can force a crash of the client process with the *panic* keyword.
+*panic* takes a string as argument.
+
+The ``return(action)`` keyword terminates the subroutine. *action* can be,
+depending on context one of
+
+* deliver
+* error
+* fetch
+* hash
+* hit_for_pass
+* lookup
+* ok
+* pass
+* pipe
+* restart
+
+Please see the list of subroutines to see what return actions are
+available where.
+
+VCL has if tests, but no loops.
+
+The contents of another VCL file may be inserted at any point in the
+code by using the *include* keyword followed by the name of the other
+file as a quoted string.
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 8d6ed9c..772e696 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -27,117 +27,11 @@ When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
 is then dynamically linked into the server process.
 
-SYNTAX
-======
+.. include:: ../include/vcl-syntax.rst
 
-The VCL syntax is very simple, and deliberately similar to C and Perl.
-Blocks are delimited by curly braces, statements end with semicolons,
-and comments may be written as in C, C++ or Perl according to your own
-preferences.
+.. include:: ../include/vcl-backends.rst
 
-In addition to the C-like assignment (=), comparison (==, !=) and
-boolean (!, && and \|\|) operators, VCL supports both regular
-expression and ACL matching using the ~ and the !~ operators.
-
-Basic strings are enclosed in " ... ", and may not contain newlines.
-
-Long strings are enclosed in {" ... "}. They may contain any
-character including ", newline and other control characters except
-for the NUL (0x00) character.
-
-Unlike C and Perl, the backslash (\) character has no special meaning
-in strings in VCL, so it can be freely used in regular expressions
-without doubling.
-
-Strings are concatenated using the '+' operator. 
-
-Assignments are introduced with the *set* keyword.  There are no
-user-defined variables; values can only be assigned to variables
-attached to backend, request or document objects.  Most of these are
-typed, and the values assigned to them must have a compatible unit
-suffix.
-
-You can use the *set* keyword to arbitrary HTTP headers. You can
-remove headers with the *remove* or *unset* keywords, which are
-synonym.
-
-You can use the *rollback* keyword to revert any changes to req at
-any time.
-
-The *synthetic* keyword is used to produce a synthetic response
-body in vcl_error. It takes a single string as argument.
-
-You can force a crash of the client process with the *panic* keyword.
-*panic* takes a string as argument.
-
-The ``return(action)`` keyword terminates the subroutine. *action* can be,
-depending on context one of
-
-* deliver
-* error
-* fetch
-* hash
-* hit_for_pass
-* lookup
-* ok
-* pass
-* pipe
-* restart
-
-Please see the list of subroutines to see what return actions are
-available where.
-
-VCL has if tests, but no loops.
-
-The contents of another VCL file may be inserted at any point in the
-code by using the *include* keyword followed by the name of the other
-file as a quoted string.
-
-Backend declarations
---------------------
-
-A backend declaration creates and initializes a named backend object:
-::
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-  }
-
-The backend object can later be used to select a backend at request time:
-::
-
-  if (req.http.host ~ "(?i)^(www.)?example.com$") {
-    set req.backend = www;
-  }
-
-To avoid overloading backend servers, .max_connections can be set to
-limit the maximum number of concurrent backend connections.
-
-The timeout parameters can be overridden in the backend declaration.
-The timeout parameters are .connect_timeout for the time to wait for a
-backend connection, .first_byte_timeout for the time to wait for the
-first byte from the backend and .between_bytes_timeout for time to
-wait between each received byte.
-
-These can be set in the declaration like this:
-::
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-    .connect_timeout = 1s;
-    .first_byte_timeout = 5s;
-    .between_bytes_timeout = 2s;
-  }
-
-To mark a backend as unhealthy after number of items have been added
-to its saintmode list ``.saintmode_threshold`` can be set to the maximum
-list size. Setting a value of 0 disables saint mode checking entirely
-for that backend.  The value in the backend declaration overrides the
-parameter.
-
-.. _reference-vcl-director:
+.. _ref-vcl-director:
 
 Directors
 ---------



From phk at varnish-cache.org  Fri Sep 14 10:34:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 14 Sep 2012 12:34:07 +0200
Subject: [master] 9c93355 Add another rant
Message-ID: <E1TCTDr-00030s-LM@project.varnish-software.com>

commit 9c93355289be9491c43bff1b5ad598933386695d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 14 10:33:58 2012 +0000

    Add another rant

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 3b49a85..2b5cb1f 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -9,6 +9,7 @@ You may or may not want to know what Poul-Henning thinks.
 .. toctree::
 	:maxdepth: 1
 
+	spdy.rst
 	http20.rst
 	varnish_does_not_hash.rst
 	thetoolsweworkwith.rst
diff --git a/doc/sphinx/phk/spdy.rst b/doc/sphinx/phk/spdy.rst
new file mode 100644
index 0000000..327518a
--- /dev/null
+++ b/doc/sphinx/phk/spdy.rst
@@ -0,0 +1,160 @@
+.. _phk_spdy:
+
+===================================
+What SPDY did to my summer vacation
+===================================
+
+It's dawning on me that I'm sort of the hipster of hipsters, in the sense
+that I tend to do things far before other people do, but totally fail to
+communicate what's going on out there in the future, and thus by the
+time the "real hipsters" catch up, I'm already somewhere different and
+more insteresting.
+
+My one lucky break was the `bikeshed email <http://bikeshed.org/>`_ where
+I actually did sit down and compose some of my thoughts, thus firmly
+sticking a stick in the ground as one of the first to seriously think
+about how you organize open source collaborations.
+
+I mention this because what I am going to write probably seems very
+unimportant for most of the Varnish users right now, but down the road,
+three, five or maybe even ten years ahead, I think it will become important.
+
+Feel free to not read it until then.
+
+The evolution of Varnish
+------------------------
+
+When we started out, seven years ago, our only and entire goal was to build
+a server-side cache better than squid.  That we did.
+
+Since then we have added stuff to Varnish, ESI:includes, gzip support,
+VMODS and I'm staring at streaming and conditional backend fetches right
+now.
+
+Varnish is a bit more than a web-cache now, but it is still, basically,
+a layer of polish you put in front of your webserver to get it too
+look and work better.
+
+Googles experiments with SPDY have forced a HTTP/2.0 effort into motion,
+but if past performance is any indication, that is not something we have
+to really worry about for a number of years, the IETF WG has still to
+manage to "clarify" RFC2616 which defines HTTP/1.1 and to say there
+is anything even remotely resembling consensus behind SPDY would be a
+downright lie.
+
+RFC2616 is from June 1999, which, to me, means that we should look at
+2035 when we design HTTP/2.0, and predicting things is well known to
+be hard, in particular with respect to the future.
+
+So what's a Varnish architect to do ?
+
+What I did this summer vacation, was to think a lot about how Varnish
+can be architected to cope with the kind of changes SPDY and maybe HTTP/2.0
+drag in:  Pipelining, multiplexing etc, without committing us to one
+particular path of science fiction about life in 2035.
+
+Profound insights often sound incredibly simplistic bordering
+trivial, until you consider the full ramifications.  The implementation
+of "Do Not Kill" is in current law is surprisingly voluminous.  (If
+you don't think so, you probably forgot to #include the Wienna
+Treaty and the convention about chemical and biological weapons.)
+
+So my insight about Varnish, that it has to become a socket-wrench like
+toolchest for doing things with HTTP traffic, will probably elicit a lot
+of "duh!" reactions, until people, including me, understand the 
+ramifications more fully.
+
+Things you cannot do with Varnish today
+---------------------------------------
+
+As much as Varnish can be bent, tweaked and coaxed into doing today,
+there are things you cannot do, or at least things which are very
+hard and very inefficient to do with Varnish.
+
+For instance we consider "a transaction" something that starts with
+a request from a client, and involves zero or more backend fetches
+of finite sized data elements.
+
+That is not how the future looks.
+
+For instance one of the things SPDY have tried out is "server push",
+where you fetch index.html and the webserver says "you'll also want
+main.css and cat.gif then" and pushes those objects on the client,
+to save the round-trip times wasted waiting for the client to ask
+for them.
+
+Today, something like that is impossible in Varnish, since objects
+are independent and you can only look up one at a time.
+
+I already can hear some of you amazing VCL wizards say "Well,
+if you inline-C grab a refcount, then restart and ..." but lets
+be honest, that's not how it should look.
+
+You should be able to do something like:
+
+	if (req.proto == "SPDY" && req.url ~ "index.html") {
+		req.obj1 = lookup(backend1, "/main.css")
+		if (req.obj1.status == 200) {
+			sess.push(req.obj1, bla, bla, bla);
+		}
+		req.obj2 = lookup(backend1, "/cat.gif")
+		if (req.obj1.status == 200) {
+			sess.push(req.obj2, bla, bla, bla);
+		}
+	}
+
+And doing that is not really *that* hard, I think.  We just need
+to keep track of all the objects we instantiate and make sure they
+disappear and die when nobody is using them any more.
+
+But a lot of the assumptions we made back in 2006 are no longer
+valid under such an architecture, but those same assumptions are
+what gives Varnish such astonishing performance, so just replacing
+them with standard CS-textbook solutions like "garbage collection"
+would make Varnish loose a lot of its lustre.
+
+As some of you know, there is a lot of modularity hidden inside
+Varnish but not quite released for public use in VCL, much of what
+is going to happen, will be polishing up and documenting that
+modularity and releasing it for you guys to have fun with, so it
+is not like we are starting from scratch or anything.
+
+But some of that modularity stands on foundations which are no longer
+firm, for instance that the initiating request exists for the
+full duration of a backend fetch.
+
+Those will take some work to fix.
+
+But, before you start to think I have a grand plan or even a clear-cut
+road map, I'd better make it absolutely clear that is not the case:
+I perceive some weird shapes in the fog of the future and I'll aim
+in that direction and either they are the doorways I suspect
+or they are trapdoors to tar-pits, time will show.
+
+I'm going to be making a lot of changes and things that used to be
+will no longer be as they used to be, but I think they will be better
+in the long run, so please bear with me, if your favourite detail
+of how Varnish works changes.
+
+Varnish is not speedy, Varnish is fast!
+---------------------------------------
+
+As I said I'm not a fan of SPDY and I sincerely hope that no bit of
+the current proposal survives unchallenged in whatever HTTP/2.0 standard
+emerges down the road.
+
+But I do want to thank the people behind that mess, not for the
+mess, but for having provoked me to spend some summertime thinking
+hard about what it is that I'm trying to do with Varnish and what
+problem Varnish is here to solve.
+
+This is going to be FUN!
+
+
+Poul-Henning 2012-09-14
+
+Author of Varnish
+
+PS: See you at `VUG6 <https://www.varnish-cache.org/vug6>`_ where I plan
+to talk more about this.
+



From phk at varnish-cache.org  Mon Sep 17 12:08:04 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 17 Sep 2012 14:08:04 +0200
Subject: [master] 232f918 Use the right size-tweaking function,
 so Ingvar doesn't have to buy 512TB ram to run Varnish on ppc64.
Message-ID: <E1TDa7Q-000249-A0@project.varnish-software.com>

commit 232f91872012c388bea98b71165bc1f15e1e9577
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 17 12:07:34 2012 +0000

    Use the right size-tweaking function, so Ingvar doesn't have to
    buy 512TB ram to run Varnish on ppc64.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index b0631a3..9359800 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -839,7 +839,7 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"50", "allocations" },
 	{ "fetch_chunksize",
-		tweak_bytes_u,
+		tweak_bytes,
 		    &mgt_param.fetch_chunksize, 4 * 1024, UINT_MAX,
 		"The default chunksize used by fetcher. "
 		"This should be bigger than the majority of objects with "
@@ -849,7 +849,7 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"128k", "bytes" },
 	{ "fetch_maxchunksize",
-		tweak_bytes_u,
+		tweak_bytes,
 		    &mgt_param.fetch_maxchunksize, 64 * 1024, UINT_MAX,
 		"The maximum chunksize we attempt to allocate from storage. "
 		"Making this too large may cause delays and storage "



From tfheen at varnish-cache.org  Tue Sep 18 09:31:22 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 18 Sep 2012 11:31:22 +0200
Subject: [3.0] 2587b1b Mark ban list as volatile
Message-ID: <E1TDu9K-0002lO-0N@project.varnish-software.com>

commit 2587b1b973b1c4e019ee74da7a93fe31af5a6186
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 18 11:21:09 2012 +0200

    Mark ban list as volatile
    
    Fixes #1194

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 5c1da57..138a1f6 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -81,7 +81,8 @@ struct ban_test {
 	const void		*arg2_spec;
 };
 
-static VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
+/* XXX phk thinks he knows why this is a good idea */
+static volatile VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
 static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;



From tfheen at varnish-cache.org  Tue Sep 18 10:35:48 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 18 Sep 2012 12:35:48 +0200
Subject: [master] c553741 Fix up list of docs to install
Message-ID: <E1TDv9g-0000TV-RV@project.varnish-software.com>

commit c553741c230684970abae1ef3fe3c7d4a01c54ca
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 18 12:35:29 2012 +0200

    Fix up list of docs to install

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 987c069..76657ec 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -127,23 +127,45 @@ EXTRA_DIST = \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
 	reference/vsm.rst \
-	tutorial/advanced_backend_servers.rst \
-	tutorial/advanced_topics.rst \
-	tutorial/backend_servers.rst \
-	tutorial/cookies.rst \
-	tutorial/esi.rst \
-	tutorial/handling_misbehaving_servers.rst \
-	tutorial/increasing_your_hitrate.rst \
 	tutorial/index.rst \
-	tutorial/logging.rst \
-	tutorial/purging.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/sizing_your_cache.rst \
+	tutorial/introduction.rst \
+	tutorial/web_accelerator.rst \
 	tutorial/starting_varnish.rst \
-	tutorial/statistics.rst \
-	tutorial/troubleshooting.rst \
-	tutorial/vary.rst \
-	tutorial/vcl.rst
+	tutorial/putting_varnish_on_port_80.rst \
+	tutorial/backend_servers.rst \
+	tutorial/now_what.rst \
+	users-guide/command-line.rst \
+	users-guide/compression.rst \
+	users-guide/configuration.rst \
+	users-guide/cookies.rst \
+	users-guide/devicedetection.rst \
+	users-guide/esi.rst \
+	users-guide/increasing-your-hitrate.rst \
+	users-guide/index.rst \
+	users-guide/operation-cli.rst \
+	users-guide/operation-logging.rst \
+	users-guide/operation.rst \
+	users-guide/operation-statistics.rst \
+	users-guide/params.rst \
+	users-guide/purging.rst \
+	users-guide/sizing-your-cache.rst \
+	users-guide/storage-backends.rst \
+	users-guide/troubleshooting.rst \
+	users-guide/vary.rst \
+	users-guide/vcl-actions.rst \
+	users-guide/vcl-backends.rst \
+	users-guide/vcl-examples.rst \
+	users-guide/vcl-hashing.rst \
+	users-guide/vcl-inline-c.rst \
+	users-guide/vcl-intro.rst \
+	users-guide/vcl.rst \
+	users-guide/vcl-saint-and-grace.rst \
+	users-guide/vcl-subs.rst \
+	users-guide/vcl-syntax.rst \
+	users-guide/vcl-variables.rst \
+	users-guide/virtualized.rst \
+	users-guide/websockets.rst
+
 
 dist-hook:
 	$(MAKE) html



From martin at varnish-cache.org  Wed Sep 19 12:47:10 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 19 Sep 2012 14:47:10 +0200
Subject: [master] 249add5 Set TCP_NODELAY on the accepted sockets to disable
 Nagle.
Message-ID: <E1TEJgM-0003P9-Ac@project.varnish-software.com>

commit 249add59fe108a5265db900ccb2d4d80666fba8a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 17 12:57:18 2012 +0200

    Set TCP_NODELAY on the accepted sockets to disable Nagle.
    
    This should not have any impact on the "normal" deliveries, as these
    will as before be done through a single writev operation.
    
    For deliveries involving chunked encoding (e.g. ESI or gunzip), this
    should prevent an unnecessary delay where the Nagle algorithm kicks in
    on the end-chunk write. This should give higher throughput and better
    utilization of a single connection, especially for clients not doing
    pipelining.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index c5831fd..ce0bfe2 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -41,6 +41,9 @@
 
 #include "config.h"
 
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
 #include "cache.h"
 #include "common/heritage.h"
 
@@ -66,7 +69,8 @@ static const struct linger linger = {
 	.l_onoff	=	0,
 };
 
-static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
+static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test,
+			need_tcpnodelay;
 
 /*--------------------------------------------------------------------
  * Some kernels have bugs/limitations with respect to which options are
@@ -80,7 +84,7 @@ sock_test(int fd)
 	struct linger lin;
 	struct timeval tv;
 	socklen_t l;
-	int i;
+	int i, tcp_nodelay;
 
 	l = sizeof lin;
 	i = getsockopt(fd, SOL_SOCKET, SO_LINGER, &lin, &l);
@@ -124,6 +128,16 @@ sock_test(int fd)
 	(void)need_rcvtimeo;
 #endif
 
+	l = sizeof tcp_nodelay;
+	i = getsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &tcp_nodelay, &l);
+	if (i) {
+		VTCP_Assert(i);
+		return;
+	}
+	assert(l == sizeof tcp_nodelay);
+	if (!tcp_nodelay)
+		need_tcpnodelay = 1;
+
 	need_test = 0;
 }
 
@@ -246,6 +260,7 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp)
 {
 	struct wrk_accept *wa;
 	const char *retval;
+	int tcp_nodelay = 1;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -275,6 +290,9 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp)
 		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_RCVTIMEO,
 		    &tv_rcvtimeo, sizeof tv_rcvtimeo));
 #endif
+	if (need_tcpnodelay)
+		VTCP_Assert(setsockopt(sp->fd, IPPROTO_TCP, TCP_NODELAY,
+		    &tcp_nodelay, sizeof tcp_nodelay));
 	return (retval);
 }
 
@@ -289,6 +307,7 @@ vca_acct(void *arg)
 #ifdef SO_SNDTIMEO_WORKS
 	double send_timeout = 0;
 #endif
+	int tcp_nodelay = 1;
 	struct listen_sock *ls;
 	double t0, now;
 	int i;
@@ -302,6 +321,8 @@ vca_acct(void *arg)
 		AZ(listen(ls->sock, cache_param->listen_depth));
 		AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER,
 		    &linger, sizeof linger));
+		AZ(setsockopt(ls->sock, IPPROTO_TCP, TCP_NODELAY,
+		    &tcp_nodelay, sizeof tcp_nodelay));
 		if (cache_param->accept_filter) {
 			i = VTCP_filter_http(ls->sock);
 			if (i)



From martin at varnish-cache.org  Wed Sep 19 13:31:02 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 19 Sep 2012 15:31:02 +0200
Subject: [master] 5422dff Include <sys/types.h> before <netinet/tcp.h> to see
 if that makes the build happy on FreeBSD.
Message-ID: <E1TEKMo-0006Q4-Ha@project.varnish-software.com>

commit 5422dff611e2ea184c42293de8cb8641aaf186cf
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 19 15:30:16 2012 +0200

    Include <sys/types.h> before <netinet/tcp.h> to see if that makes the
    build happy on FreeBSD.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index ce0bfe2..a760436 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -41,6 +41,7 @@
 
 #include "config.h"
 
+#include <sys/types.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 



From lkarsten at varnish-cache.org  Thu Sep 20 09:32:00 2012
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Thu, 20 Sep 2012 11:32:00 +0200
Subject: [3.0] cc1f635 Revert "Mark ban list as volatile".
Message-ID: <E1TEd72-0000yQ-Nk@project.varnish-software.com>

commit cc1f6351ec8f869fe1e707fd95fe08456234a081
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Sep 20 11:27:32 2012 +0200

    Revert "Mark ban list as volatile".
    
    This reverts commit 2587b1b973b1c4e019ee74da7a93fe31af5a6186.
    
    3.0 branch does not build with this commit.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 138a1f6..5c1da57 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -81,8 +81,7 @@ struct ban_test {
 	const void		*arg2_spec;
 };
 
-/* XXX phk thinks he knows why this is a good idea */
-static volatile VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
+static VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
 static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;



From apj at varnish-cache.org  Mon Sep 24 17:45:55 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 24 Sep 2012 19:45:55 +0200
Subject: [master] f0c2fd5 Fix typo
Message-ID: <E1TGCjE-00018k-04@project.varnish-software.com>

commit f0c2fd5c2784628841a280ba634f494f5fb0538c
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 24 19:45:35 2012 +0200

    Fix typo

diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
index 01d8f36..f15425d 100644
--- a/doc/sphinx/users-guide/operation-logging.rst
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -6,7 +6,7 @@ Logging in Varnish
 One of the really nice features in Varnish is how logging
 works. Instead of logging to normal log file Varnish logs to a shared
 memory segment. When the end of the segment is reached we start over,
-overwriting old data. This is much, much faster then logging to a file
+overwriting old data. This is much, much faster than logging to a file
 and it doesn't require disk space. Besides it gives you much, much
 more information when you need it.
 



From tfheen at varnish-cache.org  Wed Sep 26 08:54:58 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 26 Sep 2012 10:54:58 +0200
Subject: [master] d8e5e70 Document that lookup forces GET to the backend
Message-ID: <E1TGnOU-000246-IF@project.varnish-software.com>

commit d8e5e70b4d40fb9458fbd8f1c94d6ca7ed8f6b30
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 26 10:52:52 2012 +0200

    Document that lookup forces GET to the backend

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 772e696..e6c15a8 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -385,7 +385,8 @@ vcl_recv
   lookup  
     Look up the requested object in the cache.  Control will
     eventually pass to vcl_hit or vcl_miss, depending on whether the
-    object is in the cache.
+    object is in the cache.  The ``bereq.request`` value will be set
+    to ``GET`` regardless of the value of ``req.request``.
 
 vcl_pipe
   Called upon entering pipe mode.  In this mode, the request is passed



From tfheen at varnish-cache.org  Wed Sep 26 08:54:58 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 26 Sep 2012 10:54:58 +0200
Subject: [master] 7e5dce9 Typo
Message-ID: <E1TGnOU-00024B-Jb@project.varnish-software.com>

commit 7e5dce9d400189960c986f7a246f7983457c8fc4
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 26 10:52:58 2012 +0200

    Typo

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index e6c15a8..c87ade4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -107,7 +107,7 @@ This is useful is you are using Varnish to load balance in front of
 other Varnish caches or other web accelerators as objects won't be
 duplicated across caches.
 
-It will use the value of req.hash, just as the normal cache-lookup methods.
+It will use the value of req.hash, just as the normal cache lookup methods.
 
 
 The round-robin director