From phk at varnish-cache.org  Wed May  1 08:07:05 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 01 May 2013 10:07:05 +0200
Subject: [master] b3e1e10 Replace the remaining instances of the the "req"
 argument in VCL/VRT/VMOD with vrt_ctx arguments.
Message-ID: <E1UXS49-0001lK-GK@project.varnish-software.com>

commit b3e1e1003fba0bcd7d8534290e940e37963bdb45
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 08:06:28 2013 +0000

    Replace the remaining instances of the the "req" argument in VCL/VRT/VMOD
    with vrt_ctx arguments.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index aa2aec7..3875326 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -220,7 +220,7 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 	REPLACE(vcl->name, name);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
 	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
-	(void)vcl->conf->init_func(&ctx, NULL, NULL);
+	(void)vcl->conf->init_func(&ctx, NULL);
 	Lck_Lock(&vcl_mtx);
 	if (vcl_active == NULL)
 		vcl_active = vcl;
@@ -247,7 +247,7 @@ VCL_Nuke(struct vcls *vcl)
 	assert(vcl->conf->discard);
 	assert(vcl->conf->busy == 0);
 	VTAILQ_REMOVE(&vcl_head, vcl, list);
-	(void)vcl->conf->fini_func(&ctx, NULL, NULL);
+	(void)vcl->conf->fini_func(&ctx, NULL);
 	vcl->conf->fini_vcl(NULL);
 	free(vcl->name);
 	(void)dlclose(vcl->dlh);
@@ -421,7 +421,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	wrk->cur_method = method;
 	AN(vsl);
 	VSLb(vsl, SLT_VCL_call, "%s", VCL_Method_Name(method));
-	(void)func(&ctx, wrk, req);
+	(void)func(&ctx, wrk);
 	VSLb(vsl, SLT_VCL_return, "%s", VCL_Return_Name(wrk->handling));
 	wrk->cur_method = 0;
 	WS_Reset(wrk->aws, aws);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 0d9af43..4c44959 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -55,12 +55,14 @@ const void * const vrt_magic_string_unset = &vrt_magic_string_unset;
 /*--------------------------------------------------------------------*/
 
 const struct gethdr_s *
-VRT_MkGethdr(struct req *req, enum gethdr_e where, const char *what)
+VRT_MkGethdr(const struct vrt_ctx *ctx, enum gethdr_e where, const char *what)
 {
 	struct gethdr_s *retval;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	retval = (void*)WS_Alloc(req->wrk->aws, sizeof *retval);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	// XXX ?
+	retval = (void*)WS_Alloc(ctx->req->wrk->aws, sizeof *retval);
 	AN(retval);
 	retval->where = where;
 	retval->what = what;
@@ -71,16 +73,18 @@ VRT_MkGethdr(struct req *req, enum gethdr_e where, const char *what)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_error(struct req *req, unsigned code, const char *reason)
+VRT_error(const struct vrt_ctx *ctx, unsigned code, const char *reason)
 {
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	VSLb(req->vsl, SLT_Debug, "VCL_error(%u, %s)", code,
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	VSLb(ctx->vsl, SLT_Debug, "VCL_error(%u, %s)", code,
 	    reason ?  reason : "(null)");
 	if (code < 100 || code > 999)
 		code = 503;
-	req->err_code = (uint16_t)code;
-	req->err_reason = reason ? reason : http_StatusMessage(req->err_code);
+	ctx->req->err_code = (uint16_t)code;
+	ctx->req->err_reason =
+	    reason ? reason : http_StatusMessage(ctx->req->err_code);
 }
 
 /*--------------------------------------------------------------------*/
@@ -277,18 +281,20 @@ VRT_handling(struct worker *wrk, unsigned hand)
  */
 
 void
-VRT_hashdata(struct req *req, const char *str, ...)
+VRT_hashdata(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
 	const char *p;
 
-	HSH_AddString(req, str);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	HSH_AddString(ctx->req, str);
 	va_start(ap, str);
 	while (1) {
 		p = va_arg(ap, const char *);
 		if (p == vrt_magic_string_end)
 			break;
-		HSH_AddString(req, p);
+		HSH_AddString(ctx->req, p);
 	}
 }
 
@@ -405,27 +411,29 @@ VRT_l_beresp_saintmode(const struct vrt_ctx *ctx, double a)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_Rollback(struct req *req)
+VRT_Rollback(const struct vrt_ctx *ctx)
 {
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	HTTP_Copy(req->http, req->http0);
-	WS_Reset(req->ws, req->ws_req);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	HTTP_Copy(ctx->req->http, ctx->req->http0);
+	WS_Reset(ctx->req->ws, ctx->req->ws_req);
 }
 
 /*--------------------------------------------------------------------*/
 
 void
-VRT_synth_page(const struct req *req, unsigned flags, const char *str, ...)
+VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 {
 	va_list ap;
 	const char *p;
 	struct vsb *vsb;
 
 	(void)flags;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	vsb = SMS_Makesynth(req->obj);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
+	vsb = SMS_Makesynth(ctx->req->obj);
 	AN(vsb);
 
 	VSB_cat(vsb, str);
@@ -438,9 +446,10 @@ VRT_synth_page(const struct req *req, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	SMS_Finish(req->obj);
-	http_Unset(req->obj->http, H_Content_Length);
-	http_PrintfHeader(req->obj->http, "Content-Length: %zd", req->obj->len);
+	SMS_Finish(ctx->req->obj);
+	http_Unset(ctx->req->obj->http, H_Content_Length);
+	http_PrintfHeader(ctx->req->obj->http,
+	    "Content-Length: %zd", ctx->req->obj->len);
 }
 
 /*--------------------------------------------------------------------*/
@@ -494,9 +503,12 @@ VRT_ban_string(const char *str)
  */
 
 int
-VRT_CacheReqBody(struct req *req, long long maxsize)
+VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize)
 {
-	return (HTTP1_CacheReqBody(req, maxsize));
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	return (HTTP1_CacheReqBody(ctx->req, maxsize));
 }
 
 /*--------------------------------------------------------------------
@@ -504,15 +516,19 @@ VRT_CacheReqBody(struct req *req, long long maxsize)
  */
 
 void
-VRT_purge(const struct worker *wrk, struct req *req, double ttl, double grace)
+VRT_purge(const struct worker *wrk, const struct vrt_ctx *ctx, double ttl,
+    double grace)
 {
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	if (wrk->cur_method == VCL_MET_LOOKUP)
-		HSH_Purge(req, req->obj->objcore->objhead, ttl, grace);
+		HSH_Purge(ctx->req, ctx->req->obj->objcore->objhead,
+		    ttl, grace);
 	else if (wrk->cur_method == VCL_MET_MISS)
-		HSH_Purge(req, req->objcore->objhead, ttl, grace);
+		HSH_Purge(ctx->req, ctx->req->objcore->objhead,
+		    ttl, grace);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index e4b9750..856a057 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -62,12 +62,12 @@ VRT_re_fini(void *rep)
 }
 
 int
-VRT_re_match(struct req *req, const char *s, void *re)
+VRT_re_match(const struct vrt_ctx *ctx, const char *s, void *re)
 {
 	vre_t *t;
 	int i;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (s == NULL)
 		s = "";
 	AN(re);
@@ -76,12 +76,12 @@ VRT_re_match(struct req *req, const char *s, void *re)
 	if (i >= 0)
 		return (1);
 	if (i < VRE_ERROR_NOMATCH )
-		VSLb(req->vsl, SLT_VCL_Error, "Regexp matching returned %d", i);
+		VSLb(ctx->vsl, SLT_VCL_Error, "Regexp matching returned %d", i);
 	return (0);
 }
 
 const char *
-VRT_regsub(struct req *req, int all, const char *str, void *re,
+VRT_regsub(const struct vrt_ctx *ctx, int all, const char *str, void *re,
     const char *sub)
 {
 	int ovector[30];
@@ -94,7 +94,7 @@ VRT_regsub(struct req *req, int all, const char *str, void *re,
 	int options = 0;
 	size_t len;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(re);
 	if (str == NULL)
 		str = "";
@@ -110,12 +110,12 @@ VRT_regsub(struct req *req, int all, const char *str, void *re,
 	if (i == VRE_ERROR_NOMATCH)
 		return(str);
 	if (i < VRE_ERROR_NOMATCH ) {
-		VSLb(req->vsl, SLT_VCL_Error, "Regexp matching returned %d", i);
+		VSLb(ctx->vsl, SLT_VCL_Error, "Regexp matching returned %d", i);
 		return(str);
 	}
 
-	u = WS_Reserve(req->http->ws, 0);
-	res.e = res.b = b0 = req->http->ws->f;
+	u = WS_Reserve(ctx->ws, 0);
+	res.e = res.b = b0 = ctx->ws->f;
 	res.e += u;
 
 	do {
@@ -147,8 +147,8 @@ VRT_regsub(struct req *req, int all, const char *str, void *re,
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &cache_param->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
-			WS_Release(req->http->ws, 0);
-			VSLb(req->vsl, SLT_VCL_Error,
+			WS_Release(ctx->ws, 0);
+			VSLb(ctx->vsl, SLT_VCL_Error,
 			    "Regexp matching returned %d", i);
 			return(str);
 		}
@@ -157,10 +157,10 @@ VRT_regsub(struct req *req, int all, const char *str, void *re,
 	/* Copy suffix to match */
 	Tadd(&res, str, len+1);
 	if (res.b >= res.e) {
-		WS_Release(req->http->ws, 0);
+		WS_Release(ctx->ws, 0);
 		return (str);
 	}
 	Tcheck(res);
-	WS_ReleaseP(req->http->ws, res.b);
+	WS_ReleaseP(ctx->ws, res.b);
 	return (b0);
 }
diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc
index f78b582..809387d 100644
--- a/bin/varnishtest/tests/c00033.vtc
+++ b/bin/varnishtest/tests/c00033.vtc
@@ -25,13 +25,13 @@ varnish v1 -vcl+backend {
 
 	sub vcl_lookup {
 		if (req.method == "PURGE") {
-			C{ VRT_purge(wrk, req, 0, 0); }C
+			C{ VRT_purge(wrk, ctx, 0, 0); }C
 			error 456 "got it";
 		}
 	}
 	sub vcl_miss {
 		if (req.method == "PURGE") {
-			C{ VRT_purge(wrk, req, 0, 0); }C
+			C{ VRT_purge(wrk, ctx, 0, 0); }C
 			error 456 "got it";
 		}
 	}
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index b7c93a4..5bc57f2 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -12,7 +12,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		C{ VRT_CacheReqBody(req, 1000); }C
+		C{ VRT_CacheReqBody(ctx, 1000); }C
 		return (pass);
 	}
 	sub vcl_backend_response {
diff --git a/include/vrt.h b/include/vrt.h
index 031a9e1..063cf9a 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -186,38 +186,40 @@ void VRT_acl_log(const struct vrt_ctx *, const char *msg);
 
 /* req related */
 
-int VRT_CacheReqBody(struct req *, long long maxsize);
+int VRT_CacheReqBody(const struct vrt_ctx *, long long maxsize);
 
 /* Regexp related */
 void VRT_re_init(void **, const char *);
 void VRT_re_fini(void *);
-int VRT_re_match(struct req *, const char *, void *re);
-const char *VRT_regsub(struct req *, int all, const char *,
+int VRT_re_match(const struct vrt_ctx *, const char *, void *re);
+const char *VRT_regsub(const struct vrt_ctx *, int all, const char *,
     void *, const char *);
 
 void VRT_ban_string(const char *);
-void VRT_purge(const struct worker *, struct req *, double ttl, double grace);
+void VRT_purge(const struct worker *, const struct vrt_ctx *, double ttl,
+    double grace);
 
 void VRT_count(const struct vrt_ctx *, unsigned);
 int VRT_rewrite(const char *, const char *);
-void VRT_error(struct req *, unsigned, const char *);
+void VRT_error(const struct vrt_ctx *, unsigned, const char *);
 int VRT_switch_config(const char *);
 
-const struct gethdr_s *VRT_MkGethdr(struct req *,enum gethdr_e, const char *);
+const struct gethdr_s *VRT_MkGethdr(const struct vrt_ctx *,enum gethdr_e,
+    const char *);
 char *VRT_GetHdr(const struct vrt_ctx *, const struct gethdr_s *);
 void VRT_SetHdr(const struct vrt_ctx *, const struct gethdr_s *, const char *, ...);
 void VRT_handling(struct worker *, unsigned hand);
 
-void VRT_hashdata(struct req *, const char *str, ...);
+void VRT_hashdata(const struct vrt_ctx *, const char *str, ...);
 
 /* Simple stuff */
 int VRT_strcmp(const char *s1, const char *s2);
 void VRT_memmove(void *dst, const void *src, unsigned len);
 
-void VRT_Rollback(struct req *);
+void VRT_Rollback(const struct vrt_ctx *);
 
 /* Synthetic pages */
-void VRT_synth_page(const struct req *, unsigned flags, const char *, ...);
+void VRT_synth_page(const struct vrt_ctx *, unsigned flags, const char *, ...);
 
 /* Backend related */
 void VRT_init_dir(struct cli *, struct director **, const char *name,
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 254a3cc..509141f 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -698,7 +698,6 @@ fo = open(buildroot + "/include/vcl.h", "w")
 file_header(fo)
 
 fo.write("""
-struct sess;
 struct vrt_ctx;
 struct req;
 struct busyobj;
@@ -708,7 +707,7 @@ struct worker;
 
 typedef int vcl_init_f(struct cli *);
 typedef void vcl_fini_f(struct cli *);
-typedef int vcl_func_f(const struct vrt_ctx *ctx, struct worker *, struct req *);
+typedef int vcl_func_f(const struct vrt_ctx *ctx, struct worker *);
 """)
 
 
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 217aee4..a5bd769 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -46,7 +46,7 @@ parse_call(struct vcc *tl)
 	ExpectErr(tl, ID);
 	vcc_AddCall(tl, tl->t);
 	vcc_AddRef(tl, tl->t, SYM_SUB);
-	Fb(tl, 1, "if (VGC_function_%.*s(ctx, req))\n", PF(tl->t));
+	Fb(tl, 1, "if (VGC_function_%.*s(ctx))\n", PF(tl->t));
 	Fb(tl, 1, "\treturn (1);\n");
 	vcc_NextToken(tl);
 	return;
@@ -59,7 +59,7 @@ parse_error(struct vcc *tl)
 {
 
 	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_error(req,\n");
+	Fb(tl, 1, "VRT_error(ctx,\n");
 	if (tl->t->tok == '(') {
 		vcc_NextToken(tl);
 		vcc_Expr(tl, INT);
@@ -223,7 +223,7 @@ parse_new(struct vcc *tl)
 
 	bprintf(buf1, ", &%s, \"%s\"", sy1->name, sy1->name);
 	vcc_Eval_Func(tl, s_init, buf1, "ASDF", s_init + strlen(s_init) + 1);
-	Fd(tl, 0, "\t%s((struct req*)0, &%s);\n", s_fini, sy1->name);
+	Fd(tl, 0, "\t%s(&%s);\n", s_fini, sy1->name);
 	ExpectErr(tl, ';');
 
 	bprintf(buf1, ", %s", sy1->name);
@@ -292,7 +292,7 @@ parse_hash_data(struct vcc *tl)
 	vcc_NextToken(tl);
 	SkipToken(tl, '(');
 
-	Fb(tl, 1, "VRT_hashdata(req, ");
+	Fb(tl, 1, "VRT_hashdata(ctx, ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");
@@ -339,7 +339,7 @@ parse_rollback(struct vcc *tl)
 {
 
 	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_Rollback(req);\n");
+	Fb(tl, 1, "VRT_Rollback(ctx);\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -349,7 +349,7 @@ parse_purge(struct vcc *tl)
 {
 
 	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_purge(wrk, req, 0, 0);\n");
+	Fb(tl, 1, "VRT_purge(wrk, ctx, 0, 0);\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -359,7 +359,7 @@ parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_synth_page(req, 0, ");
+	Fb(tl, 1, "VRT_synth_page(ctx, 0, ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index bf992e2..a3de471 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -694,8 +694,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 		Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_f)\n");
 		Fc(tl, 1,
 		    "VGC_function_%s(const struct vrt_ctx *ctx,"
-		    " struct worker *wrk,"
-		    " struct req *req)\n",
+		    " struct worker *wrk)\n",
 		    method_tab[i].name);
 		AZ(VSB_finish(tl->fm[i]));
 		Fc(tl, 1, "{\n");
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index c318920..88d7ccb 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -459,7 +459,7 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym)
 	p = vcc_regexp(tl);
 	vcc_NextToken(tl);
 
-	bprintf(buf, "VRT_regsub(req, %d,\v+\n\v1,\n%s", all, p);
+	bprintf(buf, "VRT_regsub(ctx, %d,\v+\n\v1,\n%s", all, p);
 	*e = vcc_expr_edit(STRING, buf, e2, *e);
 
 	SkipToken(tl, ',');
@@ -535,7 +535,7 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 	p = args;
 	if (extra == NULL)
 		extra = "";
-	e2 = vcc_mk_expr(vcc_arg_type(&p), "%s(req%s\v+", cfunc, extra);
+	e2 = vcc_mk_expr(vcc_arg_type(&p), "%s(ctx%s\v+", cfunc, extra);
 	while (*p != '\0') {
 		e1 = NULL;
 		fmt = vcc_arg_type(&p);
@@ -1002,7 +1002,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 		re = vcc_regexp(tl);
 		ERRCHK(tl);
 		vcc_NextToken(tl);
-		bprintf(buf, "%sVRT_re_match(req, \v1, %s)", not, re);
+		bprintf(buf, "%sVRT_re_match(ctx, \v1, %s)", not, re);
 		*e = vcc_expr_edit(BOOL, buf, *e, NULL);
 		return;
 	}
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index 75e976f..ff70d64 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -237,10 +237,10 @@ vcc_Function(struct vcc *tl)
 		}
 		tl->curproc = vcc_AddProc(tl, tl->t);
 		Fh(tl, 0, "static int VGC_function_%.*s "
-		    "(struct vrt_ctx *ctx, struct req *);\n", PF(tl->t));
+		    "(struct vrt_ctx *ctx);\n", PF(tl->t));
 		Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_t)\n");
-		Fc(tl, 1, "VGC_function_%.*s(struct vrt_ctx *ctx, "
-		    "struct req *req)\n", PF(tl->t));
+		Fc(tl, 1, "VGC_function_%.*s(struct vrt_ctx *ctx)\n",
+		    PF(tl->t));
 	}
 	vcc_NextToken(tl);
 	tl->indent += INDENT;
diff --git a/lib/libvcl/vmodtool.py b/lib/libvcl/vmodtool.py
index ddec683..259f97f 100755
--- a/lib/libvcl/vmodtool.py
+++ b/lib/libvcl/vmodtool.py
@@ -246,27 +246,37 @@ class func(object):
 	def set_pfx(self, s):
 		self.pfx = s
 
-	def c_proto(self, fo):
+	def c_proto(self, fo, fini=False):
 		fo.write(ctypes[self.retval])
-		fo.write(" vmod_" + self.cnam)
-		fo.write("(struct req *")
+		fo.write(" vmod_" + self.cnam + "(")
+		p = ""
+		if not fini:
+			fo.write("const struct vrt_ctx *")
+			p = ", "
 		if self.pfx != None:
-			fo.write(self.pfx)
+			fo.write(p + self.pfx)
+			p = ", "
 		for a in self.al:
-			fo.write(", " + ctypes[a.typ])
+			fo.write(p + ctypes[a.typ])
+			p = ", "
 			if a.nam != None:
 				fo.write(" " + a.nam)
 		fo.write(");\n")
 
-	def c_typedef(self, modname):
+	def c_typedef(self, modname, fini=False):
 		s = "typedef "
 		s += ctypes[self.retval]
-		s += " td_" + modname + "_" + self.cnam
-		s += "(struct req *"
+		s += " td_" + modname + "_" + self.cnam + "("
+		p = ""
+		if not fini:
+			s += "const struct vrt_ctx *"
+			p = ", "
 		if self.pfx != None:
-			s += self.pfx
+			s += p + self.pfx
+			p = ", "
 		for a in self.al:
-			s += ", " + ctypes[a.typ]
+			s += p + ctypes[a.typ]
+			p = ", "
 		s += ");"
 		return s
 
@@ -300,10 +310,10 @@ class obj(object):
 
 	def set_modnam(self, modnam):
 		self.st = "struct vmod_" + modnam + "_" + self.nam
-		self.init.set_pfx(", " + self.st + " **, const char *")
-		self.fini.set_pfx(", " + self.st + " **")
+		self.init.set_pfx(self.st + " **, const char *")
+		self.fini.set_pfx(self.st + " **")
 		for m in self.methods:
-			m.set_pfx(", " + self.st + " *")
+			m.set_pfx(self.st + " *")
 
 	def set_init(self, f):
 		self.init = f
@@ -319,7 +329,7 @@ class obj(object):
 		l.append("/* Object " + self.nam + " */")
 		l.append(self.st + ";")
 		l.append(self.init.c_typedef(modnam) + "")
-		l.append(self.fini.c_typedef(modnam) + "")
+		l.append(self.fini.c_typedef(modnam, fini=True) + "")
 		for m in self.methods:
 			l.append(m.c_typedef(modnam) + "")
 		return l
@@ -327,7 +337,7 @@ class obj(object):
 	def c_proto(self, fo):
 		fo.write(self.st + ";\n")
 		self.init.c_proto(fo)
-		self.fini.c_proto(fo)
+		self.fini.c_proto(fo, fini = True)
 		for m in self.methods:
 			m.c_proto(fo)
 
@@ -520,7 +530,7 @@ fh = open("vcc_if.h", "w")
 file_header(fc)
 file_header(fh)
 
-fh.write('struct req;\n')
+fh.write('struct vrt_ctx;\n')
 fh.write('struct VCL_conf;\n')
 fh.write('struct vmod_priv;\n')
 fh.write("\n");
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index ee062ac..5583a4f 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -36,23 +36,23 @@
 #include "vcc_if.h"
 
 VCL_VOID __match_proto__(td_debug_panic)
-vmod_panic(struct req *req, const char *str, ...)
+vmod_panic(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
 	char *b;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	va_start(ap, str);
-	b = VRT_String(req->http->ws, "PANIC: ", str, ap);
+	b = VRT_String(ctx->ws, "PANIC: ", str, ap);
 	va_end(ap);
 	VAS_Fail("VCL", "", 0, b, 0, VAS_VCL);
 }
 
 VCL_STRING __match_proto__(td_debug_author)
-vmod_author(struct req *req, VCL_ENUM id)
+vmod_author(const struct vrt_ctx *ctx, VCL_ENUM id)
 {
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (!strcmp(id, "phk"))
 		return ("Poul-Henning");
 	if (!strcmp(id, "des"))
@@ -75,10 +75,10 @@ init_function(struct vmod_priv *priv, const struct VCL_conf *cfg)
 }
 
 VCL_VOID __match_proto__(td_debug_test_priv_call)
-vmod_test_priv_call(struct req *req, struct vmod_priv *priv)
+vmod_test_priv_call(const struct vrt_ctx *ctx, struct vmod_priv *priv)
 {
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (priv->priv == NULL) {
 		priv->priv = strdup("BAR");
 		priv->free = free;
@@ -88,9 +88,9 @@ vmod_test_priv_call(struct req *req, struct vmod_priv *priv)
 }
 
 VCL_VOID __match_proto__(td_debug_test_priv_vcl)
-vmod_test_priv_vcl(struct req *req, struct vmod_priv *priv)
+vmod_test_priv_vcl(const struct vrt_ctx *ctx, struct vmod_priv *priv)
 {
 
-        CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
         assert(!strcmp(priv->priv, "FOO"));
 }
diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c
index 2b628fa..c27b6db 100644
--- a/lib/libvmod_debug/vmod_debug_obj.c
+++ b/lib/libvmod_debug/vmod_debug_obj.c
@@ -42,12 +42,12 @@ struct vmod_debug_obj {
 };
 
 VCL_VOID
-vmod_obj__init(struct req *req, struct vmod_debug_obj **op,
+vmod_obj__init(const struct vrt_ctx *ctx, struct vmod_debug_obj **op,
     const char *vcl_name, VCL_STRING s)
 {
 	struct vmod_debug_obj *o;
 
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	(void)vcl_name;
 	(void)s;
 	AN(op);
@@ -60,10 +60,9 @@ vmod_obj__init(struct req *req, struct vmod_debug_obj **op,
 }
 
 VCL_VOID
-vmod_obj__fini(struct req *req, struct vmod_debug_obj **op)
+vmod_obj__fini(struct vmod_debug_obj **op)
 {
 
-	(void)req;
 	AN(op);
 	AN(*op);
 	FREE_OBJ(*op);
@@ -71,9 +70,10 @@ vmod_obj__fini(struct req *req, struct vmod_debug_obj **op)
 }
 
 VCL_STRING __match_proto__()
-vmod_obj_foo(struct req *req, struct vmod_debug_obj *o, VCL_STRING s)
+vmod_obj_foo(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_STRING s)
 {
-	(void)req;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	(void)s;
 	CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_OBJ_MAGIC);
 	assert(o->foobar == 42);
@@ -81,9 +81,10 @@ vmod_obj_foo(struct req *req, struct vmod_debug_obj *o, VCL_STRING s)
 }
 
 VCL_TIME __match_proto__()
-vmod_obj_date(struct req *req, struct vmod_debug_obj *o)
+vmod_obj_date(const struct vrt_ctx *ctx, struct vmod_debug_obj *o)
 {
-	(void)req;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_OBJ_MAGIC);
 	assert(o->foobar == 42);
 	return (21.4);
diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c
index 7907700..7c430ac 100644
--- a/lib/libvmod_directors/fall_back.c
+++ b/lib/libvmod_directors/fall_back.c
@@ -75,12 +75,12 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 }
 
 VCL_VOID __match_proto__()
-vmod_fallback__init(struct req *req, struct vmod_directors_fallback **rrp,
-    const char *vcl_name)
+vmod_fallback__init(const struct vrt_ctx *ctx,
+    struct vmod_directors_fallback **rrp, const char *vcl_name)
 {
 	struct vmod_directors_fallback *rr;
 
-	AZ(req);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(rrp);
 	AZ(*rrp);
 	ALLOC_OBJ(rr, VMOD_DIRECTORS_FALLBACK_MAGIC);
@@ -90,11 +90,10 @@ vmod_fallback__init(struct req *req, struct vmod_directors_fallback **rrp,
 }
 
 VCL_VOID __match_proto__()
-vmod_fallback__fini(struct req *req, struct vmod_directors_fallback **rrp)
+vmod_fallback__fini(struct vmod_directors_fallback **rrp)
 {
 	struct vmod_directors_fallback *rr;
 
-	AZ(req);
 	rr = *rrp;
 	*rrp = NULL;
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_FALLBACK_MAGIC);
@@ -103,19 +102,20 @@ vmod_fallback__fini(struct req *req, struct vmod_directors_fallback **rrp)
 }
 
 VCL_VOID __match_proto__()
-vmod_fallback_add_backend(struct req *req,
+vmod_fallback_add_backend(const struct vrt_ctx *ctx,
     struct vmod_directors_fallback *rr, VCL_BACKEND be)
 {
 
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_FALLBACK_MAGIC);
 	(void)vdir_add_backend(rr->vd, be, 0.0);
 }
 
 VCL_BACKEND __match_proto__()
-vmod_fallback_backend(struct req *req, struct vmod_directors_fallback *rr)
+vmod_fallback_backend(const struct vrt_ctx *ctx,
+    struct vmod_directors_fallback *rr)
 {
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_FALLBACK_MAGIC);
 	return (rr->vd->dir);
 }
diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c
index ac9bd41..43b4872 100644
--- a/lib/libvmod_directors/hash.c
+++ b/lib/libvmod_directors/hash.c
@@ -52,12 +52,12 @@ struct vmod_directors_hash {
 };
 
 VCL_VOID __match_proto__()
-vmod_hash__init(struct req *req, struct vmod_directors_hash **rrp,
+vmod_hash__init(const struct vrt_ctx *ctx, struct vmod_directors_hash **rrp,
     const char *vcl_name)
 {
 	struct vmod_directors_hash *rr;
 
-	AZ(req);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(rrp);
 	AZ(*rrp);
 	ALLOC_OBJ(rr, VMOD_DIRECTORS_HASH_MAGIC);
@@ -70,11 +70,10 @@ vmod_hash__init(struct req *req, struct vmod_directors_hash **rrp,
 }
 
 VCL_VOID __match_proto__()
-vmod_hash__fini(struct req *req, struct vmod_directors_hash **rrp)
+vmod_hash__fini(struct vmod_directors_hash **rrp)
 {
 	struct vmod_directors_hash *rr;
 
-	AZ(req);
 	rr = *rrp;
 	*rrp = NULL;
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC);
@@ -84,40 +83,41 @@ vmod_hash__fini(struct req *req, struct vmod_directors_hash **rrp)
 }
 
 VCL_VOID __match_proto__()
-vmod_hash_add_backend(struct req *req,
+vmod_hash_add_backend(const struct vrt_ctx *ctx,
     struct vmod_directors_hash *rr, VCL_BACKEND be, double w)
 {
 
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC);
 	(void)vdir_add_backend(rr->vd, be, w);
 }
 
 VCL_BACKEND __match_proto__()
-vmod_hash_backend(struct req *req, struct vmod_directors_hash *rr, const char *arg, ...)
+vmod_hash_backend(const struct vrt_ctx *ctx, struct vmod_directors_hash *rr,
+    const char *arg, ...)
 {
-	struct SHA256Context ctx;
+	struct SHA256Context sha_ctx;
 	va_list ap;
 	const char *p;
 	unsigned char sha256[SHA256_LEN];
 	VCL_BACKEND be;
 	double r;
 
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC);
-	SHA256_Init(&ctx);
+	SHA256_Init(&sha_ctx);
 	va_start(ap, arg);
 	p = arg;
 	while (p != vrt_magic_string_end) {
-		SHA256_Update(&ctx, arg, strlen(arg));
+		SHA256_Update(&sha_ctx, arg, strlen(arg));
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	SHA256_Final(sha256, &ctx);
+	SHA256_Final(sha256, &sha_ctx);
 
 	r = vbe32dec(sha256);
 	r = scalbn(r, -32);
-	be = vdir_pick_be(rr->vd, req->busyobj, r, rr->nloops);
+	be = vdir_pick_be(rr->vd, ctx->bo, r, rr->nloops);
 	return (be);
 }
diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c
index d51c170..a8dd8a7 100644
--- a/lib/libvmod_directors/random.c
+++ b/lib/libvmod_directors/random.c
@@ -74,12 +74,12 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 }
 
 VCL_VOID __match_proto__()
-vmod_random__init(struct req *req, struct vmod_directors_random **rrp,
+vmod_random__init(const struct vrt_ctx *ctx, struct vmod_directors_random **rrp,
     const char *vcl_name)
 {
 	struct vmod_directors_random *rr;
 
-	AZ(req);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(rrp);
 	AZ(*rrp);
 	ALLOC_OBJ(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
@@ -92,11 +92,10 @@ vmod_random__init(struct req *req, struct vmod_directors_random **rrp,
 }
 
 VCL_VOID __match_proto__()
-vmod_random__fini(struct req *req, struct vmod_directors_random **rrp)
+vmod_random__fini(struct vmod_directors_random **rrp)
 {
 	struct vmod_directors_random *rr;
 
-	AZ(req);
 	rr = *rrp;
 	*rrp = NULL;
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
@@ -106,19 +105,19 @@ vmod_random__fini(struct req *req, struct vmod_directors_random **rrp)
 }
 
 VCL_VOID __match_proto__()
-vmod_random_add_backend(struct req *req,
+vmod_random_add_backend(const struct vrt_ctx *ctx,
     struct vmod_directors_random *rr, VCL_BACKEND be, double w)
 {
 
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
 	(void)vdir_add_backend(rr->vd, be, w);
 }
 
 VCL_BACKEND __match_proto__()
-vmod_random_backend(struct req *req, struct vmod_directors_random *rr)
+vmod_random_backend(const struct vrt_ctx *ctx, struct vmod_directors_random *rr)
 {
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
 	return (rr->vd->dir);
 }
diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c
index fb4ba92..fe0e528 100644
--- a/lib/libvmod_directors/round_robin.c
+++ b/lib/libvmod_directors/round_robin.c
@@ -78,12 +78,12 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 }
 
 VCL_VOID __match_proto__()
-vmod_round_robin__init(struct req *req, struct vmod_directors_round_robin **rrp,
-    const char *vcl_name)
+vmod_round_robin__init(const struct vrt_ctx *ctx,
+    struct vmod_directors_round_robin **rrp, const char *vcl_name)
 {
 	struct vmod_directors_round_robin *rr;
 
-	AZ(req);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(rrp);
 	AZ(*rrp);
 	ALLOC_OBJ(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
@@ -93,11 +93,10 @@ vmod_round_robin__init(struct req *req, struct vmod_directors_round_robin **rrp,
 }
 
 VCL_VOID __match_proto__()
-vmod_round_robin__fini(struct req *req, struct vmod_directors_round_robin **rrp)
+vmod_round_robin__fini(struct vmod_directors_round_robin **rrp)
 {
 	struct vmod_directors_round_robin *rr;
 
-	AZ(req);
 	rr = *rrp;
 	*rrp = NULL;
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
@@ -106,19 +105,21 @@ vmod_round_robin__fini(struct req *req, struct vmod_directors_round_robin **rrp)
 }
 
 VCL_VOID __match_proto__()
-vmod_round_robin_add_backend(struct req *req,
+vmod_round_robin_add_backend(const struct vrt_ctx *ctx,
     struct vmod_directors_round_robin *rr, VCL_BACKEND be)
 {
 
-	(void)req;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
 	(void)vdir_add_backend(rr->vd, be, 0.0);
 }
 
 VCL_BACKEND __match_proto__()
-vmod_round_robin_backend(struct req *req, struct vmod_directors_round_robin *rr)
+vmod_round_robin_backend(const struct vrt_ctx *ctx,
+    struct vmod_directors_round_robin *rr)
 {
-	(void)req;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
 	return (rr->vd->dir);
 }
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index cbea944..f105135 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -46,25 +46,25 @@
 #include "vcc_if.h"
 
 VCL_VOID __match_proto__(td_std_set_ip_tos)
-vmod_set_ip_tos(struct req *req, VCL_INT tos)
+vmod_set_ip_tos(const struct vrt_ctx *ctx, VCL_INT tos)
 {
 	int itos = tos;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	VTCP_Assert(setsockopt(req->sp->fd,
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	VTCP_Assert(setsockopt(ctx->req->sp->fd,
 	    IPPROTO_IP, IP_TOS, &itos, sizeof(itos)));
 }
 
 static const char *
-vmod_updown(struct req *req, int up, const char *s, va_list ap)
+vmod_updown(const struct vrt_ctx *ctx, int up, const char *s, va_list ap)
 {
 	unsigned u;
 	char *b, *e;
 	const char *p;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	u = WS_Reserve(req->ws, 0);
-	e = b = req->ws->f;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	u = WS_Reserve(ctx->ws, 0);
+	e = b = ctx->ws->f;
 	e += u;
 	p = s;
 	while (p != vrt_magic_string_end && b < e) {
@@ -81,48 +81,48 @@ vmod_updown(struct req *req, int up, const char *s, va_list ap)
 		*b = '\0';
 	b++;
 	if (b > e) {
-		WS_Release(req->ws, 0);
+		WS_Release(ctx->ws, 0);
 		return (NULL);
 	} else {
 		e = b;
-		b = req->ws->f;
-		WS_Release(req->ws, e - b);
+		b = ctx->ws->f;
+		WS_Release(ctx->ws, e - b);
 		return (b);
 	}
 }
 
 VCL_STRING __match_proto__(td_std_toupper)
-vmod_toupper(struct req *req, const char *s, ...)
+vmod_toupper(const struct vrt_ctx *ctx, const char *s, ...)
 {
 	const char *p;
 	va_list ap;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	va_start(ap, s);
-	p = vmod_updown(req, 1, s, ap);
+	p = vmod_updown(ctx, 1, s, ap);
 	va_end(ap);
 	return (p);
 }
 
 VCL_STRING __match_proto__(td_std_tolower)
-vmod_tolower(struct req *req, const char *s, ...)
+vmod_tolower(const struct vrt_ctx *ctx, const char *s, ...)
 {
 	const char *p;
 	va_list ap;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	va_start(ap, s);
-	p = vmod_updown(req, 0, s, ap);
+	p = vmod_updown(ctx, 0, s, ap);
 	va_end(ap);
 	return (p);
 }
 
 VCL_REAL __match_proto__(td_std_random)
-vmod_random(struct req *req, VCL_REAL lo, VCL_REAL hi)
+vmod_random(const struct vrt_ctx *ctx, VCL_REAL lo, VCL_REAL hi)
 {
 	double a;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	a = drand48();
 	a *= hi - lo;
 	a += lo;
@@ -130,60 +130,56 @@ vmod_random(struct req *req, VCL_REAL lo, VCL_REAL hi)
 }
 
 VCL_VOID __match_proto__(td_std_log)
-vmod_log(struct req *req, const char *fmt, ...)
+vmod_log(const struct vrt_ctx *ctx, const char *fmt, ...)
 {
 	unsigned u;
 	va_list ap;
 	txt t;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	u = WS_Reserve(req->ws, 0);
-	t.b = req->ws->f;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	u = WS_Reserve(ctx->ws, 0);
+	t.b = ctx->ws->f;
 	va_start(ap, fmt);
 	t.e = VRT_StringList(t.b, u, fmt, ap);
 	va_end(ap);
 	if (t.e != NULL) {
 		assert(t.e > t.b);
 		t.e--;
-		VSLbt(req->vsl, SLT_VCL_Log, t);
+		VSLbt(ctx->vsl, SLT_VCL_Log, t);
 	}
-	WS_Release(req->ws, 0);
+	WS_Release(ctx->ws, 0);
 }
 
 VCL_VOID __match_proto__(td_std_syslog)
-vmod_syslog(struct req *req, VCL_INT fac, const char *fmt, ...)
+vmod_syslog(const struct vrt_ctx *ctx, VCL_INT fac, const char *fmt, ...)
 {
 	char *p;
 	unsigned u;
 	va_list ap;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	u = WS_Reserve(req->ws, 0);
-	p = req->ws->f;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	u = WS_Reserve(ctx->ws, 0);
+	p = ctx->ws->f;
 	va_start(ap, fmt);
 	p = VRT_StringList(p, u, fmt, ap);
 	va_end(ap);
 	if (p != NULL)
 		syslog((int)fac, "%s", p);
-	WS_Release(req->ws, 0);
+	WS_Release(ctx->ws, 0);
 }
 
 VCL_VOID __match_proto__(td_std_collect)
-vmod_collect(struct req *req, const struct gethdr_s *hdr)
+vmod_collect(const struct vrt_ctx *ctx, const struct gethdr_s *hdr)
 {
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (hdr->where == HDR_REQ)
-		http_CollectHdr(req->http, hdr->what);
+		http_CollectHdr(ctx->http_req, hdr->what);
 	else if (hdr->where == HDR_BEREQ) {
-		AN(req->busyobj);
-		http_CollectHdr(req->busyobj->bereq, hdr->what);
-	}
-	else if (hdr->where == HDR_BERESP) {
-		AN(req->busyobj);
-		http_CollectHdr(req->busyobj->beresp, hdr->what);
-	}
-	else if (hdr->where == HDR_RESP) {
-		http_CollectHdr(req->resp, hdr->what);
+		http_CollectHdr(ctx->http_bereq, hdr->what);
+	} else if (hdr->where == HDR_BERESP) {
+		http_CollectHdr(ctx->http_beresp, hdr->what);
+	} else if (hdr->where == HDR_RESP) {
+		http_CollectHdr(ctx->http_resp, hdr->what);
 	}
 }
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 96456b2..c763aa3 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -39,12 +39,12 @@
 #include "vcc_if.h"
 
 VCL_DURATION __match_proto__()
-vmod_duration(struct req *req, const char *p, VCL_DURATION d)
+vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
 {
 	char *e;
 	double r;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
 	if (p == NULL)
 		return (d);
@@ -89,12 +89,12 @@ vmod_duration(struct req *req, const char *p, VCL_DURATION d)
 }
 
 VCL_INT __match_proto__()
-vmod_integer(struct req *req, const char *p, VCL_INT i)
+vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
 {
 	char *e;
 	long r;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
 	if (p == NULL)
 		return (i);
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index 163926c..5ddc6bf 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -83,12 +83,12 @@ free_frfile(void *ptr)
 }
 
 VCL_STRING __match_proto__(td_std_fileread)
-vmod_fileread(struct req *req, struct vmod_priv *priv, VCL_STRING file_name)
+vmod_fileread(const struct vrt_ctx *ctx, struct vmod_priv *priv, VCL_STRING file_name)
 {
 	struct frfile *frf = NULL;
 	char *s;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(priv);
 
 	if (priv->priv != NULL) {



From phk at varnish-cache.org  Wed May  1 09:50:37 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 01 May 2013 11:50:37 +0200
Subject: [master] b6da2dc Remove the wrk argument from VCL/VRT,
 now everything we need is encapsulated in the vrt_ctx.
Message-ID: <E1UXTgL-0004k9-69@project.varnish-software.com>

commit b6da2dc74f48d6f2375d9025a21773b4a4a0c31c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 09:50:09 2013 +0000

    Remove the wrk argument from VCL/VRT, now everything we need is
    encapsulated in the vrt_ctx.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 3875326..c7f7002 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -174,6 +174,7 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 	struct vcls *vcl;
 	struct VCL_conf const *cnf;
 	struct vrt_ctx ctx;
+	unsigned hand = 0;
 
 	ASSERT_CLI();
 
@@ -220,7 +221,10 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 	REPLACE(vcl->name, name);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
 	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
-	(void)vcl->conf->init_func(&ctx, NULL);
+	ctx.method = VCL_MET_INIT;
+	ctx.handling = &hand;
+	(void)vcl->conf->init_func(&ctx);
+	assert(hand == VCL_RET_OK);
 	Lck_Lock(&vcl_mtx);
 	if (vcl_active == NULL)
 		vcl_active = vcl;
@@ -239,6 +243,7 @@ static void
 VCL_Nuke(struct vcls *vcl)
 {
 	struct vrt_ctx ctx;
+	unsigned hand = 0;
 
 	memset(&ctx, 0, sizeof ctx);
 	ctx.magic = VRT_CTX_MAGIC;
@@ -247,7 +252,10 @@ VCL_Nuke(struct vcls *vcl)
 	assert(vcl->conf->discard);
 	assert(vcl->conf->busy == 0);
 	VTAILQ_REMOVE(&vcl_head, vcl, list);
-	(void)vcl->conf->fini_func(&ctx, NULL);
+	ctx.method = VCL_MET_FINI;
+	ctx.handling = &hand;
+	(void)vcl->conf->fini_func(&ctx);
+	assert(hand == VCL_RET_OK);
 	vcl->conf->fini_vcl(NULL);
 	free(vcl->name);
 	(void)dlclose(vcl->dlh);
@@ -416,12 +424,14 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		ctx.bo = bo;
 	}
 	ctx.ws = ws;
+	ctx.method = method;
+	ctx.handling = &wrk->handling;
 	aws = WS_Snapshot(wrk->aws);
 	wrk->handling = 0;
 	wrk->cur_method = method;
 	AN(vsl);
 	VSLb(vsl, SLT_VCL_call, "%s", VCL_Method_Name(method));
-	(void)func(&ctx, wrk);
+	(void)func(&ctx);
 	VSLb(vsl, SLT_VCL_return, "%s", VCL_Return_Name(wrk->handling));
 	wrk->cur_method = 0;
 	WS_Reset(wrk->aws, aws);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 4c44959..32764fc 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -264,16 +264,12 @@ VRT_SetHdr(const struct vrt_ctx *ctx , const struct gethdr_s *hs,
 /*--------------------------------------------------------------------*/
 
 void
-VRT_handling(struct worker *wrk, unsigned hand)
+VRT_handling(const struct vrt_ctx *ctx, unsigned hand)
 {
 
-	if (wrk == NULL) {
-		assert(hand == VCL_RET_OK);
-		return;
-	}
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	assert(hand < VCL_RET_MAX);
-	wrk->handling = hand;
+	*ctx->handling = hand;
 }
 
 /*--------------------------------------------------------------------
@@ -516,17 +512,15 @@ VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize)
  */
 
 void
-VRT_purge(const struct worker *wrk, const struct vrt_ctx *ctx, double ttl,
-    double grace)
+VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 {
 
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	if (wrk->cur_method == VCL_MET_LOOKUP)
+	if (ctx->method == VCL_MET_LOOKUP)
 		HSH_Purge(ctx->req, ctx->req->obj->objcore->objhead,
 		    ttl, grace);
-	else if (wrk->cur_method == VCL_MET_MISS)
+	else if (ctx->method == VCL_MET_MISS)
 		HSH_Purge(ctx->req, ctx->req->objcore->objhead,
 		    ttl, grace);
 }
diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc
index 809387d..7f0bda3 100644
--- a/bin/varnishtest/tests/c00033.vtc
+++ b/bin/varnishtest/tests/c00033.vtc
@@ -25,13 +25,13 @@ varnish v1 -vcl+backend {
 
 	sub vcl_lookup {
 		if (req.method == "PURGE") {
-			C{ VRT_purge(wrk, ctx, 0, 0); }C
+			C{ VRT_purge(ctx, 0, 0); }C
 			error 456 "got it";
 		}
 	}
 	sub vcl_miss {
 		if (req.method == "PURGE") {
-			C{ VRT_purge(wrk, ctx, 0, 0); }C
+			C{ VRT_purge(ctx, 0, 0); }C
 			error 456 "got it";
 		}
 	}
diff --git a/include/vrt.h b/include/vrt.h
index 063cf9a..0657e4d 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -70,6 +70,9 @@ struct vrt_ctx {
 	unsigned			magic;
 #define VRT_CTX_MAGIC			0x6bb8f0db
 
+	unsigned			method;
+	unsigned			*handling;
+
 	struct vsl_log			*vsl;
 	struct VCL_conf			*vcl;
 	struct ws			*ws;
@@ -82,6 +85,7 @@ struct vrt_ctx {
 	struct busyobj			*bo;
 	struct http			*http_bereq;
 	struct http			*http_beresp;
+
 };
 
 /***********************************************************************/
@@ -196,8 +200,7 @@ const char *VRT_regsub(const struct vrt_ctx *, int all, const char *,
     void *, const char *);
 
 void VRT_ban_string(const char *);
-void VRT_purge(const struct worker *, const struct vrt_ctx *, double ttl,
-    double grace);
+void VRT_purge(const struct vrt_ctx *, double ttl, double grace);
 
 void VRT_count(const struct vrt_ctx *, unsigned);
 int VRT_rewrite(const char *, const char *);
@@ -208,7 +211,7 @@ const struct gethdr_s *VRT_MkGethdr(const struct vrt_ctx *,enum gethdr_e,
     const char *);
 char *VRT_GetHdr(const struct vrt_ctx *, const struct gethdr_s *);
 void VRT_SetHdr(const struct vrt_ctx *, const struct gethdr_s *, const char *, ...);
-void VRT_handling(struct worker *, unsigned hand);
+void VRT_handling(const struct vrt_ctx *, unsigned hand);
 
 void VRT_hashdata(const struct vrt_ctx *, const char *str, ...);
 
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 509141f..88b96fc 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -707,7 +707,7 @@ struct worker;
 
 typedef int vcl_init_f(struct cli *);
 typedef void vcl_fini_f(struct cli *);
-typedef int vcl_func_f(const struct vrt_ctx *ctx, struct worker *);
+typedef int vcl_func_f(const struct vrt_ctx *ctx);
 """)
 
 
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index a5bd769..b84df6f 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -79,7 +79,7 @@ parse_error(struct vcc *tl)
 			Fb(tl, 1, ", 0\n");
 	}
 	Fb(tl, 1, ");\n");
-	Fb(tl, 1, "VRT_handling(wrk, VCL_RET_ERROR);\n");
+	Fb(tl, 1, "VRT_handling(ctx, VCL_RET_ERROR);\n");
 	Fb(tl, 1, "return(1);\n");
 }
 
@@ -314,7 +314,7 @@ parse_return(struct vcc *tl)
 #define VCL_RET_MAC(l, U, B)						\
 	do {								\
 		if (vcc_IdIs(tl->t, #l)) {				\
-			Fb(tl, 1, "VRT_handling(wrk, VCL_RET_" #U ");\n"); \
+			Fb(tl, 1, "VRT_handling(ctx, VCL_RET_" #U ");\n"); \
 			Fb(tl, 1, "return (1);\n");			\
 			vcc_ProcAction(tl->curproc, VCL_RET_##U, tl->t);\
 			retval = 1;					\
@@ -349,7 +349,7 @@ parse_purge(struct vcc *tl)
 {
 
 	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_purge(wrk, ctx, 0, 0);\n");
+	Fb(tl, 1, "VRT_purge(ctx, 0, 0);\n");
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index a3de471..b04236f 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -693,8 +693,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	for (i = 0; i < VCL_MET_MAX; i++) {
 		Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_f)\n");
 		Fc(tl, 1,
-		    "VGC_function_%s(const struct vrt_ctx *ctx,"
-		    " struct worker *wrk)\n",
+		    "VGC_function_%s(const struct vrt_ctx *ctx)\n",
 		    method_tab[i].name);
 		AZ(VSB_finish(tl->fm[i]));
 		Fc(tl, 1, "{\n");



From phk at varnish-cache.org  Wed May  1 12:20:34 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 01 May 2013 14:20:34 +0200
Subject: [master] c3f399c Remove unused function VRT_MkGethdr()
Message-ID: <E1UXW1S-0007Uq-Dk@project.varnish-software.com>

commit c3f399caf21adf3a56ad6d4bfaca43482a66cec4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 12:20:11 2013 +0000

    Remove unused function VRT_MkGethdr()

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 32764fc..f85669d 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -54,24 +54,6 @@ const void * const vrt_magic_string_unset = &vrt_magic_string_unset;
 
 /*--------------------------------------------------------------------*/
 
-const struct gethdr_s *
-VRT_MkGethdr(const struct vrt_ctx *ctx, enum gethdr_e where, const char *what)
-{
-	struct gethdr_s *retval;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	// XXX ?
-	retval = (void*)WS_Alloc(ctx->req->wrk->aws, sizeof *retval);
-	AN(retval);
-	retval->where = where;
-	retval->what = what;
-	return (retval);
-}
-
-
-/*--------------------------------------------------------------------*/
-
 void
 VRT_error(const struct vrt_ctx *ctx, unsigned code, const char *reason)
 {
diff --git a/include/vrt.h b/include/vrt.h
index 0657e4d..05b6cde 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -207,8 +207,6 @@ int VRT_rewrite(const char *, const char *);
 void VRT_error(const struct vrt_ctx *, unsigned, const char *);
 int VRT_switch_config(const char *);
 
-const struct gethdr_s *VRT_MkGethdr(const struct vrt_ctx *,enum gethdr_e,
-    const char *);
 char *VRT_GetHdr(const struct vrt_ctx *, const struct gethdr_s *);
 void VRT_SetHdr(const struct vrt_ctx *, const struct gethdr_s *, const char *, ...);
 void VRT_handling(const struct vrt_ctx *, unsigned hand);



From phk at varnish-cache.org  Wed May  1 12:36:44 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 01 May 2013 14:36:44 +0200
Subject: [master] 80b3ecc Make VRT_String() return a const char *
Message-ID: <E1UXWH6-0000qP-VI@project.varnish-software.com>

commit 80b3ecc00e764db3351786e75b871fe1535a1260
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 12:36:21 2013 +0000

    Make VRT_String() return a const char *

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ccbd56c..865b315 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -638,7 +638,7 @@ struct req {
 	double			t_resp;
 
 	struct http_conn	htc[1];
-	char			*client_identity;
+	const char		*client_identity;
 
 	/* HTTP request */
 	struct http		*http;
@@ -1029,7 +1029,11 @@ const char *VCL_Method_Name(unsigned);
 
 /* cache_vrt.c */
 
-char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
+/*
+ * These prototypes go here, because we do not want to pollute vrt.h
+ * with va_list.  VCC never generates direct calls to them.
+ */
+const char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
 char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 
 void ESI_Deliver(struct req *);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index f85669d..a298ad7 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -116,11 +116,13 @@ vrt_selecthttp(const struct vrt_ctx *ctx, enum gethdr_e where)
 		hp = ctx->http_obj;
 		break;
 	default:
-		INCOMPL();
+		WRONG("vrt_selecthttp 'where' invalid");
 	}
 	return (hp);
 }
 
+/*--------------------------------------------------------------------*/
+
 char *
 VRT_GetHdr(const struct vrt_ctx *ctx, const struct gethdr_s *hs)
 {
@@ -136,7 +138,7 @@ VRT_GetHdr(const struct vrt_ctx *ctx, const struct gethdr_s *hs)
 }
 
 /*--------------------------------------------------------------------
- * XXX: Optimize the single element case ?
+ * Collapse a STRING_LIST in the space provided, or return NULL
  */
 
 char *
@@ -163,10 +165,10 @@ VRT_StringList(char *d, unsigned dl, const char *p, va_list ap)
 }
 
 /*--------------------------------------------------------------------
- * XXX: Optimize the single element case ?
+ * Copy and merge a STRING_LIST into a workspace.
  */
 
-char *
+const char *
 VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 {
 	char *b, *e;
@@ -196,14 +198,14 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 }
 
 /*--------------------------------------------------------------------
- * Build a string on the request workspace
+ * Copy and merge a STRING_LIST on the current workspace
  */
 
 const char *
 VRT_CollectString(const struct vrt_ctx *ctx, const char *p, ...)
 {
 	va_list ap;
-	char *b;
+	const char *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
@@ -221,7 +223,7 @@ VRT_SetHdr(const struct vrt_ctx *ctx , const struct gethdr_s *hs,
 {
 	struct http *hp;
 	va_list ap;
-	char *b;
+	const char *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(hs);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 6b79f9c..7e9f133 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -50,7 +50,7 @@ static void
 vrt_do_string(const struct http *hp, int fld,
     const char *err, const char *p, va_list ap)
 {
-	char *b;
+	const char *b;
 
 	AN(hp);
 	b = VRT_String(hp->ws, NULL, p, ap);
@@ -165,7 +165,7 @@ void
 VRT_l_client_identity(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
-	char *b;
+	const char *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
@@ -248,7 +248,7 @@ void
 VRT_l_beresp_storage(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
-	char *b;
+	const char *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
diff --git a/include/vrt.h b/include/vrt.h
index 05b6cde..1394668 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -260,5 +260,4 @@ char *VRT_REAL_string(const struct vrt_ctx *, double);
 char *VRT_TIME_string(const struct vrt_ctx *, double);
 const char *VRT_BOOL_string(unsigned);
 const char *VRT_BACKEND_string(const struct director *d);
-
 const char *VRT_CollectString(const struct vrt_ctx *, const char *p, ...);
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 5583a4f..083962f 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -39,7 +39,7 @@ VCL_VOID __match_proto__(td_debug_panic)
 vmod_panic(const struct vrt_ctx *ctx, const char *str, ...)
 {
 	va_list ap;
-	char *b;
+	const char *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	va_start(ap, str);



From phk at varnish-cache.org  Wed May  1 13:14:02 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 01 May 2013 15:14:02 +0200
Subject: [master] 7fd3aa3 Don't let the form of the argument to hash_data()
 leak into the production of the hash-key.
Message-ID: <E1UXWrC-00078D-2n@project.varnish-software.com>

commit 7fd3aa3f8fdf97185af22ce9e71c338933ee1cc0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 13:13:16 2013 +0000

    Don't let the form of the argument to hash_data() leak into the
    production of the hash-key.
    
    Fixes	#1296

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index eab4907..3f4fb79 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -155,20 +155,15 @@ HSH_DeleteObjHead(struct dstat *ds, struct objhead *oh)
 }
 
 void
-HSH_AddString(struct req *req, const char *str)
+HSH_AddString(const struct req *req, const char *str)
 {
-	int l;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	if (str == NULL)
-		str = "";
-	l = strlen(str);
-
 	AN(req->sha256ctx);
-	SHA256_Update(req->sha256ctx, str, l);
-	SHA256_Update(req->sha256ctx, "#", 1);
-
-	VSLb(req->vsl, SLT_Hash, "%s", str);
+	if (str != NULL)
+		SHA256_Update(req->sha256ctx, str, strlen(str));
+	else
+		SHA256_Update(req->sha256ctx, &str, sizeof str);
 }
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index a298ad7..8d0ee46 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -257,7 +257,7 @@ VRT_handling(const struct vrt_ctx *ctx, unsigned hand)
 }
 
 /*--------------------------------------------------------------------
- * Add an element to the array/list of hash bits.
+ * Feed data into the hash calculation
  */
 
 void
@@ -275,7 +275,13 @@ VRT_hashdata(const struct vrt_ctx *ctx, const char *str, ...)
 		if (p == vrt_magic_string_end)
 			break;
 		HSH_AddString(ctx->req, p);
+		VSLb(ctx->vsl, SLT_Hash, "%s", str);
 	}
+	/*
+	 * Add a 'field-separator' to make it more difficult to
+	 * manipulate the hash.
+	 */
+	HSH_AddString(ctx->req, NULL);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 8a3d08b..3c36b4d 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -69,7 +69,7 @@ enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **,
 void HSH_Ref(struct objcore *o);
 void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);
-void HSH_AddString(struct req *, const char *str);
+void HSH_AddString(const struct req *, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
 void HSH_Purge(struct req *, struct objhead *, double ttl, double grace);
 void HSH_config(const char *h_arg);
diff --git a/bin/varnishtest/tests/r01296.vtc b/bin/varnishtest/tests/r01296.vtc
new file mode 100644
index 0000000..f9f8de4
--- /dev/null
+++ b/bin/varnishtest/tests/r01296.vtc
@@ -0,0 +1,30 @@
+varnishtest "hash key depends on argument form to hash_data()"
+
+server s1 {
+	rxreq
+	txresp -hdr "OK: yes"
+	rxreq
+	txresp -hdr "OK: no"
+} -start
+
+varnish v1 -vcl+backend {
+
+	sub vcl_hash {
+		if (req.http.foo == "1") {
+			hash_data("123");
+		} else {
+			hash_data("1" + req.http.foo + "3");
+		}
+		return (hash);
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foo: 1"
+	rxresp
+	expect resp.http.ok == "yes"
+
+	txreq -hdr "foo: 2"
+	rxresp
+	expect resp.http.ok == "yes"
+} -run



From tfheen at varnish-cache.org  Thu May  2 08:32:08 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 02 May 2013 10:32:08 +0200
Subject: [master] cc03355 Hint storage to Transient for hit-for-pass too
Message-ID: <E1UXovw-0003ll-C4@project.varnish-software.com>

commit cc03355cf252398644c9dd707cec2d9581782059
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 2 10:15:01 2013 +0200

    Hint storage to Transient for hit-for-pass too

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index cefc1ce..097f259 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -600,7 +600,7 @@ cnt_fetchbody(struct worker *wrk, struct req *req)
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
 	if (bo->exp.ttl < cache_param->shortlived ||
-	    req->objcore == NULL)
+	    pass == 1)
 		bo->storage_hint = TRANSIENT_STORAGE;
 
 	AZ(bo->stats);



From tfheen at varnish-cache.org  Thu May  2 08:44:58 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 02 May 2013 10:44:58 +0200
Subject: [3.0] 5683344 Hint storage to Transient for hit-for-pass too
Message-ID: <E1UXp8M-0005Rn-VW@project.varnish-software.com>

commit 5683344fc3facbb7c807b2590c21a00cbfed0d81
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 2 10:32:45 2013 +0200

    Hint storage to Transient for hit-for-pass too

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index ba16417..599d0e7 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -787,7 +787,7 @@ cnt_fetchbody(struct sess *sp)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (sp->wrk->exp.ttl < params->shortlived || sp->objcore == NULL)
+	if (sp->wrk->exp.ttl < params->shortlived || pass == 1)
 		sp->wrk->storage_hint = TRANSIENT_STORAGE;
 
 	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,



From tfheen at varnish-cache.org  Thu May  2 08:44:59 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 02 May 2013 10:44:59 +0200
Subject: [3.0] 428d430 Update version number for 3.0.4 RC
Message-ID: <E1UXp8N-0005Rs-0g@project.varnish-software.com>

commit 428d4305818c4580a94f93c8d277fd75f083bdfb
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 2 10:44:54 2013 +0200

    Update version number for 3.0.4 RC

diff --git a/configure.ac b/configure.ac
index a17e139..750c7f8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2011 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [3.0.3], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [3.0.4-rc1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/varnishapi.h)
 AM_CONFIG_HEADER(config.h)
 



From tfheen at varnish-cache.org  Thu May  2 10:24:40 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 02 May 2013 12:24:40 +0200
Subject: [3.0] f762f90 Fix up version number in spec file too
Message-ID: <E1UXqgq-0005be-2T@project.varnish-software.com>

commit f762f90ddecccb374384cc35940372640c1d82ae
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 2 12:24:35 2013 +0200

    Fix up version number in spec file too

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 23a2af9..6903454 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,7 +1,7 @@
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 3.0.3
-Release: 1%{?dist}
+Version: 3.0.4
+Release: 0.rc1%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/



From tfheen at varnish-cache.org  Thu May  2 12:04:34 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 02 May 2013 14:04:34 +0200
Subject: [master] f821a2f Document %D and %T for varnishncsa
Message-ID: <E1UXsFW-0008Ko-Ds@project.varnish-software.com>

commit f821a2f7a8172259128bab679ee25df5362267ee
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 2 14:04:22 2013 +0200

    Document %D and %T for varnishncsa

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 2937b26..8cd7b05 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -55,6 +55,9 @@ The following options are available:
    	         In CLF format, i.e. a '-' rather than a 0 when no
    	         bytes are sent.
 
+	      %D
+	         Time taken to serve the request, in microseconds.
+
 	      %H 
 	         The request protocol. Defaults to HTTP/1.0 if not
                  known.
@@ -95,6 +98,9 @@ The following options are available:
 		 specified by X.  The time specification format is the
 		 same as for strftime(3).
 
+	      %T
+	         Time taken to serve the request, in seconds.
+
 	      %U
 	         The request URL without any query string. Defaults to
                  '-' if not known.



From phk at varnish-cache.org  Fri May  3 07:36:07 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 03 May 2013 09:36:07 +0200
Subject: [master] 8410629 Have calls to VCL methods provide bo,
 rather than derive it magically from req.
Message-ID: <E1UYAXH-0007xE-2V@project.varnish-software.com>

commit 8410629be00ab5024926a441b6c843038a90ecc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 07:35:40 2013 +0000

    Have calls to VCL methods provide bo, rather than derive it magically
    from req.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 097f259..9eef389 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -416,7 +416,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 		AZ(bo->do_esi);
 		AZ(bo->do_pass);
 
-		VCL_backend_response_method(req->vcl, wrk, req, NULL,
+		VCL_backend_response_method(bo->vcl, wrk, req, bo,
 		    bo->beresp->ws);
 
 		if (bo->do_pass)
@@ -924,7 +924,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 		http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
 	}
 
-	VCL_backend_fetch_method(req->vcl, wrk, req, NULL, bo->bereq->ws);
+	VCL_backend_fetch_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
 
 	if (wrk->handling == VCL_RET_FETCH) {
@@ -989,8 +989,8 @@ cnt_pass(struct worker *wrk, struct req *req)
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(req, HTTPH_R_PASS);
 
-	VCL_backend_fetch_method(req->vcl, wrk, req, NULL, bo->bereq->ws);
-	VCL_pass_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_backend_fetch_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
+	VCL_pass_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
 
 	if (wrk->handling == VCL_RET_ERROR) {
 		http_Teardown(bo->bereq);
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index c7f7002..96288b6 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -391,7 +391,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	memset(&ctx, 0, sizeof ctx);
 	ctx.magic = VRT_CTX_MAGIC;
 	if (req != NULL) {
-		AZ(bo);
+		// AZ(bo);
 		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 		CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
 		vsl = req->vsl;
@@ -403,16 +403,6 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		if (req->obj)
 			ctx.http_obj = req->obj->http;
 	}
-	if (method == VCL_MET_BACKEND_FETCH ||
-	    method == VCL_MET_PASS ||
-	    method == VCL_MET_MISS ||
-	    method == VCL_MET_PIPE ||
-	    method == VCL_MET_BACKEND_RESPONSE) {
-		/* XXX: temporary workaround */
-		AN(req);
-		bo = req->busyobj;
-		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	}
 	if (bo != NULL) {
 		// AZ(req);
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);



From phk at varnish-cache.org  Fri May  3 07:44:54 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 03 May 2013 09:44:54 +0200
Subject: [master] 10e1416 Remove req argument from vcl_backend_fetch{}
Message-ID: <E1UYAfm-0000NM-K6@project.varnish-software.com>

commit 10e141669a755387682f0b9a7d133f6ad1f3f1f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 07:44:42 2013 +0000

    Remove req argument from vcl_backend_fetch{}

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9eef389..df19c30 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -924,7 +924,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 		http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
 	}
 
-	VCL_backend_fetch_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
+	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
 
 	if (wrk->handling == VCL_RET_FETCH) {
@@ -989,7 +989,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(req, HTTPH_R_PASS);
 
-	VCL_backend_fetch_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
+	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 	VCL_pass_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
 
 	if (wrk->handling == VCL_RET_ERROR) {
diff --git a/bin/varnishtest/tests/r00781.vtc b/bin/varnishtest/tests/r00781.vtc
index d58bf29..120c363 100644
--- a/bin/varnishtest/tests/r00781.vtc
+++ b/bin/varnishtest/tests/r00781.vtc
@@ -10,7 +10,7 @@ varnish v1 -vcl+backend {
 		set req.url = req.http.foo;
 	}
 	sub vcl_backend_fetch {
-		set bereq.url = req.http.foo;
+		set bereq.url = bereq.http.foo;
 	}
 } -start
 



From phk at varnish-cache.org  Fri May  3 12:14:41 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 03 May 2013 14:14:41 +0200
Subject: [master] 1067ad1 Change req.* to bereq.* in vcl_backend_response{}
 where it makes no difference in any way.
Message-ID: <E1UYEsr-0001bv-3W@project.varnish-software.com>

commit 1067ad129b6f00cbd705586e0ac4159ed1edc746
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 12:14:12 2013 +0000

    Change req.* to bereq.* in vcl_backend_response{} where it makes no
    difference in any way.

diff --git a/bin/varnishtest/tests/e00019.vtc b/bin/varnishtest/tests/e00019.vtc
index 5731647..6a4b402 100644
--- a/bin/varnishtest/tests/e00019.vtc
+++ b/bin/varnishtest/tests/e00019.vtc
@@ -30,7 +30,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url == "bar") {
+		if (bereq.url == "bar") {
 			set beresp.do_esi = true;
 		}
 	}
diff --git a/bin/varnishtest/tests/e00024.vtc b/bin/varnishtest/tests/e00024.vtc
index ea9f6c4..98b40cd 100644
--- a/bin/varnishtest/tests/e00024.vtc
+++ b/bin/varnishtest/tests/e00024.vtc
@@ -61,7 +61,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url == "/") {
+		if (bereq.url == "/") {
 			set beresp.do_esi = true;
 		}
 	}
diff --git a/bin/varnishtest/tests/e00026.vtc b/bin/varnishtest/tests/e00026.vtc
index 6d2e50c..d4c3a39 100644
--- a/bin/varnishtest/tests/e00026.vtc
+++ b/bin/varnishtest/tests/e00026.vtc
@@ -29,7 +29,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url != "/foo") {
+		if (bereq.url != "/foo") {
 			set beresp.do_esi = true;
 		}
 	}
diff --git a/bin/varnishtest/tests/r00345.vtc b/bin/varnishtest/tests/r00345.vtc
index 772c60f..b6c819a 100644
--- a/bin/varnishtest/tests/r00345.vtc
+++ b/bin/varnishtest/tests/r00345.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 -arg "-p debug=+workspace" -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url == "/") {
+		if (bereq.url == "/") {
 			set beresp.do_esi = true;
 		}
 	}
diff --git a/bin/varnishtest/tests/r00386.vtc b/bin/varnishtest/tests/r00386.vtc
index 3edcb35..3d43b5a 100644
--- a/bin/varnishtest/tests/r00386.vtc
+++ b/bin/varnishtest/tests/r00386.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 -arg "-p debug=+workspace" -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url == "/") {
+		if (bereq.url == "/") {
 			set beresp.do_esi = true;
 		}
 	}
diff --git a/bin/varnishtest/tests/r00861.vtc b/bin/varnishtest/tests/r00861.vtc
index ff6b042..45a624d 100644
--- a/bin/varnishtest/tests/r00861.vtc
+++ b/bin/varnishtest/tests/r00861.vtc
@@ -24,7 +24,7 @@ server s1 {
 varnish v1 \
 	-vcl+backend {
 		sub vcl_backend_response {
-			if (req.url == "/1" || req.url == "/2") {
+			if (bereq.url == "/1" || bereq.url == "/2") {
 				set beresp.do_esi = true;
 				set beresp.do_gzip = true;
 			}
diff --git a/bin/varnishtest/tests/r01029.vtc b/bin/varnishtest/tests/r01029.vtc
index e388043..745fb68 100644
--- a/bin/varnishtest/tests/r01029.vtc
+++ b/bin/varnishtest/tests/r01029.vtc
@@ -14,7 +14,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_esi = true;
-		if (req.url == "/foo") {
+		if (bereq.url == "/foo") {
 			set beresp.ttl = 0s;
 		} else {
 			set beresp.ttl = 10m;
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index 7ebd036..a746852 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 		return (lookup);
 	}
 	sub vcl_backend_response {
-		set beresp.http.url = req.url;
+		set beresp.http.url = bereq.url;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r01109.vtc b/bin/varnishtest/tests/r01109.vtc
index 3b0fd23..46ef404 100644
--- a/bin/varnishtest/tests/r01109.vtc
+++ b/bin/varnishtest/tests/r01109.vtc
@@ -23,7 +23,7 @@ server s1 {
 
 varnish v1 -arg "-pfetch_chunksize=4k" -arg "-pgzip_level=0" -arg "-pthread_pool_stack=131072" -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url ~ "/test") {
+		if (bereq.url ~ "/test") {
 			set beresp.do_esi = true;
 		}
 		set beresp.do_gzip = true;
diff --git a/bin/varnishtest/tests/r01113.vtc b/bin/varnishtest/tests/r01113.vtc
index dc71417..3df7f91 100644
--- a/bin/varnishtest/tests/r01113.vtc
+++ b/bin/varnishtest/tests/r01113.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 		}
 	}
 	sub vcl_backend_response {
-		if (req.url == "/fetch") {
+		if (bereq.url == "/fetch") {
 			return(restart);
 		}
 	}
diff --git a/bin/varnishtest/tests/r01123.vtc b/bin/varnishtest/tests/r01123.vtc
index a979fe6..87d1e80 100644
--- a/bin/varnishtest/tests/r01123.vtc
+++ b/bin/varnishtest/tests/r01123.vtc
@@ -10,7 +10,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
-		if (req.url == "/") {
+		if (bereq.url == "/") {
 			set beresp.do_esi = true;
 		}
 		set beresp.do_gzip = true;
diff --git a/bin/varnishtest/tests/v00033.vtc b/bin/varnishtest/tests/v00033.vtc
index 4494c32..d3eecc3 100644
--- a/bin/varnishtest/tests/v00033.vtc
+++ b/bin/varnishtest/tests/v00033.vtc
@@ -14,7 +14,7 @@ varnish v1 -vcl+backend {
 		set beresp.http.foo =
 		    storage.nowhere.free_space +
 		    1 B + 1 KB + 1 MB + 1GB + 1TB;
-		if (req.url == "/foo") {
+		if (bereq.url == "/foo") {
 			set beresp.storage = "Transient";
 		}
 	}



From phk at varnish-cache.org  Fri May  3 12:51:35 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 03 May 2013 14:51:35 +0200
Subject: [master] 30d19c8 Eliminate req.* from vcl_backend_response{}
Message-ID: <E1UYFSZ-0005cS-2e@project.varnish-software.com>

commit 30d19c8995a5cdef3663c68f9db7e053f6a45920
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 12:51:15 2013 +0000

    Eliminate req.* from vcl_backend_response{}

diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc
index 3d3e146..7126f6c 100644
--- a/bin/varnishtest/tests/e00015.vtc
+++ b/bin/varnishtest/tests/e00015.vtc
@@ -23,10 +23,12 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_backend_response {
+	sub vcl_recv {
 	    	if(req.url == "/") {
  	    	   set req.esi = false;
 		}
+	}
+	sub vcl_backend_response {
 		set beresp.do_esi = true;
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00956.vtc b/bin/varnishtest/tests/r00956.vtc
index 6c53905..bcbb938 100644
--- a/bin/varnishtest/tests/r00956.vtc
+++ b/bin/varnishtest/tests/r00956.vtc
@@ -45,5 +45,6 @@ client c1 {
 	expect resp.bodylen == 40
 	# XXX: should be: < 5
 	expect resp.http.foo != 7.000
+	expect resp.http.foo != 6.000
 	expect resp.http.bar == 7.000
 } -run



From phk at varnish-cache.org  Sat May  4 12:45:28 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 04 May 2013 14:45:28 +0200
Subject: [master] d2d9ddb Don't much about with req.* in vcl_backend_response{}
Message-ID: <E1UYbqC-0007m6-SW@project.varnish-software.com>

commit d2d9ddb4e9fecf43330c708d86162ad126330220
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 4 12:33:55 2013 +0000

    Don't much about with req.* in vcl_backend_response{}

diff --git a/bin/varnishtest/tests/e00016.vtc b/bin/varnishtest/tests/e00016.vtc
index 44f7d6a..3871915 100644
--- a/bin/varnishtest/tests/e00016.vtc
+++ b/bin/varnishtest/tests/e00016.vtc
@@ -28,11 +28,13 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+	sub vcl_deliver {
  	    	set req.esi = true;	
 	    	if(req.url == "/body") {
 		   set req.esi = false;
 		} 
-		set beresp.do_esi = true;
 	}
 } -start
 



From phk at varnish-cache.org  Sat May  4 12:45:28 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 04 May 2013 14:45:28 +0200
Subject: [master] 5ddc4a3 Restructure slightly to fit v4 realities.
Message-ID: <E1UYbqC-0007mD-Tz@project.varnish-software.com>

commit 5ddc4a3aa2ce508954ae16c248eccbc869697fdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 4 12:37:00 2013 +0000

    Restructure slightly to fit v4 realities.

diff --git a/bin/varnishtest/tests/r00412.vtc b/bin/varnishtest/tests/r00412.vtc
index 0e49607..4cb988b 100644
--- a/bin/varnishtest/tests/r00412.vtc
+++ b/bin/varnishtest/tests/r00412.vtc
@@ -15,13 +15,12 @@ varnish v1 -vcl+backend {
 		if (beresp.status == 303) {
 			set beresp.ttl = 60 s;
 			set beresp.http.X-Magic-Redirect = "1";
-			set req.url = beresp.http.Location;
-			return (restart);
+			return (deliver);
 		}
 	}
-	sub vcl_lookup {
-		if (obj.http.X-Magic-Redirect == "1") {
-			set req.url = obj.http.Location;
+	sub vcl_deliver {
+		if (resp.http.X-Magic-Redirect == "1") {
+			set req.url = resp.http.Location;
 			return (restart);
 		}
 	}



From phk at varnish-cache.org  Sat May  4 12:45:28 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 04 May 2013 14:45:28 +0200
Subject: [master] 467f45d Don't mess with req.* in vcl_backend_response{}
Message-ID: <E1UYbqC-0007mL-Vj@project.varnish-software.com>

commit 467f45d6cb24d0723fffdd5c79de1dd3c1605717
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 4 12:45:00 2013 +0000

    Don't mess with req.* in vcl_backend_response{}

diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc
index dd3a303..26f3384 100644
--- a/bin/varnishtest/tests/v00025.vtc
+++ b/bin/varnishtest/tests/v00025.vtc
@@ -17,10 +17,7 @@ sub vcl_deliver {
 }
 
 sub vcl_backend_response {
-	if (req.esi) {
-		set beresp.http.esi = "yes";
-	}
-	set beresp.http.backend = req.backend;
+	set beresp.http.backend = bereq.backend;
 	if (beresp.ttl > 3 s) {
 		set beresp.http.ttl = "long";
 	} else {



From phk at varnish-cache.org  Mon May  6 08:18:54 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 10:18:54 +0200
Subject: [master] 282559d Don't use req.restarts in vcl_backend_response{},
 it's a bad example and does nothing to make the tests objective any
 clearer.
Message-ID: <E1UZGdK-0006Gf-E6@project.varnish-software.com>

commit 282559d9f8b34055cdc8c8db47f4c8542e10405c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 08:18:16 2013 +0000

    Don't use req.restarts in vcl_backend_response{}, it's a bad example
    and does nothing to make the tests objective any clearer.

diff --git a/bin/varnishtest/tests/r01091.vtc b/bin/varnishtest/tests/r01091.vtc
index dbf3820..f50274f 100644
--- a/bin/varnishtest/tests/r01091.vtc
+++ b/bin/varnishtest/tests/r01091.vtc
@@ -3,9 +3,6 @@ varnishtest "Test fallback director with saint mode"
 server s1 {
         rxreq
         txresp -hdr "Foo: 1"
-        accept
-        rxreq
-        txresp -hdr "Foo: 1"
 } -start
 
 server s2 {
@@ -26,11 +23,18 @@ varnish v1 -vcl+backend {
                 set req.backend = f1.backend();
         }
         sub vcl_backend_response {
-                if(req.restarts < 1) {
+		if (beresp.http.foo == "1") {
+                        set beresp.uncacheable = true;
                         set beresp.saintmode = 1h;
-                        return(restart);
+                        set beresp.status = 606;
+                        return(deliver);
                 }
         }
+	sub vcl_deliver {
+		if (resp.status == 606) {
+                        return(restart);
+		}
+	}
 } -start
 
 client c1 {



From phk at varnish-cache.org  Mon May  6 08:23:45 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 10:23:45 +0200
Subject: [master] cb93291 Don't use req.http to signal data for the test-case
 checks.
Message-ID: <E1UZGi1-0006g4-TW@project.varnish-software.com>

commit cb932911a63ea1a91d1aad73618f2cc89d1cd3b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 08:23:25 2013 +0000

    Don't use req.http to signal data for the test-case checks.

diff --git a/bin/varnishtest/tests/r00956.vtc b/bin/varnishtest/tests/r00956.vtc
index bcbb938..c8196d1 100644
--- a/bin/varnishtest/tests/r00956.vtc
+++ b/bin/varnishtest/tests/r00956.vtc
@@ -8,8 +8,8 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.ttl = 10s;
-		set req.http.foo = beresp.ttl;
-		set req.http.bar = "xxx";
+		set beresp.http.fooB = beresp.ttl;
+		set beresp.http.barB = "xxx";
 	}
 	sub vcl_lookup {
 		set req.http.foo = obj.ttl;
@@ -28,15 +28,18 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.bodylen == 40
-	expect resp.http.foo == 10.000
-	expect resp.http.bar == "xxx"
+	expect resp.http.fooB == 10.000
+	expect resp.http.barB == "xxx"
 
 	delay 2
 	txreq
 	rxresp
 	expect resp.bodylen == 40
 	# XXX: should be: < 8
+	expect resp.http.fooB == 10.000
+	expect resp.http.barB == "xxx"
 	expect resp.http.foo != 10.000
+	expect resp.http.foo != 9.000
 	expect resp.http.bar == 7.000
 
 	delay 2
@@ -44,6 +47,8 @@ client c1 {
 	rxresp
 	expect resp.bodylen == 40
 	# XXX: should be: < 5
+	expect resp.http.fooB == 10.000
+	expect resp.http.barB == "xxx"
 	expect resp.http.foo != 7.000
 	expect resp.http.foo != 6.000
 	expect resp.http.bar == 7.000



From phk at varnish-cache.org  Mon May  6 08:30:21 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 10:30:21 +0200
Subject: [master] 0e967d6 Don't return error from vcl_backend_response{}
Message-ID: <E1UZGoP-0007Hu-5p@project.varnish-software.com>

commit 0e967d6a97c62844d70744b09fa423a792ea3e17
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 08:30:06 2013 +0000

    Don't return error from vcl_backend_response{}

diff --git a/bin/varnishtest/tests/r00915.vtc b/bin/varnishtest/tests/r00915.vtc
index c7aacc8..bcb1510 100644
--- a/bin/varnishtest/tests/r00915.vtc
+++ b/bin/varnishtest/tests/r00915.vtc
@@ -13,7 +13,10 @@ varnish v1 \
 	-vcl+backend { 
 
 	sub vcl_backend_response {
-		error(751);
+		set beresp.uncacheable = false;
+		set beresp.ttl = 0s;
+		set beresp.status = 751;
+		return (deliver);
 	}
 } -start 
 



From phk at varnish-cache.org  Mon May  6 09:44:00 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 11:44:00 +0200
Subject: [master] 6c4f5f8 Don't use error in vcl_backend_response{}
Message-ID: <E1UZHxg-0005fy-QL@project.varnish-software.com>

commit 6c4f5f874d5104b99dc0ae336c4181f9f2f8447a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 08:48:57 2013 +0000

    Don't use error in vcl_backend_response{}

diff --git a/bin/varnishtest/tests/b00018.vtc b/bin/varnishtest/tests/b00018.vtc
index 06f4ed1..02d4a93 100644
--- a/bin/varnishtest/tests/b00018.vtc
+++ b/bin/varnishtest/tests/b00018.vtc
@@ -1,4 +1,4 @@
-varnishtest "Check that error in vcl_backend_response works"
+varnishtest "Check that synth response in vcl_backend_response works"
 
 server s1 {
 	rxreq 
@@ -8,7 +8,11 @@ server s1 {
 varnish v1 -vcl+backend { 
 	sub vcl_backend_response {
 	    set beresp.http.Foo = "bar";
-	    error 523 "not ok";
+	    set beresp.status = 523;
+	    set beresp.response = "not ok";
+	    set beresp.uncacheable = true;
+	    set beresp.ttl = 0s;
+	    return (deliver);
 	}
 } -start
 
@@ -18,4 +22,6 @@ client c1 {
 	expect resp.status == 523
 } -run
 
+delay 10
+
 varnish v1 -expect n_object == 0



From phk at varnish-cache.org  Mon May  6 09:44:00 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 11:44:00 +0200
Subject: [master] 1392c94 Don't restart from vcl_backend_response{}
Message-ID: <E1UZHxg-0005g3-RZ@project.varnish-software.com>

commit 1392c941584563ec2591184b9514ff8d38bdb412
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 09:06:39 2013 +0000

    Don't restart from vcl_backend_response{}

diff --git a/bin/varnishtest/tests/b00019.vtc b/bin/varnishtest/tests/b00019.vtc
index d393ab1..91c0e68 100644
--- a/bin/varnishtest/tests/b00019.vtc
+++ b/bin/varnishtest/tests/b00019.vtc
@@ -21,7 +21,7 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_backend_response {
+	sub vcl_lookup {
 	       return (restart);
 	}
 



From phk at varnish-cache.org  Mon May  6 09:44:00 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 11:44:00 +0200
Subject: [master] c68e22e Don't do restart from vcl_backend_fetch{}
Message-ID: <E1UZHxg-0005g9-Sc@project.varnish-software.com>

commit c68e22ec4e810e8bb59f539d04aebcdccc1c56f5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 09:43:48 2013 +0000

    Don't do restart from vcl_backend_fetch{}

diff --git a/bin/varnishtest/tests/c00009.vtc b/bin/varnishtest/tests/c00009.vtc
index e4705bb..7e881ab 100644
--- a/bin/varnishtest/tests/c00009.vtc
+++ b/bin/varnishtest/tests/c00009.vtc
@@ -23,6 +23,13 @@ varnish v1 -vcl+backend {
 
 	sub vcl_backend_response {
 		if (beresp.status != 200) {
+			set beresp.uncacheable = true;
+			set beresp.ttl = 0s;
+			return (deliver);
+		}
+	}
+	sub vcl_deliver {
+		if (resp.status != 200) {
 			return (restart);
 		}
 	}
diff --git a/bin/varnishtest/tests/c00029.vtc b/bin/varnishtest/tests/c00029.vtc
index 07111ba..985d595 100644
--- a/bin/varnishtest/tests/c00029.vtc
+++ b/bin/varnishtest/tests/c00029.vtc
@@ -40,12 +40,17 @@ varnish v1 -arg "-p saintmode_threshold=2" -vcl+backend {
 	sub vcl_backend_response {
 		if (beresp.http.X-Saint == "yes") {
 			set beresp.saintmode = 20s;
-			return(restart);
+			set beresp.uncacheable = true;
+			set beresp.ttl = 0s;
+			return(deliver);
 		}
 		set beresp.grace = 1h;
 		set beresp.ttl = 1s;
 	}
 	sub vcl_deliver {
+		if (resp.http.X-saint == "yes") {
+			return (restart);
+		}
 		set resp.http.X-Restarts = req.restarts;
 	}
  } -start
diff --git a/bin/varnishtest/tests/c00030.vtc b/bin/varnishtest/tests/c00030.vtc
index 9cfec57..51c5042 100644
--- a/bin/varnishtest/tests/c00030.vtc
+++ b/bin/varnishtest/tests/c00030.vtc
@@ -47,12 +47,17 @@ varnish v1 -arg "-p saintmode_threshold=10" -vcl {
 	sub vcl_backend_response {
 		if (beresp.http.X-Saint == "yes") {
 			set beresp.saintmode = 20s;
-			return (restart);
+			set beresp.uncacheable = true;
+			set beresp.ttl = 0s;
+			return(deliver);
 		}
 		set beresp.grace = 1h;
 		set beresp.ttl = 1s;
 	}
 	sub vcl_deliver {
+		if (resp.http.X-saint == "yes") {
+			return (restart);
+		}
 		set resp.http.X-Restarts = req.restarts;
 	}
  } -start
diff --git a/bin/varnishtest/tests/r01113.vtc b/bin/varnishtest/tests/r01113.vtc
index 3df7f91..debeb90 100644
--- a/bin/varnishtest/tests/r01113.vtc
+++ b/bin/varnishtest/tests/r01113.vtc
@@ -20,11 +20,6 @@ varnish v1 -vcl+backend {
 			return(restart);
 		}
 	}
-	sub vcl_backend_response {
-		if (bereq.url == "/fetch") {
-			return(restart);
-		}
-	}
 	sub vcl_deliver {
 		if (req.url == "/deliver") {
 			return(restart);
@@ -51,11 +46,6 @@ client c1 {
 	expect resp.status == 503
 } -run
 client c1 {
-	txreq -url /fetch
-	rxresp
-	expect resp.status == 503
-} -run
-client c1 {
 	txreq -url /deliver
 	rxresp
 	expect resp.status == 503
diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc
index 35fc3d2..3ed6e98 100644
--- a/bin/varnishtest/tests/s00003.vtc
+++ b/bin/varnishtest/tests/s00003.vtc
@@ -22,10 +22,18 @@ varnish v1 -vcl+backend {
 		if (beresp.http.foo == "2")
 		{
 			set beresp.saintmode = 2s;
-			return (restart);
+			set beresp.status = 606;
+			set beresp.uncacheable = true;
+			set beresp.ttl = 0s;
+			return (deliver);
 		}
 		return(deliver);
 	}
+	sub vcl_deliver {
+		if (resp.status == 606) {
+			return (restart);
+		}
+	}
 } -start
 
 client c1 {



From phk at varnish-cache.org  Mon May  6 11:05:03 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 13:05:03 +0200
Subject: [master] 344c017 Fix a testcase to not restart from
 vcl_backend_reponse{} and fix the bug (category: "How did _that_ ever work")
 this uncovered.
Message-ID: <E1UZJE7-0004LE-Qd@project.varnish-software.com>

commit 344c0175b8b42aa32b416bcb648bab22825b7fcf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 11:04:16 2013 +0000

    Fix a testcase to not restart from vcl_backend_reponse{} and
    fix the bug (category: "How did _that_ ever work") this uncovered.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index df19c30..b68f2ea 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -167,7 +167,8 @@ cnt_prepresp(struct worker *wrk, struct req *req)
 			break;
 		if (bo != NULL) {
 			AN(bo->do_stream);
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+			assert(req->obj == bo->fetch_obj);
+			req->obj = NULL;
 			VBO_DerefBusyObj(wrk, &req->busyobj);
 		} else {
 			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index 5bc57f2..ae98b48 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -15,13 +15,15 @@ varnish v1 -vcl+backend {
 		C{ VRT_CacheReqBody(ctx, 1000); }C
 		return (pass);
 	}
-	sub vcl_backend_response {
-		if (beresp.http.foo == "BAR") {
+	sub vcl_deliver {
+		if (resp.http.foo == "BAR") {
 			return (restart);
 		}
 	}
 } -start
 
+varnish v1 -cliok "param.set debug +syncvsl"
+
 client c1 {
 	txreq -body "FOO"
 	rxresp



From phk at varnish-cache.org  Mon May  6 11:05:03 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 13:05:03 +0200
Subject: [master] e9cb3ae Remove this testcase, it is no longer relevant.
Message-ID: <E1UZJE7-0004L9-Ou@project.varnish-software.com>

commit e9cb3ae22ef6a0bb030891d15447d81afd5a512a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 09:45:44 2013 +0000

    Remove this testcase, it is no longer relevant.

diff --git a/bin/varnishtest/tests/r01253.vtc b/bin/varnishtest/tests/r01253.vtc
deleted file mode 100644
index ce348ae..0000000
--- a/bin/varnishtest/tests/r01253.vtc
+++ /dev/null
@@ -1,26 +0,0 @@
-varnishtest "restarting a pass in vcl_backend_response should not panic"
-
-server s1 {
-	rxreq
-	txresp -status 304 -body "1234"
-	accept
-	rxreq
-	txresp -status 200 -body "56"
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_recv {
-		return (pass);
-	}
-	sub vcl_backend_response {
-		if (beresp.status != 200) {
-			return (restart);
-		}
-	}
-} -start
-
-client c1 {
-	txreq 
-	rxresp
-	expect resp.bodylen == 2
-} -run



From phk at varnish-cache.org  Mon May  6 11:17:40 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 13:17:40 +0200
Subject: [master] 0e8db99 Eliminate error and restart as possible returns from
 vcl_backend_response{}
Message-ID: <E1UZJQK-0005ax-3Q@project.varnish-software.com>

commit 0e8db99b6f3070be9a605846a6dbb4ac01425630
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 11:16:54 2013 +0000

    Eliminate error and restart as possible returns from vcl_backend_response{}
    
    We may want a better way to return synthetic objects from
    vcl_backend_reponse{}, it looks a bit clumsy in the vtc's so far.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index b68f2ea..77a4688 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -417,7 +417,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 		AZ(bo->do_esi);
 		AZ(bo->do_pass);
 
-		VCL_backend_response_method(bo->vcl, wrk, req, bo,
+		VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
 		    bo->beresp->ws);
 
 		if (bo->do_pass)
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 88b96fc..3dc15e2 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -85,7 +85,7 @@ returns =(
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
 	('lookup',		"C", ('error', 'restart', 'pass', 'deliver',)),
 	('backend_fetch',	"B", ('error', 'fetch', 'pass',)),
-	('backend_response',	"B", ('error', 'restart', 'deliver',)),
+	('backend_response',	"B", ('deliver',)),
 	('deliver',		"C", ('restart', 'deliver',)),
 	('error',		"C", ('restart', 'deliver',)),
 	('init',		"", ('ok',)),



From phk at varnish-cache.org  Mon May  6 11:25:04 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 06 May 2013 13:25:04 +0200
Subject: [master] 2a49a1f Remove 'error' as a possible return from
 vcl_backend_fetch{}
Message-ID: <E1UZJXU-00063m-HG@project.varnish-software.com>

commit 2a49a1fa1bfeba3fd31987e939bf63312d2bc683
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 6 11:24:42 2013 +0000

    Remove 'error' as a possible return from vcl_backend_fetch{}

diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 3dc15e2..0f6bfe8 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -84,7 +84,7 @@ returns =(
 	('hash',		"C", ('hash',)),
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
 	('lookup',		"C", ('error', 'restart', 'pass', 'deliver',)),
-	('backend_fetch',	"B", ('error', 'fetch', 'pass',)),
+	('backend_fetch',	"B", ('fetch', 'pass',)),
 	('backend_response',	"B", ('deliver',)),
 	('deliver',		"C", ('restart', 'deliver',)),
 	('error',		"C", ('restart', 'deliver',)),



From perbu at varnish-cache.org  Mon May  6 14:19:53 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 06 May 2013 16:19:53 +0200
Subject: [master] d596b0c Documentation error wrt gzip Fixes #1281
Message-ID: <E1UZMGf-00038A-DL@project.varnish-software.com>

commit d596b0c048cb43676667980002f9289dc95ba245
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon May 6 16:15:47 2013 +0200

    Documentation error wrt gzip
    Fixes #1281

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 5f66f87..01534dc 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -458,8 +458,8 @@ const struct parspec mgt_parspec[] = {
 		0,
 		"on", "bool" },
 	{ "http_gzip_support", tweak_bool, &mgt_param.http_gzip_support, 0, 0,
-		"Enable gzip support. When enabled Varnish will compress "
-		"uncompressed objects before they are stored in the cache. "
+		"Enable gzip support. When enabled Varnish request compressed "
+		"objects from the backend and store them compressed. "
 		"If a client does not support gzip encoding Varnish will "
 		"uncompress compressed objects on demand. Varnish will also "
 		"rewrite the Accept-Encoding header of clients indicating "
@@ -469,7 +469,7 @@ const struct parspec mgt_parspec[] = {
 		"Accept-Encoding header removed. For more information on how "
 		"gzip is implemented please see the chapter on gzip in the "
 		"Varnish reference.",
-		EXPERIMENTAL,
+		0,
 		"on", "bool" },
 	{ "gzip_level", tweak_uint, &mgt_param.gzip_level, 0, 9,
 		"Gzip compression level: 0=debug, 1=fast, 9=best",



From perbu at varnish-cache.org  Mon May  6 14:19:53 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 06 May 2013 16:19:53 +0200
Subject: [master] 6ed1b68 Make the gzip docs a bit clearer
Message-ID: <E1UZMGf-00038G-Er@project.varnish-software.com>

commit 6ed1b68bb3c8f127d8df7b7377d80579af04adba
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon May 6 16:16:20 2013 +0200

    Make the gzip docs a bit clearer

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index b5f5dec..01a52ad 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -716,8 +716,10 @@ beresp.do_esi
   false. Set it to true to parse the object for ESI directives. Will
   only be honored if req.esi is true.
 
-beresp.do_gzip
-  Boolean. Gzip the object before storing it. Defaults to false.
+beresp.do_gzip 
+  Boolean. Gzip the object before storing it. Defaults to false. When
+  http_gzip_support is on Varnish will request already compressed
+  content from the backend and as such compression in Varnish is not needed.
 
 beresp.do_gunzip
   Boolean. Unzip the object before storing it in the cache.  Defaults



From phk at varnish-cache.org  Tue May  7 06:50:43 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 07 May 2013 08:50:43 +0200
Subject: [master] 3592eed Aim the axe at vcl_pass{}
Message-ID: <E1UZbjX-0007L4-Pq@project.varnish-software.com>

commit 3592eeda903c0769ccab6fb2f4a729ad44827388
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 7 06:50:28 2013 +0000

    Aim the axe at vcl_pass{}

diff --git a/bin/varnishtest/tests/b00035.vtc b/bin/varnishtest/tests/b00035.vtc
index 9d49f4f..dbe8a6f 100644
--- a/bin/varnishtest/tests/b00035.vtc
+++ b/bin/varnishtest/tests/b00035.vtc
@@ -14,7 +14,7 @@ varnish v1 -vcl+backend {
 		return (pass);
 	}
 
-	sub vcl_pass {
+	sub vcl_backend_fetch {
 		set bereq.method = bereq.request + "C";
 		set bereq.request = bereq.method + "D";
 	}
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index f8228a9..865e16a 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -7,8 +7,9 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		if (req.http.X-Banned == "check") { remove req.http.X-Banned; }
-		elseif (req.restarts == 0) {
+		if (req.http.X-Banned == "check") {
+			remove req.http.X-Banned;
+		} elseif (req.restarts == 0) {
 		        set req.http.X-Banned = "check";
 			if (req.http.x-pass) {
 				return (pass);
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 2553638..a89ecb6 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -14,7 +14,7 @@ varnish v1 -vcl {
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_miss { error 100 req.url ; }
-	sub vcl_pass { error 100 "the butter please" ; }
+	sub vcl_lookup { error 100 "the butter please" ; }
 	sub vcl_backend_response { error beresp.status req.url; }
 }
 



From phk at varnish-cache.org  Tue May  7 08:13:41 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 07 May 2013 10:13:41 +0200
Subject: [master] 298475e Make the default return from recv and hash methods
 point to the next method we'll end up in.
Message-ID: <E1UZd1p-0004qB-5F@project.varnish-software.com>

commit 298475ebb70ba6ab4a25116e2292bc33d77fd51c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 7 08:13:17 2013 +0000

    Make the default return from recv and hash methods point to the
    next method we'll end up in.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 77a4688..90a4a1a 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -1172,7 +1172,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 	req->sha256ctx = &sha256ctx;	/* so HSH_AddString() can find it */
 	SHA256_Init(req->sha256ctx);
 	VCL_hash_method(req->vcl, wrk, req, NULL, req->http->ws);
-	assert(wrk->handling == VCL_RET_HASH);
+	assert(wrk->handling == VCL_RET_LOOKUP);
 	SHA256_Final(req->digest, req->sha256ctx);
 	req->sha256ctx = NULL;
 
@@ -1182,7 +1182,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 		req->wantbody = 1;
 
 	switch(recv_handling) {
-	case VCL_RET_LOOKUP:
+	case VCL_RET_HASH:
 		req->req_step = R_STP_LOOKUP;
 		return (REQ_FSM_MORE);
 	case VCL_RET_PIPE:
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index f4137b4..4918349 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -66,7 +66,7 @@ sub vcl_recv {
         /* Not cacheable by default */
         return (pass);
     }
-    return (lookup);
+    return (hash);
 }
 
 sub vcl_pipe {
@@ -90,7 +90,7 @@ sub vcl_hash {
     } else {
         hash_data(server.ip);
     }
-    return (hash);
+    return (lookup);
 }
 
 sub vcl_lookup {
diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc
index 7f0bda3..b88acfa 100644
--- a/bin/varnishtest/tests/c00033.vtc
+++ b/bin/varnishtest/tests/c00033.vtc
@@ -19,7 +19,7 @@ varnish v1 -vcl+backend {
 
 	sub vcl_recv {
 		if (req.method == "PURGE") {
-			return (lookup);
+			return (hash);
 		}
 	}
 
diff --git a/bin/varnishtest/tests/r00444.vtc b/bin/varnishtest/tests/r00444.vtc
index 85bdc15..2448b93 100644
--- a/bin/varnishtest/tests/r00444.vtc
+++ b/bin/varnishtest/tests/r00444.vtc
@@ -22,7 +22,7 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "POST") {
 			/* Lookup so we find any cached object */
-			return (lookup);
+			return (hash);
 		}
 	}
 	sub vcl_lookup {
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index 865e16a..b3a1299 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -14,7 +14,7 @@ varnish v1 -vcl+backend {
 			if (req.http.x-pass) {
 				return (pass);
 			} else {
-				return (lookup);
+				return (hash);
 			}
 		}
 	}
@@ -23,7 +23,7 @@ varnish v1 -vcl+backend {
 		## Check if they have a ban in the cache, or if they are going to be banned in cache.
 		if (req.http.X-Banned) {
 			hash_data(client.ip);
-			return (hash);
+			return (lookup);
 		}
 	}
 
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index a746852..f8aa7fb 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -18,7 +18,7 @@ varnish v1 -vcl+backend {
 			ban("obj.http.url ~ /");
 			error 201 "banned";
 		}
-		return (lookup);
+		return (hash);
 	}
 	sub vcl_backend_response {
 		set beresp.http.url = bereq.url;
diff --git a/bin/varnishtest/tests/r01296.vtc b/bin/varnishtest/tests/r01296.vtc
index f9f8de4..1683d92 100644
--- a/bin/varnishtest/tests/r01296.vtc
+++ b/bin/varnishtest/tests/r01296.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl+backend {
 		} else {
 			hash_data("1" + req.http.foo + "3");
 		}
-		return (hash);
+		return (lookup);
 	}
 } -start
 
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 0f6bfe8..349500a 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -78,10 +78,10 @@ tokens = {
 # Our methods and actions
 
 returns =(
-	('recv',		"C", ('error', 'pass', 'pipe', 'lookup',)),
+	('recv',		"C", ('error', 'pass', 'pipe', 'hash',)),
 	('pipe',		"C", ('error', 'pipe',)),
 	('pass',		"C", ('error', 'restart', 'pass',)),
-	('hash',		"C", ('hash',)),
+	('hash',		"C", ('lookup',)),
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
 	('lookup',		"C", ('error', 'restart', 'pass', 'deliver',)),
 	('backend_fetch',	"B", ('fetch', 'pass',)),



From tfheen at varnish-cache.org  Tue May  7 08:21:09 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 07 May 2013 10:21:09 +0200
Subject: [3.0] 59320c4 Document changes from 3.0.3 to 3.0.4-rc1
Message-ID: <E1UZd93-0005GM-4E@project.varnish-software.com>

commit 59320c472b3869bbe50dbbf4eb34cef44b545bf0
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri May 3 14:49:57 2013 +0200

    Document changes from 3.0.3 to 3.0.4-rc1

diff --git a/doc/changes.rst b/doc/changes.rst
index 99955cd..827a287 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,3 +1,61 @@
+================================
+Changes from 3.0.3 to 3.0.4 rc 1
+================================
+
+varnishd
+--------
+
+- Fix error handling when uncompressing fetched objects for ESI
+  processing. `Bug #1184`
+- Be clearer about which timeout was reached in logs.
+- Correctly decrement n_waitinglist counter.  `Bug #1261`
+- Turn off Nagle/set TCP_NODELAY.
+- Avoid panic on malformed Vary headers.  `Bug #1275`
+- Increase the maximum length of backend names.  `Bug #1224`
+- Add support for banning on http.status.  `Bug #1076`
+- Make hit-for-pass correctly prefer the transient storage.
+
+.. _bug #1076: http://varnish-cache.org/trac/ticket/1076
+.. _bug #1184: http://varnish-cache.org/trac/ticket/1184
+.. _bug #1224: http://varnish-cache.org/trac/ticket/1224
+.. _bug #1261: http://varnish-cache.org/trac/ticket/1261
+.. _bug #1275: http://varnish-cache.org/trac/ticket/1275
+
+
+varnishlog
+----------
+
+- If -m, but neither -b or -c is given, assume both.  This filters out
+  a lot of noise when -m is used to filter.  `Bug #1071`
+
+.. _bug #1071: http://varnish-cache.org/trac/ticket/1071
+
+varnishadm
+----------
+
+- Improve tab completion and require libedit/readline to build.
+
+varnishtop
+----------
+
+- Reopen log file if Varnish is restarted.
+
+varnishncsa
+-----------
+
+- Handle file descriptors above 64k (by ignoring them).  Prevents a
+  crash in some cases with corrupted shared memory logs.
+- Add %D and %T support for more timing information.
+
+Other
+-----
+
+- Documentation updates.
+- Fixes for OSX
+- Disable PCRE JIT-er, since it's broken in some PCRE versions, at
+  least on i386.
+- Make libvarnish prefer exact hits when looking for VSL tags.
+
 ===========================
 Changes from 3.0.2 to 3.0.3
 ===========================



From tfheen at varnish-cache.org  Tue May  7 08:21:09 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 07 May 2013 10:21:09 +0200
Subject: [3.0] e15fcc9 Set the waiter pipe as non-blocking and record overflows
Message-ID: <E1UZd93-0005GR-5n@project.varnish-software.com>

commit e15fcc9c207a56f270b449d2e35208e1bc7768cf
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Apr 9 08:25:26 2013 +0200

    Set the waiter pipe as non-blocking and record overflows
    
    Fixes #1285

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 5108a55..790ed62 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -394,6 +394,7 @@ vca_close_session(struct sess *sp, const char *why)
 void
 vca_return_session(struct sess *sp)
 {
+	int written = 0;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->obj);
@@ -406,8 +407,16 @@ vca_return_session(struct sess *sp)
 	if (VTCP_nonblocking(sp->fd)) {
 		vca_close_session(sp, "remote closed");
 		SES_Delete(sp);
-	} else if (vca_act->pass == NULL)
-		assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	} else if (vca_act->pass == NULL) {
+		written = write(vca_pipes[1], &sp, sizeof sp);
+		if (written != sizeof sp && (errno == EAGAIN || errno == EWOULDBLOCK)) {
+			VSC_C_main->sess_pipe_overflow++;
+			vca_close_session(sp, "session pipe overflow");
+			SES_Delete(sp);
+			return;
+		}
+		assert (written == sizeof sp);
+	}
 	else
 		vca_act->pass(sp);
 }
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index fe64961..108e48d 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -38,7 +38,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
-#include <fcntl.h>
 #include <stdlib.h>
 #include <unistd.h>
 
@@ -224,20 +223,11 @@ vca_sess_timeout_ticker(void *arg)
 static void
 vca_epoll_init(void)
 {
-	int i;
-
-	i = fcntl(vca_pipes[0], F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(vca_pipes[0], F_SETFL, i);
-	assert(i != -1);
-
 	AZ(pipe(dotimer_pipe));
-	i = fcntl(dotimer_pipe[0], F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(dotimer_pipe[0], F_SETFL, i);
-	assert(i != -1);
+
+	AZ(vnonblocking(vca_pipes[0]));
+	AZ(vnonblocking(vca_pipes[1]));
+	AZ(vnonblocking(dotimer_pipe[0]));
 
 	AZ(pthread_create(&vca_epoll_timeout_thread,
 	    NULL, vca_sess_timeout_ticker, NULL));
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index ea0e2fc..48fad20 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -38,7 +38,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
-#include <fcntl.h>
 #include <stdlib.h>
 #include <unistd.h>
 
@@ -208,11 +207,8 @@ vca_kqueue_init(void)
 {
 	int i;
 
-	i = fcntl(vca_pipes[0], F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(vca_pipes[0], F_SETFL, i);
-	assert(i != -1);
+	AZ(vnonblocking(vca_pipes[0]));
+	AZ(vnonblocking(vca_pipes[1]));
 
 	AZ(pthread_create(&vca_kqueue_thread, NULL, vca_kqueue_main, NULL));
 }
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index b9ad213..13120dd 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -199,6 +199,7 @@ vca_poll_init(void)
 {
 
 	vca_pollspace(256);
+	AZ(vnonblocking(vca_pipes[1]));
 	AZ(pthread_create(&vca_poll_thread, NULL, vca_main, NULL));
 }
 
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 364dd2c..6973080 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -250,9 +250,14 @@ static void
 vca_ports_pass(struct sess *sp)
 {
 	int r;
-	while((r = port_send(solaris_dport, 0, sp)) == -1 &&
-		errno == EAGAIN);
-	AZ(r);
+       r = port_send(vws->dport, 0, TRUST_ME(sp));
+       if (r == -1 && errno == EAGAIN) {
+	       VSC_C_main->sess_pipe_overflow++;
+	       vca_close_session(sp, "session pipe overflow");
+	       SES_Delete(sp);
+	       return;
+       }
+       AZ(r);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/include/libvarnish.h b/include/libvarnish.h
index b457d02..d7ae763 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -112,6 +112,7 @@ int seed_random(void);
 int vtmpfile(char *);
 char *vreadfile(const char *pfx, const char *fn, ssize_t *sz);
 char *vreadfd(int fd, ssize_t *sz);
+int vnonblocking(int fd);
 
 /* Safe printf into a fixed-size buffer */
 #define bprintf(buf, fmt, ...)						\
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index dddb235..592d236 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -167,6 +167,8 @@ VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
 VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
 VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
 
+VSC_F(sess_pipe_overflow,		uint64_t, 0, 'c', "Dropped sessions due to session pipe overflow", "")
+
 #endif
 
 /**********************************************************************/
diff --git a/lib/libvarnish/vtmpfile.c b/lib/libvarnish/vtmpfile.c
index 327790a..b51c29c 100644
--- a/lib/libvarnish/vtmpfile.c
+++ b/lib/libvarnish/vtmpfile.c
@@ -140,3 +140,16 @@ vreadfile(const char *pfx, const char *fn, ssize_t *sz)
 	errno = err;
 	return (r);
 }
+
+int
+vnonblocking(int fd)
+{
+	int i;
+
+	i = fcntl(fd, F_GETFL);
+	assert(i != -1);
+	i |= O_NONBLOCK;
+	i = fcntl(fd, F_SETFL, i);
+	assert(i != -1);
+	return (i);
+}



From tfheen at varnish-cache.org  Tue May  7 08:21:09 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 07 May 2013 10:21:09 +0200
Subject: [3.0] 0754c07 Fix up spec file
Message-ID: <E1UZd93-0005GY-74@project.varnish-software.com>

commit 0754c07e8ddd512b8fb85cda2756aa7a90228720
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue May 7 10:21:04 2013 +0200

    Fix up spec file

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 6903454..2771688 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,3 +1,4 @@
+%define v_rc rc1
 Summary: High-performance HTTP accelerator
 Name: varnish
 Version: 3.0.4



From phk at varnish-cache.org  Tue May  7 11:35:40 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 07 May 2013 13:35:40 +0200
Subject: [master] ea55fc0 Remove old bandaid from VCL1->VCL2 transition.
Message-ID: <E1UZgBI-0003fs-AD@project.varnish-software.com>

commit ea55fc01e04871abeeec989d87ccea601038994e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 7 11:35:22 2013 +0000

    Remove old bandaid from VCL1->VCL2 transition.

diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index b84df6f..057e151 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -276,17 +276,6 @@ parse_ban(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_new_syntax(struct vcc *tl)
-{
-
-	VSB_printf(tl->sb, "Please change \"%.*s\" to \"return(%.*s)\".\n",
-	    PF(tl->t), PF(tl->t));
-	vcc_ErrWhere(tl, tl->t);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
 parse_hash_data(struct vcc *tl)
 {
 	vcc_NextToken(tl);
@@ -379,11 +368,6 @@ static struct action_table {
             VCL_MET_MISS | VCL_MET_LOOKUP | VCL_MET_BACKEND_RESPONSE
 	},
 
-#define VCL_RET_MAC(l, U, B)						\
-	{ #l,			parse_new_syntax },
-#include "tbl/vcl_returns.h"
-#undef VCL_RET_MAC
-
 	/* Keep list sorted from here */
 	{ "call",		parse_call },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },



From perbu at varnish-cache.org  Wed May  8 07:10:46 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 08 May 2013 09:10:46 +0200
Subject: [master] ebd66cf missing cmd line options
Message-ID: <E1UZyWV-0002ml-04@project.varnish-software.com>

commit ebd66cf01661f95e10dbe64d01675b44509f0fd7
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed May 8 09:10:27 2013 +0200

    missing cmd line options

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 3c66c7b..da056ed 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -875,8 +875,8 @@ usage(void)
 {
 
 	fprintf(stderr,
-	    "usage: varnishncsa %s [-aDV] [-n varnish_name] "
-	    "[-P file] [-w file]\n", VSL_USAGE);
+	    "usage: varnishncsa %s [-afDV] [-n varnish_name] "
+	    "[-P file] [-w file] [-F format] \n", VSL_USAGE);
 	exit(1);
 }
 



From phk at varnish-cache.org  Sun May 12 11:08:44 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 12 May 2013 13:08:44 +0200
Subject: [master] fa014c8 Change the "error" syntax to:
Message-ID: <E1UbU8y-0001EX-Bo@project.varnish-software.com>

commit fa014c869726b540ba86d51ff0f777c82a10c5c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 12 11:08:06 2013 +0000

    Change the "error" syntax to:
    
    	return (error(status_code [ , response_string ]));

diff --git a/bin/varnishtest/tests/b00016.vtc b/bin/varnishtest/tests/b00016.vtc
index 93be72b..56bbd33 100644
--- a/bin/varnishtest/tests/b00016.vtc
+++ b/bin/varnishtest/tests/b00016.vtc
@@ -6,7 +6,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_recv {
-	    error 200 "ok";
+	    return(error(200,"ok"));
 	}
 
 	sub vcl_error {
@@ -33,7 +33,7 @@ varnish v1 -vcl {
 
 	sub vcl_recv {
 		set req.backend = bar.backend();
-		error 200 "ok";
+		return(error(200,"ok"));
 	}
 
 	sub vcl_error {
@@ -59,7 +59,7 @@ varnish v1 -vcl {
 
 	sub vcl_recv {
 	    set req.backend = baz.backend();
-	    error 200 "ok";
+	    return (error(200,"ok"));
 	}
 
 	sub vcl_error {
diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc
index 85ec39c..053fffd 100644
--- a/bin/varnishtest/tests/b00017.vtc
+++ b/bin/varnishtest/tests/b00017.vtc
@@ -5,7 +5,7 @@ varnish v1 -vcl {
 		.host = "${bad_ip}"; .port = "9080";
 	}
 	sub vcl_recv {
-		error 888;
+		return (error(888));
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/b00028.vtc b/bin/varnishtest/tests/b00028.vtc
index b91c274..0745959 100644
--- a/bin/varnishtest/tests/b00028.vtc
+++ b/bin/varnishtest/tests/b00028.vtc
@@ -11,12 +11,12 @@ varnish v1 -vcl+backend {
 		if (beresp.http.foo ~ "bar") {
 			set beresp.http.foo1 = "1";
 		} else {
-			error 999;
+			set beresp.status = 999;
 		}
 		if (beresp.http.bar !~ "bar") {
 			set beresp.http.bar1 = "2";
 		} else {
-			error 999;
+			set beresp.status = 999;
 		}
 	}
 
@@ -25,6 +25,7 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
+	expect resp.status == "200"
 	expect resp.http.foo1 == "1"
 	expect resp.http.bar1 == "2"
 } -run
diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc
index fcd0356..9703cc7 100644
--- a/bin/varnishtest/tests/b00030.vtc
+++ b/bin/varnishtest/tests/b00030.vtc
@@ -10,7 +10,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		error 200 req.grace;
+		return (error(200,req.grace));
 	}
 	sub vcl_error {
 		set obj.http.x-timestamp = now;
diff --git a/bin/varnishtest/tests/c00022.vtc b/bin/varnishtest/tests/c00022.vtc
index 64511fa..328ef92 100644
--- a/bin/varnishtest/tests/c00022.vtc
+++ b/bin/varnishtest/tests/c00022.vtc
@@ -22,11 +22,11 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "PURGE") {
 			ban ("req.url == " + req.url);
-			error 410;
+			return (error(410));
 		}
 		if (req.method == "PURGESTR") {
 			ban ("" + req.http.ban);
-			error 410;
+			return (error(410));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc
index b88acfa..13f69f8 100644
--- a/bin/varnishtest/tests/c00033.vtc
+++ b/bin/varnishtest/tests/c00033.vtc
@@ -26,13 +26,13 @@ varnish v1 -vcl+backend {
 	sub vcl_lookup {
 		if (req.method == "PURGE") {
 			C{ VRT_purge(ctx, 0, 0); }C
-			error 456 "got it";
+			return (error(456, "got it"));
 		}
 	}
 	sub vcl_miss {
 		if (req.method == "PURGE") {
 			C{ VRT_purge(ctx, 0, 0); }C
-			error 456 "got it";
+			return (error(456, "got it"));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00558.vtc b/bin/varnishtest/tests/r00558.vtc
index 0efb9e9..3a8ba97 100644
--- a/bin/varnishtest/tests/r00558.vtc
+++ b/bin/varnishtest/tests/r00558.vtc
@@ -1,4 +1,5 @@
 varnishtest "error from vcl_recv{} has no numeric code"
+# XXX: V4 makes this an obsolete test ?
 
 
 server s1 {
@@ -6,7 +7,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		return (error);
+		return (error(501));
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r00769.vtc b/bin/varnishtest/tests/r00769.vtc
index 2306346..838ef17 100644
--- a/bin/varnishtest/tests/r00769.vtc
+++ b/bin/varnishtest/tests/r00769.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.url ~ "^/test1") {
-			error 700;
+			return (error(700));
 		}
 	}
 	sub vcl_backend_response {
diff --git a/bin/varnishtest/tests/r00896.vtc b/bin/varnishtest/tests/r00896.vtc
index 280a176..b559075 100644
--- a/bin/varnishtest/tests/r00896.vtc
+++ b/bin/varnishtest/tests/r00896.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 	    if (req.http.host ~ "^(abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij|abcdefghij)") {
-		error 500 "not ok";
+		return (error(500,"not ok"));
 		}
 	}
 } -start 
diff --git a/bin/varnishtest/tests/r00936.vtc b/bin/varnishtest/tests/r00936.vtc
index 176cc7c..99e1d65 100644
--- a/bin/varnishtest/tests/r00936.vtc
+++ b/bin/varnishtest/tests/r00936.vtc
@@ -9,7 +9,7 @@ varnish v1 -errvcl {'synthetic': not a valid action in method 'vcl_recv'} {
 	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_recv {
-		synthetic "HELLOO"; return (error);
+		synthetic "HELLOO"; return (error(503));
 	}
 }
 
diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc
index b3a1299..72317de 100644
--- a/bin/varnishtest/tests/r00965.vtc
+++ b/bin/varnishtest/tests/r00965.vtc
@@ -32,11 +32,15 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_miss {
-		if (req.http.X-Banned == "check") { error 988 "restarting"; }
+		if (req.http.X-Banned == "check") {
+			return (error(988,"restarting"));
+		}
 	}
 
 	sub vcl_pass {
-		if (req.http.X-Banned == "check") { error 988 "restarting"; }
+		if (req.http.X-Banned == "check") {
+			return (error(988,"restarting"));
+		}
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r01027.vtc b/bin/varnishtest/tests/r01027.vtc
index 347f623..d6d0991 100644
--- a/bin/varnishtest/tests/r01027.vtc
+++ b/bin/varnishtest/tests/r01027.vtc
@@ -1,9 +1,9 @@
 varnishtest "Test if you can error in vcl_deliver"
 
-varnish v1 -errvcl {'error': not a valid action in method 'vcl_deliver'} {
+varnish v1 -errvcl {Invalid return "error"} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_deliver {
-		error 201 "ok";
+		return (error(201,"ok"));
 	}
 }
 
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index f8aa7fb..4f95d70 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -16,7 +16,7 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "BAN") {
 			ban("obj.http.url ~ /");
-			error 201 "banned";
+			return (error(201,"banned"));
 		}
 		return (hash);
 	}
diff --git a/bin/varnishtest/tests/r01031.vtc b/bin/varnishtest/tests/r01031.vtc
index 435f973..27bbab2 100644
--- a/bin/varnishtest/tests/r01031.vtc
+++ b/bin/varnishtest/tests/r01031.vtc
@@ -6,7 +6,7 @@ varnish v1 -vcl {
 	}
 
 	sub vcl_recv {
-		error 200 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+		return (error(200,"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 	}
 	sub vcl_error {
 		return(deliver);
diff --git a/bin/varnishtest/tests/r01283.vtc b/bin/varnishtest/tests/r01283.vtc
index eaa7b81..76fa434 100644
--- a/bin/varnishtest/tests/r01283.vtc
+++ b/bin/varnishtest/tests/r01283.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -arg "-p nuke_limit=0" -storage "-sTransient=malloc,1m" -vcl+backend {
 	sub vcl_recv {
 		if (req.http.x-do-error) {
-			error 500;
+			return (error(500));
 		}
 	}
 	sub vcl_backend_response {
diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc
index 89119e4..b815127 100644
--- a/bin/varnishtest/tests/v00011.vtc
+++ b/bin/varnishtest/tests/v00011.vtc
@@ -12,7 +12,7 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "PURGE") {
 			ban("req.url ~ ^/$");
-			error 209 "foo";
+			return (error(209,"foo"));
 		}
 	}
 
diff --git a/bin/varnishtest/tests/v00014.vtc b/bin/varnishtest/tests/v00014.vtc
index 206e8bc..5477a98 100644
--- a/bin/varnishtest/tests/v00014.vtc
+++ b/bin/varnishtest/tests/v00014.vtc
@@ -26,9 +26,9 @@ varnish v1 -vcl {
 
 	sub vcl_recv {
 		if (req.backend.healthy) {
-			error 200 "Backend healthy";
+			return(error(200,"Backend healthy"));
 		} else {
-			error 500 "Backend sick";
+			return(error(500,"Backend sick"));
 		}
 	}
 } -start
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index a89ecb6..6b6fa1f 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -8,14 +8,8 @@ varnish v1 -vcl {
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_backend_response { error beresp.status ; }
-}
-
-varnish v1 -vcl {
-	backend b { .host = "127.0.0.1"; }
-	sub vcl_miss { error 100 req.url ; }
-	sub vcl_lookup { error 100 "the butter please" ; }
-	sub vcl_backend_response { error beresp.status req.url; }
+	sub vcl_miss { return(error(100,req.url)); }
+	sub vcl_lookup { return(error(100,"the butter please")); }
 }
 
 varnish v1 -errvcl {Variable 'server.port' is read only.} {
@@ -108,3 +102,13 @@ varnish v1 -errvcl {Unknown token '<<' when looking for STRING_LIST} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_error { synthetic << "foo"; }
 }
+
+varnish v1 -errvcl {Syntax has changed, use:} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_recv { error 200; }
+}
+
+varnish v1 -errvcl {Syntax has changed, use:} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_recv { return(error); }
+}
diff --git a/bin/varnishtest/tests/v00024.vtc b/bin/varnishtest/tests/v00024.vtc
index 9c2040c..5ef311a 100644
--- a/bin/varnishtest/tests/v00024.vtc
+++ b/bin/varnishtest/tests/v00024.vtc
@@ -9,7 +9,7 @@ server s1 {
 varnish v1 -vcl+backend { 
 	sub vcl_recv {
 		if (req.http.etag == req.http.if-none-match) {
-			error 400 "FOO";
+			return(error(400,"FOO"));
 		}
 	}
 } -start
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 057e151..f53248b 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -58,29 +58,13 @@ static void
 parse_error(struct vcc *tl)
 {
 
-	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_error(ctx,\n");
-	if (tl->t->tok == '(') {
-		vcc_NextToken(tl);
-		vcc_Expr(tl, INT);
-		if (tl->t->tok == ',') {
-			Fb(tl, 1, ",\n");
-			vcc_NextToken(tl);
-			vcc_Expr(tl, STRING);
-		} else
-			Fb(tl, 1, ", 0\n");
-		SkipToken(tl, ')');
-	} else {
-		vcc_Expr(tl, INT);
-		if (tl->t->tok != ';') {
-			Fb(tl, 1, ",\n");
-			vcc_Expr(tl, STRING);
-		} else
-			Fb(tl, 1, ", 0\n");
-	}
-	Fb(tl, 1, ");\n");
-	Fb(tl, 1, "VRT_handling(ctx, VCL_RET_ERROR);\n");
-	Fb(tl, 1, "return(1);\n");
+	VSB_printf(tl->sb,
+	    "Syntax has changed, use:\n"
+	    "\treturn(error(999));\n"
+	    "or\n"
+	    "\treturn(error(999, \"Response text\"));\n");
+	vcc_ErrWhere(tl, tl->t);
+	return;
 }
 
 /*--------------------------------------------------------------------*/
@@ -300,6 +284,44 @@ parse_return(struct vcc *tl)
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 
+	/* 'error' gets special handling, to allow optional status/response */
+	if (vcc_IdIs(tl->t, "error")) {
+		vcc_NextToken(tl);
+		if (tl->t->tok == ')') {
+			VSB_printf(tl->sb,
+			    "Syntax has changed, use:\n"
+			    "\treturn(error(999));\n"
+			    "or\n"
+			    "\treturn(error(999, \"Response text\"));\n");
+			vcc_ErrWhere(tl, tl->t);
+			return;
+		}
+		ExpectErr(tl, '(');
+		vcc_NextToken(tl);
+		Fb(tl, 1, "VRT_error(ctx,\n");
+		tl->indent += INDENT;
+		vcc_Expr(tl, INT);
+		ERRCHK(tl);
+		Fb(tl, 1, ",\n");
+		if (tl->t->tok == ',') {
+			vcc_NextToken(tl);
+			vcc_Expr(tl, STRING);
+			ERRCHK(tl);
+		} else {
+			Fb(tl, 1, "(const char*)0\n");
+		}
+		tl->indent -= INDENT;
+		ExpectErr(tl, ')');
+		vcc_NextToken(tl);
+		Fb(tl, 1, ");\n");
+		Fb(tl, 1, "VRT_handling(ctx, VCL_RET_ERROR);\n");
+		Fb(tl, 1, "return (1);\n");
+		vcc_ProcAction(tl->curproc, VCL_RET_ERROR, tl->t);
+		ExpectErr(tl, ')');
+		vcc_NextToken(tl);
+		return;
+	}
+
 #define VCL_RET_MAC(l, U, B)						\
 	do {								\
 		if (vcc_IdIs(tl->t, #l)) {				\
@@ -363,23 +385,19 @@ static struct action_table {
 	action_f		*func;
 	unsigned		bitmask;
 } action_table[] = {
-	{ "error",		parse_error,
-	    VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH |
-            VCL_MET_MISS | VCL_MET_LOOKUP | VCL_MET_BACKEND_RESPONSE
-	},
-
 	/* Keep list sorted from here */
+	{ "ban",		parse_ban },
 	{ "call",		parse_call },
+	{ "error",		parse_error },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
-	{ "ban",		parse_ban },
+	{ "new",		parse_new, VCL_MET_INIT},
+	{ "purge",		parse_purge, VCL_MET_MISS | VCL_MET_LOOKUP },
 	{ "remove",		parse_unset }, /* backward compatibility */
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },
 	{ "synthetic",		parse_synthetic, VCL_MET_ERROR },
 	{ "unset",		parse_unset },
-	{ "purge",		parse_purge, VCL_MET_MISS | VCL_MET_LOOKUP },
-	{ "new",		parse_new, VCL_MET_INIT},
 	{ NULL,			NULL }
 };
 



From phk at varnish-cache.org  Sun May 12 21:05:55 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 12 May 2013 23:05:55 +0200
Subject: [master] adab8d8 Make it possible to return(purge) in vcl_recv{}.
Message-ID: <E1UbdSt-0005gr-8S@project.varnish-software.com>

commit adab8d84727227565b7f3a92a78f10448ab21420
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 12 21:04:39 2013 +0000

    Make it possible to return(purge) in vcl_recv{}.
    
    After the purge, you will end up in vcl_purge{} from where it will
    (at some future date) be possible to order a fresh fetch of the
    object.
    
    Doing purge from vcl_recv{} solves half of the object existence
    problems in vcl_lookup{}.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 3f4fb79..e9ef4fc 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -348,6 +348,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	Lck_AssertHeld(&oh->mtx);
 
 	if (always_insert) {
+		/* XXX: should we do predictive Vary in this case ? */
 		/* Insert new objcore in objecthead and release mutex */
 		*bocp = hsh_insert_busyobj(wrk, oh);
 		/* NB: no deref of objhead, new object inherits reference */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 90a4a1a..3970e63 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -1182,6 +1182,9 @@ cnt_recv(struct worker *wrk, struct req *req)
 		req->wantbody = 1;
 
 	switch(recv_handling) {
+	case VCL_RET_PURGE:
+		req->req_step = R_STP_PURGE;
+		return (REQ_FSM_MORE);
 	case VCL_RET_HASH:
 		req->req_step = R_STP_LOOKUP;
 		return (REQ_FSM_MORE);
@@ -1205,6 +1208,44 @@ cnt_recv(struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
+ * PURGE
+ * Find the objhead, purge it and ask VCL if we should fetch or
+ * just return.
+ * XXX: fetching not implemented yet.
+ */
+
+static enum req_fsm_nxt
+cnt_purge(struct worker *wrk, struct req *req)
+{
+	struct objcore *oc, *boc;
+	enum lookup_e lr;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(req->objcore);
+
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AZ(req->busyobj);
+
+	VRY_Prep(req);
+
+	AZ(req->objcore);
+	lr = HSH_Lookup(req, &oc, &boc, 1, 1);
+	assert (lr == HSH_MISS);
+	AZ(oc);
+	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
+	VRY_Finish(req, NULL);
+
+	HSH_Purge(req, boc->objhead, 0, 0);
+
+	AZ(HSH_Deref(&wrk->stats, boc, NULL));
+
+	VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws);
+	req->req_step = R_STP_ERROR;
+	return (REQ_FSM_MORE);
+}
+
+/*--------------------------------------------------------------------
  * Central state engine dispatcher.
  *
  * Kick the session around until it has had enough.
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 4918349..488cf0c 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -93,6 +93,10 @@ sub vcl_hash {
     return (lookup);
 }
 
+sub vcl_purge {
+    return (error(200, "Purged"));
+}
+
 sub vcl_lookup {
 /*
     if (!obj) {
diff --git a/bin/varnishtest/tests/b00036.vtc b/bin/varnishtest/tests/b00036.vtc
new file mode 100644
index 0000000..cb0b6e5
--- /dev/null
+++ b/bin/varnishtest/tests/b00036.vtc
@@ -0,0 +1,37 @@
+varnishtest "builtin purge from vcl_recv{}"
+
+server s1 {
+	rxreq
+	txresp -hdr "foo: 1"
+	rxreq
+	txresp -hdr "foo: 2"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.request == "PURGE") {
+			return (purge);
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.foo == 1
+
+	txreq
+	rxresp
+	expect resp.http.foo == 1
+
+	txreq -req PURGE
+	rxresp
+	expect resp.msg == "Purged"
+} -run
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.foo == 2
+
+} -run
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index ece22cb..a4c19b0 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -41,6 +41,7 @@ REQ_STEP(recv,		RECV,		(wrk, req))
 REQ_STEP(pipe,		PIPE,		(wrk, req))
 REQ_STEP(pass,		PASS,		(wrk, req))
 REQ_STEP(lookup,	LOOKUP,		(wrk, req))
+REQ_STEP(purge,		PURGE,		(wrk, req))
 REQ_STEP(miss,		MISS,		(wrk, req))
 REQ_STEP(fetch,		FETCH,		(wrk, req))
 REQ_STEP(fetchbody,	FETCHBODY,	(wrk, req))
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 349500a..1402e13 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -78,10 +78,11 @@ tokens = {
 # Our methods and actions
 
 returns =(
-	('recv',		"C", ('error', 'pass', 'pipe', 'hash',)),
+	('recv',		"C", ('error', 'pass', 'pipe', 'hash', 'purge',)),
 	('pipe',		"C", ('error', 'pipe',)),
 	('pass',		"C", ('error', 'restart', 'pass',)),
 	('hash',		"C", ('lookup',)),
+	('purge',		"C", ('error', 'fetch',)),
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
 	('lookup',		"C", ('error', 'restart', 'pass', 'deliver',)),
 	('backend_fetch',	"B", ('fetch', 'pass',)),



From phk at varnish-cache.org  Mon May 13 09:08:46 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 13 May 2013 11:08:46 +0200
Subject: [master] 255fe95 Change args to http_FilterReq() to a lower level.
Message-ID: <E1UbokQ-0001ki-6g@project.varnish-software.com>

commit 255fe9517fe968be78fb2fe2ed58f0cf93b1c512
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 09:08:15 2013 +0000

    Change args to http_FilterReq() to a lower level.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 865b315..a28bf98 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -856,7 +856,7 @@ void HTTP_Init(void);
 void http_ClrHeader(struct http *to);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
-void http_FilterReq(const struct req *, unsigned how);
+void http_FilterReq(struct http *to, const struct http *fm, unsigned how);
 void http_FilterResp(const struct http *fm, struct http *to, unsigned how);
 void http_PutProtocol(const struct http *to, const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 3970e63..0066c5c 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -772,27 +772,27 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	switch (lr) {
 	case HSH_EXP:
-VSLb(req->vsl, SLT_Debug, "XXXX EXP\n");
+		VSLb(req->vsl, SLT_Debug, "XXXX EXP\n");
 		AN(oc);
 		AZ(boc);
 		break;
 	case HSH_EXPBUSY:
-VSLb(req->vsl, SLT_Debug, "XXXX EXPBUSY\n");
+		VSLb(req->vsl, SLT_Debug, "XXXX EXPBUSY\n");
 		AN(oc);
 		AN(boc);
 		if (VDI_Healthy(req->director, req->digest)) {
-VSLb(req->vsl, SLT_Debug, "deref oc\n");
+			VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY deref oc\n");
 			(void)HSH_Deref(&wrk->stats, oc, NULL);
 			oc = boc;
 			boc = NULL;
 		} else {
-VSLb(req->vsl, SLT_Debug, "drop boc\n");
+			VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY drop boc\n");
 			(void)HSH_Deref(&wrk->stats, boc, NULL);
 			boc = NULL;
 		}
 		break;
 	case HSH_MISS:
-VSLb(req->vsl, SLT_Debug, "XXXX MISS\n");
+		VSLb(req->vsl, SLT_Debug, "XXXX MISS\n");
 		AZ(oc);
 		AN(boc);
 		oc = boc;
@@ -800,7 +800,7 @@ VSLb(req->vsl, SLT_Debug, "XXXX MISS\n");
 		AN(oc->flags & OC_F_BUSY);
 		break;
 	case HSH_HIT:
-VSLb(req->vsl, SLT_Debug, "XXXX HIT\n");
+		VSLb(req->vsl, SLT_Debug, "XXXX HIT\n");
 		AN(oc);
 		AZ(boc);
 		break;
@@ -913,7 +913,9 @@ cnt_miss(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(req, HTTPH_R_FETCH);
+	http_FilterReq(bo->bereq, req->http, HTTPH_R_FETCH);
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 	http_ForceGet(bo->bereq);
 	if (cache_param->http_gzip_support) {
 		/*
@@ -988,7 +990,9 @@ cnt_pass(struct worker *wrk, struct req *req)
 	bo = req->busyobj;
 	bo->refcount = 2;
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(req, HTTPH_R_PASS);
+	http_FilterReq(bo->bereq, req->http, HTTPH_R_PASS);
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 	VCL_pass_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
@@ -1047,7 +1051,9 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	req->busyobj = VBO_GetBusyObj(wrk, req);
 	bo = req->busyobj;
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(req, 0);
+	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
 	VCL_pipe_method(req->vcl, wrk, req, NULL, req->http->ws);
 



From phk at varnish-cache.org  Mon May 13 09:09:24 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 13 May 2013 11:09:24 +0200
Subject: [master] 4614058 Forgotten file in last commit:
Message-ID: <E1Ubol2-0001p1-Vt@project.varnish-software.com>

commit 4614058b12987fe726bc713fcd8e475e169c5c2c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 09:09:02 2013 +0000

    Forgotten file in last commit:
    
    Change args to http_FilterReq() to lower level.

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index e9f9e22..82d9ecf 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -585,21 +585,18 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how)
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterReq(const struct req *req, unsigned how)
+http_FilterReq(struct http *to, const struct http *fm, unsigned how)
 {
-	struct http *hp;
-
-	hp = req->busyobj->bereq;
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
 
-	http_linkh(hp, req->http, HTTP_HDR_METHOD);
-	http_linkh(hp, req->http, HTTP_HDR_URL);
+	http_linkh(to, fm, HTTP_HDR_METHOD);
+	http_linkh(to, fm, HTTP_HDR_URL);
 	if (how == HTTPH_R_FETCH)
-		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
+		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
-		http_linkh(hp, req->http, HTTP_HDR_PROTO);
-	http_filterfields(hp, req->http, how);
-	http_PrintfHeader(hp, "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
+		http_linkh(to, fm, HTTP_HDR_PROTO);
+	http_filterfields(to, fm, how);
 }
 
 /*--------------------------------------------------------------------*/



From phk at varnish-cache.org  Mon May 13 09:29:02 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 13 May 2013 11:29:02 +0200
Subject: [master] c8c3d31 Change args to FetchHdr() to lower levels.
Message-ID: <E1Ubp42-0003PK-NL@project.varnish-software.com>

commit c8c3d314ce532ff2a93cef20fa9ecfef966cea24
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 09:28:47 2013 +0000

    Change args to FetchHdr() to lower levels.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a28bf98..0049eec 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -814,7 +814,8 @@ void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
-int FetchHdr(struct req *req, int need_host_hdr, int sendbody);
+int FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
+    int need_host_hdr, int sendbody);
 void FetchBody(struct worker *w, void *bo);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 951e367..fd74b0f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -357,21 +357,18 @@ fetch_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
  */
 
 int
-FetchHdr(struct req *req, int need_host_hdr, int sendbody)
+FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
+    int need_host_hdr, int sendbody)
 {
 	struct vbc *vc;
-	struct worker *wrk;
-	struct busyobj *bo;
 	struct http *hp;
 	enum htc_status_e hs;
 	int retry = -1;
 	int i, first;
 	struct http_conn *htc;
 
-	wrk = req->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	htc = &bo->htc;
 
@@ -385,7 +382,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody)
 
 	bo->vbc = VDI_GetFd(NULL, bo);
 	if (bo->vbc == NULL) {
-		VSLb(req->vsl, SLT_FetchError, "no backend connection");
+		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 		return (-1);
 	}
 	vc = bo->vbc;
@@ -408,8 +405,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody)
 	i = 0;
 
 	if (sendbody) {
-		i = HTTP1_IterateReqBody(req,
-		    fetch_iter_req_body, NULL);
+		i = HTTP1_IterateReqBody(req, fetch_iter_req_body, NULL);
 		if (req->req_body_status == REQ_BODY_DONE)
 			retry = -1;
 	} else {
@@ -417,8 +413,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody)
 	}
 
 	if (WRW_FlushRelease(wrk) || i != 0) {
-		VSLb(req->vsl, SLT_FetchError,
-		    "backend write error: %d (%s)",
+		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
 		VDI_CloseFd(&bo->vbc);
 		/* XXX: other cleanup ? */
@@ -440,7 +435,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody)
 	do {
 		hs = HTTP1_Rx(htc);
 		if (hs == HTTP1_OVERFLOW) {
-			VSLb(req->vsl, SLT_FetchError,
+			VSLb(bo->vsl, SLT_FetchError,
 			    "http %sread error: overflow",
 			    first ? "first " : "");
 			VDI_CloseFd(&bo->vbc);
@@ -448,8 +443,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody)
 			return (-1);
 		}
 		if (hs == HTTP1_ERROR_EOF) {
-			VSLb(req->vsl, SLT_FetchError,
-			    "http %sread error: EOF",
+			VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
 			    first ? "first " : "");
 			VDI_CloseFd(&bo->vbc);
 			/* XXX: other cleanup ? */
@@ -466,7 +460,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody)
 	hp = bo->beresp;
 
 	if (HTTP1_DissectResponse(hp, htc)) {
-		VSLb(req->vsl, SLT_FetchError, "http format error");
+		VSLb(bo->vsl, SLT_FetchError, "http format error");
 		VDI_CloseFd(&bo->vbc);
 		/* XXX: other cleanup ? */
 		return (-1);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 0066c5c..24c4cf1 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -371,7 +371,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	req->acct_req.fetch++;
 
-	i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
+	i = FetchHdr(wrk, bo, req, need_host_hdr,
+	    req->objcore->objhead == NULL);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -379,7 +380,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
+		i = FetchHdr(wrk, bo, req, need_host_hdr,
+		    req->objcore->objhead == NULL);
 	}
 
 	if (i) {



From phk at varnish-cache.org  Mon May 13 10:35:16 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 13 May 2013 12:35:16 +0200
Subject: [master] 8e8a894 Give busyobj its own "t_fetch" timestamp,
 (== req->t_req) and only pass req to FetchHdr if we intend to send
 the body over.
Message-ID: <E1Ubq68-00086n-Oe@project.varnish-software.com>

commit 8e8a8942bd154b36bf81da45cf5fdccd7bfc3562
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 10:34:40 2013 +0000

    Give busyobj its own "t_fetch" timestamp, (== req->t_req) and
    only pass req to FetchHdr if we intend to send the body over.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0049eec..4a5cc3a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -501,6 +501,7 @@ struct busyobj {
 	 * is recycled.
 	 */
 	unsigned		refcount;
+	double			t_fetch;
 
 	uint8_t			*vary;
 	unsigned		is_gzip;
@@ -815,7 +816,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
 int FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
-    int need_host_hdr, int sendbody);
+    int need_host_hdr);
 void FetchBody(struct worker *w, void *bo);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index ffe108a..5124dc4 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -145,6 +145,8 @@ VBO_GetBusyObj(struct worker *wrk, struct req *req)
 	bo->vcl = req->vcl;
 	VCL_Ref(bo->vcl);
 
+	bo->t_fetch = req->t_req;
+
 	return (bo);
 }
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index fd74b0f..62d0815 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -358,7 +358,7 @@ fetch_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
 
 int
 FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
-    int need_host_hdr, int sendbody)
+    int need_host_hdr)
 {
 	struct vbc *vc;
 	struct http *hp;
@@ -368,15 +368,11 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
 	struct http_conn *htc;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	htc = &bo->htc;
 
-	AN(req->director);
-	AZ(req->obj);
-
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	AN(req->objcore->flags & OC_F_BUSY);
+	AN(bo->director);
 
 	hp = bo->bereq;
 
@@ -398,18 +394,16 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
 		VDI_AddHostHeader(bo->bereq, vc);
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
-	WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req);	/* XXX t_resp ? */
+	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_fetch);
 	(void)HTTP1_Write(wrk, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
 
-	if (sendbody) {
+	if (req != NULL) {
 		i = HTTP1_IterateReqBody(req, fetch_iter_req_body, NULL);
 		if (req->req_body_status == REQ_BODY_DONE)
 			retry = -1;
-	} else {
-		i = HTTP1_DiscardReqBody(req);
 	}
 
 	if (WRW_FlushRelease(wrk) || i != 0) {
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 24c4cf1..3c87406 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -371,8 +371,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	req->acct_req.fetch++;
 
-	i = FetchHdr(wrk, bo, req, need_host_hdr,
-	    req->objcore->objhead == NULL);
+	i = FetchHdr(wrk, bo, req->objcore->objhead == NULL ? req : NULL,
+	    need_host_hdr);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -380,10 +380,13 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(wrk, bo, req, need_host_hdr,
-		    req->objcore->objhead == NULL);
+		i = FetchHdr(wrk, bo,
+		    req->objcore->objhead == NULL ? req : NULL, need_host_hdr);
 	}
 
+	if (req->objcore->objhead != NULL)
+		(void)HTTP1_DiscardReqBody(req);	// XXX
+
 	if (i) {
 		wrk->handling = VCL_RET_ERROR;
 		req->err_code = 503;



From phk at varnish-cache.org  Mon May 13 11:44:00 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 13 May 2013 13:44:00 +0200
Subject: [master] 5647c26 Move the "need_host_hdr" test to cache_fetch.c
Message-ID: <E1UbrAe-0004Yw-QC@project.varnish-software.com>

commit 5647c265ad01b23cf408089a9f9b94cb2bc47af1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 11:43:44 2013 +0000

    Move the "need_host_hdr" test to cache_fetch.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4a5cc3a..9b96e4e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -815,8 +815,7 @@ void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
-int FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
-    int need_host_hdr);
+int FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req);
 void FetchBody(struct worker *w, void *bo);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 62d0815..4bacbf1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -357,8 +357,7 @@ fetch_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
  */
 
 int
-FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
-    int need_host_hdr)
+FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 {
 	struct vbc *vc;
 	struct http *hp;
@@ -366,6 +365,7 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
 	int retry = -1;
 	int i, first;
 	struct http_conn *htc;
+        int need_host_hdr;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -374,6 +374,8 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req,
 
 	AN(bo->director);
 
+	need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
+
 	hp = bo->bereq;
 
 	bo->vbc = VDI_GetFd(NULL, bo);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 3c87406..c9e0e05 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -350,7 +350,7 @@ DOT fetch -> fetchbody [style=bold,color=blue]
 static enum req_fsm_nxt
 cnt_fetch(struct worker *wrk, struct req *req)
 {
-	int i, need_host_hdr;
+	int i;
 	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -367,12 +367,9 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
-	need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
-
 	req->acct_req.fetch++;
 
-	i = FetchHdr(wrk, bo, req->objcore->objhead == NULL ? req : NULL,
-	    need_host_hdr);
+	i = FetchHdr(wrk, bo, req->objcore->objhead == NULL ? req : NULL);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -381,7 +378,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
 		i = FetchHdr(wrk, bo,
-		    req->objcore->objhead == NULL ? req : NULL, need_host_hdr);
+		    req->objcore->objhead == NULL ? req : NULL);
 	}
 
 	if (req->objcore->objhead != NULL)



From phk at varnish-cache.org  Mon May 13 12:30:43 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 13 May 2013 14:30:43 +0200
Subject: [master] 43b4bec Make VRY_Create() operate on the busyobj and
 specifically on the bereq.* rather than the req.*,
 since that's what the backend saw.
Message-ID: <E1Ubrtr-0007QZ-NN@project.varnish-software.com>

commit 43b4bec0860f45c75d8d097a5d8a6cf6362c764e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 12:30:07 2013 +0000

    Make VRY_Create() operate on the busyobj and specifically on the
    bereq.* rather than the req.*, since that's what the backend saw.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9b96e4e..aa83773 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1007,7 +1007,7 @@ void RES_BuildHttp(struct req *);
 void RES_WriteObj(struct req *);
 
 /* cache_vary.c */
-int VRY_Create(struct req *req, const struct http *hp, struct vsb **psb);
+int VRY_Create(struct busyobj *bo, struct vsb **psb);
 int VRY_Match(struct req *, const uint8_t *vary);
 void VRY_Validate(const uint8_t *vary);
 void VRY_Prep(struct req *);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c9e0e05..a1baa0e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -576,7 +576,7 @@ cnt_fetchbody(struct worker *wrk, struct req *req)
 
 	/* Create Vary instructions */
 	if (req->objcore->objhead != NULL) {
-		varyl = VRY_Create(req, bo->beresp, &vary);
+		varyl = VRY_Create(bo, &vary);
 		if (varyl > 0) {
 			AN(vary);
 			assert(varyl == VSB_len(vary));
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index ba87608..1ec8548 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -71,18 +71,21 @@
  */
 
 int
-VRY_Create(struct req *req, const struct http *hp, struct vsb **psb)
+VRY_Create(struct busyobj *bo, struct vsb **psb)
 {
 	char *v, *p, *q, *h, *e;
 	struct vsb *sb, *sbh;
 	unsigned l;
 	int error = 0;
 
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->bereq, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->beresp, HTTP_MAGIC);
 	AN(psb);
 	AZ(*psb);
 
 	/* No Vary: header, no worries */
-	if (!http_GetHdr(hp, H_Vary, &v))
+	if (!http_GetHdr(bo->beresp, H_Vary, &v))
 		return (0);
 
 	/* For vary matching string */
@@ -102,7 +105,7 @@ VRY_Create(struct req *req, const struct http *hp, struct vsb **psb)
 			continue;
 
 		if (q - p > INT8_MAX) {
-			VSLb(req->vsl, SLT_Error,
+			VSLb(bo->vsl, SLT_Error,
 			    "Vary header name length exceeded");
 			error = 1;
 			break;
@@ -114,7 +117,7 @@ VRY_Create(struct req *req, const struct http *hp, struct vsb **psb)
 		    (char)(1 + (q - p)), (int)(q - p), p, 0);
 		AZ(VSB_finish(sbh));
 
-		if (http_GetHdr(req->http, VSB_data(sbh), &h)) {
+		if (http_GetHdr(bo->bereq, VSB_data(sbh), &h)) {
 			AZ(vct_issp(*h));
 			/* Trim trailing space */
 			e = strchr(h, '\0');
@@ -123,7 +126,7 @@ VRY_Create(struct req *req, const struct http *hp, struct vsb **psb)
 			/* Encode two byte length and contents */
 			l = e - h;
 			if (l > 0xffff - 1) {
-				VSLb(req->vsl, SLT_Error,
+				VSLb(bo->vsl, SLT_Error,
 				    "Vary header maximum length exceeded");
 				error = 1;
 				break;
@@ -143,7 +146,7 @@ VRY_Create(struct req *req, const struct http *hp, struct vsb **psb)
 		if (*q == '\0')
 			break;
 		if (*q != ',') {
-			VSLb(req->vsl, SLT_Error, "Malformed Vary header");
+			VSLb(bo->vsl, SLT_Error, "Malformed Vary header");
 			error = 1;
 			break;
 		}



From phk at varnish-cache.org  Tue May 14 09:10:05 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 14 May 2013 11:10:05 +0200
Subject: [master] 2165342 Add panic dump facility for threads which have a
 busyobj rather than a req.
Message-ID: <E1UcBFF-0004hB-OY@project.varnish-software.com>

commit 21653426c21010ac5c226f50940d8afca9032962
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 14 09:09:33 2013 +0000

    Add panic dump facility for threads which have a busyobj rather than a req.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index aa83773..d7f29f5 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -913,6 +913,8 @@ uint32_t VXID_Get(struct vxid_pool *v);
 extern volatile struct params * cache_param;
 void THR_SetName(const char *name);
 const char* THR_GetName(void);
+void THR_SetBusyobj(const struct busyobj *);
+struct busyobj * THR_GetBusyobj(void);
 void THR_SetRequest(const struct req *);
 struct req * THR_GetRequest(void);
 
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index dfdbfa5..0aaf8b9 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -52,6 +52,21 @@ volatile struct params	*cache_param;
  */
 
 static pthread_key_t req_key;
+static pthread_key_t bo_key;
+
+void
+THR_SetBusyobj(const struct busyobj *bo)
+{
+
+	AZ(pthread_setspecific(bo_key, bo));
+}
+
+struct busyobj *
+THR_GetBusyobj(void)
+{
+
+	return (pthread_getspecific(bo_key));
+}
 
 void
 THR_SetRequest(const struct req *req)
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 14c14b3..824c53a 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -396,6 +396,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 {
 	const char *q;
 	struct req *req;
+	struct busyobj *bo;
 
 	AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released,
 						  we're going to die
@@ -446,6 +447,11 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 			pan_req(req);
 			VSL_Flush(req->vsl, 0);
 		}
+		bo = THR_GetBusyobj();
+		if (bo != NULL) {
+			pan_busyobj(bo);
+			VSL_Flush(bo->vsl, 0);
+		}
 	}
 	VSB_printf(pan_vsp, "\n");
 	VSB_bcat(pan_vsp, "", 1);	/* NUL termination */



From phk at varnish-cache.org  Tue May 14 09:48:17 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 14 May 2013 11:48:17 +0200
Subject: [master] 20f9078 Interesting: on FreeBSD uninitialized pthread keys
 default to the first one created. I would have skipped #0 if it was me...
Message-ID: <E1UcBqD-00081Z-JD@project.varnish-software.com>

commit 20f90789d4cb002260fb3e7341dd172abc68366f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 14 09:47:05 2013 +0000

    Interesting:  on FreeBSD uninitialized pthread keys default to the
    first one created.  I would have skipped #0 if it was me...

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 0aaf8b9..fbbfd2d 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -195,6 +195,7 @@ child_main(void)
 	cache_param = heritage.param;
 
 	AZ(pthread_key_create(&req_key, NULL));
+	AZ(pthread_key_create(&bo_key, NULL));
 	AZ(pthread_key_create(&name_key, NULL));
 
 	THR_SetName("cache-main");



From perbu at varnish-cache.org  Tue May 14 14:18:52 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Tue, 14 May 2013 16:18:52 +0200
Subject: [2.1] 410228c Fix ancient bug in the 2.1 doctree Fixes #1302
Message-ID: <E1UcG44-0003Vv-39@project.varnish-software.com>

commit 410228ce5572b87ec4e3f67398e29f3602497031
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue May 14 16:18:20 2013 +0200

    Fix ancient bug in the 2.1 doctree
    Fixes #1302

diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index 26b86fa..acdb7ff 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -277,7 +277,7 @@ probably confuse people if we made the default case-insentive.
 
 To make a PCRE regex case insensitive, put ``(?i)`` at the start::
 
-	if (req.http.host ~ "?iexample.com$") {
+	if (req.http.host ~ "(?i)example.com$") {
 		...
 	}
 



From perbu at varnish-cache.org  Wed May 15 11:18:57 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 15 May 2013 13:18:57 +0200
Subject: [master] bcdf4f1 Make the reason phrase optional. Basically the
 backend can drop the reason (OK) when responding to a probe without Varnish
 declaring the backend sick. Reported by astrosti on
 https://www.varnish-cache.org/forum/topic/961
Message-ID: <E1UcZjV-0006Vx-CP@project.varnish-software.com>

commit bcdf4f12891eb7ecba395bdf268f8be4c9faa255
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed May 15 13:17:09 2013 +0200

    Make the reason phrase optional.
    Basically the backend can drop the reason (OK) when responding to
    a probe without Varnish declaring the backend sick.
    Reported by astrosti on https://www.varnish-cache.org/forum/topic/961

diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 4780165..cd9a215 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -225,7 +225,7 @@ vbp_poke(struct vbp_target *vt)
 
 	i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf);
 
-	if (i == 2 && resp == vt->probe.exp_status)
+	if ((i == 1 || i == 2) && resp == vt->probe.exp_status)
 		vt->happy |= 1;
 }
 



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] eeef8c2 Add management shared memory counters. Add
 MGT.uptime management process uptime counter.
Message-ID: <E1Ucb5x-0004D4-Cv@project.varnish-software.com>

commit eeef8c2bff79590237e362b8d8dc9263854d97b8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jan 7 15:01:54 2013 +0100

    Add management shared memory counters. Add MGT.uptime management
    process uptime counter.

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 8f0fd3b..eb729de 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -30,6 +30,7 @@
 
 #include <stdint.h>
 
+#include "vapi/vsc_int.h"
 #include "common/common.h"
 
 struct cli;
@@ -62,6 +63,8 @@ void mgt_cli_secret(const char *S_arg);
 void mgt_cli_close_all(void);
 
 /* mgt_main.c */
+extern struct VSC_C_mgt	*VSC_C_mgt;
+extern struct VSC_C_mgt static_VSC_C_mgt;
 struct choice {
 	const char      *name;
 	const void	*ptr;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 6c4780d..2c971fb 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -86,6 +86,7 @@ static struct vev	*ev_listen;
 static struct vlu	*child_std_vlu;
 
 static struct vsb *child_panic = NULL;
+static double mgt_uptime_t0 = 0.;
 
 /* XXX: Doesn't really belong here, but only place we use it */
 static inline int
@@ -649,6 +650,20 @@ mgt_sigint(const struct vev *e, int what)
 	exit (2);
 }
 
+/*--------------------------------------------------------------------*/
+
+static int __match_proto__(vev_cb_f)
+mgt_uptime(const struct vev *e, int what)
+{
+
+	(void)e;
+	(void)what;
+	AN(VSC_C_mgt);
+	VSC_C_mgt->uptime = static_VSC_C_mgt.uptime =
+	    VTIM_mono() - mgt_uptime_t0;
+	return (0);
+}
+
 /*=====================================================================
  * This thread is the master thread in the management process.
  * The relatively simple task is to start and stop the child process
@@ -662,6 +677,14 @@ MGT_Run(void)
 	struct vev *e;
 	int i;
 
+	mgt_uptime_t0 = VTIM_mono();
+	e = vev_new();
+	XXXAN(e);
+	e->callback = mgt_uptime;
+	e->timeout = 1.0;
+	e->name = "mgt_uptime";
+	AZ(vev_add(mgt_evb, e));
+
 	e = vev_new();
 	XXXAN(e);
 	e->sig = SIGTERM;
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index a111481..5031eab 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -69,6 +69,8 @@ pid_t			mgt_pid;
 struct vev_base		*mgt_evb;
 int			exit_status = 0;
 struct vsb		*vident;
+struct VSC_C_mgt	static_VSC_C_mgt;
+struct VSC_C_mgt	*VSC_C_mgt;
 
 /*--------------------------------------------------------------------*/
 
@@ -330,6 +332,10 @@ main(int argc, char * const *argv)
 	char **av;
 	unsigned clilim;
 
+	/* Set up the mgt counters */
+	memset(&static_VSC_C_mgt, 0, sizeof static_VSC_C_mgt);
+	VSC_C_mgt = &static_VSC_C_mgt;
+
 	/*
 	 * Start out by closing all unwanted file descriptors we might
 	 * have inherited from sloppy process control daemons.
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 2e92df6..a0aaa9d 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -45,7 +45,6 @@
 #include "common/params.h"
 
 #include "flopen.h"
-#include "vapi/vsc_int.h"
 #include "vapi/vsm_int.h"
 #include "vmb.h"
 
@@ -240,6 +239,12 @@ mgt_SHM_Create(void)
 	    heritage.panic_str_len, PAN_CLASS, "", "");
 	AN(heritage.panic_str);
 
+	/* Copy management counters to shm and update pointer */
+	VSC_C_mgt = VSM_common_alloc(heritage.vsm,
+	    sizeof *VSC_C_mgt, VSC_CLASS, VSC_TYPE_MGT, "");
+	AN(VSC_C_mgt);
+	*VSC_C_mgt = static_VSC_C_mgt;
+
 	if (rename(fnbuf, VSM_FILENAME)) {
 		fprintf(stderr, "Rename failed %s -> %s: %s\n",
 		    fnbuf, VSM_FILENAME, strerror(errno));
@@ -261,6 +266,9 @@ void
 mgt_SHM_Destroy(int keep)
 {
 
+	/* Point mgt counters back at static version */
+	VSC_C_mgt = &static_VSC_C_mgt;
+
 	AN(heritage.vsm);
 	if (keep)
 		(void)rename(VSM_FILENAME, VSM_FILENAME ".keep");
diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h
index ae42507..e1ccb52 100644
--- a/include/tbl/vsc_all.h
+++ b/include/tbl/vsc_all.h
@@ -28,6 +28,11 @@
  */
 
 
+VSC_DO(MGT, mgt, VSC_TYPE_MGT)
+#define VSC_DO_MGT
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_MGT
+VSC_DONE(MGT, mgt, VSC_TYPE_MGT)
 
 VSC_DO(LCK, lck, VSC_TYPE_LCK)
 #define VSC_DO_LCK
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 1ab16ff..e763225 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -59,6 +59,17 @@
 
 /**********************************************************************/
 
+#ifdef VSC_DO_MGT
+
+VSC_F(uptime,			uint64_t, 0, 'c',
+    "Management process uptime",
+	"Uptime in seconds of the management process"
+)
+
+#endif
+
+/**********************************************************************/
+
 #ifdef VSC_DO_LCK
 
 VSC_F(creat,			uint64_t, 0, 'a',
diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h
index 95346e3..b98df57 100644
--- a/include/vapi/vsc_int.h
+++ b/include/vapi/vsc_int.h
@@ -30,6 +30,7 @@
 
 #define VSC_CLASS		"Stat"
 
+#define VSC_TYPE_MGT		"MGT"
 #define VSC_TYPE_MAIN		""
 #define VSC_TYPE_SMA		"SMA"
 #define VSC_TYPE_SMF		"SMF"



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] b922ac9 Add MGT.child_{start|exit|stop|died|dump|panic}
 counters
Message-ID: <E1Ucb5x-0004DA-Dy@project.varnish-software.com>

commit b922ac98ba05a56a89ff29ea3065c437c0a05940
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jan 7 16:06:39 2013 +0100

    Add MGT.child_{start|exit|stop|died|dump|panic} counters

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 2c971fb..1ae4599 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -401,6 +401,7 @@ mgt_launch_child(struct cli *cli)
 	}
 	assert(pid > 1);
 	REPORT(LOG_NOTICE, "child (%jd) Started", (intmax_t)pid);
+	VSC_C_mgt->child_start = ++static_VSC_C_mgt.child_start;
 
 	/* Close stuff the child got */
 	closex(&heritage.std_fd);
@@ -511,15 +512,21 @@ mgt_reap_child(void)
 	if (WIFEXITED(status) && WEXITSTATUS(status)) {
 		VSB_printf(vsb, " status=%d", WEXITSTATUS(status));
 		exit_status |= 0x20;
+		if (WEXITSTATUS(status) == 1)
+			VSC_C_mgt->child_exit = ++static_VSC_C_mgt.child_exit;
+		else
+			VSC_C_mgt->child_stop = ++static_VSC_C_mgt.child_stop;
 	}
 	if (WIFSIGNALED(status)) {
 		VSB_printf(vsb, " signal=%d", WTERMSIG(status));
 		exit_status |= 0x40;
+		VSC_C_mgt->child_died = ++static_VSC_C_mgt.child_died;
 	}
 #ifdef WCOREDUMP
 	if (WCOREDUMP(status)) {
 		VSB_printf(vsb, " (core dumped)");
 		exit_status |= 0x80;
+		VSC_C_mgt->child_dump = ++static_VSC_C_mgt.child_dump;
 	}
 #endif
 	AZ(VSB_finish(vsb));
@@ -530,6 +537,7 @@ mgt_reap_child(void)
 	if (heritage.panic_str[0] != '\0') {
 		mgt_panic_record(r);
 		mgt_SHM_Destroy(1);
+		VSC_C_mgt->child_panic = ++static_VSC_C_mgt.child_panic;
 	} else {
 		mgt_SHM_Destroy(0);
 	}
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index e763225..017398f 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -65,6 +65,31 @@ VSC_F(uptime,			uint64_t, 0, 'c',
     "Management process uptime",
 	"Uptime in seconds of the management process"
 )
+VSC_F(child_start,		uint64_t, 0, 'c',
+    "Child process started",
+	"Number of times the child process has been started"
+)
+VSC_F(child_exit,		uint64_t, 0, 'c',
+    "Child process normal exit",
+	"Number of times the child process has been cleanly stopped"
+)
+VSC_F(child_stop,		uint64_t, 0, 'c',
+    "Child process unexpected exit",
+	"Number of times the child process has exited with an unexpected"
+	" return code"
+)
+VSC_F(child_died,		uint64_t, 0, 'c',
+    "Child process died (signal)",
+	"Number of times the child process has died due to signals"
+)
+VSC_F(child_dump,		uint64_t, 0, 'c',
+    "Child process core dumped",
+	"Number of times the child process has produced core dumps"
+)
+VSC_F(child_panic,		uint64_t, 0, 'c',
+    "Child process panic",
+	"Number of times the management process has caught a child panic"
+)
 
 #endif
 



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] b3a461f Add SLT_Begin records at start of every new vxid
 transaction. Format: <type> <parent-vxid>
Message-ID: <E1Ucb5x-0004DG-Ez@project.varnish-software.com>

commit b3a461f3dd75353490a7c84e7142b41565f005d5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Apr 25 13:41:54 2013 +0200

    Add SLT_Begin records at start of every new vxid transaction.
    Format: <type> <parent-vxid>
    
    Add SLT_End records at the end of all vxid transactions.
    
    Add SLT_Link records whenever one initiates a new child transation.
    Format: <type-of-child> <child-vxid>
    
    This is used by VSL api to form structured log output.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 5124dc4..47e61b3 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -129,7 +129,7 @@ VBO_GetBusyObj(struct worker *wrk, struct req *req)
 	sz = cache_param->vsl_buffer;
 	VSL_Setup(bo->vsl, p, sz);
 	bo->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_BACKENDMARKER;
-	VSLb(bo->vsl, SLT_Link, "req %u", req->vsl->wid & VSL_IDENTMASK);
+	VSLb(bo->vsl, SLT_Begin, "bereq %u", req->vsl->wid & VSL_IDENTMASK);
 	VSLb(req->vsl, SLT_Link, "bereq %u", bo->vsl->wid & VSL_IDENTMASK);
 	p += sz;
 	p = (void*)PRNDUP(p);
@@ -182,6 +182,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	if (r)
 		return;
 
+	VSLb(bo->vsl, SLT_End, "%s", "");
 	VSL_Flush(bo->vsl, 0);
 
 	if (oc != NULL) {
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index d5cf9e6..041626f 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -60,7 +60,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 	wrk_ws_wm = WS_Snapshot(wrk->aws); /* XXX ? */
 
 	req = SES_GetReq(wrk, preq->sp);
-	VSLb(req->vsl, SLT_Link, "req %u", preq->vsl->wid & VSL_IDENTMASK);
+	VSLb(req->vsl, SLT_Begin, "esireq %u", preq->vsl->wid & VSL_IDENTMASK);
 	VSLb(preq->vsl, SLT_Link, "esireq %u", req->vsl->wid & VSL_IDENTMASK);
 	req->esi_level = preq->esi_level + 1;
 
@@ -114,6 +114,9 @@ ved_include(struct req *preq, const char *src, const char *host)
 		(void)usleep(10000);
 	}
 
+	VSLb(req->vsl, SLT_End, "%s", "");
+	req->vsl->wid = 0;
+
 	/* Reset the workspace */
 	WS_Reset(wrk->aws, wrk_ws_wm);	/* XXX ? */
 
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index ecc3583..90ffc21 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -252,8 +252,13 @@ http1_dissect(struct worker *wrk, struct req *req)
 	 * Cache_req_fsm zeros the vxid once a requests is processed.
 	 * Allocate a new one only now that we know will need it.
 	 */
-	if (req->vsl->wid == 0)
+	if (req->vsl->wid == 0) {
 		req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER;
+		VSLb(req->vsl, SLT_Begin, "req %u",
+		    req->sp->vxid & VSL_IDENTMASK);
+		VSL(SLT_Link, req->sp->vxid, "req %u",
+		    req->vsl->wid & VSL_IDENTMASK);
+	}
 
 	/* Borrow VCL reference from worker thread */
 	VCL_Refresh(&wrk->vcl);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index a1baa0e..07d0742 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -1349,6 +1349,7 @@ CNT_Request(struct worker *wrk, struct req *req)
 		 * Nuke the VXID, cache_http1_fsm.c::http1_dissect() will
 		 * allocate a new one when necessary.
 		 */
+		VSLb(req->vsl, SLT_End, "%s", "");
 		req->vsl->wid = 0;
 	}
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1630007..f117459 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -195,6 +195,7 @@ ses_vsl_socket(struct sess *sp, const char *lsockname)
 		strcpy(laddr, "-");
 		strcpy(lport, "-");
 	}
+	VSL(SLT_Begin, sp->vxid, "sess");
 	VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d",
 	    sp->addr, sp->port, lsockname, laddr, lport, sp->t_open, sp->fd);
 }
@@ -329,6 +330,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f %ju %ju %ju %ju %ju %ju",
 	    sess_close_2str(sp->reason, 0), now - sp->t_open, b->req,
 	    b->pipe, b->pass, b->fetch, b->hdrbytes, b->bodybytes);
+	VSL(SLT_End, sp->vxid, "%s", "");
 
 	MPL_Free(pp->mpl_sess, sp);
 }
@@ -383,7 +385,8 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 	sz = cache_param->workspace_thread;
 	VSL_Setup(req->vsl, p, sz);
 	req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER;
-	VSLb(req->vsl, SLT_Link, "sess %u", sp->vxid & VSL_IDENTMASK);
+	VSLb(req->vsl, SLT_Begin, "req %u", sp->vxid & VSL_IDENTMASK);
+	VSL(SLT_Link, req->sp->vxid, "req %u", req->vsl->wid & VSL_IDENTMASK);
 	p += sz;
 	p = (void*)PRNDUP(p);
 
@@ -417,6 +420,9 @@ SES_ReleaseReq(struct req *req)
 	AN(pp->pool);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	MPL_AssertSane(req);
+	if (req->vsl->wid != 0)
+		/* Non-released VXID - assume it was from a req */
+		VSLb(req->vsl, SLT_End, "%s", "");
 	VSL_Flush(req->vsl, 0);
 	req->sp = NULL;
 	MPL_Free(pp->mpl_req, req);
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 3f43bc1..25ba677 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -161,12 +161,26 @@ SLTM(VCL_Error, "", "")
 
 SLTM(Gzip, "G(un)zip performed on object", "")
 
-SLTM(Link, "Linkage between different VXIDs",
-	"Links this records VXID to its parent VXID\n"
-	"The first field gives the type of the parent:\n"
+SLTM(Link, "Links to a child VXID",
+	"Links this VXID to any child VXID it initiates\n"
+	"The first field gives the type of the child:\n"
 	"    req     Request\n"
-	"    sess    Session\n"
 	"    bereq   Backend request\n"
 	"    esireq  ESI subrequest\n"
-	"The second field gives the VXID if the parent.\n"
+	"The second field gives the VXID of the child.\n"
+)
+
+SLTM(Begin, "Marks the start of a VXID",
+    "The first record of a VXID transaction.\n"
+    "The first field gives the type of the transaction:\n"
+    "    sess	Session\n"
+    "    req	Request\n"
+    "    bereq	Backend request\n"
+    "    esireq	ESI subrequest\n"
+    "The second field gives the VXID of the parent that initiated this"
+    " transaction. For Session transactions this field is blank.\n"
+)
+
+SLTM(End, "Marks the end of a VXID",
+    "The last record of a VXID transaction.\n"
 )



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] d9f7cd5 Keep an age counter in VSM_Head that will show the
 age of this SHM segment.
Message-ID: <E1Ucb5x-0004DO-GD@project.varnish-software.com>

commit d9f7cd52dcce9bdb20ec5fe00b23f75f7b7f416e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Jan 9 11:17:21 2013 +0100

    Keep an age counter in VSM_Head that will show the age of this SHM
    segment.
    
    The API will use this counter to check for abandonment.
    
    Make use of the SHM age attribute in VSM_Abandoned.
    
    This gives a less expensive way (without doing stat syscall) to check
    for abandonment. If there's been more than 2 seconds between calls to
    VSM_Abandoned and the age attribute hasn't changed in between, the
    stat on the file will be performed to see if the SHM file has changed.
    
    VSM_Abandoned will be the standard way to check for SHM changes in API
    utilities, and when it returns true, the application is supposed to
    call VSM_Close and then VSM_Open to reopen the SHM file.
    
    Update Makefile.am in various places for build

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index 7941eb2..22825cd 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -17,7 +17,7 @@ varnishadm_CFLAGS = @LIBEDIT_CFLAGS@
 varnishadm_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
-	${PTHREAD_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM}
+	${PTHREAD_LIBS} ${RT_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM}
 
 varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 8140734..978b54d 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -115,6 +115,7 @@ void VSM_common_free(struct vsm_sc *sc, void *ptr);
 void VSM_common_delete(struct vsm_sc **sc);
 void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from);
 void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats);
+void VSM_common_ageupdate(struct vsm_sc *sc);
 
 /*---------------------------------------------------------------------
  * Generic power-2 rounding macros
diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index e53972c..6f7f5a3 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -66,6 +66,7 @@ struct vsm_sc {
 	char				*b;
 	ssize_t				len;
 	struct VSM_head			*head;
+	double				t0;
 	VTAILQ_HEAD(,vsm_range)		r_used;
 	VTAILQ_HEAD(,vsm_range)		r_cooling;
 	VTAILQ_HEAD(,vsm_range)		r_free;
@@ -139,6 +140,7 @@ VSM_common_new(void *p, ssize_t l)
 	VTAILQ_INIT(&sc->r_bogus);
 	sc->b = p;
 	sc->len = l;
+	sc->t0 = VTIM_mono();
 
 	sc->head = (void *)sc->b;
 	/* This should not be necessary, but just in case...*/
@@ -377,3 +379,15 @@ VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from)
 		memcpy(p, vr->chunk + 1, vr->chunk->len);
 	}
 }
+
+/*--------------------------------------------------------------------
+ * Update age
+ */
+
+void
+VSM_common_ageupdate(struct vsm_sc *sc)
+{
+
+	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
+	sc->head->age = VTIM_mono() - sc->t0;
+}
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 1ae4599..9397891 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -669,6 +669,8 @@ mgt_uptime(const struct vev *e, int what)
 	AN(VSC_C_mgt);
 	VSC_C_mgt->uptime = static_VSC_C_mgt.uptime =
 	    VTIM_mono() - mgt_uptime_t0;
+	if (heritage.vsm != NULL)
+		VSM_common_ageupdate(heritage.vsm);
 	return (0);
 }
 
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 3d849f6..91719e8 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -14,7 +14,7 @@ varnishhist_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	-lm \
-	${CURSES_LIBS} ${PTHREAD_LIBS}
+	${CURSES_LIBS} ${RT_LIBS} ${PTHREAD_LIBS}
 
 varnishhist.1: $(top_srcdir)/doc/sphinx/reference/varnishhist.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index 23e1fe4..f6416f7 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -17,7 +17,7 @@ varnishlog_SOURCES = \
 varnishlog_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	${PTHREAD_LIBS}
+	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
 varnishlog.1: $(top_srcdir)/doc/sphinx/reference/varnishlog.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am
index e9c30fa..7db8e89 100644
--- a/bin/varnishncsa/Makefile.am
+++ b/bin/varnishncsa/Makefile.am
@@ -19,7 +19,7 @@ varnishncsa_SOURCES = \
 varnishncsa_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	${PTHREAD_LIBS}
+	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
 varnishncsa.1: $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index df7b2b8..59c6a0d 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -15,7 +15,7 @@ varnishreplay_SOURCES = \
 varnishreplay_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	${PTHREAD_LIBS} ${NET_LIBS} ${LIBM}
+	${RT_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM}
 
 varnishreplay.1: $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index 98ea717..1a38329 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -17,7 +17,7 @@ varnishstat_SOURCES = \
 varnishstat_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	${CURSES_LIBS} ${RT_LIBS} ${PTHREAD_LIBS}
+	${CURSES_LIBS} ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
 varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index 362acf5..1a6ecc4 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -14,7 +14,7 @@ varnishtop_SOURCES = varnishtop.c \
 varnishtop_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	${CURSES_LIBS} ${PTHREAD_LIBS}
+	${CURSES_LIBS} ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
 varnishtop.1: $(top_srcdir)/doc/sphinx/reference/varnishtop.rst
 if HAVE_RST2MAN
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 800cb76..9d9ef5e 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -100,7 +100,7 @@ int VSM_Open(struct VSM_data *vd);
 	 *	<0 on failure, VSM_Error() returns diagnostic string
 	 */
 
-int VSM_Abandoned(const struct VSM_data *vd);
+int VSM_Abandoned(struct VSM_data *vd);
 	/*
 	 * Find out if the VSM file has been abandoned or closed and should
 	 * be reopened.  This function calls stat(2) and should only be
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index 7e44604..a615ffd 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -53,11 +53,12 @@
  *	If a manager is started and finds and old abandoned VSM segment
  *	it will zero the alloc_seq in it, before replacing the file.
  *
- * Subscribers will have to monitor two things to make sure they have
- * the current VSM instance:  The alloc_seq field and the dev+inode
- * of the path-name.  The former check is by far the cheaper and the
- * latter check should only be employed when lack of activity in the
- * VSM segment raises suspicion that something has happened.
+ * Subscribers will have to monitor three things to make sure they look at
+ * the right thing: The alloc_seq field, the age counter and the dev+inode
+ * of the path-name.  The former check is by far the cheaper, the second
+ * can be used to check that Varnishd is still alive and the last check
+ * should only be employed when lack of activity in the VSM segment raises
+ * suspicion that something has happened.
  *
  * The allocations ("chunks") in the VSM forms a linked list, starting with
  * VSM_head->first, with the first/next fields being byte offsets relative
@@ -114,6 +115,7 @@ struct VSM_head {
 	ssize_t			shm_size;
 	ssize_t			first;		/* Offset, first chunk */
 	unsigned		alloc_seq;
+	uint64_t		age;
 };
 
 #endif /* VSM_INT_H_INCLUDED */
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 181d1c1..2e5de91 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -22,6 +22,7 @@ libvarnishapi_la_SOURCES = \
 	../libvarnish/vmb.c \
 	../libvarnish/vre.c \
 	../libvarnish/vsb.c \
+	../libvarnish/vtim.c \
 	../libvarnish/vsha256.c \
 	vsm.c \
 	vsl_arg.c \
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 5640868..272cbd6 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -84,3 +84,10 @@ LIBVARNISHAPI_1.2 {
 	VSM_Get;
 	# Variables:
 } LIBVARNISHAPI_1.0;
+
+LIBVARNISHAPI_1.3 {
+  global:
+	# Functions:
+	VSM_Abandoned;
+	# Variables:
+} LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 08d8eb6..ff42be5 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -48,6 +48,7 @@
 
 #include "vapi/vsm.h"
 #include "vapi/vsm_int.h"
+#include "vtim.h"
 #include "vin.h"
 #include "vsb.h"
 #include "vsm_api.h"
@@ -219,6 +220,8 @@ VSM_Open(struct VSM_data *vd)
 	vd->head = v;
 	vd->b = v;
 	vd->e = vd->b + slh.shm_size;
+	vd->age_ok = vd->head->age;
+	vd->t_ok = VTIM_mono();
 
 	return (0);
 }
@@ -246,23 +249,37 @@ VSM_Close(struct VSM_data *vd)
 /*--------------------------------------------------------------------*/
 
 int
-VSM_Abandoned(const struct VSM_data *vd)
+VSM_Abandoned(struct VSM_data *vd)
 {
 	struct stat st;
+	double now;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
 	if (vd->head == NULL)
+		/* Not open */
 		return (1);
-
 	if (!vd->head->alloc_seq)
+		/* Flag of abandonment set by mgt */
 		return (1);
-	if (!stat(vd->fname, &st))
-		return (1);
-	if (st.st_dev != vd->fstat.st_dev)
-		return (1);
-	if (st.st_ino != vd->fstat.st_ino)
+	if (vd->head->age < vd->age_ok)
+		/* Age going backwards */
 		return (1);
+	now = VTIM_mono();
+	if (vd->head->age == vd->age_ok && now - vd->t_ok > 2.) {
+		/* No age change for 2 seconds, stat the file */
+		if (!stat(vd->fname, &st))
+			return (1);
+		if (st.st_dev != vd->fstat.st_dev)
+			return (1);
+		if (st.st_ino != vd->fstat.st_ino)
+			return (1);
+		vd->t_ok = now;
+	} else if (vd->head->age > vd->age_ok) {
+		/* It is aging, update timestamps */
+		vd->t_ok = now;
+		vd->age_ok = vd->head->age;
+	}
 	return (0);
 }
 
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index af99798..2942494 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -47,6 +47,9 @@ struct VSM_data {
 	char			*b;
 	char			*e;
 
+	uint64_t		age_ok;
+	double			t_ok;
+
 	struct vsc		*vsc;
 	struct vsl		*vsl;
 };



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 162a06e Disable building while api is in flux
Message-ID: <E1Ucb5x-0004Dd-Hm@project.varnish-software.com>

commit 162a06eac5d9bc233bd003759538aad5156f0709
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 29 14:10:08 2013 +0200

    Disable building while api is in flux

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 9906c5e..3465413 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -1,7 +1,10 @@
 #
 
-SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
+# Disabling building of the tools while api is in flux
+
+#SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
+SUBDIRS = varnishd
 
 if HAVE_CURSES
-SUBDIRS += varnishhist varnishstat varnishtop
+#SUBDIRS += varnishhist varnishstat varnishtop
 endif



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 23f828a Various VSM changes
Message-ID: <E1Ucb5x-0004Dn-JA@project.varnish-software.com>

commit 23f828a74b9d1e31dc6b245952f4d28fe1912bc2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Oct 19 10:55:21 2012 +0200

    Various VSM changes
    
    Rename VSM_FOREACH_SAFE as it isn't really safe in the vqueue sense,
    and removing chunks from the VSM from the client library isn't
    applicable anyway.
    
    Make the VSM chunk marker sizes defines.
    
    Check for shm open in VSM__itern
    
    Revisit logic of VSM__itern: Make sure that the VSM_fantom is either
    changed to the next, or not changed at all.
    
    Add comments.
    
    Keep copies of class, type and ident in VSM_fantom, and use these when
    revalidating fantoms.
    
    Do a proper VSM_Open in varnishstat
    
    Add a VSM_ResetError function
    
    Differentiate between missing HEAD marker and zero alloc sequence. The
    latter is an indication Varnish isn't running, the former a corrupt
    file.
    
    Fix newline on diag messages for consistency.
    
    Fix return value check inversion bug on stat call in VSM_Abandoned()
    
    Don't assert on VSM_Open on an already open VSM. Report success instead.

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 47cee40..6adae1d 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -290,6 +290,10 @@ main(int argc, char * const *argv)
 		}
 	}
 
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
+		exit(1);
+	}
 	VSC_C_main = VSC_Main(vd);
 	AN(VSC_C_main);
 
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 9d9ef5e..c405168 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -35,6 +35,8 @@
 #ifndef VAPI_VSM_H_INCLUDED
 #define VAPI_VSM_H_INCLUDED
 
+#include "vsm_int.h"
+
 struct VSM_chunk;
 struct VSM_data;
 
@@ -47,6 +49,9 @@ struct VSM_fantom {
 	void			*b;		/* first byte of payload */
 	void			*e;		/* first byte past payload */
 	uintptr_t		priv;		/* VSM private */
+	char			class[VSM_MARKER_LEN];
+	char			type[VSM_MARKER_LEN];
+	char			ident[VSM_IDENT_LEN];
 };
 
 /*---------------------------------------------------------------------
@@ -75,6 +80,11 @@ const char *VSM_Error(const struct VSM_data *vd);
 	 * Return the latest error message.
 	 */
 
+void VSM_ResetError(struct VSM_data *vd);
+	/*
+	 * Reset any error message.
+	 */
+
 #define VSM_n_USAGE	"[-n varnish_name]"
 
 int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
@@ -94,9 +104,9 @@ const char *VSM_Name(const struct VSM_data *vd);
 int VSM_Open(struct VSM_data *vd);
 	/*
 	 * Attempt to open and map the VSM file.
-	 * If diag is non-zero, diagnostics are emitted.
+	 *
 	 * Returns:
-	 *	0 on success
+	 *	0 on success, or the VSM log was already open
 	 *	<0 on failure, VSM_Error() returns diagnostic string
 	 */
 
@@ -121,7 +131,7 @@ void VSM_Close(struct VSM_data *vd);
 void VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf);
 int VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf);
 
-#define VSM_FOREACH_SAFE(vf, vd) \
+#define VSM_FOREACH(vf, vd) \
     for(VSM__iter0((vd), (vf)); VSM__itern((vd), (vf));)
 	/*
 	 * Iterate over all chunks in shared memory
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index a615ffd..2d1eab6 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -97,20 +97,22 @@
 #define VSM_INT_H_INCLUDED
 
 #define VSM_FILENAME		"_.vsm"
+#define VSM_MARKER_LEN	8
+#define VSM_IDENT_LEN	128
 
 struct VSM_chunk {
 #define VSM_CHUNK_MARKER	"VSMCHUNK"
-	char			marker[8];
+	char			marker[VSM_MARKER_LEN];
 	ssize_t			len;		/* Incl VSM_chunk */
 	ssize_t			next;		/* Offset in shmem */
-	char			class[8];
-	char			type[8];
-	char			ident[128];
+	char			class[VSM_MARKER_LEN];
+	char			type[VSM_MARKER_LEN];
+	char			ident[VSM_IDENT_LEN];
 };
 
 struct VSM_head {
 #define VSM_HEAD_MARKER		"VSMHEAD0"	/* Incr. as version# */
-	char			marker[8];
+	char			marker[VSM_MARKER_LEN];
 	ssize_t			hdrsize;
 	ssize_t			shm_size;
 	ssize_t			first;		/* Offset, first chunk */
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 272cbd6..aef4221 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -89,5 +89,6 @@ LIBVARNISHAPI_1.3 {
   global:
 	# Functions:
 	VSM_Abandoned;
+	VSM_ResetError;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 67f0690..bf63816 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -297,7 +297,7 @@ vsc_build_pt_list(struct VSM_data *vd)
 
 	vsc_delete_pts(vsc);
 
-	VSM_FOREACH_SAFE(&vf, vd) {
+	VSM_FOREACH(&vf, vd) {
 		if (strcmp(vf.chunk->class, VSC_CLASS))
 			continue;
 		/*lint -save -e525 -e539 */
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index ff42be5..a957293 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -111,6 +111,20 @@ VSM_Error(const struct VSM_data *vd)
 
 /*--------------------------------------------------------------------*/
 
+void
+VSM_ResetError(struct VSM_data *vd)
+{
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+
+	if (vd->diag == NULL)
+		return;
+	VSB_delete(vd->diag);
+	vd->diag = NULL;
+}
+
+/*--------------------------------------------------------------------*/
+
 int
 VSM_n_Arg(struct VSM_data *vd, const char *opt)
 {
@@ -174,7 +188,10 @@ VSM_Open(struct VSM_data *vd)
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
-	AZ(vd->head);
+	if (vd->head != NULL)
+		/* Already open */
+		return (0);
+
 	if (!vd->n_opt)
 		(void)VSM_n_Arg(vd, "");
 
@@ -202,13 +219,20 @@ VSM_Open(struct VSM_data *vd)
 		    vd->fname, strerror(errno)));
 	}
 
-	if (memcmp(slh.marker, VSM_HEAD_MARKER, sizeof slh.marker) ||
-	    slh.alloc_seq == 0) {
+	if (memcmp(slh.marker, VSM_HEAD_MARKER, sizeof slh.marker)) {
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
 		return (vsm_diag(vd, "Not a VSM file %s\n", vd->fname));
 	}
 
+	if (slh.alloc_seq == 0) {
+		AZ(close(vd->vsm_fd));
+		vd->vsm_fd = -1;
+		return (vsm_diag(vd,
+			"Abandoned VSM file (Varnish not running?) %s\n",
+			vd->fname));
+	}
+
 	v = mmap(NULL, slh.shm_size,
 	    PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsm_fd, 0);
 	if (v == MAP_FAILED) {
@@ -268,7 +292,7 @@ VSM_Abandoned(struct VSM_data *vd)
 	now = VTIM_mono();
 	if (vd->head->age == vd->age_ok && now - vd->t_ok > 2.) {
 		/* No age change for 2 seconds, stat the file */
-		if (!stat(vd->fname, &st))
+		if (stat(vd->fname, &st))
 			return (1);
 		if (st.st_dev != vd->fstat.st_dev)
 			return (1);
@@ -295,46 +319,50 @@ VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf)
 	memset(vf, 0, sizeof *vf);
 }
 
-/* XXX: revisit, logic is unclear */
 int
 VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
-	void *p;
+	struct VSM_chunk *c = NULL;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vf);
 
+	if (!vd->head)
+		return (0);	/* Not open */
 	if (vd->head->alloc_seq == 0)
 		return (0);	/* abandoned VSM */
 	else if (vf->priv != 0) {
+		/* get next chunk */
 		if (vf->priv != vd->head->alloc_seq)
-			return (0);
+			return (0); /* changes during iteration */
 		if (vf->chunk->len == 0)
-			return (0);
+			return (0); /* free'd during iteration */
 		if (vf->chunk->next == 0)
-			return (0);
-		p = (void*)(vd->b + vf->chunk->next);
-		assert(p != vf->chunk);
-		vf->chunk = p;
+			return (0); /* last */
+		c = (struct VSM_chunk *)(vd->b + vf->chunk->next);
+		assert(c != vf->chunk);
 	} else if (vd->head->first == 0) {
-		return (0);
+		return (0);	/* empty vsm */
 	} else {
+		/* get first chunk */
 		AZ(vf->chunk);
-		vf->chunk = (void*)(vd->b + vd->head->first);
+		c = (struct VSM_chunk *)(vd->b + vd->head->first);
 	}
-	if (memcmp(vf->chunk->marker, VSM_CHUNK_MARKER,
-	    sizeof vf->chunk->marker))
-		return (0);
+	AN(c);
+	if (memcmp(c->marker, VSM_CHUNK_MARKER, sizeof c->marker))
+		return (0);	/* XXX - assert? */
+
+	vf->chunk = c;
 	vf->priv = vd->head->alloc_seq;
 	vf->b = (void*)(vf->chunk + 1);
 	vf->e = (char*)vf->b + vf->chunk->len;
+	strncpy(vf->class, vf->chunk->class, sizeof vf->class);
+	vf->class[sizeof vf->class - 1] = '\0';
+	strncpy(vf->type, vf->chunk->type, sizeof vf->type);
+	vf->type[sizeof vf->type - 1] = '\0';
+	strncpy(vf->ident, vf->chunk->ident, sizeof vf->ident);
+	vf->ident[sizeof vf->ident - 1] = '\0';
 
-	if (vf->priv == 0)
-		return (0);	/* abandoned VSM */
-	if (vf->b == vf->e)
-		return (0);	/* freed chunk */
-	AN(vf->priv);
-	AN(vf->chunk);
 	return (1);
 }
 
@@ -353,11 +381,17 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 		return (0);
 	if (vf->priv == vd->head->alloc_seq)
 		return (1);
-	VSM_FOREACH_SAFE(&f2, vd) {
-		if (f2.chunk == vf->chunk && f2.b == vf->b && f2.e == vf->e) {
-			vf->priv = vd->head->alloc_seq;
-			return (2);
-		}
+	VSM_FOREACH(&f2, vd) {
+		if (f2.chunk != vf->chunk || f2.b != vf->b || f2.e != vf->e)
+			continue;
+		if (strcmp(f2.class, vf->class))
+			continue;
+		if (strcmp(f2.type, vf->type))
+			continue;
+		if (strcmp(f2.ident, vf->ident))
+			continue;
+		vf->priv = vd->head->alloc_seq;
+		return (2);
 	}
 	return (0);
 }
@@ -368,12 +402,12 @@ VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	VSM_FOREACH_SAFE(vf, vd) {
-		if (strcmp(vf->chunk->class, class))
+	VSM_FOREACH(vf, vd) {
+		if (strcmp(vf->class, class))
 			continue;
-		if (type != NULL && strcmp(vf->chunk->type, type))
+		if (type != NULL && strcmp(vf->type, type))
 			continue;
-		if (ident != NULL && strcmp(vf->chunk->ident, ident))
+		if (ident != NULL && strcmp(vf->ident, ident))
 			continue;
 		return (1);
 	}



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 3a54afe Various VSC/varnishstat changes
Message-ID: <E1Ucb5x-0004Dy-Ke@project.varnish-software.com>

commit 3a54afe5762a1d07ce886b15eed04a797bd1e3b3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jan 21 16:02:04 2013 +0100

    Various VSC/varnishstat changes
    
    Don't do automagic reopening in VSC
    
    Rename vf variables to fantom
    
    Keep a list of fantoms for the points, and point the points to
    these. This gets rid of the fantom struct on each point, and allows
    for StillValid checking on groups at a time instead for each point.
    
    Rename point's class to type to be consistent with VSM terminology
    
    More class->type renames
    
    Drop the type and ident pointers from VSC_point, and use the fantom's
    pointers instead.
    
    Reenable building of varnishstat
    
    Don't assert on VSM file closed in VSC_Iter. It's safe to proceed
    anyways and will produce a zero result in that case. This makes the
    function consistent with how the other functions behave.

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 3465413..4adc0bb 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -7,4 +7,5 @@ SUBDIRS = varnishd
 
 if HAVE_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop
+SUBDIRS += varnishstat
 endif
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 6adae1d..df49caa 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -58,10 +58,10 @@ do_xml_cb(void *priv, const struct VSC_point * const pt)
 	val = *(const volatile uint64_t*)pt->ptr;
 
 	printf("\t<stat>\n");
-	if (strcmp(pt->class, ""))
-		printf("\t\t<type>%s</type>\n", pt->class);
-	if (strcmp(pt->ident, ""))
-		printf("\t\t<ident>%s</ident>\n", pt->ident);
+	if (strcmp(pt->fantom->type, ""))
+		printf("\t\t<type>%s</type>\n", pt->fantom->type);
+	if (strcmp(pt->fantom->ident, ""))
+		printf("\t\t<ident>%s</ident>\n", pt->fantom->ident);
 	printf("\t\t<name>%s</name>\n", pt->desc->name);
 	printf("\t\t<value>%ju</value>\n", val);
 	printf("\t\t<flag>%c</flag>\n", pt->desc->flag);
@@ -104,14 +104,16 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 
 	printf("\t\"");
 	/* build the JSON key name.  */
-	if (pt->class[0])
-		printf("%s.", pt->class);
-	if (pt->ident[0])
-		printf("%s.", pt->ident);
+	if (pt->fantom->type[0])
+		printf("%s.", pt->fantom->type);
+	if (pt->fantom->ident[0])
+		printf("%s.", pt->fantom->ident);
 	printf("%s\": {", pt->desc->name);
 
-	if (strcmp(pt->class, "")) printf("\"type\": \"%s\", ",  pt->class);
-	if (strcmp(pt->ident, "")) printf("\"ident\": \"%s\", ", pt->ident);
+	if (strcmp(pt->fantom->type, "")) printf("\"type\": \"%s\", ",
+	    pt->fantom->type);
+	if (strcmp(pt->fantom->ident, "")) printf("\"ident\": \"%s\", ",
+	    pt->fantom->ident);
 
 	printf("\"value\": %ju, ", val);
 
@@ -163,10 +165,10 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 	i = 0;
-	if (strcmp(pt->class, ""))
-		i += printf("%s.", pt->class);
-	if (strcmp(pt->ident, ""))
-		i += printf("%s.", pt->ident);
+	if (strcmp(pt->fantom->type, ""))
+		i += printf("%s.", pt->fantom->type);
+	if (strcmp(pt->fantom->ident, ""))
+		i += printf("%s.", pt->fantom->ident);
 	i += printf("%s", pt->desc->name);
 	if (i >= op->pad)
 		op->pad = i + 1;
@@ -199,10 +201,10 @@ do_list_cb(void *priv, const struct VSC_point * const pt)
 
 	(void)priv;
 	i = 0;
-	if (strcmp(pt->class, ""))
-		i += fprintf(stderr, "%s.", pt->class);
-	if (strcmp(pt->ident, ""))
-		i += fprintf(stderr, "%s.", pt->ident);
+	if (strcmp(pt->fantom->type, ""))
+		i += fprintf(stderr, "%s.", pt->fantom->type);
+	if (strcmp(pt->fantom->ident, ""))
+		i += fprintf(stderr, "%s.", pt->fantom->ident);
 	i += fprintf(stderr, "%s", pt->desc->name);
 	if (i < 30)
 		fprintf(stderr, "%*s", i - 30, "");
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 5997096..f74c33e 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -94,12 +94,12 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 	pt->flag = sp->desc->flag;
 
 	*buf = '\0';
-	if (strcmp(sp->class, "")) {
-		strcat(buf, sp->class);
+	if (strcmp(sp->fantom->type, "")) {
+		strcat(buf, sp->fantom->type);
 		strcat(buf, ".");
 	}
-	if (strcmp(sp->ident, "")) {
-		strcat(buf, sp->ident);
+	if (strcmp(sp->fantom->ident, "")) {
+		strcat(buf, sp->fantom->ident);
 		strcat(buf, ".");
 	}
 	strcat(buf, sp->desc->name);
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 16a1583..d21cb43 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -74,11 +74,9 @@ struct VSC_desc {
 };
 
 struct VSC_point {
-	const char *class;		/* stat struct type		*/
-	const char *ident;		/* stat struct ident		*/
 	const struct VSC_desc *desc;	/* point description		*/
 	const volatile void *ptr;	/* field value			*/
-	struct VSM_fantom *vf;
+	struct VSM_fantom *fantom;
 };
 
 typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index bf63816..31b297f 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -48,11 +48,17 @@
 #include "vqueue.h"
 #include "vsm_api.h"
 
+struct vsc_vf {
+	unsigned		magic;
+#define VSC_VF_MAGIC		0x516519f8
+	VTAILQ_ENTRY(vsc_vf)	list;
+	struct VSM_fantom	fantom;
+};
+
 struct vsc_pt {
 	unsigned		magic;
 #define VSC_PT_MAGIC		0xa4ff159a
 	VTAILQ_ENTRY(vsc_pt)	list;
-	struct VSM_fantom	vf;
 	struct VSC_point	point;
 };
 
@@ -62,10 +68,10 @@ struct vsc_sf {
 	VTAILQ_ENTRY(vsc_sf)	list;
 	int			flags;
 #define VSC_SF_EXCL		(1 << 0)
-#define VSC_SF_CL_WC		(1 << 1)
+#define VSC_SF_TY_WC		(1 << 1)
 #define VSC_SF_ID_WC		(1 << 2)
 #define VSC_SF_NM_WC		(1 << 3)
-	char			*class;
+	char			*type;
 	char			*ident;
 	char			*name;
 };
@@ -74,6 +80,7 @@ struct vsc {
 	unsigned		magic;
 #define VSC_MAGIC		0x3373554a
 
+	VTAILQ_HEAD(, vsc_vf)	vf_list;
 	VTAILQ_HEAD(, vsc_pt)	pt_list;
 	VTAILQ_HEAD(, vsc_sf)	sf_list;
 	struct VSM_fantom	main_fantom;
@@ -90,8 +97,9 @@ vsc_setup(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	if (vd->vsc == NULL) {
 		ALLOC_OBJ(vd->vsc, VSC_MAGIC);
-		VTAILQ_INIT(&vd->vsc->sf_list);
+		VTAILQ_INIT(&vd->vsc->vf_list);
 		VTAILQ_INIT(&vd->vsc->pt_list);
+		VTAILQ_INIT(&vd->vsc->sf_list);
 	}
 	CHECK_OBJ_NOTNULL(vd->vsc, VSC_MAGIC);
 	return (vd->vsc);
@@ -102,13 +110,19 @@ vsc_setup(struct VSM_data *vd)
 static void
 vsc_delete_pts(struct vsc *vsc)
 {
+	struct vsc_vf *vf;
 	struct vsc_pt *pt;
 
-	while(!VTAILQ_EMPTY(&vsc->pt_list)) {
+	while (!VTAILQ_EMPTY(&vsc->pt_list)) {
 		pt = VTAILQ_FIRST(&vsc->pt_list);
 		VTAILQ_REMOVE(&vsc->pt_list, pt, list);
 		FREE_OBJ(pt);
 	}
+	while (!VTAILQ_EMPTY(&vsc->vf_list)) {
+		vf = VTAILQ_FIRST(&vsc->vf_list);
+		VTAILQ_REMOVE(&vsc->vf_list, vf, list);
+		FREE_OBJ(vf);
+	}
 }
 
 static void
@@ -116,10 +130,10 @@ vsc_delete_sfs(struct vsc *vsc)
 {
 	struct vsc_sf *sf;
 
-	while(!VTAILQ_EMPTY(&vsc->sf_list)) {
+	while (!VTAILQ_EMPTY(&vsc->sf_list)) {
 		sf = VTAILQ_FIRST(&vsc->sf_list);
 		VTAILQ_REMOVE(&vsc->sf_list, sf, list);
-		free(sf->class);
+		free(sf->type);
 		free(sf->ident);
 		free(sf->name);
 		FREE_OBJ(sf);
@@ -169,7 +183,7 @@ vsc_f_arg(struct VSM_data *vd, const char *opt)
 		if (q != NULL) {
 			*q++ = '\0';
 			if (*p != '\0')
-				REPLACE(sf->class, p);
+				REPLACE(sf->type, p);
 			p = q;
 			if (*p != '\0') {
 				q = strchr(p, '.');
@@ -186,11 +200,11 @@ vsc_f_arg(struct VSM_data *vd, const char *opt)
 		}
 
 		/* Check for wildcards */
-		if (sf->class != NULL) {
-			q = strchr(sf->class, '*');
+		if (sf->type != NULL) {
+			q = strchr(sf->type, '*');
 			if (q != NULL && q[1] == '\0') {
 				*q = '\0';
-				sf->flags |= VSC_SF_CL_WC;
+				sf->flags |= VSC_SF_TY_WC;
 			}
 		}
 		if (sf->ident != NULL) {
@@ -233,9 +247,8 @@ VSC_Main(struct VSM_data *vd)
 {
 	struct vsc *vsc = vsc_setup(vd);
 
-	if (!vd->head && VSM_Open(vd))
-		return (NULL);
-	if (!VSM_Get(vd, &vsc->main_fantom, VSC_CLASS, "", ""))
+	if (!VSM_StillValid(vd, &vsc->main_fantom) &&
+	    !VSM_Get(vd, &vsc->main_fantom, VSC_CLASS, "", ""))
 		return (NULL);
 	return ((void*)vsc->main_fantom.b);
 }
@@ -244,37 +257,42 @@ VSC_Main(struct VSM_data *vd)
  */
 
 static void
-vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
-    const struct VSC_desc *desc, const volatile void *ptr,
-    const struct VSM_fantom *vf)
+vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom)
+{
+	struct vsc_vf *vf;
+
+	ALLOC_OBJ(vf, VSC_VF_MAGIC);
+	AN(vf);
+	vf->fantom = *fantom;
+	VTAILQ_INSERT_TAIL(&vsc->vf_list, vf, list);
+}
+
+static void
+vsc_add_pt(struct vsc *vsc, const struct VSC_desc *desc,
+    const volatile void *ptr, struct VSM_fantom *fantom)
 {
 	struct vsc_pt *pt;
 
 	ALLOC_OBJ(pt, VSC_PT_MAGIC);
 	AN(pt);
-	pt->point.class = class;
-	pt->point.ident = ident;
 	pt->point.desc = desc;
 	pt->point.ptr = ptr;
-	pt->point.vf = &pt->vf;
-	pt->vf = *vf;
+	pt->point.fantom = fantom;
 	VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
 }
 
 #define VSC_DO(U,l,t)							\
 	static void							\
-	iter_##l(struct vsc *vsc, struct VSM_fantom *vf,		\
-	    const struct VSC_desc *descs)				\
+	iter_##l(struct vsc *vsc, const struct VSC_desc *descs,		\
+	    struct VSM_fantom *fantom)					\
 	{								\
 		struct VSC_C_##l *st;					\
-		const char *class = t;					\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
-		st = vf->b;						\
+		st = fantom->b;
 
 #define VSC_F(nn,tt,ll,ff,dd,ee)					\
-		vsc_add_pt(vsc, class, vf->chunk->ident, descs++,	\
-		    &st->nn, vf);
+		vsc_add_pt(vsc, descs++, &st->nn, fantom);
 
 #define VSC_DONE(U,l,t)							\
 	}
@@ -293,19 +311,24 @@ static void
 vsc_build_pt_list(struct VSM_data *vd)
 {
 	struct vsc *vsc = vsc_setup(vd);
-	struct VSM_fantom vf;
+	struct VSM_fantom fantom;
+	struct vsc_vf *vf;
 
 	vsc_delete_pts(vsc);
 
-	VSM_FOREACH(&vf, vd) {
-		if (strcmp(vf.chunk->class, VSC_CLASS))
+	VSM_FOREACH(&fantom, vd) {
+		if (strcmp(fantom.class, VSC_CLASS))
 			continue;
+		vsc_add_vf(vsc, &fantom);
+	}
+
+	VTAILQ_FOREACH(vf, &vsc->vf_list, list) {
 		/*lint -save -e525 -e539 */
 #define VSC_F(n,t,l,f,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
-		if (!strcmp(vf.chunk->type, t))			\
-			iter_##l(vsc, &vf, VSC_desc_##l);
+		if (!strcmp(vf->fantom.type, t))		\
+			iter_##l(vsc, VSC_desc_##l, &vf->fantom);
 #include "tbl/vsc_all.h"
 #undef VSC_F
 #undef VSC_DO
@@ -344,10 +367,10 @@ vsc_filter_pt_list(struct VSM_data *vd)
 	VTAILQ_INIT(&pt_list);
 	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
 		VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) {
-			if (iter_test(sf->class, pt->point.class,
-			    sf->flags & VSC_SF_CL_WC))
+			if (iter_test(sf->type, pt->point.fantom->type,
+			    sf->flags & VSC_SF_TY_WC))
 				continue;
-			if (iter_test(sf->ident, pt->point.ident,
+			if (iter_test(sf->ident, pt->point.fantom->ident,
 			    sf->flags & VSC_SF_ID_WC))
 				continue;
 			if (iter_test(sf->name, pt->point.desc->name,
@@ -376,20 +399,11 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 	int i;
 
 	if (1 != VSM_StillValid(vd, &vsc->iter_fantom)) {
-		if (!VSM_Get(vd, &vsc->iter_fantom, VSC_CLASS, "", "")) {
-			VSM_Close(vd);
-			if (!vd->head && VSM_Open(vd))
-				return (-1);
-			if (!VSM_Get(vd, &vsc->iter_fantom, VSC_CLASS, "", ""))
-				return (-1);
-		}
-		AN(vd->head);
 		/* Tell app that list will be nuked */
 		(void)func(priv, NULL);
 		vsc_build_pt_list(vd);
 		vsc_filter_pt_list(vd);
 	}
-	AN(vd->head);
 	VTAILQ_FOREACH(pt, &vsc->pt_list, list) {
 		i = func(priv, &pt->point);
 		if (i)



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 87c20ed Make the VSC types a tbl defined list,
 with descriptions.
Message-ID: <E1Ucb5x-0004E9-Lx@project.varnish-software.com>

commit 87c20ed8a665a2eaaf2fcfca9249df61f4478a9e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 24 13:23:33 2013 +0100

    Make the VSC types a tbl defined list, with descriptions.
    
    Add global variables for the type name strings.
    
    Compile in static VSC type descriptions in libvarnishapi

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 18699e5..de6b28a 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -50,6 +50,7 @@ varnishd_SOURCES = \
 	cache/cache_wrw.c \
 	cache/cache_ws.c \
 	common/common_vsm.c \
+	common/common_vsc.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
 	hash/hash_mgt.c \
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 15e1104..5b68bfb 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -206,7 +206,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	    vb->ipv4_addr == NULL ? "" : vb->ipv4_addr,
 	    vb->ipv6_addr == NULL ? "" : vb->ipv6_addr, vb->port);
 
-	b->vsc = VSM_Alloc(sizeof *b->vsc, VSC_CLASS, VSC_TYPE_VBE, buf);
+	b->vsc = VSM_Alloc(sizeof *b->vsc, VSC_CLASS, VSC_type_vbe, buf);
 	b->vsc->vcls++;
 
 	VTAILQ_INIT(&b->connlist);
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index bd34046..37fa115 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -204,7 +204,7 @@ LCK_Init(void)
 	AZ(pthread_mutex_init(&lck_mtx, NULL));
 #define LOCK(nam)						\
 	lck_##nam = VSM_Alloc(sizeof(struct VSC_C_lck),		\
-	   VSC_CLASS, VSC_TYPE_LCK, #nam);
+	   VSC_CLASS, VSC_type_lck, #nam);
 #include "tbl/locks.h"
 #undef LOCK
 }
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 5f06750..fded87b 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -237,7 +237,7 @@ MPL_New(const char *name,
 	Lck_New(&mpl->mtx, lck_mempool);
 	/* XXX: prealloc min_pool */
 	mpl->vsc = VSM_Alloc(sizeof *mpl->vsc,
-	    VSC_CLASS, VSC_TYPE_MEMPOOL, mpl->name);
+	    VSC_CLASS, VSC_type_mempool, mpl->name);
 	AN(mpl->vsc);
 	AZ(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
 	AZ(pthread_detach(mpl->thread));
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 198e225..5dbd44f 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -378,7 +378,7 @@ VSM_Init(void)
 	vsl_ptr = vsl_start + 1;
 
 	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
-	    VSC_CLASS, VSC_TYPE_MAIN, "");
+	    VSC_CLASS, VSC_type_main, "");
 	AN(VSC_C_main);
 
 	vsl_wrap();
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
new file mode 100644
index 0000000..7078595
--- /dev/null
+++ b/bin/varnishd/common/common_vsc.c
@@ -0,0 +1,34 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* Build the static vsc type names */
+#define VSC_TYPE_F(n,t,l,e,d)		const char *VSC_type_##n = t;
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index a0aaa9d..507eb84 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -241,7 +241,7 @@ mgt_SHM_Create(void)
 
 	/* Copy management counters to shm and update pointer */
 	VSC_C_mgt = VSM_common_alloc(heritage.vsm,
-	    sizeof *VSC_C_mgt, VSC_CLASS, VSC_TYPE_MGT, "");
+	    sizeof *VSC_C_mgt, VSC_CLASS, VSC_type_mgt, "");
 	AN(VSC_C_mgt);
 	*VSC_C_mgt = static_VSC_C_mgt;
 
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 3d422fe..ac0b6c4 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -433,7 +433,7 @@ smf_open(const struct stevedore *st)
 
 	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
 	sc->stats = VSM_Alloc(sizeof *sc->stats,
-	    VSC_CLASS, VSC_TYPE_SMF, st->ident);
+	    VSC_CLASS, VSC_type_smf, st->ident);
 	Lck_New(&sc->mtx, lck_smf);
 	Lck_Lock(&sc->mtx);
 	smf_open_chunk(sc, sc->filesize, 0, &fail, &sum);
diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
index 88cb948..adb6cd2 100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -238,7 +238,7 @@ sma_open(const struct stevedore *st)
 	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
 	Lck_New(&sma_sc->sma_mtx, lck_sma);
 	sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats,
-	    VSC_CLASS, VSC_TYPE_SMA, st->ident);
+	    VSC_CLASS, VSC_type_sma, st->ident);
 	memset(sma_sc->stats, 0, sizeof *sma_sc->stats);
 	if (sma_sc->sma_max != SIZE_MAX)
 		sma_sc->stats->g_space = sma_sc->sma_max;
diff --git a/include/Makefile.am b/include/Makefile.am
index cb10bc4..6c3f687 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -18,6 +18,7 @@ nobase_pkginclude_HEADERS = \
 	tbl/vcc_types.h \
 	tbl/vcl_returns.h \
 	tbl/vrt_stv_var.h \
+	tbl/vsc_types.h \
 	tbl/vsc_all.h \
 	tbl/vsc_fields.h \
 	tbl/vsc_f_main.h \
diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h
index e1ccb52..deb5eaf 100644
--- a/include/tbl/vsc_all.h
+++ b/include/tbl/vsc_all.h
@@ -28,42 +28,42 @@
  */
 
 
-VSC_DO(MGT, mgt, VSC_TYPE_MGT)
+VSC_DO(MGT, mgt, VSC_type_mgt)
 #define VSC_DO_MGT
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_MGT
-VSC_DONE(MGT, mgt, VSC_TYPE_MGT)
+VSC_DONE(MGT, mgt, VSC_type_mgt)
 
-VSC_DO(LCK, lck, VSC_TYPE_LCK)
+VSC_DO(LCK, lck, VSC_type_lck)
 #define VSC_DO_LCK
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_LCK
-VSC_DONE(LCK, lck, VSC_TYPE_LCK)
+VSC_DONE(LCK, lck, VSC_type_lck)
 
-VSC_DO(MAIN, main, VSC_TYPE_MAIN)
+VSC_DO(MAIN, main, VSC_type_main)
 #include "tbl/vsc_f_main.h"
-VSC_DONE(MAIN, main, VSC_TYPE_MAIN)
+VSC_DONE(MAIN, main, VSC_type_main)
 
-VSC_DO(SMA, sma, VSC_TYPE_SMA)
+VSC_DO(SMA, sma, VSC_type_sma)
 #define VSC_DO_SMA
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_SMA
-VSC_DONE(SMA, sma, VSC_TYPE_SMA)
+VSC_DONE(SMA, sma, VSC_type_sma)
 
-VSC_DO(SMF, smf, VSC_TYPE_SMF)
+VSC_DO(SMF, smf, VSC_type_smf)
 #define VSC_DO_SMF
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_SMF
-VSC_DONE(SMF, smf, VSC_TYPE_SMF)
+VSC_DONE(SMF, smf, VSC_type_smf)
 
-VSC_DO(VBE, vbe, VSC_TYPE_VBE)
+VSC_DO(VBE, vbe, VSC_type_vbe)
 #define VSC_DO_VBE
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_VBE
-VSC_DONE(VBE, vbe, VSC_TYPE_VBE)
+VSC_DONE(VBE, vbe, VSC_type_vbe)
 
-VSC_DO(MEMPOOL, mempool, VSC_TYPE_MEMPOOL)
+VSC_DO(MEMPOOL, mempool, VSC_type_mempool)
 #define VSC_DO_MEMPOOL
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_MEMPOOL
-VSC_DONE(MEMPOOL, mempool, VSC_TYPE_MEMPOOL)
+VSC_DONE(MEMPOOL, mempool, VSC_type_mempool)
diff --git a/include/tbl/vsc_types.h b/include/tbl/vsc_types.h
new file mode 100644
index 0000000..0fcb0dc
--- /dev/null
+++ b/include/tbl/vsc_types.h
@@ -0,0 +1,61 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Fields (n, l, e, d):
+ *    n - Name:		Field name, in C-source
+ *    t - Type:		Type name, in shm chunk
+ *    l - Label:	Display name, in stats programs
+ *    e - Explanation:	Short description of this counter type
+ *    d - Description:	Long description of this counter type
+ *
+ */
+
+
+VSC_TYPE_F(main,	"MAIN",		"",		"Child",
+    "Child process main counters"
+)
+VSC_TYPE_F(mgt,		"MGT",		"MGT",		"Master",
+    "Management process counters"
+)
+VSC_TYPE_F(sma,		"SMA",		"SMA",		"Storage malloc",
+    "Malloc storage counters"
+)
+VSC_TYPE_F(smf,		"SMF",		"SMF",		"Storage file",
+    "File storage counters"
+)
+VSC_TYPE_F(lck,		"LCK",		"LCK",		"Lock",
+    "Mutex lock counters"
+)
+VSC_TYPE_F(mempool,	"MEMPOOL",	"MEMPOOL",	"Memory pool",
+    "Memory pool counters"
+)
+VSC_TYPE_F(vbe,		"VBE",		"VBE",		"Backend",
+    "Backend counters"
+)
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index d21cb43..6c56626 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -65,6 +65,12 @@ struct VSC_C_main *VSC_Main(struct VSM_data *vd);
 	 * returns NULL until child has been started.
 	 */
 
+struct VSC_type_desc {
+	const char *label;		/* label */
+	const char *sdesc;		/* short description */
+	const char *ldesc;		/* long description */
+};
+
 struct VSC_desc {
 	const char *name;		/* field name			*/
 	const char *fmt;		/* field format ("uint64_t")	*/
@@ -96,14 +102,20 @@ int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
 	 */
 
 /**********************************************************************
- * Precompiled VSC_desc's for all know VSCs.
+ * Precompiled VSC_type_desc's and VSC_desc's for all know VSCs.
  */
-#define VSC_F(n,t,l,f,d,e)
+
+#define VSC_TYPE_F(n,t,l,e,d) \
+	extern const struct VSC_type_desc VSC_type_desc_##n;
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
+
 #define VSC_DO(U,l,t) extern const struct VSC_desc VSC_desc_##l[];
+#define VSC_F(n,t,l,f,d,e)
 #define VSC_DONE(U,l,t)
 #include "tbl/vsc_all.h"
-#undef VSC_F
 #undef VSC_DO
+#undef VSC_F
 #undef VSC_DONE
 
 #endif /* VAPI_VSC_H_INCLUDED */
diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h
index b98df57..2109379 100644
--- a/include/vapi/vsc_int.h
+++ b/include/vapi/vsc_int.h
@@ -30,21 +30,16 @@
 
 #define VSC_CLASS		"Stat"
 
-#define VSC_TYPE_MGT		"MGT"
-#define VSC_TYPE_MAIN		""
-#define VSC_TYPE_SMA		"SMA"
-#define VSC_TYPE_SMF		"SMF"
-#define VSC_TYPE_VBE		"VBE"
-#define VSC_TYPE_LCK		"LCK"
-#define VSC_TYPE_MEMPOOL	"MEMPOOL"
-
-#define VSC_F(n, t, l, f, e, d)	t n;
-
-#define VSC_DO(u,l,t) struct VSC_C_##l {
-#define VSC_DONE(u,l,t) };
+/* Forward declare the static vsc type names */
+#define VSC_TYPE_F(n,t,l,e,d)		extern const char *VSC_type_##n;
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
 
+/* Define the vsc type structs */
+#define VSC_DO(u,l,t)			struct VSC_C_##l {
+#define VSC_F(n,t,l,f,e,d)			t n;
+#define VSC_DONE(u,l,t)			};
 #include "tbl/vsc_all.h"
-
 #undef VSC_DO
 #undef VSC_F
 #undef VSC_DONE
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 31b297f..b40e35e 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -248,7 +248,7 @@ VSC_Main(struct VSM_data *vd)
 	struct vsc *vsc = vsc_setup(vd);
 
 	if (!VSM_StillValid(vd, &vsc->main_fantom) &&
-	    !VSM_Get(vd, &vsc->main_fantom, VSC_CLASS, "", ""))
+	    !VSM_Get(vd, &vsc->main_fantom, VSC_CLASS, VSC_type_main, ""))
 		return (NULL);
 	return ((void*)vsc->main_fantom.b);
 }
@@ -416,10 +416,19 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
  * Build the static point descriptions
  */
 
-#define VSC_F(n,t,l,f,d,e)	{#n,#t,f,d,e},
-#define VSC_DO(U,l,t) const struct VSC_desc VSC_desc_##l[] = {
-#define VSC_DONE(U,l,t) };
+#define VSC_TYPE_F(n,t,l,e,d)	const char *VSC_type_##n = t;
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
+
+#define VSC_TYPE_F(n,t,l,e,d)			\
+	const struct VSC_type_desc VSC_type_desc_##n = {l,e,d};
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
+
+#define VSC_DO(U,l,t)		const struct VSC_desc VSC_desc_##l[] = {
+#define VSC_F(n,t,l,f,d,e)		{#n,#t,f,d,e},
+#define VSC_DONE(U,l,t)		};
 #include "tbl/vsc_all.h"
-#undef VSC_F
 #undef VSC_DO
+#undef VSC_F
 #undef VSC_DONE



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] bcb6b21 Add VSC_section information with VSC type
 description to VSC_point
Message-ID: <E1Ucb5x-0004EL-Nh@project.varnish-software.com>

commit bcb6b218394b4a11746df73975ae9a7318379317
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Feb 27 13:58:39 2013 +0100

    Add VSC_section information with VSC type description to VSC_point

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index df49caa..e06fe2c 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -50,18 +50,20 @@ static int
 do_xml_cb(void *priv, const struct VSC_point * const pt)
 {
 	uint64_t val;
+	const struct VSC_section *sec;
 
 	(void)priv;
 	if (pt == NULL)
 		return (0);
 	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
+	sec = pt->section;
 
 	printf("\t<stat>\n");
-	if (strcmp(pt->fantom->type, ""))
-		printf("\t\t<type>%s</type>\n", pt->fantom->type);
-	if (strcmp(pt->fantom->ident, ""))
-		printf("\t\t<ident>%s</ident>\n", pt->fantom->ident);
+	if (strcmp(sec->fantom->type, ""))
+		printf("\t\t<type>%s</type>\n", sec->fantom->type);
+	if (strcmp(sec->fantom->ident, ""))
+		printf("\t\t<ident>%s</ident>\n", sec->fantom->ident);
 	printf("\t\t<name>%s</name>\n", pt->desc->name);
 	printf("\t\t<value>%ju</value>\n", val);
 	printf("\t\t<flag>%c</flag>\n", pt->desc->flag);
@@ -92,6 +94,7 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 {
 	uint64_t val;
 	int *jp;
+	const struct VSC_section *sec;
 
 	if (pt == NULL)
 		return (0);
@@ -99,21 +102,22 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 	jp = priv;
 	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
+	sec = pt->section;
 
 	if (*jp) *jp = 0; else printf(",\n");
 
 	printf("\t\"");
 	/* build the JSON key name.  */
-	if (pt->fantom->type[0])
-		printf("%s.", pt->fantom->type);
-	if (pt->fantom->ident[0])
-		printf("%s.", pt->fantom->ident);
+	if (sec->fantom->type[0])
+		printf("%s.", sec->fantom->type);
+	if (sec->fantom->ident[0])
+		printf("%s.", sec->fantom->ident);
 	printf("%s\": {", pt->desc->name);
 
-	if (strcmp(pt->fantom->type, "")) printf("\"type\": \"%s\", ",
-	    pt->fantom->type);
-	if (strcmp(pt->fantom->ident, "")) printf("\"ident\": \"%s\", ",
-	    pt->fantom->ident);
+	if (strcmp(sec->fantom->type, "")) printf("\"type\": \"%s\", ",
+	    sec->fantom->type);
+	if (strcmp(sec->fantom->ident, "")) printf("\"ident\": \"%s\", ",
+	    sec->fantom->ident);
 
 	printf("\"value\": %ju, ", val);
 
@@ -158,17 +162,19 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 	struct once_priv *op;
 	uint64_t val;
 	int i;
+	const struct VSC_section *sec;
 
 	if (pt == NULL)
 		return (0);
 	op = priv;
 	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
+	sec = pt->section;
 	i = 0;
-	if (strcmp(pt->fantom->type, ""))
-		i += printf("%s.", pt->fantom->type);
-	if (strcmp(pt->fantom->ident, ""))
-		i += printf("%s.", pt->fantom->ident);
+	if (strcmp(sec->fantom->type, ""))
+		i += printf("%s.", sec->fantom->type);
+	if (strcmp(sec->fantom->ident, ""))
+		i += printf("%s.", sec->fantom->ident);
 	i += printf("%s", pt->desc->name);
 	if (i >= op->pad)
 		op->pad = i + 1;
@@ -198,13 +204,15 @@ static int
 do_list_cb(void *priv, const struct VSC_point * const pt)
 {
 	int i;
+	const struct VSC_section * sec;
 
 	(void)priv;
+	sec = pt->section;
 	i = 0;
-	if (strcmp(pt->fantom->type, ""))
-		i += fprintf(stderr, "%s.", pt->fantom->type);
-	if (strcmp(pt->fantom->ident, ""))
-		i += fprintf(stderr, "%s.", pt->fantom->ident);
+	if (strcmp(sec->fantom->type, ""))
+		i += fprintf(stderr, "%s.", sec->fantom->type);
+	if (strcmp(sec->fantom->ident, ""))
+		i += fprintf(stderr, "%s.", sec->fantom->ident);
 	i += fprintf(stderr, "%s", pt->desc->name);
 	if (i < 30)
 		fprintf(stderr, "%*s", i - 30, "");
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index f74c33e..f1ecb77 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -94,12 +94,12 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 	pt->flag = sp->desc->flag;
 
 	*buf = '\0';
-	if (strcmp(sp->fantom->type, "")) {
-		strcat(buf, sp->fantom->type);
+	if (strcmp(sp->section->fantom->type, "")) {
+		strcat(buf, sp->section->fantom->type);
 		strcat(buf, ".");
 	}
-	if (strcmp(sp->fantom->ident, "")) {
-		strcat(buf, sp->fantom->ident);
+	if (strcmp(sp->section->fantom->ident, "")) {
+		strcat(buf, sp->section->fantom->ident);
 		strcat(buf, ".");
 	}
 	strcat(buf, sp->desc->name);
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 6c56626..0c6b595 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -65,12 +65,23 @@ struct VSC_C_main *VSC_Main(struct VSM_data *vd);
 	 * returns NULL until child has been started.
 	 */
 
+struct VSC_type_desc;
+struct VSC_section;
+struct VSC_desc;
+struct VSC_point;
+
 struct VSC_type_desc {
 	const char *label;		/* label */
 	const char *sdesc;		/* short description */
 	const char *ldesc;		/* long description */
 };
 
+struct VSC_section {
+	const struct VSC_type_desc *desc;
+	struct VSM_fantom *fantom;
+	const struct VSC_point * const *points;
+};
+
 struct VSC_desc {
 	const char *name;		/* field name			*/
 	const char *fmt;		/* field format ("uint64_t")	*/
@@ -82,7 +93,7 @@ struct VSC_desc {
 struct VSC_point {
 	const struct VSC_desc *desc;	/* point description		*/
 	const volatile void *ptr;	/* field value			*/
-	struct VSM_fantom *fantom;
+	const struct VSC_section *section;
 };
 
 typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index b40e35e..caf80df 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -53,6 +53,7 @@ struct vsc_vf {
 #define VSC_VF_MAGIC		0x516519f8
 	VTAILQ_ENTRY(vsc_vf)	list;
 	struct VSM_fantom	fantom;
+	struct VSC_section	section;
 };
 
 struct vsc_pt {
@@ -108,30 +109,39 @@ vsc_setup(struct VSM_data *vd)
 /*--------------------------------------------------------------------*/
 
 static void
-vsc_delete_pts(struct vsc *vsc)
+vsc_delete_vf_list(struct vsc *vsc)
 {
 	struct vsc_vf *vf;
+
+	while (!VTAILQ_EMPTY(&vsc->vf_list)) {
+		vf = VTAILQ_FIRST(&vsc->vf_list);
+		CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC);
+		VTAILQ_REMOVE(&vsc->vf_list, vf, list);
+		FREE_OBJ(vf);
+	}
+}
+
+static void
+vsc_delete_pt_list(struct vsc *vsc)
+{
 	struct vsc_pt *pt;
 
 	while (!VTAILQ_EMPTY(&vsc->pt_list)) {
 		pt = VTAILQ_FIRST(&vsc->pt_list);
+		CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC);
 		VTAILQ_REMOVE(&vsc->pt_list, pt, list);
 		FREE_OBJ(pt);
 	}
-	while (!VTAILQ_EMPTY(&vsc->vf_list)) {
-		vf = VTAILQ_FIRST(&vsc->vf_list);
-		VTAILQ_REMOVE(&vsc->vf_list, vf, list);
-		FREE_OBJ(vf);
-	}
 }
 
 static void
-vsc_delete_sfs(struct vsc *vsc)
+vsc_delete_sf_list(struct vsc *vsc)
 {
 	struct vsc_sf *sf;
 
 	while (!VTAILQ_EMPTY(&vsc->sf_list)) {
 		sf = VTAILQ_FIRST(&vsc->sf_list);
+		CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC);
 		VTAILQ_REMOVE(&vsc->sf_list, sf, list);
 		free(sf->type);
 		free(sf->ident);
@@ -149,8 +159,9 @@ VSC_Delete(struct VSM_data *vd)
 	vsc = vd->vsc;
 	vd->vsc = NULL;
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-	vsc_delete_sfs(vsc);
-	vsc_delete_pts(vsc);
+	vsc_delete_sf_list(vsc);
+	vsc_delete_pt_list(vsc);
+	vsc_delete_vf_list(vsc);
 	FREE_OBJ(vsc);
 }
 
@@ -257,42 +268,48 @@ VSC_Main(struct VSM_data *vd)
  */
 
 static void
-vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom)
+vsc_add_vf(struct vsc *vsc, struct VSM_fantom *fantom,
+    const struct VSC_type_desc *desc)
 {
 	struct vsc_vf *vf;
 
 	ALLOC_OBJ(vf, VSC_VF_MAGIC);
 	AN(vf);
 	vf->fantom = *fantom;
+	vf->section.desc = desc;
+	vf->section.fantom = &vf->fantom;
+
 	VTAILQ_INSERT_TAIL(&vsc->vf_list, vf, list);
 }
 
 static void
-vsc_add_pt(struct vsc *vsc, const struct VSC_desc *desc,
-    const volatile void *ptr, struct VSM_fantom *fantom)
+vsc_add_pt(struct vsc *vsc, const volatile void *ptr,
+    const struct VSC_desc *desc, struct vsc_vf *vf)
 {
 	struct vsc_pt *pt;
 
 	ALLOC_OBJ(pt, VSC_PT_MAGIC);
 	AN(pt);
+
 	pt->point.desc = desc;
 	pt->point.ptr = ptr;
-	pt->point.fantom = fantom;
+	pt->point.section = &vf->section;
+
 	VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
 }
 
 #define VSC_DO(U,l,t)							\
 	static void							\
 	iter_##l(struct vsc *vsc, const struct VSC_desc *descs,		\
-	    struct VSM_fantom *fantom)					\
+	    struct vsc_vf *vf)						\
 	{								\
 		struct VSC_C_##l *st;					\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
-		st = fantom->b;
+		st = vf->fantom.b;
 
 #define VSC_F(nn,tt,ll,ff,dd,ee)					\
-		vsc_add_pt(vsc, descs++, &st->nn, fantom);
+		vsc_add_pt(vsc, &st->nn, descs++, vf);
 
 #define VSC_DONE(U,l,t)							\
 	}
@@ -308,30 +325,45 @@ vsc_add_pt(struct vsc *vsc, const struct VSC_desc *desc,
 #include <stdio.h>
 
 static void
-vsc_build_pt_list(struct VSM_data *vd)
+vsc_build_vf_list(struct VSM_data *vd)
 {
 	struct vsc *vsc = vsc_setup(vd);
 	struct VSM_fantom fantom;
-	struct vsc_vf *vf;
 
-	vsc_delete_pts(vsc);
+	vsc_delete_pt_list(vsc);
+	vsc_delete_vf_list(vsc);
 
 	VSM_FOREACH(&fantom, vd) {
 		if (strcmp(fantom.class, VSC_CLASS))
 			continue;
-		vsc_add_vf(vsc, &fantom);
+#define VSC_TYPE_F(n,t,l,e,d)					\
+		if (!strcmp(fantom.type, t))			\
+			vsc_add_vf(vsc, &fantom,		\
+			    &VSC_type_desc_##n);
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
 	}
+}
+
+static void
+vsc_build_pt_list(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	struct vsc_vf *vf;
+
+	vsc_delete_pt_list(vsc);
 
 	VTAILQ_FOREACH(vf, &vsc->vf_list, list) {
 		/*lint -save -e525 -e539 */
-#define VSC_F(n,t,l,f,d,e)
-#define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
+		CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC);		\
 		if (!strcmp(vf->fantom.type, t))		\
-			iter_##l(vsc, VSC_desc_##l, &vf->fantom);
+			iter_##l(vsc, VSC_desc_##l, vf);
+#define VSC_F(n,t,l,f,d,e)
+#define VSC_DONE(a,b,c)
 #include "tbl/vsc_all.h"
-#undef VSC_F
 #undef VSC_DO
+#undef VSC_F
 #undef VSC_DONE
 		/*lint -restore */
 	}
@@ -359,6 +391,7 @@ vsc_filter_pt_list(struct VSM_data *vd)
 	struct vsc *vsc = vsc_setup(vd);
 	struct vsc_sf *sf;
 	struct vsc_pt *pt, *pt2;
+	const struct VSC_section *sec;
 	VTAILQ_HEAD(, vsc_pt)	pt_list;
 
 	if (VTAILQ_EMPTY(&vsc->sf_list))
@@ -366,11 +399,14 @@ vsc_filter_pt_list(struct VSM_data *vd)
 
 	VTAILQ_INIT(&pt_list);
 	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
+		CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC);
 		VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) {
-			if (iter_test(sf->type, pt->point.fantom->type,
+			CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC);
+			sec = pt->point.section;
+			if (iter_test(sf->type, sec->fantom->type,
 			    sf->flags & VSC_SF_TY_WC))
 				continue;
-			if (iter_test(sf->ident, pt->point.fantom->ident,
+			if (iter_test(sf->ident, sec->fantom->ident,
 			    sf->flags & VSC_SF_ID_WC))
 				continue;
 			if (iter_test(sf->name, pt->point.desc->name,
@@ -384,7 +420,7 @@ vsc_filter_pt_list(struct VSM_data *vd)
 			}
 		}
 	}
-	vsc_delete_pts(vsc);
+	vsc_delete_pt_list(vsc);
 	VTAILQ_CONCAT(&vsc->pt_list, &pt_list, list);
 }
 
@@ -401,6 +437,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 	if (1 != VSM_StillValid(vd, &vsc->iter_fantom)) {
 		/* Tell app that list will be nuked */
 		(void)func(priv, NULL);
+		vsc_build_vf_list(vd);
 		vsc_build_pt_list(vd);
 		vsc_filter_pt_list(vd);
 	}
@@ -413,7 +450,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 }
 
 /*--------------------------------------------------------------------
- * Build the static point descriptions
+ * Build the static type and point descriptions
  */
 
 #define VSC_TYPE_F(n,t,l,e,d)	const char *VSC_type_##n = t;



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 35a816e Order the reported point list in the same order as
 the corresponding VSC type defined in vsc_types.h
Message-ID: <E1Ucb5x-0004EW-Ps@project.varnish-software.com>

commit 35a816ea48f6f16a9aada1cf1ae703718645f02e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Feb 27 14:27:23 2013 +0100

    Order the reported point list in the same order as the corresponding
    VSC type defined in vsc_types.h
    
    Rearrange the VSC types according to relevance

diff --git a/include/tbl/vsc_types.h b/include/tbl/vsc_types.h
index 0fcb0dc..57c631a 100644
--- a/include/tbl/vsc_types.h
+++ b/include/tbl/vsc_types.h
@@ -35,6 +35,9 @@
  *    e - Explanation:	Short description of this counter type
  *    d - Description:	Long description of this counter type
  *
+ * The order in which the types are defined in this file determines the
+ * order in which counters are reported in the API, and then also the
+ * display order in varnishstat.
  */
 
 
@@ -44,18 +47,18 @@ VSC_TYPE_F(main,	"MAIN",		"",		"Child",
 VSC_TYPE_F(mgt,		"MGT",		"MGT",		"Master",
     "Management process counters"
 )
+VSC_TYPE_F(mempool,	"MEMPOOL",	"MEMPOOL",	"Memory pool",
+    "Memory pool counters"
+)
 VSC_TYPE_F(sma,		"SMA",		"SMA",		"Storage malloc",
     "Malloc storage counters"
 )
 VSC_TYPE_F(smf,		"SMF",		"SMF",		"Storage file",
     "File storage counters"
 )
-VSC_TYPE_F(lck,		"LCK",		"LCK",		"Lock",
-    "Mutex lock counters"
-)
-VSC_TYPE_F(mempool,	"MEMPOOL",	"MEMPOOL",	"Memory pool",
-    "Memory pool counters"
-)
 VSC_TYPE_F(vbe,		"VBE",		"VBE",		"Backend",
     "Backend counters"
 )
+VSC_TYPE_F(lck,		"LCK",		"LCK",		"Lock",
+    "Mutex lock counters"
+)
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 0c6b595..f2c4d15 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -77,9 +77,10 @@ struct VSC_type_desc {
 };
 
 struct VSC_section {
+	const char *type;
+	const char *ident;
 	const struct VSC_type_desc *desc;
 	struct VSM_fantom *fantom;
-	const struct VSC_point * const *points;
 };
 
 struct VSC_desc {
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index caf80df..c2054c8 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -48,12 +48,20 @@
 #include "vqueue.h"
 #include "vsm_api.h"
 
+enum {
+#define VSC_TYPE_F(n,t,l,e,d) \
+	VSC_type_order_##n,
+#include "tbl/vsc_types.h"
+#undef VSC_TYPE_F
+};
+
 struct vsc_vf {
 	unsigned		magic;
 #define VSC_VF_MAGIC		0x516519f8
 	VTAILQ_ENTRY(vsc_vf)	list;
 	struct VSM_fantom	fantom;
 	struct VSC_section	section;
+	int			order;
 };
 
 struct vsc_pt {
@@ -269,17 +277,27 @@ VSC_Main(struct VSM_data *vd)
 
 static void
 vsc_add_vf(struct vsc *vsc, struct VSM_fantom *fantom,
-    const struct VSC_type_desc *desc)
+    const struct VSC_type_desc *desc, int order)
 {
-	struct vsc_vf *vf;
+	struct vsc_vf *vf, *vf2;
 
 	ALLOC_OBJ(vf, VSC_VF_MAGIC);
 	AN(vf);
 	vf->fantom = *fantom;
+	vf->section.type = vf->fantom.type;
+	vf->section.ident = vf->fantom.ident;
 	vf->section.desc = desc;
 	vf->section.fantom = &vf->fantom;
+	vf->order = order;
 
-	VTAILQ_INSERT_TAIL(&vsc->vf_list, vf, list);
+	VTAILQ_FOREACH(vf2, &vsc->vf_list, list) {
+		if (vf->order < vf2->order)
+			break;
+	}
+	if (vf2 != NULL)
+		VTAILQ_INSERT_BEFORE(vf2, vf, list);
+	else
+		VTAILQ_INSERT_TAIL(&vsc->vf_list, vf, list);
 }
 
 static void
@@ -336,10 +354,10 @@ vsc_build_vf_list(struct VSM_data *vd)
 	VSM_FOREACH(&fantom, vd) {
 		if (strcmp(fantom.class, VSC_CLASS))
 			continue;
-#define VSC_TYPE_F(n,t,l,e,d)					\
-		if (!strcmp(fantom.type, t))			\
-			vsc_add_vf(vsc, &fantom,		\
-			    &VSC_type_desc_##n);
+#define VSC_TYPE_F(n,t,l,e,d)						\
+		if (!strcmp(fantom.type, t))				\
+			vsc_add_vf(vsc, &fantom, &VSC_type_desc_##n,	\
+			    VSC_type_order_##n);
 #include "tbl/vsc_types.h"
 #undef VSC_TYPE_F
 	}
@@ -391,7 +409,6 @@ vsc_filter_pt_list(struct VSM_data *vd)
 	struct vsc *vsc = vsc_setup(vd);
 	struct vsc_sf *sf;
 	struct vsc_pt *pt, *pt2;
-	const struct VSC_section *sec;
 	VTAILQ_HEAD(, vsc_pt)	pt_list;
 
 	if (VTAILQ_EMPTY(&vsc->sf_list))
@@ -402,11 +419,10 @@ vsc_filter_pt_list(struct VSM_data *vd)
 		CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC);
 		VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) {
 			CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC);
-			sec = pt->point.section;
-			if (iter_test(sf->type, sec->fantom->type,
+			if (iter_test(sf->type, pt->point.section->type,
 			    sf->flags & VSC_SF_TY_WC))
 				continue;
-			if (iter_test(sf->ident, sec->fantom->ident,
+			if (iter_test(sf->ident, pt->point.section->ident,
 			    sf->flags & VSC_SF_ID_WC))
 				continue;
 			if (iter_test(sf->name, pt->point.desc->name,



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 12cbc69 Change varnishtest to use new api
Message-ID: <E1Ucb5x-0004Ef-T1@project.varnish-software.com>

commit 12cbc690d9b30c8056fbc0bdec4bd9bdac69399b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 28 13:01:18 2013 +0100

    Change varnishtest to use new api
    
    Clear the thread specific log pointer on log close
    
    Reenable building of varnishtest

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 4adc0bb..2c1b6a6 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -3,7 +3,7 @@
 # Disabling building of the tools while api is in flux
 
 #SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishd
+SUBDIRS = varnishd varnishtest
 
 if HAVE_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 1fc3760..e90b305 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -88,6 +88,8 @@ vtc_logclose(struct vtclog *vl)
 {
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
+	if (pthread_getspecific(log_key) == vl)
+		AZ(pthread_setspecific(log_key, NULL));
 	VSB_delete(vl->vsb);
 	AZ(pthread_mutex_destroy(&vl->mtx));
 	FREE_OBJ(vl);
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 9857c24..4e6f8d4 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -678,7 +678,7 @@ varnish_vclbackend(struct varnish *v, const char *vcl)
  */
 
 struct stat_priv {
-	const char *target;
+	char target[256];
 	uintmax_t val;
 };
 
@@ -691,18 +691,18 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 
 	if (pt == NULL)
 		return(0);
-	if (strcmp(pt->class, "")) {
-		i = strlen(pt->class);
-		if (memcmp(pt->class, p, i))
+	if (strcmp(pt->section->type, "")) {
+		i = strlen(pt->section->type);
+		if (memcmp(pt->section->type, p, i))
 			return (0);
 		p += i;
 		if (*p != '.')
 			return (0);
 		p++;
 	}
-	if (strcmp(pt->ident, "")) {
-		i = strlen(pt->ident);
-		if (memcmp(pt->ident, p, i))
+	if (strcmp(pt->section->ident, "")) {
+		i = strlen(pt->section->ident);
+		if (memcmp(pt->section->ident, p, i))
 			return (0);
 		p += i;
 		if (*p != '.')
@@ -722,32 +722,32 @@ varnish_expect(const struct varnish *v, char * const *av) {
 	uint64_t ref;
 	int good;
 	char *p;
-	int i, j;
+	int i;
+	const char *prefix = "";
 	struct stat_priv sp;
 
-	good = -1;
-
-	sp.target = av[0];
+	if (NULL == strchr(av[0], '.'))
+		prefix = "MAIN.";
+	snprintf(sp.target, sizeof sp.target, "%s%s", prefix, av[0]);
+	sp.target[sizeof sp.target - 1] = '\0';
 	sp.val = 0;
 	ref = 0;
+	good = 0;
 	for (i = 0; i < 10; i++, (void)usleep(100000)) {
+		if (VSM_Abandoned(v->vd)) {
+			VSM_Close(v->vd);
+			good = VSM_Open(v->vd);
+		}
+		if (good < 0)
+			continue;
 
 		good = VSC_Iter(v->vd, do_stat_cb, &sp);
-		if (good < 0) {
-			VSM_Close(v->vd);
-			j = VSM_Open(v->vd);
-			if (j == 0)
-				continue;
-			do {
-				(void)usleep(100000);
-				j = VSM_Open(v->vd);
-				i++;
-			} while(i < 10 && j < 0);
-			if (j < 0)
-				break;
+		if (!good) {
+			good = -2;
+			continue;
 		}
-		good = 0;
 
+		good = 0;
 		ref = strtoumax(av[2], &p, 0);
 		if (ref == UINTMAX_MAX || *p)
 			vtc_log(v->vl, 0, "Syntax error in number (%s)", av[2]);
@@ -764,6 +764,9 @@ varnish_expect(const struct varnish *v, char * const *av) {
 			break;
 	}
 	if (good == -1) {
+		vtc_log(v->vl, 0, "VSM error: %s", VSM_Error(v->vd));
+		VSM_ResetError(v->vd);
+	} else if (good == -2) {
 		vtc_log(v->vl, 0, "stats field %s unknown", av[0]);
 	} else if (good) {
 		vtc_log(v->vl, 2, "as expected: %s (%ju) %s %s",



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] e60c07c Some VSC functions added
Message-ID: <E1Ucb5x-0004El-UC@project.varnish-software.com>

commit e60c07cec4e61b6e814ac1b41b3dd3c2ee7ee1b9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Mar 13 13:37:32 2013 +0100

    Some VSC functions added
    
    Add a VSC_Mgt() function to get direct access to the management counters
    
    Add VSC_MgtValid(), VSC_MainValid(), VSC_IterValid() functions that
    call VSM_StillValid on the fantom used to produce them.
    
    Use the iter fantom when making the pointer lists, so VSM_IterValid()
    returns the right result.

diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index f2c4d15..4d21be2 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -59,6 +59,12 @@ int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
 	 *	 1 Handled.
 	 */
 
+struct VSC_C_mgt *VSC_Mgt(struct VSM_data *vd);
+	/*
+	 * return Management stats structure
+	 * returns NULL until management process has finished initialization.
+	 */
+
 struct VSC_C_main *VSC_Main(struct VSM_data *vd);
 	/*
 	 * return Main stats structure
@@ -113,6 +119,37 @@ int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
 	 *	0:	Done
 	 */
 
+int VSC_MgtValid(struct VSM_data *vd);
+	/*
+	 * Call VSM_StillValid on the fantom used to find the management
+	 * counters
+	 *
+	 * Returns:
+	 *	0: fantom is not valid any more
+	 *	1: fantom is still the same.
+	 *	2: a fantom with same dimensions exist.
+	 */
+
+int VSC_MainValid(struct VSM_data *vd);
+	/*
+	 * Call VSM_StillValid on the fantom used to find the main counters.
+	 *
+	 * Returns:
+	 *	0: fantom is not valid any more
+	 *	1: fantom is still the same.
+	 *	2: a fantom with same dimensions exist.
+	 */
+
+int VSC_IterValid(struct VSM_data *vd);
+	/*
+	 * Call VSM_StillValid on the fantom used to produce the last
+	 * VSC_Iter results.
+	 *
+	 * Returns:
+	 *	0: fantom is not valid any more
+	 *	1: fantom is still the same.
+	 */
+
 /**********************************************************************
  * Precompiled VSC_type_desc's and VSC_desc's for all know VSCs.
  */
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index aef4221..6892b86 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -90,5 +90,9 @@ LIBVARNISHAPI_1.3 {
 	# Functions:
 	VSM_Abandoned;
 	VSM_ResetError;
+	VSC_Mgt;
+	VSC_MgtValid;
+	VSC_MainValid;
+	VSC_IterValid;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index c2054c8..fc35d28 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -92,6 +92,7 @@ struct vsc {
 	VTAILQ_HEAD(, vsc_vf)	vf_list;
 	VTAILQ_HEAD(, vsc_pt)	pt_list;
 	VTAILQ_HEAD(, vsc_sf)	sf_list;
+	struct VSM_fantom	mgt_fantom;
 	struct VSM_fantom	main_fantom;
 	struct VSM_fantom	iter_fantom;
 };
@@ -261,6 +262,19 @@ VSC_Arg(struct VSM_data *vd, int arg, const char *opt)
 
 /*--------------------------------------------------------------------*/
 
+struct VSC_C_mgt *
+VSC_Mgt(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+
+	if (!VSM_StillValid(vd, &vsc->mgt_fantom) &&
+	    !VSM_Get(vd, &vsc->mgt_fantom, VSC_CLASS, VSC_type_mgt, ""))
+		return (NULL);
+	return ((void*)vsc->mgt_fantom.b);
+}
+
+/*--------------------------------------------------------------------*/
+
 struct VSC_C_main *
 VSC_Main(struct VSM_data *vd)
 {
@@ -276,7 +290,7 @@ VSC_Main(struct VSM_data *vd)
  */
 
 static void
-vsc_add_vf(struct vsc *vsc, struct VSM_fantom *fantom,
+vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom,
     const struct VSC_type_desc *desc, int order)
 {
 	struct vsc_vf *vf, *vf2;
@@ -346,18 +360,17 @@ static void
 vsc_build_vf_list(struct VSM_data *vd)
 {
 	struct vsc *vsc = vsc_setup(vd);
-	struct VSM_fantom fantom;
 
 	vsc_delete_pt_list(vsc);
 	vsc_delete_vf_list(vsc);
 
-	VSM_FOREACH(&fantom, vd) {
-		if (strcmp(fantom.class, VSC_CLASS))
+	VSM_FOREACH(&vsc->iter_fantom, vd) {
+		if (strcmp(vsc->iter_fantom.class, VSC_CLASS))
 			continue;
 #define VSC_TYPE_F(n,t,l,e,d)						\
-		if (!strcmp(fantom.type, t))				\
-			vsc_add_vf(vsc, &fantom, &VSC_type_desc_##n,	\
-			    VSC_type_order_##n);
+		if (!strcmp(vsc->iter_fantom.type, t))			\
+			vsc_add_vf(vsc, &vsc->iter_fantom,		\
+			    &VSC_type_desc_##n, VSC_type_order_##n);
 #include "tbl/vsc_types.h"
 #undef VSC_TYPE_F
 	}
@@ -466,6 +479,47 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+int
+VSC_MgtValid(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	fprintf(stderr, "VSC_MgtValid called priv=%ju\n",
+		vsc->mgt_fantom.priv);
+	return (VSM_StillValid(vd, &vsc->mgt_fantom));
+}
+
+int
+VSC_MainValid(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	fprintf(stderr, "VSC_MainValid called priv=%ju\n",
+		vsc->main_fantom.priv);
+	return (VSM_StillValid(vd, &vsc->main_fantom));
+}
+
+int
+VSC_IterValid(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	int v;
+	fprintf(stderr, "VSC_IterValid called priv=%ju\n",
+		vsc->iter_fantom.priv);
+	v = VSM_StillValid(vd, &vsc->iter_fantom);
+	if (v == 2) {
+		/* There's been changes, reiteration needed. Clear fantom
+		   so subsequent calls will also fail */
+		memset(&vsc->iter_fantom, 0, sizeof vsc->iter_fantom);
+		v = 0;
+	}
+	fprintf(stderr, "VSC_IterValid returns %d priv=%ju\n", v,
+		vsc->iter_fantom.priv);
+
+	return (v);
+}
+
+/*--------------------------------------------------------------------
  * Build the static type and point descriptions
  */
 



From martin at varnish-cache.org  Wed May 15 12:46:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:13 +0200
Subject: [master] 8aec529 New varnishstat curses implementation
Message-ID: <E1Ucb5x-0004Et-VN@project.varnish-software.com>

commit 8aec529fadd7cb97ceccb8bef455676c39886661
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Mar 13 14:20:56 2013 +0100

    New varnishstat curses implementation

diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index 1a38329..6060438 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -12,7 +12,8 @@ varnishstat_SOURCES = \
 	varnishstat.c \
 	varnishstat_curses.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c
+	$(top_builddir)/lib/libvarnish/version.c \
+	$(top_builddir)/lib/libvarnish/vtim.c
 
 varnishstat_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index e06fe2c..fcd9bc3 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -192,7 +192,8 @@ do_once(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main)
 	struct once_priv op;
 
 	memset(&op, 0, sizeof op);
-	op.up = VSC_C_main->uptime;
+	if (VSC_C_main != NULL)
+		op.up = VSC_C_main->uptime;
 	op.pad = 18;
 
 	(void)VSC_Iter(vd, do_once_cb, &op);
@@ -267,7 +268,6 @@ main(int argc, char * const *argv)
 {
 	int c;
 	struct VSM_data *vd;
-	const struct VSC_C_main *VSC_C_main;
 	int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0;
 
 	vd = VSM_New();
@@ -304,11 +304,8 @@ main(int argc, char * const *argv)
 		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
 	}
-	VSC_C_main = VSC_Main(vd);
-	AN(VSC_C_main);
-
 	if (!(xml || json || once)) {
-		do_curses(vd, VSC_C_main, delay);
+		do_curses(vd, delay);
 		exit(0);
 	}
 
@@ -318,7 +315,7 @@ main(int argc, char * const *argv)
 		else if (json)
 			do_json(vd);
 		else if (once)
-			do_once(vd, VSC_C_main);
+			do_once(vd, VSC_Main(vd));
 		else {
 			assert(0);
 		}
diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h
index c229ed2..1f66476 100644
--- a/bin/varnishstat/varnishstat.h
+++ b/bin/varnishstat/varnishstat.h
@@ -35,6 +35,4 @@
 #include "vas.h"
 #include "vcs.h"
 
-
-void do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
-    int delay);
+void do_curses(struct VSM_data *vd, int delay);
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index f1ecb77..5d19dc1 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -1,10 +1,11 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2013 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
  * Author: Dag-Erling Sm?rgrav <des at des.no>
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -30,250 +31,863 @@
  * Statistics output program
  */
 
+
 #include "config.h"
 
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <signal.h>
+#include <time.h>
 #include <sys/time.h>
+#include <poll.h>
+#include <stdint.h>
+
+#include "vas.h"
+#include "miniobj.h"
+#include "vqueue.h"
+#include "vapi/vsm.h"
+#include "vapi/vsc.h"
+#include "vtim.h"
+
+#include "varnishstat.h"
 
 #ifdef HAVE_NCURSES_CURSES_H
 #  include <ncurses/curses.h>
-#endif
-
-#ifndef HAVE_NCURSES_CURSES_H
+#else
 #  ifdef HAVE_CURSES_H
 #    include <curses.h>
 #  endif
 #endif
-#include <errno.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <unistd.h>
 
-#include "varnishstat.h"
+#define LINES_STATUS		3
+#define LINES_BAR_T		1
+#define LINES_BAR_B		1
+#define LINES_INFO		3
+#define LINES_POINTS_MIN	3
 
-#include "vapi/vsc.h"
-#include "vapi/vsm.h"
-#include "vqueue.h"
+#define N_COL			6
+#define COLW			13
+#define COLW_NAME_MIN		24
 
-#if 0
-#define AC(x) assert((x) != ERR)
-#else
-#define AC(x) x
-#endif
+struct ma {
+	unsigned n, nmax;
+	double acc;
+};
 
 struct pt {
-	VTAILQ_ENTRY(pt)	next;
-	const volatile uint64_t	*ptr;
-	uint64_t		ref;
+	unsigned		magic;
+#define PT_MAGIC		0x41698E4F
+	VTAILQ_ENTRY(pt)	list;
+
+	const struct VSC_point	*vpt;
+
+	char			*key;
+	char			*name;
 	int			flag;
+	const volatile uint64_t	*ptr;
+
 	char			seen;
-	char			*name;
+
+	uint64_t		cur, last;
+	double			t_cur, t_last;
+	double			chg, avg;
+
+	struct ma		ma_10, ma_100, ma_1000;
 };
 
-static VTAILQ_HEAD(, pt) pthead = VTAILQ_HEAD_INITIALIZER(pthead);
+static VTAILQ_HEAD(, pt) ptlist = VTAILQ_HEAD_INITIALIZER(ptlist);
+static int n_ptlist = 0;
+static int n_ptarray = 0;
+static struct pt **ptarray = NULL;
+static const struct VSC_C_mgt *VSC_C_mgt = NULL;
+static const struct VSC_C_main *VSC_C_main = NULL;
+
+static int l_status, l_bar_t, l_points, l_bar_b, l_info;
+static int colw_name = COLW_NAME_MIN;
+static WINDOW *w_status = NULL;
+static WINDOW *w_bar_t = NULL;
+static WINDOW *w_points = NULL;
+static WINDOW *w_bar_b = NULL;
+static WINDOW *w_info = NULL;
+
+static int keep_running = 1;
+static int show_info = 1;
+static int hide_unseen = 1;
+static int page_start = 0;
+static int current = 0;
+static int rebuild = 0;
+static int redraw = 0;
+static int sample = 0;
+static double t_sample = 0.;
+static double interval = 1.;
+
+static void
+update_ma(struct ma *ma, double val)
+{
+	AN(ma);
+	AN(ma->nmax);
+	if (ma->n < ma->nmax)
+		ma->n++;
+	ma->acc += (val - ma->acc) / (double)ma->n;
+}
+
+static void
+update_position(void)
+{
+	int old_current, old_page_start;
+
+	old_current = current;
+	old_page_start = page_start;
+
+	if (n_ptarray == 0) {
+		current = 0;
+		page_start = 0;
+	} else {
+		if (current < 0)
+			current = 0;
+		if (current > n_ptarray - 1)
+			current = n_ptarray - 1;
+		if (current < page_start)
+			page_start = current;
+		if (current > page_start + (l_points - 1))
+			page_start = current - (l_points - 1);
+		if (page_start < 0)
+			page_start = 0;
+		if (page_start > n_ptarray - 1)
+			page_start = n_ptarray - 1;
+	}
+
+	if (current != old_current || page_start != old_page_start)
+		redraw = 1;
+}
+
+static void
+delete_pt_array(void)
+{
+	if (ptarray != NULL)
+		free(ptarray);
+	ptarray = NULL;
+	n_ptarray = 0;
+
+	update_position();
+}
+
+static void
+build_pt_array(void)
+{
+	int i;
+	struct pt *pt;
+	struct pt *pt_current = NULL;
+	int current_line = 0;
+
+	if (current < n_ptarray) {
+		pt_current = ptarray[current];
+		current_line = current - page_start;
+	}
+
+	if (ptarray != NULL)
+		delete_pt_array();
+	AZ(n_ptarray);
+	ptarray = calloc(n_ptlist, sizeof *ptarray);
+	AN(ptarray);
+
+	VTAILQ_FOREACH(pt, &ptlist, list) {
+		CHECK_OBJ_NOTNULL(pt, PT_MAGIC);
+		if (!pt->seen && hide_unseen)
+			continue;
+		assert(n_ptarray < n_ptlist);
+		ptarray[n_ptarray++] = pt;
+	}
+	assert(n_ptarray <= n_ptlist);
+
+	for (i = 0; pt_current != NULL && i < n_ptarray; i++)
+		if (ptarray[i] == pt_current)
+			break;
+	current = i;
+	page_start = current - current_line;
+	update_position();
+
+	rebuild = 0;
+	redraw = 1;
+}
+
+static void
+delete_pt_list(void)
+{
+	struct pt *pt;
+	unsigned i = 0;
+
+	delete_pt_array();
+
+	while (!VTAILQ_EMPTY(&ptlist)) {
+		pt = VTAILQ_FIRST(&ptlist);
+		CHECK_OBJ_NOTNULL(pt, PT_MAGIC);
+		VTAILQ_REMOVE(&ptlist, pt, list);
+		free(pt->name);
+		FREE_OBJ(pt);
+		i++;
+	}
+	assert(i == n_ptlist);
+	n_ptlist = 0;
+
+	update_position();
+}
+
+struct pt_priv {
+	unsigned		magic;
+#define PT_PRIV_MAGIC		0x34ACBAD6
+	VTAILQ_HEAD(, pt)	ptlist;
+	unsigned		n_ptlist;
+};
 
 static int
-do_curses_cb(void *priv, const struct VSC_point * const sp)
+build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 {
+	struct pt_priv *pt_priv;
 	struct pt *pt;
 	char buf[128];
 
-	(void)priv;
-	if (sp == NULL)
+	if (vpt == NULL)
 		return (0);
-	assert(!strcmp(sp->desc->fmt, "uint64_t"));
 
-	pt = calloc(sizeof *pt, 1);
+	CAST_OBJ_NOTNULL(pt_priv, priv, PT_PRIV_MAGIC);
+
+	assert(!strcmp(vpt->desc->fmt, "uint64_t"));
+	snprintf(buf, sizeof buf, "%s.%s.%s", vpt->section->type,
+	    vpt->section->ident, vpt->desc->name);
+	buf[sizeof buf - 1] = '\0';
+
+	VTAILQ_FOREACH(pt, &ptlist, list) {
+		CHECK_OBJ_NOTNULL(pt, PT_MAGIC);
+		AN(pt->key);
+		if (strcmp(buf, pt->key))
+			continue;
+		VTAILQ_REMOVE(&ptlist, pt, list);
+		AN(n_ptlist);
+		n_ptlist--;
+		pt->vpt = vpt;
+		VTAILQ_INSERT_TAIL(&pt_priv->ptlist, pt, list);
+		pt_priv->n_ptlist++;
+		return (0);
+	}
+	AZ(pt);
+
+	ALLOC_OBJ(pt, PT_MAGIC);
 	AN(pt);
-	VTAILQ_INSERT_TAIL(&pthead, pt, next);
 
-	pt->ptr = sp->ptr;
-	pt->ref = *pt->ptr;
-	pt->flag = sp->desc->flag;
+	pt->key = strdup(buf);
+	AN(pt->key);
 
 	*buf = '\0';
-	if (strcmp(sp->section->fantom->type, "")) {
-		strcat(buf, sp->section->fantom->type);
+	if (strcmp(vpt->section->type, "")) {
+		strcat(buf, vpt->section->type);
 		strcat(buf, ".");
 	}
-	if (strcmp(sp->section->fantom->ident, "")) {
-		strcat(buf, sp->section->fantom->ident);
+	if (strcmp(vpt->section->ident, "")) {
+		strcat(buf, vpt->section->ident);
 		strcat(buf, ".");
 	}
-	strcat(buf, sp->desc->name);
-	strcat(buf, " - ");
-	strcat(buf, sp->desc->sdesc);
+	strcat(buf, vpt->desc->name);
 	pt->name = strdup(buf);
 	AN(pt->name);
+
+	pt->vpt = vpt;
+
+	pt->ptr = vpt->ptr;
+	pt->last = *pt->ptr;
+	pt->flag = vpt->desc->flag;
+	if (pt->flag == 'a')
+		pt->flag = 'c';
+	if (pt->flag == 'i')
+		pt->flag = 'g';
+
+	pt->ma_10.nmax = 10;
+	pt->ma_100.nmax = 100;
+	pt->ma_1000.nmax = 1000;
+
+	VTAILQ_INSERT_TAIL(&pt_priv->ptlist, pt, list);
+	pt_priv->n_ptlist++;
+
 	return (0);
 }
 
 static void
-prep_pts(struct VSM_data *vd)
+build_pt_list(struct VSM_data *vd)
 {
-	struct pt *pt, *pt2;
+	struct pt_priv pt_priv;
+	int i;
+	struct pt *pt_current = NULL;
+	int current_line = 0;
 
-	VTAILQ_FOREACH_SAFE(pt, &pthead, next, pt2) {
-		VTAILQ_REMOVE(&pthead, pt, next);
-		free(pt->name);
-		free(pt);
+	if (current < n_ptarray) {
+		pt_current = ptarray[current];
+		current_line = current - page_start;
 	}
 
-	(void)VSC_Iter(vd, do_curses_cb, NULL);
+	pt_priv.magic = PT_PRIV_MAGIC;
+	VTAILQ_INIT(&pt_priv.ptlist);
+	pt_priv.n_ptlist = 0;
+
+	(void)VSC_Iter(vd, build_pt_list_cb, &pt_priv);
+	delete_pt_list();
+	AN(VTAILQ_EMPTY(&ptlist));
+	AZ(n_ptlist);
+	VTAILQ_CONCAT(&ptlist, &pt_priv.ptlist, list);
+	n_ptlist = pt_priv.n_ptlist;
+	build_pt_array();
+
+	for (i = 0; pt_current != NULL && i < n_ptarray; i++)
+		if (ptarray[i] == pt_current)
+			break;
+	current = i;
+	page_start = current - current_line;
+	update_position();
 }
 
 static void
-myexp(double *acc, double val, unsigned *n, unsigned nmax)
+sample_points(void)
 {
+	struct pt *pt;
+
+	t_sample = VTIM_mono();
+	sample = 0;
+	redraw = 1;
+
+	VTAILQ_FOREACH(pt, &ptlist, list) {
+		AN(pt->vpt);
+		AN(pt->ptr);
+		if (*pt->ptr == 0 && !pt->seen)
+			continue;
+		if (!pt->seen) {
+			pt->seen = 1;
+			rebuild = 1;
+		}
+		pt->last = pt->cur;
+		pt->cur = *pt->ptr;
+		pt->t_last = pt->t_cur;
+		pt->t_cur = VTIM_mono();
 
-	if (*n < nmax)
-		(*n)++;
-	(*acc) += (val - *acc) / (double)*n;
+		if (pt->t_last)
+			pt->chg = (pt->cur - (intmax_t)pt->last) /
+			    (pt->t_cur - pt->t_last);
+
+		if (pt->flag == 'g') {
+			pt->avg = 0.;
+			update_ma(&pt->ma_10, pt->cur);
+			update_ma(&pt->ma_100, pt->cur);
+			update_ma(&pt->ma_1000, pt->cur);
+		} else if (pt->flag == 'c') {
+			if (VSC_C_main != NULL && VSC_C_main->uptime)
+				pt->avg = pt->cur / VSC_C_main->uptime;
+			else
+				pt->avg = 0.;
+			if (pt->t_last) {
+				update_ma(&pt->ma_10, pt->chg);
+				update_ma(&pt->ma_100, pt->chg);
+				update_ma(&pt->ma_1000, pt->chg);
+			}
+		}
+	}
 }
 
-void
-do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
-    int delay)
+static void
+make_windows(void)
 {
-	intmax_t ju;
-	struct timeval tv;
-	double tt, lt, lhit, hit, lmiss, miss, hr, mr, ratio, up;
-	double a1, a2, a3;
-	unsigned n1, n2, n3;
-	time_t rt;
-	int ch, line;
-	struct pt *pt;
+	int Y, X;
+	int y;
+	int y_status, y_bar_t, y_points, y_bar_b, y_info;
 
-	(void)initscr();
-	AC(raw());
-	AC(noecho());
-	AC(nonl());
-	AC(intrflush(stdscr, FALSE));
-	(void)curs_set(0);	/* XXX: too many implementations are bogus */
-
-	while (1) {
-		/*
-		 * Initialization goes in outher loop
-		 */
-		prep_pts(vd);
-		AC(erase());
-		AC(refresh());
-
-		a1 = a2 = a3 = 0.0;
-		n1 = n2 = n3 = 0;
-		lt = 0;
-		lhit = 0;
-		lmiss = 0;
-
-		while (1) {
-			/*
-			 * Break to outher loop if we need to re-read file.
-			 * Only check if it looks like nothing is happening.
-			 */
-			AZ(gettimeofday(&tv, NULL));
-			tt = tv.tv_usec * 1e-6 + tv.tv_sec;
-			lt = tt - lt;
-
-			rt = VSC_C_main->uptime;
-			up = rt;
-
-			AC(mvprintw(0, 0, "%*s", COLS - 1, VSM_Name(vd)));
-			AC(mvprintw(0, 0, "%d+%02d:%02d:%02d", rt / 86400,
-			    (rt % 86400) / 3600, (rt % 3600) / 60, rt % 60));
-
-			hit = VSC_C_main->cache_hit;
-			miss = VSC_C_main->cache_miss;
-			hr = (hit - lhit) / lt;
-			mr = (miss - lmiss) / lt;
-			lhit = hit;
-			lmiss = miss;
-			if (hr + mr != 0) {
-				ratio = hr / (hr + mr);
-				myexp(&a1, ratio, &n1, 10);
-				myexp(&a2, ratio, &n2, 100);
-				myexp(&a3, ratio, &n3, 1000);
-			}
-			AC(mvprintw(1, 0, "Hitrate ratio: %8u %8u %8u",
-			    n1, n2, n3));
-			AC(mvprintw(2, 0, "Hitrate avg:   %8.4f %8.4f %8.4f",
-			    a1, a2, a3));
-
-			line = 3;
-			VTAILQ_FOREACH(pt, &pthead, next) {
-				ju = *pt->ptr;
-				if (ju == 0 && !pt->seen)
-					continue;
-				pt->seen = 1;
-				line++;
-				if (line >= LINES)
-					break;
-				if (pt->flag == 'a' || pt->flag == 'c') {
-					AC(mvprintw(line, 0,
-					    "%12ju %12.2f %12.2f %s\n",
-					    ju, (ju - (intmax_t)pt->ref)/lt,
-					    ju / up, pt->name));
-					pt->ref = ju;
-				} else if (pt->flag == 'b') {
-					AC(mvprintw(line, 0, "  %010.10jx <",
-					    (ju >> 24) & 0xffffffffffLL));
-					for (ch = 0x800000; ch; ch >>= 1)
-						if (ju & ch)
-							AC(addstr("V"));
-						else
-							AC(addstr("_"));
-					AC(addstr(" "));
-					AC(addstr(pt->name));
-				} else {
-					AC(mvprintw(line, 0,
-					    "%12ju %12s %12s %s\n",
-					    ju, ".  ", ".  ", pt->name));
-				}
-			}
-			lt = tt;
-			AC(refresh());
-			timeout(delay * 1000);
-			switch ((ch = getch())) {
-			case ERR:
-				break;
-#ifdef KEY_RESIZE
-			case KEY_RESIZE:
-				AC(erase());
-				break;
-#endif
-			case '\014': /* Ctrl-L */
-			case '\024': /* Ctrl-T */
-				AC(redrawwin(stdscr));
-				AC(refresh());
-				break;
-			case '\003': /* Ctrl-C */
-				AZ(raise(SIGINT));
-				break;
-			case '\032': /* Ctrl-Z */
-				AZ(raise(SIGTSTP));
-				break;
-			case '\021': /* Ctrl-Q */
-			case 'Q':
-			case 'q':
-				AC(endwin());
-				exit(0);
-			case '0':
-			case '1':
-			case '2':
-			case '3':
-			case '4':
-			case '5':
-			case '6':
-			case '7':
-			case '8':
-			case '9':
-				delay = 1U << (ch - '0');
-				break;
-			default:
-				AC(beep());
-				break;
+	if (w_status) {
+		delwin(w_status);
+		w_status = NULL;
+	}
+	if (w_bar_t) {
+		delwin(w_bar_t);
+		w_bar_t = NULL;
+	}
+	if (w_points) {
+		delwin(w_points);
+		w_points = NULL;
+	}
+	if (w_bar_b) {
+		delwin(w_bar_b);
+		w_bar_b = NULL;
+	}
+	if (w_info) {
+		delwin(w_info);
+		w_info = NULL;
+	}
+
+	Y = LINES;
+	X = COLS;
+
+	l_status = LINES_STATUS;
+	l_bar_t = LINES_BAR_T;
+	l_bar_b = LINES_BAR_B;
+	l_info = (show_info ? LINES_INFO : 0);
+	l_points = Y - (l_status + l_bar_t + l_bar_b + l_info);
+	if (l_points < LINES_POINTS_MIN) {
+		l_points += l_info;
+		l_info = 0;
+	}
+	if (l_points < LINES_POINTS_MIN)
+		l_points = LINES_POINTS_MIN;
+
+	y = 0;
+	y_status = y;
+	y += l_status;
+	y_bar_t = y;
+	y += l_bar_t;
+	y_points = y;
+	y += l_points;
+	y_bar_b = y;
+	y += l_bar_b;
+	y_info = y;
+	y += l_info;
+	assert(y >= Y);
+
+	w_status = newwin(l_status, X, y_status, 0);
+	AN(w_status);
+	nodelay(w_status, 1);
+	keypad(w_status, 1);
+	wnoutrefresh(w_status);
+
+	w_bar_t = newwin(l_bar_t, X, y_bar_t, 0);
+	AN(w_bar_t);
+	wbkgd(w_bar_t, A_REVERSE);
+	wnoutrefresh(w_bar_t);
+
+	w_points = newwin(l_points, X, y_points, 0);
+	AN(w_points);
+	wnoutrefresh(w_points);
+
+	w_bar_b = newwin(l_bar_b, X, y_bar_b, 0);
+	AN(w_bar_b);
+	wbkgd(w_bar_b, A_REVERSE);
+	wnoutrefresh(w_bar_b);
+
+	if (l_info) {
+		w_info = newwin(l_info, X, y_info, 0);
+		AN(w_info);
+		wnoutrefresh(w_info);
+	}
+
+	if (X - COLW_NAME_MIN > N_COL * COLW)
+		colw_name = X - (N_COL * COLW);
+	else
+		colw_name = COLW_NAME_MIN;
+
+	redraw = 1;
+}
+
+static void
+draw_status(void)
+{
+	time_t up_mgt = 0;
+	time_t up_chld = 0;
+
+	AN(w_status);
+
+	werase(w_status);
+	if (VSC_C_mgt != NULL)
+		up_mgt = VSC_C_mgt->uptime;
+	if (VSC_C_main != NULL)
+		up_chld = VSC_C_main->uptime;
+
+	mvwprintw(w_status, 0, 0, "Uptime mgt:   %d+%02d:%02d:%02d",
+	    up_mgt / 86400, (up_mgt % 86400) / 3600, (up_mgt % 3600) / 60,
+	    up_mgt % 60);
+	mvwprintw(w_status, 1, 0, "Uptime child: %d+%02d:%02d:%02d",
+	    up_chld / 86400, (up_chld % 86400) / 3600, (up_chld % 3600) / 60,
+	    up_chld % 60);
+
+	wnoutrefresh(w_status);
+}
+
+static void
+draw_bar_t(void)
+{
+	int X, x;
+	enum {
+		COL_CUR,
+		COL_CHG,
+		COL_AVG,
+		COL_MA10,
+		COL_MA100,
+		COL_MA1000,
+		COL_LAST
+	} col;
+
+	AN(w_bar_t);
+
+	X = getmaxx(w_bar_t);
+	x = 0;
+	werase(w_bar_t);
+	if (page_start > 0)
+		mvwaddch(w_bar_t, 0, x, ACS_UARROW);
+	x += 2;
+	mvwprintw(w_bar_t, 0, x, "%.*s", colw_name - 2, "NAME");
+	x += colw_name - 2;
+	col = 0;
+	while (col < COL_LAST) {
+		if (X - x < COLW)
+			break;
+		switch (col) {
+		case COL_CUR:
+			mvwprintw(w_bar_t, 0, x, " %12.12s", "CURRENT");
+			break;
+		case COL_CHG:
+			mvwprintw(w_bar_t, 0, x, " %12.12s", "CHANGE");
+			break;
+		case COL_AVG:
+			mvwprintw(w_bar_t, 0, x, " %12.12s", "AVERAGE");
+			break;
+		case COL_MA10:
+			mvwprintw(w_bar_t, 0, x, " %12.12s", "AVG_10");
+			break;
+		case COL_MA100:
+			mvwprintw(w_bar_t, 0, x, " %12.12s", "AVG_100");
+			break;
+		case COL_MA1000:
+			mvwprintw(w_bar_t, 0, x, " %12.12s", "AVG_1000");
+			break;
+		default:
+			break;
+		}
+		x += COLW;
+		col++;
+	}
+
+	wnoutrefresh(w_bar_t);
+}
+
+static void
+draw_line_default(WINDOW *w, int y, int x, int X, struct pt *pt)
+{
+	enum {
+		COL_CUR,
+		COL_CHG,
+		COL_AVG,
+		COL_MA10,
+		COL_MA100,
+		COL_MA1000,
+		COL_LAST
+	} col;
+
+	AN(w);
+	AN(pt);
+
+	col = 0;
+	while (col < COL_LAST) {
+		if (X - x < COLW)
+			break;
+		switch (col) {
+		case COL_CUR:
+			mvwprintw(w, y, x, " %12ju", (uintmax_t)pt->cur);
+			break;
+		case COL_CHG:
+			if (pt->t_last)
+				mvwprintw(w, y, x, " %12.2f", pt->chg);
+			else
+				mvwprintw(w, y, x, " %12s", ".  ");
+			break;
+		case COL_AVG:
+			if (pt->avg)
+				mvwprintw(w, y, x, " %12.2f", pt->avg);
+			else
+				mvwprintw(w, y, x, " %12s", ".  ");
+			break;
+		case COL_MA10:
+			mvwprintw(w, y, x, " %12.2f", pt->ma_10.acc);
+			break;
+		case COL_MA100:
+			mvwprintw(w, y, x, " %12.2f", pt->ma_100.acc);
+			break;
+		case COL_MA1000:
+			mvwprintw(w, y, x, " %12.2f", pt->ma_1000.acc);
+			break;
+		default:
+			break;
+		}
+		x += COLW;
+		col++;
+	}
+}
+
+static void
+draw_line_bitmap(WINDOW *w, int y, int x, int X, struct pt *pt)
+{
+	int ch;
+	enum {
+		COL_VAL,
+		COL_MAP,
+		COL_LAST
+	} col;
+
+	AN(w);
+	AN(pt);
+	assert(pt->flag == 'b');
+
+	col = 0;
+	while (col < COL_LAST) {
+		switch (col) {
+		case COL_VAL:
+			if (X - x < COLW)
+				return;
+			mvwprintw(w, y, x, "   %010.10jx",
+			    (pt->cur >> 24) & 0xffffffffffLL);
+			x += COLW;
+			break;
+		case COL_MAP:
+			if (X - x < 2 * COLW)
+				return;
+			x += (2 * COLW) - 24;
+			for (ch = 0x800000; ch; ch >>= 1) {
+				if (pt->cur & ch)
+					mvwaddch(w, y, x, 'V');
+				else
+					mvwaddch(w, y, x, '_');
+				x++;
 			}
+			break;
+		default:
+			x += COLW;
+			break;
+		}
+		col++;
+	}
+}
+
+static void
+draw_line(WINDOW *w, int y, struct pt *pt)
+{
+	int x, X;
+
+	assert(colw_name >= COLW_NAME_MIN);
+	X = getmaxx(w);
+	x = 0;
+	if (strlen(pt->name) > colw_name)
+		mvwprintw(w, y, x, "%.*s...", colw_name - 3, pt->name);
+	else
+		mvwprintw(w, y, x, "%.*s", colw_name, pt->name);
+	x += colw_name;
+
+	if (pt->flag == 'b')
+		draw_line_bitmap(w, y, x, X, pt);
+	else
+		draw_line_default(w, y, x, X, pt);
+}
+
+static void
+draw_points(void)
+{
+	int Y, X;
+	int line;
+	int n;
+
+	AN(w_points);
+
+	werase(w_points);
+	if (n_ptarray == 0) {
+		wnoutrefresh(w_points);
+		return;
+	}
+
+	assert(current >= 0);
+	assert(current < n_ptarray);
+	assert(page_start >= 0);
+	assert(page_start < n_ptarray);
+	assert(current >= page_start);
+	assert(current - page_start < l_points);
+
+	getmaxyx(w_points, Y, X);
+	(void)Y;
+	(void)X;
+	for (line = 0; line < l_points; line++) {
+		n = line + page_start;
+		if (n >= n_ptarray)
+			break;
+		if (n == current)
+			wattron(w_points, A_BOLD);
+		draw_line(w_points, line, ptarray[n]);
+		if (n == current)
+			wattroff(w_points, A_BOLD);
+	}
+	wnoutrefresh(w_points);
+}
+
+static void
+draw_bar_b(void)
+{
+	int x;
+
+	AN(w_bar_b);
+
+	x = 0;
+	werase(w_bar_b);
+	if (page_start + l_points < n_ptarray)
+		mvwaddch(w_bar_b, 0, x, ACS_DARROW);
+	x += 2;
+	if (current < n_ptarray - 1)
+		mvwprintw(w_bar_b, 0, x, "%s", ptarray[current]->name);
+
+	wnoutrefresh(w_bar_b);
+}
+
+static void
+draw_info(void)
+{
+	int Y, X;
+
+	if (w_info == NULL)
+		return;
+
+	getmaxyx(w_info, Y, X);
+	mvwprintw(w_info, 0, 0, "infotest Y=%d X=%d", Y, X);
+	wnoutrefresh(w_info);
+}
+
+static void
+draw_screen(void)
+{
+	draw_status();
+	draw_bar_t();
+	draw_points();
+	draw_bar_b();
+	draw_info();
+	doupdate();
+	redraw = 0;
+}
+
+static void
+handle_keypress(int ch)
+{
+	fprintf(stderr, "key: 0x%x\n", ch);
+
+	switch (ch) {
+	case KEY_UP:
+		if (current == 0)
+			return;
+		current--;
+		break;
+	case KEY_DOWN:
+		if (current == n_ptarray - 1)
+			return;
+		current++;
+		break;
+	case KEY_PPAGE:
+	case 'b':
+		current -= l_points;
+		page_start -= l_points;
+		break;
+	case KEY_NPAGE:
+	case ' ':
+		current += l_points;
+		if (page_start + l_points < n_ptarray - 1)
+			page_start += l_points;
+		break;
+	case 'g':
+		current = 0;
+		page_start = 0;
+		break;
+	case 'G':
+		current = n_ptarray - 1;
+		page_start = current - l_points + 1;
+		break;
+	case 'h':
+		hide_unseen = 1 - hide_unseen;
+		rebuild = 1;
+		break;
+	case 'q':
+	case 'Q':
+		keep_running = 0;
+		return;
+	case '\024':		/* Ctrl-T */
+		sample = 1;
+		return;
+	default:
+		return;
+	}
+
+	update_position();
+	redraw = 1;
+}
+
+void
+do_curses(struct VSM_data *vd, int delay)
+{
+	struct pollfd pollfd;
+	long timeout;
+	int ch;
+	double now;
+
+	AN(freopen("errlog", "w", stderr));
+	setbuf(stderr, NULL);
+
+	(void)delay;
+
+	initscr();
+	raw();
+	noecho();
+	nonl();
+	curs_set(0);
+
+	pollfd.fd = STDIN_FILENO;
+	pollfd.events = POLLIN;
+
+	make_windows();
+	doupdate();
+
+	VSC_C_mgt = VSC_Mgt(vd);
+	VSC_C_main = VSC_Main(vd);
+	while (keep_running) {
+		if (VSM_Abandoned(vd)) {
+			fprintf(stderr, "abandoned\n");
+			delete_pt_list();
+			VSM_Close(vd);
+			if (VSM_Open(vd) < 0)
+				fprintf(stderr, "VSM_Open failed: %s\n",
+				    VSM_Error(vd));
+		}
+		VSC_C_mgt = VSC_Mgt(vd);
+		VSC_C_main = VSC_Main(vd);
+		if (!VSC_IterValid(vd)) {
+			fprintf(stderr, "iter not valid\n");
+			build_pt_list(vd);
+		} else
+			fprintf(stderr, "iter valid\n");
+
+		now = VTIM_mono();
+		if (now - t_sample > interval)
+			sample = 1;
+		if (sample)
+			sample_points();
+		if (rebuild)
+			build_pt_array();
+		if (redraw)
+			draw_screen();
+
+		timeout = (t_sample + interval - now) * 1000;
+		if (timeout > 0)
+			(void)poll(&pollfd, 1, timeout);
+
+		switch (ch = wgetch(w_status)) {
+		case ERR:
+			break;
+		case KEY_RESIZE:
+			make_windows();
+			update_position();
+			break;
+		default:
+			handle_keypress(ch);
+			break;
 		}
 	}
+	VSM_Close(vd);
+	AZ(endwin());
+
+	fclose(stderr);
 }



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 4cf3e41 Add verbosity levels to all Varnish counters
Message-ID: <E1Ucb5y-0004F5-0Z@project.varnish-software.com>

commit 4cf3e419bc842012ed011522b139603b7a240ea7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 14 16:07:06 2013 +0100

    Add verbosity levels to all Varnish counters

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index d7f29f5..b86f4c0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -243,7 +243,7 @@ struct acct {
 
 #define L0(t, n)
 #define L1(t, n)		t n;
-#define VSC_F(n, t, l, f, e,d)	L##l(t, n)
+#define VSC_F(n,t,l,f,v,e,d)	L##l(t, n)
 struct dstat {
 #include "tbl/vsc_f_main.h"
 };
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 7b65479..ec18df9 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -49,7 +49,7 @@ wrk_sumstat(struct worker *w)
 	Lck_AssertHeld(&wstat_mtx);
 #define L0(n)
 #define L1(n) (VSC_C_main->n += w->stats.n)
-#define VSC_F(n, t, l, f, d, e) L##l(n);
+#define VSC_F(n, t, l, f, v, d, e) L##l(n);
 #include "tbl/vsc_f_main.h"
 #undef VSC_F
 #undef L0
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 6506668..a42579b 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -61,7 +61,7 @@
  * Globals, not related to traffic
  */
 
-VSC_F(uptime,			uint64_t, 0, 'a',
+VSC_F(uptime,			uint64_t, 0, 'a', info,
     "Child process uptime",
 	""
 )
@@ -71,61 +71,61 @@ VSC_F(uptime,			uint64_t, 0, 'a',
  * Sessions
  */
 
-VSC_F(sess_conn,		uint64_t, 1, 'c',
+VSC_F(sess_conn,		uint64_t, 1, 'c', info,
     "Sessions accepted",
 	"Count of sessions succesfully accepted"
 )
 
-VSC_F(sess_drop,		uint64_t, 1, 'c',
+VSC_F(sess_drop,		uint64_t, 1, 'c', info,
     "Sessions dropped",
 	"Count of sessions silently dropped due to lack of worker thread."
 )
 
-VSC_F(sess_fail,		uint64_t, 1, 'c',
+VSC_F(sess_fail,		uint64_t, 1, 'c', info,
     "Session accept failures",
 	"Count of failures to accept TCP connection."
 	"  Either the client changed its mind, or the kernel ran out of"
 	" some resource like filedescriptors."
 )
 
-VSC_F(sess_pipe_overflow,		uint64_t, 1, 'c',
+VSC_F(sess_pipe_overflow,	uint64_t, 1, 'c', info,
     "Session pipe overflow",
 	"Count of sessions dropped due to the session pipe overflowing."
 )
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(client_req_400,		uint64_t, 1, 'a',
+VSC_F(client_req_400,		uint64_t, 1, 'a', info,
     "Client requests received, subject to 400 errors",
 	"400 means we couldn't make sense of the request, it was"
 	" malformed in some drastic way."
 )
 
-VSC_F(client_req_413,		uint64_t, 1, 'a',
+VSC_F(client_req_413,		uint64_t, 1, 'a', info,
     "Client requests received, subject to 413 errors",
 	"413 means that HTTP headers execeeded length or count limits."
 )
 
-VSC_F(client_req_417,		uint64_t, 1, 'a',
+VSC_F(client_req_417,		uint64_t, 1, 'a', info,
     "Client requests received, subject to 417 errors",
 	"417 means that something went wrong with an Expect: header."
 )
 
-VSC_F(client_req,		uint64_t, 1, 'a',
+VSC_F(client_req,		uint64_t, 1, 'a', info,
     "Good Client requests received",
 	""
 )
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(cache_hit,		uint64_t, 1, 'a',
+VSC_F(cache_hit,		uint64_t, 1, 'a', info,
     "Cache hits",
 	"Count of cache hits. "
 	"  A cache hit indicates that an object has been delivered to a"
 	"  client without fetching it from a backend server."
 )
 
-VSC_F(cache_hitpass,		uint64_t, 1, 'a',
+VSC_F(cache_hitpass,		uint64_t, 1, 'a', info,
     "Cache hits for pass",
 	"Count of hits for pass"
 	"  A cache hit for pass indicates that Varnish is going to"
@@ -134,7 +134,7 @@ VSC_F(cache_hitpass,		uint64_t, 1, 'a',
 	"  decision is being used."
 )
 
-VSC_F(cache_miss,		uint64_t, 1, 'a',
+VSC_F(cache_miss,		uint64_t, 1, 'a', info,
     "Cache misses",
 	"Count of misses"
 	"  A cache miss indicates the object was fetched from the"
@@ -143,33 +143,33 @@ VSC_F(cache_miss,		uint64_t, 1, 'a',
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(backend_conn,		uint64_t, 0, 'a',
+VSC_F(backend_conn,		uint64_t, 0, 'a', info,
     "Backend conn. success",
 	""
 )
 
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a',
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a', info,
     "Backend conn. not attempted",
 	""
 )
-VSC_F(backend_busy,		uint64_t, 0, 'a',
+VSC_F(backend_busy,		uint64_t, 0, 'a', info,
     "Backend conn. too many",
 	""
 )
-VSC_F(backend_fail,		uint64_t, 0, 'a',
+VSC_F(backend_fail,		uint64_t, 0, 'a', info,
     "Backend conn. failures",
 	""
 )
-VSC_F(backend_reuse,		uint64_t, 0, 'a',
+VSC_F(backend_reuse,		uint64_t, 0, 'a', info,
     "Backend conn. reuses",
 	"Count of backend connection reuses"
 	"  This counter is increased whenever we reuse a recycled connection."
 )
-VSC_F(backend_toolate,		uint64_t, 0, 'a',
+VSC_F(backend_toolate,		uint64_t, 0, 'a', info,
     "Backend conn. was closed",
 	""
 )
-VSC_F(backend_recycle,		uint64_t, 0, 'a',
+VSC_F(backend_recycle,		uint64_t, 0, 'a', info,
     "Backend conn. recycles",
 	"Count of backend connection recycles"
 	"  This counter is increased whenever we have a keep-alive"
@@ -177,7 +177,7 @@ VSC_F(backend_recycle,		uint64_t, 0, 'a',
 	"  It has not yet been used, but it might be, unless the backend"
 	"  closes it."
 )
-VSC_F(backend_retry,		uint64_t, 0, 'a',
+VSC_F(backend_retry,		uint64_t, 0, 'a', info,
     "Backend conn. retry",
 	""
 )
@@ -186,51 +186,51 @@ VSC_F(backend_retry,		uint64_t, 0, 'a',
  * Backend fetch statistics
  */
 
-VSC_F(fetch_head,		uint64_t, 1, 'c',
+VSC_F(fetch_head,		uint64_t, 1, 'c', info,
     "Fetch no body (HEAD)",
 	"beresp with no body because the request is HEAD."
 )
-VSC_F(fetch_length,		uint64_t, 1, 'c',
+VSC_F(fetch_length,		uint64_t, 1, 'c', info,
     "Fetch with Length",
 	"beresp with Content-Length."
 )
-VSC_F(fetch_chunked,		uint64_t, 1, 'c',
+VSC_F(fetch_chunked,		uint64_t, 1, 'c', info,
     "Fetch chunked",
 	"beresp with Chunked."
 )
-VSC_F(fetch_eof,		uint64_t, 1, 'c',
+VSC_F(fetch_eof,		uint64_t, 1, 'c', info,
     "Fetch EOF",
 	"beresp with EOF from lack of other info."
 )
-VSC_F(fetch_bad,		uint64_t, 1, 'c',
+VSC_F(fetch_bad,		uint64_t, 1, 'c', info,
     "Fetch bad T-E",
 	"beresp failed due to unknown Transfer-Encoding."
 )
-VSC_F(fetch_close,		uint64_t, 1, 'c',
+VSC_F(fetch_close,		uint64_t, 1, 'c', info,
     "Fetch wanted close",
 	"beresp with EOF due to Connection: Close."
 )
-VSC_F(fetch_oldhttp,		uint64_t, 1, 'c',
+VSC_F(fetch_oldhttp,		uint64_t, 1, 'c', info,
     "Fetch pre HTTP/1.1 closed",
 	"beresp with EOF due to HTTP < 1.1"
 )
-VSC_F(fetch_zero,		uint64_t, 1, 'c',
+VSC_F(fetch_zero,		uint64_t, 1, 'c', info,
     "Fetch zero len body",
 	"beresp with EOF due to keep-live but neither Chunked or Len."
 )
-VSC_F(fetch_1xx,		uint64_t, 1, 'c',
+VSC_F(fetch_1xx,		uint64_t, 1, 'c', info,
     "Fetch no body (1xx)",
 	"beresp with no body because of 1XX response."
 )
-VSC_F(fetch_204,		uint64_t, 1, 'c',
+VSC_F(fetch_204,		uint64_t, 1, 'c', info,
     "Fetch no body (204)",
 	"beresp with no body because of 204 response."
 )
-VSC_F(fetch_304,		uint64_t, 1, 'c',
+VSC_F(fetch_304,		uint64_t, 1, 'c', info,
     "Fetch no body (304)",
 	"beresp with no body because of 304 response."
 )
-VSC_F(fetch_failed,		uint64_t, 1, 'c',
+VSC_F(fetch_failed,		uint64_t, 1, 'c', info,
     "Fetch body failed",
 	"beresp body fetch failed."
 )
@@ -241,68 +241,68 @@ VSC_F(fetch_failed,		uint64_t, 1, 'c',
  *
  */
 
-VSC_F(pools,			uint64_t, 0, 'g',
+VSC_F(pools,			uint64_t, 0, 'g', info,
     "Number of thread pools",
 	"Number of thread pools.  See also param wthread_pools."
 	"  NB: Presently pools cannot be removed once created."
 )
 
-VSC_F(threads,			uint64_t, 0, 'g',
+VSC_F(threads,			uint64_t, 0, 'g', info,
     "Total number of threads",
 	"Number of threads in all pools."
 	"  See also params thread_pools, thread_pool_min & thread_pool_max."
 )
 
-VSC_F(threads_limited,		uint64_t, 0, 'c',
+VSC_F(threads_limited,		uint64_t, 0, 'c', info,
     "Threads hit max",
 	"Number of times more threads were needed, but limit was reached"
 	" in a thread pool."
 	"  See also param thread_pool_max."
 )
 
-VSC_F(threads_created,		uint64_t, 0, 'c',
+VSC_F(threads_created,		uint64_t, 0, 'c', info,
     "Threads created",
 	"Total number of threads created in all pools."
 )
 
-VSC_F(threads_destroyed,	uint64_t, 0, 'c',
+VSC_F(threads_destroyed,	uint64_t, 0, 'c', info,
     "Threads destoryed",
 	"Total number of threads destroyed in all pools."
 )
 
-VSC_F(threads_failed,		uint64_t, 0, 'c',
+VSC_F(threads_failed,		uint64_t, 0, 'c', info,
     "Thread creation failed",
 	"Number of times creating a thread failed."
 	"  See VSL::Debug for diagnostics."
 	"  See also param thread_fail_delay."
 )
 
-VSC_F(thread_queue_len,		uint64_t, 0, 'g',
+VSC_F(thread_queue_len,		uint64_t, 0, 'g', info,
     "Length of session queue",
 	"Length of session queue waiting for threads."
 	"  NB: Only updates once per second."
 	"  See also param queue_max."
 )
 
-VSC_F(busy_sleep,		uint64_t, 1, 'c',
+VSC_F(busy_sleep,		uint64_t, 1, 'c', info,
     "Number of requests sent to sleep on busy objhdr",
 	"Number of requests sent to sleep without a worker threads because"
 	" they found a busy object."
 )
 
-VSC_F(busy_wakeup,		uint64_t, 1, 'c',
+VSC_F(busy_wakeup,		uint64_t, 1, 'c', info,
     "Number of requests woken after sleep on busy objhdr",
 	"Number of requests taken of the busy object sleep list and"
 	" and rescheduled."
 )
 
-VSC_F(sess_queued,		uint64_t, 0, 'c',
+VSC_F(sess_queued,		uint64_t, 0, 'c', info,
     "Sessions queued for thread",
 	"Number of times session was queued waiting for a thread."
 	"  See also param queue_max."
 )
 
-VSC_F(sess_dropped,		uint64_t, 0, 'c',
+VSC_F(sess_dropped,		uint64_t, 0, 'c', info,
     "Sessions dropped for thread",
 	"Number of times session was dropped because the queue were too"
 	" long already."
@@ -311,214 +311,214 @@ VSC_F(sess_dropped,		uint64_t, 0, 'c',
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(n_object,			uint64_t, 1, 'i',
+VSC_F(n_object,			uint64_t, 1, 'i', info,
     "N struct object",
 	""
 )
-VSC_F(n_vampireobject,		uint64_t, 1, 'i',
+VSC_F(n_vampireobject,		uint64_t, 1, 'i', diag,
     "N unresurrected objects",
 	""
 )
-VSC_F(n_objectcore,		uint64_t, 1, 'i',
+VSC_F(n_objectcore,		uint64_t, 1, 'i', info,
     "N struct objectcore",
 	""
 )
-VSC_F(n_objecthead,		uint64_t, 1, 'i',
+VSC_F(n_objecthead,		uint64_t, 1, 'i', info,
     "N struct objecthead",
 	""
 )
-VSC_F(n_waitinglist,		uint64_t, 1, 'i',
+VSC_F(n_waitinglist,		uint64_t, 1, 'i', debug,
     "N struct waitinglist",
 	""
 )
 
-VSC_F(n_backend,		uint64_t, 0, 'i',
+VSC_F(n_backend,		uint64_t, 0, 'i', info,
     "N backends",
 	""
 )
 
-VSC_F(n_expired,		uint64_t, 0, 'i',
+VSC_F(n_expired,		uint64_t, 0, 'i', info,
     "N expired objects",
 	""
 )
-VSC_F(n_lru_nuked,		uint64_t, 0, 'i',
+VSC_F(n_lru_nuked,		uint64_t, 0, 'i', info,
     "N LRU nuked objects",
 	""
 )
-VSC_F(n_lru_moved,		uint64_t, 0, 'i',
+VSC_F(n_lru_moved,		uint64_t, 0, 'i', diag,
     "N LRU moved objects",
 	""
 )
 
-VSC_F(losthdr,			uint64_t, 0, 'a',
+VSC_F(losthdr,			uint64_t, 0, 'a', info,
     "HTTP header overflows",
 	""
 )
 
-VSC_F(s_sess,			uint64_t, 1, 'a',
+VSC_F(s_sess,			uint64_t, 1, 'a', info,
     "Total Sessions",
 	""
 )
-VSC_F(s_req,			uint64_t, 1, 'a',
+VSC_F(s_req,			uint64_t, 1, 'a', info,
     "Total Requests",
 	""
 )
-VSC_F(s_pipe,			uint64_t, 1, 'a',
+VSC_F(s_pipe,			uint64_t, 1, 'a', info,
     "Total pipe",
 	""
 )
-VSC_F(s_pass,			uint64_t, 1, 'a',
+VSC_F(s_pass,			uint64_t, 1, 'a', info,
     "Total pass",
 	""
 )
-VSC_F(s_fetch,			uint64_t, 1, 'a',
+VSC_F(s_fetch,			uint64_t, 1, 'a', info,
     "Total fetch",
 	""
 )
-VSC_F(s_error,			uint64_t, 1, 'a',
+VSC_F(s_error,			uint64_t, 1, 'a', info,
     "Total error",
 	""
 )
-VSC_F(s_hdrbytes,		uint64_t, 1, 'a',
+VSC_F(s_hdrbytes,		uint64_t, 1, 'a', info,
     "Total header bytes",
 	""
 )
-VSC_F(s_bodybytes,		uint64_t, 1, 'a',
+VSC_F(s_bodybytes,		uint64_t, 1, 'a', info,
     "Total body bytes",
 	""
 )
 
-VSC_F(sess_closed,		uint64_t, 1, 'a',
+VSC_F(sess_closed,		uint64_t, 1, 'a', info,
     "Session Closed",
 	""
 )
-VSC_F(sess_pipeline,		uint64_t, 1, 'a',
+VSC_F(sess_pipeline,		uint64_t, 1, 'a', info,
     "Session Pipeline",
 	""
 )
-VSC_F(sess_readahead,		uint64_t, 1, 'a',
+VSC_F(sess_readahead,		uint64_t, 1, 'a', info,
     "Session Read Ahead",
 	""
 )
-VSC_F(sess_herd,		uint64_t, 1, 'a',
+VSC_F(sess_herd,		uint64_t, 1, 'a', diag,
     "Session herd",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(shm_records,		uint64_t, 0, 'a',
+VSC_F(shm_records,		uint64_t, 0, 'a', diag,
     "SHM records",
 	""
 )
-VSC_F(shm_writes,		uint64_t, 0, 'a',
+VSC_F(shm_writes,		uint64_t, 0, 'a', diag,
     "SHM writes",
 	""
 )
-VSC_F(shm_flushes,		uint64_t, 0, 'a',
+VSC_F(shm_flushes,		uint64_t, 0, 'a', diag,
     "SHM flushes due to overflow",
 	""
 )
-VSC_F(shm_cont,			uint64_t, 0, 'a',
+VSC_F(shm_cont,			uint64_t, 0, 'a', diag,
     "SHM MTX contention",
 	""
 )
-VSC_F(shm_cycles,		uint64_t, 0, 'a',
+VSC_F(shm_cycles,		uint64_t, 0, 'a', diag,
     "SHM cycles through buffer",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(sms_nreq,			uint64_t, 0, 'a',
+VSC_F(sms_nreq,			uint64_t, 0, 'a', diag,
     "SMS allocator requests",
 	""
 )
-VSC_F(sms_nobj,			uint64_t, 0, 'i',
+VSC_F(sms_nobj,			uint64_t, 0, 'i', diag,
     "SMS outstanding allocations",
 	""
 )
-VSC_F(sms_nbytes,		uint64_t, 0, 'i',
+VSC_F(sms_nbytes,		uint64_t, 0, 'i', diag,
     "SMS outstanding bytes",
 	""
 )
-VSC_F(sms_balloc,		uint64_t, 0, 'i',
+VSC_F(sms_balloc,		uint64_t, 0, 'i', diag,
     "SMS bytes allocated",
 	""
 )
-VSC_F(sms_bfree,		uint64_t, 0, 'i',
+VSC_F(sms_bfree,		uint64_t, 0, 'i', diag,
     "SMS bytes freed",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(backend_req,		uint64_t, 0, 'a',
+VSC_F(backend_req,		uint64_t, 0, 'a', info,
     "Backend requests made",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(n_vcl,			uint64_t, 0, 'a',
+VSC_F(n_vcl,			uint64_t, 0, 'a', info,
     "N vcl total",
 	""
 )
-VSC_F(n_vcl_avail,		uint64_t, 0, 'a',
+VSC_F(n_vcl_avail,		uint64_t, 0, 'a', diag,
     "N vcl available",
 	""
 )
-VSC_F(n_vcl_discard,		uint64_t, 0, 'a',
+VSC_F(n_vcl_discard,		uint64_t, 0, 'a', diag,
     "N vcl discarded",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(bans,			uint64_t, 0, 'g',
+VSC_F(bans,			uint64_t, 0, 'g', info,
    "Count of bans",
 	"Number of all bans in system, including bans superseded"
 	" by newer bans and bans already checked by the ban-lurker."
 )
-VSC_F(bans_gone,		uint64_t, 0, 'g',
+VSC_F(bans_gone,		uint64_t, 0, 'g', diag,
     "Number of bans marked 'gone'",
 	"Number of bans which are no longer active, either because they"
 	" got checked by the ban-lurker or superseded by newer identical bans."
 )
-VSC_F(bans_req,			uint64_t, 0, 'g',
+VSC_F(bans_req,			uint64_t, 0, 'g', diag,
     "Number of bans using req.*",
 	"Number of bans which use req.* variables.  These bans can not"
 	" be washed by the ban-lurker."
 )
-VSC_F(bans_added,		uint64_t, 0, 'c',
+VSC_F(bans_added,		uint64_t, 0, 'c', diag,
     "Bans added",
 	"Counter of bans added to ban list."
 )
-VSC_F(bans_deleted,		uint64_t, 0, 'c',
+VSC_F(bans_deleted,		uint64_t, 0, 'c', diag,
     "Bans deleted",
 	"Counter of bans deleted from ban list."
 )
 
-VSC_F(bans_tested,		uint64_t, 0, 'c',
+VSC_F(bans_tested,		uint64_t, 0, 'c', diag,
     "Bans tested against objects",
 	"Count of how many bans and objects have been tested against"
 	" each other."
 )
-VSC_F(bans_tests_tested,	uint64_t, 0, 'c',
+VSC_F(bans_tests_tested,	uint64_t, 0, 'c', diag,
     "Ban tests tested against objects",
 	"Count of how many tests and objects have been tested against"
 	" each other.  'ban req.url == foo && req.http.host == bar'"
 	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
 )
-VSC_F(bans_dups,		uint64_t, 0, 'c',
+VSC_F(bans_dups,		uint64_t, 0, 'c', diag,
     "Bans superseded by other bans",
 	"Count of bans replaced by later identical bans."
 )
-VSC_F(bans_persisted_bytes,	uint64_t, 0, 'g',
+VSC_F(bans_persisted_bytes,	uint64_t, 0, 'g', diag,
     "Bytes used by the persisted ban lists",
         "Number of bytes used by the persisted ban lists."
 )
-VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g',
+VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g', diag,
     "Extra bytes in persisted ban lists due to fragmentation",
         "Number of extra bytes accumulated through dropped and"
 	" gone bans in the persistent ban lists."
@@ -526,96 +526,96 @@ VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g',
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(hcb_nolock,		uint64_t, 1, 'a',
+VSC_F(hcb_nolock,		uint64_t, 1, 'a', debug,
     "HCB Lookups without lock",
 	""
 )
-VSC_F(hcb_lock,			uint64_t, 0, 'a',
+VSC_F(hcb_lock,			uint64_t, 0, 'a', debug,
     "HCB Lookups with lock",
 	""
 )
-VSC_F(hcb_insert,		uint64_t, 0, 'a',
+VSC_F(hcb_insert,		uint64_t, 0, 'a', debug,
     "HCB Inserts",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(esi_errors,		uint64_t, 0, 'a',
+VSC_F(esi_errors,		uint64_t, 0, 'a', diag,
     "ESI parse errors (unlock)",
 	""
 )
-VSC_F(esi_warnings,		uint64_t, 0, 'a',
+VSC_F(esi_warnings,		uint64_t, 0, 'a', diag,
     "ESI parse warnings (unlock)",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(dir_dns_lookups,		uint64_t, 0, 'a',
+VSC_F(dir_dns_lookups,		uint64_t, 0, 'a', diag,
     "DNS director lookups",
 	""
 )
-VSC_F(dir_dns_failed,		uint64_t, 0, 'a',
+VSC_F(dir_dns_failed,		uint64_t, 0, 'a', diag,
     "DNS director failed lookups",
 	""
 )
-VSC_F(dir_dns_hit,		uint64_t, 0, 'a',
+VSC_F(dir_dns_hit,		uint64_t, 0, 'a', diag,
     "DNS director cached lookups hit",
 	""
 )
-VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a',
+VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', debug,
     "DNS director full dnscache",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(vmods,			uint64_t, 0, 'i',
+VSC_F(vmods,			uint64_t, 0, 'i', info,
     "Loaded VMODs",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(n_gzip,			uint64_t, 0, 'a',
+VSC_F(n_gzip,			uint64_t, 0, 'a', info,
     "Gzip operations",
 	""
 )
-VSC_F(n_gunzip,			uint64_t, 0, 'a',
+VSC_F(n_gunzip,			uint64_t, 0, 'a', info,
     "Gunzip operations",
 	""
 )
 
 /*--------------------------------------------------------------------*/
 
-VSC_F(vsm_free,			uint64_t, 0, 'g',
+VSC_F(vsm_free,			uint64_t, 0, 'g', diag,
     "Free VSM space",
 	"Number of bytes free in the shared memory used to communicate"
 	" with tools like varnishstat, varnishlog etc."
 )
 
-VSC_F(vsm_used,			uint64_t, 0, 'g',
+VSC_F(vsm_used,			uint64_t, 0, 'g', diag,
     "Used VSM space",
 	"Number of bytes used in the shared memory used to communicate"
 	" with tools like varnishstat, varnishlog etc."
 )
 
-VSC_F(vsm_cooling,		uint64_t, 0, 'g',
+VSC_F(vsm_cooling,		uint64_t, 0, 'g', debug,
     "Cooling VSM space",
 	"Number of bytes which will soon (max 1 minute) be freed"
 	" in the shared memory used to communicate"
 	" with tools like varnishstat, varnishlog etc."
 )
 
-VSC_F(vsm_overflow,		uint64_t, 0, 'g',
+VSC_F(vsm_overflow,		uint64_t, 0, 'g', diag,
     "Overflow VSM space",
 	"Number of bytes which does not fit"
 	" in the shared memory used to communicate"
 	" with tools like varnishstat, varnishlog etc."
 )
 
-VSC_F(vsm_overflowed,		uint64_t, 0, 'c',
+VSC_F(vsm_overflowed,		uint64_t, 0, 'c', diag,
     "Overflowed VSM space",
 	"Total number of bytes which did not fit"
 	" in the shared memory used to communicate"
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 017398f..85fe2e4 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -28,7 +28,7 @@
  *
  * Definition of all shared memory statistics below.
  *
- * Fields (n, t, l, f, e, d):
+ * Fields (n, t, l, f, v, e, d):
  *    n - Name:		Field name, in C-source and stats programs
  *    t - Type:		C-type, uint64_t, unless marked in 'f'
  *    l - Local:	Local counter in worker thread.
@@ -38,6 +38,7 @@
  *				'c' - Counter, never decreases.
  *				'g' - Gauge, goes up and down
  *				'i' - Integer (deprecated, use 'g')
+ *    v - Verbosity:	Counter verbosity level (see vsc_levels.h)
  *    e - Explantion:	Short explanation of field (for screen use)
  *    d - Description:	Long explanation of field (for doc use)
  *
@@ -61,32 +62,32 @@
 
 #ifdef VSC_DO_MGT
 
-VSC_F(uptime,			uint64_t, 0, 'c',
+VSC_F(uptime,			uint64_t, 0, 'c', info,
     "Management process uptime",
 	"Uptime in seconds of the management process"
 )
-VSC_F(child_start,		uint64_t, 0, 'c',
+VSC_F(child_start,		uint64_t, 0, 'c', diag,
     "Child process started",
 	"Number of times the child process has been started"
 )
-VSC_F(child_exit,		uint64_t, 0, 'c',
+VSC_F(child_exit,		uint64_t, 0, 'c', diag,
     "Child process normal exit",
 	"Number of times the child process has been cleanly stopped"
 )
-VSC_F(child_stop,		uint64_t, 0, 'c',
+VSC_F(child_stop,		uint64_t, 0, 'c', diag,
     "Child process unexpected exit",
 	"Number of times the child process has exited with an unexpected"
 	" return code"
 )
-VSC_F(child_died,		uint64_t, 0, 'c',
+VSC_F(child_died,		uint64_t, 0, 'c', diag,
     "Child process died (signal)",
 	"Number of times the child process has died due to signals"
 )
-VSC_F(child_dump,		uint64_t, 0, 'c',
+VSC_F(child_dump,		uint64_t, 0, 'c', diag,
     "Child process core dumped",
 	"Number of times the child process has produced core dumps"
 )
-VSC_F(child_panic,		uint64_t, 0, 'c',
+VSC_F(child_panic,		uint64_t, 0, 'c', diag,
     "Child process panic",
 	"Number of times the management process has caught a child panic"
 )
@@ -97,15 +98,15 @@ VSC_F(child_panic,		uint64_t, 0, 'c',
 
 #ifdef VSC_DO_LCK
 
-VSC_F(creat,			uint64_t, 0, 'a',
+VSC_F(creat,			uint64_t, 0, 'a', debug,
     "Created locks",
 	""
 )
-VSC_F(destroy,			uint64_t, 0, 'a',
+VSC_F(destroy,			uint64_t, 0, 'a', debug,
     "Destroyed locks",
 	""
 )
-VSC_F(locks,			uint64_t, 0, 'a',
+VSC_F(locks,			uint64_t, 0, 'a', debug,
     "Lock Operations",
 	""
 )
@@ -117,31 +118,31 @@ VSC_F(locks,			uint64_t, 0, 'a',
  */
 
 #if defined(VSC_DO_SMA) || defined (VSC_DO_SMF)
-VSC_F(c_req,			uint64_t, 0, 'a',
+VSC_F(c_req,			uint64_t, 0, 'a', info,
     "Allocator requests",
 	""
 )
-VSC_F(c_fail,			uint64_t, 0, 'a',
+VSC_F(c_fail,			uint64_t, 0, 'a', info,
     "Allocator failures",
 	""
 )
-VSC_F(c_bytes,			uint64_t, 0, 'a',
+VSC_F(c_bytes,			uint64_t, 0, 'a', info,
     "Bytes allocated",
 	""
 )
-VSC_F(c_freed,			uint64_t, 0, 'a',
+VSC_F(c_freed,			uint64_t, 0, 'a', info,
     "Bytes freed",
 	""
 )
-VSC_F(g_alloc,			uint64_t, 0, 'i',
+VSC_F(g_alloc,			uint64_t, 0, 'i', info,
     "Allocations outstanding",
 	""
 )
-VSC_F(g_bytes,			uint64_t, 0, 'i',
+VSC_F(g_bytes,			uint64_t, 0, 'i', info,
     "Bytes outstanding",
 	""
 )
-VSC_F(g_space,			uint64_t, 0, 'i',
+VSC_F(g_space,			uint64_t, 0, 'i', info,
     "Bytes available",
 	""
 )
@@ -157,15 +158,15 @@ VSC_F(g_space,			uint64_t, 0, 'i',
 /**********************************************************************/
 
 #ifdef VSC_DO_SMF
-VSC_F(g_smf,			uint64_t, 0, 'i',
+VSC_F(g_smf,			uint64_t, 0, 'i', info,
     "N struct smf",
 	""
 )
-VSC_F(g_smf_frag,		uint64_t, 0, 'i',
+VSC_F(g_smf_frag,		uint64_t, 0, 'i', info,
     "N small free smf",
 	""
 )
-VSC_F(g_smf_large,		uint64_t, 0, 'i',
+VSC_F(g_smf_large,		uint64_t, 0, 'i', info,
     "N large free smf",
 	""
 )
@@ -175,11 +176,11 @@ VSC_F(g_smf_large,		uint64_t, 0, 'i',
 
 #ifdef VSC_DO_VBE
 
-VSC_F(vcls,			uint64_t, 0, 'i',
+VSC_F(vcls,			uint64_t, 0, 'i', debug,
     "VCL references",
 	""
 )
-VSC_F(happy,			uint64_t, 0, 'b',
+VSC_F(happy,			uint64_t, 0, 'b', info,
     "Happy health probes",
 	""
 )
@@ -189,47 +190,47 @@ VSC_F(happy,			uint64_t, 0, 'b',
 /**********************************************************************/
 #ifdef VSC_DO_MEMPOOL
 
-VSC_F(live,			uint64_t, 0, 'g',
+VSC_F(live,			uint64_t, 0, 'g', debug,
     "In use",
 	""
 )
-VSC_F(pool,			uint64_t, 0, 'g',
+VSC_F(pool,			uint64_t, 0, 'g', debug,
     "In Pool",
 	""
 )
-VSC_F(sz_wanted,		uint64_t, 0, 'g',
+VSC_F(sz_wanted,		uint64_t, 0, 'g', debug,
     "Size requested",
 	""
 )
-VSC_F(sz_needed,		uint64_t, 0, 'g',
+VSC_F(sz_needed,		uint64_t, 0, 'g', debug,
     "Size allocated",
 	""
 )
-VSC_F(allocs,			uint64_t, 0, 'c',
+VSC_F(allocs,			uint64_t, 0, 'c', debug,
     "Allocations",
 	""
 )
-VSC_F(frees,			uint64_t, 0, 'c',
+VSC_F(frees,			uint64_t, 0, 'c', debug,
     "Frees",
 	""
 )
-VSC_F(recycle,			uint64_t, 0, 'c',
+VSC_F(recycle,			uint64_t, 0, 'c', debug,
     "Recycled from pool",
 	""
 )
-VSC_F(timeout,			uint64_t, 0, 'c',
+VSC_F(timeout,			uint64_t, 0, 'c', debug,
     "Timed out from pool",
 	""
 )
-VSC_F(toosmall,			uint64_t, 0, 'c',
+VSC_F(toosmall,			uint64_t, 0, 'c', debug,
     "Too small to recycle",
 	""
 )
-VSC_F(surplus,			uint64_t, 0, 'c',
+VSC_F(surplus,			uint64_t, 0, 'c', debug,
     "Too many for pool",
 	""
 )
-VSC_F(randry,			uint64_t, 0, 'c',
+VSC_F(randry,			uint64_t, 0, 'c', debug,
     "Pool ran dry",
 	""
 )
diff --git a/include/tbl/vsc_levels.h b/include/tbl/vsc_levels.h
new file mode 100644
index 0000000..9471a9a
--- /dev/null
+++ b/include/tbl/vsc_levels.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Fields (v, l, e, d):
+ *    v - Verbosity lvl:	Field name, in C-source
+ *    l - Label:		Display name, in stats programs
+ *    e - Explanation:		Short description of this counter type
+ *    d - Description:		Long description of this counter type
+ */
+
+VSC_LEVEL_F(info,	"INFO",		"Informational counters",
+    "Counters giving runtime information")
+VSC_LEVEL_F(diag,	"DIAG",		"Diagnostic counters",
+    "Counters giving diagnostic information")
+VSC_LEVEL_F(debug,	"DEBUG",	"Debug counters",
+    "Counters giving Varnish internals debug information")
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 4d21be2..9aefac2 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -71,11 +71,19 @@ struct VSC_C_main *VSC_Main(struct VSM_data *vd);
 	 * returns NULL until child has been started.
 	 */
 
+struct VSC_level_desc;
 struct VSC_type_desc;
 struct VSC_section;
 struct VSC_desc;
 struct VSC_point;
 
+struct VSC_level_desc {
+	unsigned verbosity;
+	const char *label;		/* label */
+	const char *sdesc;		/* short description */
+	const char *ldesc;		/* long description */
+};
+
 struct VSC_type_desc {
 	const char *label;		/* label */
 	const char *sdesc;		/* short description */
@@ -95,6 +103,7 @@ struct VSC_desc {
 	int flag;			/* 'c' = counter, 'g' = gauge	*/
 	const char *sdesc;		/* short description		*/
 	const char *ldesc;		/* long description		*/
+	const struct VSC_level_desc *level;
 };
 
 struct VSC_point {
@@ -150,17 +159,24 @@ int VSC_IterValid(struct VSM_data *vd);
 	 *	1: fantom is still the same.
 	 */
 
+const struct VSC_level_desc *VSC_LevelDesc(unsigned level);
+
 /**********************************************************************
  * Precompiled VSC_type_desc's and VSC_desc's for all know VSCs.
  */
 
+#define VSC_LEVEL_F(v,l,e,d) \
+	extern const struct VSC_level_desc VSC_level_desc_##v;
+#include "tbl/vsc_levels.h"
+#undef VSC_LEVEL_F
+
 #define VSC_TYPE_F(n,t,l,e,d) \
 	extern const struct VSC_type_desc VSC_type_desc_##n;
 #include "tbl/vsc_types.h"
 #undef VSC_TYPE_F
 
 #define VSC_DO(U,l,t) extern const struct VSC_desc VSC_desc_##l[];
-#define VSC_F(n,t,l,f,d,e)
+#define VSC_F(n,t,l,f,v,d,e)
 #define VSC_DONE(U,l,t)
 #include "tbl/vsc_all.h"
 #undef VSC_DO
diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h
index 2109379..3583d2b 100644
--- a/include/vapi/vsc_int.h
+++ b/include/vapi/vsc_int.h
@@ -30,6 +30,15 @@
 
 #define VSC_CLASS		"Stat"
 
+/* vsc levels */
+enum VSC_level_e {
+#define VSC_LEVEL_F(v,l,e,d)		\
+	VSC_level_##v,
+#include "tbl/vsc_levels.h"
+#undef VSC_LEVEL_F
+	VSC_level_MAX,
+};
+
 /* Forward declare the static vsc type names */
 #define VSC_TYPE_F(n,t,l,e,d)		extern const char *VSC_type_##n;
 #include "tbl/vsc_types.h"
@@ -37,7 +46,7 @@
 
 /* Define the vsc type structs */
 #define VSC_DO(u,l,t)			struct VSC_C_##l {
-#define VSC_F(n,t,l,f,e,d)			t n;
+#define VSC_F(n,t,l,f,v,e,d)			t n;
 #define VSC_DONE(u,l,t)			};
 #include "tbl/vsc_all.h"
 #undef VSC_DO
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 6892b86..34a851d 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -94,5 +94,6 @@ LIBVARNISHAPI_1.3 {
 	VSC_MgtValid;
 	VSC_MainValid;
 	VSC_IterValid;
+	VSC_LevelDesc;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index fc35d28..97fc953 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -340,7 +340,7 @@ vsc_add_pt(struct vsc *vsc, const volatile void *ptr,
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
 		st = vf->fantom.b;
 
-#define VSC_F(nn,tt,ll,ff,dd,ee)					\
+#define VSC_F(nn,tt,ll,ff,vv,dd,ee)			\
 		vsc_add_pt(vsc, &st->nn, descs++, vf);
 
 #define VSC_DONE(U,l,t)							\
@@ -390,7 +390,7 @@ vsc_build_pt_list(struct VSM_data *vd)
 		CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC);		\
 		if (!strcmp(vf->fantom.type, t))		\
 			iter_##l(vsc, VSC_desc_##l, vf);
-#define VSC_F(n,t,l,f,d,e)
+#define VSC_F(n,t,l,f,v,d,e)
 #define VSC_DONE(a,b,c)
 #include "tbl/vsc_all.h"
 #undef VSC_DO
@@ -519,10 +519,30 @@ VSC_IterValid(struct VSM_data *vd)
 	return (v);
 }
 
+const struct VSC_level_desc *
+VSC_LevelDesc(unsigned level)
+{
+	switch (level) {
+#define VSC_LEVEL_F(v,l,e,d)	\
+	case VSC_level_##v:	\
+		return (&VSC_level_desc_##v);
+#include "tbl/vsc_levels.h"
+#undef VSC_LEVEL_F
+	default:
+		return (NULL);
+	}
+}
+
 /*--------------------------------------------------------------------
- * Build the static type and point descriptions
+ * Build the static level, type and point descriptions
  */
 
+#define VSC_LEVEL_F(v,l,e,d)			\
+	const struct VSC_level_desc VSC_level_desc_##v = \
+		{VSC_level_##v, l, e, d};
+#include "tbl/vsc_levels.h"
+#undef VSC_LEVEL_F
+
 #define VSC_TYPE_F(n,t,l,e,d)	const char *VSC_type_##n = t;
 #include "tbl/vsc_types.h"
 #undef VSC_TYPE_F
@@ -533,7 +553,7 @@ VSC_IterValid(struct VSM_data *vd)
 #undef VSC_TYPE_F
 
 #define VSC_DO(U,l,t)		const struct VSC_desc VSC_desc_##l[] = {
-#define VSC_F(n,t,l,f,d,e)		{#n,#t,f,d,e},
+#define VSC_F(n,t,l,f,v,d,e)		{#n,#t,f,d,e,&VSC_level_desc_##v},
 #define VSC_DONE(U,l,t)		};
 #include "tbl/vsc_all.h"
 #undef VSC_DO
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
index 4ea94be..d79453b 100644
--- a/man/vsc2rst.c
+++ b/man/vsc2rst.c
@@ -3,8 +3,17 @@
 
 #include <stdio.h>
 
-#define P(x, ...) printf(x "\n", ##__VA_ARGS__)
-#define VSC_F(n, t, l, f, e, d) printf("%s ? %s\n\t%s\n\n", #n, e, d);
+#define VSC_LEVEL_F(v,l,e,d)		\
+	static const char VSC_level_##v[] = l;
+#include "tbl/vsc_levels.h"
+#undef VSC_LEVEL_F
+
+#define P(x, ...)			\
+	printf(x "\n", ##__VA_ARGS__)
+#define VSC_LEVEL_F(v,l,e,d)		\
+	printf("%s ? %s\n\t%s\n\n", l, e, d);
+#define VSC_F(n, t, l, f, v, e, d)	\
+	printf("%s ? %s (%s)\n\t%s\n\n", #n, e, VSC_level_##v, d);
 
 int main(int argc, char **argv)
 {
@@ -25,6 +34,12 @@ int main(int argc, char **argv)
 	P(":Manual section: 7");
 	P("");
 
+	P("COUNTER LEVELS");
+	P("==============");
+	P("");
+#include "tbl/vsc_levels.h"
+
+	P("");
 	P("MAIN COUNTERS");
 	P("=============");
 	P("");



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 285036a Add verbosity filter to interactive varnishstat
Message-ID: <E1Ucb5y-0004FC-1f@project.varnish-software.com>

commit 285036abc7c053569b25c55890383c9994b16d7b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 14 16:08:26 2013 +0100

    Add verbosity filter to interactive varnishstat
    
    Show UNSEEN in bottom bar when not hiding unseened counters

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 5d19dc1..d81c5bc 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -113,6 +113,7 @@ static WINDOW *w_points = NULL;
 static WINDOW *w_bar_b = NULL;
 static WINDOW *w_info = NULL;
 
+static int verbosity = VSC_level_info;
 static int keep_running = 1;
 static int show_info = 1;
 static int hide_unseen = 1;
@@ -198,6 +199,8 @@ build_pt_array(void)
 		CHECK_OBJ_NOTNULL(pt, PT_MAGIC);
 		if (!pt->seen && hide_unseen)
 			continue;
+		if (pt->vpt->desc->level->verbosity > verbosity)
+			continue;
 		assert(n_ptarray < n_ptlist);
 		ptarray[n_ptarray++] = pt;
 	}
@@ -722,11 +725,13 @@ draw_points(void)
 static void
 draw_bar_b(void)
 {
-	int x;
+	int x, X;
+	const struct VSC_level_desc *level;
 
 	AN(w_bar_b);
 
 	x = 0;
+	X = getmaxx(w_bar_b);
 	werase(w_bar_b);
 	if (page_start + l_points < n_ptarray)
 		mvwaddch(w_bar_b, 0, x, ACS_DARROW);
@@ -734,6 +739,13 @@ draw_bar_b(void)
 	if (current < n_ptarray - 1)
 		mvwprintw(w_bar_b, 0, x, "%s", ptarray[current]->name);
 
+	level = VSC_LevelDesc(verbosity);
+	if (level != NULL)
+		mvwprintw(w_bar_b, 0, X - 7, "%7s", level->label);
+	X -= 7;
+	if (!hide_unseen)
+		mvwprintw(w_bar_b, 0, X - 6, "%6s", "UNSEEN");
+
 	wnoutrefresh(w_bar_b);
 }
 
@@ -801,6 +813,12 @@ handle_keypress(int ch)
 		hide_unseen = 1 - hide_unseen;
 		rebuild = 1;
 		break;
+	case 'v':
+		verbosity++;
+		if (VSC_LevelDesc(verbosity) == NULL)
+			verbosity = 0;
+		rebuild = 1;
+		break;
 	case 'q':
 	case 'Q':
 		keep_running = 0;



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 507a101 Add VSL Query/Dispatch structured log messages
 facility
Message-ID: <E1Ucb5y-0004GK-EX@project.varnish-software.com>

commit 507a101956fcd27f2e6e3ba1e403684b32040bc6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Apr 24 09:49:42 2013 +0200

    Add VSL Query/Dispatch structured log messages facility
    
    Move vtree.h from varnishtop to include directory so it's available for anyone.
    
    Simple proof-of-concept single regex against any log line query
    implementation.

diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index 1a6ecc4..b2f5c89 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -8,8 +8,7 @@ dist_man_MANS = varnishtop.1
 
 varnishtop_SOURCES = varnishtop.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c \
-	vtree.h
+	$(top_builddir)/lib/libvarnish/version.c
 
 varnishtop_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/bin/varnishtop/vtree.h b/bin/varnishtop/vtree.h
deleted file mode 100644
index 7692d12..0000000
--- a/bin/varnishtop/vtree.h
+++ /dev/null
@@ -1,763 +0,0 @@
-/*	$NetBSD: tree.h,v 1.8 2004/03/28 19:38:30 provos Exp $	*/
-/*	$OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $	*/
-/* $FreeBSD: release/9.0.0/sys/sys/tree.h 189204 2009-03-01 04:57:23Z bms $ */
-
-/*-
- * Copyright 2002 Niels Provos <provos at citi.umich.edu>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef	_VTREE_H_
-#define	_VTREE_H_
-
-/*
- * This file defines data structures for different types of trees:
- * splay trees and red-black trees.
- *
- * A splay tree is a self-organizing data structure.  Every operation
- * on the tree causes a splay to happen.  The splay moves the requested
- * node to the root of the tree and partly rebalances it.
- *
- * This has the benefit that request locality causes faster lookups as
- * the requested nodes move to the top of the tree.  On the other hand,
- * every lookup causes memory writes.
- *
- * The Balance Theorem bounds the total access time for m operations
- * and n inserts on an initially empty tree as O((m + n)lg n).  The
- * amortized cost for a sequence of m accesses to a splay tree is O(lg n);
- *
- * A red-black tree is a binary search tree with the node color as an
- * extra attribute.  It fulfills a set of conditions:
- *	- every search path from the root to a leaf consists of the
- *	  same number of black nodes,
- *	- each red node (except for the root) has a black parent,
- *	- each leaf node is black.
- *
- * Every operation on a red-black tree is bounded as O(lg n).
- * The maximum height of a red-black tree is 2lg (n+1).
- */
-
-#define VSPLAY_HEAD(name, type)						\
-struct name {								\
-	struct type *sph_root; /* root of the tree */			\
-}
-
-#define VSPLAY_INITIALIZER(root)						\
-	{ NULL }
-
-#define VSPLAY_INIT(root) do {						\
-	(root)->sph_root = NULL;					\
-} while (/*CONSTCOND*/ 0)
-
-#define VSPLAY_ENTRY(type)						\
-struct {								\
-	struct type *spe_left; /* left element */			\
-	struct type *spe_right; /* right element */			\
-}
-
-#define VSPLAY_LEFT(elm, field)		(elm)->field.spe_left
-#define VSPLAY_RIGHT(elm, field)		(elm)->field.spe_right
-#define VSPLAY_ROOT(head)		(head)->sph_root
-#define VSPLAY_EMPTY(head)		(VSPLAY_ROOT(head) == NULL)
-
-/* VSPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold VSPLAY_{RIGHT,LEFT} */
-#define VSPLAY_ROTATE_RIGHT(head, tmp, field) do {			\
-	VSPLAY_LEFT((head)->sph_root, field) = VSPLAY_RIGHT(tmp, field);	\
-	VSPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
-	(head)->sph_root = tmp;						\
-} while (/*CONSTCOND*/ 0)
-
-#define VSPLAY_ROTATE_LEFT(head, tmp, field) do {			\
-	VSPLAY_RIGHT((head)->sph_root, field) = VSPLAY_LEFT(tmp, field);	\
-	VSPLAY_LEFT(tmp, field) = (head)->sph_root;			\
-	(head)->sph_root = tmp;						\
-} while (/*CONSTCOND*/ 0)
-
-#define VSPLAY_LINKLEFT(head, tmp, field) do {				\
-	VSPLAY_LEFT(tmp, field) = (head)->sph_root;			\
-	tmp = (head)->sph_root;						\
-	(head)->sph_root = VSPLAY_LEFT((head)->sph_root, field);		\
-} while (/*CONSTCOND*/ 0)
-
-#define VSPLAY_LINKRIGHT(head, tmp, field) do {				\
-	VSPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
-	tmp = (head)->sph_root;						\
-	(head)->sph_root = VSPLAY_RIGHT((head)->sph_root, field);	\
-} while (/*CONSTCOND*/ 0)
-
-#define VSPLAY_ASSEMBLE(head, node, left, right, field) do {		\
-	VSPLAY_RIGHT(left, field) = VSPLAY_LEFT((head)->sph_root, field);	\
-	VSPLAY_LEFT(right, field) = VSPLAY_RIGHT((head)->sph_root, field);\
-	VSPLAY_LEFT((head)->sph_root, field) = VSPLAY_RIGHT(node, field);	\
-	VSPLAY_RIGHT((head)->sph_root, field) = VSPLAY_LEFT(node, field);	\
-} while (/*CONSTCOND*/ 0)
-
-/* Generates prototypes and inline functions */
-
-#define VSPLAY_PROTOTYPE(name, type, field, cmp)				\
-void name##_VSPLAY(struct name *, struct type *);			\
-void name##_VSPLAY_MINMAX(struct name *, int);				\
-struct type *name##_VSPLAY_INSERT(struct name *, struct type *);		\
-struct type *name##_VSPLAY_REMOVE(struct name *, struct type *);		\
-									\
-/* Finds the node with the same key as elm */				\
-static __inline struct type *						\
-name##_VSPLAY_FIND(struct name *head, struct type *elm)			\
-{									\
-	if (VSPLAY_EMPTY(head))						\
-		return(NULL);						\
-	name##_VSPLAY(head, elm);					\
-	if ((cmp)(elm, (head)->sph_root) == 0)				\
-		return (head->sph_root);				\
-	return (NULL);							\
-}									\
-									\
-static __inline struct type *						\
-name##_VSPLAY_NEXT(struct name *head, struct type *elm)			\
-{									\
-	name##_VSPLAY(head, elm);					\
-	if (VSPLAY_RIGHT(elm, field) != NULL) {				\
-		elm = VSPLAY_RIGHT(elm, field);				\
-		while (VSPLAY_LEFT(elm, field) != NULL) {		\
-			elm = VSPLAY_LEFT(elm, field);			\
-		}							\
-	} else								\
-		elm = NULL;						\
-	return (elm);							\
-}									\
-									\
-static __inline struct type *						\
-name##_VSPLAY_MIN_MAX(struct name *head, int val)			\
-{									\
-	name##_VSPLAY_MINMAX(head, val);					\
-        return (VSPLAY_ROOT(head));					\
-}
-
-/* Main splay operation.
- * Moves node close to the key of elm to top
- */
-#define VSPLAY_GENERATE(name, type, field, cmp)				\
-struct type *								\
-name##_VSPLAY_INSERT(struct name *head, struct type *elm)		\
-{									\
-    if (VSPLAY_EMPTY(head)) {						\
-	    VSPLAY_LEFT(elm, field) = VSPLAY_RIGHT(elm, field) = NULL;	\
-    } else {								\
-	    int __comp;							\
-	    name##_VSPLAY(head, elm);					\
-	    __comp = (cmp)(elm, (head)->sph_root);			\
-	    if(__comp < 0) {						\
-		    VSPLAY_LEFT(elm, field) = VSPLAY_LEFT((head)->sph_root, field);\
-		    VSPLAY_RIGHT(elm, field) = (head)->sph_root;		\
-		    VSPLAY_LEFT((head)->sph_root, field) = NULL;		\
-	    } else if (__comp > 0) {					\
-		    VSPLAY_RIGHT(elm, field) = VSPLAY_RIGHT((head)->sph_root, field);\
-		    VSPLAY_LEFT(elm, field) = (head)->sph_root;		\
-		    VSPLAY_RIGHT((head)->sph_root, field) = NULL;	\
-	    } else							\
-		    return ((head)->sph_root);				\
-    }									\
-    (head)->sph_root = (elm);						\
-    return (NULL);							\
-}									\
-									\
-struct type *								\
-name##_VSPLAY_REMOVE(struct name *head, struct type *elm)		\
-{									\
-	struct type *__tmp;						\
-	if (VSPLAY_EMPTY(head))						\
-		return (NULL);						\
-	name##_VSPLAY(head, elm);					\
-	if ((cmp)(elm, (head)->sph_root) == 0) {			\
-		if (VSPLAY_LEFT((head)->sph_root, field) == NULL) {	\
-			(head)->sph_root = VSPLAY_RIGHT((head)->sph_root, field);\
-		} else {						\
-			__tmp = VSPLAY_RIGHT((head)->sph_root, field);	\
-			(head)->sph_root = VSPLAY_LEFT((head)->sph_root, field);\
-			name##_VSPLAY(head, elm);			\
-			VSPLAY_RIGHT((head)->sph_root, field) = __tmp;	\
-		}							\
-		return (elm);						\
-	}								\
-	return (NULL);							\
-}									\
-									\
-void									\
-name##_VSPLAY(struct name *head, struct type *elm)			\
-{									\
-	struct type __node, *__left, *__right, *__tmp;			\
-	int __comp;							\
-\
-	VSPLAY_LEFT(&__node, field) = VSPLAY_RIGHT(&__node, field) = NULL;\
-	__left = __right = &__node;					\
-\
-	while ((__comp = (cmp)(elm, (head)->sph_root)) != 0) {		\
-		if (__comp < 0) {					\
-			__tmp = VSPLAY_LEFT((head)->sph_root, field);	\
-			if (__tmp == NULL)				\
-				break;					\
-			if ((cmp)(elm, __tmp) < 0){			\
-				VSPLAY_ROTATE_RIGHT(head, __tmp, field);	\
-				if (VSPLAY_LEFT((head)->sph_root, field) == NULL)\
-					break;				\
-			}						\
-			VSPLAY_LINKLEFT(head, __right, field);		\
-		} else if (__comp > 0) {				\
-			__tmp = VSPLAY_RIGHT((head)->sph_root, field);	\
-			if (__tmp == NULL)				\
-				break;					\
-			if ((cmp)(elm, __tmp) > 0){			\
-				VSPLAY_ROTATE_LEFT(head, __tmp, field);	\
-				if (VSPLAY_RIGHT((head)->sph_root, field) == NULL)\
-					break;				\
-			}						\
-			VSPLAY_LINKRIGHT(head, __left, field);		\
-		}							\
-	}								\
-	VSPLAY_ASSEMBLE(head, &__node, __left, __right, field);		\
-}									\
-									\
-/* Splay with either the minimum or the maximum element			\
- * Used to find minimum or maximum element in tree.			\
- */									\
-void name##_VSPLAY_MINMAX(struct name *head, int __comp) \
-{									\
-	struct type __node, *__left, *__right, *__tmp;			\
-\
-	VSPLAY_LEFT(&__node, field) = VSPLAY_RIGHT(&__node, field) = NULL;\
-	__left = __right = &__node;					\
-\
-	while (1) {							\
-		if (__comp < 0) {					\
-			__tmp = VSPLAY_LEFT((head)->sph_root, field);	\
-			if (__tmp == NULL)				\
-				break;					\
-			if (__comp < 0){				\
-				VSPLAY_ROTATE_RIGHT(head, __tmp, field);	\
-				if (VSPLAY_LEFT((head)->sph_root, field) == NULL)\
-					break;				\
-			}						\
-			VSPLAY_LINKLEFT(head, __right, field);		\
-		} else if (__comp > 0) {				\
-			__tmp = VSPLAY_RIGHT((head)->sph_root, field);	\
-			if (__tmp == NULL)				\
-				break;					\
-			if (__comp > 0) {				\
-				VSPLAY_ROTATE_LEFT(head, __tmp, field);	\
-				if (VSPLAY_RIGHT((head)->sph_root, field) == NULL)\
-					break;				\
-			}						\
-			VSPLAY_LINKRIGHT(head, __left, field);		\
-		}							\
-	}								\
-	VSPLAY_ASSEMBLE(head, &__node, __left, __right, field);		\
-}
-
-#define VSPLAY_NEGINF	-1
-#define VSPLAY_INF	1
-
-#define VSPLAY_INSERT(name, x, y)	name##_VSPLAY_INSERT(x, y)
-#define VSPLAY_REMOVE(name, x, y)	name##_VSPLAY_REMOVE(x, y)
-#define VSPLAY_FIND(name, x, y)		name##_VSPLAY_FIND(x, y)
-#define VSPLAY_NEXT(name, x, y)		name##_VSPLAY_NEXT(x, y)
-#define VSPLAY_MIN(name, x)		(VSPLAY_EMPTY(x) ? NULL	\
-					: name##_VSPLAY_MIN_MAX(x, VSPLAY_NEGINF))
-#define VSPLAY_MAX(name, x)		(VSPLAY_EMPTY(x) ? NULL	\
-					: name##_VSPLAY_MIN_MAX(x, VSPLAY_INF))
-
-#define VSPLAY_FOREACH(x, name, head)					\
-	for ((x) = VSPLAY_MIN(name, head);				\
-	     (x) != NULL;						\
-	     (x) = VSPLAY_NEXT(name, head, x))
-
-/* Macros that define a red-black tree */
-#define VRB_HEAD(name, type)						\
-struct name {								\
-	struct type *rbh_root; /* root of the tree */			\
-}
-
-#define VRB_INITIALIZER(root)						\
-	{ NULL }
-
-#define VRB_INIT(root) do {						\
-	(root)->rbh_root = NULL;					\
-} while (/*CONSTCOND*/ 0)
-
-#define VRB_BLACK	0
-#define VRB_RED		1
-#define VRB_ENTRY(type)							\
-struct {								\
-	struct type *rbe_left;		/* left element */		\
-	struct type *rbe_right;		/* right element */		\
-	struct type *rbe_parent;	/* parent element */		\
-	int rbe_color;			/* node color */		\
-}
-
-#define VRB_LEFT(elm, field)		(elm)->field.rbe_left
-#define VRB_RIGHT(elm, field)		(elm)->field.rbe_right
-#define VRB_PARENT(elm, field)		(elm)->field.rbe_parent
-#define VRB_COLOR(elm, field)		(elm)->field.rbe_color
-#define VRB_ROOT(head)			(head)->rbh_root
-#define VRB_EMPTY(head)			(VRB_ROOT(head) == NULL)
-
-#define VRB_SET(elm, parent, field) do {					\
-	VRB_PARENT(elm, field) = parent;					\
-	VRB_LEFT(elm, field) = VRB_RIGHT(elm, field) = NULL;		\
-	VRB_COLOR(elm, field) = VRB_RED;					\
-} while (/*CONSTCOND*/ 0)
-
-#define VRB_SET_BLACKRED(black, red, field) do {				\
-	VRB_COLOR(black, field) = VRB_BLACK;				\
-	VRB_COLOR(red, field) = VRB_RED;					\
-} while (/*CONSTCOND*/ 0)
-
-#ifndef VRB_AUGMENT
-#define VRB_AUGMENT(x)	do {} while (0)
-#endif
-
-#define VRB_ROTATE_LEFT(head, elm, tmp, field) do {			\
-	(tmp) = VRB_RIGHT(elm, field);					\
-	if ((VRB_RIGHT(elm, field) = VRB_LEFT(tmp, field)) != NULL) {	\
-		VRB_PARENT(VRB_LEFT(tmp, field), field) = (elm);		\
-	}								\
-	VRB_AUGMENT(elm);						\
-	if ((VRB_PARENT(tmp, field) = VRB_PARENT(elm, field)) != NULL) {	\
-		if ((elm) == VRB_LEFT(VRB_PARENT(elm, field), field))	\
-			VRB_LEFT(VRB_PARENT(elm, field), field) = (tmp);	\
-		else							\
-			VRB_RIGHT(VRB_PARENT(elm, field), field) = (tmp);	\
-	} else								\
-		(head)->rbh_root = (tmp);				\
-	VRB_LEFT(tmp, field) = (elm);					\
-	VRB_PARENT(elm, field) = (tmp);					\
-	VRB_AUGMENT(tmp);						\
-	if ((VRB_PARENT(tmp, field)))					\
-		VRB_AUGMENT(VRB_PARENT(tmp, field));			\
-} while (/*CONSTCOND*/ 0)
-
-#define VRB_ROTATE_RIGHT(head, elm, tmp, field) do {			\
-	(tmp) = VRB_LEFT(elm, field);					\
-	if ((VRB_LEFT(elm, field) = VRB_RIGHT(tmp, field)) != NULL) {	\
-		VRB_PARENT(VRB_RIGHT(tmp, field), field) = (elm);		\
-	}								\
-	VRB_AUGMENT(elm);						\
-	if ((VRB_PARENT(tmp, field) = VRB_PARENT(elm, field)) != NULL) {	\
-		if ((elm) == VRB_LEFT(VRB_PARENT(elm, field), field))	\
-			VRB_LEFT(VRB_PARENT(elm, field), field) = (tmp);	\
-		else							\
-			VRB_RIGHT(VRB_PARENT(elm, field), field) = (tmp);	\
-	} else								\
-		(head)->rbh_root = (tmp);				\
-	VRB_RIGHT(tmp, field) = (elm);					\
-	VRB_PARENT(elm, field) = (tmp);					\
-	VRB_AUGMENT(tmp);						\
-	if ((VRB_PARENT(tmp, field)))					\
-		VRB_AUGMENT(VRB_PARENT(tmp, field));			\
-} while (/*CONSTCOND*/ 0)
-
-/* Generates prototypes and inline functions */
-#define	VRB_PROTOTYPE(name, type, field, cmp)				\
-	VRB_PROTOTYPE_INTERNAL(name, type, field, cmp,)
-#define	VRB_PROTOTYPE_STATIC(name, type, field, cmp)			\
-	VRB_PROTOTYPE_INTERNAL(name, type, field, cmp, __unused static)
-#define VRB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr)		\
-attr void name##_VRB_INSERT_COLOR(struct name *, struct type *);		\
-attr void name##_VRB_REMOVE_COLOR(struct name *, struct type *, struct type *);\
-attr struct type *name##_VRB_REMOVE(struct name *, struct type *);	\
-attr struct type *name##_VRB_INSERT(struct name *, struct type *);	\
-attr struct type *name##_VRB_FIND(struct name *, struct type *);		\
-attr struct type *name##_VRB_NFIND(struct name *, struct type *);	\
-attr struct type *name##_VRB_NEXT(struct type *);			\
-attr struct type *name##_VRB_PREV(struct type *);			\
-attr struct type *name##_VRB_MINMAX(struct name *, int);			\
-									\
-
-/* Main rb operation.
- * Moves node close to the key of elm to top
- */
-#define	VRB_GENERATE(name, type, field, cmp)				\
-	VRB_GENERATE_INTERNAL(name, type, field, cmp,)
-#define	VRB_GENERATE_STATIC(name, type, field, cmp)			\
-	VRB_GENERATE_INTERNAL(name, type, field, cmp, __unused static)
-#define VRB_GENERATE_INTERNAL(name, type, field, cmp, attr)		\
-attr void								\
-name##_VRB_INSERT_COLOR(struct name *head, struct type *elm)		\
-{									\
-	struct type *parent, *gparent, *tmp;				\
-	while ((parent = VRB_PARENT(elm, field)) != NULL &&		\
-	    VRB_COLOR(parent, field) == VRB_RED) {			\
-		gparent = VRB_PARENT(parent, field);			\
-		if (parent == VRB_LEFT(gparent, field)) {		\
-			tmp = VRB_RIGHT(gparent, field);			\
-			if (tmp && VRB_COLOR(tmp, field) == VRB_RED) {	\
-				VRB_COLOR(tmp, field) = VRB_BLACK;	\
-				VRB_SET_BLACKRED(parent, gparent, field);\
-				elm = gparent;				\
-				continue;				\
-			}						\
-			if (VRB_RIGHT(parent, field) == elm) {		\
-				VRB_ROTATE_LEFT(head, parent, tmp, field);\
-				tmp = parent;				\
-				parent = elm;				\
-				elm = tmp;				\
-			}						\
-			VRB_SET_BLACKRED(parent, gparent, field);	\
-			VRB_ROTATE_RIGHT(head, gparent, tmp, field);	\
-		} else {						\
-			tmp = VRB_LEFT(gparent, field);			\
-			if (tmp && VRB_COLOR(tmp, field) == VRB_RED) {	\
-				VRB_COLOR(tmp, field) = VRB_BLACK;	\
-				VRB_SET_BLACKRED(parent, gparent, field);\
-				elm = gparent;				\
-				continue;				\
-			}						\
-			if (VRB_LEFT(parent, field) == elm) {		\
-				VRB_ROTATE_RIGHT(head, parent, tmp, field);\
-				tmp = parent;				\
-				parent = elm;				\
-				elm = tmp;				\
-			}						\
-			VRB_SET_BLACKRED(parent, gparent, field);	\
-			VRB_ROTATE_LEFT(head, gparent, tmp, field);	\
-		}							\
-	}								\
-	VRB_COLOR(head->rbh_root, field) = VRB_BLACK;			\
-}									\
-									\
-attr void								\
-name##_VRB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \
-{									\
-	struct type *tmp;						\
-	while ((elm == NULL || VRB_COLOR(elm, field) == VRB_BLACK) &&	\
-	    elm != VRB_ROOT(head)) {					\
-		if (VRB_LEFT(parent, field) == elm) {			\
-			tmp = VRB_RIGHT(parent, field);			\
-			if (VRB_COLOR(tmp, field) == VRB_RED) {		\
-				VRB_SET_BLACKRED(tmp, parent, field);	\
-				VRB_ROTATE_LEFT(head, parent, tmp, field);\
-				tmp = VRB_RIGHT(parent, field);		\
-			}						\
-			if ((VRB_LEFT(tmp, field) == NULL ||		\
-			    VRB_COLOR(VRB_LEFT(tmp, field), field) == VRB_BLACK) &&\
-			    (VRB_RIGHT(tmp, field) == NULL ||		\
-			    VRB_COLOR(VRB_RIGHT(tmp, field), field) == VRB_BLACK)) {\
-				VRB_COLOR(tmp, field) = VRB_RED;		\
-				elm = parent;				\
-				parent = VRB_PARENT(elm, field);		\
-			} else {					\
-				if (VRB_RIGHT(tmp, field) == NULL ||	\
-				    VRB_COLOR(VRB_RIGHT(tmp, field), field) == VRB_BLACK) {\
-					struct type *oleft;		\
-					if ((oleft = VRB_LEFT(tmp, field)) \
-					    != NULL)			\
-						VRB_COLOR(oleft, field) = VRB_BLACK;\
-					VRB_COLOR(tmp, field) = VRB_RED;	\
-					VRB_ROTATE_RIGHT(head, tmp, oleft, field);\
-					tmp = VRB_RIGHT(parent, field);	\
-				}					\
-				VRB_COLOR(tmp, field) = VRB_COLOR(parent, field);\
-				VRB_COLOR(parent, field) = VRB_BLACK;	\
-				if (VRB_RIGHT(tmp, field))		\
-					VRB_COLOR(VRB_RIGHT(tmp, field), field) = VRB_BLACK;\
-				VRB_ROTATE_LEFT(head, parent, tmp, field);\
-				elm = VRB_ROOT(head);			\
-				break;					\
-			}						\
-		} else {						\
-			tmp = VRB_LEFT(parent, field);			\
-			if (VRB_COLOR(tmp, field) == VRB_RED) {		\
-				VRB_SET_BLACKRED(tmp, parent, field);	\
-				VRB_ROTATE_RIGHT(head, parent, tmp, field);\
-				tmp = VRB_LEFT(parent, field);		\
-			}						\
-			if ((VRB_LEFT(tmp, field) == NULL ||		\
-			    VRB_COLOR(VRB_LEFT(tmp, field), field) == VRB_BLACK) &&\
-			    (VRB_RIGHT(tmp, field) == NULL ||		\
-			    VRB_COLOR(VRB_RIGHT(tmp, field), field) == VRB_BLACK)) {\
-				VRB_COLOR(tmp, field) = VRB_RED;		\
-				elm = parent;				\
-				parent = VRB_PARENT(elm, field);		\
-			} else {					\
-				if (VRB_LEFT(tmp, field) == NULL ||	\
-				    VRB_COLOR(VRB_LEFT(tmp, field), field) == VRB_BLACK) {\
-					struct type *oright;		\
-					if ((oright = VRB_RIGHT(tmp, field)) \
-					    != NULL)			\
-						VRB_COLOR(oright, field) = VRB_BLACK;\
-					VRB_COLOR(tmp, field) = VRB_RED;	\
-					VRB_ROTATE_LEFT(head, tmp, oright, field);\
-					tmp = VRB_LEFT(parent, field);	\
-				}					\
-				VRB_COLOR(tmp, field) = VRB_COLOR(parent, field);\
-				VRB_COLOR(parent, field) = VRB_BLACK;	\
-				if (VRB_LEFT(tmp, field))		\
-					VRB_COLOR(VRB_LEFT(tmp, field), field) = VRB_BLACK;\
-				VRB_ROTATE_RIGHT(head, parent, tmp, field);\
-				elm = VRB_ROOT(head);			\
-				break;					\
-			}						\
-		}							\
-	}								\
-	if (elm)							\
-		VRB_COLOR(elm, field) = VRB_BLACK;			\
-}									\
-									\
-attr struct type *							\
-name##_VRB_REMOVE(struct name *head, struct type *elm)			\
-{									\
-	struct type *child, *parent, *old = elm;			\
-	int color;							\
-	if (VRB_LEFT(elm, field) == NULL)				\
-		child = VRB_RIGHT(elm, field);				\
-	else if (VRB_RIGHT(elm, field) == NULL)				\
-		child = VRB_LEFT(elm, field);				\
-	else {								\
-		struct type *left;					\
-		elm = VRB_RIGHT(elm, field);				\
-		while ((left = VRB_LEFT(elm, field)) != NULL)		\
-			elm = left;					\
-		child = VRB_RIGHT(elm, field);				\
-		parent = VRB_PARENT(elm, field);				\
-		color = VRB_COLOR(elm, field);				\
-		if (child)						\
-			VRB_PARENT(child, field) = parent;		\
-		if (parent) {						\
-			if (VRB_LEFT(parent, field) == elm)		\
-				VRB_LEFT(parent, field) = child;		\
-			else						\
-				VRB_RIGHT(parent, field) = child;	\
-			VRB_AUGMENT(parent);				\
-		} else							\
-			VRB_ROOT(head) = child;				\
-		if (VRB_PARENT(elm, field) == old)			\
-			parent = elm;					\
-		(elm)->field = (old)->field;				\
-		if (VRB_PARENT(old, field)) {				\
-			if (VRB_LEFT(VRB_PARENT(old, field), field) == old)\
-				VRB_LEFT(VRB_PARENT(old, field), field) = elm;\
-			else						\
-				VRB_RIGHT(VRB_PARENT(old, field), field) = elm;\
-			VRB_AUGMENT(VRB_PARENT(old, field));		\
-		} else							\
-			VRB_ROOT(head) = elm;				\
-		VRB_PARENT(VRB_LEFT(old, field), field) = elm;		\
-		if (VRB_RIGHT(old, field))				\
-			VRB_PARENT(VRB_RIGHT(old, field), field) = elm;	\
-		if (parent) {						\
-			left = parent;					\
-			do {						\
-				VRB_AUGMENT(left);			\
-			} while ((left = VRB_PARENT(left, field)) != NULL); \
-		}							\
-		goto color;						\
-	}								\
-	parent = VRB_PARENT(elm, field);					\
-	color = VRB_COLOR(elm, field);					\
-	if (child)							\
-		VRB_PARENT(child, field) = parent;			\
-	if (parent) {							\
-		if (VRB_LEFT(parent, field) == elm)			\
-			VRB_LEFT(parent, field) = child;			\
-		else							\
-			VRB_RIGHT(parent, field) = child;		\
-		VRB_AUGMENT(parent);					\
-	} else								\
-		VRB_ROOT(head) = child;					\
-color:									\
-	if (color == VRB_BLACK)						\
-		name##_VRB_REMOVE_COLOR(head, parent, child);		\
-	return (old);							\
-}									\
-									\
-/* Inserts a node into the RB tree */					\
-attr struct type *							\
-name##_VRB_INSERT(struct name *head, struct type *elm)			\
-{									\
-	struct type *tmp;						\
-	struct type *parent = NULL;					\
-	int comp = 0;							\
-	tmp = VRB_ROOT(head);						\
-	while (tmp) {							\
-		parent = tmp;						\
-		comp = (cmp)(elm, parent);				\
-		if (comp < 0)						\
-			tmp = VRB_LEFT(tmp, field);			\
-		else if (comp > 0)					\
-			tmp = VRB_RIGHT(tmp, field);			\
-		else							\
-			return (tmp);					\
-	}								\
-	VRB_SET(elm, parent, field);					\
-	if (parent != NULL) {						\
-		if (comp < 0)						\
-			VRB_LEFT(parent, field) = elm;			\
-		else							\
-			VRB_RIGHT(parent, field) = elm;			\
-		VRB_AUGMENT(parent);					\
-	} else								\
-		VRB_ROOT(head) = elm;					\
-	name##_VRB_INSERT_COLOR(head, elm);				\
-	return (NULL);							\
-}									\
-									\
-/* Finds the node with the same key as elm */				\
-attr struct type *							\
-name##_VRB_FIND(struct name *head, struct type *elm)			\
-{									\
-	struct type *tmp = VRB_ROOT(head);				\
-	int comp;							\
-	while (tmp) {							\
-		comp = cmp(elm, tmp);					\
-		if (comp < 0)						\
-			tmp = VRB_LEFT(tmp, field);			\
-		else if (comp > 0)					\
-			tmp = VRB_RIGHT(tmp, field);			\
-		else							\
-			return (tmp);					\
-	}								\
-	return (NULL);							\
-}									\
-									\
-/* Finds the first node greater than or equal to the search key */	\
-attr struct type *							\
-name##_VRB_NFIND(struct name *head, struct type *elm)			\
-{									\
-	struct type *tmp = VRB_ROOT(head);				\
-	struct type *res = NULL;					\
-	int comp;							\
-	while (tmp) {							\
-		comp = cmp(elm, tmp);					\
-		if (comp < 0) {						\
-			res = tmp;					\
-			tmp = VRB_LEFT(tmp, field);			\
-		}							\
-		else if (comp > 0)					\
-			tmp = VRB_RIGHT(tmp, field);			\
-		else							\
-			return (tmp);					\
-	}								\
-	return (res);							\
-}									\
-									\
-/* ARGSUSED */								\
-attr struct type *							\
-name##_VRB_NEXT(struct type *elm)					\
-{									\
-	if (VRB_RIGHT(elm, field)) {					\
-		elm = VRB_RIGHT(elm, field);				\
-		while (VRB_LEFT(elm, field))				\
-			elm = VRB_LEFT(elm, field);			\
-	} else {							\
-		if (VRB_PARENT(elm, field) &&				\
-		    (elm == VRB_LEFT(VRB_PARENT(elm, field), field)))	\
-			elm = VRB_PARENT(elm, field);			\
-		else {							\
-			while (VRB_PARENT(elm, field) &&			\
-			    (elm == VRB_RIGHT(VRB_PARENT(elm, field), field)))\
-				elm = VRB_PARENT(elm, field);		\
-			elm = VRB_PARENT(elm, field);			\
-		}							\
-	}								\
-	return (elm);							\
-}									\
-									\
-/* ARGSUSED */								\
-attr struct type *							\
-name##_VRB_PREV(struct type *elm)					\
-{									\
-	if (VRB_LEFT(elm, field)) {					\
-		elm = VRB_LEFT(elm, field);				\
-		while (VRB_RIGHT(elm, field))				\
-			elm = VRB_RIGHT(elm, field);			\
-	} else {							\
-		if (VRB_PARENT(elm, field) &&				\
-		    (elm == VRB_RIGHT(VRB_PARENT(elm, field), field)))	\
-			elm = VRB_PARENT(elm, field);			\
-		else {							\
-			while (VRB_PARENT(elm, field) &&			\
-			    (elm == VRB_LEFT(VRB_PARENT(elm, field), field)))\
-				elm = VRB_PARENT(elm, field);		\
-			elm = VRB_PARENT(elm, field);			\
-		}							\
-	}								\
-	return (elm);							\
-}									\
-									\
-attr struct type *							\
-name##_VRB_MINMAX(struct name *head, int val)				\
-{									\
-	struct type *tmp = VRB_ROOT(head);				\
-	struct type *parent = NULL;					\
-	while (tmp) {							\
-		parent = tmp;						\
-		if (val < 0)						\
-			tmp = VRB_LEFT(tmp, field);			\
-		else							\
-			tmp = VRB_RIGHT(tmp, field);			\
-	}								\
-	return (parent);						\
-}
-
-#define VRB_NEGINF	-1
-#define VRB_INF	1
-
-#define VRB_INSERT(name, x, y)	name##_VRB_INSERT(x, y)
-#define VRB_REMOVE(name, x, y)	name##_VRB_REMOVE(x, y)
-#define VRB_FIND(name, x, y)	name##_VRB_FIND(x, y)
-#define VRB_NFIND(name, x, y)	name##_VRB_NFIND(x, y)
-#define VRB_NEXT(name, x, y)	name##_VRB_NEXT(y)
-#define VRB_PREV(name, x, y)	name##_VRB_PREV(y)
-#define VRB_MIN(name, x)		name##_VRB_MINMAX(x, VRB_NEGINF)
-#define VRB_MAX(name, x)		name##_VRB_MINMAX(x, VRB_INF)
-
-#define VRB_FOREACH(x, name, head)					\
-	for ((x) = VRB_MIN(name, head);					\
-	     (x) != NULL;						\
-	     (x) = name##_VRB_NEXT(x))
-
-#define VRB_FOREACH_FROM(x, name, y)					\
-	for ((x) = (y);							\
-	    ((x) != NULL) && ((y) = name##_VRB_NEXT(x), (x) != NULL);	\
-	     (x) = (y))
-
-#define VRB_FOREACH_SAFE(x, name, head, y)				\
-	for ((x) = VRB_MIN(name, head);					\
-	    ((x) != NULL) && ((y) = name##_VRB_NEXT(x), (x) != NULL);	\
-	     (x) = (y))
-
-#define VRB_FOREACH_REVERSE(x, name, head)				\
-	for ((x) = VRB_MAX(name, head);					\
-	     (x) != NULL;						\
-	     (x) = name##_VRB_PREV(x))
-
-#define VRB_FOREACH_REVERSE_FROM(x, name, y)				\
-	for ((x) = (y);							\
-	    ((x) != NULL) && ((y) = name##_VRB_PREV(x), (x) != NULL);	\
-	     (x) = (y))
-
-#define VRB_FOREACH_REVERSE_SAFE(x, name, head, y)			\
-	for ((x) = VRB_MAX(name, head);					\
-	    ((x) != NULL) && ((y) = name##_VRB_PREV(x), (x) != NULL);	\
-	     (x) = (y))
-
-#endif	/* _VTREE_H_ */
diff --git a/include/Makefile.am b/include/Makefile.am
index 6c3f687..ff33292 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -57,7 +57,8 @@ nobase_noinst_HEADERS = \
 	vsub.h \
 	vss.h \
 	vtcp.h \
-	vtim.h
+	vtim.h \
+	vtree.h
 
 # Headers for use with vmods
 pkgdataincludedir = $(pkgdatadir)/include
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 54014cd..023b65f 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -1,9 +1,10 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2013 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -48,9 +49,27 @@
 			VSL_x_USAGE
 
 struct VSL_data;
+struct VSLQ;
 
 struct VSL_cursor {
 	const uint32_t		*ptr; /* Record pointer */
+
+	/* If not -1, the vxid of all records in this set */
+	int32_t			vxid;
+
+	/* For set cursors, the depth level of these records */
+	unsigned level;
+
+	/* Nonzero if pointer values from this cursor are still valid
+	   after next call to VSL_Next */
+	unsigned shmptr_ok;
+};
+
+enum VSL_grouping_e {
+	VSL_g_raw,
+	VSL_g_vxid,
+	VSL_g_request,
+	VSL_g_session,
 };
 
 extern const char *VSL_tags[256];
@@ -155,13 +174,138 @@ int VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c);
 	 *	0:	No match
 	 */
 
-int VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *file);
+int VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
+	/*
+	 * Print the log record pointed to by cursor to stream.
+	 *
+	 * Format: <vxid> <tag> <type> <data>
+	 *
+	 * Arguments:
+	 *   vsl: The VSL_data context
+	 *     c: A VSL_cursor
+	 *    fo: A FILE* pointer
+	 *
+	 * Return values:
+	 *	0:	OK
+	 *     -5:	I/O write error - see errno
+	 */
+
+int VSL_PrintLevel(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
 	/*
 	 * Print the log record pointed to by cursor to stream.
 	 *
+	 * Format: <level> <tag> <type> <data>
+	 *
+	 * Arguments:
+	 *   vsl: The VSL_data context
+	 *     c: A VSL_cursor
+	 *    fo: A FILE* pointer
+	 *
 	 * Return values:
 	 *	0:	OK
 	 *     -5:	I/O write error - see errno
 	 */
 
+int VSL_PrintAll(struct VSL_data *vsl, struct VSL_cursor *c, void *fo);
+	/*
+	 * Calls VSL_Next on c until c is exhausted. In turn calls
+	 * prints all records where VSL_Match returns true.
+	 *
+	 * If c->vxid == -1, calls VSL_PrintVXID on each record. Else
+	 * prints a VXID header and calls VSL_PrintLevel on each record.
+	 *
+	 * Arguments:
+	 *   vsl: The VSL_data context
+	 *     c: A VSL_cursor
+	 *    fo: A FILE* pointer, stdout if NULL
+	 *
+	 * Return values:
+	 *	0:	OK
+	 *    !=0:	Return value from either VSL_Next or VSL_Print
+	 */
+
+int VSL_PrintSet(struct VSL_data *vsl, struct VSL_cursor *cp[], void *fo);
+	/*
+	 * Calls VSL_PrintAll on each cursor in cp[]. If any cursor in cp
+	 * has vxid != -1 it will end with a double line break as a set
+	 * delimeter.
+	 *
+	 * Arguments:
+	 *   vsl: The VSL_data context
+	 *    cp: A NULL-terminated array of VSL_cursor pointers
+	 *    fo: A FILE* pointer, stdout if NULL
+	 *
+	 * Return values:
+	 *	0:	OK
+	 *    !=0:	Return value from either VSL_Next or VSL_PrintAll
+	 */
+
+struct VSLQ *VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp,
+    enum VSL_grouping_e grouping, const char *query);
+	/*
+	 * Create a new query context using cp. On success cp is NULLed,
+	 * and will be deleted when deleting the query.
+	 *
+	 * Arguments:
+	 *       vsl: The VSL_data context
+	 *        cp: The cursor to use
+	 *  grouping: VXID grouping to report on
+	 *     query: Query match expression
+	 *
+	 * Return values:
+	 *  non-NULL: OK
+	 *      NULL: Error - see VSL_Error
+	 */
+
+void VSLQ_Delete(struct VSLQ **pvslq);
+	/*
+	 * Delete the query pointed to by pvslq, freeing up the resources
+	 */
+
+typedef int VSLQ_dispatch_f(struct VSL_data *vsl, struct VSL_cursor *cp[],
+    void *priv);
+	/*
+	 * The callback function type for use with VSLQ_Dispatch.
+	 *
+	 * Arguments:
+	 *   vsl: The VSL_data context
+	 *  cp[]: A NULL terminated array of pointer to cursors. Each cursor
+	 *        will iterate over the log records of a single VXID
+	 *  priv: The priv argument from VSL_Dispatch
+	 *
+	 * Return value:
+	 *     0: OK - continue
+	 *   !=0: Makes VSLQ_Dispatch return with this return value immediatly
+	 */
+
+int VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
+	/*
+	 * Process log and call func for each set matching the specified
+	 * query
+	 *
+	 * Arguments:
+	 *  vslq: The VSLQ query
+	 *  func: The callback function to call. Can be NULL to ignore records.
+	 *  priv: An argument passed to func
+	 *
+	 * Return values:
+	 *     0: No more log records available
+	 *   !=0: The return value from either VSL_Next() or func
+	 */
+
+int VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
+	/*
+	 * Flush any pending record sets from the query.
+	 *
+	 * Arguments:
+	 *  vslq: The VSL context
+	 *  func: The callback function to call. Pass NULL to discard the
+	 *        pending messages
+	 *  priv: An argument passed to func
+	 *
+	 * Return values:
+	 *     0: OK
+	 *   !=0: The return value from func
+	 */
+
 #endif /* VAPI_VSL_H_INCLUDED */
diff --git a/include/vtree.h b/include/vtree.h
new file mode 100644
index 0000000..7692d12
--- /dev/null
+++ b/include/vtree.h
@@ -0,0 +1,763 @@
+/*	$NetBSD: tree.h,v 1.8 2004/03/28 19:38:30 provos Exp $	*/
+/*	$OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $	*/
+/* $FreeBSD: release/9.0.0/sys/sys/tree.h 189204 2009-03-01 04:57:23Z bms $ */
+
+/*-
+ * Copyright 2002 Niels Provos <provos at citi.umich.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef	_VTREE_H_
+#define	_VTREE_H_
+
+/*
+ * This file defines data structures for different types of trees:
+ * splay trees and red-black trees.
+ *
+ * A splay tree is a self-organizing data structure.  Every operation
+ * on the tree causes a splay to happen.  The splay moves the requested
+ * node to the root of the tree and partly rebalances it.
+ *
+ * This has the benefit that request locality causes faster lookups as
+ * the requested nodes move to the top of the tree.  On the other hand,
+ * every lookup causes memory writes.
+ *
+ * The Balance Theorem bounds the total access time for m operations
+ * and n inserts on an initially empty tree as O((m + n)lg n).  The
+ * amortized cost for a sequence of m accesses to a splay tree is O(lg n);
+ *
+ * A red-black tree is a binary search tree with the node color as an
+ * extra attribute.  It fulfills a set of conditions:
+ *	- every search path from the root to a leaf consists of the
+ *	  same number of black nodes,
+ *	- each red node (except for the root) has a black parent,
+ *	- each leaf node is black.
+ *
+ * Every operation on a red-black tree is bounded as O(lg n).
+ * The maximum height of a red-black tree is 2lg (n+1).
+ */
+
+#define VSPLAY_HEAD(name, type)						\
+struct name {								\
+	struct type *sph_root; /* root of the tree */			\
+}
+
+#define VSPLAY_INITIALIZER(root)						\
+	{ NULL }
+
+#define VSPLAY_INIT(root) do {						\
+	(root)->sph_root = NULL;					\
+} while (/*CONSTCOND*/ 0)
+
+#define VSPLAY_ENTRY(type)						\
+struct {								\
+	struct type *spe_left; /* left element */			\
+	struct type *spe_right; /* right element */			\
+}
+
+#define VSPLAY_LEFT(elm, field)		(elm)->field.spe_left
+#define VSPLAY_RIGHT(elm, field)		(elm)->field.spe_right
+#define VSPLAY_ROOT(head)		(head)->sph_root
+#define VSPLAY_EMPTY(head)		(VSPLAY_ROOT(head) == NULL)
+
+/* VSPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold VSPLAY_{RIGHT,LEFT} */
+#define VSPLAY_ROTATE_RIGHT(head, tmp, field) do {			\
+	VSPLAY_LEFT((head)->sph_root, field) = VSPLAY_RIGHT(tmp, field);	\
+	VSPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
+	(head)->sph_root = tmp;						\
+} while (/*CONSTCOND*/ 0)
+
+#define VSPLAY_ROTATE_LEFT(head, tmp, field) do {			\
+	VSPLAY_RIGHT((head)->sph_root, field) = VSPLAY_LEFT(tmp, field);	\
+	VSPLAY_LEFT(tmp, field) = (head)->sph_root;			\
+	(head)->sph_root = tmp;						\
+} while (/*CONSTCOND*/ 0)
+
+#define VSPLAY_LINKLEFT(head, tmp, field) do {				\
+	VSPLAY_LEFT(tmp, field) = (head)->sph_root;			\
+	tmp = (head)->sph_root;						\
+	(head)->sph_root = VSPLAY_LEFT((head)->sph_root, field);		\
+} while (/*CONSTCOND*/ 0)
+
+#define VSPLAY_LINKRIGHT(head, tmp, field) do {				\
+	VSPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
+	tmp = (head)->sph_root;						\
+	(head)->sph_root = VSPLAY_RIGHT((head)->sph_root, field);	\
+} while (/*CONSTCOND*/ 0)
+
+#define VSPLAY_ASSEMBLE(head, node, left, right, field) do {		\
+	VSPLAY_RIGHT(left, field) = VSPLAY_LEFT((head)->sph_root, field);	\
+	VSPLAY_LEFT(right, field) = VSPLAY_RIGHT((head)->sph_root, field);\
+	VSPLAY_LEFT((head)->sph_root, field) = VSPLAY_RIGHT(node, field);	\
+	VSPLAY_RIGHT((head)->sph_root, field) = VSPLAY_LEFT(node, field);	\
+} while (/*CONSTCOND*/ 0)
+
+/* Generates prototypes and inline functions */
+
+#define VSPLAY_PROTOTYPE(name, type, field, cmp)				\
+void name##_VSPLAY(struct name *, struct type *);			\
+void name##_VSPLAY_MINMAX(struct name *, int);				\
+struct type *name##_VSPLAY_INSERT(struct name *, struct type *);		\
+struct type *name##_VSPLAY_REMOVE(struct name *, struct type *);		\
+									\
+/* Finds the node with the same key as elm */				\
+static __inline struct type *						\
+name##_VSPLAY_FIND(struct name *head, struct type *elm)			\
+{									\
+	if (VSPLAY_EMPTY(head))						\
+		return(NULL);						\
+	name##_VSPLAY(head, elm);					\
+	if ((cmp)(elm, (head)->sph_root) == 0)				\
+		return (head->sph_root);				\
+	return (NULL);							\
+}									\
+									\
+static __inline struct type *						\
+name##_VSPLAY_NEXT(struct name *head, struct type *elm)			\
+{									\
+	name##_VSPLAY(head, elm);					\
+	if (VSPLAY_RIGHT(elm, field) != NULL) {				\
+		elm = VSPLAY_RIGHT(elm, field);				\
+		while (VSPLAY_LEFT(elm, field) != NULL) {		\
+			elm = VSPLAY_LEFT(elm, field);			\
+		}							\
+	} else								\
+		elm = NULL;						\
+	return (elm);							\
+}									\
+									\
+static __inline struct type *						\
+name##_VSPLAY_MIN_MAX(struct name *head, int val)			\
+{									\
+	name##_VSPLAY_MINMAX(head, val);					\
+        return (VSPLAY_ROOT(head));					\
+}
+
+/* Main splay operation.
+ * Moves node close to the key of elm to top
+ */
+#define VSPLAY_GENERATE(name, type, field, cmp)				\
+struct type *								\
+name##_VSPLAY_INSERT(struct name *head, struct type *elm)		\
+{									\
+    if (VSPLAY_EMPTY(head)) {						\
+	    VSPLAY_LEFT(elm, field) = VSPLAY_RIGHT(elm, field) = NULL;	\
+    } else {								\
+	    int __comp;							\
+	    name##_VSPLAY(head, elm);					\
+	    __comp = (cmp)(elm, (head)->sph_root);			\
+	    if(__comp < 0) {						\
+		    VSPLAY_LEFT(elm, field) = VSPLAY_LEFT((head)->sph_root, field);\
+		    VSPLAY_RIGHT(elm, field) = (head)->sph_root;		\
+		    VSPLAY_LEFT((head)->sph_root, field) = NULL;		\
+	    } else if (__comp > 0) {					\
+		    VSPLAY_RIGHT(elm, field) = VSPLAY_RIGHT((head)->sph_root, field);\
+		    VSPLAY_LEFT(elm, field) = (head)->sph_root;		\
+		    VSPLAY_RIGHT((head)->sph_root, field) = NULL;	\
+	    } else							\
+		    return ((head)->sph_root);				\
+    }									\
+    (head)->sph_root = (elm);						\
+    return (NULL);							\
+}									\
+									\
+struct type *								\
+name##_VSPLAY_REMOVE(struct name *head, struct type *elm)		\
+{									\
+	struct type *__tmp;						\
+	if (VSPLAY_EMPTY(head))						\
+		return (NULL);						\
+	name##_VSPLAY(head, elm);					\
+	if ((cmp)(elm, (head)->sph_root) == 0) {			\
+		if (VSPLAY_LEFT((head)->sph_root, field) == NULL) {	\
+			(head)->sph_root = VSPLAY_RIGHT((head)->sph_root, field);\
+		} else {						\
+			__tmp = VSPLAY_RIGHT((head)->sph_root, field);	\
+			(head)->sph_root = VSPLAY_LEFT((head)->sph_root, field);\
+			name##_VSPLAY(head, elm);			\
+			VSPLAY_RIGHT((head)->sph_root, field) = __tmp;	\
+		}							\
+		return (elm);						\
+	}								\
+	return (NULL);							\
+}									\
+									\
+void									\
+name##_VSPLAY(struct name *head, struct type *elm)			\
+{									\
+	struct type __node, *__left, *__right, *__tmp;			\
+	int __comp;							\
+\
+	VSPLAY_LEFT(&__node, field) = VSPLAY_RIGHT(&__node, field) = NULL;\
+	__left = __right = &__node;					\
+\
+	while ((__comp = (cmp)(elm, (head)->sph_root)) != 0) {		\
+		if (__comp < 0) {					\
+			__tmp = VSPLAY_LEFT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if ((cmp)(elm, __tmp) < 0){			\
+				VSPLAY_ROTATE_RIGHT(head, __tmp, field);	\
+				if (VSPLAY_LEFT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			VSPLAY_LINKLEFT(head, __right, field);		\
+		} else if (__comp > 0) {				\
+			__tmp = VSPLAY_RIGHT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if ((cmp)(elm, __tmp) > 0){			\
+				VSPLAY_ROTATE_LEFT(head, __tmp, field);	\
+				if (VSPLAY_RIGHT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			VSPLAY_LINKRIGHT(head, __left, field);		\
+		}							\
+	}								\
+	VSPLAY_ASSEMBLE(head, &__node, __left, __right, field);		\
+}									\
+									\
+/* Splay with either the minimum or the maximum element			\
+ * Used to find minimum or maximum element in tree.			\
+ */									\
+void name##_VSPLAY_MINMAX(struct name *head, int __comp) \
+{									\
+	struct type __node, *__left, *__right, *__tmp;			\
+\
+	VSPLAY_LEFT(&__node, field) = VSPLAY_RIGHT(&__node, field) = NULL;\
+	__left = __right = &__node;					\
+\
+	while (1) {							\
+		if (__comp < 0) {					\
+			__tmp = VSPLAY_LEFT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if (__comp < 0){				\
+				VSPLAY_ROTATE_RIGHT(head, __tmp, field);	\
+				if (VSPLAY_LEFT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			VSPLAY_LINKLEFT(head, __right, field);		\
+		} else if (__comp > 0) {				\
+			__tmp = VSPLAY_RIGHT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if (__comp > 0) {				\
+				VSPLAY_ROTATE_LEFT(head, __tmp, field);	\
+				if (VSPLAY_RIGHT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			VSPLAY_LINKRIGHT(head, __left, field);		\
+		}							\
+	}								\
+	VSPLAY_ASSEMBLE(head, &__node, __left, __right, field);		\
+}
+
+#define VSPLAY_NEGINF	-1
+#define VSPLAY_INF	1
+
+#define VSPLAY_INSERT(name, x, y)	name##_VSPLAY_INSERT(x, y)
+#define VSPLAY_REMOVE(name, x, y)	name##_VSPLAY_REMOVE(x, y)
+#define VSPLAY_FIND(name, x, y)		name##_VSPLAY_FIND(x, y)
+#define VSPLAY_NEXT(name, x, y)		name##_VSPLAY_NEXT(x, y)
+#define VSPLAY_MIN(name, x)		(VSPLAY_EMPTY(x) ? NULL	\
+					: name##_VSPLAY_MIN_MAX(x, VSPLAY_NEGINF))
+#define VSPLAY_MAX(name, x)		(VSPLAY_EMPTY(x) ? NULL	\
+					: name##_VSPLAY_MIN_MAX(x, VSPLAY_INF))
+
+#define VSPLAY_FOREACH(x, name, head)					\
+	for ((x) = VSPLAY_MIN(name, head);				\
+	     (x) != NULL;						\
+	     (x) = VSPLAY_NEXT(name, head, x))
+
+/* Macros that define a red-black tree */
+#define VRB_HEAD(name, type)						\
+struct name {								\
+	struct type *rbh_root; /* root of the tree */			\
+}
+
+#define VRB_INITIALIZER(root)						\
+	{ NULL }
+
+#define VRB_INIT(root) do {						\
+	(root)->rbh_root = NULL;					\
+} while (/*CONSTCOND*/ 0)
+
+#define VRB_BLACK	0
+#define VRB_RED		1
+#define VRB_ENTRY(type)							\
+struct {								\
+	struct type *rbe_left;		/* left element */		\
+	struct type *rbe_right;		/* right element */		\
+	struct type *rbe_parent;	/* parent element */		\
+	int rbe_color;			/* node color */		\
+}
+
+#define VRB_LEFT(elm, field)		(elm)->field.rbe_left
+#define VRB_RIGHT(elm, field)		(elm)->field.rbe_right
+#define VRB_PARENT(elm, field)		(elm)->field.rbe_parent
+#define VRB_COLOR(elm, field)		(elm)->field.rbe_color
+#define VRB_ROOT(head)			(head)->rbh_root
+#define VRB_EMPTY(head)			(VRB_ROOT(head) == NULL)
+
+#define VRB_SET(elm, parent, field) do {					\
+	VRB_PARENT(elm, field) = parent;					\
+	VRB_LEFT(elm, field) = VRB_RIGHT(elm, field) = NULL;		\
+	VRB_COLOR(elm, field) = VRB_RED;					\
+} while (/*CONSTCOND*/ 0)
+
+#define VRB_SET_BLACKRED(black, red, field) do {				\
+	VRB_COLOR(black, field) = VRB_BLACK;				\
+	VRB_COLOR(red, field) = VRB_RED;					\
+} while (/*CONSTCOND*/ 0)
+
+#ifndef VRB_AUGMENT
+#define VRB_AUGMENT(x)	do {} while (0)
+#endif
+
+#define VRB_ROTATE_LEFT(head, elm, tmp, field) do {			\
+	(tmp) = VRB_RIGHT(elm, field);					\
+	if ((VRB_RIGHT(elm, field) = VRB_LEFT(tmp, field)) != NULL) {	\
+		VRB_PARENT(VRB_LEFT(tmp, field), field) = (elm);		\
+	}								\
+	VRB_AUGMENT(elm);						\
+	if ((VRB_PARENT(tmp, field) = VRB_PARENT(elm, field)) != NULL) {	\
+		if ((elm) == VRB_LEFT(VRB_PARENT(elm, field), field))	\
+			VRB_LEFT(VRB_PARENT(elm, field), field) = (tmp);	\
+		else							\
+			VRB_RIGHT(VRB_PARENT(elm, field), field) = (tmp);	\
+	} else								\
+		(head)->rbh_root = (tmp);				\
+	VRB_LEFT(tmp, field) = (elm);					\
+	VRB_PARENT(elm, field) = (tmp);					\
+	VRB_AUGMENT(tmp);						\
+	if ((VRB_PARENT(tmp, field)))					\
+		VRB_AUGMENT(VRB_PARENT(tmp, field));			\
+} while (/*CONSTCOND*/ 0)
+
+#define VRB_ROTATE_RIGHT(head, elm, tmp, field) do {			\
+	(tmp) = VRB_LEFT(elm, field);					\
+	if ((VRB_LEFT(elm, field) = VRB_RIGHT(tmp, field)) != NULL) {	\
+		VRB_PARENT(VRB_RIGHT(tmp, field), field) = (elm);		\
+	}								\
+	VRB_AUGMENT(elm);						\
+	if ((VRB_PARENT(tmp, field) = VRB_PARENT(elm, field)) != NULL) {	\
+		if ((elm) == VRB_LEFT(VRB_PARENT(elm, field), field))	\
+			VRB_LEFT(VRB_PARENT(elm, field), field) = (tmp);	\
+		else							\
+			VRB_RIGHT(VRB_PARENT(elm, field), field) = (tmp);	\
+	} else								\
+		(head)->rbh_root = (tmp);				\
+	VRB_RIGHT(tmp, field) = (elm);					\
+	VRB_PARENT(elm, field) = (tmp);					\
+	VRB_AUGMENT(tmp);						\
+	if ((VRB_PARENT(tmp, field)))					\
+		VRB_AUGMENT(VRB_PARENT(tmp, field));			\
+} while (/*CONSTCOND*/ 0)
+
+/* Generates prototypes and inline functions */
+#define	VRB_PROTOTYPE(name, type, field, cmp)				\
+	VRB_PROTOTYPE_INTERNAL(name, type, field, cmp,)
+#define	VRB_PROTOTYPE_STATIC(name, type, field, cmp)			\
+	VRB_PROTOTYPE_INTERNAL(name, type, field, cmp, __unused static)
+#define VRB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr)		\
+attr void name##_VRB_INSERT_COLOR(struct name *, struct type *);		\
+attr void name##_VRB_REMOVE_COLOR(struct name *, struct type *, struct type *);\
+attr struct type *name##_VRB_REMOVE(struct name *, struct type *);	\
+attr struct type *name##_VRB_INSERT(struct name *, struct type *);	\
+attr struct type *name##_VRB_FIND(struct name *, struct type *);		\
+attr struct type *name##_VRB_NFIND(struct name *, struct type *);	\
+attr struct type *name##_VRB_NEXT(struct type *);			\
+attr struct type *name##_VRB_PREV(struct type *);			\
+attr struct type *name##_VRB_MINMAX(struct name *, int);			\
+									\
+
+/* Main rb operation.
+ * Moves node close to the key of elm to top
+ */
+#define	VRB_GENERATE(name, type, field, cmp)				\
+	VRB_GENERATE_INTERNAL(name, type, field, cmp,)
+#define	VRB_GENERATE_STATIC(name, type, field, cmp)			\
+	VRB_GENERATE_INTERNAL(name, type, field, cmp, __unused static)
+#define VRB_GENERATE_INTERNAL(name, type, field, cmp, attr)		\
+attr void								\
+name##_VRB_INSERT_COLOR(struct name *head, struct type *elm)		\
+{									\
+	struct type *parent, *gparent, *tmp;				\
+	while ((parent = VRB_PARENT(elm, field)) != NULL &&		\
+	    VRB_COLOR(parent, field) == VRB_RED) {			\
+		gparent = VRB_PARENT(parent, field);			\
+		if (parent == VRB_LEFT(gparent, field)) {		\
+			tmp = VRB_RIGHT(gparent, field);			\
+			if (tmp && VRB_COLOR(tmp, field) == VRB_RED) {	\
+				VRB_COLOR(tmp, field) = VRB_BLACK;	\
+				VRB_SET_BLACKRED(parent, gparent, field);\
+				elm = gparent;				\
+				continue;				\
+			}						\
+			if (VRB_RIGHT(parent, field) == elm) {		\
+				VRB_ROTATE_LEFT(head, parent, tmp, field);\
+				tmp = parent;				\
+				parent = elm;				\
+				elm = tmp;				\
+			}						\
+			VRB_SET_BLACKRED(parent, gparent, field);	\
+			VRB_ROTATE_RIGHT(head, gparent, tmp, field);	\
+		} else {						\
+			tmp = VRB_LEFT(gparent, field);			\
+			if (tmp && VRB_COLOR(tmp, field) == VRB_RED) {	\
+				VRB_COLOR(tmp, field) = VRB_BLACK;	\
+				VRB_SET_BLACKRED(parent, gparent, field);\
+				elm = gparent;				\
+				continue;				\
+			}						\
+			if (VRB_LEFT(parent, field) == elm) {		\
+				VRB_ROTATE_RIGHT(head, parent, tmp, field);\
+				tmp = parent;				\
+				parent = elm;				\
+				elm = tmp;				\
+			}						\
+			VRB_SET_BLACKRED(parent, gparent, field);	\
+			VRB_ROTATE_LEFT(head, gparent, tmp, field);	\
+		}							\
+	}								\
+	VRB_COLOR(head->rbh_root, field) = VRB_BLACK;			\
+}									\
+									\
+attr void								\
+name##_VRB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \
+{									\
+	struct type *tmp;						\
+	while ((elm == NULL || VRB_COLOR(elm, field) == VRB_BLACK) &&	\
+	    elm != VRB_ROOT(head)) {					\
+		if (VRB_LEFT(parent, field) == elm) {			\
+			tmp = VRB_RIGHT(parent, field);			\
+			if (VRB_COLOR(tmp, field) == VRB_RED) {		\
+				VRB_SET_BLACKRED(tmp, parent, field);	\
+				VRB_ROTATE_LEFT(head, parent, tmp, field);\
+				tmp = VRB_RIGHT(parent, field);		\
+			}						\
+			if ((VRB_LEFT(tmp, field) == NULL ||		\
+			    VRB_COLOR(VRB_LEFT(tmp, field), field) == VRB_BLACK) &&\
+			    (VRB_RIGHT(tmp, field) == NULL ||		\
+			    VRB_COLOR(VRB_RIGHT(tmp, field), field) == VRB_BLACK)) {\
+				VRB_COLOR(tmp, field) = VRB_RED;		\
+				elm = parent;				\
+				parent = VRB_PARENT(elm, field);		\
+			} else {					\
+				if (VRB_RIGHT(tmp, field) == NULL ||	\
+				    VRB_COLOR(VRB_RIGHT(tmp, field), field) == VRB_BLACK) {\
+					struct type *oleft;		\
+					if ((oleft = VRB_LEFT(tmp, field)) \
+					    != NULL)			\
+						VRB_COLOR(oleft, field) = VRB_BLACK;\
+					VRB_COLOR(tmp, field) = VRB_RED;	\
+					VRB_ROTATE_RIGHT(head, tmp, oleft, field);\
+					tmp = VRB_RIGHT(parent, field);	\
+				}					\
+				VRB_COLOR(tmp, field) = VRB_COLOR(parent, field);\
+				VRB_COLOR(parent, field) = VRB_BLACK;	\
+				if (VRB_RIGHT(tmp, field))		\
+					VRB_COLOR(VRB_RIGHT(tmp, field), field) = VRB_BLACK;\
+				VRB_ROTATE_LEFT(head, parent, tmp, field);\
+				elm = VRB_ROOT(head);			\
+				break;					\
+			}						\
+		} else {						\
+			tmp = VRB_LEFT(parent, field);			\
+			if (VRB_COLOR(tmp, field) == VRB_RED) {		\
+				VRB_SET_BLACKRED(tmp, parent, field);	\
+				VRB_ROTATE_RIGHT(head, parent, tmp, field);\
+				tmp = VRB_LEFT(parent, field);		\
+			}						\
+			if ((VRB_LEFT(tmp, field) == NULL ||		\
+			    VRB_COLOR(VRB_LEFT(tmp, field), field) == VRB_BLACK) &&\
+			    (VRB_RIGHT(tmp, field) == NULL ||		\
+			    VRB_COLOR(VRB_RIGHT(tmp, field), field) == VRB_BLACK)) {\
+				VRB_COLOR(tmp, field) = VRB_RED;		\
+				elm = parent;				\
+				parent = VRB_PARENT(elm, field);		\
+			} else {					\
+				if (VRB_LEFT(tmp, field) == NULL ||	\
+				    VRB_COLOR(VRB_LEFT(tmp, field), field) == VRB_BLACK) {\
+					struct type *oright;		\
+					if ((oright = VRB_RIGHT(tmp, field)) \
+					    != NULL)			\
+						VRB_COLOR(oright, field) = VRB_BLACK;\
+					VRB_COLOR(tmp, field) = VRB_RED;	\
+					VRB_ROTATE_LEFT(head, tmp, oright, field);\
+					tmp = VRB_LEFT(parent, field);	\
+				}					\
+				VRB_COLOR(tmp, field) = VRB_COLOR(parent, field);\
+				VRB_COLOR(parent, field) = VRB_BLACK;	\
+				if (VRB_LEFT(tmp, field))		\
+					VRB_COLOR(VRB_LEFT(tmp, field), field) = VRB_BLACK;\
+				VRB_ROTATE_RIGHT(head, parent, tmp, field);\
+				elm = VRB_ROOT(head);			\
+				break;					\
+			}						\
+		}							\
+	}								\
+	if (elm)							\
+		VRB_COLOR(elm, field) = VRB_BLACK;			\
+}									\
+									\
+attr struct type *							\
+name##_VRB_REMOVE(struct name *head, struct type *elm)			\
+{									\
+	struct type *child, *parent, *old = elm;			\
+	int color;							\
+	if (VRB_LEFT(elm, field) == NULL)				\
+		child = VRB_RIGHT(elm, field);				\
+	else if (VRB_RIGHT(elm, field) == NULL)				\
+		child = VRB_LEFT(elm, field);				\
+	else {								\
+		struct type *left;					\
+		elm = VRB_RIGHT(elm, field);				\
+		while ((left = VRB_LEFT(elm, field)) != NULL)		\
+			elm = left;					\
+		child = VRB_RIGHT(elm, field);				\
+		parent = VRB_PARENT(elm, field);				\
+		color = VRB_COLOR(elm, field);				\
+		if (child)						\
+			VRB_PARENT(child, field) = parent;		\
+		if (parent) {						\
+			if (VRB_LEFT(parent, field) == elm)		\
+				VRB_LEFT(parent, field) = child;		\
+			else						\
+				VRB_RIGHT(parent, field) = child;	\
+			VRB_AUGMENT(parent);				\
+		} else							\
+			VRB_ROOT(head) = child;				\
+		if (VRB_PARENT(elm, field) == old)			\
+			parent = elm;					\
+		(elm)->field = (old)->field;				\
+		if (VRB_PARENT(old, field)) {				\
+			if (VRB_LEFT(VRB_PARENT(old, field), field) == old)\
+				VRB_LEFT(VRB_PARENT(old, field), field) = elm;\
+			else						\
+				VRB_RIGHT(VRB_PARENT(old, field), field) = elm;\
+			VRB_AUGMENT(VRB_PARENT(old, field));		\
+		} else							\
+			VRB_ROOT(head) = elm;				\
+		VRB_PARENT(VRB_LEFT(old, field), field) = elm;		\
+		if (VRB_RIGHT(old, field))				\
+			VRB_PARENT(VRB_RIGHT(old, field), field) = elm;	\
+		if (parent) {						\
+			left = parent;					\
+			do {						\
+				VRB_AUGMENT(left);			\
+			} while ((left = VRB_PARENT(left, field)) != NULL); \
+		}							\
+		goto color;						\
+	}								\
+	parent = VRB_PARENT(elm, field);					\
+	color = VRB_COLOR(elm, field);					\
+	if (child)							\
+		VRB_PARENT(child, field) = parent;			\
+	if (parent) {							\
+		if (VRB_LEFT(parent, field) == elm)			\
+			VRB_LEFT(parent, field) = child;			\
+		else							\
+			VRB_RIGHT(parent, field) = child;		\
+		VRB_AUGMENT(parent);					\
+	} else								\
+		VRB_ROOT(head) = child;					\
+color:									\
+	if (color == VRB_BLACK)						\
+		name##_VRB_REMOVE_COLOR(head, parent, child);		\
+	return (old);							\
+}									\
+									\
+/* Inserts a node into the RB tree */					\
+attr struct type *							\
+name##_VRB_INSERT(struct name *head, struct type *elm)			\
+{									\
+	struct type *tmp;						\
+	struct type *parent = NULL;					\
+	int comp = 0;							\
+	tmp = VRB_ROOT(head);						\
+	while (tmp) {							\
+		parent = tmp;						\
+		comp = (cmp)(elm, parent);				\
+		if (comp < 0)						\
+			tmp = VRB_LEFT(tmp, field);			\
+		else if (comp > 0)					\
+			tmp = VRB_RIGHT(tmp, field);			\
+		else							\
+			return (tmp);					\
+	}								\
+	VRB_SET(elm, parent, field);					\
+	if (parent != NULL) {						\
+		if (comp < 0)						\
+			VRB_LEFT(parent, field) = elm;			\
+		else							\
+			VRB_RIGHT(parent, field) = elm;			\
+		VRB_AUGMENT(parent);					\
+	} else								\
+		VRB_ROOT(head) = elm;					\
+	name##_VRB_INSERT_COLOR(head, elm);				\
+	return (NULL);							\
+}									\
+									\
+/* Finds the node with the same key as elm */				\
+attr struct type *							\
+name##_VRB_FIND(struct name *head, struct type *elm)			\
+{									\
+	struct type *tmp = VRB_ROOT(head);				\
+	int comp;							\
+	while (tmp) {							\
+		comp = cmp(elm, tmp);					\
+		if (comp < 0)						\
+			tmp = VRB_LEFT(tmp, field);			\
+		else if (comp > 0)					\
+			tmp = VRB_RIGHT(tmp, field);			\
+		else							\
+			return (tmp);					\
+	}								\
+	return (NULL);							\
+}									\
+									\
+/* Finds the first node greater than or equal to the search key */	\
+attr struct type *							\
+name##_VRB_NFIND(struct name *head, struct type *elm)			\
+{									\
+	struct type *tmp = VRB_ROOT(head);				\
+	struct type *res = NULL;					\
+	int comp;							\
+	while (tmp) {							\
+		comp = cmp(elm, tmp);					\
+		if (comp < 0) {						\
+			res = tmp;					\
+			tmp = VRB_LEFT(tmp, field);			\
+		}							\
+		else if (comp > 0)					\
+			tmp = VRB_RIGHT(tmp, field);			\
+		else							\
+			return (tmp);					\
+	}								\
+	return (res);							\
+}									\
+									\
+/* ARGSUSED */								\
+attr struct type *							\
+name##_VRB_NEXT(struct type *elm)					\
+{									\
+	if (VRB_RIGHT(elm, field)) {					\
+		elm = VRB_RIGHT(elm, field);				\
+		while (VRB_LEFT(elm, field))				\
+			elm = VRB_LEFT(elm, field);			\
+	} else {							\
+		if (VRB_PARENT(elm, field) &&				\
+		    (elm == VRB_LEFT(VRB_PARENT(elm, field), field)))	\
+			elm = VRB_PARENT(elm, field);			\
+		else {							\
+			while (VRB_PARENT(elm, field) &&			\
+			    (elm == VRB_RIGHT(VRB_PARENT(elm, field), field)))\
+				elm = VRB_PARENT(elm, field);		\
+			elm = VRB_PARENT(elm, field);			\
+		}							\
+	}								\
+	return (elm);							\
+}									\
+									\
+/* ARGSUSED */								\
+attr struct type *							\
+name##_VRB_PREV(struct type *elm)					\
+{									\
+	if (VRB_LEFT(elm, field)) {					\
+		elm = VRB_LEFT(elm, field);				\
+		while (VRB_RIGHT(elm, field))				\
+			elm = VRB_RIGHT(elm, field);			\
+	} else {							\
+		if (VRB_PARENT(elm, field) &&				\
+		    (elm == VRB_RIGHT(VRB_PARENT(elm, field), field)))	\
+			elm = VRB_PARENT(elm, field);			\
+		else {							\
+			while (VRB_PARENT(elm, field) &&			\
+			    (elm == VRB_LEFT(VRB_PARENT(elm, field), field)))\
+				elm = VRB_PARENT(elm, field);		\
+			elm = VRB_PARENT(elm, field);			\
+		}							\
+	}								\
+	return (elm);							\
+}									\
+									\
+attr struct type *							\
+name##_VRB_MINMAX(struct name *head, int val)				\
+{									\
+	struct type *tmp = VRB_ROOT(head);				\
+	struct type *parent = NULL;					\
+	while (tmp) {							\
+		parent = tmp;						\
+		if (val < 0)						\
+			tmp = VRB_LEFT(tmp, field);			\
+		else							\
+			tmp = VRB_RIGHT(tmp, field);			\
+	}								\
+	return (parent);						\
+}
+
+#define VRB_NEGINF	-1
+#define VRB_INF	1
+
+#define VRB_INSERT(name, x, y)	name##_VRB_INSERT(x, y)
+#define VRB_REMOVE(name, x, y)	name##_VRB_REMOVE(x, y)
+#define VRB_FIND(name, x, y)	name##_VRB_FIND(x, y)
+#define VRB_NFIND(name, x, y)	name##_VRB_NFIND(x, y)
+#define VRB_NEXT(name, x, y)	name##_VRB_NEXT(y)
+#define VRB_PREV(name, x, y)	name##_VRB_PREV(y)
+#define VRB_MIN(name, x)		name##_VRB_MINMAX(x, VRB_NEGINF)
+#define VRB_MAX(name, x)		name##_VRB_MINMAX(x, VRB_INF)
+
+#define VRB_FOREACH(x, name, head)					\
+	for ((x) = VRB_MIN(name, head);					\
+	     (x) != NULL;						\
+	     (x) = name##_VRB_NEXT(x))
+
+#define VRB_FOREACH_FROM(x, name, y)					\
+	for ((x) = (y);							\
+	    ((x) != NULL) && ((y) = name##_VRB_NEXT(x), (x) != NULL);	\
+	     (x) = (y))
+
+#define VRB_FOREACH_SAFE(x, name, head, y)				\
+	for ((x) = VRB_MIN(name, head);					\
+	    ((x) != NULL) && ((y) = name##_VRB_NEXT(x), (x) != NULL);	\
+	     (x) = (y))
+
+#define VRB_FOREACH_REVERSE(x, name, head)				\
+	for ((x) = VRB_MAX(name, head);					\
+	     (x) != NULL;						\
+	     (x) = name##_VRB_PREV(x))
+
+#define VRB_FOREACH_REVERSE_FROM(x, name, y)				\
+	for ((x) = (y);							\
+	    ((x) != NULL) && ((y) = name##_VRB_PREV(x), (x) != NULL);	\
+	     (x) = (y))
+
+#define VRB_FOREACH_REVERSE_SAFE(x, name, head, y)			\
+	for ((x) = VRB_MAX(name, head);					\
+	    ((x) != NULL) && ((y) = name##_VRB_PREV(x), (x) != NULL);	\
+	     (x) = (y))
+
+#endif	/* _VTREE_H_ */
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 6069534..ac4fabc 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -27,6 +27,8 @@ libvarnishapi_la_SOURCES = \
 	vsm.c \
 	vsl_arg.c \
 	vsl_cursor.c \
+	vsl_dispatch.c \
+	vsl_query.c \
 	vsl.c \
 	vsc.c \
 	libvarnishapi.map
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index a25aa94..6099c4e 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -104,6 +104,13 @@ LIBVARNISHAPI_1.3 {
 	VSL_DeleteCursor;
 	VSL_Next;
 	VSL_Match;
-	VSL_Print;
+	VSL_PrintVXID;
+	VSL_PrintLevel;
+	VSL_PrintAll;
+	VSL_PrintSet;
+	VSLQ_New;
+	VSLQ_Delete;
+	VSLQ_Dispatch;
+	VSLQ_Flush;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 963afce..994d76d 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -155,7 +155,7 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c)
 }
 
 int
-VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
+VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 {
 	enum VSL_tag_e tag;
 	uint32_t vxid;
@@ -200,3 +200,101 @@ VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 		return (-5);
 	return (0);
 }
+
+int
+VSL_PrintLevel(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
+{
+	enum VSL_tag_e tag;
+	unsigned len, lvl;
+	const char *data;
+	int type;
+	int i;
+
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	if (c == NULL || c->ptr == NULL)
+		return (0);
+	if (fo == NULL)
+		fo = stdout;
+
+	tag = VSL_TAG(c->ptr);
+	len = VSL_LEN(c->ptr);
+	type = VSL_CLIENT(c->ptr) ? 'c' : VSL_BACKEND(c->ptr) ? 'b' : '-';
+	data = VSL_CDATA(c->ptr);
+	lvl = c->level;
+	if (tag == SLT_Debug) {
+		i = fprintf(fo, "%2u %-15s %c \"", lvl, VSL_tags[tag],
+		    type);
+		if (i < 0)
+			return (-5);
+		while (len-- > 0) {
+			if (*data >= ' ' && *data <= '~')
+				i = fprintf(fo, "%c", *data);
+			else
+				i = fprintf(fo, "%%%02x",
+				    (unsigned char)*data);
+			if (i < 0)
+				return (-5);
+			data++;
+		}
+		i = fprintf(fo, "\"\n");
+		if (i < 0)
+			return (-5);
+		return (0);
+	}
+	i = fprintf(fo, "%2u %-15s %c %.*s\n",
+	    lvl, VSL_tags[tag], type, (int)len, data);
+	if (i < 0)
+		return (-5);
+	return (0);
+}
+
+int
+VSL_PrintAll(struct VSL_data *vsl, struct VSL_cursor *c, void *fo)
+{
+	int i;
+
+	if (c == NULL)
+		return (0);
+	if (c->vxid >= 0) {
+		i = fprintf(fo, "vv VXID: %11u vv\n", c->vxid);
+		if (i < 0)
+			return (-5);
+	}
+	while (1) {
+		i = VSL_Next(c);
+		if (i <= 0)
+			return (i);
+		if (!VSL_Match(vsl, c))
+			continue;
+		if (c->vxid < 0)
+			i = VSL_PrintVXID(vsl, c, fo);
+		else
+			i = VSL_PrintLevel(vsl, c, fo);
+		if (i != 0)
+			return (i);
+	}
+}
+
+int
+VSL_PrintSet(struct VSL_data *vsl, struct VSL_cursor *cp[], void *fo)
+{
+	int i;
+	int delim = 0;
+	struct VSL_cursor *c;
+
+	c = cp[0];
+	while (c) {
+		if (c->vxid >= 0)
+			delim = 1;
+		i = VSL_PrintAll(vsl, c, fo);
+		if (i)
+			return (i);
+		c = *++cp;
+	}
+	if (delim) {
+		i = fprintf(fo, "\n");
+		if (i < 0)
+			return (-5);
+	}
+	return (0);
+}
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index f03812e..a03aef7 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -37,10 +37,12 @@
 
 int vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
     __printflike(2, 3);
+int vsl_skip(struct VSL_cursor *c, ssize_t words);
 
 typedef void vslc_delete_f(void *);
 typedef int vslc_next_f(void *);
 typedef int vslc_reset_f(void *);
+typedef int vslc_skip_f(void *, ssize_t words);
 
 struct vslc {
 	struct VSL_cursor		c;
@@ -50,6 +52,7 @@ struct vslc {
 	vslc_delete_f			*delete;
 	vslc_next_f			*next;
 	vslc_reset_f			*reset;
+	vslc_skip_f			*skip;
 };
 
 struct VSL_data {
@@ -66,3 +69,10 @@ struct VSL_data {
 	struct vbitmap			*vbm_select;
 	struct vbitmap			*vbm_supress;
 };
+
+/* vsl_query.c */
+struct vslq_query;
+struct vslq_query *vslq_newquery(struct VSL_data *vsl,
+    enum VSL_grouping_e grouping, const char *query);
+void vslq_deletequery(struct vslq_query **pquery);
+int vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[]);
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index f0d535b..bf4d1ec 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -169,6 +169,23 @@ vslc_vsm_reset(void *cursor)
 	return (0);
 }
 
+static int
+vslc_vsm_skip(void *cursor, ssize_t words)
+{
+	struct vslc_vsm *c;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_VSM_MAGIC);
+	if (words < 0)
+		return (-1);
+
+	c->next += words;
+	assert(c->next >= c->head->log);
+	assert(c->next < c->end);
+	c->c.c.ptr = NULL;
+
+	return (0);
+}
+
 struct VSL_cursor *
 VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 {
@@ -199,10 +216,13 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 		vsl_diag(vsl, "Out of memory\n");
 		return (NULL);
 	}
+	c->c.c.vxid = -1;	/* N/A to this cursor type */
+	c->c.c.shmptr_ok = 1;
 	c->c.magic = VSLC_MAGIC;
 	c->c.delete = vslc_vsm_delete;
 	c->c.next = vslc_vsm_next;
 	c->c.reset = vslc_vsm_reset;
+	c->c.skip = vslc_vsm_skip;
 
 	c->vsm = vsm;
 	c->vf = vf;
@@ -347,6 +367,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 		vsl_diag(vsl, "Out of memory\n");
 		return (NULL);
 	}
+	c->c.c.vxid = -1;	/* N/A to this cursor type */
 	c->c.magic = VSLC_MAGIC;
 	c->c.delete = vslc_file_delete;
 	c->c.next = vslc_file_next;
@@ -391,3 +412,14 @@ VSL_Next(struct VSL_cursor *cursor)
 	AN(c->next);
 	return ((c->next)(c));
 }
+
+int
+vsl_skip(struct VSL_cursor *cursor, ssize_t words)
+{
+	struct vslc *c;
+
+	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
+	if (c->skip == NULL)
+		return (-1);
+	return ((c->skip)(c, words));
+}
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
new file mode 100644
index 0000000..61b0518
--- /dev/null
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -0,0 +1,924 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdint.h>
+
+#include "vas.h"
+#include "miniobj.h"
+#include "vqueue.h"
+#include "vtree.h"
+#include "vtim.h"
+
+#include "vapi/vsl.h"
+#include "vsl_api.h"
+
+#define VTX_CACHE 10
+#define VTX_BUFSIZE_MIN 64
+
+enum vtx_type_e {
+	vtx_t_unknown,
+	vtx_t_sess,
+	vtx_t_req,
+	vtx_t_esireq,
+	vtx_t_bereq,
+};
+
+enum vtx_link_e {
+	vtx_l_sess,
+	vtx_l_req,
+	vtx_l_esireq,
+	vtx_l_bereq,
+};
+
+struct vtx_key {
+	unsigned		vxid;
+	VRB_ENTRY(vtx_key)	entry;
+};
+VRB_HEAD(vtx_tree, vtx_key);
+
+struct vtx {
+	struct vtx_key		key;
+	unsigned		magic;
+#define VTX_MAGIC		0xACC21D09
+	VTAILQ_ENTRY(vtx)	list_child;
+	VTAILQ_ENTRY(vtx)	list_incomplete;
+
+	double			t_start;
+	unsigned		flags;
+#define VTX_F_SHM		0x1
+#define VTX_F_COMPLETE		0x2
+#define VTX_F_READY		0x4
+
+	enum vtx_type_e		type;
+
+	struct vtx		*parent;
+	VTAILQ_HEAD(,vtx)	child;
+	unsigned		n_child;
+	unsigned		n_childready;
+	unsigned		n_descend;
+
+	const uint32_t		*start;
+	ssize_t			len;
+	unsigned		index;
+
+	uint32_t		*buf;
+	ssize_t			bufsize;
+};
+
+struct vslc_raw {
+	struct vslc		c;
+	unsigned		magic;
+#define VSLC_RAW_MAGIC		0x247EBD44
+
+	const uint32_t		*start;
+	ssize_t			len;
+	const uint32_t		*next;
+};
+
+struct vslc_vtx {
+	struct vslc		c;
+	unsigned		magic;
+#define VSLC_VTX_MAGIC		0x74C6523F
+
+	struct vtx		*vtx;
+	const uint32_t		*next;
+};
+
+struct VSLQ {
+	unsigned		magic;
+#define VSLQ_MAGIC		0x23A8BE97
+
+	struct VSL_data		*vsl;
+	struct VSL_cursor	*c;
+	struct vslq_query	*query;
+
+	enum VSL_grouping_e	grouping;
+
+	struct vtx_tree		tree;
+
+	VTAILQ_HEAD(,vtx)	incomplete;
+	unsigned		n_incomplete;
+
+	VTAILQ_HEAD(,vtx)	cache;
+	unsigned		n_cache;
+};
+
+static inline int
+vtx_keycmp(const struct vtx_key *a, const struct vtx_key *b)
+{
+	if (a->vxid < b->vxid)
+		return (-1);
+	if (a->vxid > b->vxid)
+		return (1);
+	return (0);
+}
+
+VRB_PROTOTYPE(vtx_tree, vtx_key, entry, vtx_keycmp);
+VRB_GENERATE(vtx_tree, vtx_key, entry, vtx_keycmp);
+
+static int
+vtx_diag(struct vtx *vtx, const char *fmt, ...)
+{
+	va_list ap;
+
+	/* XXX: Prepend diagnostic message on vtx as a synthetic log
+	   record. For now print to stderr */
+	fprintf(stderr, "vtx_diag <%u>: ", vtx->key.vxid);
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+	fprintf(stderr, "\n");
+
+	return (-1);
+}
+
+static int
+vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr, const char *reason)
+{
+	return (vtx_diag(vtx, "%s (%s: %.*s)", reason, VSL_tags[VSL_TAG(ptr)],
+		(int)VSL_LEN(ptr), VSL_CDATA(ptr)));
+}
+
+static int
+vslc_raw_next(void *cursor)
+{
+	struct vslc_raw *c;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_RAW_MAGIC);
+
+	assert(c->next >= c->start);
+	assert(c->next <= c->start + c->len);
+	if (c->next < c->start + c->len) {
+		c->c.c.ptr = c->next;
+		c->next = VSL_NEXT(c->next);
+		return (1);
+	}
+	return (0);
+}
+
+static int
+vslc_raw_reset(void *cursor)
+{
+	struct vslc_raw *c;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_RAW_MAGIC);
+
+	assert(c->next >= c->start);
+	assert(c->next <= c->start + c->len);
+	c->next = c->start;
+	c->c.c.ptr = NULL;
+
+	return (0);
+}
+
+static int
+vslc_vtx_next(void *cursor)
+{
+	struct vslc_vtx *c;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_VTX_MAGIC);
+
+	assert(c->next >= c->vtx->start);
+	assert(c->next <= c->vtx->start + c->vtx->len);
+	if (c->next < c->vtx->start + c->vtx->len) {
+		c->c.c.ptr = c->next;
+		c->next = VSL_NEXT(c->next);
+		return (1);
+	}
+	return (0);
+}
+
+static int
+vslc_vtx_reset(void *cursor)
+{
+	struct vslc_vtx *c;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_VTX_MAGIC);
+
+	assert(c->next >= c->vtx->start);
+	assert(c->next <= c->vtx->start + c->vtx->len);
+	c->next = c->vtx->start;
+	c->c.c.ptr = NULL;
+
+	return (0);
+}
+
+static void
+vslc_vtx_setup(struct vslc_vtx *c, struct vtx *vtx, unsigned level)
+{
+	AN(c);
+	AN(vtx);
+
+	memset(c, 0, sizeof *c);
+	c->c.c.vxid = vtx->key.vxid;
+	c->c.c.level = level;
+	c->c.magic = VSLC_MAGIC;
+	c->c.next = vslc_vtx_next;
+	c->c.reset = vslc_vtx_reset;
+
+	c->magic = VSLC_VTX_MAGIC;
+	c->vtx = vtx;
+	c->next = c->vtx->start;
+}
+
+static struct vtx *
+vtx_new(struct VSLQ *vslq)
+{
+	struct vtx *vtx;
+
+	AN(vslq);
+	if (vslq->n_cache) {
+		AZ(VTAILQ_EMPTY(&vslq->cache));
+		vtx = VTAILQ_FIRST(&vslq->cache);
+		VTAILQ_REMOVE(&vslq->cache, vtx, list_child);
+		vslq->n_cache--;
+	} else {
+		ALLOC_OBJ(vtx, VTX_MAGIC);
+		AN(vtx);
+	}
+
+	vtx->key.vxid = 0;
+	vtx->t_start = VTIM_mono();
+	vtx->flags = 0;
+	vtx->type = vtx_t_unknown;
+	vtx->parent = NULL;
+	VTAILQ_INIT(&vtx->child);
+	vtx->n_child = 0;
+	vtx->n_childready = 0;
+	vtx->n_descend = 0;
+	vtx->start = vtx->buf;
+	vtx->len = 0;
+	vtx->index = 0;
+
+	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_incomplete);
+	vslq->n_incomplete++;
+
+	return (vtx);
+}
+
+static void
+vtx_free(struct vtx **pvtx)
+{
+	struct vtx *vtx;
+
+	AN(pvtx);
+	vtx = *pvtx;
+	*pvtx = NULL;
+
+	free(vtx->buf);
+	FREE_OBJ(vtx);
+}
+
+static void
+vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
+{
+	struct vtx *vtx;
+	struct vtx *child;
+
+	AN(vslq);
+	AN(pvtx);
+
+	vtx = *pvtx;
+	*pvtx = NULL;
+	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+
+	AN(vtx->flags & VTX_F_COMPLETE);
+	AN(vtx->flags & VTX_F_READY);
+	AZ(vtx->parent);
+
+	while (!VTAILQ_EMPTY(&vtx->child)) {
+		child = VTAILQ_FIRST(&vtx->child);
+		assert(child->parent == vtx);
+		AN(vtx->n_child);
+		assert(vtx->n_descend >= child->n_descend + 1);
+		VTAILQ_REMOVE(&vtx->child, child, list_child);
+		child->parent = NULL;
+		vtx->n_child--;
+		vtx->n_descend -= child->n_descend + 1;
+		vtx_retire(vslq, &child);
+		AZ(child);
+	}
+	AZ(vtx->n_child);
+	AZ(vtx->n_descend);
+	AN(VRB_REMOVE(vtx_tree, &vslq->tree, &vtx->key));
+
+	if (vslq->n_cache < VTX_CACHE) {
+		VTAILQ_INSERT_HEAD(&vslq->cache, vtx, list_child);
+		vslq->n_cache++;
+	} else {
+		vtx_free(&vtx);
+		AZ(vtx);
+	}
+}
+
+static struct vtx *
+vtx_lori(struct VSLQ *vslq, unsigned vxid)
+{
+	struct vtx *vtx;
+	struct vtx_key lkey, *key;
+
+	AN(vslq);
+	lkey.vxid = vxid;
+	key = VRB_FIND(vtx_tree, &vslq->tree, &lkey);
+	if (key != NULL) {
+		CAST_OBJ_NOTNULL(vtx, (void *)key, VTX_MAGIC);
+		return (vtx);
+	}
+
+	vtx = vtx_new(vslq);
+	AN(vtx);
+	vtx->key.vxid = vxid;
+	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
+	return (vtx);
+}
+
+static void
+vtx_append(struct vtx *vtx, const uint32_t *ptr, ssize_t len, int shmptr_ok)
+{
+	ssize_t bufsize;
+	const uint32_t *ptr2;
+	ssize_t len2;
+
+	AN(vtx);
+
+	if (vtx->flags & VTX_F_SHM) {
+		assert(vtx->start != vtx->buf);
+		ptr2 = vtx->start;
+		vtx->start = vtx->buf;
+		len2 = vtx->len;
+		vtx->len = 0;
+		vtx->flags &= ~VTX_F_SHM;
+		vtx_append(vtx, ptr2, len2, 0);
+	}
+
+	if (len == 0)
+		return;
+	AN(ptr);
+
+	if (shmptr_ok && vtx->len == 0) {
+		vtx->start = ptr;
+		vtx->len = len;
+		vtx->flags |= VTX_F_SHM;
+		return;
+	}
+
+	bufsize = vtx->bufsize;
+	if (bufsize == 0)
+		bufsize = VTX_BUFSIZE_MIN;
+	while (vtx->len + len > bufsize)
+		bufsize *= 2;
+	if (bufsize != vtx->bufsize) {
+		vtx->buf = realloc(vtx->buf, 4 * bufsize);
+		AN(vtx->buf);
+		vtx->bufsize = bufsize;
+		vtx->start = vtx->buf;
+	}
+	memcpy(&vtx->buf[vtx->len], ptr, 4 * len);
+	vtx->len += len;
+}
+
+static struct vtx *
+vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
+{
+	struct vtx *ready;
+
+	AN(vslq);
+	AN(vtx->flags & VTX_F_COMPLETE);
+	AZ(vtx->flags & VTX_F_READY);
+
+	if (vtx->type == vtx_t_unknown)
+		vtx_diag(vtx, "vtx of unknown type marked complete");
+
+	ready = vtx;
+	while (1) {
+		if (ready->flags & VTX_F_COMPLETE &&
+		    ready->n_child == ready->n_childready)
+			ready->flags |= VTX_F_READY;
+		else
+			break;
+		if (ready->parent == NULL)
+			break;
+		ready = ready->parent;
+		ready->n_childready++;
+		assert(ready->n_child >= ready->n_childready);
+	}
+
+	if (ready->flags & VTX_F_READY && ready->parent == NULL)
+		/* Top level vtx ready */
+		return (ready);
+
+	if (vtx->flags & VTX_F_SHM) {
+		/* Not ready, append zero to make sure it's not a shm
+		   reference */
+		vtx_append(vtx, NULL, 0, 0);
+		AZ(vtx->flags & VTX_F_SHM);
+	}
+
+	return (NULL);
+}
+
+static int
+vtx_parsetag_bl(const char *str, unsigned strlen, enum vtx_type_e *ptype,
+    unsigned *pvxid)
+{
+	char ibuf[strlen + 1];
+	char tbuf[7];
+	unsigned vxid;
+	int i;
+	enum vtx_type_e type = vtx_t_unknown;
+
+	AN(str);
+	memcpy(ibuf, str, strlen);
+	ibuf[strlen] = '\0';
+	i = sscanf(ibuf, "%6s %u", tbuf, &vxid);
+	if (i < 1)
+		return (-1);
+	if (!strcmp(tbuf, "sess"))
+		type = vtx_t_sess;
+	else if (!strcmp(tbuf, "req"))
+		type = vtx_t_req;
+	else if (!strcmp(tbuf, "esireq"))
+		type = vtx_t_esireq;
+	else if (!strcmp(tbuf, "bereq"))
+		type = vtx_t_bereq;
+	else
+		return (-1);
+	if (i == 1)
+		vxid = 0;
+	if (ptype)
+		*ptype = type;
+	if (pvxid)
+		*pvxid = vxid;
+	return (i);
+}
+
+static void
+vtx_set_parent(struct vtx *parent, struct vtx *child)
+{
+
+	AN(parent);
+	AN(child);
+	AZ(child->parent);
+	child->parent = parent;
+	VTAILQ_INSERT_TAIL(&parent->child, child, list_child);
+	parent->n_child++;
+	do
+		parent->n_descend += 1 + child->n_descend;
+	while ((parent = parent->parent));
+}
+
+static int
+vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
+{
+	int i;
+	enum vtx_type_e p_type;
+	unsigned p_vxid;
+	struct vtx *p_vtx;
+
+	assert(VSL_TAG(ptr) == SLT_Begin);
+
+	if (vtx->flags & VTX_F_READY)
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
+
+	i = vtx_parsetag_bl(VSL_CDATA(ptr), VSL_LEN(ptr), &p_type, &p_vxid);
+	if (i < 1)
+		return (vtx_diag_tag(vtx, ptr, "parse error"));
+
+	/* Check/set vtx type */
+	assert(p_type != vtx_t_unknown);
+	if (vtx->type != vtx_t_unknown && vtx->type != p_type)
+		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
+	vtx->type = p_type;
+
+	if (i == 1 || p_vxid == 0)
+		return (0);
+
+	/* Lookup and check parent vtx */
+	p_vtx = vtx_lori(vslq, p_vxid);
+	AN(p_vtx);
+	if (vtx->parent == p_vtx)
+		/* Link already exists */
+		return (0);
+	if (vtx->parent != NULL)
+		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
+	if (p_vtx->flags & VTX_F_READY)
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
+
+	vtx_set_parent(p_vtx, vtx);
+
+	return (0);
+}
+
+static int
+vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
+{
+	int i;
+	enum vtx_type_e c_type;
+	unsigned c_vxid;
+	struct vtx *c_vtx;
+
+	assert(VSL_TAG(ptr) == SLT_Link);
+
+	if (vtx->flags & VTX_F_READY)
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
+
+	i = vtx_parsetag_bl(VSL_CDATA(ptr), VSL_LEN(ptr), &c_type, &c_vxid);
+	if (i < 2)
+		return (vtx_diag_tag(vtx, ptr, "parse error"));
+	assert(i == 2);
+
+	/* Lookup and check child vtx */
+	c_vtx = vtx_lori(vslq, c_vxid);
+	AN(c_vtx);
+	if (c_vtx->parent == vtx)
+		/* Link already exists */
+		return (0);
+	if (c_vtx->parent != NULL)
+		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
+	if (c_vtx->flags & VTX_F_READY)
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
+	assert(c_type != vtx_t_unknown);
+	if (c_vtx->type != vtx_t_unknown && c_vtx->type != c_type)
+		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
+	c_vtx->type = c_type;
+
+	vtx_set_parent(vtx, c_vtx);
+
+	return (0);
+}
+
+static struct vtx *
+vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
+{
+	const uint32_t *ptr;
+	enum VSL_tag_e tag;
+	int complete;
+
+	complete = (vtx->flags & VTX_F_COMPLETE ? 1 : 0);
+	ptr = vtx->start + vtx->index;
+	assert(ptr <= vtx->start + vtx->len);
+	for (; ptr < vtx->start + vtx->len; ptr = VSL_NEXT(ptr)) {
+		tag = VSL_TAG(ptr);
+
+		if (complete) {
+			vtx_diag(vtx, "late log rec");
+			continue;
+		}
+
+		if (vtx->type == vtx_t_unknown && tag != SLT_Begin)
+			vtx_diag_tag(vtx, ptr, "early log rec");
+
+		switch (tag) {
+		case SLT_Begin:
+			(void)vtx_scan_begintag(vslq, vtx, ptr);
+			break;
+
+		case SLT_Link:
+			(void)vtx_scan_linktag(vslq, vtx, ptr);
+			break;
+
+		case SLT_End:
+			complete = 1;
+			break;
+
+		default:
+			break;
+		}
+	}
+	vtx->index = ptr - vtx->start;
+	assert(vtx->index <= vtx->len);
+
+	if (!complete && vtx->flags & VTX_F_SHM) {
+		/* Append zero to make sure it's not a shm reference */
+		vtx_append(vtx, NULL, 0, 0);
+		AZ(vtx->flags & VTX_F_SHM);
+	}
+
+	if (complete) {
+		VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
+		vtx->flags |= VTX_F_COMPLETE;
+		AN(vslq->n_incomplete);
+		vslq->n_incomplete--;
+		return (vtx_check_ready(vslq, vtx));
+	}
+
+	return (NULL);
+}
+
+static struct vtx *
+vtx_force(struct VSLQ *vslq, struct vtx *vtx, const char *reason)
+{
+	AZ(vtx->flags & VTX_F_COMPLETE);
+	AZ(vtx->flags & VTX_F_READY);
+	vtx_diag(vtx, reason);
+
+	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
+	vtx->flags |= VTX_F_COMPLETE;
+	AN(vslq->n_incomplete);
+	vslq->n_incomplete--;
+
+	return (vtx_check_ready(vslq, vtx));
+}
+
+static int
+vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
+    void *priv)
+{
+	unsigned n = vtx->n_descend + 1;
+	unsigned i, j;
+	struct vslc_vtx c[n];
+	struct VSL_cursor *cp[n + 1];
+
+	AN(vslq);
+	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+
+	if (func == NULL)
+		return (0);
+	if (vslq->grouping == VSL_g_session &&
+	    vtx->type != vtx_t_sess)
+		return (0);
+	if (vslq->grouping == VSL_g_request &&
+	    vtx->type != vtx_t_req)
+		return (0);
+
+	i = j = 0;
+	vslc_vtx_setup(&c[i], vtx, 0);
+	i++;
+	while (j < i) {
+		vtx = VTAILQ_FIRST(&c[j].vtx->child);
+		while (vtx) {
+			assert(i < n);
+			vslc_vtx_setup(&c[i], vtx, c[j].c.c.level + 1);
+			i++;
+			vtx = VTAILQ_NEXT(vtx, list_child);
+		}
+		j++;
+	}
+	assert(i == n);
+
+	/* Reverse order */
+	for (i = 0; i < n; i++)
+		cp[i] = &c[n - i - 1].c.c;
+	cp[i] = NULL;
+
+	/* Query test goes here */
+	if (vslq->query == NULL ? 1 : vslq_runquery(vslq->query, cp))
+		return ((func)(vslq->vsl, cp, priv));
+	else
+		return (0);
+}
+
+struct VSLQ *
+VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp,
+    enum VSL_grouping_e grouping, const char *querystring)
+{
+	struct vslq_query *query;
+	struct VSLQ *vslq;
+
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	AN(cp);
+	if (grouping > VSL_g_session) {
+		(void)vsl_diag(vsl, "Illegal query grouping");
+		return (NULL);
+	}
+	if (querystring != NULL) {
+		query = vslq_newquery(vsl, grouping, querystring);
+		if (query == NULL)
+			return (NULL);
+	} else
+		query = NULL;
+
+	ALLOC_OBJ(vslq, VSLQ_MAGIC);
+	AN(vslq);
+	vslq->vsl = vsl;
+	vslq->c = *cp;
+	*cp = NULL;
+	vslq->grouping = grouping;
+	vslq->query = query;
+	VRB_INIT(&vslq->tree);
+	VTAILQ_INIT(&vslq->incomplete);
+	VTAILQ_INIT(&vslq->cache);
+
+	return (vslq);
+}
+
+void
+VSLQ_Delete(struct VSLQ **pvslq)
+{
+	struct VSLQ *vslq;
+	struct vtx *vtx;
+
+	AN(pvslq);
+	vslq = *pvslq;
+	*pvslq = NULL;
+	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
+
+	(void)VSLQ_Flush(vslq, NULL, NULL);
+	AZ(vslq->n_incomplete);
+	VSL_DeleteCursor(vslq->c);
+	vslq->c = NULL;
+
+	if (vslq->query != NULL)
+		vslq_deletequery(&vslq->query);
+	AZ(vslq->query);
+
+	while (!VTAILQ_EMPTY(&vslq->cache)) {
+		AN(vslq->n_cache);
+		vtx = VTAILQ_FIRST(&vslq->cache);
+		VTAILQ_REMOVE(&vslq->cache, vtx, list_child);
+		vslq->n_cache--;
+		vtx_free(&vtx);
+		AZ(vtx);
+	}
+
+	FREE_OBJ(vslq);
+}
+
+static int
+vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+{
+	struct vslc_raw rawc;
+	struct VSL_cursor *c;
+	struct VSL_cursor *pc[2];
+	int i;
+
+	assert(vslq->grouping == VSL_g_raw);
+	c = vslq->c;
+
+	memset(&rawc, 0, sizeof rawc);
+	rawc.c.c.vxid = -1;
+	rawc.c.magic = VSLC_MAGIC;
+	rawc.c.next = vslc_raw_next;
+	rawc.c.reset = vslc_raw_reset;
+	rawc.magic = VSLC_RAW_MAGIC;
+	pc[0] = &rawc.c.c;
+	pc[1] = NULL;
+
+	while (1) {
+		i = VSL_Next(c);
+		if (i <= 0)
+			break;
+		AN(c->ptr);
+		if (func == NULL)
+			continue;
+		rawc.start = c->ptr;
+		rawc.len = VSL_NEXT(c->ptr) - c->ptr;
+		rawc.next = rawc.start;
+		rawc.c.c.ptr = NULL;
+
+		/* Query check goes here */
+		i = 0;
+		if (vslq->query == NULL ? 1 : vslq_runquery(vslq->query, pc))
+			i = (func)(vslq->vsl, pc, priv);
+		if (i)
+			break;
+	}
+
+	return (i);
+}
+
+int
+VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+{
+	struct VSL_cursor *c;
+	int i;
+	enum VSL_tag_e tag;
+	const uint32_t *ptr;
+	ssize_t len;
+	unsigned vxid;
+	struct vtx *vtx;
+	double now;
+
+	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
+
+	if (vslq->grouping == VSL_g_raw)
+		return (vslq_raw(vslq, func, priv));
+
+	c = vslq->c;
+	while (1) {
+		i = VSL_Next(c);
+		if (i != 1)
+			break;
+		tag = VSL_TAG(c->ptr);
+		if (tag == SLT__Batch) {
+			ptr = VSL_NEXT(c->ptr);
+			len = VSL_WORDS(c->ptr[1]);
+			AZ(vsl_skip(c, len));
+		} else {
+			ptr = c->ptr;
+			len = VSL_NEXT(ptr) - ptr;
+		}
+		vxid = VSL_ID(ptr);
+		if (vxid == 0)
+			continue;
+		vtx = vtx_lori(vslq, vxid);
+		AN(vtx);
+		vtx_append(vtx, ptr, len, c->shmptr_ok);
+		vtx = vtx_scan(vslq, vtx);
+		if (vtx) {
+			AN(vtx->flags & VTX_F_READY);
+			i = vslq_callback(vslq, vtx, func, priv);
+			vtx_retire(vslq, &vtx);
+			AZ(vtx);
+			if (i)
+				break;
+		}
+	}
+	if (i)
+		return (i);
+
+	now = VTIM_mono();
+	while ((vtx = VTAILQ_FIRST(&vslq->incomplete)) &&
+	    now - vtx->t_start > 120.) {
+		/* XXX: Make timeout configurable through options and
+		   provide a sane default */
+		AZ(vtx->flags & VTX_F_COMPLETE);
+		vtx = vtx_force(vslq, vtx, "incomplete - timeout");
+		if (vtx) {
+			AN(vtx->flags & VTX_F_READY);
+			i = vslq_callback(vslq, vtx, func, priv);
+			vtx_retire(vslq, &vtx);
+			AZ(vtx);
+			if (i)
+				break;
+		}
+	}
+	if (i)
+		return (i);
+
+	while (vslq->n_incomplete > 1000) {
+		/* XXX: Make limit configurable through options and
+		   provide a sane default */
+		vtx = VTAILQ_FIRST(&vslq->incomplete);
+		AN(vtx);
+		AZ(vtx->flags & VTX_F_COMPLETE);
+		vtx = vtx_force(vslq, vtx, "incomplete - store overflow");
+		if (vtx) {
+			AN(vtx->flags & VTX_F_READY);
+			i = vslq_callback(vslq, vtx, func, priv);
+			vtx_retire(vslq, &vtx);
+			AZ(vtx);
+			if (i)
+				break;
+		}
+	}
+
+	return (i);
+}
+
+int
+VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+{
+	struct vtx *vtx;
+	int i = 0;
+
+	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
+
+	while (vslq->n_incomplete) {
+		vtx = VTAILQ_FIRST(&vslq->incomplete);
+		AN(vtx);
+		AZ(vtx->flags & VTX_F_COMPLETE);
+		vtx = vtx_force(vslq, vtx, "incomplete - flushing");
+		if (vtx) {
+			AN(vtx->flags & VTX_F_READY);
+			i = vslq_callback(vslq, vtx, func, priv);
+			vtx_retire(vslq, &vtx);
+			AZ(vtx);
+			if (i)
+				break;
+		}
+	}
+	return (i);
+}
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
new file mode 100644
index 0000000..b4a403c
--- /dev/null
+++ b/lib/libvarnishapi/vsl_query.c
@@ -0,0 +1,122 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "vas.h"
+#include "miniobj.h"
+#include "vre.h"
+
+#include "vapi/vsl.h"
+#include "vsl_api.h"
+
+struct vslq_query {
+	unsigned		magic;
+#define VSLQ_QUERY_MAGIC	0x122322A5
+
+	vre_t			*regex;
+};
+
+struct vslq_query *
+vslq_newquery(struct VSL_data *vsl, enum VSL_grouping_e grouping,
+    const char *querystring)
+{
+	struct vslq_query *query;
+	const char *error;
+	int pos;
+	vre_t *regex;
+
+	(void)grouping;
+	AN(querystring);
+	regex = VRE_compile(querystring, 0, &error, &pos);
+	if (regex == NULL) {
+		vsl_diag(vsl, "failed to compile regex at pos %d: %s",
+		    pos, error);
+		return (NULL);
+	}
+
+	ALLOC_OBJ(query, VSLQ_QUERY_MAGIC);
+	query->regex = regex;
+	return (query);
+}
+
+void
+vslq_deletequery(struct vslq_query **pquery)
+{
+	struct vslq_query *query;
+
+	AN(pquery);
+	query = *pquery;
+	*pquery = NULL;
+	CHECK_OBJ_NOTNULL(query, VSLQ_QUERY_MAGIC);
+
+	AN(query->regex);
+	VRE_free(&query->regex);
+	AZ(query->regex);
+
+	FREE_OBJ(query);
+}
+
+int
+vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[])
+{
+	struct VSL_cursor *c;
+	int i, len;
+	const char *data;
+
+	CHECK_OBJ_NOTNULL(query, VSLQ_QUERY_MAGIC);
+	AN(query->regex);
+
+	c = cp[0];
+	while (c) {
+		while (1) {
+			i = VSL_Next(c);
+			if (i == 0)
+				break;
+			assert(i == 1);
+			AN(c->ptr);
+			len = VSL_LEN(c->ptr);
+			data = VSL_CDATA(c->ptr);
+			i = VRE_exec(query->regex, data, len, 0, 0, NULL, 0,
+			    NULL);
+			if (i != VRE_ERROR_NOMATCH) {
+				AZ(VSL_ResetCursor(c));
+				return (1);
+			}
+		}
+		AZ(VSL_ResetCursor(c));
+		c = *++cp;
+	}
+
+	return (0);
+}



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 64ad584 Make varnishlog use VSL Query/Dispatch
Message-ID: <E1Ucb5y-0004GX-GU@project.varnish-software.com>

commit 64ad5841387017ba9a4d6adf3f95a191a76588b4
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Apr 30 15:52:44 2013 +0200

    Make varnishlog use VSL Query/Dispatch

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index 6e22d30..c669e70 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -62,33 +62,12 @@ error(int status, const char *fmt, ...)
 	va_start(ap, fmt);
 	vfprintf(stderr, fmt, ap); /* XXX: syslog on daemon */
 	va_end(ap);
+	fprintf(stderr, "\n");
 
 	if (status)
 		exit(status);
 }
 
-static struct VSL_cursor *
-cursor_vsm(struct VSL_data *vsl, struct VSM_data *vsm, int status, int front)
-{
-	struct VSL_cursor *c;
-
-	AN(vsm);
-	if (VSM_Open(vsm)) {
-		if (status)
-			error(status, "VSM: %s", VSM_Error(vsm));
-		VSM_ResetError(vsm);
-		return (NULL);
-	}
-	c = VSL_CursorVSM(vsl, vsm, front);
-	if (c == NULL) {
-		if (status)
-			error(status, "VSL: %s", VSL_Error(vsl));
-		VSL_ResetError(vsl);
-		return (NULL);
-	}
-	return (c);
-}
-
 static void
 usage(void)
 {
@@ -96,90 +75,128 @@ usage(void)
 	exit(1);
 }
 
-static void
-do_raw(struct VSL_data *vsl, struct VSM_data *vsm)
-{
-	struct VSL_cursor *c;
-	int i;
-
-	c = cursor_vsm(vsl, vsm, 1, 1);
-	AN(c);
-
-	i = 0;
-	while (1) {
-		while (c == NULL) {
-			c = cursor_vsm(vsl, vsm, 0, 1);
-			if (c != NULL) {
-				error(0, "Log reopened\n");
-				break;
-			}
-			VTIM_sleep(1.);
-		}
-
-		i = VSL_Next(c);
-		if (i == 1) {
-			/* Got new record */
-			if (VSL_Match(vsl, c))
-				(void)VSL_Print(vsl, c, stdout);
-		} else if (i == 0) {
-			/* Nothing to do but wait */
-			VTIM_sleep(0.01);
-		} else if (i == -1) {
-			/* EOF */
-			break;
-		} else if (i == -2) {
-			/* Abandoned - try reconnect */
-			error(0, "Log abandoned, reopening\n");
-			VSL_DeleteCursor(c);
-			c = NULL;
-			VSM_Close(vsm);
-		} else if (i < -2) {
-			/* Overrun - bail */
-			error(1, "Log overrun\n");
-		}
-	}
-
-	VSL_DeleteCursor(c);
-}
-
 int
 main(int argc, char * const *argv)
 {
+	char optchar;
+	int d_opt = 0;
+	char *g_arg = NULL;
+	char *r_arg = NULL;
+
 	struct VSL_data *vsl;
 	struct VSM_data *vsm;
-	char c;
-	char *r_arg = NULL;
+	struct VSL_cursor *c;
+	struct VSLQ *q;
+	enum VSL_grouping_e grouping = VSL_g_vxid;
+	int i;
 
 	vsl = VSL_New();
 	AN(vsl);
 	vsm = VSM_New();
 	AN(vsm);
 
-	while ((c = getopt(argc, argv, "n:r:")) != -1) {
-		switch (c) {
-		case 'r':
-			r_arg = optarg;
+	while ((optchar = getopt(argc, argv, "dg:n:r:")) != -1) {
+		switch (optchar) {
+		case 'd':
+			d_opt = 1;
+			break;
+		case 'g':
+			/* Grouping mode */
+			g_arg = optarg;
 			break;
 		case 'n':
+			/* Instance name */
 			if (VSM_n_Arg(vsm, optarg) > 0)
 				break;
+		case 'r':
+			/* Read from file */
+			r_arg = optarg;
+			break;
 		default:
 			usage();
 		}
 	}
 
+	if (g_arg) {
+		if (!strcmp(g_arg, "raw"))
+			grouping = VSL_g_raw;
+		else if (!strcmp(g_arg, "vxid"))
+			grouping = VSL_g_vxid;
+		else if (!strcmp(g_arg, "request"))
+			grouping = VSL_g_request;
+		else if (!strcmp(g_arg, "session"))
+			grouping = VSL_g_session;
+		else
+			error(1, "Wrong -g argument: %s", g_arg);
+	}
+
+	/* Create cursor */
 	if (r_arg) {
 		/* XXX */
+		error (1, "-r not implemented");
 	} else {
 		if (VSM_Open(vsm))
-			error(1, VSM_Error(vsm));
+			error(1, "VSM_Open: %s", VSM_Error(vsm));
+		if (!(c = VSL_CursorVSM(vsl, vsm, !d_opt)))
+			error(1, "VSL_CursorVSM: %s", VSL_Error(vsl));
 	}
+	AN(c);
 
-	do_raw(vsl, vsm);
+	/* Create query */
+	q = VSLQ_New(vsl, &c, grouping, argv[optind]);
+	if (q == NULL)
+		error(1, "VSLQ_New: %s", VSL_Error(vsl));
+	AZ(c);
 
+	while (1) {
+		while (!r_arg && q == NULL) {
+			VTIM_sleep(0.1);
+			if (VSM_Open(vsm)) {
+				VSM_ResetError(vsm);
+				continue;
+			}
+			c = VSL_CursorVSM(vsl, vsm, 1);
+			if (c == NULL) {
+				VSL_ResetError(vsl);
+				continue;
+			}
+			q = VSLQ_New(vsl, &c, grouping, argv[optind]);
+			AN(q);
+			AZ(c);
+		}
+
+		i = VSLQ_Dispatch(q, VSL_PrintSet, stdout);
+		if (i == 0) {
+			/* Nothing to do but wait */
+			VTIM_sleep(0.01);
+		} else if (i == -1) {
+			/* EOF */
+			break;
+		} else if (i <= -2) {
+			/* XXX: Make continuation optional */
+			VSLQ_Flush(q, VSL_PrintSet, stdout);
+			VSLQ_Delete(&q);
+			AZ(q);
+			if (i == -2) {
+				/* Abandoned */
+				error(0, "Log abandoned - reopening");
+				VSM_Close(vsm);
+			} else if (i < -2) {
+				/* Overrun */
+				error(0, "Log overrun");
+			}
+		} else {
+			error(1, "Unexpected: %d", i);
+		}
+	}
+
+	if (q != NULL) {
+		VSLQ_Flush(q, VSL_PrintSet, stdout);
+		VSLQ_Delete(&q);
+		AZ(q);
+	}
 	VSL_Delete(vsl);
-	if (vsm)
-		VSM_Delete(vsm);
+	VSM_Delete(vsm);
 
 	exit(0);
 }



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] b4f9d54 Add a VSL_Check function to check a record pointers
 validity.
Message-ID: <E1Ucb5y-0004Gk-Hv@project.varnish-software.com>

commit b4f9d54697eb54a0eae60b106eb7962824bcf97a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 1 17:32:03 2013 +0200

    Add a VSL_Check function to check a record pointers validity.
    
    Introduce a VSLC_ptr type to represent a record pointer.

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 64b24b8..80f9674 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -227,12 +227,12 @@ varnishlog_thread(void *priv)
 		} else if (i != 1)
 			break;
 
-		tag = VSL_TAG(c->ptr);
-		vxid = VSL_ID(c->ptr);
-		len = VSL_LEN(c->ptr);
-		type = VSL_CLIENT(c->ptr) ? 'c' : VSL_BACKEND(c->ptr) ?
+		tag = VSL_TAG(c->rec.ptr);
+		vxid = VSL_ID(c->rec.ptr);
+		len = VSL_LEN(c->rec.ptr);
+		type = VSL_CLIENT(c->rec.ptr) ? 'c' : VSL_BACKEND(c->rec.ptr) ?
 		    'b' : '-';
-		data = VSL_CDATA(c->ptr);
+		data = VSL_CDATA(c->rec.ptr);
 		v->vsl_tag_count[tag]++;
 		vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", vxid,
 		    VSL_tags[tag], type, (int)len, data);
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 023b65f..4bb1d60 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -51,8 +51,14 @@
 struct VSL_data;
 struct VSLQ;
 
-struct VSL_cursor {
+struct VSLC_ptr {
 	const uint32_t		*ptr; /* Record pointer */
+	unsigned		priv;
+};
+
+struct VSL_cursor {
+	/* The record this cursor points to */
+	struct VSLC_ptr		rec;
 
 	/* If not -1, the vxid of all records in this set */
 	int32_t			vxid;
@@ -151,6 +157,18 @@ int VSL_ResetCursor(struct VSL_cursor *c);
 	 *    -1: Operation not supported
 	 */
 
+int VSL_Check(const struct VSL_cursor *c, const struct VSLC_ptr *ptr);
+	/*
+	 * Check if the VSLC_ptr structure points to a value that is still
+	 * valid:
+	 *
+	 * Return values:
+	 *    -1: Operation not supported
+	 *     0: Not valid
+	 *     1: Valid - warning level
+	 *     2: Valid
+	 */
+
 int VSL_Next(struct VSL_cursor *c);
 	/*
 	 * Return raw pointer to next VSL record.
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index c42198a..f65dc9f 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -68,9 +68,9 @@
 struct VSL_head {
 #define VSL_HEAD_MARKER		"VSLHEAD0"	/* Incr. as version# */
 	char			marker[VSM_MARKER_LEN];
-	ssize_t			segments[VSL_SEGMENTS];
-	unsigned		segment;	/* Current varnishd segment */
-	unsigned		seq;		/* Non-zero seq number */
+	volatile ssize_t	segments[VSL_SEGMENTS];
+	volatile unsigned	segment;	/* Current varnishd segment */
+	volatile unsigned	seq;		/* Non-zero seq number */
 	uint32_t		log[];
 };
 
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 994d76d..25e1898 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -140,9 +140,9 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c)
 	enum VSL_tag_e tag;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-	if (c == NULL || c->ptr == NULL)
+	if (c == NULL || c->rec.ptr == NULL)
 		return (0);
-	tag = VSL_TAG(c->ptr);
+	tag = VSL_TAG(c->rec.ptr);
 	if (tag <= SLT__Bogus || tag >= SLT__Reserved)
 		return (0);
 	if (vbit_test(vsl->vbm_select, tag))
@@ -165,15 +165,16 @@ VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-	if (c == NULL || c->ptr == NULL)
+	if (c == NULL || c->rec.ptr == NULL)
 		return (0);
 	if (fo == NULL)
 		fo = stdout;
-	tag = VSL_TAG(c->ptr);
-	vxid = VSL_ID(c->ptr);
-	len = VSL_LEN(c->ptr);
-	type = VSL_CLIENT(c->ptr) ? 'c' : VSL_BACKEND(c->ptr) ? 'b' : '-';
-	data = VSL_CDATA(c->ptr);
+	tag = VSL_TAG(c->rec.ptr);
+	vxid = VSL_ID(c->rec.ptr);
+	len = VSL_LEN(c->rec.ptr);
+	type = VSL_CLIENT(c->rec.ptr) ? 'c' : VSL_BACKEND(c->rec.ptr) ?
+	    'b' : '-';
+	data = VSL_CDATA(c->rec.ptr);
 
 	if (tag == SLT_Debug) {
 		i = fprintf(fo, "%10u %-15s %c \"", vxid, VSL_tags[tag], type);
@@ -211,15 +212,16 @@ VSL_PrintLevel(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-	if (c == NULL || c->ptr == NULL)
+	if (c == NULL || c->rec.ptr == NULL)
 		return (0);
 	if (fo == NULL)
 		fo = stdout;
 
-	tag = VSL_TAG(c->ptr);
-	len = VSL_LEN(c->ptr);
-	type = VSL_CLIENT(c->ptr) ? 'c' : VSL_BACKEND(c->ptr) ? 'b' : '-';
-	data = VSL_CDATA(c->ptr);
+	tag = VSL_TAG(c->rec.ptr);
+	len = VSL_LEN(c->rec.ptr);
+	type = VSL_CLIENT(c->rec.ptr) ? 'c' : VSL_BACKEND(c->rec.ptr) ?
+	    'b' : '-';
+	data = VSL_CDATA(c->rec.ptr);
 	lvl = c->level;
 	if (tag == SLT_Debug) {
 		i = fprintf(fo, "%2u %-15s %c \"", lvl, VSL_tags[tag],
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index a03aef7..039d57b 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -35,6 +35,11 @@
 
 #define VSL_FILE_HEAD			"VSL"
 
+struct vslc_shmptr {
+	uint32_t	*ptr;
+	unsigned	priv;
+};
+
 int vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
     __printflike(2, 3);
 int vsl_skip(struct VSL_cursor *c, ssize_t words);
@@ -43,6 +48,8 @@ typedef void vslc_delete_f(void *);
 typedef int vslc_next_f(void *);
 typedef int vslc_reset_f(void *);
 typedef int vslc_skip_f(void *, ssize_t words);
+typedef int vslc_ref_f(void *, struct vslc_shmptr *ptr);
+typedef int vslc_check_f(const void *, const struct VSLC_ptr *ptr);
 
 struct vslc {
 	struct VSL_cursor		c;
@@ -53,6 +60,7 @@ struct vslc {
 	vslc_next_f			*next;
 	vslc_reset_f			*reset;
 	vslc_skip_f			*skip;
+	vslc_check_f			*check;
 };
 
 struct VSL_data {
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index bf4d1ec..1712ade 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -55,11 +55,10 @@ struct vslc_vsm {
 	struct VSM_data			*vsm;
 	struct VSM_fantom		vf;
 
-	volatile const struct VSL_head	*head;
-	volatile const uint32_t		*next;
+	const struct VSL_head		*head;
 	const uint32_t			*end;
 	ssize_t				segsize;
-	unsigned			seq;
+	struct VSLC_ptr			next;
 };
 
 static void
@@ -72,74 +71,101 @@ vslc_vsm_delete(void *cursor)
 }
 
 static int
-vslc_vsm_next(void *cursor)
+vslc_vsm_check(const void *cursor, const struct VSLC_ptr *ptr)
 {
-	struct vslc_vsm *c;
-	int diff;
-	unsigned segment;
-	uint32_t t;
+	const struct vslc_vsm *c;
+	unsigned seqdiff, segment, segdiff;
 
 	CAST_OBJ_NOTNULL(c, cursor, VSLC_VSM_MAGIC);
-	CHECK_OBJ_NOTNULL(c->vsm, VSM_MAGIC);
 
-	/* Assert pointers */
-	AN(c->next);
-	assert(c->next >= c->head->log);
-	assert(c->next < c->end);
+	if (ptr->ptr == NULL)
+		return (0);
 
 	/* Check sequence number */
-	diff = c->head->seq - c->seq;
-	if (c->head->seq < c->seq)
+	seqdiff = c->head->seq - ptr->priv;
+	if (c->head->seq < ptr->priv)
 		/* Wrap around skips 0 */
-		diff -= 1;
-	if (diff > 1)
-		return (-3);
+		seqdiff -= 1;
+	if (seqdiff > 1)
+		/* Too late */
+		return (0);
 
 	/* Check overrun */
-	segment = (c->next - c->head->log) / c->segsize;
+	segment = (ptr->ptr - c->head->log) / c->segsize;
 	if (segment >= VSL_SEGMENTS)
+		/* Rounding error spills to last segment */
 		segment = VSL_SEGMENTS - 1;
-	diff = (segment - c->head->segment) % VSL_SEGMENTS;
-	if (0 < diff && diff <= 2)
+	segdiff = (segment - c->head->segment) % VSL_SEGMENTS;
+	if (segdiff == 0 && seqdiff == 0)
+		/* In same segment, but close to tail */
+		return (2);
+	if (segdiff <= 2)
+		/* Too close to continue */
+		return (0);
+	if (segdiff <= 4)
+		/* Warning level */
+		return (1);
+	/* Safe */
+	return (2);
+}
+
+static int
+vslc_vsm_next(void *cursor)
+{
+	struct vslc_vsm *c;
+	int i;
+	uint32_t t;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_VSM_MAGIC);
+	CHECK_OBJ_NOTNULL(c->vsm, VSM_MAGIC);
+
+	/* Assert pointers */
+	AN(c->next.ptr);
+	assert(c->next.ptr >= c->head->log);
+	assert(c->next.ptr < c->end);
+
+	i = vslc_vsm_check(c, &c->next);
+	if (i <= 0)
+		/* Overrun */
 		return (-3);
 
 	/* Check VSL fantom and abandonment */
-	if (*c->next == VSL_ENDMARKER) {
+	if (*(volatile const uint32_t *)c->next.ptr == VSL_ENDMARKER) {
 		if (!VSM_StillValid(c->vsm, &c->vf) ||
 		    VSM_Abandoned(c->vsm))
 			return (-2);
 	}
 
 	while (1) {
-		assert(c->next >= c->head->log);
-		assert(c->next < c->end);
+		assert(c->next.ptr >= c->head->log);
+		assert(c->next.ptr < c->end);
 		AN(c->head->seq);
-		t = *c->next;
+		t = *(volatile const uint32_t *)c->next.ptr;
 		AN(t);
 
 		if (t == VSL_WRAPMARKER) {
 			/* Wrap around not possible at front */
-			assert(c->next != c->head->log);
-			c->next = c->head->log;
+			assert(c->next.ptr != c->head->log);
+			c->next.ptr = c->head->log;
 			continue;
 		}
 
 		if (t == VSL_ENDMARKER) {
-			if (c->next != c->head->log &&
-			    c->seq != c->head->seq) {
+			if (c->next.ptr != c->head->log &&
+			    c->next.priv != c->head->seq) {
 				/* ENDMARKER not at front and seq wrapped */
 				/* XXX: assert on this? */
-				c->next = c->head->log;
+				c->next.ptr = c->head->log;
 				continue;
 			}
 			return (0);
 		}
 
-		if (c->next == c->head->log)
-			c->seq = c->head->seq;
+		if (c->next.ptr == c->head->log)
+			c->next.priv = c->head->seq;
 
-		c->c.c.ptr = (void*)(uintptr_t)c->next; /* Loose volatile */
-		c->next = VSL_NEXT(c->next);
+		c->c.c.rec = c->next;
+		c->next.ptr = VSL_NEXT(c->next.ptr);
 		return (1);
 	}
 }
@@ -162,9 +188,9 @@ vslc_vsm_reset(void *cursor)
 	if (c->head->segments[segment] < 0)
 		segment = 0;
 	assert(c->head->segments[segment] >= 0);
-	c->next = c->head->log + c->head->segments[segment];
-	c->seq = c->head->seq;
-	c->c.c.ptr = NULL;
+	c->next.ptr = c->head->log + c->head->segments[segment];
+	c->next.priv = c->head->seq;
+	c->c.c.rec.ptr = NULL;
 
 	return (0);
 }
@@ -178,10 +204,10 @@ vslc_vsm_skip(void *cursor, ssize_t words)
 	if (words < 0)
 		return (-1);
 
-	c->next += words;
-	assert(c->next >= c->head->log);
-	assert(c->next < c->end);
-	c->c.c.ptr = NULL;
+	c->next.ptr += words;
+	assert(c->next.ptr >= c->head->log);
+	assert(c->next.ptr < c->end);
+	c->c.c.rec.ptr = NULL;
 
 	return (0);
 }
@@ -223,6 +249,7 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 	c->c.next = vslc_vsm_next;
 	c->c.reset = vslc_vsm_reset;
 	c->c.skip = vslc_vsm_skip;
+	c->c.check = vslc_vsm_check;
 
 	c->vsm = vsm;
 	c->vf = vf;
@@ -232,10 +259,12 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 
 	if (tail) {
 		/* Locate tail of log */
-		c->next = c->head->log + c->head->segments[c->head->segment];
-		while (c->next < c->end && *c->next != VSL_ENDMARKER)
-			c->next = VSL_NEXT(c->next);
-		c->seq = c->head->seq;
+		c->next.ptr = c->head->log +
+		    c->head->segments[c->head->segment];
+		while (c->next.ptr < c->end &&
+		    *(volatile const uint32_t *)c->next.ptr != VSL_ENDMARKER)
+			c->next.ptr = VSL_NEXT(c->next.ptr);
+		c->next.priv = c->head->seq;
 	} else
 		AZ(vslc_vsm_reset(&c->c));
 
@@ -293,7 +322,7 @@ vslc_file_next(void *cursor)
 		return (c->error);
 
 	do {
-		c->c.c.ptr = NULL;
+		c->c.c.rec.ptr = NULL;
 		assert(c->buflen >= 2 * 4);
 		i = vslc_file_readn(c->fd, c->buf, 2 * 4);
 		if (i < 0)
@@ -313,8 +342,9 @@ vslc_file_next(void *cursor)
 		if (i == 0)
 			return (-1);	/* EOF */
 		assert(i == l - 2 * 4);
-		c->c.c.ptr = c->buf;
-	} while (c->c.c.ptr != NULL && VSL_TAG(c->c.c.ptr) == SLT__Batch);
+		c->c.c.rec.ptr = c->buf;
+	} while (c->c.c.rec.ptr != NULL &&
+	    VSL_TAG(c->c.c.rec.ptr) == SLT__Batch);
 	return (1);
 }
 
@@ -423,3 +453,14 @@ vsl_skip(struct VSL_cursor *cursor, ssize_t words)
 		return (-1);
 	return ((c->skip)(c, words));
 }
+
+int
+VSL_Check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
+{
+	const struct vslc *c;
+
+	CAST_OBJ_NOTNULL(c, (const void *)cursor, VSLC_MAGIC);
+	if (c->check == NULL)
+		return (-1);
+	return ((c->check)(c, ptr));
+}
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 61b0518..c5389b9 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -181,7 +181,7 @@ vslc_raw_next(void *cursor)
 	assert(c->next >= c->start);
 	assert(c->next <= c->start + c->len);
 	if (c->next < c->start + c->len) {
-		c->c.c.ptr = c->next;
+		c->c.c.rec.ptr = c->next;
 		c->next = VSL_NEXT(c->next);
 		return (1);
 	}
@@ -198,7 +198,7 @@ vslc_raw_reset(void *cursor)
 	assert(c->next >= c->start);
 	assert(c->next <= c->start + c->len);
 	c->next = c->start;
-	c->c.c.ptr = NULL;
+	c->c.c.rec.ptr = NULL;
 
 	return (0);
 }
@@ -213,7 +213,7 @@ vslc_vtx_next(void *cursor)
 	assert(c->next >= c->vtx->start);
 	assert(c->next <= c->vtx->start + c->vtx->len);
 	if (c->next < c->vtx->start + c->vtx->len) {
-		c->c.c.ptr = c->next;
+		c->c.c.rec.ptr = c->next;
 		c->next = VSL_NEXT(c->next);
 		return (1);
 	}
@@ -230,7 +230,7 @@ vslc_vtx_reset(void *cursor)
 	assert(c->next >= c->vtx->start);
 	assert(c->next <= c->vtx->start + c->vtx->len);
 	c->next = c->vtx->start;
-	c->c.c.ptr = NULL;
+	c->c.c.rec.ptr = NULL;
 
 	return (0);
 }
@@ -790,13 +790,13 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		i = VSL_Next(c);
 		if (i <= 0)
 			break;
-		AN(c->ptr);
+		AN(c->rec.ptr);
 		if (func == NULL)
 			continue;
-		rawc.start = c->ptr;
-		rawc.len = VSL_NEXT(c->ptr) - c->ptr;
+		rawc.start = c->rec.ptr;
+		rawc.len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
 		rawc.next = rawc.start;
-		rawc.c.c.ptr = NULL;
+		rawc.c.c.rec.ptr = NULL;
 
 		/* Query check goes here */
 		i = 0;
@@ -831,13 +831,13 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		i = VSL_Next(c);
 		if (i != 1)
 			break;
-		tag = VSL_TAG(c->ptr);
+		tag = VSL_TAG(c->rec.ptr);
 		if (tag == SLT__Batch) {
-			ptr = VSL_NEXT(c->ptr);
-			len = VSL_WORDS(c->ptr[1]);
+			ptr = VSL_NEXT(c->rec.ptr);
+			len = VSL_WORDS(c->rec.ptr[1]);
 			AZ(vsl_skip(c, len));
 		} else {
-			ptr = c->ptr;
+			ptr = c->rec.ptr;
 			len = VSL_NEXT(ptr) - ptr;
 		}
 		vxid = VSL_ID(ptr);
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index b4a403c..852f8d3 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -104,9 +104,9 @@ vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[])
 			if (i == 0)
 				break;
 			assert(i == 1);
-			AN(c->ptr);
-			len = VSL_LEN(c->ptr);
-			data = VSL_CDATA(c->ptr);
+			AN(c->rec.ptr);
+			len = VSL_LEN(c->rec.ptr);
+			data = VSL_CDATA(c->rec.ptr);
 			i = VRE_exec(query->regex, data, len, 0, 0, NULL, 0,
 			    NULL);
 			if (i != VRE_ERROR_NOMATCH) {



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 6361987 Use a static jump table for the cursors.
Message-ID: <E1Ucb5y-0004Gt-JG@project.varnish-software.com>

commit 636198778809de2098cfe7357a055618d70d4e79
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 1 18:05:07 2013 +0200

    Use a static jump table for the cursors.

diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 039d57b..eb9e543 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -51,11 +51,7 @@ typedef int vslc_skip_f(void *, ssize_t words);
 typedef int vslc_ref_f(void *, struct vslc_shmptr *ptr);
 typedef int vslc_check_f(const void *, const struct VSLC_ptr *ptr);
 
-struct vslc {
-	struct VSL_cursor		c;
-	unsigned			magic;
-#define VSLC_MAGIC			0x5007C0DE
-
+struct vslc_tbl {
 	vslc_delete_f			*delete;
 	vslc_next_f			*next;
 	vslc_reset_f			*reset;
@@ -63,6 +59,14 @@ struct vslc {
 	vslc_check_f			*check;
 };
 
+struct vslc {
+	struct VSL_cursor		c;
+	unsigned			magic;
+#define VSLC_MAGIC			0x5007C0DE
+
+	const struct vslc_tbl		*tbl;
+};
+
 struct VSL_data {
 	unsigned			magic;
 #undef VSL_MAGIC
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 1712ade..f4a7738 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -212,6 +212,14 @@ vslc_vsm_skip(void *cursor, ssize_t words)
 	return (0);
 }
 
+static struct vslc_tbl vslc_vsm_tbl = {
+	.delete		= vslc_vsm_delete,
+	.next		= vslc_vsm_next,
+	.reset		= vslc_vsm_reset,
+	.skip		= vslc_vsm_skip,
+	.check		= vslc_vsm_check,
+};
+
 struct VSL_cursor *
 VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 {
@@ -245,11 +253,7 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 	c->c.c.vxid = -1;	/* N/A to this cursor type */
 	c->c.c.shmptr_ok = 1;
 	c->c.magic = VSLC_MAGIC;
-	c->c.delete = vslc_vsm_delete;
-	c->c.next = vslc_vsm_next;
-	c->c.reset = vslc_vsm_reset;
-	c->c.skip = vslc_vsm_skip;
-	c->c.check = vslc_vsm_check;
+	c->c.tbl = & vslc_vsm_tbl;
 
 	c->vsm = vsm;
 	c->vf = vf;
@@ -356,6 +360,14 @@ vslc_file_reset(void *cursor)
 	return (-1);
 }
 
+static struct vslc_tbl vslc_file_tbl = {
+	.delete		= vslc_file_delete,
+	.next		= vslc_file_next,
+	.reset		= vslc_file_reset,
+	.skip		= NULL,
+	.check		= NULL,
+};
+
 struct VSL_cursor *
 VSL_CursorFile(struct VSL_data *vsl, const char *name)
 {
@@ -399,9 +411,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 	}
 	c->c.c.vxid = -1;	/* N/A to this cursor type */
 	c->c.magic = VSLC_MAGIC;
-	c->c.delete = vslc_file_delete;
-	c->c.next = vslc_file_next;
-	c->c.reset = vslc_file_reset;
+	c->c.tbl = &vslc_file_tbl;
 
 	c->fd = fd;
 	c->buflen = BUFSIZ;
@@ -417,9 +427,9 @@ VSL_DeleteCursor(struct VSL_cursor *cursor)
 	struct vslc *c;
 
 	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
-	if (c->delete == NULL)
+	if (c->tbl->delete == NULL)
 		return;
-	(c->delete)(c);
+	(c->tbl->delete)(c);
 }
 
 int
@@ -428,9 +438,9 @@ VSL_ResetCursor(struct VSL_cursor *cursor)
 	struct vslc *c;
 
 	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
-	if (c->reset == NULL)
+	if (c->tbl->reset == NULL)
 		return (-1);
-	return ((c->reset)(c));
+	return ((c->tbl->reset)(c));
 }
 
 int
@@ -439,8 +449,8 @@ VSL_Next(struct VSL_cursor *cursor)
 	struct vslc *c;
 
 	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
-	AN(c->next);
-	return ((c->next)(c));
+	AN(c->tbl->next);
+	return ((c->tbl->next)(c));
 }
 
 int
@@ -449,9 +459,9 @@ vsl_skip(struct VSL_cursor *cursor, ssize_t words)
 	struct vslc *c;
 
 	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
-	if (c->skip == NULL)
+	if (c->tbl->skip == NULL)
 		return (-1);
-	return ((c->skip)(c, words));
+	return ((c->tbl->skip)(c, words));
 }
 
 int
@@ -460,7 +470,7 @@ VSL_Check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 	const struct vslc *c;
 
 	CAST_OBJ_NOTNULL(c, (const void *)cursor, VSLC_MAGIC);
-	if (c->check == NULL)
+	if (c->tbl->check == NULL)
 		return (-1);
-	return ((c->check)(c, ptr));
+	return ((c->tbl->check)(c, ptr));
 }
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index c5389b9..265e1e5 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -203,6 +203,14 @@ vslc_raw_reset(void *cursor)
 	return (0);
 }
 
+static struct vslc_tbl vslc_raw_tbl = {
+	.delete	= NULL,
+	.next	= vslc_raw_next,
+	.reset	= vslc_raw_reset,
+	.skip	= NULL,
+	.check	= NULL,
+};
+
 static int
 vslc_vtx_next(void *cursor)
 {
@@ -235,6 +243,14 @@ vslc_vtx_reset(void *cursor)
 	return (0);
 }
 
+static struct vslc_tbl vslc_vtx_tbl = {
+	.delete	= NULL,
+	.next	= vslc_vtx_next,
+	.reset	= vslc_vtx_reset,
+	.skip	= NULL,
+	.check	= NULL,
+};
+
 static void
 vslc_vtx_setup(struct vslc_vtx *c, struct vtx *vtx, unsigned level)
 {
@@ -245,8 +261,7 @@ vslc_vtx_setup(struct vslc_vtx *c, struct vtx *vtx, unsigned level)
 	c->c.c.vxid = vtx->key.vxid;
 	c->c.c.level = level;
 	c->c.magic = VSLC_MAGIC;
-	c->c.next = vslc_vtx_next;
-	c->c.reset = vslc_vtx_reset;
+	c->c.tbl = &vslc_vtx_tbl;
 
 	c->magic = VSLC_VTX_MAGIC;
 	c->vtx = vtx;
@@ -780,8 +795,7 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	memset(&rawc, 0, sizeof rawc);
 	rawc.c.c.vxid = -1;
 	rawc.c.magic = VSLC_MAGIC;
-	rawc.c.next = vslc_raw_next;
-	rawc.c.reset = vslc_raw_reset;
+	rawc.c.tbl = &vslc_raw_tbl;
 	rawc.magic = VSLC_RAW_MAGIC;
 	pc[0] = &rawc.c.c;
 	pc[1] = NULL;



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] cef9da1 Make Dispatch use VSL_Check to handle SHM ptrs.
Message-ID: <E1Ucb5y-0004H6-LD@project.varnish-software.com>

commit cef9da177a5445d04306cbde5a8756734f8ab8cc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun May 12 23:55:39 2013 +0200

    Make Dispatch use VSL_Check to handle SHM ptrs.
    
    This allows Dispatch to run completely in SHM with no copying for most
    cases, also when transactions come in multiple batches.

diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index f65dc9f..8765b84 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -89,6 +89,8 @@ struct VSL_head {
 #define VSL_BACKEND(ptr)	(((ptr)[1]) & VSL_BACKENDMARKER)
 #define VSL_DATA(ptr)		((char*)((ptr)+2))
 #define VSL_CDATA(ptr)		((const char*)((ptr)+2))
+#define VSL_BATCHLEN(ptr)	((ptr)[1])
+#define VSL_BATCHID(ptr)	(VSL_ID((ptr) + 2))
 
 #define VSL_ENDMARKER	(((uint32_t)SLT__Reserved << 24) | 0x454545) /* "EEE" */
 #define VSL_WRAPMARKER	(((uint32_t)SLT__Reserved << 24) | 0x575757) /* "WWW" */
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 265e1e5..8c3bfe2 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -46,6 +46,9 @@
 
 #define VTX_CACHE 10
 #define VTX_BUFSIZE_MIN 64
+#define VTX_CHUNKS 3
+
+struct vtx;
 
 enum vtx_type_e {
 	vtx_t_unknown,
@@ -62,6 +65,33 @@ enum vtx_link_e {
 	vtx_l_bereq,
 };
 
+struct vslc_raw {
+	struct vslc		c;
+	unsigned		magic;
+#define VSLC_RAW_MAGIC		0x247EBD44
+
+	const uint32_t		*start;
+	ssize_t			len;
+	const uint32_t		*next;
+};
+
+struct vtx_chunk {
+	struct VSLC_ptr		start;
+	ssize_t			len;
+	ssize_t			offset;
+};
+
+struct vslc_vtx {
+	struct vslc		c;
+	unsigned		magic;
+#define VSLC_VTX_MAGIC		0x74C6523F
+
+	struct vtx		*vtx;
+
+	unsigned		chunk;	/* Current chunk */
+	ssize_t			offset; /* Offset of next record */
+};
+
 struct vtx_key {
 	unsigned		vxid;
 	VRB_ENTRY(vtx_key)	entry;
@@ -74,12 +104,12 @@ struct vtx {
 #define VTX_MAGIC		0xACC21D09
 	VTAILQ_ENTRY(vtx)	list_child;
 	VTAILQ_ENTRY(vtx)	list_incomplete;
+	VTAILQ_ENTRY(vtx)	list_shm;
 
 	double			t_start;
 	unsigned		flags;
-#define VTX_F_SHM		0x1
-#define VTX_F_COMPLETE		0x2
-#define VTX_F_READY		0x4
+#define VTX_F_COMPLETE		0x1
+#define VTX_F_READY		0x2
 
 	enum vtx_type_e		type;
 
@@ -89,33 +119,16 @@ struct vtx {
 	unsigned		n_childready;
 	unsigned		n_descend;
 
-	const uint32_t		*start;
+	struct vslc_vtx		c;
 	ssize_t			len;
-	unsigned		index;
+
+	struct vtx_chunk	chunk[VTX_CHUNKS];
+	unsigned		n_chunk;
 
 	uint32_t		*buf;
 	ssize_t			bufsize;
 };
 
-struct vslc_raw {
-	struct vslc		c;
-	unsigned		magic;
-#define VSLC_RAW_MAGIC		0x247EBD44
-
-	const uint32_t		*start;
-	ssize_t			len;
-	const uint32_t		*next;
-};
-
-struct vslc_vtx {
-	struct vslc		c;
-	unsigned		magic;
-#define VSLC_VTX_MAGIC		0x74C6523F
-
-	struct vtx		*vtx;
-	const uint32_t		*next;
-};
-
 struct VSLQ {
 	unsigned		magic;
 #define VSLQ_MAGIC		0x23A8BE97
@@ -131,6 +144,8 @@ struct VSLQ {
 	VTAILQ_HEAD(,vtx)	incomplete;
 	unsigned		n_incomplete;
 
+	VTAILQ_HEAD(,vtx)	shmlist;
+
 	VTAILQ_HEAD(,vtx)	cache;
 	unsigned		n_cache;
 };
@@ -215,17 +230,37 @@ static int
 vslc_vtx_next(void *cursor)
 {
 	struct vslc_vtx *c;
+	struct vtx_chunk *chunk;
 
 	CAST_OBJ_NOTNULL(c, cursor, VSLC_VTX_MAGIC);
 
-	assert(c->next >= c->vtx->start);
-	assert(c->next <= c->vtx->start + c->vtx->len);
-	if (c->next < c->vtx->start + c->vtx->len) {
-		c->c.c.rec.ptr = c->next;
-		c->next = VSL_NEXT(c->next);
+	assert (c->offset <= c->vtx->len);
+	if (c->offset == c->vtx->len)
+		return (0);
+
+	if (c->vtx->n_chunk == 0) {
+		/* Buffer */
+		AN(c->vtx->buf);
+		assert(c->offset < c->vtx->bufsize);
+		c->c.c.rec.ptr = c->vtx->buf + c->offset;
+		c->offset += VSL_NEXT(c->c.c.rec.ptr) - c->c.c.rec.ptr;
 		return (1);
 	}
-	return (0);
+
+	/* Shmptr chunks */
+	assert(c->chunk < c->vtx->n_chunk);
+	chunk = &c->vtx->chunk[c->chunk];
+	assert(c->offset >= chunk->offset);
+	assert(c->offset <= chunk->offset + chunk->len);
+	if (c->offset == chunk->offset + chunk->len) {
+		c->chunk++;
+		chunk = &c->vtx->chunk[c->chunk];
+	}
+	AN(chunk->start.ptr);
+	c->c.c.rec.ptr = chunk->start.ptr + c->offset - chunk->offset;
+	c->offset += VSL_NEXT(c->c.c.rec.ptr) - c->c.c.rec.ptr;
+
+	return (1);
 }
 
 static int
@@ -234,10 +269,8 @@ vslc_vtx_reset(void *cursor)
 	struct vslc_vtx *c;
 
 	CAST_OBJ_NOTNULL(c, cursor, VSLC_VTX_MAGIC);
-
-	assert(c->next >= c->vtx->start);
-	assert(c->next <= c->vtx->start + c->vtx->len);
-	c->next = c->vtx->start;
+	c->chunk = 0;
+	c->offset = 0;
 	c->c.c.rec.ptr = NULL;
 
 	return (0);
@@ -251,23 +284,6 @@ static struct vslc_tbl vslc_vtx_tbl = {
 	.check	= NULL,
 };
 
-static void
-vslc_vtx_setup(struct vslc_vtx *c, struct vtx *vtx, unsigned level)
-{
-	AN(c);
-	AN(vtx);
-
-	memset(c, 0, sizeof *c);
-	c->c.c.vxid = vtx->key.vxid;
-	c->c.c.level = level;
-	c->c.magic = VSLC_MAGIC;
-	c->c.tbl = &vslc_vtx_tbl;
-
-	c->magic = VSLC_VTX_MAGIC;
-	c->vtx = vtx;
-	c->next = c->vtx->start;
-}
-
 static struct vtx *
 vtx_new(struct VSLQ *vslq)
 {
@@ -282,6 +298,10 @@ vtx_new(struct VSLQ *vslq)
 	} else {
 		ALLOC_OBJ(vtx, VTX_MAGIC);
 		AN(vtx);
+		vtx->c.c.magic = VSLC_MAGIC;
+		vtx->c.c.tbl = &vslc_vtx_tbl;
+		vtx->c.magic = VSLC_VTX_MAGIC;
+		vtx->c.vtx = vtx;
 	}
 
 	vtx->key.vxid = 0;
@@ -293,9 +313,10 @@ vtx_new(struct VSLQ *vslq)
 	vtx->n_child = 0;
 	vtx->n_childready = 0;
 	vtx->n_descend = 0;
-	vtx->start = vtx->buf;
+	(void)vslc_vtx_reset(&vtx->c);
 	vtx->len = 0;
-	vtx->index = 0;
+	memset(&vtx->chunk, 0, sizeof vtx->chunk);
+	vtx->n_chunk = 0;
 
 	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_incomplete);
 	vslq->n_incomplete++;
@@ -349,6 +370,9 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 	AZ(vtx->n_descend);
 	AN(VRB_REMOVE(vtx_tree, &vslq->tree, &vtx->key));
 
+	if (vtx->n_chunk)
+		VTAILQ_REMOVE(&vslq->shmlist, vtx, list_shm);
+
 	if (vslq->n_cache < VTX_CACHE) {
 		VTAILQ_INSERT_HEAD(&vslq->cache, vtx, list_child);
 		vslq->n_cache++;
@@ -375,52 +399,81 @@ vtx_lori(struct VSLQ *vslq, unsigned vxid)
 	vtx = vtx_new(vslq);
 	AN(vtx);
 	vtx->key.vxid = vxid;
+	vtx->c.c.c.vxid = vxid;
 	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
 	return (vtx);
 }
 
 static void
-vtx_append(struct vtx *vtx, const uint32_t *ptr, ssize_t len, int shmptr_ok)
+vtx_set_bufsize(struct vtx *vtx, ssize_t len)
 {
-	ssize_t bufsize;
-	const uint32_t *ptr2;
-	ssize_t len2;
 
 	AN(vtx);
+	assert(len >= 0);
+	if (vtx->bufsize >= len)
+		return;
+	if (vtx->bufsize == 0)
+		vtx->bufsize = VTX_BUFSIZE_MIN;
+	while (vtx->bufsize < len)
+		vtx->bufsize *= 2;
+	vtx->buf = realloc(vtx->buf, sizeof (uint32_t) * vtx->bufsize);
+}
 
-	if (vtx->flags & VTX_F_SHM) {
-		assert(vtx->start != vtx->buf);
-		ptr2 = vtx->start;
-		vtx->start = vtx->buf;
-		len2 = vtx->len;
-		vtx->len = 0;
-		vtx->flags &= ~VTX_F_SHM;
-		vtx_append(vtx, ptr2, len2, 0);
-	}
+static void
+vtx_buffer(struct VSLQ *vslq, struct vtx *vtx)
+{
+	int i;
+
+	AN(vtx->n_chunk);
+	AN(vtx->len);
+
+	vtx_set_bufsize(vtx, vtx->len);
+	AN(vtx->buf);
+	assert(vtx->bufsize >= vtx->len);
+
+	for (i = 0; i < vtx->n_chunk; i++)
+		memcpy(vtx->buf + vtx->chunk[i].offset, vtx->chunk[i].start.ptr,
+		    sizeof (uint32_t) * vtx->chunk[i].len);
+
+	memset(&vtx->chunk, 0, sizeof vtx->chunk);
+	VTAILQ_REMOVE(&vslq->shmlist, vtx, list_shm);
+	vtx->n_chunk = 0;
+}
 
+static void
+vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start,
+    ssize_t len, int copy)
+{
+
+	AN(vtx);
 	if (len == 0)
 		return;
-	AN(ptr);
-
-	if (shmptr_ok && vtx->len == 0) {
-		vtx->start = ptr;
-		vtx->len = len;
-		vtx->flags |= VTX_F_SHM;
+	AN(start);
+
+	if (vtx->len > 0 && vtx->n_chunk == 0)
+		/* Can't mix buffer and shmptr */
+		copy = 1;
+
+	if (!copy && vtx->n_chunk < VTX_CHUNKS) {
+		/* Add shmptr chunk */
+		AZ(vtx->chunk[vtx->n_chunk].len);
+		vtx->chunk[vtx->n_chunk].start = *start;
+		vtx->chunk[vtx->n_chunk].len = len;
+		vtx->chunk[vtx->n_chunk].offset = vtx->len;
+		vtx->len += len;
+		if (vtx->n_chunk == 0)
+			VTAILQ_INSERT_TAIL(&vslq->shmlist, vtx, list_shm);
+		vtx->n_chunk++;
 		return;
 	}
 
-	bufsize = vtx->bufsize;
-	if (bufsize == 0)
-		bufsize = VTX_BUFSIZE_MIN;
-	while (vtx->len + len > bufsize)
-		bufsize *= 2;
-	if (bufsize != vtx->bufsize) {
-		vtx->buf = realloc(vtx->buf, 4 * bufsize);
-		AN(vtx->buf);
-		vtx->bufsize = bufsize;
-		vtx->start = vtx->buf;
-	}
-	memcpy(&vtx->buf[vtx->len], ptr, 4 * len);
+	/* Append to buffer */
+	vtx_set_bufsize(vtx, vtx->len + len);
+	if (vtx->n_chunk)
+		vtx_buffer(vslq, vtx);
+	AZ(vtx->n_chunk);
+	AN(vtx->buf);
+	memcpy(vtx->buf + vtx->len, start->ptr, sizeof (uint32_t) * len);
 	vtx->len += len;
 }
 
@@ -454,13 +507,6 @@ vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
 		/* Top level vtx ready */
 		return (ready);
 
-	if (vtx->flags & VTX_F_SHM) {
-		/* Not ready, append zero to make sure it's not a shm
-		   reference */
-		vtx_append(vtx, NULL, 0, 0);
-		AZ(vtx->flags & VTX_F_SHM);
-	}
-
 	return (NULL);
 }
 
@@ -518,7 +564,7 @@ static int
 vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 {
 	int i;
-	enum vtx_type_e p_type;
+	enum vtx_type_e type;
 	unsigned p_vxid;
 	struct vtx *p_vtx;
 
@@ -527,25 +573,31 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	if (vtx->flags & VTX_F_READY)
 		return (vtx_diag_tag(vtx, ptr, "link too late"));
 
-	i = vtx_parsetag_bl(VSL_CDATA(ptr), VSL_LEN(ptr), &p_type, &p_vxid);
+	i = vtx_parsetag_bl(VSL_CDATA(ptr), VSL_LEN(ptr), &type, &p_vxid);
 	if (i < 1)
 		return (vtx_diag_tag(vtx, ptr, "parse error"));
 
 	/* Check/set vtx type */
-	assert(p_type != vtx_t_unknown);
-	if (vtx->type != vtx_t_unknown && vtx->type != p_type)
+	assert(type != vtx_t_unknown);
+	if (vtx->type != vtx_t_unknown && vtx->type != type)
 		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
-	vtx->type = p_type;
+	vtx->type = type;
 
 	if (i == 1 || p_vxid == 0)
 		return (0);
 
+	if (vslq->grouping == VSL_g_vxid)
+		return (0);	/* No links */
+	if (vslq->grouping == VSL_g_request && vtx->type == vtx_t_req)
+		return (0);	/* No links */
+
 	/* Lookup and check parent vtx */
 	p_vtx = vtx_lori(vslq, p_vxid);
 	AN(p_vtx);
 	if (vtx->parent == p_vtx)
 		/* Link already exists */
 		return (0);
+
 	if (vtx->parent != NULL)
 		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
 	if (p_vtx->flags & VTX_F_READY)
@@ -574,6 +626,11 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		return (vtx_diag_tag(vtx, ptr, "parse error"));
 	assert(i == 2);
 
+	if (vslq->grouping == VSL_g_vxid)
+		return (0);	/* No links */
+	if (vslq->grouping == VSL_g_request && vtx->type == vtx_t_sess)
+		return (0);	/* No links */
+
 	/* Lookup and check child vtx */
 	c_vtx = vtx_lori(vslq, c_vxid);
 	AN(c_vtx);
@@ -584,7 +641,6 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
 	if (c_vtx->flags & VTX_F_READY)
 		return (vtx_diag_tag(vtx, ptr, "link too late"));
-	assert(c_type != vtx_t_unknown);
 	if (c_vtx->type != vtx_t_unknown && c_vtx->type != c_type)
 		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
 	c_vtx->type = c_type;
@@ -599,16 +655,17 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 {
 	const uint32_t *ptr;
 	enum VSL_tag_e tag;
-	int complete;
+	struct vtx *ret = NULL;
 
-	complete = (vtx->flags & VTX_F_COMPLETE ? 1 : 0);
-	ptr = vtx->start + vtx->index;
-	assert(ptr <= vtx->start + vtx->len);
-	for (; ptr < vtx->start + vtx->len; ptr = VSL_NEXT(ptr)) {
+	while (vslc_vtx_next(&vtx->c) == 1) {
+		ptr = vtx->c.c.c.rec.ptr;
 		tag = VSL_TAG(ptr);
 
-		if (complete) {
-			vtx_diag(vtx, "late log rec");
+		if (tag == SLT__Batch)
+			continue;
+
+		if (vtx->flags & VTX_F_COMPLETE) {
+			vtx_diag_tag(vtx, ptr, "late log rec");
 			continue;
 		}
 
@@ -625,31 +682,21 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 			break;
 
 		case SLT_End:
-			complete = 1;
+			AZ(vtx->flags & VTX_F_COMPLETE);
+			AZ(ret);
+			VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
+			vtx->flags |= VTX_F_COMPLETE;
+			AN(vslq->n_incomplete);
+			vslq->n_incomplete--;
+			ret = vtx_check_ready(vslq, vtx);
 			break;
 
 		default:
 			break;
 		}
 	}
-	vtx->index = ptr - vtx->start;
-	assert(vtx->index <= vtx->len);
-
-	if (!complete && vtx->flags & VTX_F_SHM) {
-		/* Append zero to make sure it's not a shm reference */
-		vtx_append(vtx, NULL, 0, 0);
-		AZ(vtx->flags & VTX_F_SHM);
-	}
 
-	if (complete) {
-		VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
-		vtx->flags |= VTX_F_COMPLETE;
-		AN(vslq->n_incomplete);
-		vslq->n_incomplete--;
-		return (vtx_check_ready(vslq, vtx));
-	}
-
-	return (NULL);
+	return (ret);
 }
 
 static struct vtx *
@@ -672,9 +719,9 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
     void *priv)
 {
 	unsigned n = vtx->n_descend + 1;
+	struct vslc_vtx *cp[n];
+	struct VSL_cursor *CP[n + 1];
 	unsigned i, j;
-	struct vslc_vtx c[n];
-	struct VSL_cursor *cp[n + 1];
 
 	AN(vslq);
 	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
@@ -689,13 +736,17 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 		return (0);
 
 	i = j = 0;
-	vslc_vtx_setup(&c[i], vtx, 0);
+	cp[i] = &vtx->c;
+	vslc_vtx_reset(cp[i]);
+	cp[i]->c.c.level = 0;
 	i++;
 	while (j < i) {
-		vtx = VTAILQ_FIRST(&c[j].vtx->child);
+		vtx = VTAILQ_FIRST(&cp[j]->vtx->child);
 		while (vtx) {
 			assert(i < n);
-			vslc_vtx_setup(&c[i], vtx, c[j].c.c.level + 1);
+			cp[i] = &vtx->c;
+			vslc_vtx_reset(cp[i]);
+			cp[i]->c.c.level = cp[j]->c.c.level + 1;
 			i++;
 			vtx = VTAILQ_NEXT(vtx, list_child);
 		}
@@ -705,12 +756,12 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 
 	/* Reverse order */
 	for (i = 0; i < n; i++)
-		cp[i] = &c[n - i - 1].c.c;
-	cp[i] = NULL;
+		CP[i] = &cp[n - i - 1]->c.c;
+	CP[i] = NULL;
 
 	/* Query test goes here */
-	if (vslq->query == NULL ? 1 : vslq_runquery(vslq->query, cp))
-		return ((func)(vslq->vsl, cp, priv));
+	if (vslq->query == NULL ? 1 : vslq_runquery(vslq->query, CP))
+		return ((func)(vslq->vsl, CP, priv));
 	else
 		return (0);
 }
@@ -744,6 +795,7 @@ VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp,
 	vslq->query = query;
 	VRB_INIT(&vslq->tree);
 	VTAILQ_INIT(&vslq->incomplete);
+	VTAILQ_INIT(&vslq->shmlist);
 	VTAILQ_INIT(&vslq->cache);
 
 	return (vslq);
@@ -829,7 +881,6 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	struct VSL_cursor *c;
 	int i;
 	enum VSL_tag_e tag;
-	const uint32_t *ptr;
 	ssize_t len;
 	unsigned vxid;
 	struct vtx *vtx;
@@ -842,24 +893,38 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 
 	c = vslq->c;
 	while (1) {
+		/* Check shmlist and buffer on warning */
+		while ((vtx = VTAILQ_FIRST(&vslq->shmlist))) {
+			AN(vtx->n_chunk);
+			i = VSL_Check(c, &vtx->chunk[0].start);
+			if (i == 2)
+				break;
+			else if (i == 1)
+				vtx_buffer(vslq, vtx);
+			else
+				/* Too late */
+				return (-3);
+		}
+
 		i = VSL_Next(c);
 		if (i != 1)
-			break;
+			return (i);
 		tag = VSL_TAG(c->rec.ptr);
 		if (tag == SLT__Batch) {
-			ptr = VSL_NEXT(c->rec.ptr);
-			len = VSL_WORDS(c->rec.ptr[1]);
-			AZ(vsl_skip(c, len));
+			vxid = VSL_BATCHID(c->rec.ptr);
+			len = VSL_END(c->rec.ptr, VSL_BATCHLEN(c->rec.ptr)) -
+			    c->rec.ptr;
 		} else {
-			ptr = c->rec.ptr;
-			len = VSL_NEXT(ptr) - ptr;
+			vxid = VSL_ID(c->rec.ptr);
+			len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
 		}
-		vxid = VSL_ID(ptr);
 		if (vxid == 0)
 			continue;
 		vtx = vtx_lori(vslq, vxid);
 		AN(vtx);
-		vtx_append(vtx, ptr, len, c->shmptr_ok);
+		vtx_append(vslq, vtx, &c->rec, len, VSL_Check(c, &c->rec) != 2);
+		if (tag == SLT__Batch)
+			AZ(vsl_skip(c, VSL_WORDS(VSL_BATCHLEN(c->rec.ptr))));
 		vtx = vtx_scan(vslq, vtx);
 		if (vtx) {
 			AN(vtx->flags & VTX_F_READY);
@@ -867,7 +932,7 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 			vtx_retire(vslq, &vtx);
 			AZ(vtx);
 			if (i)
-				break;
+				return (i);
 		}
 	}
 	if (i)



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] f2e1f42 Drop the VSL_{Main|Mgt|Iter}Valid functions.
Message-ID: <E1Ucb5y-0004HI-Md@project.varnish-software.com>

commit f2e1f42f6ddb68936785d9ddb7c608bd572fee21
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 13 14:11:35 2013 +0200

    Drop the VSL_{Main|Mgt|Iter}Valid functions.
    
    Let the functions returning counters take an optional fantom pointer,
    that can subsequently be used with VSM_StillValid to check for
    changes.
    
    Improve the VSC api documentation.
    
    Add missing VSM_StillValid funciton to the .map file.

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index fcd9bc3..894710c 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -82,7 +82,7 @@ do_xml(struct VSM_data *vd)
 	now = time(NULL);
 	(void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now));
 	printf("<varnishstat timestamp=\"%s\">\n", time_stamp);
-	(void)VSC_Iter(vd, do_xml_cb, NULL);
+	(void)VSC_Iter(vd, NULL, do_xml_cb, NULL);
 	printf("</varnishstat>\n");
 }
 
@@ -143,7 +143,7 @@ do_json(struct VSM_data *vd)
 
 	(void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now));
 	printf("\t\"timestamp\": \"%s\",\n", time_stamp);
-	(void)VSC_Iter(vd, do_json_cb, &jp);
+	(void)VSC_Iter(vd, NULL, do_json_cb, &jp);
 	printf("\n}\n");
 	fflush(stdout);
 }
@@ -196,7 +196,7 @@ do_once(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main)
 		op.up = VSC_C_main->uptime;
 	op.pad = 18;
 
-	(void)VSC_Iter(vd, do_once_cb, &op);
+	(void)VSC_Iter(vd, NULL, do_once_cb, &op);
 }
 
 /*--------------------------------------------------------------------*/
@@ -228,7 +228,7 @@ list_fields(struct VSM_data *vd)
 	fprintf(stderr, "Field name                     Description\n");
 	fprintf(stderr, "----------                     -----------\n");
 
-	(void)VSC_Iter(vd, do_list_cb, NULL);
+	(void)VSC_Iter(vd, NULL, do_list_cb, NULL);
 }
 
 /*--------------------------------------------------------------------*/
@@ -315,7 +315,7 @@ main(int argc, char * const *argv)
 		else if (json)
 			do_json(vd);
 		else if (once)
-			do_once(vd, VSC_Main(vd));
+			do_once(vd, VSC_Main(vd, NULL));
 		else {
 			assert(0);
 		}
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d81c5bc..89b5812 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -318,7 +318,7 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 }
 
 static void
-build_pt_list(struct VSM_data *vd)
+build_pt_list(struct VSM_data *vd, struct VSM_fantom *fantom)
 {
 	struct pt_priv pt_priv;
 	int i;
@@ -334,7 +334,7 @@ build_pt_list(struct VSM_data *vd)
 	VTAILQ_INIT(&pt_priv.ptlist);
 	pt_priv.n_ptlist = 0;
 
-	(void)VSC_Iter(vd, build_pt_list_cb, &pt_priv);
+	(void)VSC_Iter(vd, fantom, build_pt_list_cb, &pt_priv);
 	delete_pt_list();
 	AN(VTAILQ_EMPTY(&ptlist));
 	AZ(n_ptlist);
@@ -841,6 +841,7 @@ do_curses(struct VSM_data *vd, int delay)
 	long timeout;
 	int ch;
 	double now;
+	struct VSM_fantom f_main, f_mgt, f_iter;
 
 	AN(freopen("errlog", "w", stderr));
 	setbuf(stderr, NULL);
@@ -859,8 +860,8 @@ do_curses(struct VSM_data *vd, int delay)
 	make_windows();
 	doupdate();
 
-	VSC_C_mgt = VSC_Mgt(vd);
-	VSC_C_main = VSC_Main(vd);
+	VSC_C_mgt = VSC_Mgt(vd, &f_mgt);
+	VSC_C_main = VSC_Main(vd, &f_main);
 	while (keep_running) {
 		if (VSM_Abandoned(vd)) {
 			fprintf(stderr, "abandoned\n");
@@ -870,13 +871,10 @@ do_curses(struct VSM_data *vd, int delay)
 				fprintf(stderr, "VSM_Open failed: %s\n",
 				    VSM_Error(vd));
 		}
-		VSC_C_mgt = VSC_Mgt(vd);
-		VSC_C_main = VSC_Main(vd);
-		if (!VSC_IterValid(vd)) {
-			fprintf(stderr, "iter not valid\n");
-			build_pt_list(vd);
-		} else
-			fprintf(stderr, "iter valid\n");
+		VSC_C_mgt = VSC_Mgt(vd, &f_mgt);
+		VSC_C_main = VSC_Main(vd, &f_main);
+		if (VSM_StillValid(vd, &f_iter) != 1)
+			build_pt_list(vd, &f_iter);
 
 		now = VTIM_mono();
 		if (now - t_sample > interval)
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 80f9674..55d2747 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -774,7 +774,7 @@ varnish_expect(const struct varnish *v, char * const *av) {
 		if (good < 0)
 			continue;
 
-		good = VSC_Iter(v->vd, do_stat_cb, &sp);
+		good = VSC_Iter(v->vd, NULL, do_stat_cb, &sp);
 		if (!good) {
 			good = -2;
 			continue;
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 9aefac2..550ca55 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -59,16 +59,61 @@ int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
 	 *	 1 Handled.
 	 */
 
-struct VSC_C_mgt *VSC_Mgt(struct VSM_data *vd);
+struct VSC_C_mgt *VSC_Mgt(struct VSM_data *vd, struct VSM_fantom *fantom);
 	/*
-	 * return Management stats structure
-	 * returns NULL until management process has finished initialization.
+	 * Looks up and returns the management stats structure. If fantom
+	 * is non-NULL, it can if successful later be used with
+	 * VSM_StillValid. Returns NULL until the management process has
+	 * finished initialization.
+	 *
+	 * The returned structure is valid for at most 60 seconds after
+	 * VSM_StillValid(,fantom) starts returning VSM_invalid. Using the
+	 * pointer after this event gives undefined behavior.
+	 *
+	 * Arguments:
+	 *	    vd: The VSM_data context
+	 *	fantom: Pointer to a fantom. Can be NULL.
+	 *
+	 * Return values:
+	 *      NULL: Failure
+	 *  non-NULL: Success
 	 */
 
-struct VSC_C_main *VSC_Main(struct VSM_data *vd);
+struct VSC_C_main *VSC_Main(struct VSM_data *vd, struct VSM_fantom *fantom);
 	/*
-	 * return Main stats structure
-	 * returns NULL until child has been started.
+	 * Looks up and returns the main stats structure. If fantom is
+	 * non-NULL, it can if successful later be used with
+	 * VSM_StillValid. Returns NULL until child has been started.
+	 *
+	 * The returned structure is valid for at most 60 seconds after
+	 * VSM_StillValid(,fantom) starts returning VSM_invalid. Using the
+	 * pointer after this event gives undefined behavior.
+	 *
+	 * Arguments:
+	 *	    vd: The VSM_data context
+	 *	fantom: Pointer to a fantom. Can be NULL.
+	 *
+	 * Return values:
+	 *      NULL: Failure
+	 *  non-NULL: Success
+	 */
+
+void *VSC_Get(struct VSM_data *vd, struct VSM_fantom *fantom, const char *type,
+    const char *ident);
+	/*
+	 * Looks up the given VSC type and identifier. If fantom is
+	 * non-NULL, it can if successful later be used with
+	 * VSM_StillValid.
+	 *
+	 * Arguments:
+	 *	vd:	The VSM_data context
+	 *	fantom: Pointer to a fantom. Can be NULL.
+	 *	type:	The type of the counter segment
+	 *	ident:	The identifier of the counter segment
+	 *
+	 * Return values:
+	 *  NULL:	Failure
+	 *  non-NULL:	A void pointer to the stats structure.
 	 */
 
 struct VSC_level_desc;
@@ -114,51 +159,36 @@ struct VSC_point {
 
 typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
 
-int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
+int VSC_Iter(struct VSM_data *vd, struct VSM_fantom *fantom, VSC_iter_f *func,
+    void *priv);
 	/*
 	 * Iterate over all statistics counters, calling "func" for
 	 * each counter not suppressed by any "-f" arguments.
 	 *
+	 * fantom points to a struct VSM_fantom. If non-NULL, it can be
+	 * used with VSM_StillValid to check the validity of the points
+	 * returned.
+	 *
+	 * The returned points are valid for at most 60 seconds after
+	 * VSM_StillValid(,fantom) starts returning anything but
+	 * VSM_valid, or until the next call to VSC_Iter. Using the point
+	 * values after any of these events gives undefined behavior.
+	 *
 	 * Func is called with pt == NULL, whenever VSM allocations
 	 * change (child restart, allocations/deallocations)
 	 *
+	 * Arguments:
+	 *	    vd: The VSM_data context
+	 *	fantom: Pointer to a fantom. Can be NULL.
+	 *	  func: The callback function
+	 *	  priv: Passed as argument to func
+	 *
 	 * Returns:
 	 *	!=0:	func returned non-zero
 	 *	-1:	No VSC's available
 	 *	0:	Done
 	 */
 
-int VSC_MgtValid(struct VSM_data *vd);
-	/*
-	 * Call VSM_StillValid on the fantom used to find the management
-	 * counters
-	 *
-	 * Returns:
-	 *	0: fantom is not valid any more
-	 *	1: fantom is still the same.
-	 *	2: a fantom with same dimensions exist.
-	 */
-
-int VSC_MainValid(struct VSM_data *vd);
-	/*
-	 * Call VSM_StillValid on the fantom used to find the main counters.
-	 *
-	 * Returns:
-	 *	0: fantom is not valid any more
-	 *	1: fantom is still the same.
-	 *	2: a fantom with same dimensions exist.
-	 */
-
-int VSC_IterValid(struct VSM_data *vd);
-	/*
-	 * Call VSM_StillValid on the fantom used to produce the last
-	 * VSC_Iter results.
-	 *
-	 * Returns:
-	 *	0: fantom is not valid any more
-	 *	1: fantom is still the same.
-	 */
-
 const struct VSC_level_desc *VSC_LevelDesc(unsigned level);
 
 /**********************************************************************
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 6099c4e..5d8101b 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -90,10 +90,8 @@ LIBVARNISHAPI_1.3 {
 	# Functions:
 	VSM_Abandoned;
 	VSM_ResetError;
+	VSM_StillValid;
 	VSC_Mgt;
-	VSC_MgtValid;
-	VSC_MainValid;
-	VSC_IterValid;
 	VSC_LevelDesc;
 	VSL_New;
 	VSL_Delete;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 97fc953..48fe32d 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -92,8 +92,6 @@ struct vsc {
 	VTAILQ_HEAD(, vsc_vf)	vf_list;
 	VTAILQ_HEAD(, vsc_pt)	pt_list;
 	VTAILQ_HEAD(, vsc_sf)	sf_list;
-	struct VSM_fantom	mgt_fantom;
-	struct VSM_fantom	main_fantom;
 	struct VSM_fantom	iter_fantom;
 };
 
@@ -263,32 +261,40 @@ VSC_Arg(struct VSM_data *vd, int arg, const char *opt)
 /*--------------------------------------------------------------------*/
 
 struct VSC_C_mgt *
-VSC_Mgt(struct VSM_data *vd)
+VSC_Mgt(struct VSM_data *vd, struct VSM_fantom *fantom)
 {
-	struct vsc *vsc = vsc_setup(vd);
 
-	if (!VSM_StillValid(vd, &vsc->mgt_fantom) &&
-	    !VSM_Get(vd, &vsc->mgt_fantom, VSC_CLASS, VSC_type_mgt, ""))
-		return (NULL);
-	return ((void*)vsc->mgt_fantom.b);
+	return (VSC_Get(vd, fantom, VSC_type_mgt, ""));
 }
 
 /*--------------------------------------------------------------------*/
 
 struct VSC_C_main *
-VSC_Main(struct VSM_data *vd)
+VSC_Main(struct VSM_data *vd, struct VSM_fantom *fantom)
 {
-	struct vsc *vsc = vsc_setup(vd);
 
-	if (!VSM_StillValid(vd, &vsc->main_fantom) &&
-	    !VSM_Get(vd, &vsc->main_fantom, VSC_CLASS, VSC_type_main, ""))
-		return (NULL);
-	return ((void*)vsc->main_fantom.b);
+	return (VSC_Get(vd, fantom, VSC_type_main, ""));
 }
 
 /*--------------------------------------------------------------------
  */
 
+void *
+VSC_Get(struct VSM_data *vd, struct VSM_fantom *fantom, const char *type,
+    const char *ident)
+{
+	struct VSM_fantom f2;
+
+	if (fantom == NULL)
+		fantom = &f2;
+	if (!VSM_StillValid(vd, fantom) &&
+	    !VSM_Get(vd, fantom, VSC_CLASS, type, ident))
+		return (NULL);
+	return ((void*)fantom->b);
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 vsc_add_vf(struct vsc *vsc, const struct VSM_fantom *fantom,
     const struct VSC_type_desc *desc, int order)
@@ -457,7 +463,8 @@ vsc_filter_pt_list(struct VSM_data *vd)
  */
 
 int
-VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
+VSC_Iter(struct VSM_data *vd, struct VSM_fantom *fantom, VSC_iter_f *func,
+    void *priv)
 {
 	struct vsc *vsc = vsc_setup(vd);
 	struct vsc_pt *pt;
@@ -470,6 +477,8 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 		vsc_build_pt_list(vd);
 		vsc_filter_pt_list(vd);
 	}
+	if (fantom != NULL)
+		*fantom = vsc->iter_fantom;
 	VTAILQ_FOREACH(pt, &vsc->pt_list, list) {
 		i = func(priv, &pt->point);
 		if (i)
@@ -481,44 +490,6 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 /*--------------------------------------------------------------------
  */
 
-int
-VSC_MgtValid(struct VSM_data *vd)
-{
-	struct vsc *vsc = vsc_setup(vd);
-	fprintf(stderr, "VSC_MgtValid called priv=%ju\n",
-		vsc->mgt_fantom.priv);
-	return (VSM_StillValid(vd, &vsc->mgt_fantom));
-}
-
-int
-VSC_MainValid(struct VSM_data *vd)
-{
-	struct vsc *vsc = vsc_setup(vd);
-	fprintf(stderr, "VSC_MainValid called priv=%ju\n",
-		vsc->main_fantom.priv);
-	return (VSM_StillValid(vd, &vsc->main_fantom));
-}
-
-int
-VSC_IterValid(struct VSM_data *vd)
-{
-	struct vsc *vsc = vsc_setup(vd);
-	int v;
-	fprintf(stderr, "VSC_IterValid called priv=%ju\n",
-		vsc->iter_fantom.priv);
-	v = VSM_StillValid(vd, &vsc->iter_fantom);
-	if (v == 2) {
-		/* There's been changes, reiteration needed. Clear fantom
-		   so subsequent calls will also fail */
-		memset(&vsc->iter_fantom, 0, sizeof vsc->iter_fantom);
-		v = 0;
-	}
-	fprintf(stderr, "VSC_IterValid returns %d priv=%ju\n", v,
-		vsc->iter_fantom.priv);
-
-	return (v);
-}
-
 const struct VSC_level_desc *
 VSC_LevelDesc(unsigned level)
 {



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 3e5f375 Add an enum describing the return values from
 VSM_StillValid
Message-ID: <E1Ucb5y-0004HU-Oa@project.varnish-software.com>

commit 3e5f3757c00a31ece2affb6bf70e32c066ecdee6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 13 14:33:45 2013 +0200

    Add an enum describing the return values from VSM_StillValid
    
    Use the enum values where VSM_StillValid is used.

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 89b5812..f45a43a 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -873,7 +873,7 @@ do_curses(struct VSM_data *vd, int delay)
 		}
 		VSC_C_mgt = VSC_Mgt(vd, &f_mgt);
 		VSC_C_main = VSC_Main(vd, &f_main);
-		if (VSM_StillValid(vd, &f_iter) != 1)
+		if (VSM_valid != VSM_StillValid(vd, &f_iter))
 			build_pt_list(vd, &f_iter);
 
 		now = VTIM_mono();
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index f6f7f9e..ef1024c 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -139,17 +139,23 @@ int VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf);
 	 * vd = "struct VSM_data *"
 	 */
 
-int VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf);
+enum VSM_valid_e {
+	VSM_invalid,
+	VSM_valid,
+	VSM_similar,
+};
+
+enum VSM_valid_e VSM_StillValid(const struct VSM_data *vd,
+    struct VSM_fantom *vf);
 	/*
 	 * This is a cheap syscall-less check to see if the fantom is still
 	 * valid.  Further checking with VSM_Abandoned() may be a good
 	 * idea.
 	 *
 	 * Return:
-	 *	0: fantom is not valid any more.
-	 *	1: fantom is still the same.
-	 *	2: a fantom with same dimensions exist in same position,
-	 *	   check class/type/ident
+	 *   VSM_invalid: fantom is not valid any more.
+	 *   VSM_valid:   fantom is still the same.
+	 *   VSM_similar: a fantom with same dimensions exist in same position.
 	 */
 
 int VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 48fe32d..06c11ce 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -287,7 +287,7 @@ VSC_Get(struct VSM_data *vd, struct VSM_fantom *fantom, const char *type,
 
 	if (fantom == NULL)
 		fantom = &f2;
-	if (!VSM_StillValid(vd, fantom) &&
+	if (VSM_invalid == VSM_StillValid(vd, fantom) &&
 	    !VSM_Get(vd, fantom, VSC_CLASS, type, ident))
 		return (NULL);
 	return ((void*)fantom->b);
@@ -470,7 +470,7 @@ VSC_Iter(struct VSM_data *vd, struct VSM_fantom *fantom, VSC_iter_f *func,
 	struct vsc_pt *pt;
 	int i;
 
-	if (1 != VSM_StillValid(vd, &vsc->iter_fantom)) {
+	if (VSM_valid != VSM_StillValid(vd, &vsc->iter_fantom)) {
 		/* Tell app that list will be nuked */
 		(void)func(priv, NULL);
 		vsc_build_vf_list(vd);
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index f4a7738..01808a4 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -131,7 +131,7 @@ vslc_vsm_next(void *cursor)
 
 	/* Check VSL fantom and abandonment */
 	if (*(volatile const uint32_t *)c->next.ptr == VSL_ENDMARKER) {
-		if (!VSM_StillValid(c->vsm, &c->vf) ||
+		if (VSM_invalid == VSM_StillValid(c->vsm, &c->vf) ||
 		    VSM_Abandoned(c->vsm))
 			return (-2);
 	}
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index b40e3d4..60679fd 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -367,7 +367,7 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 
 /*--------------------------------------------------------------------*/
 
-int
+enum VSM_valid_e
 VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
 	struct VSM_fantom f2;
@@ -375,11 +375,11 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vf);
 	if (!vd->head)
-		return (0);
+		return (VSM_invalid);
 	if (!vd->head->alloc_seq)
-		return (0);
+		return (VSM_invalid);
 	if (vf->priv == vd->head->alloc_seq)
-		return (1);
+		return (VSM_valid);
 	VSM_FOREACH(&f2, vd) {
 		if (f2.chunk != vf->chunk || f2.b != vf->b || f2.e != vf->e)
 			continue;
@@ -390,9 +390,9 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 		if (strcmp(f2.ident, vf->ident))
 			continue;
 		vf->priv = vd->head->alloc_seq;
-		return (2);
+		return (VSM_similar);
 	}
-	return (0);
+	return (VSM_invalid);
 }
 
 int



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 4e373b6 Add documentation about behavior on VSM_Close.
Message-ID: <E1Ucb5y-0004Hh-Q4@project.varnish-software.com>

commit 4e373b6ad4d80b5e4e2c4cd7694dacae8dd72805
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 13 14:39:06 2013 +0200

    Add documentation about behavior on VSM_Close.

diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index ef1024c..42d6e8d 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -124,7 +124,10 @@ int VSM_Abandoned(struct VSM_data *vd);
 
 void VSM_Close(struct VSM_data *vd);
 	/*
-	 * Close and unmap shared memory, if open.
+	 * Close and unmap shared memory, if open. Any reference to
+	 * previously returned memory areas will cause segmentation
+	 * fault. This includes any VSC counter areas or any VSL SHM
+	 * record references.
 	 */
 
 



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] ddced7a Don't reverse order of transactions in callback
Message-ID: <E1Ucb5y-0004ID-UZ@project.varnish-software.com>

commit ddced7a2efd2a57d4808db6cfd71329f3260ef90
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 14 13:22:53 2013 +0200

    Don't reverse order of transactions in callback

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 8c3bfe2..98a438d 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -719,8 +719,7 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
     void *priv)
 {
 	unsigned n = vtx->n_descend + 1;
-	struct vslc_vtx *cp[n];
-	struct VSL_cursor *CP[n + 1];
+	struct vslc_vtx *cp[n + 1];
 	unsigned i, j;
 
 	AN(vslq);
@@ -735,6 +734,7 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 	    vtx->type != vtx_t_req)
 		return (0);
 
+	/* Build cursor array */
 	i = j = 0;
 	cp[i] = &vtx->c;
 	vslc_vtx_reset(cp[i]);
@@ -753,17 +753,15 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 		j++;
 	}
 	assert(i == n);
-
-	/* Reverse order */
-	for (i = 0; i < n; i++)
-		CP[i] = &cp[n - i - 1]->c.c;
-	CP[i] = NULL;
+	cp[i] = NULL;
 
 	/* Query test goes here */
-	if (vslq->query == NULL ? 1 : vslq_runquery(vslq->query, CP))
-		return ((func)(vslq->vsl, CP, priv));
-	else
+	if (vslq->query != NULL &&
+	    vslq_runquery(vslq->query, (struct VSL_cursor **)cp))
 		return (0);
+
+	/* Callback */
+	return ((func)(vslq->vsl, (struct VSL_cursor **)cp, priv));
 }
 
 struct VSLQ *



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] d6eeed4 Draw the info segment in the curses app
Message-ID: <E1Ucb5y-0004I4-TK@project.varnish-software.com>

commit d6eeed453970ff8e1dfdcdad5e09f48819a712ef
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 13 15:43:11 2013 +0200

    Draw the info segment in the curses app

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index de3ea95..a2bf8a3 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -752,13 +752,18 @@ draw_bar_b(void)
 static void
 draw_info(void)
 {
-	int Y, X;
 
 	if (w_info == NULL)
 		return;
 
-	getmaxyx(w_info, Y, X);
-	mvwprintw(w_info, 0, 0, "infotest Y=%d X=%d", Y, X);
+	werase(w_info);
+	if (current < n_ptarray - 1) {
+		/* XXX: Word wrapping, and overflow handling? */
+		mvwprintw(w_info, 0, 0, "%s:",
+		    ptarray[current]->vpt->desc->sdesc);
+		mvwprintw(w_info, 1, 0, "%s",
+		    ptarray[current]->vpt->desc->ldesc);
+	}
 	wnoutrefresh(w_info);
 }
 



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] e1706e6 Remove some debugging printf's from varnishstat
Message-ID: <E1Ucb5y-0004Ht-S0@project.varnish-software.com>

commit e1706e6f8d38bc2f617153b1834ebb2d1fed8c25
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 13 14:52:06 2013 +0200

    Remove some debugging printf's from varnishstat

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index f45a43a..de3ea95 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -777,8 +777,6 @@ draw_screen(void)
 static void
 handle_keypress(int ch)
 {
-	fprintf(stderr, "key: 0x%x\n", ch);
-
 	switch (ch) {
 	case KEY_UP:
 		if (current == 0)
@@ -843,9 +841,6 @@ do_curses(struct VSM_data *vd, int delay)
 	double now;
 	struct VSM_fantom f_main, f_mgt, f_iter;
 
-	AN(freopen("errlog", "w", stderr));
-	setbuf(stderr, NULL);
-
 	(void)delay;
 
 	initscr();
@@ -864,7 +859,6 @@ do_curses(struct VSM_data *vd, int delay)
 	VSC_C_main = VSC_Main(vd, &f_main);
 	while (keep_running) {
 		if (VSM_Abandoned(vd)) {
-			fprintf(stderr, "abandoned\n");
 			delete_pt_list();
 			VSM_Close(vd);
 			if (VSM_Open(vd) < 0)
@@ -904,6 +898,4 @@ do_curses(struct VSM_data *vd, int delay)
 	}
 	VSM_Close(vd);
 	AZ(endwin());
-
-	fclose(stderr);
 }



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 4b3dd53 Beginnings of new varnishlog
Message-ID: <E1Ucb5y-0004Fi-82@project.varnish-software.com>

commit 4b3dd535ee24754c62a4dede191f40204387ff6d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Apr 28 17:01:01 2013 +0200

    Beginnings of new varnishlog

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 4adc0bb..226c670 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -3,7 +3,7 @@
 # Disabling building of the tools while api is in flux
 
 #SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishd
+SUBDIRS = varnishd varnishlog
 
 if HAVE_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index f6416f7..b163259 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -12,7 +12,8 @@ varnishlog_SOURCES = \
 	$(top_builddir)/lib/libvarnish/flopen.c \
 	$(top_builddir)/lib/libvarnish/version.c \
 	$(top_builddir)/lib/libvarnish/vsb.c \
-	$(top_builddir)/lib/libvarnish/vpf.c
+	$(top_builddir)/lib/libvarnish/vpf.c \
+	$(top_builddir)/lib/libvarnish/vtim.c
 
 varnishlog_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index 270855f..6e22d30 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -1,9 +1,10 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2013 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -31,357 +32,154 @@
 
 #include "config.h"
 
-#include <errno.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <stdint.h>
-#include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
+#include <stdio.h>
 #include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdint.h>
 
-#include "vapi/vsl.h"
 #include "vapi/vsm.h"
+#include "vapi/vsl.h"
 #include "vas.h"
 #include "vcs.h"
 #include "vpf.h"
 #include "vsb.h"
+#include "vtim.h"
 
 #include "compat/daemon.h"
 
-static int	b_flag = 0, c_flag = 0;
-
-/* Ordering-----------------------------------------------------------*/
-
-static struct vsb	*ob[65536];
-static unsigned char	flg[65536];
-static enum VSL_tag_e   last[65536];
-static uint64_t       bitmap[65536];
-#define F_INVCL		(1 << 0)
+static void error(int status, const char *fmt, ...)
+	__printflike(2, 3);
 
 static void
-h_order_finish(int fd, struct VSM_data *vd)
+error(int status, const char *fmt, ...)
 {
+	va_list ap;
 
-	AZ(VSB_finish(ob[fd]));
-	if (VSB_len(ob[fd]) > 1 && VSL_Matched(vd, bitmap[fd])) {
-		printf("%s", VSB_data(ob[fd]));
-	}
-	bitmap[fd] = 0;
-	VSB_clear(ob[fd]);
-}
-
-static void
-clean_order(struct VSM_data *vd)
-{
-	unsigned u;
+	AN(fmt);
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap); /* XXX: syslog on daemon */
+	va_end(ap);
 
-	for (u = 0; u < 65536; u++) {
-		if (ob[u] == NULL)
-			continue;
-		AZ(VSB_finish(ob[u]));
-		if (VSB_len(ob[u]) > 1 && VSL_Matched(vd, bitmap[u])) {
-			printf("%s\n", VSB_data(ob[u]));
-		}
-		flg[u] = 0;
-		bitmap[u] = 0;
-		VSB_clear(ob[u]);
-	}
+	if (status)
+		exit(status);
 }
 
-static int
-h_order(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
-    unsigned spec, const char *ptr, uint64_t bm)
+static struct VSL_cursor *
+cursor_vsm(struct VSL_data *vsl, struct VSM_data *vsm, int status, int front)
 {
-	char type;
-
-	struct VSM_data *vd = priv;
-
-	/* XXX: Just ignore any fd not inside the bitmap */
-	if (fd >= sizeof bitmap / sizeof bitmap[0])
-		return (0);
-
-	bitmap[fd] |= bm;
-
-	type = (spec & VSL_S_CLIENT) ? 'c' :
-	    (spec & VSL_S_BACKEND) ? 'b' : '-';
-
-	if (!(spec & (VSL_S_CLIENT|VSL_S_BACKEND))) {
-		if (!b_flag && !c_flag)
-			(void)VSL_H_Print(stdout, tag, fd, len, spec, ptr, bm);
-		return (0);
-	}
-	if (ob[fd] == NULL) {
-		ob[fd] = VSB_new_auto();
-		assert(ob[fd] != NULL);
-	}
-	if ((tag == SLT_BackendOpen || tag == SLT_SessOpen ||
-		(tag == SLT_ReqStart &&
-		    last[fd] != SLT_SessOpen &&
-		    last[fd] != SLT_VCL_acl) ||
-		(tag == SLT_BackendXID &&
-		    last[fd] != SLT_BackendOpen)) &&
-	    VSB_len(ob[fd]) != 0) {
-		/*
-		 * This is the start of a new request, yet we haven't seen
-		 * the end of the previous one.  Spit it out anyway before
-		 * starting on the new one.
-		 */
-		if (last[fd] != SLT_SessClose)
-			VSB_printf(ob[fd], "%5u %-12s %c %s\n",
-			    fd, "Interrupted", type, VSL_tags[tag]);
-		h_order_finish(fd, vd);
-	}
-
-	last[fd] = tag;
-
-	switch (tag) {
-	case SLT_VCL_call:
-		if (flg[fd] & F_INVCL)
-			VSB_cat(ob[fd], "\n");
-		else
-			flg[fd] |= F_INVCL;
-		VSB_printf(ob[fd], "%5u %-12s %c %.*s",
-		    fd, VSL_tags[tag], type, (int)len, ptr);
-		return (0);
-	case SLT_VCL_trace:
-	case SLT_VCL_return:
-		if (flg[fd] & F_INVCL) {
-			VSB_cat(ob[fd], " ");
-			VSB_bcat(ob[fd], ptr, len);
-			return (0);
-		}
-		break;
-	default:
-		break;
-	}
-	if (flg[fd] & F_INVCL) {
-		VSB_cat(ob[fd], "\n");
-		flg[fd] &= ~F_INVCL;
-	}
-	VSB_printf(ob[fd], "%5u %-12s %c %.*s\n",
-	    fd, VSL_tags[tag], type, (int)len, ptr);
-	switch (tag) {
-	case SLT_ReqEnd:
-	case SLT_BackendClose:
-	case SLT_BackendReuse:
-		h_order_finish(fd, vd);
-		break;
-	default:
-		break;
-	}
-	return (0);
+	struct VSL_cursor *c;
+
+	AN(vsm);
+	if (VSM_Open(vsm)) {
+		if (status)
+			error(status, "VSM: %s", VSM_Error(vsm));
+		VSM_ResetError(vsm);
+		return (NULL);
+	}
+	c = VSL_CursorVSM(vsl, vsm, front);
+	if (c == NULL) {
+		if (status)
+			error(status, "VSL: %s", VSL_Error(vsl));
+		VSL_ResetError(vsl);
+		return (NULL);
+	}
+	return (c);
 }
 
 static void
-do_order(struct VSM_data *vd)
+usage(void)
 {
-	int i;
-
-	if (!b_flag) {
-		VSL_Select(vd, SLT_SessOpen);
-		VSL_Select(vd, SLT_SessClose);
-		VSL_Select(vd, SLT_ReqEnd);
-	}
-	if (!c_flag) {
-		VSL_Select(vd, SLT_BackendOpen);
-		VSL_Select(vd, SLT_BackendClose);
-		VSL_Select(vd, SLT_BackendReuse);
-	}
-	while (1) {
-		i = VSL_Dispatch(vd, h_order, vd);
-		if (i == 0) {
-			clean_order(vd);
-			AZ(fflush(stdout));
-		}
-		else if (i < 0)
-			break;
-	}
-	clean_order(vd);
+	fprintf(stderr, "usage: varnishlog ...\n");
+	exit(1);
 }
 
-/*--------------------------------------------------------------------*/
-
-static volatile sig_atomic_t reopen;
-
 static void
-sighup(int sig)
+do_raw(struct VSL_data *vsl, struct VSM_data *vsm)
 {
+	struct VSL_cursor *c;
+	int i;
 
-	(void)sig;
-	reopen = 1;
-}
-
-static int
-open_log(const char *w_arg, int a_flag)
-{
-	int fd, flags;
-
-	flags = (a_flag ? O_APPEND : O_TRUNC) | O_WRONLY | O_CREAT;
-#ifdef O_LARGEFILE
-	flags |= O_LARGEFILE;
-#endif
-	if (!strcmp(w_arg, "-"))
-		fd = STDOUT_FILENO;
-	else
-		fd = open(w_arg, flags, 0644);
-	if (fd < 0) {
-		perror(w_arg);
-		exit(1);
-	}
-	return (fd);
-}
-
-static void
-do_write(struct VSM_data *vd, const char *w_arg, int a_flag)
-{
-	int fd, i, l;
-	uint32_t *p;
+	c = cursor_vsm(vsl, vsm, 1, 1);
+	AN(c);
 
-	fd = open_log(w_arg, a_flag);
-	XXXAN(fd >= 0);
-	(void)signal(SIGHUP, sighup);
+	i = 0;
 	while (1) {
-		i = VSL_NextSLT(vd, &p, NULL);
-		if (i < 0)
-			break;
-		if (i > 0) {
-			l = VSL_LEN(p);
-			i = write(fd, p, 8L + VSL_WORDS(l) * 4L);
-			if (i < 0) {
-				perror(w_arg);
-				exit(1);
+		while (c == NULL) {
+			c = cursor_vsm(vsl, vsm, 0, 1);
+			if (c != NULL) {
+				error(0, "Log reopened\n");
+				break;
 			}
+			VTIM_sleep(1.);
 		}
-		if (reopen) {
-			AZ(close(fd));
-			fd = open_log(w_arg, a_flag);
-			XXXAN(fd >= 0);
-			reopen = 0;
+
+		i = VSL_Next(c);
+		if (i == 1) {
+			/* Got new record */
+			if (VSL_Match(vsl, c))
+				(void)VSL_Print(vsl, c, stdout);
+		} else if (i == 0) {
+			/* Nothing to do but wait */
+			VTIM_sleep(0.01);
+		} else if (i == -1) {
+			/* EOF */
+			break;
+		} else if (i == -2) {
+			/* Abandoned - try reconnect */
+			error(0, "Log abandoned, reopening\n");
+			VSL_DeleteCursor(c);
+			c = NULL;
+			VSM_Close(vsm);
+		} else if (i < -2) {
+			/* Overrun - bail */
+			error(1, "Log overrun\n");
 		}
 	}
-	exit(0);
-}
-
-/*--------------------------------------------------------------------*/
 
-static void
-usage(void)
-{
-	fprintf(stderr, "usage: varnishlog "
-	    "%s [-aDV] [-o [tag regex]] [-n varnish_name]"
-	    " [-P file] [-w file]\n", VSL_USAGE);
-	exit(1);
+	VSL_DeleteCursor(c);
 }
 
 int
 main(int argc, char * const *argv)
 {
-	int c;
-	int a_flag = 0, D_flag = 0, O_flag = 0, u_flag = 0, m_flag = 0;
-	const char *P_arg = NULL;
-	const char *w_arg = NULL;
-	struct vpf_fh *pfh = NULL;
-	struct VSM_data *vd;
+	struct VSL_data *vsl;
+	struct VSM_data *vsm;
+	char c;
+	char *r_arg = NULL;
 
-	vd = VSM_New();
+	vsl = VSL_New();
+	AN(vsl);
+	vsm = VSM_New();
+	AN(vsm);
 
-	while ((c = getopt(argc, argv, VSL_ARGS "aDP:uVw:oO")) != -1) {
+	while ((c = getopt(argc, argv, "n:r:")) != -1) {
 		switch (c) {
-		case 'a':
-			a_flag = 1;
-			break;
-		case 'b':
-			b_flag = 1;
-			AN(VSL_Arg(vd, c, optarg));
-			break;
-		case 'c':
-			c_flag = 1;
-			AN(VSL_Arg(vd, c, optarg));
+		case 'r':
+			r_arg = optarg;
 			break;
-		case 'D':
-			D_flag = 1;
-			break;
-		case 'o': /* ignored for compatibility with older versions */
-			break;
-		case 'O':
-			O_flag = 1;
-			break;
-		case 'P':
-			P_arg = optarg;
-			break;
-		case 'u':
-			u_flag = 1;
-			break;
-		case 'V':
-			VCS_Message("varnishlog");
-			exit(0);
-		case 'w':
-			w_arg = optarg;
-			break;
-		case 'm':
-			m_flag = 1;
-			/* FALLTHROUGH */
-		default:
-			if (VSL_Arg(vd, c, optarg) > 0)
+		case 'n':
+			if (VSM_n_Arg(vsm, optarg) > 0)
 				break;
+		default:
 			usage();
 		}
 	}
 
-	/* If we're matching, we want either -b or -c, apply both if
-	 * none are given.  This prevents spurious noise in the log
-	 * output. */
-	if (b_flag == 0 && c_flag == 0 && m_flag) {
-		b_flag = 1;
-		AN(VSL_Arg(vd, 'b', NULL));
-		c_flag = 1;
-		AN(VSL_Arg(vd, 'c', NULL));
-	}
-
-	if (O_flag && m_flag)
-		usage();
-
-	if ((argc - optind) > 0)
-		usage();
-
-	if (VSM_Open(vd)) {
-		fprintf(stderr, "%s\n", VSM_Error(vd));
-		exit(1);
-	}
-
-	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
-		perror(P_arg);
-		exit(1);
+	if (r_arg) {
+		/* XXX */
+	} else {
+		if (VSM_Open(vsm))
+			error(1, VSM_Error(vsm));
 	}
 
-	if (D_flag && varnish_daemon(0, 0) == -1) {
-		perror("daemon()");
-		if (pfh != NULL)
-			VPF_Remove(pfh);
-		exit(1);
-	}
-
-	if (pfh != NULL)
-		VPF_Write(pfh);
-
-	if (w_arg != NULL)
-		do_write(vd, w_arg, a_flag);
+	do_raw(vsl, vsm);
 
-	if (u_flag)
-		setbuf(stdout, NULL);
-
-	if (!O_flag)
-		do_order(vd);
-
-	while (VSL_Dispatch(vd, VSL_H_Print, stdout) >= 0) {
-		if (fflush(stdout) != 0) {
-			perror("stdout");
-			break;
-		}
-	}
+	VSL_Delete(vsl);
+	if (vsm)
+		VSM_Delete(vsm);
 
-	if (pfh != NULL)
-		VPF_Remove(pfh);
 	exit(0);
 }



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] a099300 Beginnings of new VSL api
Message-ID: <E1Ucb5y-0004Fa-6R@project.varnish-software.com>

commit a0993006bb06c3684847057bd3dc692f2d03c04f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Apr 10 13:47:39 2013 +0200

    Beginnings of new VSL api
    
    Remove automagic reopening logic from vsl
    
    Make vsl use the new VSL_head structure
    
    Remove most of the old functionality

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index db8b953..6ee0e33 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -28,131 +28,129 @@
  *
  * This is the public API for the VSL access.
  *
- * VSL is a "subclass" of VSM.
- *
- * VSL can either read from VSM or from a file.
- *
- * When reading from a file, the filename is passed in with:
- *	VSL_Arg(vd, "r", "/some/file");
- * and once VSL_Dispatch()/VSL_NextSLT() will indicate EOF by returning -2.
- * Another file can then be opened with VSL_Arg() and processed.
- *
  */
 
 #ifndef VAPI_VSL_H_INCLUDED
 #define VAPI_VSL_H_INCLUDED
 
-#include "vapi/vsl_int.h"
+#include <stdio.h>
 
-struct VSM_data;
+#include "vapi/vsm.h"
+#include "vapi/vsl_int.h"
 
-/*---------------------------------------------------------------------
- * VSL level access functions
- */
+#define VSL_ARGS	"i:x:"
 
-#define VSL_ARGS	"bCcdI:i:k:n:r:s:X:x:m:"
-#define VSL_b_USAGE	"[-b]"
-#define VSL_c_USAGE	"[-c]"
-#define VSL_C_USAGE	"[-C]"
-#define VSL_d_USAGE	"[-d]"
 #define VSL_i_USAGE	"[-i tag]"
-#define VSL_I_USAGE	"[-I regexp]"
-#define VSL_k_USAGE	"[-k keep]"
-#define VSL_m_USAGE	"[-m tag:regex]"
-#define VSL_n_USAGE	VSM_n_USAGE
-#define VSL_r_USAGE	"[-r file]"
-#define VSL_s_USAGE	"[-s skip]"
 #define VSL_x_USAGE	"[-x tag]"
-#define VSL_X_USAGE	"[-X regexp]"
 
-#define VSL_USAGE	"[-bCcd] "		\
+#define VSL_USAGE	"[...] "		\
 			VSL_i_USAGE " "		\
-			VSL_I_USAGE " "		\
-			VSL_k_USAGE " "		\
-			VSL_m_USAGE " "		\
-			VSL_n_USAGE " "		\
-			VSL_r_USAGE " "		\
-			VSL_s_USAGE " "		\
-			VSL_X_USAGE " "		\
 			VSL_x_USAGE
 
-int VSL_Arg(struct VSM_data *vd, int arg, const char *opt);
+struct VSL_data;
+
+struct VSL_cursor {
+	const uint32_t		*ptr; /* Record pointer */
+};
+
+extern const char *VSL_tags[256];
+	/*
+	 * Tag to string array.  Contains NULL for invalid tags.
+	 */
+
+int VSL_Name2Tag(const char *name, int l);
+	/*
+	 * Convert string to tag number (= enum VSL_tag_e)
+	 *
+	 * Return values:
+	 *	>=0:	Tag number
+	 *	-1:	No tag matches
+	 *	-2:	Multiple tags match substring
+	 */
+
+struct VSL_data *VSL_New(void);
+int VSL_Arg(struct VSL_data *vsl, int opt, const char *arg);
 	/*
 	 * Handle standard log-presenter arguments
 	 * Return:
-	 *	-1 error, VSM_Error() returns diagnostic string
+	 *	-1 error, VSL_Error() returns diagnostic string
 	 *	 0 not handled
 	 *	 1 Handled.
 	 */
 
-typedef int VSL_handler_f(void *priv, enum VSL_tag_e tag, unsigned fd,
-    unsigned len, unsigned spec, const char *ptr, uint64_t bitmap);
+void VSL_Delete(struct VSL_data *vsl);
 	/*
-	 * This is the call-back function you must provide.
-	 *	priv is whatever you asked for it to be.
-	 *	tag is the SLT_mumble tag
-	 *	fd is the filedescriptor associated with this record
-	 *	len is the length of the data at ptr
-	 *	spec are the VSL_S_* flags
-	 *	ptr points to the data, beware of non-printables.
-	 *	bitmap is XXX ???
+	 * Delete a VSL context, freeing up the resources
 	 */
 
-#define VSL_S_CLIENT	(1 << 0)
-#define VSL_S_BACKEND	(1 << 1)
-
-VSL_handler_f VSL_H_Print;
+const char *VSL_Error(const struct VSL_data *vsl);
 	/*
-	 * This call-back function will printf() the record to the FILE *
-	 * specified in priv.
+	 * Return the latest error message.
 	 */
 
-void VSL_Select(struct VSM_data *vd, enum VSL_tag_e tag);
+void VSL_ResetError(struct VSL_data *vsl);
 	/*
-	 * This adds tags which shall always be selected, similar to using
-	 * the '-i' option.
-	 * VSL_Select()/-i takes precedence over all other filtering.
+	 * Reset any error message.
 	 */
 
-int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
+struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm,
+    int tail);
+       /*
+        * Set the cursor pointed to by cursor up as a raw cursor in the
+        * log. If tail is non-zero, it will point to the tail of the
+        * log. Is tail is zero, it will point close to the head of the
+        * log, at least 2 segments away from the head.
+	*
+	* Return values:
+	* non-NULL: Pointer to cursor
+	*     NULL: Error, see VSL_Error
+        */
+
+struct VSL_cursor *VSL_CursorFile(struct VSL_data *vsl, const char *name);
 	/*
-	 * Call func(priv, ...) for all filtered VSL records.
+	 * Create a cursor pointing to the beginning of the binary VSL log
+	 * in file name. If name is '-' reads from stdin.
 	 *
 	 * Return values:
-	 *	!=0:	Non-zero return value from func()
-	 *	0:	no VSL records.
-	 *	-1:	VSL chunk was abandoned.
-	 *	-2:	End of file (-r) / -k arg exhausted / "done"
+	 * non-NULL: Pointer to cursor
+	 *     NULL: Error, see VSL_Error
 	 */
 
-int VSL_NextSLT(struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
+void VSL_DeleteCursor(struct VSL_cursor *c);
 	/*
-	 * Return raw pointer to next filtered VSL record.
-	 *
-	 * Return values:
-	 *	1:	Valid VSL record at *pp
-	 *	0:	no VSL records
-	 *	-1:	VSL chunk was abandoned
-	 *	-2:	End of file (-r) / -k arg exhausted / "done"
+	 * Delete the cursor pointed to by c
 	 */
 
-int VSL_Matched(struct VSM_data *vd, uint64_t bitmap);
+int VSL_Next(struct VSL_cursor *c);
 	/*
+	 * Return raw pointer to next VSL record.
+	 *
+	 * Return values:
+	 *	1:	Cursor points to next log record
+	 *	0:	End of log
+	 *     -1:	End of file (-r) (XXX / -k arg exhausted / "done")
+	 *     -2:	Remote abandoned or closed
+	 *     -3:	Overrun
+	 *     -4:	I/O read error - see errno
 	 */
 
-int VSL_Name2Tag(const char *name, int l);
+int VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c);
 	/*
-	 * Convert string to tag number (= enum VSL_tag_e)
+	 * Returns true if the record pointed to by cursor matches the
+	 * record current record selectors
 	 *
-	 * Return values:
-	 *	>=0:	Tag number
-	 *	-1:	No tag matches
-	 *	-2:	Multiple tags match substring
+	 * Return value:
+	 *	1:	Match
+	 *	0:	No match
 	 */
 
-extern const char *VSL_tags[256];
+int VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *file);
 	/*
-	 * Tag to string array.  Contains NULL for invalid tags.
+	 * Print the log record pointed to by cursor to stream.
+	 *
+	 * Return values:
+	 *	0:	OK
+	 *     -5:	I/O write error - see errno
 	 */
 
 #endif /* VAPI_VSL_H_INCLUDED */
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index c34706b..c42198a 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -88,6 +88,7 @@ struct VSL_head {
 #define VSL_CLIENT(ptr)		(((ptr)[1]) & VSL_CLIENTMARKER)
 #define VSL_BACKEND(ptr)	(((ptr)[1]) & VSL_BACKENDMARKER)
 #define VSL_DATA(ptr)		((char*)((ptr)+2))
+#define VSL_CDATA(ptr)		((const char*)((ptr)+2))
 
 #define VSL_ENDMARKER	(((uint32_t)SLT__Reserved << 24) | 0x454545) /* "EEE" */
 #define VSL_WRAPMARKER	(((uint32_t)SLT__Reserved << 24) | 0x575757) /* "WWW" */
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index c405168..f6f7f9e 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -148,7 +148,8 @@ int VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf);
 	 * Return:
 	 *	0: fantom is not valid any more.
 	 *	1: fantom is still the same.
-	 *	2: a fantom with same dimensions exist, check class/type/ident
+	 *	2: a fantom with same dimensions exist in same position,
+	 *	   check class/type/ident
 	 */
 
 int VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 2e5de91..6069534 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -26,6 +26,7 @@ libvarnishapi_la_SOURCES = \
 	../libvarnish/vsha256.c \
 	vsm.c \
 	vsl_arg.c \
+	vsl_cursor.c \
 	vsl.c \
 	vsc.c \
 	libvarnishapi.map
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 34a851d..a25aa94 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -95,5 +95,15 @@ LIBVARNISHAPI_1.3 {
 	VSC_MainValid;
 	VSC_IterValid;
 	VSC_LevelDesc;
+	VSL_New;
+	VSL_Delete;
+	VSL_Error;
+	VSL_ResetError;
+	VSL_CursorVSM;
+	VSL_CursorFile;
+	VSL_DeleteCursor;
+	VSL_Next;
+	VSL_Match;
+	VSL_Print;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index cb9e6eb..963afce 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -1,9 +1,10 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2013 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -33,6 +34,7 @@
 #include <sys/types.h>
 
 #include <errno.h>
+#include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -43,12 +45,14 @@
 #include "vas.h"
 #include "vdef.h"
 
-#include "vapi/vsl.h"
 #include "vapi/vsm.h"
+#include "vapi/vsl.h"
 #include "vapi/vsm_int.h"
+#include "vin.h"
 #include "vbm.h"
 #include "vmb.h"
 #include "vre.h"
+#include "vsb.h"
 #include "vsl_api.h"
 #include "vsm_api.h"
 
@@ -60,334 +64,139 @@ const char *VSL_tags[256] = {
 #  undef SLTM
 };
 
-/*--------------------------------------------------------------------*/
-
-struct vsl *
-vsl_Setup(struct VSM_data *vd)
+int
+vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
 {
-	struct vsl *vsl;
+	va_list ap;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (vd->vsl == NULL) {
-		ALLOC_OBJ(vd->vsl, VSL_MAGIC);
-		AN(vd->vsl);
-		vsl = vd->vsl;
-		vsl->regflags = 0;
-		vsl->vbm_supress = vbit_init(256);
-		vsl->vbm_select = vbit_init(256);
-		vsl->r_fd = -1;
-		vsl->num_matchers = 0;
-		VTAILQ_INIT(&vsl->matchers);
-	}
-	CHECK_OBJ_NOTNULL(vd->vsl, VSL_MAGIC);
-	return (vd->vsl);
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	AN(fmt);
+
+	if (vsl->diag == NULL)
+		vsl->diag = VSB_new_auto();
+	AN(vsl->diag);
+	VSB_clear(vsl->diag);
+	va_start(ap, fmt);
+	VSB_vprintf(vsl->diag, fmt, ap);
+	va_end(ap);
+	AZ(VSB_finish(vsl->diag));
+	return (-1);
 }
 
-/*--------------------------------------------------------------------
- * Called from VSM_Delete()
- */
-
-void
-VSL_Delete(struct VSM_data *vd)
+struct VSL_data *
+VSL_New(void)
 {
-	struct vsl *vsl;
+	struct VSL_data *vsl;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	vd->vsl = NULL;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	ALLOC_OBJ(vsl, VSL_MAGIC);
+	if (vsl == NULL)
+		return (NULL);
 
-	if (vsl->r_fd > STDIN_FILENO)
-		(void)close(vsl->r_fd);
-	vbit_destroy(vsl->vbm_supress);
-	vbit_destroy(vsl->vbm_select);
-	free(vsl->rbuf);
-	FREE_OBJ(vsl);
-}
+	vsl->vbm_select = vbit_init(256);
+	vsl->vbm_supress = vbit_init(256);
 
-/*--------------------------------------------------------------------*/
+	return (vsl);
+}
 
 void
-VSL_Select(struct VSM_data *vd, enum VSL_tag_e tag)
+VSL_Delete(struct VSL_data *vsl)
 {
-	struct vsl *vsl = vsl_Setup(vd);
-
-	vbit_set(vsl->vbm_select, (int)tag);
-}
 
-/*--------------------------------------------------------------------
- */
-
-static int
-vsl_open(struct VSM_data *vd)
-{
-	struct vsl *vsl = vsl_Setup(vd);
-	int i;
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
-	assert(vsl->r_fd < 0);
-	if (vd->head == NULL) {
-		i = VSM_Open(vd);
-		if (i)
-			return (i);
-	}
-	if (!VSM_Get(vd, &vsl->vf, VSL_CLASS, NULL, NULL)) {
-		VSM_Close(vd);
-		return (vsm_diag(vd, "No VSL chunk found "
-		    " (child not started ?)\n"));
-	}
-	vsl->log_start = vsl->vf.b;
-	vsl->log_end = vsl->vf.e;
-	vsl->log_ptr = vsl->log_start + 1;
-	if (!vsl->d_opt) {
-		while (vsl->log_ptr < vsl->log_end &&
-		    *vsl->log_ptr != VSL_ENDMARKER)
-			vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
-	}
-	if (vsl->log_ptr >= vsl->log_end)
-		vsl->log_ptr = vsl->log_start + 1;
-	vsl->last_seq = vsl->log_start[0];
-	return (0);
+	vbit_destroy(vsl->vbm_select);
+	vbit_destroy(vsl->vbm_supress);
+	VSL_ResetError(vsl);
+	FREE_OBJ(vsl);
 }
 
-/*--------------------------------------------------------------------
- */
-
-static void
-vsl_close(struct VSM_data *vd)
+const char *
+VSL_Error(const struct VSL_data *vsl)
 {
-	struct vsl *vsl = vsl_Setup(vd);
 
-	assert(vsl->r_fd < 0);
-	VSM_Close(vd);
-	memset(&vsl->vf, 0, sizeof vsl->vf);
-	vsl->log_start = NULL;
-	vsl->log_end = NULL;
-	vsl->log_ptr = NULL;
-}
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
-/*--------------------------------------------------------------------
- * Return the next log record, if there is one
- *
- * Return:
- *	<0: error
- *	0: no record
- *	>0: record available at pp
- */
+	if (vsl->diag == NULL)
+		return (NULL);
+	else
+		return (VSB_data(vsl->diag));
+}
 
-static int
-vsl_nextslt(struct VSM_data *vd, uint32_t **pp)
+void
+VSL_ResetError(struct VSL_data *vsl)
 {
-	struct vsl *vsl = vsl_Setup(vd);
-	unsigned l;
-	uint32_t t;
-	int i;
-
-	*pp = NULL;
-	if (vsl->r_fd != -1) {
-		assert(vsl->rbuflen >= 8);
-		i = read(vsl->r_fd, vsl->rbuf, 8);
-		if (i == 0)
-			return (-2);
-		if (i != 8)
-			return (-1);
-		l = 2 + VSL_WORDS(VSL_LEN(vsl->rbuf));
-		if (vsl->rbuflen < l) {
-			l += 256;
-			vsl->rbuf = realloc(vsl->rbuf, l * 4L);
-			assert(vsl->rbuf != NULL);
-			vsl->rbuflen = l;
-		}
-		i = read(vsl->r_fd, vsl->rbuf + 2, l * 4L - 8L);
-		if (i != (l * 4L - 8L))
-			return (-1);
-		*pp = vsl->rbuf;
-		return (1);
-	}
-
-	if (vsl->log_ptr == NULL && vsl_open(vd))
-		return (0);
-
-	while (1) {
-		assert(vsl->log_ptr >= vsl->log_start + 1);
-		assert(vsl->log_ptr < vsl->log_end);
-		t = *vsl->log_ptr;
-
-		if (t == VSL_WRAPMARKER) {
-			/* Wrap around not possible at front */
-			if (vsl->log_ptr == vsl->log_start + 1)
-				return (-1);
-			vsl->log_ptr = vsl->log_start + 1;
-			continue;
-		}
 
-		if (t == VSL_ENDMARKER) {
-			if (vsl->log_ptr != vsl->log_start + 1 &&
-			    vsl->last_seq != vsl->log_start[0]) {
-				/* ENDMARKER not at front and seq wrapped */
-				vsl->log_ptr = vsl->log_start + 1;
-				continue;
-			}
-			return (0);
-		}
-
-		if (t == 0) {
-			/* Uninitialized VSL */
-			return (0);
-		}
-
-		if (vsl->log_ptr == vsl->log_start + 1)
-			vsl->last_seq = vsl->log_start[0];
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
-		*pp = (void*)(uintptr_t)vsl->log_ptr; /* Loose volatile */
-		vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
-		return (1);
-	}
+	if (vsl->diag == NULL)
+		return;
+	VSB_delete(vsl->diag);
+	vsl->diag = NULL;
 }
 
 int
-VSL_NextSLT(struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
+VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c)
 {
-	struct vsl *vsl = vsl_Setup(vd);
-	uint32_t *p;
-	unsigned char t;
-	int i;
-	struct vsl_re_match *vrm;
-	int j;
-
-	if (bits != NULL)
-		*bits = 0;
-
-	while (1) {
-		i = vsl_nextslt(vd, &p);
-		if (i < 0)
-			return (i);
-		if (i == 0) {
-			if (vsl->d_opt || vsl->r_fd >= 0)
-				return (i);
-			if (!VSM_StillValid(vd, &vsl->vf))
-				vsl_close(vd);
-			return (i);
-		}
+	enum VSL_tag_e tag;
 
-		t = VSL_TAG(p);
-		if (t == SLT__Batch) {
-			continue;
-		} else if (vbit_test(vsl->vbm_select, t)) {
-			/* nothing */
-		} else if (vbit_test(vsl->vbm_supress, t)) {
-			continue;
-		} else if (vsl->b_opt && !VSL_BACKEND(p)) {
-			continue;
-		} else if (vsl->c_opt && !VSL_CLIENT(p)) {
-			continue;
-		} else if (vsl->regincl != NULL) {
-			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0, NULL);
-			if (i == VRE_ERROR_NOMATCH)
-				continue;
-		} else if (vsl->regexcl != NULL) {
-			i = VRE_exec(vsl->regexcl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0, NULL);
-			if (i != VRE_ERROR_NOMATCH)
-				continue;
-		}
-
-		if (vsl->skip) {
-			--vsl->skip;
-			continue;
-		} else if (vsl->keep) {
-			if (--vsl->keep == 0)
-				return (-2);
-		}
-
-		if (bits != NULL) {
-			j = 0;
-			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
-				if (vrm->tag == t) {
-					i = VRE_exec(vrm->re, VSL_DATA(p),
-					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
-					if (i >= 0)	/* XXX ?? */
-						*bits |= (uintmax_t)1 << j;
-				}
-				j++;
-			}
-		}
-		*pp = p;
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	if (c == NULL || c->ptr == NULL)
+		return (0);
+	tag = VSL_TAG(c->ptr);
+	if (tag <= SLT__Bogus || tag >= SLT__Reserved)
+		return (0);
+	if (vbit_test(vsl->vbm_select, tag))
 		return (1);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
-{
-	int i;
-	unsigned u, l, s;
-	uint32_t *p;
-	uint64_t bitmap;
+	else if (vbit_test(vsl->vbm_supress, tag))
+		return (0);
 
-	while (1) {
-		i = VSL_NextSLT(vd, &p, &bitmap);
-		if (i <= 0)
-			return (i);
-		u = VSL_ID(p);
-		l = VSL_LEN(p);
-		s = 0;
-		if (VSL_CLIENT(p))
-			s |= VSL_S_CLIENT;
-		if (VSL_BACKEND(p))
-			s |= VSL_S_BACKEND;
-		i = func(priv, (enum VSL_tag_e)VSL_TAG(p),
-		    u, l, s, VSL_DATA(p), bitmap);
-		if (i)
-			return (i);
-	}
+	/* Default show */
+	return (1);
 }
 
-/*--------------------------------------------------------------------*/
-
 int
-VSL_H_Print(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
-    unsigned spec, const char *ptr, uint64_t bitmap)
+VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 {
-	FILE *fo = priv;
+	enum VSL_tag_e tag;
+	uint32_t vxid;
+	unsigned len;
+	const char *data;
 	int type;
+	int i;
 
-	(void) bitmap;
-	assert(fo != NULL);
-
-	type = (spec & VSL_S_CLIENT) ? 'c' :
-	    (spec & VSL_S_BACKEND) ? 'b' : '-';
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	if (c == NULL || c->ptr == NULL)
+		return (0);
+	if (fo == NULL)
+		fo = stdout;
+	tag = VSL_TAG(c->ptr);
+	vxid = VSL_ID(c->ptr);
+	len = VSL_LEN(c->ptr);
+	type = VSL_CLIENT(c->ptr) ? 'c' : VSL_BACKEND(c->ptr) ? 'b' : '-';
+	data = VSL_CDATA(c->ptr);
 
 	if (tag == SLT_Debug) {
-		fprintf(fo, "%5u %-12s %c \"", fd, VSL_tags[tag], type);
+		i = fprintf(fo, "%10u %-15s %c \"", vxid, VSL_tags[tag], type);
+		if (i < 0)
+			return (-5);
 		while (len-- > 0) {
-			if (*ptr >= ' ' && *ptr <= '~')
-				fprintf(fo, "%c", *ptr);
+			if (*data >= ' ' && *data <= '~')
+				i = fprintf(fo, "%c", *data);
 			else
-				fprintf(fo, "%%%02x", (unsigned char)*ptr);
-			ptr++;
+				i = fprintf(fo, "%%%02x", (unsigned char)*data);
+			if (i < 0)
+				return (-5);
+			data++;
 		}
-		fprintf(fo, "\"\n");
+		i = fprintf(fo, "\"\n");
+		if (i < 0)
+			return (-5);
 		return (0);
 	}
-	fprintf(fo, "%5u %-12s %c %.*s\n",
-	    fd, VSL_tags[tag], type, (int)len, ptr);
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
 
-int
-VSL_Matched(struct VSM_data *vd, uint64_t bitmap)
-{
-	struct vsl *vsl = vsl_Setup(vd);
-
-	if (vsl->num_matchers > 0) {
-		uint64_t t;
-		t = vsl->num_matchers | (vsl->num_matchers - 1);
-		return (bitmap == t);
-	}
-	return (1);
+	i = fprintf(fo, "%10u %-15s %c %.*s\n",
+	    vxid, VSL_tags[tag], type, (int)len, data);
+	if (i < 0)
+		return (-5);
+	return (0);
 }
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index b203ae8..8b70098 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -1,9 +1,10 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2013 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -28,64 +29,38 @@
  *
  */
 
+#include "vdef.h"
 #include "vqueue.h"
+#include "vapi/vsm.h"
 
-#define	SLEEP_USEC			(50*1000)
-#define	TIMEOUT_USEC			(5*1000*1000)
+#define VSL_FILE_HEAD			"VSL"
 
+int vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
+    __printflike(2, 3);
 
-struct vsl_re_match {
-	unsigned			magic;
-#define VSL_RE_MATCH_MAGIC		0x4013151e
-	int				tag;
-	vre_t				*re;
-	VTAILQ_ENTRY(vsl_re_match)	next;
-};
-
-struct vsl {
-	unsigned		magic;
-#define VSL_MAGIC		0x7a31db38
-
-	struct VSM_fantom	vf;
-
-	/* Stuff relating the log records below here */
-
-	volatile uint32_t	*log_start;
-	volatile uint32_t	*log_end;
-	volatile uint32_t	*log_ptr;
+typedef void vslc_delete_f(void *);
+typedef int vslc_next_f(void *);
 
-	volatile uint32_t	last_seq;
-
-	/* for -r option */
-	int			r_fd;
-	unsigned		rbuflen;
-	uint32_t		*rbuf;
-
-	int			b_opt;
-	int			c_opt;
-	int			d_opt;
+struct vslc {
+	struct VSL_cursor		c;
+	unsigned			magic;
+#define VSLC_MAGIC			0x5007C0DE
 
-	unsigned		flags;
-#define F_SEEN_IX		(1 << 0)
+	vslc_delete_f			*delete;
+	vslc_next_f			*next;
+};
 
-	/*
-	 * Bit map of programatically selected tags, that cannot be suppressed.
-	 * This way programs can make sure they will see certain tags, even
-	 * if the user tries to supress them with -x/-X
-	 */
-	struct vbitmap		*vbm_select;	/* index: tag */
+struct VSL_data {
+	unsigned			magic;
+#undef VSL_MAGIC
+#define VSL_MAGIC			0x8E6C92AA
 
-	/* Bit map of tags selected/supressed with -[iIxX] options */
-	struct vbitmap		*vbm_supress;	/* index: tag */
+	struct vsb			*diag;
 
-	int			regflags;
-	vre_t			*regincl;
-	vre_t			*regexcl;
-	int			num_matchers;
-	VTAILQ_HEAD(, vsl_re_match) matchers;
+	unsigned			flags;
+#define F_SEEN_ix			(1 << 0)
 
-	unsigned long		skip;
-	unsigned long		keep;
+	/* Bitmaps of -ix selected tags */
+	struct vbitmap			*vbm_select;
+	struct vbitmap			*vbm_supress;
 };
-
-struct vsl *vsl_Setup(struct VSM_data *vd);
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index e10567c..72c1f22 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -83,71 +83,20 @@ VSL_Name2Tag(const char *name, int l)
 	return (n);
 }
 
-
-/*--------------------------------------------------------------------*/
-
-static int
-vsl_r_arg(struct VSM_data *vd, const char *opt)
-{
-	struct vsl *vsl = vsl_Setup(vd);
-
-	if (vsl->r_fd > STDIN_FILENO)
-		(void)close(vsl->r_fd);
-	if (!strcmp(opt, "-"))
-		vsl->r_fd = STDIN_FILENO;
-	else
-		vsl->r_fd = open(opt, O_RDONLY);
-	if (vsl->r_fd < 0)
-		return (vsm_diag(vd,
-		    "Could not open %s: %s", opt, strerror(errno)));
-	if (vsl->rbuflen == 0) {
-		vsl->rbuflen = BUFSIZ;
-		vsl->rbuf = malloc(vsl->rbuflen);
-		AN(vsl->rbuf);
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
 static int
-vsl_IX_arg(struct VSM_data *vd, const char *opt, int arg)
+vsl_ix_arg(struct VSL_data *vsl, int opt, const char *arg)
 {
-	struct vsl *vsl = vsl_Setup(vd);
-	vre_t **rp;
-	const char *error;
-	int erroroffset;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (arg == 'I')
-		rp = &vsl->regincl;
-	else
-		rp = &vsl->regexcl;
-	if (*rp != NULL)
-		return (vsm_diag(vd, "Option %c can only be given once", arg));
-	*rp = VRE_compile(opt, vsl->regflags, &error, &erroroffset);
-	if (*rp == NULL)
-		return (vsm_diag(vd, "Illegal regex: %s\n", error));
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-vsl_ix_arg(struct VSM_data *vd, const char *opt, int arg)
-{
-	struct vsl *vsl = vsl_Setup(vd);
 	int i, l;
 	const char *b, *e;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	/* If first option is 'i', set all bits for supression */
-	if (arg == 'i' && !(vsl->flags & F_SEEN_IX))
+	if (opt == 'i' && !(vsl->flags & F_SEEN_ix))
 		for (i = 0; i < 256; i++)
 			vbit_set(vsl->vbm_supress, i);
-	vsl->flags |= F_SEEN_IX;
+	vsl->flags |= F_SEEN_ix;
 
-	for (b = opt; *b; b = e) {
+	for (b = arg; *b; b = e) {
 		while (isspace(*b))
 			b++;
 		e = strchr(b, ',');
@@ -160,123 +109,28 @@ vsl_ix_arg(struct VSM_data *vd, const char *opt, int arg)
 			l--;
 		i = VSL_Name2Tag(b, l);
 		if (i >= 0) {
-			if (arg == 'x')
+			if (opt == 'x')
 				vbit_set(vsl->vbm_supress, i);
 			else
 				vbit_clr(vsl->vbm_supress, i);
 		} else if (i == -2) {
-			return (vsm_diag(vd,
-			    "\"%*.*s\" matches multiple tags\n", l, l, b));
+			return (vsl_diag(vsl,
+			    "-%c: \"%*.*s\" matches multiple tags\n",
+			    (char)opt, l, l, b));
 		} else {
-			return (vsm_diag(vd,
-			    "Could not match \"%*.*s\" to any tag\n", l, l, b));
+			return (vsl_diag(vsl,
+			    "-%c: Could not match \"%*.*s\" to any tag\n",
+			    (char)opt, l, l, b));
 		}
 	}
 	return (1);
 }
 
-/*--------------------------------------------------------------------*/
-
-
-static int
-vsl_m_arg(struct VSM_data *vd, const char *opt)
-{
-	struct vsl *vsl = vsl_Setup(vd);
-	struct vsl_re_match *m;
-	const char *error;
-	char *o, *regex;
-	int erroroffset;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-
-	if (!strchr(opt, ':'))
-		return (vsm_diag(vd,
-		    "No : found in -m option %s\n", opt));
-
-	o = strdup(opt);
-	AN(o);
-	regex = strchr(o, ':');
-	*regex = '\0';
-	regex++;
-
-	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
-	AN(m);
-	m->tag = VSL_Name2Tag(o, -1);
-	if (m->tag < 0) {
-		(void)vsm_diag(vd, "Illegal tag %s specified\n", o);
-		free(o);
-		FREE_OBJ(m);
-		return (-1);
-	}
-	/* Get tag, regex */
-	m->re = VRE_compile(regex, vsl->regflags, &error, &erroroffset);
-	if (m->re == NULL) {
-		(void)vsm_diag(vd, "Illegal regex: %s\n", error);
-		free(o);
-		FREE_OBJ(m);
-		return (-1);
-	}
-	vsl->num_matchers++;
-	VTAILQ_INSERT_TAIL(&vsl->matchers, m, next);
-	free(o);
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-vsl_s_arg(struct VSM_data *vd, const char *opt)
-{
-	struct vsl *vsl = vsl_Setup(vd);
-	char *end;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (*opt == '\0')
-		return (vsm_diag(vd, "number required for -s\n"));
-	vsl->skip = strtoul(opt, &end, 10);
-	if (*end != '\0')
-		return (vsm_diag(vd, "invalid number for -k\n"));
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-vsl_k_arg(struct VSM_data *vd, const char *opt)
-{
-	struct vsl *vsl = vsl_Setup(vd);
-	char *end;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (*opt == '\0')
-		return (vsm_diag(vd, "number required for -k\n"));
-	vsl->keep = strtoul(opt, &end, 10);
-	if (*end != '\0')
-		return (vsm_diag(vd, "invalid number for -k\n"));
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
 int
-VSL_Arg(struct VSM_data *vd, int arg, const char *opt)
+VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 {
-	struct vsl *vsl = vsl_Setup(vd);
-
-	switch (arg) {
-	case 'b': vsl->b_opt = !vsl->b_opt; return (1);
-	case 'c': vsl->c_opt = !vsl->c_opt; return (1);
-	case 'd':
-		vsl->d_opt = !vsl->d_opt;
-		return (1);
-	case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg));
-	case 'k': return (vsl_k_arg(vd, opt));
-	case 'n': return (VSM_n_Arg(vd, opt));
-	case 'r': return (vsl_r_arg(vd, opt));
-	case 's': return (vsl_s_arg(vd, opt));
-	case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg));
-	case 'm': return (vsl_m_arg(vd, opt));
-	case 'C': vsl->regflags = VRE_CASELESS; return (1);
+	switch (opt) {
+	case 'i': case'x': return (vsl_ix_arg(vsl, opt, arg));
 	default:
 		return (0);
 	}
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
new file mode 100644
index 0000000..65262d1
--- /dev/null
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -0,0 +1,359 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "vas.h"
+#include "vdef.h"
+#include "miniobj.h"
+#include "vapi/vsm.h"
+#include "vsm_api.h"
+#include "vapi/vsl.h"
+#include "vsl_api.h"
+
+struct vslc_vsm {
+	struct vslc			c;
+	unsigned			magic;
+#define VSLC_VSM_MAGIC			0x4D3903A6
+
+	struct VSM_data			*vsm;
+	struct VSM_fantom		vf;
+
+	volatile const struct VSL_head	*head;
+	volatile const uint32_t		*next;
+	const uint32_t			*end;
+	ssize_t				segsize;
+	unsigned			seq;
+};
+
+static void
+vslc_vsm_delete(void *cursor)
+{
+	struct vslc_vsm *c;
+
+	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_VSM_MAGIC);
+	FREE_OBJ(c);
+}
+
+static int
+vslc_vsm_next(void *cursor)
+{
+	struct vslc_vsm *c;
+	int diff;
+	unsigned segment;
+	uint32_t t;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_VSM_MAGIC);
+	CHECK_OBJ_NOTNULL(c->vsm, VSM_MAGIC);
+
+	/* Assert pointers */
+	AN(c->next);
+	assert(c->next >= c->head->log);
+	assert(c->next < c->end);
+
+	/* Check sequence number */
+	diff = c->head->seq - c->seq;
+	if (c->head->seq < c->seq)
+		/* Wrap around skips 0 */
+		diff -= 1;
+	if (diff > 1)
+		return (-3);
+
+	/* Check overrun */
+	segment = (c->next - c->head->log) / c->segsize;
+	if (segment >= VSL_SEGMENTS)
+		segment = VSL_SEGMENTS - 1;
+	diff = (segment - c->head->segment) % VSL_SEGMENTS;
+	if (0 < diff && diff <= 2)
+		return (-3);
+
+	/* Check VSL fantom and abandonment */
+	if (*c->next == VSL_ENDMARKER) {
+		if (!VSM_StillValid(c->vsm, &c->vf) ||
+		    VSM_Abandoned(c->vsm))
+			return (-2);
+	}
+
+	while (1) {
+		assert(c->next >= c->head->log);
+		assert(c->next < c->end);
+		AN(c->head->seq);
+		t = *c->next;
+		AN(t);
+
+		if (t == VSL_WRAPMARKER) {
+			/* Wrap around not possible at front */
+			assert(c->next != c->head->log);
+			c->next = c->head->log;
+			continue;
+		}
+
+		if (t == VSL_ENDMARKER) {
+			if (c->next != c->head->log &&
+			    c->seq != c->head->seq) {
+				/* ENDMARKER not at front and seq wrapped */
+				/* XXX: assert on this? */
+				c->next = c->head->log;
+				continue;
+			}
+			return (0);
+		}
+
+		if (c->next == c->head->log)
+			c->seq = c->head->seq;
+
+		c->c.c.ptr = (void*)(uintptr_t)c->next; /* Loose volatile */
+		c->next = VSL_NEXT(c->next);
+		return (1);
+	}
+}
+
+struct VSL_cursor *
+VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
+{
+	struct vslc_vsm *c;
+	struct VSM_fantom vf;
+	struct VSL_head *head;
+	unsigned segment;
+
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	CHECK_OBJ_NOTNULL(vsm, VSM_MAGIC);
+
+	if (!VSM_Get(vsm, &vf, VSL_CLASS, "", "")) {
+		vsl_diag(vsl, "No VSL chunk found (child not started ?)\n");
+		return (NULL);
+	}
+
+	head = vf.b;
+	if (memcmp(head->marker, VSL_HEAD_MARKER, sizeof head->marker)) {
+		vsl_diag(vsl, "Not a VSL chunk\n");
+		return (NULL);
+	}
+	if (head->seq == 0) {
+		vsl_diag(vsl, "VSL chunk not initialized\n");
+		return (NULL);
+	}
+
+	ALLOC_OBJ(c, VSLC_VSM_MAGIC);
+	if (c == NULL) {
+		vsl_diag(vsl, "Out of memory\n");
+		return (NULL);
+	}
+	c->c.magic = VSLC_MAGIC;
+	c->c.delete = vslc_vsm_delete;
+	c->c.next = vslc_vsm_next;
+
+	c->vsm = vsm;
+	c->vf = vf;
+	c->head = head;
+	c->end = vf.e;
+	c->segsize = (c->end - c->head->log) / VSL_SEGMENTS;
+
+	if (tail) {
+		/* Locate tail of log */
+		c->next = c->head->log + c->head->segments[c->head->segment];
+		while (c->next < c->end && *c->next != VSL_ENDMARKER)
+			c->next = VSL_NEXT(c->next);
+	} else {
+		/*
+		 * Starting (VSL_SEGMENTS - 3) behind varnishd. This way
+		 * even if varnishd wraps immediately, we'll still have a
+		 * full segment worth of log before the general constraint
+		 * of at least 2 segments apart will be broken
+		 */
+		segment = (c->head->segment + 3) % VSL_SEGMENTS;
+		if (c->head->segments[segment] < 0)
+			segment = 0;
+		assert(c->head->segments[segment] >= 0);
+		c->next = c->head->log + c->head->segments[segment];
+	}
+	c->seq = c->head->seq;
+
+	return (&c->c.c);
+}
+
+struct vslc_file {
+	struct vslc			c;
+	unsigned			magic;
+#define VSLC_FILE_MAGIC			0x1D65FFEF
+
+	int				error;
+	int				fd;
+	ssize_t				buflen;
+	uint32_t			*buf;
+};
+
+static void
+vslc_file_delete(void *cursor)
+{
+	struct vslc_file *c;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_FILE_MAGIC);
+	if (c->fd > STDIN_FILENO)
+		(void)close(c->fd);
+	if (c->buf != NULL)
+		free(c->buf);
+	FREE_OBJ(c);
+}
+
+static ssize_t
+vslc_file_readn(int fd, void *buf, size_t n)
+{
+	size_t t = 0;
+	ssize_t l;
+
+	while (t < n) {
+		l = read(fd, (char *)buf + t, n - t);
+		if (l <= 0)
+			return (l);
+		t += l;
+	}
+	return (t);
+}
+
+static int
+vslc_file_next(void *cursor)
+{
+	struct vslc_file *c;
+	ssize_t i, l;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_FILE_MAGIC);
+
+	if (c->error)
+		return (c->error);
+
+	do {
+		c->c.c.ptr = NULL;
+		assert(c->buflen >= 2 * 4);
+		i = vslc_file_readn(c->fd, c->buf, 2 * 4);
+		if (i < 0)
+			return (-4);	/* I/O error */
+		if (i == 0)
+			return (-1);	/* EOF */
+		assert(i == 2 * 4);
+		l = (2 + VSL_WORDS(VSL_LEN(c->buf))) * 4;
+		if (c->buflen < l) {
+			c->buf = realloc(c->buf, 2 * l);
+			AN(c->buf);
+			c->buflen = 2 * l;
+		}
+		i = vslc_file_readn(c->fd, c->buf + 2, l - 2 * 4);
+		if (i < 0)
+			return (-4);	/* I/O error */
+		if (i == 0)
+			return (-1);	/* EOF */
+		assert(i == l - 2 * 4);
+		c->c.c.ptr = c->buf;
+	} while (c->c.c.ptr != NULL && VSL_TAG(c->c.c.ptr) == SLT__Batch);
+	return (1);
+}
+
+struct VSL_cursor *
+VSL_CursorFile(struct VSL_data *vsl, const char *name)
+{
+	struct vslc_file *c;
+	int fd;
+	char buf[4];
+	ssize_t i;
+
+	if (!strcmp(name, "-")) {
+		fd = open(name, O_RDONLY);
+		if (fd < 0) {
+			vsl_diag(vsl, "Could not open %s: %s\n", name,
+			    strerror(errno));
+			return (NULL);
+		}
+	} else
+		fd = STDIN_FILENO;
+
+	i = vslc_file_readn(fd, buf, sizeof buf);
+	if (i <= 0) {
+		if (fd > STDIN_FILENO)
+			(void)close(fd);
+		vsl_diag(vsl, "VSL file read error: %s\n",
+		    i < 0 ? strerror(errno) : "EOF");
+		return (NULL);
+	}
+	assert(i == sizeof buf);
+	if (memcmp(buf, VSL_FILE_HEAD, sizeof buf)) {
+		if (fd > STDIN_FILENO)
+			(void)close(fd);
+		vsl_diag(vsl, "Not a VSL file: %s\n", name);
+		return (NULL);
+	}
+
+	ALLOC_OBJ(c, VSLC_FILE_MAGIC);
+	if (c == NULL) {
+		if (fd > STDIN_FILENO)
+			(void)close(fd);
+		vsl_diag(vsl, "Out of memory\n");
+		return (NULL);
+	}
+	c->c.magic = VSLC_MAGIC;
+	c->c.delete = vslc_file_delete;
+	c->c.next = vslc_file_next;
+
+	c->fd = fd;
+	c->buflen = BUFSIZ;
+	c->buf = malloc(c->buflen);
+	AN(c->buf);
+
+	return (&c->c.c);
+}
+
+void
+VSL_DeleteCursor(struct VSL_cursor *cursor)
+{
+	struct vslc *c;
+
+	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
+	if (c->delete == NULL)
+		return;
+	(c->delete)(c);
+}
+
+int
+VSL_Next(struct VSL_cursor *cursor)
+{
+	struct vslc *c;
+
+	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
+	AN(c->next);
+	return ((c->next)(c));
+}
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index a957293..b40e3d4 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -163,8 +163,7 @@ VSM_Delete(struct VSM_data *vd)
 	free(vd->fname);
 	if (vd->vsc != NULL)
 		VSC_Delete(vd);
-	if (vd->vsl != NULL)
-		VSL_Delete(vd);
+	VSM_ResetError(vd);
 	FREE_OBJ(vd);
 }
 
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 2942494..04ebd7f 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -57,4 +57,3 @@ struct VSM_data {
 int vsm_diag(struct VSM_data *vd, const char *fmt, ...)
     __printflike(2, 3);
 void VSC_Delete(struct VSM_data *vd);
-void VSL_Delete(struct VSM_data *vd);



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] ef23d99 New VSL log format,
 with fraction indexes available to safely jump into the log.
Message-ID: <E1Ucb5y-0004FP-4j@project.varnish-software.com>

commit ef23d9958e79a7851c5cc890fccf05bcf98c86e0
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Apr 10 13:06:49 2013 +0200

    New VSL log format, with fraction indexes available to safely jump
    into the log.

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 5dbd44f..278b29d 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -44,9 +44,12 @@
 static pthread_mutex_t vsl_mtx;
 static pthread_mutex_t vsm_mtx;
 
-static uint32_t			*vsl_start;
+static struct VSL_head		*vsl_head;
 static const uint32_t		*vsl_end;
 static uint32_t			*vsl_ptr;
+static unsigned			vsl_segment;
+static ssize_t			vsl_segsize;
+static unsigned			vsl_seq;
 
 struct VSC_C_main       *VSC_C_main;
 
@@ -93,17 +96,21 @@ static void
 vsl_wrap(void)
 {
 
-	assert(vsl_ptr >= vsl_start + 1);
+	assert(vsl_ptr >= vsl_head->log);
 	assert(vsl_ptr < vsl_end);
-	vsl_start[1] = VSL_ENDMARKER;
+	vsl_head->log[0] = VSL_ENDMARKER;
 	do
-		vsl_start[0]++;
-	while (vsl_start[0] == 0);
+		vsl_seq++;
+	while (vsl_seq == 0);
+	vsl_head->seq = vsl_seq;
+	vsl_head->segments[0] = 0;
 	VWMB();
-	if (vsl_ptr != vsl_start + 1) {
+	if (vsl_ptr != vsl_head->log) {
 		*vsl_ptr = VSL_WRAPMARKER;
-		vsl_ptr = vsl_start + 1;
+		vsl_ptr = vsl_head->log;
 	}
+	vsl_segment = 0;
+	vsl_head->segment = vsl_segment;
 	VSC_C_main->shm_cycles++;
 }
 
@@ -116,6 +123,7 @@ vsl_get(unsigned len, unsigned records, unsigned flushes)
 {
 	uint32_t *p;
 	int err;
+	unsigned old_segment;
 
 	err = pthread_mutex_trylock(&vsl_mtx);
 	if (err == EBUSY) {
@@ -137,11 +145,25 @@ vsl_get(unsigned len, unsigned records, unsigned flushes)
 
 	p = vsl_ptr;
 	vsl_ptr = VSL_END(vsl_ptr, len);
+	assert(vsl_ptr < vsl_end);
+	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
 
 	*vsl_ptr = VSL_ENDMARKER;
 
-	assert(vsl_ptr < vsl_end);
-	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
+	old_segment = vsl_segment;
+	while ((vsl_ptr - vsl_head->log) / vsl_segsize > vsl_segment) {
+		if (vsl_segment == VSL_SEGMENTS - 1)
+			break;
+		vsl_segment++;
+		vsl_head->segments[vsl_segment] = vsl_ptr - vsl_head->log;
+	}
+	if (old_segment != vsl_segment) {
+		/* Write memory barrier to ensure ENDMARKER and new table
+		   values are seen before new segment number */
+		VWMB();
+		vsl_head->segment = vsl_segment;
+	}
+
 	AZ(pthread_mutex_unlock(&vsl_mtx));
 
 	return (p);
@@ -252,7 +274,7 @@ VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
 
 	assert(vsl->wlp < vsl->wle);
 
-	/* Wrap if necessary */
+	/* Flush if necessary */
 	if (VSL_END(vsl->wlp, l) >= vsl->wle)
 		VSL_Flush(vsl, 1);
 	assert(VSL_END(vsl->wlp, l) < vsl->wle);
@@ -298,7 +320,7 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
 
 	mlen = cache_param->shm_reclen;
 
-	/* Wrap if we cannot fit a full size record */
+	/* Flush if we cannot fit a full size record */
 	if (VSL_END(vsl->wlp, mlen) >= vsl->wle)
 		VSL_Flush(vsl, 1);
 
@@ -357,34 +379,37 @@ vsm_cleaner(void *priv)
 void
 VSM_Init(void)
 {
-	uint32_t *vsl_log_start;
+	int i;
 	pthread_t tp;
 
 	AZ(pthread_mutex_init(&vsl_mtx, NULL));
 	AZ(pthread_mutex_init(&vsm_mtx, NULL));
 
-	vsl_log_start = VSM_Alloc(cache_param->vsl_space, VSL_CLASS, "", "");
-	AN(vsl_log_start);
-	vsl_log_start[1] = VSL_ENDMARKER;
+	vsl_head = VSM_Alloc(cache_param->vsl_space, VSL_CLASS, "", "");
+	AN(vsl_head);
+	vsl_end = vsl_head->log +
+	    (cache_param->vsl_space - sizeof *vsl_head) / sizeof *vsl_end;
+	vsl_segsize = (vsl_end - vsl_head->log) / VSL_SEGMENTS;
+
+	memset(vsl_head, 0, sizeof *vsl_head);
+	memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
+	vsl_head->segments[0] = 0;
+	for (i = 1; i < VSL_SEGMENTS; i++)
+		vsl_head->segments[i] = -1;
+	vsl_ptr = vsl_head->log;
+	*vsl_ptr = VSL_ENDMARKER;
+
 	VWMB();
 	do
-		*vsl_log_start = random() & 0xffff;
-	while (*vsl_log_start == 0);
+		vsl_seq = random();
+	while (vsl_seq == 0);
+	vsl_head->seq = vsl_seq;
 	VWMB();
 
-	vsl_start = vsl_log_start;
-	vsl_end = vsl_start +
-	    cache_param->vsl_space / (unsigned)sizeof *vsl_end;
-	vsl_ptr = vsl_start + 1;
-
 	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
 	    VSC_CLASS, VSC_type_main, "");
 	AN(VSC_C_main);
-
-	vsl_wrap();
-	// VSM_head->starttime = (intmax_t)VTIM_real();
 	memset(VSC_C_main, 0, sizeof *VSC_C_main);
-	// VSM_head->child_pid = getpid();
 
 	AZ(pthread_create(&tp, NULL, vsm_cleaner, NULL));
 }
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index d50f9fc..c34706b 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -34,15 +34,27 @@
 #ifndef VAPI_VSL_FMT_H_INCLUDED
 #define VAPI_VSL_FMT_H_INCLUDED
 
+#include "vapi/vsm_int.h"
+
 #define VSL_CLASS		"Log"
+#define VSL_SEGMENTS		8
 
 /*
  * Shared memory log format
  *
- * The log is structured as an array of 32bit unsigned integers.
+ * The segments array has index values providing safe entry points into
+ * the log, where each element N gives the index of the first log record
+ * in the Nth fraction of the log. An index value of -1 indicated that no
+ * log records in this fraction exists.
+ *
+ * The segment member shows the current segment where Varnish is currently
+ * appending log data.
+ *
+ * The seq member contains a non-zero seq number randomly initialized,
+ * which increases whenever writing the log starts from the front.
  *
- * The first integer contains a non-zero serial number, which changes
- * whenever writing the log starts from the front.
+ * The log member points to an array of 32bit unsigned integers containing
+ * log records.
  *
  * Each logrecord consist of:
  *	[n]		= ((type & 0xff) << 24) | (length & 0xffff)
@@ -53,6 +65,15 @@
  * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c
  */
 
+struct VSL_head {
+#define VSL_HEAD_MARKER		"VSLHEAD0"	/* Incr. as version# */
+	char			marker[VSM_MARKER_LEN];
+	ssize_t			segments[VSL_SEGMENTS];
+	unsigned		segment;	/* Current varnishd segment */
+	unsigned		seq;		/* Non-zero seq number */
+	uint32_t		log[];
+};
+
 #define VSL_CLIENTMARKER	(1U<<30)
 #define VSL_BACKENDMARKER	(1U<<31)
 #define VSL_IDENTMASK		(~(3U<<30))



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 607b7af Update varnishadm to use new api
Message-ID: <E1Ucb5y-0004Ft-9W@project.varnish-software.com>

commit 607b7af79c370960de611ddc1e9044d119aca576
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Apr 28 22:09:12 2013 +0200

    Update varnishadm to use new api

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 226c670..8e481e5 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -3,7 +3,7 @@
 # Disabling building of the tools while api is in flux
 
 #SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishd varnishlog
+SUBDIRS = varnishadm varnishd varnishlog
 
 if HAVE_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop
diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 532ab72..4029178 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -326,26 +326,29 @@ n_arg_sock(const char *n_arg)
 {
 	char *T_arg = NULL, *T_start = NULL;
 	char *S_arg = NULL;
-	struct VSM_data *vsd;
+	struct VSM_data *vsm;
 	char *p;
 	int sock;
 	struct VSM_fantom vt;
 
-	vsd = VSM_New();
-	assert(VSL_Arg(vsd, 'n', n_arg));
-	if (VSM_Open(vsd)) {
-		fprintf(stderr, "%s\n", VSM_Error(vsd));
+	vsm = VSM_New();
+	if (VSM_n_Arg(vsm, n_arg) < 0) {
+		fprintf(stderr, "%s\n", VSM_Error(vsm));
+		return (-1);
+	}
+	if (VSM_Open(vsm)) {
+		fprintf(stderr, "%s\n", VSM_Error(vsm));
 		return (-1);
 	}
 
-	if (!VSM_Get(vsd, &vt, "Arg", "-T", "")) {
+	if (!VSM_Get(vsm, &vt, "Arg", "-T", "")) {
 		fprintf(stderr, "No -T arg in shared memory\n");
 		return (-1);
 	}
 	AN(vt.b);
 	T_start = T_arg = strdup(vt.b);
 
-	if (VSM_Get(vsd, &vt, "Arg", "-S", "")) {
+	if (VSM_Get(vsm, &vt, "Arg", "-S", "")) {
 		AN(vt.b);
 		S_arg = strdup(vt.b);
 	}



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] f6e0586 Make varnishtest work again with new VSL api
Message-ID: <E1Ucb5y-0004Fz-Ay@project.varnish-software.com>

commit f6e05865f900e0c9392fce875d33ea9b36a1052a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 29 15:13:39 2013 +0200

    Make varnishtest work again with new VSL api

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 8e481e5..02bfe66 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -3,7 +3,7 @@
 # Disabling building of the tools while api is in flux
 
 #SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishadm varnishd varnishlog
+SUBDIRS = varnishadm varnishd varnishlog varnishtest
 
 if HAVE_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 4e6f8d4..64b24b8 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -48,6 +48,7 @@
 #include "vcli.h"
 #include "vss.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 struct varnish {
 	unsigned		magic;
@@ -172,44 +173,76 @@ wait_running(const struct varnish *v)
 }
 
 /**********************************************************************
- * Varnishlog gatherer + thread
+ * Varnishlog gatherer thread
  */
 
-static int
-h_addlog(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
-    unsigned spec, const char *ptr, uint64_t bitmap)
+static void *
+varnishlog_thread(void *priv)
 {
 	struct varnish *v;
-	int type;
+	struct VSL_data *vsl;
+	struct VSM_data *vsm;
+	struct VSL_cursor *c;
+	enum VSL_tag_e tag;
+	uint32_t vxid;
+	unsigned len;
+	const char *data;
+	int type, i;
 
-	(void) bitmap;
-
-	type = (spec & VSL_S_CLIENT) ? 'c' :
-	    (spec & VSL_S_BACKEND) ? 'b' : '-';
 	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
 
-	v->vsl_tag_count[tag]++;
+	vsl = VSL_New();
+	AN(vsl);
+	vsm = VSM_New();
+	AN(vsm);
+	(void)VSM_n_Arg(vsm, v->workdir);
 
-	vtc_log(v->vl, 4, "vsl| %5u %-12s %c %.*s", fd,
-	    VSL_tags[tag], type, len, ptr);
-	v->vsl_sleep = 100;
-	return (0);
-}
+	c = NULL;
+	while (v->pid) {
+		if (c == NULL) {
+			VTIM_sleep(0.1);
+			if (VSM_Open(vsm)) {
+				VSM_ResetError(vsm);
+				continue;
+			}
+			c = VSL_CursorVSM(vsl, vsm, 1);
+			if (c == NULL) {
+				VSL_ResetError(vsl);
+				continue;
+			}
+		}
+		AN(c);
 
-static void *
-varnishlog_thread(void *priv)
-{
-	struct varnish *v;
-	struct VSM_data	*vsl;
+		i = VSL_Next(c);
+		if (i == 0) {
+			/* Nothing to do but wait */
+			VTIM_sleep(0.01);
+			continue;
+		} else if (i == -2) {
+			/* Abandoned - try reconnect */
+			VSL_DeleteCursor(c);
+			c = NULL;
+			VSM_Close(vsm);
+			continue;
+		} else if (i != 1)
+			break;
 
-	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
-	vsl = VSM_New();
-	(void)VSL_Arg(vsl, 'n', v->workdir);
-	while (v->pid) {
-		if (VSL_Dispatch(vsl, h_addlog, v) <= 0)
-			usleep(100000);
+		tag = VSL_TAG(c->ptr);
+		vxid = VSL_ID(c->ptr);
+		len = VSL_LEN(c->ptr);
+		type = VSL_CLIENT(c->ptr) ? 'c' : VSL_BACKEND(c->ptr) ?
+		    'b' : '-';
+		data = VSL_CDATA(c->ptr);
+		v->vsl_tag_count[tag]++;
+		vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", vxid,
+		    VSL_tags[tag], type, (int)len, data);
 	}
-	VSM_Delete(vsl);
+
+	if (c)
+		VSL_DeleteCursor(c);
+	VSL_Delete(vsl);
+	VSM_Delete(vsm);
+
 	return (NULL);
 }
 
@@ -449,7 +482,7 @@ varnish_launch(struct varnish *v)
 		vtc_log(v->vl, 0, "CLI auth command failed: %u %s", u, r);
 	free(r);
 
-	(void)VSL_Arg(v->vd, 'n', v->workdir);
+	(void)VSM_n_Arg(v->vd, v->workdir);
 	AZ(VSM_Open(v->vd));
 }
 



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 5c8b67f Work on print out of transaction statements.
Message-ID: <E1Ucb5y-0004IP-WB@project.varnish-software.com>

commit 5c8b67f8f4a7a924c920304380ae73bed22f515b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 13:32:52 2013 +0200

    Work on print out of transaction statements.
    
    Document and be consistent about how log statements are printed.
    
    Make the dispatch-reported data be of struct VSL_transactions, and
    move the transaction metadata there.
    
    Add a verbose option.

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index c669e70..c48bb1b 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -95,7 +95,7 @@ main(int argc, char * const *argv)
 	vsm = VSM_New();
 	AN(vsm);
 
-	while ((optchar = getopt(argc, argv, "dg:n:r:")) != -1) {
+	while ((optchar = getopt(argc, argv, "dg:n:r:v")) != -1) {
 		switch (optchar) {
 		case 'd':
 			d_opt = 1;
@@ -113,7 +113,8 @@ main(int argc, char * const *argv)
 			r_arg = optarg;
 			break;
 		default:
-			usage();
+			if (!VSL_Arg(vsl, optchar, optarg))
+				usage();
 		}
 	}
 
@@ -165,7 +166,7 @@ main(int argc, char * const *argv)
 			AZ(c);
 		}
 
-		i = VSLQ_Dispatch(q, VSL_PrintSet, stdout);
+		i = VSLQ_Dispatch(q, VSL_PrintTransactions, stdout);
 		if (i == 0) {
 			/* Nothing to do but wait */
 			VTIM_sleep(0.01);
@@ -174,7 +175,7 @@ main(int argc, char * const *argv)
 			break;
 		} else if (i <= -2) {
 			/* XXX: Make continuation optional */
-			VSLQ_Flush(q, VSL_PrintSet, stdout);
+			VSLQ_Flush(q, VSL_PrintTransactions, stdout);
 			VSLQ_Delete(&q);
 			AZ(q);
 			if (i == -2) {
@@ -191,7 +192,7 @@ main(int argc, char * const *argv)
 	}
 
 	if (q != NULL) {
-		VSLQ_Flush(q, VSL_PrintSet, stdout);
+		VSLQ_Flush(q, VSL_PrintTransactions, stdout);
 		VSLQ_Delete(&q);
 		AZ(q);
 	}
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 4bb1d60..2da4684 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -59,16 +59,22 @@ struct VSLC_ptr {
 struct VSL_cursor {
 	/* The record this cursor points to */
 	struct VSLC_ptr		rec;
+};
 
-	/* If not -1, the vxid of all records in this set */
-	int32_t			vxid;
-
-	/* For set cursors, the depth level of these records */
-	unsigned level;
+enum VSL_transaction_e {
+	VSL_t_unknown,
+	VSL_t_sess,
+	VSL_t_req,
+	VSL_t_esireq,
+	VSL_t_bereq,
+	VSL_t_raw,
+};
 
-	/* Nonzero if pointer values from this cursor are still valid
-	   after next call to VSL_Next */
-	unsigned shmptr_ok;
+struct VSL_transaction {
+	unsigned		level;
+	int32_t			vxid;
+	enum VSL_transaction_e	type;
+	struct VSL_cursor	*c;
 };
 
 enum VSL_grouping_e {
@@ -192,11 +198,13 @@ int VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c);
 	 *	0:	No match
 	 */
 
-int VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
+int VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
 	/*
 	 * Print the log record pointed to by cursor to stream.
 	 *
-	 * Format: <vxid> <tag> <type> <data>
+	 * Format: (t=type)
+	 * 1234567890 12345678901234 1 ...
+	 *       vxid tag            t content
 	 *
 	 * Arguments:
 	 *   vsl: The VSL_data context
@@ -208,11 +216,13 @@ int VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
 	 *     -5:	I/O write error - see errno
 	 */
 
-int VSL_PrintLevel(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
+int VSL_PrintTerse(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
 	/*
 	 * Print the log record pointed to by cursor to stream.
 	 *
-	 * Format: <level> <tag> <type> <data>
+	 * Format:
+	 * 12345678901234 ...
+	 * tag            content
 	 *
 	 * Arguments:
 	 *   vsl: The VSL_data context
@@ -227,10 +237,7 @@ int VSL_PrintLevel(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
 int VSL_PrintAll(struct VSL_data *vsl, struct VSL_cursor *c, void *fo);
 	/*
 	 * Calls VSL_Next on c until c is exhausted. In turn calls
-	 * prints all records where VSL_Match returns true.
-	 *
-	 * If c->vxid == -1, calls VSL_PrintVXID on each record. Else
-	 * prints a VXID header and calls VSL_PrintLevel on each record.
+	 * VSL_Print on all records where VSL_Match returns true.
 	 *
 	 * Arguments:
 	 *   vsl: The VSL_data context
@@ -242,11 +249,14 @@ int VSL_PrintAll(struct VSL_data *vsl, struct VSL_cursor *c, void *fo);
 	 *    !=0:	Return value from either VSL_Next or VSL_Print
 	 */
 
-int VSL_PrintSet(struct VSL_data *vsl, struct VSL_cursor *cp[], void *fo);
+int VSL_PrintTransactions(struct VSL_data *vsl,
+    struct VSL_transaction *ptrans[], void *fo);
 	/*
-	 * Calls VSL_PrintAll on each cursor in cp[]. If any cursor in cp
-	 * has vxid != -1 it will end with a double line break as a set
-	 * delimeter.
+	 * Prints out each transaction in the array ptrans. For
+	 * transactions of level > 0 it will print a header before the log
+	 * records. The records will for level == 0 (single records) or if
+	 * v_opt is set, be printed by VSL_Print. Else VSL_PrintTerse is
+	 * used.
 	 *
 	 * Arguments:
 	 *   vsl: The VSL_data context
@@ -255,7 +265,7 @@ int VSL_PrintSet(struct VSL_data *vsl, struct VSL_cursor *cp[], void *fo);
 	 *
 	 * Return values:
 	 *	0:	OK
-	 *    !=0:	Return value from either VSL_Next or VSL_PrintAll
+	 *    !=0:	Return value from either VSL_Next or VSL_Print
 	 */
 
 struct VSLQ *VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp,
@@ -280,16 +290,15 @@ void VSLQ_Delete(struct VSLQ **pvslq);
 	 * Delete the query pointed to by pvslq, freeing up the resources
 	 */
 
-typedef int VSLQ_dispatch_f(struct VSL_data *vsl, struct VSL_cursor *cp[],
-    void *priv);
+typedef int VSLQ_dispatch_f(struct VSL_data *vsl,
+    struct VSL_transaction *trans[], void *priv);
 	/*
 	 * The callback function type for use with VSLQ_Dispatch.
 	 *
 	 * Arguments:
-	 *   vsl: The VSL_data context
-	 *  cp[]: A NULL terminated array of pointer to cursors. Each cursor
-	 *        will iterate over the log records of a single VXID
-	 *  priv: The priv argument from VSL_Dispatch
+	 *      vsl: The VSL_data context
+	 *  trans[]: A NULL terminated array of pointers to VSL_transaction.
+	 *     priv: The priv argument from VSL_Dispatch
 	 *
 	 * Return value:
 	 *     0: OK - continue
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 5d8101b..df2e644 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -102,10 +102,10 @@ LIBVARNISHAPI_1.3 {
 	VSL_DeleteCursor;
 	VSL_Next;
 	VSL_Match;
-	VSL_PrintVXID;
-	VSL_PrintLevel;
+	VSL_Print;
+	VSL_PrintTerse;
 	VSL_PrintAll;
-	VSL_PrintSet;
+	VSL_PrintTransactions;
 	VSLQ_New;
 	VSLQ_Delete;
 	VSLQ_Dispatch;
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 25e1898..a9b1726 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -154,15 +154,30 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c)
 	return (1);
 }
 
+const char *VSL_transactions[256] = {
+	/*                 12345678901234 */
+	[VSL_t_unknown] = "<< Unknown  >>",
+	[VSL_t_sess]	= "<< Session  >>",
+	[VSL_t_req]	= "<< Request  >>",
+	[VSL_t_esireq]	= "<< ESI-req  >>",
+	[VSL_t_bereq]	= "<< BeReq    >>",
+	[VSL_t_raw]	= "<< Record   >>",
+};
+
+#define VSL_PRINT(...)					\
+	do {						\
+		if (0 > fprintf(__VA_ARGS__))		\
+			return (-5);			\
+	} while (0)
+
 int
-VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
+VSL_Print(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 {
 	enum VSL_tag_e tag;
 	uint32_t vxid;
 	unsigned len;
 	const char *data;
 	int type;
-	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	if (c == NULL || c->rec.ptr == NULL)
@@ -177,76 +192,51 @@ VSL_PrintVXID(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	data = VSL_CDATA(c->rec.ptr);
 
 	if (tag == SLT_Debug) {
-		i = fprintf(fo, "%10u %-15s %c \"", vxid, VSL_tags[tag], type);
-		if (i < 0)
-			return (-5);
+		VSL_PRINT(fo, "%10u %-14s %c \"", vxid, VSL_tags[tag], type);
 		while (len-- > 0) {
 			if (*data >= ' ' && *data <= '~')
-				i = fprintf(fo, "%c", *data);
+				VSL_PRINT(fo, "%c", *data);
 			else
-				i = fprintf(fo, "%%%02x", (unsigned char)*data);
-			if (i < 0)
-				return (-5);
+				VSL_PRINT(fo, "%%%02x", (unsigned char)*data);
 			data++;
 		}
-		i = fprintf(fo, "\"\n");
-		if (i < 0)
-			return (-5);
-		return (0);
-	}
+		VSL_PRINT(fo, "\"\n");
+	} else
+		VSL_PRINT(fo, "%10u %-14s %c %.*s\n", vxid, VSL_tags[tag],
+		    type, (int)len, data);
 
-	i = fprintf(fo, "%10u %-15s %c %.*s\n",
-	    vxid, VSL_tags[tag], type, (int)len, data);
-	if (i < 0)
-		return (-5);
 	return (0);
 }
 
 int
-VSL_PrintLevel(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
+VSL_PrintTerse(struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 {
 	enum VSL_tag_e tag;
-	unsigned len, lvl;
+	unsigned len;
 	const char *data;
-	int type;
-	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	if (c == NULL || c->rec.ptr == NULL)
 		return (0);
 	if (fo == NULL)
 		fo = stdout;
-
 	tag = VSL_TAG(c->rec.ptr);
 	len = VSL_LEN(c->rec.ptr);
-	type = VSL_CLIENT(c->rec.ptr) ? 'c' : VSL_BACKEND(c->rec.ptr) ?
-	    'b' : '-';
 	data = VSL_CDATA(c->rec.ptr);
-	lvl = c->level;
+
 	if (tag == SLT_Debug) {
-		i = fprintf(fo, "%2u %-15s %c \"", lvl, VSL_tags[tag],
-		    type);
-		if (i < 0)
-			return (-5);
+		VSL_PRINT(fo, "%-14s \"", VSL_tags[tag]);
 		while (len-- > 0) {
 			if (*data >= ' ' && *data <= '~')
-				i = fprintf(fo, "%c", *data);
+				VSL_PRINT(fo, "%c", *data);
 			else
-				i = fprintf(fo, "%%%02x",
-				    (unsigned char)*data);
-			if (i < 0)
-				return (-5);
+				VSL_PRINT(fo, "%%%02x", (unsigned char)*data);
 			data++;
 		}
-		i = fprintf(fo, "\"\n");
-		if (i < 0)
-			return (-5);
-		return (0);
-	}
-	i = fprintf(fo, "%2u %-15s %c %.*s\n",
-	    lvl, VSL_tags[tag], type, (int)len, data);
-	if (i < 0)
-		return (-5);
+		VSL_PRINT(fo, "\"\n");
+	} else
+		VSL_PRINT(fo, "%-14s %.*s\n", VSL_tags[tag], (int)len, data);
+
 	return (0);
 }
 
@@ -257,46 +247,76 @@ VSL_PrintAll(struct VSL_data *vsl, struct VSL_cursor *c, void *fo)
 
 	if (c == NULL)
 		return (0);
-	if (c->vxid >= 0) {
-		i = fprintf(fo, "vv VXID: %11u vv\n", c->vxid);
-		if (i < 0)
-			return (-5);
-	}
 	while (1) {
 		i = VSL_Next(c);
 		if (i <= 0)
 			return (i);
 		if (!VSL_Match(vsl, c))
 			continue;
-		if (c->vxid < 0)
-			i = VSL_PrintVXID(vsl, c, fo);
-		else
-			i = VSL_PrintLevel(vsl, c, fo);
+		i = VSL_Print(vsl, c, fo);
 		if (i != 0)
 			return (i);
 	}
 }
 
 int
-VSL_PrintSet(struct VSL_data *vsl, struct VSL_cursor *cp[], void *fo)
+VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction *pt[],
+    void *fo)
 {
+	struct VSL_transaction *t;
 	int i;
 	int delim = 0;
-	struct VSL_cursor *c;
+	int verbose;
+
+	(void)vsl;
+	if (pt[0] == NULL)
+		return (0);
+
+	t = pt[0];
+	while (t) {
+		verbose = 0;
+		if (t->level == 0 || vsl->v_opt)
+			verbose = 1;
 
-	c = cp[0];
-	while (c) {
-		if (c->vxid >= 0)
+		if (t->level) {
+			/* Print header */
+			if (t->level > 3)
+				VSL_PRINT(fo, "*%1.1u* ", t->level);
+			else
+				VSL_PRINT(fo, "%-3.*s ", t->level, "***");
+			VSL_PRINT(fo, "%*.s%-14s %*.s%-10u\n",
+			    verbose ? 10 + 1 : 0, " ",
+			    VSL_transactions[t->type],
+			    verbose ? 1 + 1 : 0, " ",
+			    t->vxid);
 			delim = 1;
-		i = VSL_PrintAll(vsl, c, fo);
-		if (i)
-			return (i);
-		c = *++cp;
-	}
-	if (delim) {
-		i = fprintf(fo, "\n");
-		if (i < 0)
-			return (-5);
+		}
+
+		while (1) {
+			/* Print records */
+			i = VSL_Next(t->c);
+			if (i < 0)
+				return (i);
+			if (i == 0)
+				break;
+			if (!VSL_Match(vsl, t->c))
+				continue;
+			if (t->level > 3)
+				VSL_PRINT(fo, "-%1.1u- ", t->level);
+			else if (t->level)
+				VSL_PRINT(fo, "%-3.*s ", t->level, "---");
+			if (verbose)
+				i = VSL_Print(vsl, t->c, fo);
+			else
+				i = VSL_PrintTerse(vsl, t->c, fo);
+			if (i != 0)
+				return (i);
+		}
+		t = *++pt;
 	}
+
+	if (delim)
+		VSL_PRINT(fo, "\n");;
+
 	return (0);
 }
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index eb9e543..b2f9fb7 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -80,6 +80,8 @@ struct VSL_data {
 	/* Bitmaps of -ix selected tags */
 	struct vbitmap			*vbm_select;
 	struct vbitmap			*vbm_supress;
+
+	int				v_opt;
 };
 
 /* vsl_query.c */
@@ -87,4 +89,4 @@ struct vslq_query;
 struct vslq_query *vslq_newquery(struct VSL_data *vsl,
     enum VSL_grouping_e grouping, const char *query);
 void vslq_deletequery(struct vslq_query **pquery);
-int vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[]);
+int vslq_runquery(struct vslq_query *query, struct VSL_transaction *ptrans[]);
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 72c1f22..fcbc528 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -131,6 +131,7 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 {
 	switch (opt) {
 	case 'i': case'x': return (vsl_ix_arg(vsl, opt, arg));
+	case 'v': vsl->v_opt = 1; return (1);
 	default:
 		return (0);
 	}
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 01808a4..9bee281 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -250,8 +250,6 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 		vsl_diag(vsl, "Out of memory\n");
 		return (NULL);
 	}
-	c->c.c.vxid = -1;	/* N/A to this cursor type */
-	c->c.c.shmptr_ok = 1;
 	c->c.magic = VSLC_MAGIC;
 	c->c.tbl = & vslc_vsm_tbl;
 
@@ -409,7 +407,6 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 		vsl_diag(vsl, "Out of memory\n");
 		return (NULL);
 	}
-	c->c.c.vxid = -1;	/* N/A to this cursor type */
 	c->c.magic = VSLC_MAGIC;
 	c->c.tbl = &vslc_file_tbl;
 
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 98a438d..7053500 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -50,21 +50,6 @@
 
 struct vtx;
 
-enum vtx_type_e {
-	vtx_t_unknown,
-	vtx_t_sess,
-	vtx_t_req,
-	vtx_t_esireq,
-	vtx_t_bereq,
-};
-
-enum vtx_link_e {
-	vtx_l_sess,
-	vtx_l_req,
-	vtx_l_esireq,
-	vtx_l_bereq,
-};
-
 struct vslc_raw {
 	struct vslc		c;
 	unsigned		magic;
@@ -111,7 +96,7 @@ struct vtx {
 #define VTX_F_COMPLETE		0x1
 #define VTX_F_READY		0x2
 
-	enum vtx_type_e		type;
+	enum VSL_transaction_e	type;
 
 	struct vtx		*parent;
 	VTAILQ_HEAD(,vtx)	child;
@@ -307,7 +292,7 @@ vtx_new(struct VSLQ *vslq)
 	vtx->key.vxid = 0;
 	vtx->t_start = VTIM_mono();
 	vtx->flags = 0;
-	vtx->type = vtx_t_unknown;
+	vtx->type = VSL_t_unknown;
 	vtx->parent = NULL;
 	VTAILQ_INIT(&vtx->child);
 	vtx->n_child = 0;
@@ -399,7 +384,6 @@ vtx_lori(struct VSLQ *vslq, unsigned vxid)
 	vtx = vtx_new(vslq);
 	AN(vtx);
 	vtx->key.vxid = vxid;
-	vtx->c.c.c.vxid = vxid;
 	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
 	return (vtx);
 }
@@ -486,7 +470,7 @@ vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
 	AN(vtx->flags & VTX_F_COMPLETE);
 	AZ(vtx->flags & VTX_F_READY);
 
-	if (vtx->type == vtx_t_unknown)
+	if (vtx->type == VSL_t_unknown)
 		vtx_diag(vtx, "vtx of unknown type marked complete");
 
 	ready = vtx;
@@ -511,14 +495,14 @@ vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
 }
 
 static int
-vtx_parsetag_bl(const char *str, unsigned strlen, enum vtx_type_e *ptype,
+vtx_parsetag_bl(const char *str, unsigned strlen, enum VSL_transaction_e *ptype,
     unsigned *pvxid)
 {
 	char ibuf[strlen + 1];
 	char tbuf[7];
 	unsigned vxid;
 	int i;
-	enum vtx_type_e type = vtx_t_unknown;
+	enum VSL_transaction_e type = VSL_t_unknown;
 
 	AN(str);
 	memcpy(ibuf, str, strlen);
@@ -527,13 +511,13 @@ vtx_parsetag_bl(const char *str, unsigned strlen, enum vtx_type_e *ptype,
 	if (i < 1)
 		return (-1);
 	if (!strcmp(tbuf, "sess"))
-		type = vtx_t_sess;
+		type = VSL_t_sess;
 	else if (!strcmp(tbuf, "req"))
-		type = vtx_t_req;
+		type = VSL_t_req;
 	else if (!strcmp(tbuf, "esireq"))
-		type = vtx_t_esireq;
+		type = VSL_t_esireq;
 	else if (!strcmp(tbuf, "bereq"))
-		type = vtx_t_bereq;
+		type = VSL_t_bereq;
 	else
 		return (-1);
 	if (i == 1)
@@ -564,7 +548,7 @@ static int
 vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 {
 	int i;
-	enum vtx_type_e type;
+	enum VSL_transaction_e type;
 	unsigned p_vxid;
 	struct vtx *p_vtx;
 
@@ -578,8 +562,8 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		return (vtx_diag_tag(vtx, ptr, "parse error"));
 
 	/* Check/set vtx type */
-	assert(type != vtx_t_unknown);
-	if (vtx->type != vtx_t_unknown && vtx->type != type)
+	assert(type != VSL_t_unknown);
+	if (vtx->type != VSL_t_unknown && vtx->type != type)
 		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
 	vtx->type = type;
 
@@ -588,7 +572,7 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 
 	if (vslq->grouping == VSL_g_vxid)
 		return (0);	/* No links */
-	if (vslq->grouping == VSL_g_request && vtx->type == vtx_t_req)
+	if (vslq->grouping == VSL_g_request && vtx->type == VSL_t_req)
 		return (0);	/* No links */
 
 	/* Lookup and check parent vtx */
@@ -612,7 +596,7 @@ static int
 vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 {
 	int i;
-	enum vtx_type_e c_type;
+	enum VSL_transaction_e c_type;
 	unsigned c_vxid;
 	struct vtx *c_vtx;
 
@@ -628,7 +612,7 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 
 	if (vslq->grouping == VSL_g_vxid)
 		return (0);	/* No links */
-	if (vslq->grouping == VSL_g_request && vtx->type == vtx_t_sess)
+	if (vslq->grouping == VSL_g_request && vtx->type == VSL_t_sess)
 		return (0);	/* No links */
 
 	/* Lookup and check child vtx */
@@ -641,7 +625,7 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
 	if (c_vtx->flags & VTX_F_READY)
 		return (vtx_diag_tag(vtx, ptr, "link too late"));
-	if (c_vtx->type != vtx_t_unknown && c_vtx->type != c_type)
+	if (c_vtx->type != VSL_t_unknown && c_vtx->type != c_type)
 		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
 	c_vtx->type = c_type;
 
@@ -669,7 +653,7 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 			continue;
 		}
 
-		if (vtx->type == vtx_t_unknown && tag != SLT_Begin)
+		if (vtx->type == VSL_t_unknown && tag != SLT_Begin)
 			vtx_diag_tag(vtx, ptr, "early log rec");
 
 		switch (tag) {
@@ -719,7 +703,9 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
     void *priv)
 {
 	unsigned n = vtx->n_descend + 1;
-	struct vslc_vtx *cp[n + 1];
+	struct VSL_transaction trans[n];
+	struct VSL_transaction *ptrans[n + 1];
+	struct vslc_vtx *c;
 	unsigned i, j;
 
 	AN(vslq);
@@ -728,40 +714,46 @@ vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 	if (func == NULL)
 		return (0);
 	if (vslq->grouping == VSL_g_session &&
-	    vtx->type != vtx_t_sess)
+	    vtx->type != VSL_t_sess)
 		return (0);
 	if (vslq->grouping == VSL_g_request &&
-	    vtx->type != vtx_t_req)
+	    vtx->type != VSL_t_req)
 		return (0);
 
-	/* Build cursor array */
-	i = j = 0;
-	cp[i] = &vtx->c;
-	vslc_vtx_reset(cp[i]);
-	cp[i]->c.c.level = 0;
-	i++;
+	/* Build transaction array */
+	vslc_vtx_reset(&vtx->c);
+	trans[0].level = 1;
+	trans[0].vxid = vtx->key.vxid;
+	trans[0].type = vtx->type;
+	trans[0].c = &vtx->c.c.c;
+	i = 1;
+	j = 0;
 	while (j < i) {
-		vtx = VTAILQ_FIRST(&cp[j]->vtx->child);
-		while (vtx) {
+		CAST_OBJ_NOTNULL(c, (void *)trans[j].c, VSLC_VTX_MAGIC);
+		VTAILQ_FOREACH(vtx, &c->vtx->child, list_child) {
 			assert(i < n);
-			cp[i] = &vtx->c;
-			vslc_vtx_reset(cp[i]);
-			cp[i]->c.c.level = cp[j]->c.c.level + 1;
+			vslc_vtx_reset(&vtx->c);
+			trans[i].level = trans[j].level + 1;
+			trans[i].vxid = vtx->key.vxid;
+			trans[i].type = vtx->type;
+			trans[i].c = &vtx->c.c.c;
 			i++;
-			vtx = VTAILQ_NEXT(vtx, list_child);
 		}
 		j++;
 	}
 	assert(i == n);
-	cp[i] = NULL;
+
+	/* Build pointer array */
+	for (i = 0; i < n; i++)
+		ptrans[i] = &trans[i];
+	ptrans[i] = NULL;
 
 	/* Query test goes here */
-	if (vslq->query != NULL &&
-	    vslq_runquery(vslq->query, (struct VSL_cursor **)cp))
+	if (vslq->query != NULL && vslq_runquery(vslq->query, ptrans))
 		return (0);
 
 	/* Callback */
-	return ((func)(vslq->vsl, (struct VSL_cursor **)cp, priv));
+	return ((func)(vslq->vsl, ptrans, priv));
 }
 
 struct VSLQ *
@@ -835,20 +827,23 @@ static int
 vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
 	struct vslc_raw rawc;
+	struct VSL_transaction trans;
+	struct VSL_transaction *ptrans[2];
 	struct VSL_cursor *c;
-	struct VSL_cursor *pc[2];
 	int i;
 
 	assert(vslq->grouping == VSL_g_raw);
 	c = vslq->c;
 
 	memset(&rawc, 0, sizeof rawc);
-	rawc.c.c.vxid = -1;
 	rawc.c.magic = VSLC_MAGIC;
 	rawc.c.tbl = &vslc_raw_tbl;
 	rawc.magic = VSLC_RAW_MAGIC;
-	pc[0] = &rawc.c.c;
-	pc[1] = NULL;
+	trans.level = 0;
+	trans.type = VSL_t_raw;
+	trans.c = &rawc.c.c;
+	ptrans[0] = &trans;
+	ptrans[1] = NULL;
 
 	while (1) {
 		i = VSL_Next(c);
@@ -861,11 +856,14 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		rawc.len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
 		rawc.next = rawc.start;
 		rawc.c.c.rec.ptr = NULL;
+		trans.vxid = VSL_ID(c->rec.ptr);
 
 		/* Query check goes here */
-		i = 0;
-		if (vslq->query == NULL ? 1 : vslq_runquery(vslq->query, pc))
-			i = (func)(vslq->vsl, pc, priv);
+		if (vslq->query != NULL && vslq_runquery(vslq->query, ptrans))
+			continue;
+
+		/* Callback */
+		i = (func)(vslq->vsl, ptrans, priv);
 		if (i)
 			break;
 	}
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 852f8d3..20efff4 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -88,8 +88,9 @@ vslq_deletequery(struct vslq_query **pquery)
 }
 
 int
-vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[])
+vslq_runquery(struct vslq_query *query, struct VSL_transaction *ptrans[])
 {
+	struct VSL_transaction *t;
 	struct VSL_cursor *c;
 	int i, len;
 	const char *data;
@@ -97,8 +98,9 @@ vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[])
 	CHECK_OBJ_NOTNULL(query, VSLQ_QUERY_MAGIC);
 	AN(query->regex);
 
-	c = cp[0];
-	while (c) {
+	t = ptrans[0];
+	while (t) {
+		c = t->c;
 		while (1) {
 			i = VSL_Next(c);
 			if (i == 0)
@@ -115,7 +117,7 @@ vslq_runquery(struct vslq_query *query, struct VSL_cursor *cp[])
 			}
 		}
 		AZ(VSL_ResetCursor(c));
-		c = *++cp;
+		t = *++ptrans;
 	}
 
 	return (0);



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] 445ba2b Implement VSL_ResetCursor
Message-ID: <E1Ucb5y-0004G6-Cg@project.varnish-software.com>

commit 445ba2bbf59e5bae7b1df283471d9de8d1251481
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 29 17:12:17 2013 +0200

    Implement VSL_ResetCursor

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 6ee0e33..54014cd 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -121,6 +121,17 @@ void VSL_DeleteCursor(struct VSL_cursor *c);
 	 * Delete the cursor pointed to by c
 	 */
 
+int VSL_ResetCursor(struct VSL_cursor *c);
+	/*
+	 * Reset the cursor position to the head, so that the next call to
+	 * VSL_Next returns the first record. For VSM cursor, it will
+	 * point close to the head of the log, but at least 2 segments away
+	 * from the tail.
+	 *
+	 * Return values:
+	 *    -1: Operation not supported
+	 */
+
 int VSL_Next(struct VSL_cursor *c);
 	/*
 	 * Return raw pointer to next VSL record.
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 8b70098..f03812e 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -40,6 +40,7 @@ int vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
 
 typedef void vslc_delete_f(void *);
 typedef int vslc_next_f(void *);
+typedef int vslc_reset_f(void *);
 
 struct vslc {
 	struct VSL_cursor		c;
@@ -48,6 +49,7 @@ struct vslc {
 
 	vslc_delete_f			*delete;
 	vslc_next_f			*next;
+	vslc_reset_f			*reset;
 };
 
 struct VSL_data {
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 65262d1..f0d535b 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -144,13 +144,37 @@ vslc_vsm_next(void *cursor)
 	}
 }
 
+static int
+vslc_vsm_reset(void *cursor)
+{
+	struct vslc_vsm *c;
+	unsigned segment;
+
+	CAST_OBJ_NOTNULL(c, cursor, VSLC_VSM_MAGIC);
+
+	/*
+	 * Starting (VSL_SEGMENTS - 3) behind varnishd. This way
+	 * even if varnishd wraps immediately, we'll still have a
+	 * full segment worth of log before the general constraint
+	 * of at least 2 segments apart will be broken
+	 */
+	segment = (c->head->segment + 3) % VSL_SEGMENTS;
+	if (c->head->segments[segment] < 0)
+		segment = 0;
+	assert(c->head->segments[segment] >= 0);
+	c->next = c->head->log + c->head->segments[segment];
+	c->seq = c->head->seq;
+	c->c.c.ptr = NULL;
+
+	return (0);
+}
+
 struct VSL_cursor *
 VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 {
 	struct vslc_vsm *c;
 	struct VSM_fantom vf;
 	struct VSL_head *head;
-	unsigned segment;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	CHECK_OBJ_NOTNULL(vsm, VSM_MAGIC);
@@ -178,6 +202,7 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 	c->c.magic = VSLC_MAGIC;
 	c->c.delete = vslc_vsm_delete;
 	c->c.next = vslc_vsm_next;
+	c->c.reset = vslc_vsm_reset;
 
 	c->vsm = vsm;
 	c->vf = vf;
@@ -190,20 +215,9 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 		c->next = c->head->log + c->head->segments[c->head->segment];
 		while (c->next < c->end && *c->next != VSL_ENDMARKER)
 			c->next = VSL_NEXT(c->next);
-	} else {
-		/*
-		 * Starting (VSL_SEGMENTS - 3) behind varnishd. This way
-		 * even if varnishd wraps immediately, we'll still have a
-		 * full segment worth of log before the general constraint
-		 * of at least 2 segments apart will be broken
-		 */
-		segment = (c->head->segment + 3) % VSL_SEGMENTS;
-		if (c->head->segments[segment] < 0)
-			segment = 0;
-		assert(c->head->segments[segment] >= 0);
-		c->next = c->head->log + c->head->segments[segment];
-	}
-	c->seq = c->head->seq;
+		c->seq = c->head->seq;
+	} else
+		AZ(vslc_vsm_reset(&c->c));
 
 	return (&c->c.c);
 }
@@ -284,6 +298,14 @@ vslc_file_next(void *cursor)
 	return (1);
 }
 
+static int
+vslc_file_reset(void *cursor)
+{
+	(void)cursor;
+	/* XXX: Implement me */
+	return (-1);
+}
+
 struct VSL_cursor *
 VSL_CursorFile(struct VSL_data *vsl, const char *name)
 {
@@ -328,6 +350,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 	c->c.magic = VSLC_MAGIC;
 	c->c.delete = vslc_file_delete;
 	c->c.next = vslc_file_next;
+	c->c.reset = vslc_file_reset;
 
 	c->fd = fd;
 	c->buflen = BUFSIZ;
@@ -349,6 +372,17 @@ VSL_DeleteCursor(struct VSL_cursor *cursor)
 }
 
 int
+VSL_ResetCursor(struct VSL_cursor *cursor)
+{
+	struct vslc *c;
+
+	CAST_OBJ_NOTNULL(c, (void *)cursor, VSLC_MAGIC);
+	if (c->reset == NULL)
+		return (-1);
+	return ((c->reset)(c));
+}
+
+int
 VSL_Next(struct VSL_cursor *cursor)
 {
 	struct vslc *c;



From martin at varnish-cache.org  Wed May 15 12:46:15 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:15 +0200
Subject: [master] ba19e06 Documentation updates
Message-ID: <E1Ucb5z-0004Ia-2X@project.varnish-software.com>

commit ba19e06a10f95672415905627f813c0a26ddbae1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 14:41:16 2013 +0200

    Documentation updates

diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 550ca55..e77ef64 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -60,30 +60,12 @@ int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
 	 */
 
 struct VSC_C_mgt *VSC_Mgt(struct VSM_data *vd, struct VSM_fantom *fantom);
-	/*
-	 * Looks up and returns the management stats structure. If fantom
-	 * is non-NULL, it can if successful later be used with
-	 * VSM_StillValid. Returns NULL until the management process has
-	 * finished initialization.
-	 *
-	 * The returned structure is valid for at most 60 seconds after
-	 * VSM_StillValid(,fantom) starts returning VSM_invalid. Using the
-	 * pointer after this event gives undefined behavior.
-	 *
-	 * Arguments:
-	 *	    vd: The VSM_data context
-	 *	fantom: Pointer to a fantom. Can be NULL.
-	 *
-	 * Return values:
-	 *      NULL: Failure
-	 *  non-NULL: Success
-	 */
-
 struct VSC_C_main *VSC_Main(struct VSM_data *vd, struct VSM_fantom *fantom);
 	/*
-	 * Looks up and returns the main stats structure. If fantom is
-	 * non-NULL, it can if successful later be used with
-	 * VSM_StillValid. Returns NULL until child has been started.
+	 * Looks up and returns the management stats and the child main
+	 * stats structure. If fantom is non-NULL, it can later be used
+	 * with VSM_StillValid. Returns NULL until the management process
+	 * has finished initialization or the child has been started.
 	 *
 	 * The returned structure is valid for at most 60 seconds after
 	 * VSM_StillValid(,fantom) starts returning VSM_invalid. Using the
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 42d6e8d..aa58835 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -151,9 +151,28 @@ enum VSM_valid_e {
 enum VSM_valid_e VSM_StillValid(const struct VSM_data *vd,
     struct VSM_fantom *vf);
 	/*
-	 * This is a cheap syscall-less check to see if the fantom is still
-	 * valid.  Further checking with VSM_Abandoned() may be a good
-	 * idea.
+	 * Check the validity of a previously looked up VSM_fantom.
+	 *
+	 * VSM_invalid means that the SHM chunk this fantom points to does
+	 * not exist in the log file any longer. Using the fantom's
+	 * pointer gives undefined results. Further checking with
+	 * VSM_Abandoned() may be a good idea.
+	 *
+	 * VSM_valid means that the SHM structure has not changed since
+	 * the fantom was looked up or since the last call to
+	 * VSM_StillValid().
+	 *
+	 * VSM_similar means that the SHM structure has changed, but there
+	 * is still a valid chunk present with the same the same type and
+	 * identifier. The fantom's pointers and dimensions haven't
+	 * changed. The next call to VSM_StillValid() on this fantom will
+	 * return VSM_valid.
+	 *
+	 * Applications using the fantom to monitor a single chunk can
+	 * treat VSM_similar as equal to VSM_valid.  Applications using a
+	 * fantom to monitor the SHM file for new or removed chunks,
+	 * should reiterate over the chunks on VSM_similar as the
+	 * structure has changed.
 	 *
 	 * Return:
 	 *   VSM_invalid: fantom is not valid any more.



From martin at varnish-cache.org  Wed May 15 12:46:14 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 14:46:14 +0200
Subject: [master] f370e8d Disable building varnishtest while VSL is in flux
Message-ID: <E1Ucb5y-0004FJ-2x@project.varnish-software.com>

commit f370e8d14f0fca615fbcccdafac6331dcae5d9a1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 29 14:22:14 2013 +0200

    Disable building varnishtest while VSL is in flux

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 2c1b6a6..4adc0bb 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -3,7 +3,7 @@
 # Disabling building of the tools while api is in flux
 
 #SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
-SUBDIRS = varnishd varnishtest
+SUBDIRS = varnishd
 
 if HAVE_CURSES
 #SUBDIRS += varnishhist varnishstat varnishtop



From martin at varnish-cache.org  Wed May 15 13:29:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 15:29:04 +0200
Subject: [master] 020d5e7 Fix up some Makefile things for distcheck to pass
Message-ID: <E1UcblQ-0000i5-3x@project.varnish-software.com>

commit 020d5e7c8818d8bac00f825cf675e156947ba164
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 15:27:48 2013 +0200

    Fix up some Makefile things for distcheck to pass

diff --git a/configure.ac b/configure.ac
index 4ae4e4f..f49f3fe 100644
--- a/configure.ac
+++ b/configure.ac
@@ -582,12 +582,8 @@ AC_CONFIG_FILES([
     bin/varnishadm/Makefile
     bin/varnishd/Makefile
     bin/varnishlog/Makefile
-    bin/varnishhist/Makefile
-    bin/varnishncsa/Makefile
-    bin/varnishreplay/Makefile
     bin/varnishstat/Makefile
     bin/varnishtest/Makefile
-    bin/varnishtop/Makefile
     doc/Makefile
     doc/sphinx/Makefile
     doc/sphinx/conf.py
diff --git a/include/Makefile.am b/include/Makefile.am
index ff33292..3f33a3e 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -21,6 +21,7 @@ nobase_pkginclude_HEADERS = \
 	tbl/vsc_types.h \
 	tbl/vsc_all.h \
 	tbl/vsc_fields.h \
+	tbl/vsc_levels.h \
 	tbl/vsc_f_main.h \
 	tbl/vsl_tags.h \
 	tbl/vsl_tags_http.h \



From martin at varnish-cache.org  Wed May 15 13:29:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 15:29:04 +0200
Subject: [master] 88db48c Cast through void* to remove clang warning
Message-ID: <E1UcblQ-0000iA-5i@project.varnish-software.com>

commit 88db48cb016857add18b025d961e13fcb2a13a33
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 15:28:12 2013 +0200

    Cast through void* to remove clang warning

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 60679fd..ec73a98 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -338,14 +338,14 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 			return (0); /* free'd during iteration */
 		if (vf->chunk->next == 0)
 			return (0); /* last */
-		c = (struct VSM_chunk *)(vd->b + vf->chunk->next);
+		c = (struct VSM_chunk *)(void*)(vd->b + vf->chunk->next);
 		assert(c != vf->chunk);
 	} else if (vd->head->first == 0) {
 		return (0);	/* empty vsm */
 	} else {
 		/* get first chunk */
 		AZ(vf->chunk);
-		c = (struct VSM_chunk *)(vd->b + vd->head->first);
+		c = (struct VSM_chunk *)(void*)(vd->b + vd->head->first);
 	}
 	AN(c);
 	if (memcmp(c->marker, VSM_CHUNK_MARKER, sizeof c->marker))



From martin at varnish-cache.org  Wed May 15 13:34:44 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 15:34:44 +0200
Subject: [master] eba8c98 Rename a variable to get rid of 'shadows global
 declaration' warning
Message-ID: <E1Ucbqu-000139-Cv@project.varnish-software.com>

commit eba8c98e4208bc36aa7ffb71042d146f948705e7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 15:34:09 2013 +0200

    Rename a variable to get rid of 'shadows global declaration' warning

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 7053500..8256352 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -495,18 +495,18 @@ vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
 }
 
 static int
-vtx_parsetag_bl(const char *str, unsigned strlen, enum VSL_transaction_e *ptype,
+vtx_parsetag_bl(const char *str, unsigned len, enum VSL_transaction_e *ptype,
     unsigned *pvxid)
 {
-	char ibuf[strlen + 1];
+	char ibuf[len + 1];
 	char tbuf[7];
 	unsigned vxid;
 	int i;
 	enum VSL_transaction_e type = VSL_t_unknown;
 
 	AN(str);
-	memcpy(ibuf, str, strlen);
-	ibuf[strlen] = '\0';
+	memcpy(ibuf, str, len);
+	ibuf[len] = '\0';
 	i = sscanf(ibuf, "%6s %u", tbuf, &vxid);
 	if (i < 1)
 		return (-1);



From martin at varnish-cache.org  Wed May 15 13:41:23 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 15:41:23 +0200
Subject: [master] f02f714 Remove file reading code until finished,
 to get rid of compiler warnings
Message-ID: <E1UcbxL-0001SL-Ek@project.varnish-software.com>

commit f02f714a2a2debe92092d8fb126e1e17a8aed4c9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 15:41:02 2013 +0200

    Remove file reading code until finished, to get rid of compiler warnings

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index c48bb1b..f08407c 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -81,7 +81,6 @@ main(int argc, char * const *argv)
 	char optchar;
 	int d_opt = 0;
 	char *g_arg = NULL;
-	char *r_arg = NULL;
 
 	struct VSL_data *vsl;
 	struct VSM_data *vsm;
@@ -108,10 +107,6 @@ main(int argc, char * const *argv)
 			/* Instance name */
 			if (VSM_n_Arg(vsm, optarg) > 0)
 				break;
-		case 'r':
-			/* Read from file */
-			r_arg = optarg;
-			break;
 		default:
 			if (!VSL_Arg(vsl, optchar, optarg))
 				usage();
@@ -132,16 +127,11 @@ main(int argc, char * const *argv)
 	}
 
 	/* Create cursor */
-	if (r_arg) {
-		/* XXX */
-		error (1, "-r not implemented");
-	} else {
-		if (VSM_Open(vsm))
-			error(1, "VSM_Open: %s", VSM_Error(vsm));
-		if (!(c = VSL_CursorVSM(vsl, vsm, !d_opt)))
-			error(1, "VSL_CursorVSM: %s", VSL_Error(vsl));
-	}
-	AN(c);
+	if (VSM_Open(vsm))
+		error(1, "VSM_Open: %s", VSM_Error(vsm));
+	c = VSL_CursorVSM(vsl, vsm, !d_opt);
+	if (c == NULL)
+		error(1, "VSL_CursorVSM: %s", VSL_Error(vsl));
 
 	/* Create query */
 	q = VSLQ_New(vsl, &c, grouping, argv[optind]);
@@ -150,7 +140,7 @@ main(int argc, char * const *argv)
 	AZ(c);
 
 	while (1) {
-		while (!r_arg && q == NULL) {
+		while (q == NULL) {
 			VTIM_sleep(0.1);
 			if (VSM_Open(vsm)) {
 				VSM_ResetError(vsm);



From martin at varnish-cache.org  Wed May 15 13:47:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 15:47:19 +0200
Subject: [master] 2262b6a Rename a variable to get rid of 'shadows global
 declaration' warning
Message-ID: <E1Ucc35-0002QD-0u@project.varnish-software.com>

commit 2262b6a96e0c8e816fbdf14bc70703504e6e63cc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 15:47:12 2013 +0200

    Rename a variable to get rid of 'shadows global declaration' warning

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index a2bf8a3..6fc7c8a 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -841,7 +841,7 @@ void
 do_curses(struct VSM_data *vd, int delay)
 {
 	struct pollfd pollfd;
-	long timeout;
+	long t;
 	int ch;
 	double now;
 	struct VSM_fantom f_main, f_mgt, f_iter;
@@ -885,9 +885,9 @@ do_curses(struct VSM_data *vd, int delay)
 		if (redraw)
 			draw_screen();
 
-		timeout = (t_sample + interval - now) * 1000;
-		if (timeout > 0)
-			(void)poll(&pollfd, 1, timeout);
+		t = (t_sample + interval - now) * 1000;
+		if (t > 0)
+			(void)poll(&pollfd, 1, t);
 
 		switch (ch = wgetch(w_status)) {
 		case ERR:



From martin at varnish-cache.org  Wed May 15 14:30:42 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 15 May 2013 16:30:42 +0200
Subject: [master] dd9838c Update the varnishlog documentation
Message-ID: <E1Uccj4-0004te-Vb@project.varnish-software.com>

commit dd9838c410f8050ef25dd22aa0b510bee1f0729f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 15 16:29:38 2013 +0200

    Update the varnishlog documentation

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 9a2868f..2cd4176 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -10,79 +10,321 @@ Display Varnish logs
 
 :Author: Dag-Erling Sm?rgrav
 :Author: Per Buer
-:Date:   2010-05-31
-:Version: 0.2
+:Author: Martin Blix Grydeland
+:Date:   2013-05-15
+:Version: 0.3
 :Manual section: 1
 
 
 SYNOPSIS
 ========
 
-varnishlog [-a] [-b] [-C] [-c] [-D] [-d] [-I regex] [-i tag] [-k keep] 
-[-n varnish_name] [-o] [-O] [-m tag:regex ...] [-P file] [-r file] [-s num] [-u] [-V]
-[-w file] [-X regex] [-x tag]
+varnishlog [-a] [-b] [-c] [-C] [-d] [-D] [-i tag] [-I [tag:]regex] [-k
+keep] [-n varnish_name] [-P file] [-r file] [--raw] [-s num] [-S] [-u]
+[-v] [-V] [-w file] [-x tag] [-X [tag:]regex] <query expression>
+
+OPTIONS
+=======
+
+The following options are available:
+
+-a
+
+	When writing to a file, append to it rather than overwrite it.
+
+	XXX: Not yet implemented
+
+-b
+
+	Only show backend transactions. If neither -b nor -c is
+	specified, varnishlog acts as if they both were specified.
+
+	XXX: Not yet implemented
+
+-c
+
+	Only show client transactions. If neither -b nor -c is
+	specified, varnishlog acts as if they both were present.
+
+	XXX: Not yet implemented
+
+-C
+
+	Ignore case when matching regular expressions.
+
+	XXX: Not yet implemented
+
+-D
+
+	Daemonize.
+
+	XXX: Not yet implemented
+
+-d
+
+	Process old log entries on startup. Nomally, varnishlog will
+	only process entries which are written to the log after it
+	starts.
+
+-g {session|request|vxid|raw}
+
+	The grouping of the log records. The default is to group by
+	request.
+
+-i tag
+
+	Output only this tag. Multiple -i options may be given.
+
+	If an -i option is the first of any -ix options, all tags are
+	disabled for output before -ix processing.
+
+-I [tag:]regex
+
+	Output only records matching this regular expression. If tag
+	is given, limit the regex matching to records of that
+	tag. Multiple -I options may be given.
+
+	XXX: Not yet implemented
+
+-k num
+
+	Only show the first num log transactions (or log records
+	in --raw mode)
+
+	XXX: Not yet implemented
+
+-n
+
+	Specifies the name of the varnishd instance to get logs
+	from. If -n is not specified, the host name is used.
+
+
+-P file
+
+	Write the process' PID to the specified file.
+
+	XXX: Not yet implemented
+
+-r file
+
+	Read log entries from file instaed of shared memory
+
+	XXX: Not yet implemented
+
+-s num
+
+	Skip the first num log transactions (or log records if
+	in --raw mode)
+
+	XXX: Not yet implemented
+
+-u
+
+	Unbuffered output.
+
+	XXX: Not yet implemented
+
+-v
+
+	Use verbose output on set output, giving the VXID on every log
+	line. Without this option, the VXID will only be given on the
+	header of that transaction.
+
+-V
+
+	Display the version number and exit.
+
+	XXX: Not yet implemented
+
+-w file
+
+	Write log entries to file instead of displaying them.  The
+   	file will be overwritten unless the -a option was
+   	specified. If varnishlog receives a SIGHUP while writing to a
+   	file, it will reopen the file, allowing the old one to be
+   	rotated away.
+
+	XXX: Not yet implemented
+
+-x tag
+
+	Exclude log records of this tag. Multiple -x options may be
+	given.
+
+	If an -x option is the first of any -ix options, all tags are
+	enabled for output before -ix processing.
+
+-X [tag:]regex
+
+	Do not output log records matching this regex. If tag is
+	given, limit the regex matching to records of that tag.
+	Multiple -X options may be given.
+
+	XXX: Not yet implemented
+
 
 DESCRIPTION
 ===========
 
+Varnishlog is a utility to extract and query the Varnish shared memory
+log.
 
-The varnishlog utility reads and presents varnishd(1) shared memory logs.
+Varnishlog operates on transactions. A transaction is a set of log
+lines that belongs together, e.g. a client request. Varnishlog will
+monitor the log, and collect all log records that make up a
+transaction before reporting on that transaction. Transactions can
+also be grouped, meaning backend transactions are reported together
+with the client transaction that initiated it.
+
+The grouping levels are:
+
+* Session
+
+  All transactions initiated by a client connection is reported
+  together. All log data is buffered until the client connection is
+  closed.
+
+* Request
+
+  Transactions are grouped by request, where the set will include the
+  request itself, and any backend requests or ESI-subrequests. Session
+  data is not reported. This is the default.
+
+* VXID
+
+  Transactions are not grouped, so each VXID is reported in it's
+  entirity. Sessions, requests, ESI-requests and backend requests are
+  all reported individually. Non-transactional data is not reported
+  (VXID == 0).
+
+* Raw
+
+  Every log record will make up a transaction of it's own. All data,
+  including non-transactional data will be reported.
 
-The following options are available:
 
--a          When writing to a file, append to it rather than overwrite it.
+Grouping
+========
+
+When grouping transactions, there is a hirarchy structure showing
+which transaction initiated what. The level increases by one by a
+'initiated by' relation, so for example a backend transaction will
+have one higher level than the client transaction that initiated it on
+a cache miss.
+
+Example transaction hirarchy ::
+
+  Lvl 1: Client request (cache miss)
+    Lvl 2: Backend request
+    Lvl 2: ESI subrequest (cache miss)
+      Lvl 3: Backend request
+      Lvl 3: Backend request (VCL restart)
+      Lvl 3: ESI subrequest (cache miss)
+        Lvl 4: Backend request
+    Lvl 2: ESI subrequest (cache hit)
+
+Query operators will unless limited see a grouped set of transactions
+together, and matching will be done on any log line from the complete
+set. See QUERY LANGUAGE for how to limit a match to a specific part of
+the set.
+
+Running queries in session grouping mode can potentially consume a lot
+of memory.
+
+
+QUERY LANGUAGE
+==============
+
+XXX: As a POC only a single string is accepted as a query expression,
+and this will be used as a regular expression that will be matched
+against any log line of the set. The rest of the query language is yet
+to be implemented.
+
+The query expression is given as a single command line
+argument. Additional arguments will give an error.
+
+An expression consists of a single tag, or a comparison between a tag
+and a constant.
+
+A single tag expression is considered true if there is one or more
+records with that tag in the transaction.
 
--b          Include log entries which result from communication with a backend server.  
-	    If neither -b nor -c is specified, varnishlog acts as if they both were.
+For all comparisons, the LHS must be a tag, and the RHS must be a
+constant.
 
--C          Ignore case when matching regular expressions.
+Constants must be quoted if they contain whitespace. You can use
+either single or double quotes.
 
--c          Include log entries which result from communication with a client.  
-	    If neither -b nor -c is specified, varnishlog acts as if they both were.
+A comparison expression is true if the comparison is true for one or
+more records with that tag in the transaction.
 
--D          Daemonize.
+(be)?re(q|sp).(url|request|status|response) expands to their specific
+tags.
 
--d          Process old log entries on startup.  Normally, varnishlog will only process entries 
-	    which are written to the log after it starts.
+(be)?re(q|sp).http.<header> expands to their corresponding
+(Ber|R)(eq|esp)Header tag, and for this comparison the value will be
+s/^(?i)<header>: //
 
--I regex    Include log entries which match the specified regular expression.  If 
-   	    neither -I nor -i is specified, all log entries are included.
+<tag>{n} will only match on a transaction at the nth level (see
+grouping). Levels starts counting at 0. If n is followed by a '+',
+it will only match at level n or higher. If n is followed by a '-', it
+will only match at level n or lower.
 
--i tag      Include log entries with the specified tag.  If neither -I nor -i is specified, 
-   	    all log entries are included.
+<tag>[n] will consider the value of the tag to be a white-space
+separated field list, and extract the nth field for the comparison.
 
--k num      Only show the first num log records.
+<tag>#n adds a repetition counter to this match, and is true only if
+the match is true n times. n+ means n or more, n- means n or less.
 
--m tag:regex only list transactions where tag matches regex. Multiple
-            -m options are AND-ed together. Can not be combined with 
-	    -O.
+'==', '!=', '<', '<=', '>' and '>=' are numerical comparisons. Integer
+by default, or floating point if the RHS contains a dot. LHS will be
+transformed (atoi/atof) for comparison.
 
--n          Specifies the name of the varnishd instance to get logs from.  If
-	    -n is not specified, the host name is used.
+'eq' and 'ne' are for string comparison.
 
--o          Ignored for compatibility with earlier versions.
+'~' and '!~' are PCRE regular expression comparisons.
 
--O          Do not group log entries by request ID.  Can not be
-            combined with -m.
+'not' is for negation
 
--P file     Write the process's PID to the specified file.
+'and' is concatenation
 
--r file     Read log entries from file instead of shared memory.
+'or' is alteration
 
--s num      Skip the first num log records.
+'not' has highest precedence, alternation and concatenation have equal
+precedence and associate left to right. Paranthesis can be used to
+group statements.
 
--u          Unbuffered output.
+QUERY EXAMPLES
+==============
 
--V          Display the version number and exit.
+The following commands will list the entire client transaction of
+requests where the url is "/foo" ::
 
--w file     Write log entries to file instead of displaying them.  The file 
-   	    will be overwritten unless the -a option was specified. If 
-	    varnishlog receives a SIGHUP while writing to a file, it will 
-	    reopen the file, allowing the old one to be rotated away.
+	$ varnishlog -c 'req.url eq "/foo"'
+	$ varnishlog -c 'ReqURL eq "/foo"'
 
--X regex    Exclude log entries which match the specified regular expression.
+The following command will list the URL of all requests that has a
+cookie-header ::
+
+	$ varnishlog -c -i ReqURL req.http.cookie
+	$ varnishlog -c -i ReqURL 'ReqHeader ~ "^Cookie: "'
+
+Report the User-Agent of logged in clients where the request delivery
+time exceeds exceeds 0.5 seconds ::
+
+	$ varnishlog -c -I RxHeader:User-Agent 'req.http.cookie ~
+	"logged_in" and ReqEnd[5] > 0.5'
+
+Report delivery status code of client requests that had one or more
+503 errors from backend requests ::
+
+	$ varnishlog -i TxStatus 'beresp.status == 503'
+
+Report transaction set on requests that has backend failures
+or long delivery time on their ESI subrequests ::
+
+	$ varnishlog 'beresp.status{2+} >= 500 or ReqEnd{1+}[5] > 0.5'
 
--x tag      Exclude log entries with the specified tag.
 
 TAGS
 ====
@@ -139,16 +381,6 @@ The following log entry tags are currently defined:
 * VCL_trace
 * WorkThread
 
-EXAMPLES
-========
-
-The following command line simply copies all log entries to a log file::
-
-    $ varnishlog -w /var/log/varnish.log
-
-The following command line reads that same log file and displays requests for the front page::
-
-    $ varnishlog -r /var/log/varnish.log -c -m 'RxURL:^/$'
 
 SEE ALSO
 ========
@@ -161,8 +393,10 @@ SEE ALSO
 HISTORY
 =======
 
-The varnishlog utility was developed by Poul-Henning Kamp ?phk at phk.freebsd.dk? in cooperation with Verdens Gang
-AS, Varnish Software AS and Varnish Software.  This manual page was initially written by Dag-Erling Sm?rgrav.
+The varnishlog utility was developed by Poul-Henning Kamp
+?phk at phk.freebsd.dk? in cooperation with Verdens Gang AS, Varnish
+Software AS and Varnish Software.  This manual page was initially
+written by Dag-Erling Sm?rgrav.
 
 
 COPYRIGHT
@@ -172,4 +406,4 @@ This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2011 Varnish Software AS
+* Copyright (c) 2006-2013 Varnish Software AS



From phk at varnish-cache.org  Thu May 16 19:24:01 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 16 May 2013 21:24:01 +0200
Subject: [master] 18596a6 Move fetchbody to cache_fetch.c
Message-ID: <E1Ud3mT-0008Tl-Hj@project.varnish-software.com>

commit 18596a610f5cd2f2e01d9014046a26247fa7dc96
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 16 19:23:32 2013 +0000

    Move fetchbody to cache_fetch.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b86f4c0..3d47cd8 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -812,11 +812,11 @@ int EXP_NukeOne(struct busyobj *, struct lru *lru);
 void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 
 /* cache_fetch.c */
+int VBF_Fetch(struct worker *wrk, struct req *req);
 struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
 int FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req);
-void FetchBody(struct worker *w, void *bo);
 void Fetch_Init(void);
 
 /* cache_gzip.c */
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 4bacbf1..f42f62a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -30,6 +30,7 @@
 #include "config.h"
 
 #include <inttypes.h>
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -41,6 +42,7 @@
 #include "vcli_priv.h"
 #include "vct.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 static unsigned fetchfrag;
 
@@ -472,7 +474,7 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
  * refcount on it, which we must release, when done fetching.
  */
 
-void
+static void
 FetchBody(struct worker *wrk, void *priv)
 {
 	int cls;
@@ -624,6 +626,240 @@ FetchBody(struct worker *wrk, void *priv)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+int
+VBF_Fetch(struct worker *wrk, struct req *req)
+{
+	struct http *hp, *hp2;
+	char *b;
+	uint16_t nhttp;
+	unsigned l;
+	struct vsb *vary = NULL;
+	int varyl = 0, pass;
+	struct busyobj *bo;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	bo = req->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	if (req->objcore->objhead == NULL) {
+		/* This is a pass from vcl_recv */
+		pass = 1;
+		/* VCL may have fiddled this, but that doesn't help */
+		bo->exp.ttl = -1.;
+	} else if (bo->do_pass) {
+		pass = 1;
+	} else {
+		/* regular object */
+		pass = 0;
+	}
+
+	/*
+	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+	 * object processed before it is stored.
+	 *
+	 * The backend Content-Encoding header tells us what we are going
+	 * to receive, which we classify in the following three classes:
+	 *
+	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 *	no Content-Encoding		--> object is not gzip'ed.
+	 *	anything else			--> do nothing wrt gzip
+	 *
+	 */
+
+	/* We do nothing unless the param is set */
+	if (!cache_param->http_gzip_support)
+		bo->do_gzip = bo->do_gunzip = 0;
+
+	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
+
+	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
+
+	/* It can't be both */
+	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
+
+	/* We won't gunzip unless it is gzip'ed */
+	if (bo->do_gunzip && !bo->is_gzip)
+		bo->do_gunzip = 0;
+
+	/* If we do gunzip, remove the C-E header */
+	if (bo->do_gunzip)
+		http_Unset(bo->beresp, H_Content_Encoding);
+
+	/* We wont gzip unless it is ungziped */
+	if (bo->do_gzip && !bo->is_gunzip)
+		bo->do_gzip = 0;
+
+	/* If we do gzip, add the C-E header */
+	if (bo->do_gzip)
+		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
+
+	/* But we can't do both at the same time */
+	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
+
+	/* ESI takes precedence and handles gzip/gunzip itself */
+	if (bo->do_esi)
+		bo->vfp = &vfp_esi;
+	else if (bo->do_gunzip)
+		bo->vfp = &vfp_gunzip;
+	else if (bo->do_gzip)
+		bo->vfp = &vfp_gzip;
+	else if (bo->is_gzip)
+		bo->vfp = &vfp_testgzip;
+
+	if (bo->do_esi || req->esi_level > 0)
+		bo->do_stream = 0;
+	if (!req->wantbody)
+		bo->do_stream = 0;
+
+	/* No reason to try streaming a non-existing body */
+	if (bo->htc.body_status == BS_NONE)
+		bo->do_stream = 0;
+
+	l = http_EstimateWS(bo->beresp,
+	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
+
+	/* Create Vary instructions */
+	if (req->objcore->objhead != NULL) {
+		varyl = VRY_Create(bo, &vary);
+		if (varyl > 0) {
+			AN(vary);
+			assert(varyl == VSB_len(vary));
+			l += varyl;
+		} else if (varyl < 0) {
+			/* Vary parse error */
+			AZ(vary);
+			AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
+			req->objcore = NULL;
+			VDI_CloseFd(&bo->vbc);
+			VBO_DerefBusyObj(wrk, &req->busyobj);
+			return (-1);
+		} else
+			/* No vary */
+			AZ(vary);
+	}
+
+	/*
+	 * Space for producing a Content-Length: header including padding
+	 * A billion gigabytes is enough for anybody.
+	 */
+	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
+
+	if (bo->exp.ttl < cache_param->shortlived ||
+	    pass == 1)
+		bo->storage_hint = TRANSIENT_STORAGE;
+
+	AZ(bo->stats);
+	bo->stats = &wrk->stats;
+	req->obj = STV_NewObject(bo, &req->objcore, bo->storage_hint, l,
+	    nhttp);
+	if (req->obj == NULL) {
+		/*
+		 * Try to salvage the transaction by allocating a
+		 * shortlived object on Transient storage.
+		 */
+		if (bo->exp.ttl > cache_param->shortlived)
+			bo->exp.ttl = cache_param->shortlived;
+		bo->exp.grace = 0.0;
+		bo->exp.keep = 0.0;
+		req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
+		    l, nhttp);
+	}
+	bo->stats = NULL;
+	if (req->obj == NULL) {
+		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
+		req->objcore = NULL;
+		VDI_CloseFd(&bo->vbc);
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		return (-1);
+	}
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+
+	bo->storage_hint = NULL;
+
+	AZ(bo->fetch_obj);
+	bo->fetch_obj = req->obj;
+
+	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
+		req->obj->gziped = 1;
+
+	if (vary != NULL) {
+		req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
+		    VSB_data(vary), varyl);
+		AN(req->obj->vary);
+		VRY_Validate(req->obj->vary);
+		VSB_delete(vary);
+	}
+
+	req->obj->vxid = bo->vsl->wid;
+	req->obj->response = req->err_code;
+	WS_Assert(req->obj->ws_o);
+
+	/* Filter into object */
+	hp = bo->beresp;
+	hp2 = req->obj->http;
+
+	hp2->logtag = HTTP_Obj;
+	http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
+	http_CopyHome(hp2);
+
+	if (http_GetHdr(hp, H_Last_Modified, &b))
+		req->obj->last_modified = VTIM_parse(b);
+	else
+		req->obj->last_modified = floor(bo->exp.entered);
+
+	assert(WRW_IsReleased(wrk));
+
+	/*
+	 * If we can deliver a 304 reply, we don't bother streaming.
+	 * Notice that vcl_deliver{} could still nuke the headers
+	 * that allow the 304, in which case we return 200 non-stream.
+	 */
+	if (req->obj->response == 200 &&
+	    req->http->conds &&
+	    RFC2616_Do_Cond(req))
+		bo->do_stream = 0;
+
+	/*
+	 * Ready to fetch the body
+	 */
+	bo->fetch_task.func = FetchBody;
+	bo->fetch_task.priv = bo;
+
+	assert(bo->refcount == 2);	/* one for each thread */
+
+	if (req->obj->objcore->objhead != NULL) {
+		EXP_Insert(req->obj);
+		AN(req->obj->objcore->ban);
+		AZ(req->obj->ws_o->overflow);
+		HSH_Unbusy(&wrk->stats, req->obj->objcore);
+	}
+
+	if (!bo->do_stream ||
+	    Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
+		FetchBody(wrk, bo);
+
+	if (req->obj->objcore->objhead != NULL)
+		HSH_Ref(req->obj->objcore);
+
+	if (bo->state == BOS_FINISHED) {
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+	} else if (bo->state == BOS_FAILED) {
+		/* handle early failures */
+		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		return (-1);
+	}
+
+	assert(WRW_IsReleased(wrk));
+	return (0);
+}
+
+
+/*--------------------------------------------------------------------
  * Debugging aids
  */
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 07d0742..bb18117 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -481,239 +481,19 @@ DOT fetchbody:out -> prepresp [style=bold,color=blue]
 static enum req_fsm_nxt
 cnt_fetchbody(struct worker *wrk, struct req *req)
 {
-	struct http *hp, *hp2;
-	char *b;
-	uint16_t nhttp;
-	unsigned l;
-	struct vsb *vary = NULL;
-	int varyl = 0, pass;
-	struct busyobj *bo;
+	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	assert(wrk->handling == VCL_RET_DELIVER);
-
-	if (req->objcore->objhead == NULL) {
-		/* This is a pass from vcl_recv */
-		pass = 1;
-		/* VCL may have fiddled this, but that doesn't help */
-		bo->exp.ttl = -1.;
-	} else if (bo->do_pass) {
-		pass = 1;
-	} else {
-		/* regular object */
-		pass = 0;
-	}
-
-	/*
-	 * The VCL variables beresp.do_g[un]zip tells us how we want the
-	 * object processed before it is stored.
-	 *
-	 * The backend Content-Encoding header tells us what we are going
-	 * to receive, which we classify in the following three classes:
-	 *
-	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
-	 *	no Content-Encoding		--> object is not gzip'ed.
-	 *	anything else			--> do nothing wrt gzip
-	 *
-	 */
-
-	/* We do nothing unless the param is set */
-	if (!cache_param->http_gzip_support)
-		bo->do_gzip = bo->do_gunzip = 0;
-
-	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
-
-	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
-
-	/* It can't be both */
-	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
-
-	/* We won't gunzip unless it is gzip'ed */
-	if (bo->do_gunzip && !bo->is_gzip)
-		bo->do_gunzip = 0;
-
-	/* If we do gunzip, remove the C-E header */
-	if (bo->do_gunzip)
-		http_Unset(bo->beresp, H_Content_Encoding);
-
-	/* We wont gzip unless it is ungziped */
-	if (bo->do_gzip && !bo->is_gunzip)
-		bo->do_gzip = 0;
-
-	/* If we do gzip, add the C-E header */
-	if (bo->do_gzip)
-		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
-
-	/* But we can't do both at the same time */
-	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
-
-	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (bo->do_esi)
-		bo->vfp = &vfp_esi;
-	else if (bo->do_gunzip)
-		bo->vfp = &vfp_gunzip;
-	else if (bo->do_gzip)
-		bo->vfp = &vfp_gzip;
-	else if (bo->is_gzip)
-		bo->vfp = &vfp_testgzip;
-
-	if (bo->do_esi || req->esi_level > 0)
-		bo->do_stream = 0;
-	if (!req->wantbody)
-		bo->do_stream = 0;
-
-	/* No reason to try streaming a non-existing body */
-	if (bo->htc.body_status == BS_NONE)
-		bo->do_stream = 0;
-
-	l = http_EstimateWS(bo->beresp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
-
-	/* Create Vary instructions */
-	if (req->objcore->objhead != NULL) {
-		varyl = VRY_Create(bo, &vary);
-		if (varyl > 0) {
-			AN(vary);
-			assert(varyl == VSB_len(vary));
-			l += varyl;
-		} else if (varyl < 0) {
-			/* Vary parse error */
-			AZ(vary);
-			req->err_code = 503;
-			req->req_step = R_STP_ERROR;
-			AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-			req->objcore = NULL;
-			VDI_CloseFd(&bo->vbc);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
-			return (REQ_FSM_MORE);
-		} else
-			/* No vary */
-			AZ(vary);
-	}
 
-	/*
-	 * Space for producing a Content-Length: header including padding
-	 * A billion gigabytes is enough for anybody.
-	 */
-	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
-
-	if (bo->exp.ttl < cache_param->shortlived ||
-	    pass == 1)
-		bo->storage_hint = TRANSIENT_STORAGE;
-
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
-	req->obj = STV_NewObject(bo, &req->objcore, bo->storage_hint, l,
-	    nhttp);
-	if (req->obj == NULL) {
-		/*
-		 * Try to salvage the transaction by allocating a
-		 * shortlived object on Transient storage.
-		 */
-		if (bo->exp.ttl > cache_param->shortlived)
-			bo->exp.ttl = cache_param->shortlived;
-		bo->exp.grace = 0.0;
-		bo->exp.keep = 0.0;
-		req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
-		    l, nhttp);
-	}
-	bo->stats = NULL;
-	if (req->obj == NULL) {
+	i = VBF_Fetch(wrk, req);
+	if (i < 0) {
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
-		VDI_CloseFd(&bo->vbc);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		return (REQ_FSM_MORE);
-	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-
-	bo->storage_hint = NULL;
-
-	AZ(bo->fetch_obj);
-	bo->fetch_obj = req->obj;
-
-	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
-		req->obj->gziped = 1;
-
-	if (vary != NULL) {
-		req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
-		    VSB_data(vary), varyl);
-		AN(req->obj->vary);
-		VRY_Validate(req->obj->vary);
-		VSB_delete(vary);
-	}
-
-	req->obj->vxid = bo->vsl->wid;
-	req->obj->response = req->err_code;
-	WS_Assert(req->obj->ws_o);
-
-	/* Filter into object */
-	hp = bo->beresp;
-	hp2 = req->obj->http;
-
-	hp2->logtag = HTTP_Obj;
-	http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(hp2);
-
-	if (http_GetHdr(hp, H_Last_Modified, &b))
-		req->obj->last_modified = VTIM_parse(b);
-	else
-		req->obj->last_modified = floor(bo->exp.entered);
-
-	assert(WRW_IsReleased(wrk));
-
-	/*
-	 * If we can deliver a 304 reply, we don't bother streaming.
-	 * Notice that vcl_deliver{} could still nuke the headers
-	 * that allow the 304, in which case we return 200 non-stream.
-	 */
-	if (req->obj->response == 200 &&
-	    req->http->conds &&
-	    RFC2616_Do_Cond(req))
-		bo->do_stream = 0;
-
-	/*
-	 * Ready to fetch the body
-	 */
-	bo->fetch_task.func = FetchBody;
-	bo->fetch_task.priv = bo;
-
-	assert(bo->refcount == 2);	/* one for each thread */
-
-	if (req->obj->objcore->objhead != NULL) {
-		EXP_Insert(req->obj);
-		AN(req->obj->objcore->ban);
-		AZ(req->obj->ws_o->overflow);
-		HSH_Unbusy(&wrk->stats, req->obj->objcore);
-	}
-
-	if (!bo->do_stream ||
-	    Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
-		FetchBody(wrk, bo);
-
-	if (req->obj->objcore->objhead != NULL)
-		HSH_Ref(req->obj->objcore);
-
-	if (bo->state == BOS_FINISHED) {
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-	} else if (bo->state == BOS_FAILED) {
-		/* handle early failures */
-		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		req->err_code = 503;
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
+	} else {
+		assert(WRW_IsReleased(wrk));
+		req->req_step = R_STP_PREPRESP;
 	}
-
-	assert(WRW_IsReleased(wrk));
-	req->req_step = R_STP_PREPRESP;
 	return (REQ_FSM_MORE);
 }
 



From phk at varnish-cache.org  Fri May 17 07:26:56 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 17 May 2013 09:26:56 +0200
Subject: [master] d9c9c13 Turn vcl_pass{} into a "last chance to restart/error
 before fetch" decision point.
Message-ID: <E1UdF44-0001pA-N9@project.varnish-software.com>

commit d9c9c13d41a123642e612163450cac156618decb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 17 07:26:22 2013 +0000

    Turn vcl_pass{} into a "last chance to restart/error before fetch"
    decision point.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index bb18117..77f357e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -768,6 +768,16 @@ cnt_pass(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->busyobj);
 
+	VCL_pass_method(req->vcl, wrk, req, NULL, req->http->ws);
+	if (wrk->handling == VCL_RET_ERROR) {
+		req->req_step = R_STP_ERROR;
+		return (REQ_FSM_MORE);
+	}
+	if (wrk->handling == VCL_RET_RESTART) {
+		INCOMPL();
+	}
+	assert (wrk->handling == VCL_RET_FETCH);
+
 	req->busyobj = VBO_GetBusyObj(wrk, req);
 	bo = req->busyobj;
 	bo->refcount = 2;
@@ -777,15 +787,8 @@ cnt_pass(struct worker *wrk, struct req *req)
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
-	VCL_pass_method(bo->vcl, wrk, req, bo, bo->bereq->ws);
 
-	if (wrk->handling == VCL_RET_ERROR) {
-		http_Teardown(bo->bereq);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	}
-	assert(wrk->handling == VCL_RET_PASS);
+	assert (wrk->handling == VCL_RET_FETCH);
 	req->acct_req.pass++;
 	req->req_step = R_STP_FETCH;
 
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 488cf0c..520bc06 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -80,7 +80,7 @@ sub vcl_pipe {
 }
 
 sub vcl_pass {
-    return (pass);
+    return (fetch);
 }
 
 sub vcl_hash {
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 1402e13..7671650 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -80,7 +80,7 @@ tokens = {
 returns =(
 	('recv',		"C", ('error', 'pass', 'pipe', 'hash', 'purge',)),
 	('pipe',		"C", ('error', 'pipe',)),
-	('pass',		"C", ('error', 'restart', 'pass',)),
+	('pass',		"C", ('error', 'restart', 'fetch',)),
 	('hash',		"C", ('lookup',)),
 	('purge',		"C", ('error', 'fetch',)),
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),



From phk at varnish-cache.org  Fri May 17 07:41:13 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 17 May 2013 09:41:13 +0200
Subject: [master] 78b5dab Make cnt_miss{} look more like cnt_pass{}
Message-ID: <E1UdFHt-0003UK-Bs@project.varnish-software.com>

commit 78b5dabe3e904f2f79a4365c8ae442420bc57057
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 17 07:41:00 2013 +0000

    Make cnt_miss{} look more like cnt_pass{}

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 77f357e..c02aac9 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -523,7 +523,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	struct objcore *oc, *boc;
 	struct object *o;
 	struct objhead *oh;
-	struct busyobj *bo;
 	enum lookup_e lr;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -596,16 +595,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	/* If we inserted a new object it's a miss */
 	if (oc->flags & OC_F_BUSY) {
-		AZ(req->busyobj);
-		bo = VBO_GetBusyObj(wrk, req);
-		req->busyobj = bo;
-		/* One ref for req, one for FetchBody */
-		bo->refcount = 2;
-		VRY_Finish(req, bo);
-
-		oc->busyobj = bo;
-		wrk->stats.cache_miss++;
-
 		req->objcore = oc;
 		req->req_step = R_STP_MISS;
 		return (REQ_FSM_MORE);
@@ -690,9 +679,19 @@ cnt_miss(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(req->obj);
+	AZ(req->busyobj);
+
+	bo = VBO_GetBusyObj(wrk, req);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	req->busyobj = bo;
+	/* One ref for req, one for FetchBody */
+	bo->refcount = 2;
+
+	VRY_Finish(req, bo);
+
+	req->objcore->busyobj = bo;
+	wrk->stats.cache_miss++;
 
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(bo->bereq, req->http, HTTPH_R_FETCH);



From phk at varnish-cache.org  Fri May 17 08:18:00 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 17 May 2013 10:18:00 +0200
Subject: [master] ac8de26 Use worker workspace for purge list  of objects
Message-ID: <E1UdFrU-0000Yu-QI@project.varnish-software.com>

commit ac8de26b6404a1634ba95d45655513ef706f6b68
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 17 08:17:44 2013 +0000

    Use worker workspace for purge list  of objects

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index e9ef4fc..3d3ef81 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -517,16 +517,16 @@ hsh_rush(struct dstat *ds, struct objhead *oh)
  */
 
 void
-HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace)
+HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, nobj, n;
 	struct object *o;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	spc = WS_Reserve(req->ws, 0);
-	ocp = (void*)req->ws->f;
+	spc = WS_Reserve(wrk->aws, 0);
+	ocp = (void*)wrk->aws->f;
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
 	nobj = 0;
@@ -543,7 +543,7 @@ HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace)
 			continue;
 		}
 
-		(void)oc_getobj(&req->wrk->stats, oc);
+		(void)oc_getobj(&wrk->stats, oc);
 		    /* XXX: still needed ? */
 
 		xxxassert(spc >= sizeof *ocp);
@@ -561,16 +561,16 @@ HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace)
 	for (n = 0; n < nobj; n++) {
 		oc = ocp[n];
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		o = oc_getobj(&req->wrk->stats, oc);
+		o = oc_getobj(&wrk->stats, oc);
 		if (o == NULL)
 			continue;
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		o->exp.ttl = ttl;
 		o->exp.grace = grace;
 		EXP_Rearm(o);
-		(void)HSH_Deref(&req->wrk->stats, NULL, &o);
+		(void)HSH_Deref(&wrk->stats, NULL, &o);
 	}
-	WS_Release(req->ws, 0);
+	WS_Release(wrk->aws, 0);
 }
 
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c02aac9..a0636ef 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -1026,7 +1026,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
 	VRY_Finish(req, NULL);
 
-	HSH_Purge(req, boc->objhead, 0, 0);
+	HSH_Purge(wrk, boc->objhead, 0, 0);
 
 	AZ(HSH_Deref(&wrk->stats, boc, NULL));
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 8d0ee46..3009e90 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -507,11 +507,12 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
 	if (ctx->method == VCL_MET_LOOKUP)
-		HSH_Purge(ctx->req, ctx->req->obj->objcore->objhead,
+		HSH_Purge(ctx->req->wrk, ctx->req->obj->objcore->objhead,
 		    ttl, grace);
 	else if (ctx->method == VCL_MET_MISS)
-		HSH_Purge(ctx->req, ctx->req->objcore->objhead,
+		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
 		    ttl, grace);
 }
 
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 3c36b4d..e21a96e 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -71,7 +71,7 @@ void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(const struct req *, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
-void HSH_Purge(struct req *, struct objhead *, double ttl, double grace);
+void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace);
 void HSH_config(const char *h_arg);
 struct objcore *HSH_NewObjCore(struct worker *wrk);
 



From phk at varnish-cache.org  Fri May 17 10:28:50 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 17 May 2013 12:28:50 +0200
Subject: [master] 88db6f6 Move vcl_miss{} before vcl_backend_fetch{}
Message-ID: <E1UdHu6-0007rG-0z@project.varnish-software.com>

commit 88db6f6603e5759c9085528242c13d2e9949518f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 17 10:28:29 2013 +0000

    Move vcl_miss{} before vcl_backend_fetch{}

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index a0636ef..734e091 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -357,6 +357,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
@@ -674,6 +676,7 @@ static enum req_fsm_nxt
 cnt_miss(struct worker *wrk, struct req *req)
 {
 	struct busyobj *bo;
+	int pass = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -682,14 +685,41 @@ cnt_miss(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->busyobj);
 
+	/* We optimistically expect to need this most of the time */
 	bo = VBO_GetBusyObj(wrk, req);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->busyobj = bo;
+	VRY_Finish(req, bo);
+
+	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
+	switch (wrk->handling) {
+	case VCL_RET_ERROR:
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
+		req->objcore = NULL;
+		req->req_step = R_STP_ERROR;
+		return (REQ_FSM_MORE);
+	case VCL_RET_RESTART:
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
+		req->objcore = NULL;
+		req->req_step = R_STP_RESTART;
+		return (REQ_FSM_MORE);
+	case VCL_RET_PASS:
+		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
+		req->objcore = HSH_NewObjCore(wrk);
+		pass = 1;
+		break;
+	case VCL_RET_FETCH:
+		break;
+	default:
+		WRONG("wrong return from vcl_miss{}");
+	}
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	/* One ref for req, one for FetchBody */
 	bo->refcount = 2;
 
-	VRY_Finish(req, bo);
-
+	AN (req->objcore);
 	req->objcore->busyobj = bo;
 	wrk->stats.cache_miss++;
 
@@ -697,44 +727,24 @@ cnt_miss(struct worker *wrk, struct req *req)
 	http_FilterReq(bo->bereq, req->http, HTTPH_R_FETCH);
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
-	http_ForceGet(bo->bereq);
-	if (cache_param->http_gzip_support) {
-		/*
-		 * We always ask the backend for gzip, even if the
-		 * client doesn't grok it.  We will uncompress for
-		 * the minority of clients which don't.
-		 */
-		http_Unset(bo->bereq, H_Accept_Encoding);
-		http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
+	if (!pass) {
+		http_ForceGet(bo->bereq);
+		if (cache_param->http_gzip_support) {
+			/*
+			 * We always ask the backend for gzip, even if the
+			 * client doesn't grok it.  We will uncompress for
+			 * the minority of clients which don't.
+			 */
+			http_Unset(bo->bereq, H_Accept_Encoding);
+			http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
+		}
 	}
 
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
-	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
-
-	if (wrk->handling == VCL_RET_FETCH) {
-		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-		req->req_step = R_STP_FETCH;
-		return (REQ_FSM_MORE);
-	}
 
-	AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-	req->objcore = NULL;
-	http_Teardown(bo->bereq);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
-
-	switch(wrk->handling) {
-	case VCL_RET_ERROR:
-		req->req_step = R_STP_ERROR;
-		break;
-	case VCL_RET_PASS:
-		req->req_step = R_STP_PASS;
-		break;
-	case VCL_RET_RESTART:
-		req->req_step = R_STP_RESTART;
-		break;
-	default:
-		WRONG("Illegal action in vcl_miss{}");
-	}
+	xxxassert(wrk->handling == VCL_RET_FETCH);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
 
diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc
index 79185e1..570fc93 100644
--- a/bin/varnishtest/tests/c00012.vtc
+++ b/bin/varnishtest/tests/c00012.vtc
@@ -25,7 +25,7 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 7
-	expect resp.http.x-varnish == "1004"
+	expect resp.http.x-varnish == "1003"
 } 
 
 client c1 -run
diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc
deleted file mode 100644
index 13f69f8..0000000
--- a/bin/varnishtest/tests/c00033.vtc
+++ /dev/null
@@ -1,85 +0,0 @@
-varnishtest "Inline C access to purges (could be retired)"
-
-server s1 {
-	rxreq
-	txresp -hdr "Vary: foo" -bodylen 1
-	rxreq
-	txresp -hdr "Vary: foo" -bodylen 2
-	rxreq
-	txresp -hdr "Vary: foo" -bodylen 3
-	rxreq
-	txresp -hdr "Vary: foo" -bodylen 4
-	rxreq
-	txresp -hdr "Vary: foo" -bodylen 5
-	rxreq
-	txresp -hdr "Vary: foo" -bodylen 6
-} -start
-
-varnish v1 -vcl+backend {
-
-	sub vcl_recv {
-		if (req.method == "PURGE") {
-			return (hash);
-		}
-	}
-
-	sub vcl_lookup {
-		if (req.method == "PURGE") {
-			C{ VRT_purge(ctx, 0, 0); }C
-			return (error(456, "got it"));
-		}
-	}
-	sub vcl_miss {
-		if (req.method == "PURGE") {
-			C{ VRT_purge(ctx, 0, 0); }C
-			return (error(456, "got it"));
-		}
-	}
-} -start
-
-client c1 {
-	txreq -hdr "foo: bar1"
-	rxresp
-	expect resp.bodylen == 1
-
-	txreq -hdr "foo: bar2"
-	rxresp
-	expect resp.bodylen == 2
-
-	txreq -hdr "foo: bar1"
-	rxresp
-	expect resp.bodylen == 1
-
-	txreq -hdr "foo: bar2"
-	rxresp
-	expect resp.bodylen == 2
-
-	txreq -req "PURGE" -hdr "foo: bar1"
-	rxresp
-	expect resp.status == 456
-} -run
-
-client c1 {
-	txreq -hdr "foo: bar1"
-	rxresp
-	expect resp.bodylen == 3
-
-	txreq -hdr "foo: bar2"
-	rxresp
-	expect resp.bodylen == 4
-
-	txreq -req "PURGE" -hdr "foo: bar3"
-	rxresp
-	expect resp.status == 456
-} -run
-
-client c1 {
-	txreq -hdr "foo: bar1"
-	rxresp
-	expect resp.bodylen == 5
-
-	txreq -hdr "foo: bar2"
-	rxresp
-	expect resp.bodylen == 6
-
-} -run



From phk at varnish-cache.org  Fri May 17 10:57:11 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 17 May 2013 12:57:11 +0200
Subject: [master] e16233f Unify some code from miss+pass into fetch
Message-ID: <E1UdILX-0002zT-3E@project.varnish-software.com>

commit e16233fd33248c32281a79e47f8fb8383b1326a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 17 10:56:57 2013 +0000

    Unify some code from miss+pass into fetch

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 734e091..6dd3518 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -367,6 +367,28 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
 
+	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
+	http_FilterReq(bo->bereq, req->http,
+	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
+	if (!bo->do_pass) {
+		http_ForceGet(bo->bereq);
+		if (cache_param->http_gzip_support) {
+			/*
+			 * We always ask the backend for gzip, even if the
+			 * client doesn't grok it.  We will uncompress for
+			 * the minority of clients which don't.
+			 */
+			http_Unset(bo->bereq, H_Accept_Encoding);
+			http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
+		}
+	}
+
+	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
+	xxxassert (wrk->handling == VCL_RET_FETCH);
+
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
+
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
 	req->acct_req.fetch++;
@@ -419,7 +441,6 @@ cnt_fetch(struct worker *wrk, struct req *req)
 			bo->exp.ttl = -1.;
 
 		AZ(bo->do_esi);
-		AZ(bo->do_pass);
 
 		VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
 		    bo->beresp->ws);
@@ -676,7 +697,6 @@ static enum req_fsm_nxt
 cnt_miss(struct worker *wrk, struct req *req)
 {
 	struct busyobj *bo;
-	int pass = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -707,7 +727,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	case VCL_RET_PASS:
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = HSH_NewObjCore(wrk);
-		pass = 1;
+		bo->do_pass = 1;
 		break;
 	case VCL_RET_FETCH:
 		break;
@@ -723,26 +743,6 @@ cnt_miss(struct worker *wrk, struct req *req)
 	req->objcore->busyobj = bo;
 	wrk->stats.cache_miss++;
 
-	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(bo->bereq, req->http, HTTPH_R_FETCH);
-	http_PrintfHeader(bo->bereq,
-	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
-	if (!pass) {
-		http_ForceGet(bo->bereq);
-		if (cache_param->http_gzip_support) {
-			/*
-			 * We always ask the backend for gzip, even if the
-			 * client doesn't grok it.  We will uncompress for
-			 * the minority of clients which don't.
-			 */
-			http_Unset(bo->bereq, H_Accept_Encoding);
-			http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
-		}
-	}
-
-	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
-
-	xxxassert(wrk->handling == VCL_RET_FETCH);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
@@ -786,23 +786,17 @@ cnt_pass(struct worker *wrk, struct req *req)
 		INCOMPL();
 	}
 	assert (wrk->handling == VCL_RET_FETCH);
+	req->acct_req.pass++;
 
 	req->busyobj = VBO_GetBusyObj(wrk, req);
 	bo = req->busyobj;
+	bo->do_pass = 1;
+	req->objcore = HSH_NewObjCore(wrk);
+	req->objcore->busyobj = bo;
 	bo->refcount = 2;
-	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(bo->bereq, req->http, HTTPH_R_PASS);
-	http_PrintfHeader(bo->bereq,
-	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
-
-	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
-	assert (wrk->handling == VCL_RET_FETCH);
-	req->acct_req.pass++;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->req_step = R_STP_FETCH;
-
-	req->objcore = HSH_NewObjCore(wrk);
-	req->objcore->busyobj = bo;
 	return (REQ_FSM_MORE);
 }
 



From phk at varnish-cache.org  Mon May 20 10:44:32 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 20 May 2013 12:44:32 +0200
Subject: [master] 2eb5e55 Move all of cnt_fetch() to cache_fetch.c
Message-ID: <E1UeNZw-00017R-MQ@project.varnish-software.com>

commit 2eb5e555f796caa9d05523a0fd044ecf91e60a0b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 20 10:44:10 2013 +0000

    Move all of cnt_fetch() to cache_fetch.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3d47cd8..18155e8 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -816,7 +816,6 @@ int VBF_Fetch(struct worker *wrk, struct req *req);
 struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
-int FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req);
 void Fetch_Init(void);
 
 /* cache_gzip.c */
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f42f62a..3e88210 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -40,6 +40,7 @@
 
 #include "cache_backend.h"
 #include "vcli_priv.h"
+#include "vcl.h"
 #include "vct.h"
 #include "vtcp.h"
 #include "vtim.h"
@@ -358,7 +359,7 @@ fetch_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
  *	 1 failure which can be retried.
  */
 
-int
+static int
 FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 {
 	struct vbc *vc;
@@ -628,6 +629,146 @@ FetchBody(struct worker *wrk, void *priv)
 /*--------------------------------------------------------------------
  */
 
+static int
+cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+
+	bo = req->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	AN(req->director);
+	AZ(bo->vbc);
+	AZ(bo->should_close);
+	AZ(bo->storage_hint);
+
+	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
+	http_FilterReq(bo->bereq, req->http,
+	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
+	if (!bo->do_pass) {
+		http_ForceGet(bo->bereq);
+		if (cache_param->http_gzip_support) {
+			/*
+			 * We always ask the backend for gzip, even if the
+			 * client doesn't grok it.  We will uncompress for
+			 * the minority of clients which don't.
+			 */
+			http_Unset(bo->bereq, H_Accept_Encoding);
+			http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
+		}
+	}
+
+	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
+	xxxassert (wrk->handling == VCL_RET_FETCH);
+
+	http_PrintfHeader(bo->bereq,
+	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
+
+	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
+
+	req->acct_req.fetch++;
+
+	i = FetchHdr(wrk, bo, req->objcore->objhead == NULL ? req : NULL);
+	/*
+	 * If we recycle a backend connection, there is a finite chance
+	 * that the backend closed it before we get a request to it.
+	 * Do a single retry in that case.
+	 */
+	if (i == 1) {
+		VSC_C_main->backend_retry++;
+		i = FetchHdr(wrk, bo,
+		    req->objcore->objhead == NULL ? req : NULL);
+	}
+
+	if (req->objcore->objhead != NULL)
+		(void)HTTP1_DiscardReqBody(req);	// XXX
+
+	if (i) {
+		wrk->handling = VCL_RET_ERROR;
+		req->err_code = 503;
+	} else {
+		/*
+		 * These two headers can be spread over multiple actual headers
+		 * and we rely on their content outside of VCL, so collect them
+		 * into one line here.
+		 */
+		http_CollectHdr(bo->beresp, H_Cache_Control);
+		http_CollectHdr(bo->beresp, H_Vary);
+
+		/*
+		 * Figure out how the fetch is supposed to happen, before the
+		 * headers are adultered by VCL
+		 * NB: Also sets other wrk variables
+		 */
+		bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
+
+		req->err_code = http_GetStatus(bo->beresp);
+
+		/*
+		 * What does RFC2616 think about TTL ?
+		 */
+		EXP_Clr(&bo->exp);
+		bo->exp.entered = W_TIM_real(wrk);
+		RFC2616_Ttl(bo);
+
+		/* pass from vclrecv{} has negative TTL */
+		if (req->objcore->objhead == NULL)
+			bo->exp.ttl = -1.;
+
+		AZ(bo->do_esi);
+
+		VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
+		    bo->beresp->ws);
+
+		if (bo->do_pass)
+			req->objcore->flags |= OC_F_PASS;
+
+		switch (wrk->handling) {
+		case VCL_RET_DELIVER:
+			return (0);
+		default:
+			break;
+		}
+
+		/* We are not going to fetch the body, Close the connection */
+		VDI_CloseFd(&bo->vbc);
+	}
+
+	/* Clean up partial fetch */
+	AZ(bo->vbc);
+
+	if (req->objcore->objhead != NULL ||
+	    wrk->handling == VCL_RET_RESTART ||
+	    wrk->handling == VCL_RET_ERROR) {
+		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
+		req->objcore = NULL;
+	}
+	assert(bo->refcount == 2);
+	bo->storage_hint = NULL;
+	VBO_DerefBusyObj(wrk, &bo);
+	VBO_DerefBusyObj(wrk, &req->busyobj);
+	req->director = NULL;
+
+	switch (wrk->handling) {
+	case VCL_RET_RESTART:
+		// req->req_step = R_STP_RESTART;
+		return (1);
+	case VCL_RET_ERROR:
+		// req->req_step = R_STP_ERROR;
+		return (-1);
+	default:
+		WRONG("Illegal action in vcl_fetch{}");
+	}
+}
+
+
 int
 VBF_Fetch(struct worker *wrk, struct req *req)
 {
@@ -638,6 +779,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	struct vsb *vary = NULL;
 	int varyl = 0, pass;
 	struct busyobj *bo;
+	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -645,6 +787,10 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	i = cnt_fetch(wrk, req, bo);
+	if (i)
+		return (i);
+
 	if (req->objcore->objhead == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 6dd3518..04e3415 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -335,160 +335,6 @@ cnt_error(struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
- * Fetch response headers from the backend
- *
-DOT subgraph xcluster_fetch {
-DOT	fetch [
-DOT		shape=record
-DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}}"
-DOT	]
-DOT }
-DOT fetch -> fetchbody [style=bold,color=red]
-DOT fetch -> fetchbody [style=bold,color=blue]
- */
-
-static enum req_fsm_nxt
-cnt_fetch(struct worker *wrk, struct req *req)
-{
-	int i;
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	AN(req->director);
-	AZ(bo->vbc);
-	AZ(bo->should_close);
-	AZ(bo->storage_hint);
-
-	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(bo->bereq, req->http,
-	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
-	if (!bo->do_pass) {
-		http_ForceGet(bo->bereq);
-		if (cache_param->http_gzip_support) {
-			/*
-			 * We always ask the backend for gzip, even if the
-			 * client doesn't grok it.  We will uncompress for
-			 * the minority of clients which don't.
-			 */
-			http_Unset(bo->bereq, H_Accept_Encoding);
-			http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
-		}
-	}
-
-	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
-	xxxassert (wrk->handling == VCL_RET_FETCH);
-
-	http_PrintfHeader(bo->bereq,
-	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
-
-	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
-
-	req->acct_req.fetch++;
-
-	i = FetchHdr(wrk, bo, req->objcore->objhead == NULL ? req : NULL);
-	/*
-	 * If we recycle a backend connection, there is a finite chance
-	 * that the backend closed it before we get a request to it.
-	 * Do a single retry in that case.
-	 */
-	if (i == 1) {
-		VSC_C_main->backend_retry++;
-		i = FetchHdr(wrk, bo,
-		    req->objcore->objhead == NULL ? req : NULL);
-	}
-
-	if (req->objcore->objhead != NULL)
-		(void)HTTP1_DiscardReqBody(req);	// XXX
-
-	if (i) {
-		wrk->handling = VCL_RET_ERROR;
-		req->err_code = 503;
-	} else {
-		/*
-		 * These two headers can be spread over multiple actual headers
-		 * and we rely on their content outside of VCL, so collect them
-		 * into one line here.
-		 */
-		http_CollectHdr(bo->beresp, H_Cache_Control);
-		http_CollectHdr(bo->beresp, H_Vary);
-
-		/*
-		 * Figure out how the fetch is supposed to happen, before the
-		 * headers are adultered by VCL
-		 * NB: Also sets other wrk variables
-		 */
-		bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
-
-		req->err_code = http_GetStatus(bo->beresp);
-
-		/*
-		 * What does RFC2616 think about TTL ?
-		 */
-		EXP_Clr(&bo->exp);
-		bo->exp.entered = W_TIM_real(wrk);
-		RFC2616_Ttl(bo);
-
-		/* pass from vclrecv{} has negative TTL */
-		if (req->objcore->objhead == NULL)
-			bo->exp.ttl = -1.;
-
-		AZ(bo->do_esi);
-
-		VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
-		    bo->beresp->ws);
-
-		if (bo->do_pass)
-			req->objcore->flags |= OC_F_PASS;
-
-		switch (wrk->handling) {
-		case VCL_RET_DELIVER:
-			req->req_step = R_STP_FETCHBODY;
-			return (REQ_FSM_MORE);
-		default:
-			break;
-		}
-
-		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(&bo->vbc);
-	}
-
-	/* Clean up partial fetch */
-	AZ(bo->vbc);
-
-	if (req->objcore->objhead != NULL ||
-	    wrk->handling == VCL_RET_RESTART ||
-	    wrk->handling == VCL_RET_ERROR) {
-		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
-	}
-	assert(bo->refcount == 2);
-	bo->storage_hint = NULL;
-	VBO_DerefBusyObj(wrk, &bo);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
-	req->director = NULL;
-
-	switch (wrk->handling) {
-	case VCL_RET_RESTART:
-		req->req_step = R_STP_RESTART;
-		return (REQ_FSM_MORE);
-	case VCL_RET_ERROR:
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	default:
-		WRONG("Illegal action in vcl_fetch{}");
-	}
-}
-
-/*--------------------------------------------------------------------
  * Prepare to fetch body from backend
  *
 DOT subgraph xcluster_body {
@@ -502,7 +348,7 @@ DOT fetchbody:out -> prepresp [style=bold,color=blue]
  */
 
 static enum req_fsm_nxt
-cnt_fetchbody(struct worker *wrk, struct req *req)
+cnt_fetch(struct worker *wrk, struct req *req)
 {
 	int i;
 
@@ -513,6 +359,8 @@ cnt_fetchbody(struct worker *wrk, struct req *req)
 	if (i < 0) {
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
+	} else if (i == 1) {
+		req->req_step = R_STP_RESTART;
 	} else {
 		assert(WRW_IsReleased(wrk));
 		req->req_step = R_STP_PREPRESP;
@@ -705,8 +553,12 @@ cnt_miss(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->busyobj);
 
-	/* We optimistically expect to need this most of the time */
+	/*
+	 * We optimistically expect to need this most of the time
+	 * (This allows us to put the predictive Vary directly on the bo->ws)
+	 */
 	bo = VBO_GetBusyObj(wrk, req);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->busyobj = bo;
 	VRY_Finish(req, bo);
 
@@ -743,7 +595,6 @@ cnt_miss(struct worker *wrk, struct req *req)
 	req->objcore->busyobj = bo;
 	wrk->stats.cache_miss++;
 
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
@@ -788,14 +639,14 @@ cnt_pass(struct worker *wrk, struct req *req)
 	assert (wrk->handling == VCL_RET_FETCH);
 	req->acct_req.pass++;
 
-	req->busyobj = VBO_GetBusyObj(wrk, req);
-	bo = req->busyobj;
+	bo = VBO_GetBusyObj(wrk, req);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	req->busyobj = bo;
 	bo->do_pass = 1;
 	req->objcore = HSH_NewObjCore(wrk);
 	req->objcore->busyobj = bo;
 	bo->refcount = 2;
 
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index a4c19b0..6c3f3f6 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -44,7 +44,6 @@ REQ_STEP(lookup,	LOOKUP,		(wrk, req))
 REQ_STEP(purge,		PURGE,		(wrk, req))
 REQ_STEP(miss,		MISS,		(wrk, req))
 REQ_STEP(fetch,		FETCH,		(wrk, req))
-REQ_STEP(fetchbody,	FETCHBODY,	(wrk, req))
 REQ_STEP(prepresp,	PREPRESP,	(wrk, req))
 REQ_STEP(deliver,	DELIVER,	(wrk, req))
 REQ_STEP(error,		ERROR,		(wrk, req))



From phk at varnish-cache.org  Mon May 20 13:28:50 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 20 May 2013 15:28:50 +0200
Subject: [master] afed0f6 Let req_fsm deref its own bo->ref on failure
Message-ID: <E1UeQ8w-0000os-In@project.varnish-software.com>

commit afed0f6143e39961d2156bf4172d7c17351daa46
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 20 13:28:35 2013 +0000

    Let req_fsm deref its own bo->ref on failure

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3e88210..51b3d4f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -753,7 +753,6 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	assert(bo->refcount == 2);
 	bo->storage_hint = NULL;
 	VBO_DerefBusyObj(wrk, &bo);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
 	req->director = NULL;
 
 	switch (wrk->handling) {
@@ -881,7 +880,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 			AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 			req->objcore = NULL;
 			VDI_CloseFd(&bo->vbc);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
 			return (-1);
 		} else
 			/* No vary */
@@ -919,7 +917,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
 		return (-1);
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
@@ -996,7 +993,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	} else if (bo->state == BOS_FAILED) {
 		/* handle early failures */
 		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
 		return (-1);
 	}
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 04e3415..9a4e27d 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -356,14 +356,17 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	i = VBF_Fetch(wrk, req);
-	if (i < 0) {
+	if (i == 0) {
+		assert(WRW_IsReleased(wrk));
+		req->req_step = R_STP_PREPRESP;
+	} else if (i < 0) {
+		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
-	} else if (i == 1) {
-		req->req_step = R_STP_RESTART;
 	} else {
-		assert(WRW_IsReleased(wrk));
-		req->req_step = R_STP_PREPRESP;
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		assert(i == 1);
+		req->req_step = R_STP_RESTART;
 	}
 	return (REQ_FSM_MORE);
 }



From phk at varnish-cache.org  Mon May 20 18:25:07 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 20 May 2013 20:25:07 +0200
Subject: [master] 4d4bfea Start splitting the fetch path into natural and
 (hopefully) understandable bits.
Message-ID: <E1UeUlf-0001db-Dd@project.varnish-software.com>

commit 4d4bfea9b62599ba6f4b3256d8d4bf56c2f05454
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 20 18:24:34 2013 +0000

    Start splitting the fetch path into natural and (hopefully)
    understandable bits.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 51b3d4f..3cda58e 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -627,23 +627,19 @@ FetchBody(struct worker *wrk, void *priv)
 }
 
 /*--------------------------------------------------------------------
+ * Copy req->bereq and run it by VCL::vcl_backend_fetch{}
  */
 
-static int
-cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
+static void
+vbf_make_bereq(struct worker *wrk, const struct req *req, struct busyobj *bo)
 {
-	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-
-	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	AN(req->director);
+	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
@@ -652,6 +648,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	http_FilterReq(bo->bereq, req->http,
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
 	if (!bo->do_pass) {
+		// XXX: Forcing GET should happen in vcl_miss{} ?
 		http_ForceGet(bo->bereq);
 		if (cache_param->http_gzip_support) {
 			/*
@@ -665,10 +662,18 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	}
 
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
-	xxxassert (wrk->handling == VCL_RET_FETCH);
 
 	http_PrintfHeader(bo->bereq,
-	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
+	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static int
+cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
+{
+	int i;
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
@@ -689,10 +694,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	if (req->objcore->objhead != NULL)
 		(void)HTTP1_DiscardReqBody(req);	// XXX
 
-	if (i) {
-		wrk->handling = VCL_RET_ERROR;
-		req->err_code = 503;
-	} else {
+	if (!i) {
 		/*
 		 * These two headers can be spread over multiple actual headers
 		 * and we rely on their content outside of VCL, so collect them
@@ -729,15 +731,14 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 		if (bo->do_pass)
 			req->objcore->flags |= OC_F_PASS;
 
-		switch (wrk->handling) {
-		case VCL_RET_DELIVER:
+		if (wrk->handling == VCL_RET_DELIVER)
 			return (0);
-		default:
-			break;
-		}
 
 		/* We are not going to fetch the body, Close the connection */
 		VDI_CloseFd(&bo->vbc);
+	} else {
+		wrk->handling = VCL_RET_ERROR;
+		req->err_code = 503;
 	}
 
 	/* Clean up partial fetch */
@@ -752,22 +753,19 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	}
 	assert(bo->refcount == 2);
 	bo->storage_hint = NULL;
+	bo->director = NULL;
 	VBO_DerefBusyObj(wrk, &bo);
-	req->director = NULL;
 
 	switch (wrk->handling) {
 	case VCL_RET_RESTART:
-		// req->req_step = R_STP_RESTART;
 		return (1);
 	case VCL_RET_ERROR:
-		// req->req_step = R_STP_ERROR;
 		return (-1);
 	default:
 		WRONG("Illegal action in vcl_fetch{}");
 	}
 }
 
-
 int
 VBF_Fetch(struct worker *wrk, struct req *req)
 {
@@ -783,8 +781,14 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	xxxassert(bo->refcount == 2);	// Req might abandon early ?
+	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
+
+	vbf_make_bereq(wrk, req, bo);
+	xxxassert (wrk->handling == VCL_RET_FETCH);
 
 	i = cnt_fetch(wrk, req, bo);
 	if (i)



From phk at varnish-cache.org  Wed May 22 12:25:59 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 22 May 2013 14:25:59 +0200
Subject: [master] fddbe67 Try to separate "do_pass" from "have object"
 decision points.
Message-ID: <E1Uf87D-0002kX-2v@project.varnish-software.com>

commit fddbe67126d8a51681bfbade1712cf2d88eee50d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 22 12:25:28 2013 +0000

    Try to separate "do_pass" from "have object" decision points.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3cda58e..3ab9cea 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -633,6 +633,7 @@ FetchBody(struct worker *wrk, void *priv)
 static void
 vbf_make_bereq(struct worker *wrk, const struct req *req, struct busyobj *bo)
 {
+	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -661,7 +662,10 @@ vbf_make_bereq(struct worker *wrk, const struct req *req, struct busyobj *bo)
 		}
 	}
 
+	// Don't let VCL reset do_pass
+	i = bo->do_pass;
 	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
+	bo->do_pass |= i;
 
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
@@ -679,7 +683,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 
 	req->acct_req.fetch++;
 
-	i = FetchHdr(wrk, bo, req->objcore->objhead == NULL ? req : NULL);
+	i = FetchHdr(wrk, bo, bo->do_pass ? req : NULL);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -687,8 +691,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(wrk, bo,
-		    req->objcore->objhead == NULL ? req : NULL);
+		i = FetchHdr(wrk, bo, bo->do_pass ? req : NULL);
 	}
 
 	if (req->objcore->objhead != NULL)
@@ -725,8 +728,11 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 
 		AZ(bo->do_esi);
 
+		// Don't let VCL reset do_pass
+		i = bo->do_pass;
 		VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
 		    bo->beresp->ws);
+		bo->do_pass |= i;
 
 		if (bo->do_pass)
 			req->objcore->flags |= OC_F_PASS;
@@ -774,7 +780,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	uint16_t nhttp;
 	unsigned l;
 	struct vsb *vary = NULL;
-	int varyl = 0, pass;
+	int varyl = 0;
 	struct busyobj *bo;
 	int i;
 
@@ -795,15 +801,11 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		return (i);
 
 	if (req->objcore->objhead == NULL) {
+		AN(bo->do_pass);
 		/* This is a pass from vcl_recv */
-		pass = 1;
+		bo->do_pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
-		bo->exp.ttl = -1.;
-	} else if (bo->do_pass) {
-		pass = 1;
-	} else {
-		/* regular object */
-		pass = 0;
+		// bo->exp.ttl = -1.;
 	}
 
 	/*
@@ -869,7 +871,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		bo->do_stream = 0;
 
 	l = http_EstimateWS(bo->beresp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
+	    bo->do_pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
 	if (req->objcore->objhead != NULL) {
@@ -896,8 +898,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (bo->exp.ttl < cache_param->shortlived ||
-	    pass == 1)
+	if (bo->exp.ttl < cache_param->shortlived || bo->do_pass == 1)
 		bo->storage_hint = TRANSIENT_STORAGE;
 
 	AZ(bo->stats);
@@ -950,7 +951,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	hp2 = req->obj->http;
 
 	hp2->logtag = HTTP_Obj;
-	http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
+	http_FilterResp(hp, hp2, bo->do_pass ? HTTPH_R_PASS : HTTPH_A_INS);
 	http_CopyHome(hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))



From tfheen at varnish-cache.org  Wed May 22 13:03:14 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 22 May 2013 15:03:14 +0200
Subject: [master] bf58f06 Return an error on duplicated Host headers
Message-ID: <E1Uf8hG-0006si-NU@project.varnish-software.com>

commit bf58f0613865246ab6eb812b6e8d888e4cf8ee95
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon May 6 09:17:43 2013 +0200

    Return an error on duplicated Host headers

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 18155e8..8907097 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -877,6 +877,7 @@ double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
 uint16_t http_GetStatus(const struct http *hp);
 const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
+int http_IsHdr(const txt *hh, const char *hdr);
 enum sess_close http_DoConnection(const struct http *);
 void http_CopyHome(const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 82d9ecf..3075fea 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -154,7 +154,7 @@ http_Teardown(struct http *hp)
 
 /*--------------------------------------------------------------------*/
 
-static int
+int
 http_IsHdr(const txt *hh, const char *hdr)
 {
 	unsigned l;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 00ed9bf..cf23904 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -376,6 +376,30 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 
 /*--------------------------------------------------------------------*/
 
+static int
+htc_request_check_host_hdr(struct http *hp)
+{
+	int u;
+	int seen_host = 0;
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		if (hp->hd[u].b == NULL)
+			continue;
+		AN(hp->hd[u].b);
+		AN(hp->hd[u].e);
+		if (http_IsHdr(&hp->hd[u], H_Host)) {
+			if (seen_host) {
+				VSLb(hp->vsl, SLT_Error, "Duplicated Host header");
+				return (400);
+			}
+			seen_host = 1;
+		}
+	}
+	return (0);
+}
+
+
+/*--------------------------------------------------------------------*/
+
 static void
 htc_proto_ver(struct http *hp)
 {
@@ -412,6 +436,11 @@ HTTP1_DissectRequest(struct req *req)
 	}
 	htc_proto_ver(hp);
 
+	retval = htc_request_check_host_hdr(hp);
+	if (retval != 0) {
+		return (retval);
+	}
+
 	/* RFC2616, section 5.2, point 1 */
 	if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) {
 		b = e = hp->hd[HTTP_HDR_URL].b + 7;



From phk at varnish-cache.org  Wed May 22 19:08:37 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 22 May 2013 21:08:37 +0200
Subject: [master] 486cf54 Give vcl_backend_(fetch|response){} the intended
 return methods.
Message-ID: <E1UfEOr-0008G5-UL@project.varnish-software.com>

commit 486cf5448db734ab35d3e07b9666509fe5bfab2f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 22 19:03:01 2013 +0000

    Give vcl_backend_(fetch|response){} the intended return methods.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3ab9cea..369ed21 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -763,10 +763,10 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	VBO_DerefBusyObj(wrk, &bo);
 
 	switch (wrk->handling) {
-	case VCL_RET_RESTART:
-		return (1);
 	case VCL_RET_ERROR:
 		return (-1);
+	case VCL_RET_RESTART:
+		INCOMPL();
 	default:
 		WRONG("Illegal action in vcl_fetch{}");
 	}
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9a4e27d..d8ba776 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -356,17 +356,14 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	i = VBF_Fetch(wrk, req);
-	if (i == 0) {
-		assert(WRW_IsReleased(wrk));
-		req->req_step = R_STP_PREPRESP;
-	} else if (i < 0) {
+	if (i < 0) {
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
 	} else {
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		assert(i == 1);
-		req->req_step = R_STP_RESTART;
+		AZ(i);
+		assert(WRW_IsReleased(wrk));
+		req->req_step = R_STP_PREPRESP;
 	}
 	return (REQ_FSM_MORE);
 }
diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc
index 29b1bfb..b8553e6 100644
--- a/bin/varnishtest/tests/v00017.vtc
+++ b/bin/varnishtest/tests/v00017.vtc
@@ -67,7 +67,7 @@ varnish v1 -vcl {
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { if (client.ip == "127.0.0.1") { return(pass); } }
-	sub vcl_backend_fetch { if (client.ip != "127.0.0.1") { return(pass); } }
+	sub vcl_backend_response { if (client.ip != "127.0.0.1") { return(restart); } }
 }
 
 varnish v1 -errvcl {Operator > not possible on IP} {
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 7671650..40fd1e1 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -85,8 +85,8 @@ returns =(
 	('purge',		"C", ('error', 'fetch',)),
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
 	('lookup',		"C", ('error', 'restart', 'pass', 'deliver',)),
-	('backend_fetch',	"B", ('fetch', 'pass',)),
-	('backend_response',	"B", ('deliver',)),
+	('backend_fetch',	"B", ('fetch', 'error')),
+	('backend_response',	"B", ('deliver', 'restart', 'error')),
 	('deliver',		"C", ('restart', 'deliver',)),
 	('error',		"C", ('restart', 'deliver',)),
 	('init',		"", ('ok',)),



From phk at varnish-cache.org  Wed May 22 19:58:44 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 22 May 2013 21:58:44 +0200
Subject: [master] f7592fd Polish
Message-ID: <E1UfFBM-0004l5-GZ@project.varnish-software.com>

commit f7592fd55ea8cb9eb062bf137042d0d21520f559
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 22 19:58:39 2013 +0000

    Polish

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index cf23904..c186646 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -377,7 +377,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 /*--------------------------------------------------------------------*/
 
 static int
-htc_request_check_host_hdr(struct http *hp)
+htc_request_check_host_hdr(const struct http *hp)
 {
 	int u;
 	int seen_host = 0;
@@ -388,7 +388,8 @@ htc_request_check_host_hdr(struct http *hp)
 		AN(hp->hd[u].e);
 		if (http_IsHdr(&hp->hd[u], H_Host)) {
 			if (seen_host) {
-				VSLb(hp->vsl, SLT_Error, "Duplicated Host header");
+				VSLb(hp->vsl, SLT_Error,
+				    "Duplicated Host header");
 				return (400);
 			}
 			seen_host = 1;
@@ -397,7 +398,6 @@ htc_request_check_host_hdr(struct http *hp)
 	return (0);
 }
 
-
 /*--------------------------------------------------------------------*/
 
 static void



From phk at varnish-cache.org  Wed May 22 20:42:21 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 22 May 2013 22:42:21 +0200
Subject: [master] ee3b325 Eliminate the objcore** arg from STV_MkObject() by
 putting it in busyobj.
Message-ID: <E1UfFrZ-0000Hp-Ky@project.varnish-software.com>

commit ee3b325eecbf3675ef649eea132f3816019a815b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 22 20:41:35 2013 +0000

    Eliminate the objcore** arg from STV_MkObject() by putting it in busyobj.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8907097..64f1eee 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -516,6 +516,7 @@ struct busyobj {
 	struct vbc		*vbc;
 	struct http		*bereq;
 	struct http		*beresp;
+	struct objcore		*fetch_objcore;
 	struct object		*fetch_obj;
 	uint8_t			digest[DIGEST_LEN];
 	struct exp		exp;
@@ -1080,7 +1081,7 @@ unsigned RFC2616_Req_Gzip(const struct http *);
 int RFC2616_Do_Cond(const struct req *sp);
 
 /* stevedore.c */
-struct object *STV_NewObject(struct busyobj *, struct objcore **,
+struct object *STV_NewObject(struct busyobj *,
     const char *hint, unsigned len, uint16_t nhttp);
 struct storage *STV_alloc(struct busyobj *, size_t size);
 void STV_trim(struct storage *st, size_t size, int move_ok);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 369ed21..66a7c72 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -637,8 +637,8 @@ vbf_make_bereq(struct worker *wrk, const struct req *req, struct busyobj *bo)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
 	AN(bo->director);
 	AZ(bo->vbc);
@@ -694,7 +694,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 		i = FetchHdr(wrk, bo, bo->do_pass ? req : NULL);
 	}
 
-	if (req->objcore->objhead != NULL)
+	if (bo->fetch_objcore->objhead != NULL)
 		(void)HTTP1_DiscardReqBody(req);	// XXX
 
 	if (!i) {
@@ -723,7 +723,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 		RFC2616_Ttl(bo);
 
 		/* pass from vclrecv{} has negative TTL */
-		if (req->objcore->objhead == NULL)
+		if (bo->fetch_objcore->objhead == NULL)
 			bo->exp.ttl = -1.;
 
 		AZ(bo->do_esi);
@@ -735,7 +735,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 		bo->do_pass |= i;
 
 		if (bo->do_pass)
-			req->objcore->flags |= OC_F_PASS;
+			bo->fetch_objcore->flags |= OC_F_PASS;
 
 		if (wrk->handling == VCL_RET_DELIVER)
 			return (0);
@@ -750,12 +750,13 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	/* Clean up partial fetch */
 	AZ(bo->vbc);
 
-	if (req->objcore->objhead != NULL ||
+	if (bo->fetch_objcore->objhead != NULL ||
 	    wrk->handling == VCL_RET_RESTART ||
 	    wrk->handling == VCL_RET_ERROR) {
-		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+		CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
+		bo->fetch_objcore = NULL;
 	}
 	assert(bo->refcount == 2);
 	bo->storage_hint = NULL;
@@ -793,6 +794,8 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	xxxassert(bo->refcount == 2);	// Req might abandon early ?
 	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 
+	bo->fetch_objcore = req->objcore;
+
 	vbf_make_bereq(wrk, req, bo);
 	xxxassert (wrk->handling == VCL_RET_FETCH);
 
@@ -800,7 +803,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	if (i)
 		return (i);
 
-	if (req->objcore->objhead == NULL) {
+	if (bo->fetch_objcore->objhead == NULL) {
 		AN(bo->do_pass);
 		/* This is a pass from vcl_recv */
 		bo->do_pass = 1;
@@ -874,7 +877,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
-	if (req->objcore->objhead != NULL) {
+	if (bo->fetch_objcore->objhead != NULL) {
 		varyl = VRY_Create(bo, &vary);
 		if (varyl > 0) {
 			AN(vary);
@@ -885,6 +888,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 			AZ(vary);
 			AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 			req->objcore = NULL;
+			bo->fetch_objcore = NULL;
 			VDI_CloseFd(&bo->vbc);
 			return (-1);
 		} else
@@ -903,8 +907,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 
 	AZ(bo->stats);
 	bo->stats = &wrk->stats;
-	req->obj = STV_NewObject(bo, &req->objcore, bo->storage_hint, l,
-	    nhttp);
+	req->obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
 	if (req->obj == NULL) {
 		/*
 		 * Try to salvage the transaction by allocating a
@@ -914,18 +917,19 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 			bo->exp.ttl = cache_param->shortlived;
 		bo->exp.grace = 0.0;
 		bo->exp.keep = 0.0;
-		req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
-		    l, nhttp);
+		req->obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
 	}
 	bo->stats = NULL;
 	if (req->obj == NULL) {
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
+		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
 		return (-1);
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 
+	req->objcore = NULL;
 	bo->storage_hint = NULL;
 
 	AZ(bo->fetch_obj);
@@ -974,8 +978,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	/*
 	 * Ready to fetch the body
 	 */
-	bo->fetch_task.func = FetchBody;
-	bo->fetch_task.priv = bo;
 
 	assert(bo->refcount == 2);	/* one for each thread */
 
@@ -986,9 +988,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		HSH_Unbusy(&wrk->stats, req->obj->objcore);
 	}
 
-	if (!bo->do_stream ||
-	    Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
-		FetchBody(wrk, bo);
+	FetchBody(wrk, bo);
 
 	if (req->obj->objcore->objhead != NULL)
 		HSH_Ref(req->obj->objcore);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d8ba776..e789ffe 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -273,16 +273,16 @@ cnt_error(struct worker *wrk, struct req *req)
 	req->busyobj = bo;
 	AZ(bo->stats);
 	bo->stats = &wrk->stats;
-	req->objcore = HSH_NewObjCore(wrk);
-	req->obj = STV_NewObject(bo, &req->objcore,
+	bo->fetch_objcore = HSH_NewObjCore(wrk);
+	req->obj = STV_NewObject(bo,
 	    TRANSIENT_STORAGE, cache_param->http_resp_size,
 	    (uint16_t)cache_param->http_max_hdr);
 	bo->stats = NULL;
 	if (req->obj == NULL) {
 		req->doclose = SC_OVERLOAD;
 		req->director = NULL;
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
+		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
+		bo->fetch_objcore = NULL;
 		http_Teardown(bo->beresp);
 		http_Teardown(bo->bereq);
 		VBO_DerefBusyObj(wrk, &req->busyobj);
@@ -291,6 +291,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	AZ(req->objcore);
+	AZ(bo->fetch_objcore);
 	req->obj->vxid = bo->vsl->wid;
 	req->obj->exp.entered = req->t_req;
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index fa0fdcf..407470e 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -252,7 +252,7 @@ struct stv_objsecrets {
  */
 
 struct object *
-STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
+STV_MkObject(struct stevedore *stv, struct busyobj *bo,
     void *ptr, unsigned ltot, const struct stv_objsecrets *soc)
 {
 	struct object *o;
@@ -261,8 +261,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
-	AN(ocp);
-	CHECK_OBJ_NOTNULL((*ocp), OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
 	assert(PAOK(ptr));
 	assert(PAOK(soc->wsl));
@@ -288,8 +287,8 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	VTAILQ_INIT(&o->store);
 	bo->stats->n_object++;
 
-	o->objcore = *ocp;
-	*ocp = NULL;     /* refcnt follows pointer. */
+	o->objcore = bo->fetch_objcore;
+	bo->fetch_objcore = NULL;     /* refcnt follows pointer. */
 	if (o->objcore->objhead != NULL)
 		BAN_NewObjCore(o->objcore);
 
@@ -306,14 +305,13 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 
 struct object *
 stv_default_allocobj(struct stevedore *stv, struct busyobj *bo,
-    struct objcore **ocp, unsigned ltot, const struct stv_objsecrets *soc)
+    unsigned ltot, const struct stv_objsecrets *soc)
 {
 	struct object *o;
 	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AN(ocp);
 	st = stv->alloc(stv, ltot);
 	if (st == NULL)
 		return (NULL);
@@ -322,7 +320,7 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo,
 		return (NULL);
 	}
 	ltot = st->len = st->space;
-	o = STV_MkObject(stv, bo, ocp, st->ptr, ltot, soc);
+	o = STV_MkObject(stv, bo, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 	return (o);
@@ -335,7 +333,7 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo,
  */
 
 struct object *
-STV_NewObject(struct busyobj *bo, struct objcore **ocp, const char *hint,
+STV_NewObject(struct busyobj *bo, const char *hint,
     unsigned wsl, uint16_t nhttp)
 {
 	struct object *o;
@@ -345,7 +343,6 @@ STV_NewObject(struct busyobj *bo, struct objcore **ocp, const char *hint,
 	int i;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AN(ocp);
 	assert(wsl > 0);
 	wsl = PRNDUP(wsl);
 
@@ -362,12 +359,12 @@ STV_NewObject(struct busyobj *bo, struct objcore **ocp, const char *hint,
 
 	stv = stv0 = stv_pick_stevedore(bo->vsl, &hint);
 	AN(stv->allocobj);
-	o = stv->allocobj(stv, bo, ocp, ltot, &soc);
+	o = stv->allocobj(stv, bo, ltot, &soc);
 	if (o == NULL && hint == NULL) {
 		do {
 			stv = stv_pick_stevedore(bo->vsl, &hint);
 			AN(stv->allocobj);
-			o = stv->allocobj(stv, bo, ocp, ltot, &soc);
+			o = stv->allocobj(stv, bo, ltot, &soc);
 		} while (o == NULL && stv != stv0);
 	}
 	if (o == NULL) {
@@ -375,15 +372,13 @@ STV_NewObject(struct busyobj *bo, struct objcore **ocp, const char *hint,
 		for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) {
 			if (EXP_NukeOne(bo, stv->lru) == -1)
 				break;
-			o = stv->allocobj(stv, bo, ocp, ltot, &soc);
+			o = stv->allocobj(stv, bo, ltot, &soc);
 		}
 	}
 
 	if (o == NULL) {
-		AN(*ocp);
 		return (NULL);
 	}
-	AZ(*ocp);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
 	return (o);
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index be6078b..47fabad 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -45,7 +45,7 @@ typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
 typedef void storage_trim_f(struct storage *, size_t size, int move_ok);
 typedef void storage_free_f(struct storage *);
 typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *,
-    struct objcore **, unsigned ltot, const struct stv_objsecrets *);
+    unsigned ltot, const struct stv_objsecrets *);
 typedef void storage_close_f(const struct stevedore *);
 typedef void storage_signal_close_f(const struct stevedore *);
 typedef int storage_baninfo_f(const struct stevedore *, enum baninfo event,
@@ -101,8 +101,7 @@ int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
 uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
     const char *ctx);
 struct object *STV_MkObject(struct stevedore *stv, struct busyobj *bo,
-    struct objcore **ocp, void *ptr, unsigned ltot,
-    const struct stv_objsecrets *soc);
+    void *ptr, unsigned ltot, const struct stv_objsecrets *soc);
 
 struct lru *LRU_Alloc(void);
 void LRU_Free(struct lru *lru);
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 55279c8..6c7f015 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -506,7 +506,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
  */
 
 static struct object *
-smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
+smp_allocobj(struct stevedore *stv, struct busyobj *bo, 
     unsigned ltot, const struct stv_objsecrets *soc)
 {
 	struct object *o;
@@ -517,8 +517,8 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	struct objcore *oc;
 	unsigned objidx;
 
-	AN(ocp);
-	if (*ocp == NULL)
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (bo->fetch_objcore == NULL)
 		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
 	AN(bo->exp.ttl > 0.);
@@ -532,7 +532,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	assert(st->space >= ltot);
 	ltot = st->len = st->space;
 
-	o = STV_MkObject(stv, bo, ocp, st->ptr, ltot, soc);
+	o = STV_MkObject(stv, bo, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 



From phk at varnish-cache.org  Wed May 22 20:56:10 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 22 May 2013 22:56:10 +0200
Subject: [master] eef78be Don't steal the req's ref on the busyobj
Message-ID: <E1UfG4w-0001qW-AF@project.varnish-software.com>

commit eef78bea43d2e234faf8244b8c1602fbc87c62f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 22 20:55:57 2013 +0000

    Don't steal the req's ref on the busyobj

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 66a7c72..4bf15de 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -990,12 +990,12 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 
 	FetchBody(wrk, bo);
 
+	assert(bo->refcount == 1);
+
 	if (req->obj->objcore->objhead != NULL)
 		HSH_Ref(req->obj->objcore);
 
-	if (bo->state == BOS_FINISHED) {
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-	} else if (bo->state == BOS_FAILED) {
+	if (bo->state == BOS_FAILED) {
 		/* handle early failures */
 		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
 		return (-1);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index e789ffe..6f4e49b 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -357,12 +357,15 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	i = VBF_Fetch(wrk, req);
+	AN(req->busyobj);
+	assert(req->busyobj->refcount > 0);
 	if (i < 0) {
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
 	} else {
 		AZ(i);
+		VBO_DerefBusyObj(wrk, &req->busyobj);
 		assert(WRW_IsReleased(wrk));
 		req->req_step = R_STP_PREPRESP;
 	}



From phk at varnish-cache.org  Thu May 23 07:30:24 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 09:30:24 +0200
Subject: [master] 2643a6c Polish
Message-ID: <E1UfPyi-0004xw-LW@project.varnish-software.com>

commit 2643a6cad0aed05eab549dd348e849addd9096bc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 07:30:16 2013 +0000

    Polish

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 4bf15de..3fcf0ba 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -791,7 +791,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	xxxassert(bo->refcount == 2);	// Req might abandon early ?
+	assert(bo->refcount == 2);
 	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 
 	bo->fetch_objcore = req->objcore;
@@ -803,13 +803,8 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	if (i)
 		return (i);
 
-	if (bo->fetch_objcore->objhead == NULL) {
+	if (bo->fetch_objcore->objhead == NULL) 
 		AN(bo->do_pass);
-		/* This is a pass from vcl_recv */
-		bo->do_pass = 1;
-		/* VCL may have fiddled this, but that doesn't help */
-		// bo->exp.ttl = -1.;
-	}
 
 	/*
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 6f4e49b..86586da 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -356,6 +356,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
+	AN(req->busyobj);
+	AN(req->objcore);
 	i = VBF_Fetch(wrk, req);
 	AN(req->busyobj);
 	assert(req->busyobj->refcount > 0);



From phk at varnish-cache.org  Thu May 23 07:52:52 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 09:52:52 +0200
Subject: [master] 570d235 More filing away at the req/busyobj separation
Message-ID: <E1UfQKS-0007Qu-Pp@project.varnish-software.com>

commit 570d23526ab68294a25f7637c9b833142b263ef5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 07:52:39 2013 +0000

    More filing away at the req/busyobj separation

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3fcf0ba..2a46f9b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -783,6 +783,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	struct vsb *vary = NULL;
 	int varyl = 0;
 	struct busyobj *bo;
+	struct object *obj;
 	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -802,6 +803,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	i = cnt_fetch(wrk, req, bo);
 	if (i)
 		return (i);
+	req->objcore = NULL;
 
 	if (bo->fetch_objcore->objhead == NULL) 
 		AN(bo->do_pass);
@@ -881,8 +883,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		} else if (varyl < 0) {
 			/* Vary parse error */
 			AZ(vary);
-			AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-			req->objcore = NULL;
+			AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
 			bo->fetch_objcore = NULL;
 			VDI_CloseFd(&bo->vbc);
 			return (-1);
@@ -902,8 +903,8 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 
 	AZ(bo->stats);
 	bo->stats = &wrk->stats;
-	req->obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
-	if (req->obj == NULL) {
+	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
+	if (obj == NULL) {
 		/*
 		 * Try to salvage the transaction by allocating a
 		 * shortlived object on Transient storage.
@@ -912,63 +913,63 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 			bo->exp.ttl = cache_param->shortlived;
 		bo->exp.grace = 0.0;
 		bo->exp.keep = 0.0;
-		req->obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
+		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
 	}
 	bo->stats = NULL;
-	if (req->obj == NULL) {
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
+	if (obj == NULL) {
+		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
 		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
 		return (-1);
 	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 
-	req->objcore = NULL;
 	bo->storage_hint = NULL;
 
 	AZ(bo->fetch_obj);
-	bo->fetch_obj = req->obj;
+	bo->fetch_obj = obj;
 
 	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
-		req->obj->gziped = 1;
+		obj->gziped = 1;
 
 	if (vary != NULL) {
-		req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
+		obj->vary = (void *)WS_Copy(obj->http->ws,
 		    VSB_data(vary), varyl);
-		AN(req->obj->vary);
-		VRY_Validate(req->obj->vary);
+		AN(obj->vary);
+		VRY_Validate(obj->vary);
 		VSB_delete(vary);
 	}
 
-	req->obj->vxid = bo->vsl->wid;
-	req->obj->response = req->err_code;
-	WS_Assert(req->obj->ws_o);
+	obj->vxid = bo->vsl->wid;
+	obj->response = req->err_code;
+	WS_Assert(obj->ws_o);
 
 	/* Filter into object */
 	hp = bo->beresp;
-	hp2 = req->obj->http;
+	hp2 = obj->http;
 
 	hp2->logtag = HTTP_Obj;
 	http_FilterResp(hp, hp2, bo->do_pass ? HTTPH_R_PASS : HTTPH_A_INS);
 	http_CopyHome(hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
-		req->obj->last_modified = VTIM_parse(b);
+		obj->last_modified = VTIM_parse(b);
 	else
-		req->obj->last_modified = floor(bo->exp.entered);
+		obj->last_modified = floor(bo->exp.entered);
 
 	assert(WRW_IsReleased(wrk));
 
+#if 0
 	/*
 	 * If we can deliver a 304 reply, we don't bother streaming.
 	 * Notice that vcl_deliver{} could still nuke the headers
 	 * that allow the 304, in which case we return 200 non-stream.
 	 */
-	if (req->obj->response == 200 &&
+	if (obj->response == 200 &&
 	    req->http->conds &&
 	    RFC2616_Do_Cond(req))
 		bo->do_stream = 0;
+#endif
 
 	/*
 	 * Ready to fetch the body
@@ -976,23 +977,23 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 
 	assert(bo->refcount == 2);	/* one for each thread */
 
-	if (req->obj->objcore->objhead != NULL) {
-		EXP_Insert(req->obj);
-		AN(req->obj->objcore->ban);
-		AZ(req->obj->ws_o->overflow);
-		HSH_Unbusy(&wrk->stats, req->obj->objcore);
+	if (obj->objcore->objhead != NULL) {
+		EXP_Insert(obj);
+		AN(obj->objcore->ban);
+		AZ(obj->ws_o->overflow);
+		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
 
 	FetchBody(wrk, bo);
 
 	assert(bo->refcount == 1);
 
-	if (req->obj->objcore->objhead != NULL)
-		HSH_Ref(req->obj->objcore);
+	if (obj->objcore->objhead != NULL)
+		HSH_Ref(obj->objcore);
 
 	if (bo->state == BOS_FAILED) {
 		/* handle early failures */
-		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+		(void)HSH_Deref(&wrk->stats, NULL, &obj);
 		return (-1);
 	}
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 86586da..0ac775e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -367,6 +367,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_ERROR;
 	} else {
 		AZ(i);
+		req->obj = req->busyobj->fetch_obj;
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		assert(WRW_IsReleased(wrk));
 		req->req_step = R_STP_PREPRESP;



From phk at varnish-cache.org  Thu May 23 08:13:32 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 10:13:32 +0200
Subject: [master] dc8f77f More pushing of req/busobj split
Message-ID: <E1UfQeS-0001BN-Lj@project.varnish-software.com>

commit dc8f77f7bce9d6187035fb7b1320381490f48291
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 08:13:19 2013 +0000

    More pushing of req/busobj split

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 64f1eee..1c0dbce 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -502,6 +502,7 @@ struct busyobj {
 	 */
 	unsigned		refcount;
 	double			t_fetch;
+	uint16_t		err_code;
 
 	uint8_t			*vary;
 	unsigned		is_gzip;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2a46f9b..6018879 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -681,8 +681,6 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
-	req->acct_req.fetch++;
-
 	i = FetchHdr(wrk, bo, bo->do_pass ? req : NULL);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
@@ -713,7 +711,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 		 */
 		bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
 
-		req->err_code = http_GetStatus(bo->beresp);
+		bo->err_code = http_GetStatus(bo->beresp);
 
 		/*
 		 * What does RFC2616 think about TTL ?
@@ -744,7 +742,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 		VDI_CloseFd(&bo->vbc);
 	} else {
 		wrk->handling = VCL_RET_ERROR;
-		req->err_code = 503;
+		bo->err_code = 503;
 	}
 
 	/* Clean up partial fetch */
@@ -754,8 +752,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	    wrk->handling == VCL_RET_RESTART ||
 	    wrk->handling == VCL_RET_ERROR) {
 		CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
+		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
 		bo->fetch_objcore = NULL;
 	}
 	assert(bo->refcount == 2);
@@ -796,6 +793,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 
 	bo->fetch_objcore = req->objcore;
+	req->objcore = NULL;
 
 	vbf_make_bereq(wrk, req, bo);
 	xxxassert (wrk->handling == VCL_RET_FETCH);
@@ -803,7 +801,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	i = cnt_fetch(wrk, req, bo);
 	if (i)
 		return (i);
-	req->objcore = NULL;
 
 	if (bo->fetch_objcore->objhead == NULL) 
 		AN(bo->do_pass);
@@ -941,7 +938,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	}
 
 	obj->vxid = bo->vsl->wid;
-	obj->response = req->err_code;
+	obj->response = bo->err_code;
 	WS_Assert(obj->ws_o);
 
 	/* Filter into object */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 0ac775e..9f1df09 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -358,6 +358,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	AN(req->busyobj);
 	AN(req->objcore);
+	req->acct_req.fetch++;
 	i = VBF_Fetch(wrk, req);
 	AN(req->busyobj);
 	assert(req->busyobj->refcount > 0);
@@ -367,6 +368,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_ERROR;
 	} else {
 		AZ(i);
+		req->err_code = req->busyobj->err_code;
 		req->obj = req->busyobj->fetch_obj;
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		assert(WRW_IsReleased(wrk));



From phk at varnish-cache.org  Thu May 23 08:16:54 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 10:16:54 +0200
Subject: [master] 4b3d70b Polish
Message-ID: <E1UfQhi-0001T6-RU@project.varnish-software.com>

commit 4b3d70bd496346651a768b54bb0cd3851e2138a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 08:16:49 2013 +0000

    Polish

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index c186646..3371089 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -376,7 +376,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 
 /*--------------------------------------------------------------------*/
 
-static int
+static uint16_t
 htc_request_check_host_hdr(const struct http *hp)
 {
 	int u;



From phk at varnish-cache.org  Thu May 23 08:55:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 10:55:09 +0200
Subject: [master] 1016485 Make the req responsisble for getting rid of its own
 body.
Message-ID: <E1UfRIj-0004VP-1D@project.varnish-software.com>

commit 10164855643afc42364c234428e5133143b8175d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 08:54:52 2013 +0000

    Make the req responsisble for getting rid of its own body.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 6018879..375c1d8 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -681,7 +681,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
-	i = FetchHdr(wrk, bo, bo->do_pass ? req : NULL);
+	i = FetchHdr(wrk, bo, req);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -689,12 +689,9 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(wrk, bo, bo->do_pass ? req : NULL);
+		i = FetchHdr(wrk, bo, req);
 	}
 
-	if (bo->fetch_objcore->objhead != NULL)
-		(void)HTTP1_DiscardReqBody(req);	// XXX
-
 	if (!i) {
 		/*
 		 * These two headers can be spread over multiple actual headers
@@ -798,7 +795,8 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	vbf_make_bereq(wrk, req, bo);
 	xxxassert (wrk->handling == VCL_RET_FETCH);
 
-	i = cnt_fetch(wrk, req, bo);
+	i = cnt_fetch(wrk, bo->do_pass ? req : NULL, bo);
+
 	if (i)
 		return (i);
 
@@ -858,11 +856,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	else if (bo->is_gzip)
 		bo->vfp = &vfp_testgzip;
 
-	if (bo->do_esi || req->esi_level > 0)
-		bo->do_stream = 0;
-	if (!req->wantbody)
-		bo->do_stream = 0;
-
 	/* No reason to try streaming a non-existing body */
 	if (bo->htc.body_status == BS_NONE)
 		bo->do_stream = 0;
@@ -956,18 +949,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 
 	assert(WRW_IsReleased(wrk));
 
-#if 0
-	/*
-	 * If we can deliver a 304 reply, we don't bother streaming.
-	 * Notice that vcl_deliver{} could still nuke the headers
-	 * that allow the 304, in which case we return 200 non-stream.
-	 */
-	if (obj->response == 200 &&
-	    req->http->conds &&
-	    RFC2616_Do_Cond(req))
-		bo->do_stream = 0;
-#endif
-
 	/*
 	 * Ready to fetch the body
 	 */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9f1df09..2def95e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -362,6 +362,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	i = VBF_Fetch(wrk, req);
 	AN(req->busyobj);
 	assert(req->busyobj->refcount > 0);
+	(void)HTTP1_DiscardReqBody(req);
 	if (i < 0) {
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;



From phk at varnish-cache.org  Thu May 23 13:41:14 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 15:41:14 +0200
Subject: [master] 8b89950 Collect all the beresp. policy code together
Message-ID: <E1UfVla-0007WZ-PT@project.varnish-software.com>

commit 8b89950de440fb869bdbd85869465ad2ce662a70
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 12:46:39 2013 +0000

    Collect all the beresp. policy code together

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 375c1d8..3b27ab0 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -674,6 +674,110 @@ vbf_make_bereq(struct worker *wrk, const struct req *req, struct busyobj *bo)
 /*--------------------------------------------------------------------
  */
 
+static void
+vbf_proc_resp(struct worker *wrk, struct busyobj *bo)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	/*
+	 * These two headers can be spread over multiple actual headers
+	 * and we rely on their content outside of VCL, so collect them
+	 * into one line here.
+	 */
+	http_CollectHdr(bo->beresp, H_Cache_Control);
+	http_CollectHdr(bo->beresp, H_Vary);
+
+	/*
+	 * Figure out how the fetch is supposed to happen, before the
+	 * headers are adultered by VCL
+	 * NB: Also sets other wrk variables
+	 */
+	bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
+
+	bo->err_code = http_GetStatus(bo->beresp);
+
+	/*
+	 * What does RFC2616 think about TTL ?
+	 */
+	EXP_Clr(&bo->exp);
+	bo->exp.entered = W_TIM_real(wrk);
+	RFC2616_Ttl(bo);
+
+	/* pass from vclrecv{} has negative TTL */
+	if (bo->fetch_objcore->objhead == NULL)
+		bo->exp.ttl = -1.;
+
+	AZ(bo->do_esi);
+
+	// Don't let VCL reset do_pass
+	i = bo->do_pass;
+	VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
+	    bo->beresp->ws);
+	bo->do_pass |= i;
+
+	if (bo->do_pass)
+		bo->fetch_objcore->flags |= OC_F_PASS;
+
+	/*
+	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+	 * object processed before it is stored.
+	 *
+	 * The backend Content-Encoding header tells us what we are going
+	 * to receive, which we classify in the following three classes:
+	 *
+	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 *	no Content-Encoding		--> object is not gzip'ed.
+	 *	anything else			--> do nothing wrt gzip
+	 *
+	 */
+
+	/* We do nothing unless the param is set */
+	if (!cache_param->http_gzip_support)
+		bo->do_gzip = bo->do_gunzip = 0;
+
+	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
+
+	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
+
+	/* It can't be both */
+	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
+
+	/* We won't gunzip unless it is gzip'ed */
+	if (bo->do_gunzip && !bo->is_gzip)
+		bo->do_gunzip = 0;
+
+	/* If we do gunzip, remove the C-E header */
+	if (bo->do_gunzip)
+		http_Unset(bo->beresp, H_Content_Encoding);
+
+	/* We wont gzip unless it is ungziped */
+	if (bo->do_gzip && !bo->is_gunzip)
+		bo->do_gzip = 0;
+
+	/* If we do gzip, add the C-E header */
+	if (bo->do_gzip)
+		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
+
+	/* But we can't do both at the same time */
+	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
+
+	/* ESI takes precedence and handles gzip/gunzip itself */
+	if (bo->do_esi)
+		bo->vfp = &vfp_esi;
+	else if (bo->do_gunzip)
+		bo->vfp = &vfp_gunzip;
+	else if (bo->do_gzip)
+		bo->vfp = &vfp_gzip;
+	else if (bo->is_gzip)
+		bo->vfp = &vfp_testgzip;
+
+}
+
+/*--------------------------------------------------------------------
+ */
+
 static int
 cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 {
@@ -693,44 +797,7 @@ cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
 	}
 
 	if (!i) {
-		/*
-		 * These two headers can be spread over multiple actual headers
-		 * and we rely on their content outside of VCL, so collect them
-		 * into one line here.
-		 */
-		http_CollectHdr(bo->beresp, H_Cache_Control);
-		http_CollectHdr(bo->beresp, H_Vary);
-
-		/*
-		 * Figure out how the fetch is supposed to happen, before the
-		 * headers are adultered by VCL
-		 * NB: Also sets other wrk variables
-		 */
-		bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
-
-		bo->err_code = http_GetStatus(bo->beresp);
-
-		/*
-		 * What does RFC2616 think about TTL ?
-		 */
-		EXP_Clr(&bo->exp);
-		bo->exp.entered = W_TIM_real(wrk);
-		RFC2616_Ttl(bo);
-
-		/* pass from vclrecv{} has negative TTL */
-		if (bo->fetch_objcore->objhead == NULL)
-			bo->exp.ttl = -1.;
-
-		AZ(bo->do_esi);
-
-		// Don't let VCL reset do_pass
-		i = bo->do_pass;
-		VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
-		    bo->beresp->ws);
-		bo->do_pass |= i;
-
-		if (bo->do_pass)
-			bo->fetch_objcore->flags |= OC_F_PASS;
+		vbf_proc_resp(wrk, bo);
 
 		if (wrk->handling == VCL_RET_DELIVER)
 			return (0);
@@ -803,59 +870,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	if (bo->fetch_objcore->objhead == NULL) 
 		AN(bo->do_pass);
 
-	/*
-	 * The VCL variables beresp.do_g[un]zip tells us how we want the
-	 * object processed before it is stored.
-	 *
-	 * The backend Content-Encoding header tells us what we are going
-	 * to receive, which we classify in the following three classes:
-	 *
-	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
-	 *	no Content-Encoding		--> object is not gzip'ed.
-	 *	anything else			--> do nothing wrt gzip
-	 *
-	 */
-
-	/* We do nothing unless the param is set */
-	if (!cache_param->http_gzip_support)
-		bo->do_gzip = bo->do_gunzip = 0;
-
-	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
-
-	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
-
-	/* It can't be both */
-	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
-
-	/* We won't gunzip unless it is gzip'ed */
-	if (bo->do_gunzip && !bo->is_gzip)
-		bo->do_gunzip = 0;
-
-	/* If we do gunzip, remove the C-E header */
-	if (bo->do_gunzip)
-		http_Unset(bo->beresp, H_Content_Encoding);
-
-	/* We wont gzip unless it is ungziped */
-	if (bo->do_gzip && !bo->is_gunzip)
-		bo->do_gzip = 0;
-
-	/* If we do gzip, add the C-E header */
-	if (bo->do_gzip)
-		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
-
-	/* But we can't do both at the same time */
-	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
-
-	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (bo->do_esi)
-		bo->vfp = &vfp_esi;
-	else if (bo->do_gunzip)
-		bo->vfp = &vfp_gunzip;
-	else if (bo->do_gzip)
-		bo->vfp = &vfp_gzip;
-	else if (bo->is_gzip)
-		bo->vfp = &vfp_testgzip;
-
 	/* No reason to try streaming a non-existing body */
 	if (bo->htc.body_status == BS_NONE)
 		bo->do_stream = 0;
@@ -882,15 +896,16 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 			AZ(vary);
 	}
 
+	if (bo->exp.ttl < cache_param->shortlived || bo->do_pass == 1)
+		bo->storage_hint = TRANSIENT_STORAGE;
+
+
 	/*
 	 * Space for producing a Content-Length: header including padding
 	 * A billion gigabytes is enough for anybody.
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (bo->exp.ttl < cache_param->shortlived || bo->do_pass == 1)
-		bo->storage_hint = TRANSIENT_STORAGE;
-
 	AZ(bo->stats);
 	bo->stats = &wrk->stats;
 	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
@@ -979,7 +994,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	return (0);
 }
 
-
 /*--------------------------------------------------------------------
  * Debugging aids
  */



From phk at varnish-cache.org  Thu May 23 15:54:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 23 May 2013 17:54:26 +0200
Subject: [master] cf6f2af Polish
Message-ID: <E1UfXqU-0003cK-6Q@project.varnish-software.com>

commit cf6f2af9f3e02c5b35e279fd64fc546c5111a153
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 23 15:54:17 2013 +0000

    Polish

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3b27ab0..f48fb20 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -368,7 +368,6 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	int retry = -1;
 	int i, first;
 	struct http_conn *htc;
-        int need_host_hdr;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -377,8 +376,6 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 
 	AN(bo->director);
 
-	need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
-
 	hp = bo->bereq;
 
 	bo->vbc = VDI_GetFd(NULL, bo);
@@ -395,7 +392,7 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	 * header if one is necessary.  This cannot be done in the VCL
 	 * because the backend may be chosen by a director.
 	 */
-	if (need_host_hdr)
+	if (!http_GetHdr(bo->bereq, H_Host, NULL))
 		VDI_AddHostHeader(bo->bereq, vc);
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
@@ -899,7 +896,6 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	if (bo->exp.ttl < cache_param->shortlived || bo->do_pass == 1)
 		bo->storage_hint = TRANSIENT_STORAGE;
 
-
 	/*
 	 * Space for producing a Content-Length: header including padding
 	 * A billion gigabytes is enough for anybody.



From tfheen at varnish-cache.org  Fri May 24 10:23:36 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 24 May 2013 12:23:36 +0200
Subject: [master] 22035ff Add cscope support
Message-ID: <E1Ufp9s-0002fE-Q9@project.varnish-software.com>

commit 22035ff1cddb58a1237d7c717b90e52db3784ae1
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 23 09:47:19 2013 +0200

    Add cscope support

diff --git a/.gitignore b/.gitignore
index 4ec89a6..cc3ac5c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,6 +36,7 @@ Makefile.in
 /stamp-h1
 /varnishapi.pc
 TAGS
+cscope.*out
 
 # Default vcl made from bin/varnishd/default.vcl
 /bin/varnishd/default_vcl.h
diff --git a/Makefile.am b/Makefile.am
index 44b2481..87bbe7a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -7,6 +7,7 @@ SUBDIRS += redhat
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = varnishapi.pc
 
+CLEANFILES = cscope.in.out cscope.out cscope.po.out
 EXTRA_DIST = LICENSE autogen.sh varnishapi.pc.in
 
 DISTCHECK_CONFIGURE_FLAGS = \
@@ -34,3 +35,10 @@ distcheck-hook:
 distcleancheck_listfiles = \
 	find . -type f -exec sh -c 'test -f $(srcdir)/$$1 || echo $$1' \
 		sh '{}' ';'
+
+cscope:
+	-rm -f cscope*
+	find . -name '*.[hcS]' > cscope.files
+	cscope -b
+
+.PHONY: cscope



From tfheen at varnish-cache.org  Fri May 24 10:23:36 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 24 May 2013 12:23:36 +0200
Subject: [master] c259f85 Add a -N option for use with utilities. This takes a
 VSM filename as argument, and will also disable the abandonment checks.
Message-ID: <E1Ufp9s-0002fJ-Rj@project.varnish-software.com>

commit c259f855945a6e0097956e1095ff85f8d40312d2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu May 23 15:12:28 2013 +0200

    Add a -N option for use with utilities. This takes a VSM filename as argument, and will also disable the abandonment checks.
    
    This allows reading a stale VSM file (e.g. _VSM.keep saved from a
    panic) for forensics gathering.

diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index 3c5d541..703df32 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -19,7 +19,7 @@ Varnish Cache statistics
 SYNOPSIS
 ========
 
-varnishstat [-1] [-x] [-j] [-f field_list] [-l] [-n varnish_name] [-V] [-w delay]
+varnishstat [-1] [-x] [-j] [-f field_list] [-l] [-n varnish_name] [-N filename] [-V] [-w delay]
 
 DESCRIPTION
 ===========
@@ -38,6 +38,9 @@ The following options are available:
 -n          Specifies the name of the varnishd instance to get logs from.  If -n is not specified, the host name
 	    is used.
 
+-N          Specify a the filename of a stale VSM instance. When using this
+            option the abandonment checking is disabled.
+
 -V          Display the version number and exit.
 
 -w delay    Wait delay seconds between updates.  The default is 1. Can also be used with -1, -x or -j for repeated output.
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index e77ef64..ad5ee26 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -44,8 +44,8 @@ struct VSM_fantom;
  * VSC level access functions
  */
 
-#define VSC_ARGS	"f:n:"
-#define VSC_n_USAGE	VSM_n_USAGE
+#define VSC_ARGS	"f:n:N:"
+#define VSC_n_USAGE	VSM_n_USAGE " " VSM_N_USAGE
 #define VSC_f_USAGE	"[-f field_name,...]"
 #define VSC_USAGE	VSC_n_USAGE \
 			VSC_f_USAGE
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index aa58835..caddcb9 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -96,6 +96,20 @@ int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
 	 *	 <0 on failure, VSM_Error() returns diagnostic string
 	 */
 
+#define VSM_N_USAGE	"[-N filename]"
+
+int VSM_N_Arg(struct VSM_data *vd, const char *N_arg);
+	/*
+	 * Configure the library to use the specified VSM file name. This
+	 * bypasses abandonment checks and allows looking at stale VSM
+	 * files without a running Varnish instance.
+	 *
+	 * Can also be, and normally is done through VSC_Arg()/VSL_Arg().
+	 *
+	 * Returns:
+	 *	1 on success
+	 */
+
 const char *VSM_Name(const struct VSM_data *vd);
 	/*
 	 * Return the instance name.
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 06c11ce..ec4b385 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -253,6 +253,7 @@ VSC_Arg(struct VSM_data *vd, int arg, const char *opt)
 	switch (arg) {
 	case 'f': return (vsc_f_arg(vd, opt));
 	case 'n': return (VSM_n_Arg(vd, opt));
+	case 'N': return (VSM_N_Arg(vd, opt));
 	default:
 		return (0);
 	}
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index ec73a98..f0b2602 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -141,6 +141,20 @@ VSM_n_Arg(struct VSM_data *vd, const char *opt)
 
 /*--------------------------------------------------------------------*/
 
+int
+VSM_N_Arg(struct VSM_data *vd, const char *opt)
+{
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	AN(opt);
+
+	REPLACE(vd->fname, opt);
+	vd->N_opt = 1;
+	return (1);
+}
+
+/*--------------------------------------------------------------------*/
+
 const char *
 VSM_Name(const struct VSM_data *vd)
 {
@@ -191,7 +205,7 @@ VSM_Open(struct VSM_data *vd)
 		/* Already open */
 		return (0);
 
-	if (!vd->n_opt)
+	if (!vd->n_opt && !vd->N_opt)
 		(void)VSM_n_Arg(vd, "");
 
 	AZ(vd->head);
@@ -224,7 +238,7 @@ VSM_Open(struct VSM_data *vd)
 		return (vsm_diag(vd, "Not a VSM file %s\n", vd->fname));
 	}
 
-	if (slh.alloc_seq == 0) {
+	if (!vd->N_opt && slh.alloc_seq == 0) {
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
 		return (vsm_diag(vd,
@@ -282,6 +296,9 @@ VSM_Abandoned(struct VSM_data *vd)
 	if (vd->head == NULL)
 		/* Not open */
 		return (1);
+	if (vd->N_opt)
+		/* No abandonment check should be done */
+		return (0);
 	if (!vd->head->alloc_seq)
 		/* Flag of abandonment set by mgt */
 		return (1);
@@ -328,11 +345,11 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 
 	if (!vd->head)
 		return (0);	/* Not open */
-	if (vd->head->alloc_seq == 0)
+	if (!vd->N_opt && vd->head->alloc_seq == 0)
 		return (0);	/* abandoned VSM */
-	else if (vf->priv != 0) {
+	else if (vf->chunk != NULL) {
 		/* get next chunk */
-		if (vf->priv != vd->head->alloc_seq)
+		if (!vd->N_opt && vf->priv != vd->head->alloc_seq)
 			return (0); /* changes during iteration */
 		if (vf->chunk->len == 0)
 			return (0); /* free'd during iteration */
@@ -376,7 +393,9 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 	AN(vf);
 	if (!vd->head)
 		return (VSM_invalid);
-	if (!vd->head->alloc_seq)
+	if (!vd->N_opt && !vd->head->alloc_seq)
+		return (VSM_invalid);
+	if (vf->chunk == NULL)
 		return (VSM_invalid);
 	if (vf->priv == vd->head->alloc_seq)
 		return (VSM_valid);
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 04ebd7f..0c2dbd4 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -39,6 +39,7 @@ struct VSM_data {
 
 	char			*n_opt;
 	char			*fname;
+	int			N_opt;
 
 	struct stat		fstat;
 



From tfheen at varnish-cache.org  Fri May 24 10:23:36 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 24 May 2013 12:23:36 +0200
Subject: [master] 2910838 Add test for multiple Host headers
Message-ID: <E1Ufp9s-0002fQ-T4@project.varnish-software.com>

commit 2910838bdb136e6d4fc8a2bd5ac27a323939b098
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri May 24 12:09:54 2013 +0200

    Add test for multiple Host headers

diff --git a/bin/varnishtest/tests/b00037.vtc b/bin/varnishtest/tests/b00037.vtc
new file mode 100644
index 0000000..848bdd4
--- /dev/null
+++ b/bin/varnishtest/tests/b00037.vtc
@@ -0,0 +1,15 @@
+varnishtest "Error on multiple Host headers"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txreq -hdr "Host: foo" -hdr "Host: bar"
+} -run
+
+varnish v1 -expect client_req_400 == 1



From phk at varnish-cache.org  Fri May 24 11:56:23 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 13:56:23 +0200
Subject: [master] 4efd5c9 Finally pull the remains of cnt_fetch() entirely
 into VBF_Fetch()
Message-ID: <E1Ufqbf-0003Y4-S5@project.varnish-software.com>

commit 4efd5c96934f20bd8563b1db2af3d1d86c2f41bf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 11:55:52 2013 +0000

    Finally pull the remains of cnt_fetch() entirely into VBF_Fetch()

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f48fb20..5e07962 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -772,65 +772,6 @@ vbf_proc_resp(struct worker *wrk, struct busyobj *bo)
 
 }
 
-/*--------------------------------------------------------------------
- */
-
-static int
-cnt_fetch(struct worker *wrk, struct req *req, struct busyobj *bo)
-{
-	int i;
-
-	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
-
-	i = FetchHdr(wrk, bo, req);
-	/*
-	 * If we recycle a backend connection, there is a finite chance
-	 * that the backend closed it before we get a request to it.
-	 * Do a single retry in that case.
-	 */
-	if (i == 1) {
-		VSC_C_main->backend_retry++;
-		i = FetchHdr(wrk, bo, req);
-	}
-
-	if (!i) {
-		vbf_proc_resp(wrk, bo);
-
-		if (wrk->handling == VCL_RET_DELIVER)
-			return (0);
-
-		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(&bo->vbc);
-	} else {
-		wrk->handling = VCL_RET_ERROR;
-		bo->err_code = 503;
-	}
-
-	/* Clean up partial fetch */
-	AZ(bo->vbc);
-
-	if (bo->fetch_objcore->objhead != NULL ||
-	    wrk->handling == VCL_RET_RESTART ||
-	    wrk->handling == VCL_RET_ERROR) {
-		CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
-		bo->fetch_objcore = NULL;
-	}
-	assert(bo->refcount == 2);
-	bo->storage_hint = NULL;
-	bo->director = NULL;
-	VBO_DerefBusyObj(wrk, &bo);
-
-	switch (wrk->handling) {
-	case VCL_RET_ERROR:
-		return (-1);
-	case VCL_RET_RESTART:
-		INCOMPL();
-	default:
-		WRONG("Illegal action in vcl_fetch{}");
-	}
-}
-
 int
 VBF_Fetch(struct worker *wrk, struct req *req)
 {
@@ -859,12 +800,58 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	vbf_make_bereq(wrk, req, bo);
 	xxxassert (wrk->handling == VCL_RET_FETCH);
 
-	i = cnt_fetch(wrk, bo->do_pass ? req : NULL, bo);
+	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
+
+	if (!bo->do_pass)
+		req = NULL;
+
+	i = FetchHdr(wrk, bo, req);
+	/*
+	 * If we recycle a backend connection, there is a finite chance
+	 * that the backend closed it before we get a request to it.
+	 * Do a single retry in that case.
+	 */
+	if (i == 1) {
+		VSC_C_main->backend_retry++;
+		i = FetchHdr(wrk, bo, req);
+	}
+
+	if (i) {
+		wrk->handling = VCL_RET_ERROR;
+		bo->err_code = 503;
+	} else {
+		vbf_proc_resp(wrk, bo);
+		if (wrk->handling != VCL_RET_DELIVER)
+			VDI_CloseFd(&bo->vbc);
+	}
+
+	if (wrk->handling != VCL_RET_DELIVER) {
+		/* Clean up partial fetch */
+		AZ(bo->vbc);
 
-	if (i)
-		return (i);
+		if (bo->fetch_objcore->objhead != NULL ||
+		    wrk->handling == VCL_RET_RESTART ||
+		    wrk->handling == VCL_RET_ERROR) {
+			CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
+			AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
+			bo->fetch_objcore = NULL;
+		}
+		assert(bo->refcount == 2);
+		bo->storage_hint = NULL;
+		bo->director = NULL;
+		VBO_DerefBusyObj(wrk, &bo);
 
-	if (bo->fetch_objcore->objhead == NULL) 
+		switch (wrk->handling) {
+		case VCL_RET_ERROR:
+			return (-1);
+		case VCL_RET_RESTART:
+			INCOMPL();
+		default:
+			WRONG("Illegal action in vcl_fetch{}");
+		}
+	}
+
+	if (bo->fetch_objcore->objhead == NULL)
 		AN(bo->do_pass);
 
 	/* No reason to try streaming a non-existing body */
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 6c7f015..36c20ab 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -506,7 +506,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
  */
 
 static struct object *
-smp_allocobj(struct stevedore *stv, struct busyobj *bo, 
+smp_allocobj(struct stevedore *stv, struct busyobj *bo,
     unsigned ltot, const struct stv_objsecrets *soc)
 {
 	struct object *o;



From phk at varnish-cache.org  Fri May 24 12:18:59 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 14:18:59 +0200
Subject: [master] 299d0e7 Instigate a bit of VTLAism on the Varnish Backend
 Fetch code
Message-ID: <E1UfqxX-0006GN-QR@project.varnish-software.com>

commit 299d0e7c58eb9c982a4bda8b4301ef675dffabc3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 12:18:37 2013 +0000

    Instigate a bit of VTLAism on the Varnish Backend Fetch code

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1c0dbce..82dc709 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -815,10 +815,10 @@ void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 
 /* cache_fetch.c */
 int VBF_Fetch(struct worker *wrk, struct req *req);
-struct storage *FetchStorage(struct busyobj *, ssize_t sz);
-int FetchError(struct busyobj *, const char *error);
-int FetchError2(struct busyobj *, const char *error, const char *more);
-void Fetch_Init(void);
+struct storage *VBF_GetStorage(struct busyobj *, ssize_t sz);
+int VBF_Error(struct busyobj *, const char *error);
+int VBF_Error2(struct busyobj *, const char *error, const char *more);
+void VBF_Init(void);
 
 /* cache_gzip.c */
 struct vgz;
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index d46d7c1..95439b0 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -93,7 +93,7 @@ vfp_esi_bytes_uu(struct busyobj *bo, const struct vef_priv *vef,
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		st = FetchStorage(bo, 0);
+		st = VBF_GetStorage(bo, 0);
 		if (st == NULL)
 			return (-1);
 		wl = vef_read(htc,
@@ -380,7 +380,7 @@ vfp_esi_end(void *priv)
 		retval = -1;
 
 	if (bo->vgz_rx != NULL && VGZ_Destroy(&bo->vgz_rx) != VGZ_END)
-		retval = FetchError(bo, "Gunzip+ESI Failed at the very end");
+		retval = VBF_Error(bo, "Gunzip+ESI Failed at the very end");
 
 	vsb = VEP_Finish(bo);
 
@@ -395,7 +395,7 @@ vfp_esi_end(void *priv)
 				    VSB_data(vsb), l);
 				bo->fetch_obj->esidata->len = l;
 			} else {
-				retval = FetchError(bo,
+				retval = VBF_Error(bo,
 				    "Could not allocate storage for esidata");
 			}
 		}
@@ -408,7 +408,7 @@ vfp_esi_end(void *priv)
 	if (vef->vgz != NULL) {
 		VGZ_UpdateObj(vef->vgz, bo->fetch_obj);
 		if (VGZ_Destroy(&vef->vgz) != VGZ_END)
-			retval = FetchError(bo,
+			retval = VBF_Error(bo,
 			    "ESI+Gzip Failed at the very end");
 	}
 	if (vef->ibuf != NULL)
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5e07962..110723e 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -57,7 +57,7 @@ static unsigned fetchfrag;
  */
 
 int
-FetchError2(struct busyobj *bo, const char *error, const char *more)
+VBF_Error2(struct busyobj *bo, const char *error, const char *more)
 {
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -72,9 +72,9 @@ FetchError2(struct busyobj *bo, const char *error, const char *more)
 }
 
 int
-FetchError(struct busyobj *bo, const char *error)
+VBF_Error(struct busyobj *bo, const char *error)
 {
-	return(FetchError2(bo, error, NULL));
+	return(VBF_Error2(bo, error, NULL));
 }
 
 /*--------------------------------------------------------------------
@@ -100,7 +100,7 @@ vfp_nop_begin(void *priv, size_t estimate)
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 
 	if (estimate > 0)
-		(void)FetchStorage(bo, estimate);
+		(void)VBF_GetStorage(bo, estimate);
 }
 
 /*--------------------------------------------------------------------
@@ -108,7 +108,7 @@ vfp_nop_begin(void *priv, size_t estimate)
  *
  * Process (up to) 'bytes' from the socket.
  *
- * Return -1 on error, issue FetchError()
+ * Return -1 on error, issue VBF_Error()
  *	will not be called again, once error happens.
  * Return 0 on EOF on socket even if bytes not reached.
  * Return 1 when 'bytes' have been processed.
@@ -124,7 +124,7 @@ vfp_nop_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 
 	while (bytes > 0) {
-		st = FetchStorage(bo, 0);
+		st = VBF_GetStorage(bo, 0);
 		if (st == NULL)
 			return(-1);
 		l = st->space - st->len;
@@ -182,7 +182,7 @@ static struct vfp vfp_nop = {
  */
 
 struct storage *
-FetchStorage(struct busyobj *bo, ssize_t sz)
+VBF_GetStorage(struct busyobj *bo, ssize_t sz)
 {
 	ssize_t l;
 	struct storage *st;
@@ -202,7 +202,7 @@ FetchStorage(struct busyobj *bo, ssize_t sz)
 		l = cache_param->fetch_chunksize;
 	st = STV_alloc(bo, l);
 	if (st == NULL) {
-		(void)FetchError(bo, "Could not get storage");
+		(void)VBF_Error(bo, "Could not get storage");
 		return (NULL);
 	}
 	AZ(st->len);
@@ -215,7 +215,7 @@ FetchStorage(struct busyobj *bo, ssize_t sz)
  */
 
 static ssize_t
-fetch_number(const char *nbr, int radix)
+vbf_fetch_number(const char *nbr, int radix)
 {
 	uintmax_t cll;
 	ssize_t cl;
@@ -236,20 +236,20 @@ fetch_number(const char *nbr, int radix)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl)
+vbf_fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl)
 {
 	int i;
 
 	assert(htc->body_status == BS_LENGTH);
 
 	if (cl < 0) {
-		return (FetchError(bo, "straight length field bogus"));
+		return (VBF_Error(bo, "straight length field bogus"));
 	} else if (cl == 0)
 		return (0);
 
 	i = bo->vfp->bytes(bo, htc, cl);
 	if (i <= 0)
-		return (FetchError(bo, "straight insufficient bytes"));
+		return (VBF_Error(bo, "straight insufficient bytes"));
 	return (0);
 }
 
@@ -260,7 +260,7 @@ fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl)
  */
 
 static int
-fetch_chunked(struct busyobj *bo, struct http_conn *htc)
+vbf_fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
@@ -272,17 +272,17 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 		/* Skip leading whitespace */
 		do {
 			if (HTTP1_Read(htc, buf, 1) <= 0)
-				return (FetchError(bo, "chunked read err"));
+				return (VBF_Error(bo, "chunked read err"));
 		} while (vct_islws(buf[0]));
 
 		if (!vct_ishex(buf[0]))
-			return (FetchError(bo, "chunked header non-hex"));
+			return (VBF_Error(bo, "chunked header non-hex"));
 
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
 				if (HTTP1_Read(htc, buf + u, 1) <= 0)
-					return (FetchError(bo,
+					return (VBF_Error(bo,
 					    "chunked read err"));
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
@@ -290,31 +290,31 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 		}
 
 		if (u >= sizeof buf)
-			return (FetchError(bo,"chunked header too long"));
+			return (VBF_Error(bo,"chunked header too long"));
 
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n')
 			if (HTTP1_Read(htc, buf + u, 1) <= 0)
-				return (FetchError(bo, "chunked read err"));
+				return (VBF_Error(bo, "chunked read err"));
 
 		if (buf[u] != '\n')
-			return (FetchError(bo,"chunked header no NL"));
+			return (VBF_Error(bo,"chunked header no NL"));
 
 		buf[u] = '\0';
-		cl = fetch_number(buf, 16);
+		cl = vbf_fetch_number(buf, 16);
 		if (cl < 0)
-			return (FetchError(bo,"chunked header number syntax"));
+			return (VBF_Error(bo,"chunked header number syntax"));
 
 		if (cl > 0 && bo->vfp->bytes(bo, htc, cl) <= 0)
-			return (FetchError(bo, "chunked read err"));
+			return (VBF_Error(bo, "chunked read err"));
 
 		i = HTTP1_Read(htc, buf, 1);
 		if (i <= 0)
-			return (FetchError(bo, "chunked read err"));
+			return (VBF_Error(bo, "chunked read err"));
 		if (buf[0] == '\r' && HTTP1_Read( htc, buf, 1) <= 0)
-			return (FetchError(bo, "chunked read err"));
+			return (VBF_Error(bo, "chunked read err"));
 		if (buf[0] != '\n')
-			return (FetchError(bo,"chunked tail no NL"));
+			return (VBF_Error(bo,"chunked tail no NL"));
 	} while (cl > 0);
 	return (0);
 }
@@ -322,12 +322,12 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 /*--------------------------------------------------------------------*/
 
 static void
-fetch_eof(struct busyobj *bo, struct http_conn *htc)
+vbf_fetch_eof(struct busyobj *bo, struct http_conn *htc)
 {
 
 	assert(htc->body_status == BS_EOF);
 	if (bo->vfp->bytes(bo, htc, SSIZE_MAX) < 0)
-		(void)FetchError(bo,"eof socket fail");
+		(void)VBF_Error(bo,"eof socket fail");
 }
 
 /*--------------------------------------------------------------------
@@ -335,7 +335,7 @@ fetch_eof(struct busyobj *bo, struct http_conn *htc)
  */
 
 static int __match_proto__(req_body_iter_f)
-fetch_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
+vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
 {
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -360,7 +360,7 @@ fetch_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
  */
 
 static int
-FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
+vbf_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 {
 	struct vbc *vc;
 	struct http *hp;
@@ -403,7 +403,7 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	i = 0;
 
 	if (req != NULL) {
-		i = HTTP1_IterateReqBody(req, fetch_iter_req_body, NULL);
+		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, NULL);
 		if (req->req_body_status == REQ_BODY_DONE)
 			retry = -1;
 	}
@@ -473,7 +473,7 @@ FetchHdr(struct worker *wrk, struct busyobj *bo, struct req *req)
  */
 
 static void
-FetchBody(struct worker *wrk, void *priv)
+vbf_fetch_body(struct worker *wrk, void *priv)
 {
 	int cls;
 	struct storage *st;
@@ -521,11 +521,11 @@ FetchBody(struct worker *wrk, void *priv)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cl = fetch_number(bo->h_content_length, 10);
+		cl = vbf_fetch_number(bo->h_content_length, 10);
 
 		bo->vfp->begin(bo, cl > 0 ? cl : 0);
 		if (bo->state == BOS_FETCHING)
-			cls = fetch_straight(bo, htc, cl);
+			cls = vbf_fetch_straight(bo, htc, cl);
 		mklen = 1;
 		if (bo->vfp->end(bo))
 			assert(bo->state == BOS_FAILED);
@@ -533,7 +533,7 @@ FetchBody(struct worker *wrk, void *priv)
 	case BS_CHUNKED:
 		bo->vfp->begin(bo, cl > 0 ? cl : 0);
 		if (bo->state == BOS_FETCHING)
-			cls = fetch_chunked(bo, htc);
+			cls = vbf_fetch_chunked(bo, htc);
 		mklen = 1;
 		if (bo->vfp->end(bo))
 			assert(bo->state == BOS_FAILED);
@@ -541,14 +541,14 @@ FetchBody(struct worker *wrk, void *priv)
 	case BS_EOF:
 		bo->vfp->begin(bo, cl > 0 ? cl : 0);
 		if (bo->state == BOS_FETCHING)
-			fetch_eof(bo, htc);
+			vbf_fetch_eof(bo, htc);
 		mklen = 1;
 		cls = 1;
 		if (bo->vfp->end(bo))
 			assert(bo->state == BOS_FAILED);
 		break;
 	case BS_ERROR:
-		cls = FetchError(bo, "error incompatible Transfer-Encoding");
+		cls = VBF_Error(bo, "error incompatible Transfer-Encoding");
 		mklen = 0;
 		break;
 	default:
@@ -805,7 +805,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	if (!bo->do_pass)
 		req = NULL;
 
-	i = FetchHdr(wrk, bo, req);
+	i = vbf_fetch_hdr(wrk, bo, req);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -813,7 +813,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(wrk, bo, req);
+		i = vbf_fetch_hdr(wrk, bo, req);
 	}
 
 	if (i) {
@@ -960,7 +960,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
 
-	FetchBody(wrk, bo);
+	vbf_fetch_body(wrk, bo);
 
 	assert(bo->refcount == 1);
 
@@ -1000,7 +1000,7 @@ static struct cli_proto debug_cmds[] = {
  */
 
 void
-Fetch_Init(void)
+VBF_Init(void)
 {
 
 	CLI_AddFuncs(debug_cmds);
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 425e344..3dceda2 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -206,7 +206,7 @@ VGZ_ObufStorage(struct busyobj *bo, struct vgz *vg)
 {
 	struct storage *st;
 
-	st = FetchStorage(bo, 0);
+	st = VBF_GetStorage(bo, 0);
 	if (st == NULL)
 		return (-1);
 
@@ -482,7 +482,7 @@ vfp_gunzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		if (i != VGZ_OK && i != VGZ_END)
-			return(FetchError(bo, "Gunzip data error"));
+			return(VBF_Error(bo, "Gunzip data error"));
 		VBO_extend(bo, dl);
 	}
 	assert(i == Z_OK || i == Z_STREAM_END);
@@ -504,7 +504,7 @@ vfp_gunzip_end(void *priv)
 		return(0);
 	}
 	if (VGZ_Destroy(&vg) != VGZ_END)
-		return(FetchError(bo, "Gunzip error at the very end"));
+		return(VBF_Error(bo, "Gunzip error at the very end"));
 	return (0);
 }
 
@@ -592,7 +592,7 @@ vfp_gzip_end(void *priv)
 	} while (i != Z_STREAM_END);
 	VGZ_UpdateObj(vg, bo->fetch_obj);
 	if (VGZ_Destroy(&vg) != VGZ_END)
-		return(FetchError(bo, "Gzip error at the very end"));
+		return(VBF_Error(bo, "Gzip error at the very end"));
 	return (0);
 }
 
@@ -637,7 +637,7 @@ vfp_testgzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
-		st = FetchStorage(bo, 0);
+		st = VBF_GetStorage(bo, 0);
 		if (st == NULL)
 			return(-1);
 		l = st->space - st->len;
@@ -655,9 +655,9 @@ vfp_testgzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 			VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
-				return(FetchError(bo, "Junk after gzip data"));
+				return(VBF_Error(bo, "Junk after gzip data"));
 			if (i != VGZ_OK && i != VGZ_END)
-				return(FetchError2(bo,
+				return(VBF_Error2(bo,
 				    "Invalid Gzip data", vg->vz.msg));
 		}
 	}
@@ -681,7 +681,7 @@ vfp_testgzip_end(void *priv)
 	}
 	VGZ_UpdateObj(vg, bo->fetch_obj);
 	if (VGZ_Destroy(&vg) != VGZ_END)
-		return(FetchError(bo, "TestGunzip error at the very end"));
+		return(VBF_Error(bo, "TestGunzip error at the very end"));
 	return (0);
 }
 
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index fbbfd2d..bb17c0b 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -209,7 +209,7 @@ child_main(void)
 	WAIT_Init();
 	PAN_Init();
 	CLI_Init();
-	Fetch_Init();
+	VBF_Init();
 
 	VCL_Init();
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 2def95e..a325742 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -598,7 +598,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	}
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	/* One ref for req, one for FetchBody */
+	/* One ref for req, one for vbf_fetch_body */
 	bo->refcount = 2;
 
 	AN (req->objcore);



From phk at varnish-cache.org  Fri May 24 12:41:17 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 14:41:17 +0200
Subject: [master] a1100ba Nail down the asserts where we want them.
Message-ID: <E1UfrJ7-0007eh-E0@project.varnish-software.com>

commit a1100baacff2efa81b75a9ef8e3e6b0725dc8e42
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 12:41:00 2013 +0000

    Nail down the asserts where we want them.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 110723e..143a929 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -816,6 +816,10 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		i = vbf_fetch_hdr(wrk, bo, req);
 	}
 
+	if (bo->do_pass)
+		req = NULL;
+	AZ(req);
+
 	if (i) {
 		wrk->handling = VCL_RET_ERROR;
 		bo->err_code = 503;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index a325742..1e55fcd 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -361,6 +361,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	req->acct_req.fetch++;
 	i = VBF_Fetch(wrk, req);
 	AN(req->busyobj);
+	AZ(req->objcore);
 	assert(req->busyobj->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
 	if (i < 0) {



From phk at varnish-cache.org  Fri May 24 14:24:23 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 16:24:23 +0200
Subject: [master] a6fafd2 One comment and N > 0 bugs from goal...
Message-ID: <E1Ufsut-0001SF-0H@project.varnish-software.com>

commit a6fafd240e6e25f67d52a453551e43a449d532a4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 14:24:04 2013 +0000

    One comment and N > 0 bugs from goal...

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 82dc709..77ef0d5 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -814,7 +814,7 @@ int EXP_NukeOne(struct busyobj *, struct lru *lru);
 void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 
 /* cache_fetch.c */
-int VBF_Fetch(struct worker *wrk, struct req *req);
+void VBF_Fetch(struct worker *wrk, struct req *req);
 struct storage *VBF_GetStorage(struct busyobj *, ssize_t sz);
 int VBF_Error(struct busyobj *, const char *error);
 int VBF_Error2(struct busyobj *, const char *error, const char *more);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 143a929..0aa85fe 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -772,38 +772,49 @@ vbf_proc_resp(struct worker *wrk, struct busyobj *bo)
 
 }
 
-int
-VBF_Fetch(struct worker *wrk, struct req *req)
+struct vbf_secret_handshake {
+	unsigned		magic;
+#define VBF_SECRET_HANDSHAKE_MAGIC	0x98c95172
+	struct busyobj		*bo;
+	struct req		**reqp;
+};
+
+static void
+vbf_fetch_thread(struct worker *wrk, void *priv)
 {
+	struct vbf_secret_handshake *vsh;
+	struct busyobj *bo;
+	struct req *req;
+	int i;
 	struct http *hp, *hp2;
 	char *b;
 	uint16_t nhttp;
 	unsigned l;
 	struct vsb *vary = NULL;
 	int varyl = 0;
-	struct busyobj *bo;
 	struct object *obj;
-	int i;
+
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(vsh, priv, VBF_SECRET_HANDSHAKE_MAGIC);
+	AN(vsh->reqp);
+	req = *vsh->reqp;
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	assert(bo->refcount == 2);
-	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
-
-	bo->fetch_objcore = req->objcore;
-	req->objcore = NULL;
+	bo = vsh->bo;
 
 	vbf_make_bereq(wrk, req, bo);
 	xxxassert (wrk->handling == VCL_RET_FETCH);
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
-	if (!bo->do_pass)
+	if (!bo->do_pass) {
+		AN(req);
+		AN(vsh);
 		req = NULL;
+		*vsh->reqp = NULL;
+		vsh = NULL;
+	}
 
 	i = vbf_fetch_hdr(wrk, bo, req);
 	/*
@@ -816,9 +827,15 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		i = vbf_fetch_hdr(wrk, bo, req);
 	}
 
-	if (bo->do_pass)
+	if (bo->do_pass) {
+		AN(req);
+		AN(vsh);
 		req = NULL;
+		*vsh->reqp = NULL;
+		vsh = NULL;
+	}
 	AZ(req);
+	AZ(vsh);
 
 	if (i) {
 		wrk->handling = VCL_RET_ERROR;
@@ -843,11 +860,12 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		assert(bo->refcount == 2);
 		bo->storage_hint = NULL;
 		bo->director = NULL;
-		VBO_DerefBusyObj(wrk, &bo);
 
 		switch (wrk->handling) {
 		case VCL_RET_ERROR:
-			return (-1);
+			bo->state = BOS_FAILED;
+			VBO_DerefBusyObj(wrk, &bo);
+			return;
 		case VCL_RET_RESTART:
 			INCOMPL();
 		default:
@@ -878,7 +896,8 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 			AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
 			bo->fetch_objcore = NULL;
 			VDI_CloseFd(&bo->vbc);
-			return (-1);
+			bo->state = BOS_FAILED;
+			return;
 		} else
 			/* No vary */
 			AZ(vary);
@@ -912,7 +931,8 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
 		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
-		return (-1);
+		bo->state = BOS_FAILED;
+		return;
 	}
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 
@@ -974,11 +994,43 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	if (bo->state == BOS_FAILED) {
 		/* handle early failures */
 		(void)HSH_Deref(&wrk->stats, NULL, &obj);
-		return (-1);
+		return;
 	}
 
 	assert(WRW_IsReleased(wrk));
-	return (0);
+}
+
+void
+VBF_Fetch(struct worker *wrk, struct req *req)
+{
+	struct vbf_secret_handshake vsh;
+	struct busyobj *bo;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+
+	bo = req->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	assert(bo->refcount == 2);
+	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
+
+	bo->fetch_objcore = req->objcore;
+	req->objcore = NULL;
+
+	vsh.magic = VBF_SECRET_HANDSHAKE_MAGIC;
+	vsh.bo = bo;
+	vsh.reqp = &req;
+
+	bo->fetch_task.priv = &vsh;
+	bo->fetch_task.func = vbf_fetch_thread;
+
+	// if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
+		vbf_fetch_thread(wrk, &vsh);
+	while (req != NULL) {
+		printf("XXX\n");
+		(void)usleep(100000);
+	}
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 1e55fcd..5ee6d14 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -351,7 +351,6 @@ DOT fetchbody:out -> prepresp [style=bold,color=blue]
 static enum req_fsm_nxt
 cnt_fetch(struct worker *wrk, struct req *req)
 {
-	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -359,17 +358,21 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	AN(req->busyobj);
 	AN(req->objcore);
 	req->acct_req.fetch++;
-	i = VBF_Fetch(wrk, req);
+	VBF_Fetch(wrk, req);
 	AN(req->busyobj);
 	AZ(req->objcore);
 	assert(req->busyobj->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
-	if (i < 0) {
+	while (req->busyobj->state < BOS_FAILED) {
+		printf("YYY\n");
+		(void)usleep(100000);
+	}
+	if (req->busyobj->state == BOS_FAILED) {
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
 	} else {
-		AZ(i);
+		assert (req->busyobj->state == BOS_FINISHED);
 		req->err_code = req->busyobj->err_code;
 		req->obj = req->busyobj->fetch_obj;
 		VBO_DerefBusyObj(wrk, &req->busyobj);



From phk at varnish-cache.org  Fri May 24 14:57:40 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 16:57:40 +0200
Subject: [master] e2f1dac Register the busyobj on the thread while we do
 fetch-thread duty.
Message-ID: <E1UftR6-00045a-OM@project.varnish-software.com>

commit e2f1dac1d95a1d2a9db3cf1ebcc47a33b16f61cf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 14:57:23 2013 +0000

    Register the busyobj on the thread while we do fetch-thread duty.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0aa85fe..f1ee9c2 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -802,6 +802,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	bo = vsh->bo;
+	THR_SetBusyobj(bo);
 
 	vbf_make_bereq(wrk, req, bo);
 	xxxassert (wrk->handling == VCL_RET_FETCH);
@@ -865,6 +866,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		case VCL_RET_ERROR:
 			bo->state = BOS_FAILED;
 			VBO_DerefBusyObj(wrk, &bo);
+			THR_SetBusyobj(NULL);
 			return;
 		case VCL_RET_RESTART:
 			INCOMPL();
@@ -897,6 +899,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 			bo->fetch_objcore = NULL;
 			VDI_CloseFd(&bo->vbc);
 			bo->state = BOS_FAILED;
+			THR_SetBusyobj(NULL);
 			return;
 		} else
 			/* No vary */
@@ -932,6 +935,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
 		bo->state = BOS_FAILED;
+		THR_SetBusyobj(NULL);
 		return;
 	}
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
@@ -994,10 +998,12 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	if (bo->state == BOS_FAILED) {
 		/* handle early failures */
 		(void)HSH_Deref(&wrk->stats, NULL, &obj);
+		THR_SetBusyobj(NULL);
 		return;
 	}
 
 	assert(WRW_IsReleased(wrk));
+	THR_SetBusyobj(NULL);
 }
 
 void



From phk at varnish-cache.org  Fri May 24 15:26:34 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 17:26:34 +0200
Subject: [master] c09a7d5 Write the request body to the right wrk's wrw
Message-ID: <E1Uftt4-0000Zj-O7@project.varnish-software.com>

commit c09a7d50bf18fea03b64ee23e0a6b256fdb1d89e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 15:26:18 2013 +0000

    Write the request body to the right wrk's wrw

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f1ee9c2..a564fdb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -337,13 +337,14 @@ vbf_fetch_eof(struct busyobj *bo, struct http_conn *htc)
 static int __match_proto__(req_body_iter_f)
 vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
 {
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	(void)priv;
+	CAST_OBJ_NOTNULL(wrk, priv, WORKER_MAGIC);
 
 	if (l > 0) {
-		(void)WRW_Write(req->wrk, ptr, l);
-		if (WRW_Flush(req->wrk))
+		(void)WRW_Write(wrk, ptr, l);
+		if (WRW_Flush(wrk))
 			return (-1);
 	}
 	return (0);
@@ -403,7 +404,7 @@ vbf_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	i = 0;
 
 	if (req != NULL) {
-		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, NULL);
+		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
 		if (req->req_body_status == REQ_BODY_DONE)
 			retry = -1;
 	}
@@ -1031,7 +1032,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	bo->fetch_task.priv = &vsh;
 	bo->fetch_task.func = vbf_fetch_thread;
 
-	// if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
+	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
 		vbf_fetch_thread(wrk, &vsh);
 	while (req != NULL) {
 		printf("XXX\n");



From phk at varnish-cache.org  Fri May 24 15:28:01 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 24 May 2013 17:28:01 +0200
Subject: [master] 665f1ef Oops, that bit was still two test-cases to early
Message-ID: <E1UftuT-0000gd-OJ@project.varnish-software.com>

commit 665f1ef16f4f05c3954da97231f4512926f5bfe7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 24 15:27:45 2013 +0000

    Oops, that bit was still two test-cases to early

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index a564fdb..2fed1e3 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -1032,7 +1032,7 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 	bo->fetch_task.priv = &vsh;
 	bo->fetch_task.func = vbf_fetch_thread;
 
-	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
+	// if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
 		vbf_fetch_thread(wrk, &vsh);
 	while (req != NULL) {
 		printf("XXX\n");



From phk at varnish-cache.org  Tue May 28 09:51:05 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 28 May 2013 11:51:05 +0200
Subject: [master] 9ab56d3 Mask out the identity type from the xid before
 VSL'ing.
Message-ID: <E1UhGYb-00006v-9m@project.varnish-software.com>

commit 9ab56d334fa73cd530089c2ccd98bb2c9b9bd7b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 28 09:50:43 2013 +0000

    Mask out the identity type from the xid before VSL'ing.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 80c630f..2271949 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -409,7 +409,8 @@ exp_timer(struct worker *wrk, void *priv)
 		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
 		o = oc_getobj(&wrk->stats, oc);
 		VSLb(&vsl, SLT_ExpKill, "%u %.0f",
-		    oc_getxid(&wrk->stats, oc), EXP_Ttl(NULL, o) - t);
+		    oc_getxid(&wrk->stats, oc) & VSL_IDENTMASK,
+		    EXP_Ttl(NULL, o) - t);
 		(void)HSH_Deref(&wrk->stats, oc, NULL);
 	}
 	NEEDLESS_RETURN(NULL);
@@ -454,7 +455,8 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 		return (-1);
 
 	/* XXX: bad idea for -spersistent */
-	VSLb(bo->vsl, SLT_ExpKill, "%u LRU", oc_getxid(bo->stats, oc));
+	VSLb(bo->vsl, SLT_ExpKill, "%u LRU",
+	    oc_getxid(bo->stats, oc) & VSL_IDENTMASK);
 	(void)HSH_Deref(bo->stats, oc, NULL);
 	return (1);
 }
@@ -508,7 +510,8 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 			oc = oc_array[i];
 			o = oc_getobj(&wrk->stats, oc);
 			VSLb(vsl, SLT_ExpKill, "%u %.0f LRU",
-			     oc_getxid(&wrk->stats, oc), EXP_Ttl(NULL, o) - t);
+			    oc_getxid(&wrk->stats, oc) & VSL_IDENTMASK,
+			    EXP_Ttl(NULL, o) - t);
 			EXP_Set_ttl(&o->exp, 0.);
 			(void)HSH_Deref(&wrk->stats, oc, NULL);
 		}



From phk at varnish-cache.org  Tue May 28 22:30:34 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 29 May 2013 00:30:34 +0200
Subject: [master] ffb2d8e Try to get the busyobj refcounting more right
Message-ID: <E1UhSPa-0005bb-Nj@project.varnish-software.com>

commit ffb2d8e7ac287e1624a85462aedb76a003929788
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 28 22:30:21 2013 +0000

    Try to get the busyobj refcounting more right

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2fed1e3..7358c43 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -621,7 +621,6 @@ vbf_fetch_body(struct worker *wrk, void *priv)
 	if (obj->objcore->objhead != NULL)
 		HSH_Complete(obj->objcore);
 	bo->stats = NULL;
-	VBO_DerefBusyObj(wrk, &bo);
 }
 
 /*--------------------------------------------------------------------
@@ -711,8 +710,7 @@ vbf_proc_resp(struct worker *wrk, struct busyobj *bo)
 
 	// Don't let VCL reset do_pass
 	i = bo->do_pass;
-	VCL_backend_response_method(bo->vcl, wrk, NULL, bo,
-	    bo->beresp->ws);
+	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
 	bo->do_pass |= i;
 
 	if (bo->do_pass)
@@ -866,7 +864,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		switch (wrk->handling) {
 		case VCL_RET_ERROR:
 			bo->state = BOS_FAILED;
-			VBO_DerefBusyObj(wrk, &bo);
+			VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 			THR_SetBusyobj(NULL);
 			return;
 		case VCL_RET_RESTART:
@@ -900,6 +898,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 			bo->fetch_objcore = NULL;
 			VDI_CloseFd(&bo->vbc);
 			bo->state = BOS_FAILED;
+VSL_Flush(bo->vsl, 0);
 			THR_SetBusyobj(NULL);
 			return;
 		} else
@@ -936,6 +935,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
 		bo->state = BOS_FAILED;
+		VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 		THR_SetBusyobj(NULL);
 		return;
 	}
@@ -980,7 +980,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	 * Ready to fetch the body
 	 */
 
-	assert(bo->refcount == 2);	/* one for each thread */
+	assert(bo->refcount >= 2);	/* one for each thread */
 
 	if (obj->objcore->objhead != NULL) {
 		EXP_Insert(obj);
@@ -991,18 +991,20 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	vbf_fetch_body(wrk, bo);
 
-	assert(bo->refcount == 1);
+	assert(bo->refcount >= 1);
 
 	if (obj->objcore->objhead != NULL)
 		HSH_Ref(obj->objcore);
 
 	if (bo->state == BOS_FAILED) {
 		/* handle early failures */
+		VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 		(void)HSH_Deref(&wrk->stats, NULL, &obj);
 		THR_SetBusyobj(NULL);
 		return;
 	}
 
+	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 	assert(WRW_IsReleased(wrk));
 	THR_SetBusyobj(NULL);
 }