From martin at varnish-software.com  Mon Oct  1 09:01:43 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 1 Oct 2012 11:01:43 +0200
Subject: RFC: new vcl_lookup{} proposal
In-Reply-To: <20314.1347867221@critter.freebsd.dk>
References: <20314.1347867221@critter.freebsd.dk>
Message-ID: <CANTn4coB-cBUDhgHm9ohLOsG_mef2UQ0M7asRPspH1S_MHnhbw@mail.gmail.com>

As we are moving more logic from varnishd into VCL, I believe some
rethinking with how we deal with the default_vcl logic might be in order.
This to make it easier to make the easy VCL changes and keep the default
logic around still.

A common vcl error I have observed is where you want to match on something
to perform an action, and then stop further processing on this request so
further rules won't match on the url. E.g.:

sub vcl_fetch {
    if (req.url ~ "\.(gif|jpg|css") {
        # Cache our static resources "forever"
        set beresp.ttl = 180d;
        return (deliver);
    }


}

This will ofc bypass any set-cookie checks performed by the default
vcl_fetch logic. Also the explicit return(deliver) prevents any further
application level rules that may be applied later, which is convenient. But
the problem comes if for some reason one of these static resources should
actually return a set-cookie header.

So the correct thing to do in this case

If we changed how the default vcl is structured into having all of the
logic in separate subroutines, we would in cases li

On Mon, Sep 17, 2012 at 9:33 AM, Poul-Henning Kamp <phk at phk.freebsd.dk>wrote:

>
> Right now we go directly from vcl_hash{} to either vcl_hit{}, vcl_miss{}
> or vcl_pass{} depending on the outcome, and that means, amongst other
> things, that a VCL implementation of "PURGE" as to add code to all those.
>
> Furthermore we have some semi-magic things going on behind the scenes,
> such as grace and saint modes.
>
> Assume:
>
> * Backend fetches will be done by a different thread than the one
>   processing the clients request.
>
> * vcl_hash{} always is followed by vcl_lookup{} and that the above
> mentioned
>   logic gets moved to VCL, rather than C code.
>
> * vcl_hit{} goes away.
>
> Then a plausible default vcl_lookup{} could then look like:
>
>         sub vcl_lookup {
>
>                 if (!obj) {
>                         // Miss:  Start a background fetch in another
> thread
>                         // We come back here, when the status of that fetch
>                         // is available. (ie: we go on the waiting list)
>                         return (miss);
>                 }
>
>                 if (obj.ttl <= 0s &&
>                     obj.ttl > -req.grace && obj.ttl > -obj.grace) {
>                         // We have a grace situation
>                         if (!obj.busy) {
>                                 // Nobody is fetching yet, so start a
>                                 // background fetch
>                                 background_fetch();
>                         }
>                         // NB: First client does not hang on the fetch
>                         return (deliver);
>                 }
>
>                 if (obj.ttl <= 0s) {
>                         // object is too old, fetch new one (see miss)
>                         return (miss);
>                 }
>
>                 return (deliver);
>         }
>
> Possible customizations:
>
> Purging:
>
>         if (req.request == "PURGE") {
>                 // We pressume access control was in vcl_recv{}
>                 return (purge);
>                 return (purge_all);
>         }
>
> Prefetching:
>
>         if (obj.ttl < 2s && !obj.busy) {
>                 // Close to deadline, pre-fetch
>                 background_fetch();
>         }
>
> Don't stream, only deliver complete objects:
>
>         if (!obj.complete) {
>                 wait_complete(obj);
>         }
>
> I'm seriously pondering if saintmode should become a vmod at the same
> time, in which case it would probably look something like:
>
>         sub vcl_lookup {
>                 if (obj.ttl <= 0s && !saint.ok(req.backend, req.url)) {
>                         return (deliviver)
>                 }
>         }
>
>         sub vcl_fetch {
>                 if (beresp.status >= 400) {
>                         saint.bad(bereq.backend, bereq.url);
>                 }
>         }
>
> --
> Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
> phk at FreeBSD.ORG         | TCP/IP since RFC 956
> FreeBSD committer       | BSD since 4.3-tahoe
> Never attribute to malice what can adequately be explained by incompetence.
>
> _______________________________________________
> varnish-dev mailing list
> varnish-dev at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
>



-- 
Martin Blix Grydeland
Varnish Software AS
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121001/1b03e79a/attachment.html>

From martin at varnish-software.com  Mon Oct  1 09:21:29 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 1 Oct 2012 11:21:29 +0200
Subject: RFC: new vcl_lookup{} proposal
In-Reply-To: <CANTn4coB-cBUDhgHm9ohLOsG_mef2UQ0M7asRPspH1S_MHnhbw@mail.gmail.com>
References: <20314.1347867221@critter.freebsd.dk>
 <CANTn4coB-cBUDhgHm9ohLOsG_mef2UQ0M7asRPspH1S_MHnhbw@mail.gmail.com>
Message-ID: <CANTn4coXHExVc2k8Wj8F7256rCemj056G+wMeyGt-ckMH8X4iQ@mail.gmail.com>

* Sorry about the early incomplete version of this email. I sent it
unfinished by mistake *

As we are moving more logic from varnishd into VCL, I believe some
rethinking with how we deal with the default_vcl logic might be in order.
This to make it easier to make the easy VCL changes and keep the default
logic around still.

> A common vcl error I have observed is where you want to match on something
to perform an action, and then stop further processing on this request so
further rules won't match on the url. E.g.:

> sub vcl_fetch {
    if (req.url ~ "\.(gif|jpg|css") {
        # Cache our static resources "forever"
        set beresp.ttl = 180d;
        return (deliver);
    }
}

> This will ofc bypass any set-cookie checks performed by the default
vcl_fetch logic. Also the explicit return(deliver) prevents any further
application level rules that may be applied later, which is convenient. But
the problem comes if for some reason one of these static resources should
actually return a set-cookie header.

> If we had restructured the default vcl logic bits into vcl subroutines of
their own, this would be much easier to do safe. Given that we have a
default_vcl_fetch routine that contains exactly what vcl_fetch contains
today, the above example could become:
>
> sub vcl_fetch {
    if (req.url ~ "\.(gif|jpg|css") {
        # Cache our static resources "forever"
        set beresp.ttl = 180d;
        call default_vcl_fetch; # Will not return
    }
}

The default vcl_fetch routine would simply become:

sub vcl_fetch {
    call default_vcl_fetch;
}


To take it a little further, I also think that some of the default logic
could do with a way to modify it, without having to copy all and redoing
it. If e.g. the default_vcl_recv looked like this:

sub default_vcl_recv {
    call default_vcl_recv_xff; # Handle X-Forwarded-For header creation
    call default_vcl_recv_check_method; # Pass on non-recognized http
methods
    call default_vcl_recv_check_pass; # Pass on anything but GET and HEAD
    call default_vcl_recv_check_auth; # Pass on auth and cookie headers
present
    return (lookup);
}

We could then change part of the logic only by overriding only one of the
functions. So if I had a need to do the X-Forwarded-For headers differently
for my site, I could redefine only that function. E.g.:

sub default_vcl_recv_xff {
    if (client.ip ~ acl_fw) {
        set req.http.X-Forwarded-For = req.http.X-Orig-IP;
        exit; # Exit from current subroutine (only allowed from
non-callback subs, could be implemented by a goto in libvcl)
    }
    # Default vcl would take over here doing the normal XFF handling
}

Martin

-- 
Martin Blix Grydeland
Varnish Software AS
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121001/92b0a4d2/attachment.html>

From martin at varnish-software.com  Mon Oct  1 10:26:08 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:08 +0200
Subject: [PATCH 01/13] Add a EXP_NukeLRU() function to nuke an entire LRU
 structure at a time.
Message-ID: <1349087180-11089-1-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/cache/cache.h        |    1 +
 bin/varnishd/cache/cache_expire.c |   59 +++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 15db84c..7533782 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -790,6 +790,7 @@ void EXP_Init(void);
 void EXP_Rearm(const struct object *o);
 int EXP_Touch(struct objcore *oc);
 int EXP_NukeOne(struct busyobj *, struct lru *lru);
+void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 
 /* cache_fetch.c */
 struct storage *FetchStorage(struct busyobj *, ssize_t sz);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 3d18d78..524d565 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -460,6 +460,65 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 }
 
 /*--------------------------------------------------------------------
+ * Nukes an entire LRU
+ */
+
+#define NUKEBUF 10
+
+void
+EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
+{
+	struct objcore *oc;
+	struct objcore *oc_array[NUKEBUF];
+	struct object *o;
+	int i, n;
+	double t;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	t = VTIM_real();
+	Lck_Lock(&lru->mtx);
+	while (!VTAILQ_EMPTY(&lru->lru_head)) {
+		Lck_Lock(&exp_mtx);
+		n = 0;
+		while (n < NUKEBUF) {
+			oc = VTAILQ_FIRST(&lru->lru_head);
+			if (oc == NULL)
+				break;
+			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+			assert(oc_getlru(oc) == lru);
+
+			/* Remove from the LRU and binheap */
+			VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+			assert(oc->timer_idx != BINHEAP_NOIDX);
+			binheap_delete(exp_heap, oc->timer_idx);
+			assert(oc->timer_idx == BINHEAP_NOIDX);
+
+			oc_array[n++] = oc;
+			VSC_C_main->n_lru_nuked++;
+		}
+		assert(n > 0);
+		Lck_Unlock(&exp_mtx);
+		Lck_Unlock(&lru->mtx);
+
+		for (i = 0; i < n; i++) {
+			oc = oc_array[i];
+			o = oc_getobj(&wrk->stats, oc);
+			VSLb(vsl, SLT_ExpKill, "%u %.0f LRU",
+			     oc_getxid(&wrk->stats, oc), EXP_Ttl(NULL, o) - t);
+			EXP_Set_ttl(&o->exp, 0.);
+			(void)HSH_Deref(&wrk->stats, oc, NULL);
+		}
+
+		Lck_Lock(&lru->mtx);
+	}
+	Lck_Unlock(&lru->mtx);
+
+	WRK_SumStat(wrk);
+}
+
+/*--------------------------------------------------------------------
  * BinHeap helper functions for objcore.
  */
 
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:12 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:12 +0200
Subject: [PATCH 05/13] Round to page sizes on signature syncs
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-5-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.h      |    1 +
 bin/varnishd/storage/storage_persistent_subr.c |   27 ++++++++++++++++++------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index c4837b9..aceb996 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -210,6 +210,7 @@ void smp_reset_signspace(struct smp_signspace *spc);
 void smp_copy_signspace(struct smp_signspace *dst,
 			const struct smp_signspace *src);
 void smp_trunc_signspace(struct smp_signspace *spc, uint32_t len);
+void smp_msync(void *addr, size_t length);
 
 void smp_newsilo(struct smp_sc *sc);
 int smp_valid_silo(struct smp_sc *sc);
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index 9933541..f73ae06 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -156,13 +156,7 @@ smp_reset_sign(struct smp_signctx *ctx)
 void
 smp_sync_sign(const struct smp_signctx *ctx)
 {
-	int i;
-
-	/* XXX: round to pages */
-	i = msync((void*)ctx->ss, ctx->ss->length + SMP_SIGN_SPACE, MS_SYNC);
-	if (i && 0)
-		fprintf(stderr, "SyncSign(%p %s) = %d %s\n",
-		    ctx->ss, ctx->id, i, strerror(errno));
+	smp_msync(ctx->ss, SMP_SIGN_SPACE + ctx->ss->length);
 }
 
 /*--------------------------------------------------------------------
@@ -268,6 +262,25 @@ smp_new_signspace(const struct smp_sc *sc, struct smp_signspace *spc,
 }
 
 /*--------------------------------------------------------------------
+ * Force a write of a memory block (rounded to nearest pages) to
+ * the backing store.
+ */
+
+void
+smp_msync(void *addr, size_t length)
+{
+	uintptr_t start, end;
+	int pagesize;
+
+	pagesize = getpagesize();
+	assert(pagesize > 0 && PWR2(pagesize));
+	start = RDN2((uintptr_t)addr, pagesize);
+	end = RUP2((uintptr_t)addr + length, pagesize);
+	assert(start < end);
+	AZ(msync((void *)start, end - start, MS_SYNC));
+}
+
+/*--------------------------------------------------------------------
  * Initialize a Silo with a valid but empty structure.
  *
  * XXX: more intelligent sizing of things.
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:09 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:09 +0200
Subject: [PATCH 02/13] Create and use a smp_signspace structure to have range
 checking on the growing signed data structures.
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-2-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c      |   23 +++---
 bin/varnishd/storage/storage_persistent.h      |   35 +++++++--
 bin/varnishd/storage/storage_persistent_silo.c |   10 +--
 bin/varnishd/storage/storage_persistent_subr.c |   90 ++++++++++++++++++++----
 4 files changed, 126 insertions(+), 32 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 5d31caf..b54f497 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -68,13 +68,14 @@ static VTAILQ_HEAD(,smp_sc)	silos = VTAILQ_HEAD_INITIALIZER(silos);
  */
 
 static void
-smp_appendban(struct smp_sc *sc, struct smp_signctx *ctx,
+smp_appendban(struct smp_sc *sc, struct smp_signspace *spc,
     uint32_t len, const uint8_t *ban)
 {
 	uint8_t *ptr, *ptr2;
 
 	(void)sc;
-	ptr = ptr2 = SIGN_END(ctx);
+	ptr = ptr2 = SIGNSPACE_FRONT(spc);
+	assert(SIGNSPACE_FREE(spc) >= 4 + 4 + len);
 
 	memcpy(ptr, "BAN", 4);
 	ptr += 4;
@@ -85,7 +86,7 @@ smp_appendban(struct smp_sc *sc, struct smp_signctx *ctx,
 	memcpy(ptr, ban, len);
 	ptr += len;
 
-	smp_append_sign(ctx, ptr2, ptr - ptr2);
+	smp_append_signspace(spc, ptr - ptr2);
 }
 
 /* Trust that cache_ban.c takes care of locking */
@@ -106,7 +107,7 @@ SMP_NewBan(const uint8_t *ban, unsigned ln)
  */
 
 static int
-smp_open_bans(struct smp_sc *sc, struct smp_signctx *ctx)
+smp_open_bans(struct smp_sc *sc, struct smp_signspace *spc)
 {
 	uint8_t *ptr, *pe;
 	uint32_t length;
@@ -114,11 +115,11 @@ smp_open_bans(struct smp_sc *sc, struct smp_signctx *ctx)
 
 	ASSERT_CLI();
 	(void)sc;
-	i = smp_chk_sign(ctx);
+	i = smp_chk_signspace(spc);
 	if (i)
 		return (i);
-	ptr = SIGN_DATA(ctx);
-	pe = ptr + ctx->ss->length;
+	ptr = SIGNSPACE_DATA(spc);
+	pe = SIGNSPACE_FRONT(spc);
 
 	while (ptr < pe) {
 		if (memcmp(ptr, "BAN", 4)) {
@@ -148,7 +149,7 @@ smp_open_bans(struct smp_sc *sc, struct smp_signctx *ctx)
  */
 
 static int
-smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
+smp_open_segs(struct smp_sc *sc, struct smp_signspace *spc)
 {
 	uint64_t length, l;
 	struct smp_segptr *ss, *se;
@@ -156,12 +157,12 @@ smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
 	int i, n = 0;
 
 	ASSERT_CLI();
-	i = smp_chk_sign(ctx);
+	i = smp_chk_signspace(spc);
 	if (i)
 		return (i);
 
-	ss = SIGN_DATA(ctx);
-	length = ctx->ss->length;
+	ss = SIGNSPACE_DATA(spc);
+	length = SIGNSPACE_LEN(spc);
 
 	if (length == 0) {
 		/* No segments */
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index c817faf..ec97d3e 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -55,6 +55,21 @@ struct smp_signctx {
 	const char		*id;
 };
 
+/*
+ * A space wrapped by a signature
+ *
+ * A signspace is a chunk of the silo that is wrapped by a
+ * signature. It has attributes for size, so range checking can be
+ * performed.
+ *
+ */
+
+struct smp_signspace {
+	struct smp_signctx	ctx;
+	uint8_t			*start;
+	uint64_t		size;
+};
+
 struct smp_sc;
 
 /* XXX: name confusion with on-media version ? */
@@ -116,10 +131,10 @@ struct smp_sc {
 	VTAILQ_ENTRY(smp_sc)	list;
 
 	struct smp_signctx	idn;
-	struct smp_signctx	ban1;
-	struct smp_signctx	ban2;
-	struct smp_signctx	seg1;
-	struct smp_signctx	seg2;
+	struct smp_signspace	ban1;
+	struct smp_signspace	ban2;
+	struct smp_signspace	seg1;
+	struct smp_signspace	seg2;
 
 	struct ban		*tailban;
 
@@ -161,6 +176,11 @@ struct smp_sc {
 #define SIGN_DATA(ctx)	((void *)((ctx)->ss + 1))
 #define SIGN_END(ctx)	((void *)((int8_t *)SIGN_DATA(ctx) + (ctx)->ss->length))
 
+#define SIGNSPACE_DATA(spc)	(SIGN_DATA(&(spc)->ctx))
+#define SIGNSPACE_FRONT(spc)	(SIGN_END(&(spc)->ctx))
+#define SIGNSPACE_LEN(spc)	((spc)->ctx.ss->length)
+#define SIGNSPACE_FREE(spc)	((spc)->size - SIGNSPACE_LEN(spc))
+
 /* storage_persistent_mgt.c */
 
 void smp_mgt_init(struct stevedore *parent, int ac, char * const *av);
@@ -181,6 +201,13 @@ int smp_chk_sign(struct smp_signctx *ctx);
 void smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len);
 void smp_reset_sign(struct smp_signctx *ctx);
 void smp_sync_sign(const struct smp_signctx *ctx);
+
+void smp_def_signspace(const struct smp_sc *sc, struct smp_signspace *spc,
+		       uint64_t off, uint64_t size, const char *id);
+int smp_chk_signspace(struct smp_signspace *spc);
+void smp_append_signspace(struct smp_signspace *spc, uint32_t len);
+void smp_reset_signspace(struct smp_signspace *spc);
+
 void smp_newsilo(struct smp_sc *sc);
 int smp_valid_silo(struct smp_sc *sc);
 
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index ff53af8..6ed80d2 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -58,15 +58,15 @@
  */
 
 static void
-smp_save_seg(const struct smp_sc *sc, struct smp_signctx *ctx)
+smp_save_seg(const struct smp_sc *sc, struct smp_signspace *spc)
 {
 	struct smp_segptr *ss;
 	struct smp_seg *sg;
 	uint64_t length;
 
 	Lck_AssertHeld(&sc->mtx);
-	smp_reset_sign(ctx);
-	ss = SIGN_DATA(ctx);
+	smp_reset_signspace(spc);
+	ss = SIGNSPACE_DATA(spc);
 	length = 0;
 	VTAILQ_FOREACH(sg, &sc->segments, list) {
 		assert(sg->p.offset < sc->mediasize);
@@ -75,8 +75,8 @@ smp_save_seg(const struct smp_sc *sc, struct smp_signctx *ctx)
 		ss++;
 		length += sizeof *ss;
 	}
-	smp_append_sign(ctx, SIGN_DATA(ctx), length);
-	smp_sync_sign(ctx);
+	smp_append_signspace(spc, length);
+	smp_sync_sign(&spc->ctx);
 }
 
 void
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index 004bba3..ac93544 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -179,6 +179,64 @@ smp_new_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
 }
 
 /*--------------------------------------------------------------------
+ * Define a signature space by location, size and identifier
+ */
+
+void
+smp_def_signspace(const struct smp_sc *sc, struct smp_signspace *spc,
+		  uint64_t off, uint64_t size, const char *id)
+{
+	smp_def_sign(sc, &spc->ctx, off, id);
+	spc->start = SIGN_DATA(&spc->ctx);
+	spc->size = size - SMP_SIGN_SPACE;
+}
+
+/*--------------------------------------------------------------------
+ * Check that a signspace's signature space is good, leave state ready
+ * for append
+ */
+
+int
+smp_chk_signspace(struct smp_signspace *spc)
+{
+	return (smp_chk_sign(&spc->ctx));
+}
+
+/*--------------------------------------------------------------------
+ * Append data to a signature space
+ */
+
+void
+smp_append_signspace(struct smp_signspace *spc, uint32_t len)
+{
+	assert(len <= SIGNSPACE_FREE(spc));
+	smp_append_sign(&spc->ctx, SIGNSPACE_FRONT(spc), len);
+}
+
+/*--------------------------------------------------------------------
+ * Reset a signature space to empty, prepare for appending.
+ */
+
+void
+smp_reset_signspace(struct smp_signspace *spc)
+{
+	smp_reset_sign(&spc->ctx);
+}
+
+/*--------------------------------------------------------------------
+ * Create a new signature space and force the signature to backing store.
+ */
+
+static void
+smp_new_signspace(const struct smp_sc *sc, struct smp_signspace *spc,
+		  uint64_t off, uint64_t size, const char *id)
+{
+	smp_new_sign(sc, &spc->ctx, off, id);
+	spc->start = SIGN_DATA(&spc->ctx);
+	spc->size = size - SMP_SIGN_SPACE;
+}
+
+/*--------------------------------------------------------------------
  * Initialize a Silo with a valid but empty structure.
  *
  * XXX: more intelligent sizing of things.
@@ -220,10 +278,14 @@ smp_newsilo(struct smp_sc *sc)
 	si->stuff[SMP_END_STUFF] = si->mediasize;
 	assert(si->stuff[SMP_SPC_STUFF] < si->stuff[SMP_END_STUFF]);
 
-	smp_new_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
-	smp_new_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
-	smp_new_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
-	smp_new_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+	smp_new_signspace(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF],
+			  smp_stuff_len(sc, SMP_BAN1_STUFF), "BAN 1");
+	smp_new_signspace(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF],
+			  smp_stuff_len(sc, SMP_BAN2_STUFF), "BAN 2");
+	smp_new_signspace(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF],
+			  smp_stuff_len(sc, SMP_SEG1_STUFF), "SEG 1");
+	smp_new_signspace(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF],
+			  smp_stuff_len(sc, SMP_SEG2_STUFF), "SEG 2");
 
 	smp_append_sign(&sc->idn, si, sizeof *si);
 	smp_sync_sign(&sc->idn);
@@ -282,20 +344,24 @@ smp_valid_silo(struct smp_sc *sc)
 	assert(smp_stuff_len(sc, SMP_BAN1_STUFF) ==
 	  smp_stuff_len(sc, SMP_BAN2_STUFF));
 
-	smp_def_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
-	smp_def_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
-	smp_def_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
-	smp_def_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+	smp_def_signspace(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF],
+			  smp_stuff_len(sc, SMP_BAN1_STUFF), "BAN 1");
+	smp_def_signspace(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF],
+			  smp_stuff_len(sc, SMP_BAN2_STUFF), "BAN 2");
+	smp_def_signspace(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF],
+			  smp_stuff_len(sc, SMP_SEG1_STUFF), "SEG 1");
+	smp_def_signspace(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF],
+			  smp_stuff_len(sc, SMP_SEG2_STUFF), "SEG 2");
 
 	/* We must have one valid BAN table */
-	i = smp_chk_sign(&sc->ban1);
-	j = smp_chk_sign(&sc->ban2);
+	i = smp_chk_signspace(&sc->ban1);
+	j = smp_chk_signspace(&sc->ban2);
 	if (i && j)
 		return (100 + i * 10 + j);
 
 	/* We must have one valid SEG table */
-	i = smp_chk_sign(&sc->seg1);
-	j = smp_chk_sign(&sc->seg2);
+	i = smp_chk_signspace(&sc->seg1);
+	j = smp_chk_signspace(&sc->seg2);
 	if (i && j)
 		return (200 + i * 10 + j);
 	return (0);
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:10 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:10 +0200
Subject: [PATCH 03/13] Add smp_copy_signspace() and smp_trunc_signspace()
 utility functions
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-3-git-send-email-martin@varnish-software.com>

smp_copy_signspace() will copy the sign data of one space onto another.

smp_trunc_signspace() will truncate the signed space to len bytes and
resign the space.
---
 bin/varnishd/storage/storage_persistent.h      |    3 +++
 bin/varnishd/storage/storage_persistent_subr.c |   31 ++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index ec97d3e..c4837b9 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -207,6 +207,9 @@ void smp_def_signspace(const struct smp_sc *sc, struct smp_signspace *spc,
 int smp_chk_signspace(struct smp_signspace *spc);
 void smp_append_signspace(struct smp_signspace *spc, uint32_t len);
 void smp_reset_signspace(struct smp_signspace *spc);
+void smp_copy_signspace(struct smp_signspace *dst,
+			const struct smp_signspace *src);
+void smp_trunc_signspace(struct smp_signspace *spc, uint32_t len);
 
 void smp_newsilo(struct smp_sc *sc);
 int smp_valid_silo(struct smp_sc *sc);
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index ac93544..f7c748e 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -224,6 +224,37 @@ smp_reset_signspace(struct smp_signspace *spc)
 }
 
 /*--------------------------------------------------------------------
+ * Copy the contents of one signspace to another. Prepare for
+ * appending.
+ */
+
+void
+smp_copy_signspace(struct smp_signspace *dst, const struct smp_signspace *src)
+{
+	assert(SIGNSPACE_LEN(src) <= dst->size);
+	smp_reset_signspace(dst);
+	memcpy(SIGNSPACE_DATA(dst), SIGNSPACE_DATA(src), SIGNSPACE_LEN(src));
+	smp_append_signspace(dst, SIGNSPACE_LEN(src));
+	assert(SIGNSPACE_LEN(src) == SIGNSPACE_LEN(dst));
+}
+
+/*--------------------------------------------------------------------
+ * Reapplies the sign over the len first bytes of the
+ * signspace. Prepares for appending.
+ */
+
+void
+smp_trunc_signspace(struct smp_signspace *spc, uint32_t len)
+{
+	assert(len <= SIGNSPACE_LEN(spc));
+	spc->ctx.ss->length = 0;
+	SHA256_Init(&spc->ctx.ctx);
+	SHA256_Update(&spc->ctx.ctx, spc->ctx.ss,
+		      offsetof(struct smp_sign, length));
+	smp_append_signspace(spc, len);
+}
+
+/*--------------------------------------------------------------------
  * Create a new signature space and force the signature to backing store.
  */
 
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:11 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:11 +0200
Subject: [PATCH 04/13] Sync the complete sign area on smp_sync_sign
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-4-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent_subr.c |    2 +-
 include/persistent.h                           |    2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index f7c748e..9933541 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -159,7 +159,7 @@ smp_sync_sign(const struct smp_signctx *ctx)
 	int i;
 
 	/* XXX: round to pages */
-	i = msync((void*)ctx->ss, ctx->ss->length + SHA256_LEN, MS_SYNC);
+	i = msync((void*)ctx->ss, ctx->ss->length + SMP_SIGN_SPACE, MS_SYNC);
 	if (i && 0)
 		fprintf(stderr, "SyncSign(%p %s) = %d %s\n",
 		    ctx->ss, ctx->id, i, strerror(errno));
diff --git a/include/persistent.h b/include/persistent.h
index ff9207e..f0cb7f5 100644
--- a/include/persistent.h
+++ b/include/persistent.h
@@ -111,6 +111,8 @@ struct smp_sign {
 	char			ident[8];
 	uint32_t		unique;
 	uint64_t		mapped;
+	/* The length field is the length of the signed data only
+	 * (does not include struct smp_sign) */
 	uint64_t		length;		/* NB: Must be last */
 };
 
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:13 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:13 +0200
Subject: [PATCH 06/13] Free the LRU object and set free_offset when dropping
 empty segments in smp_close_seg()
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-6-git-send-email-martin@varnish-software.com>

Fixes: #1146
---
 bin/varnishd/storage/storage_persistent_silo.c |    6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 6ed80d2..d433cde 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -253,9 +253,11 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	sc->cur_seg = NULL;
 
 	if (sg->nalloc == 0) {
-		/* XXX: if segment is empty, delete instead */
+		/* If segment is empty, delete instead */
+		sc->free_offset = sg->p.offset;
 		VTAILQ_REMOVE(&sc->segments, sg, list);
-		free(sg);
+		LRU_Free(sg->lru);
+		FREE_OBJ(sg);
 		return;
 	}
 
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:14 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:14 +0200
Subject: [PATCH 07/13] Don't exit on full silo
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-7-git-send-email-martin@varnish-software.com>

This then also breaks the previous expectation that cur_seg would
always be non-NULL. Change the code to take this into account.
---
 bin/varnishd/storage/storage_persistent.c      |   24 +++++---
 bin/varnishd/storage/storage_persistent_silo.c |   71 ++++++++++++------------
 2 files changed, 54 insertions(+), 41 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index b54f497..744a5da 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -368,7 +368,9 @@ smp_close(const struct stevedore *st)
 
 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
 	Lck_Lock(&sc->mtx);
-	smp_close_seg(sc, sc->cur_seg);
+	if (sc->cur_seg != NULL)
+		smp_close_seg(sc, sc->cur_seg);
+	AZ(sc->cur_seg);
 	Lck_Unlock(&sc->mtx);
 
 	/* XXX: reap thread */
@@ -393,7 +395,6 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
 	struct smp_sc *sc;
 	struct storage *ss;
 	struct smp_seg *sg;
-	unsigned tries;
 	uint64_t left, extra;
 
 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
@@ -411,14 +412,22 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
 	Lck_Lock(&sc->mtx);
 	sg = NULL;
 	ss = NULL;
-	for (tries = 0; tries < 3; tries++) {
+
+	left = 0;
+	if (sc->cur_seg != NULL)
 		left = smp_spaceleft(sc, sc->cur_seg);
-		if (left >= extra + min_size)
-			break;
-		smp_close_seg(sc, sc->cur_seg);
+	if (left < extra + min_size) {
+		if (sc->cur_seg != NULL)
+			smp_close_seg(sc, sc->cur_seg);
 		smp_new_seg(sc);
+		if (sc->cur_seg != NULL)
+			left = smp_spaceleft(sc, sc->cur_seg);
+		else
+			left = 0;
 	}
+
 	if (left >= extra + min_size)  {
+		AN(sc->cur_seg);
 		if (left < extra + max_size)
 			max_size = IRNDN(sc, left - extra);
 
@@ -611,7 +620,8 @@ debug_persistent(struct cli *cli, const char * const * av, void *priv)
 	}
 	Lck_Lock(&sc->mtx);
 	if (!strcmp(av[3], "sync")) {
-		smp_close_seg(sc, sc->cur_seg);
+		if (sc->cur_seg != NULL)
+			smp_close_seg(sc, sc->cur_seg);
 		smp_new_seg(sc);
 	} else if (!strcmp(av[3], "dump")) {
 		debug_report_silo(cli, sc, 1);
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index d433cde..0014647 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -171,48 +171,50 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 void
 smp_new_seg(struct smp_sc *sc)
 {
-	struct smp_seg *sg, *sg2;
+	struct smp_seg tmpsg;
+	struct smp_seg *sg;
 
+	AZ(sc->cur_seg);
 	Lck_AssertHeld(&sc->mtx);
-	ALLOC_OBJ(sg, SMP_SEG_MAGIC);
-	AN(sg);
-	sg->sc = sc;
-	sg->lru = LRU_Alloc();
-	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
 
 	/* XXX: find where it goes in silo */
 
-	sg->p.offset = sc->free_offset;
-	// XXX: align */
-	assert(sg->p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
-	assert(sg->p.offset < sc->mediasize);
-
-	sg->p.length = sc->aim_segl;
-	sg->p.length &= ~7;
-
-	if (smp_segend(sg) > sc->mediasize) {
-		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
-		sg->p.offset = sc->free_offset;
-		sg2 = VTAILQ_FIRST(&sc->segments);
-		if (smp_segend(sg) > sg2->p.offset) {
-			printf("Out of space in persistent silo\n");
-			printf("Committing suicide, restart will make space\n");
-			exit (0);
-		}
+	memset(&tmpsg, 0, sizeof tmpsg);
+	tmpsg.magic = SMP_SEG_MAGIC;
+	tmpsg.sc = sc;
+	tmpsg.p.offset = sc->free_offset;
+	/* XXX: align */
+	assert(tmpsg.p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
+	assert(tmpsg.p.offset < sc->mediasize);
+
+	tmpsg.p.length = sc->aim_segl;
+	tmpsg.p.length &= ~7;
+
+	if (smp_segend(&tmpsg) > sc->mediasize)
+		/* XXX: Consider truncation in this case */
+		tmpsg.p.offset = sc->ident->stuff[SMP_SPC_STUFF];
+
+	assert(smp_segend(&tmpsg) <= sc->mediasize);
+
+	sg = VTAILQ_FIRST(&sc->segments);
+	if (sg != NULL && tmpsg.p.offset <= sg->p.offset) {
+		if (smp_segend(&tmpsg) > sg->p.offset)
+			/* No more space, return (cur_seg will be NULL) */
+			/* XXX: Consider truncation instead of failing */
+			return;
+		assert(smp_segend(&tmpsg) <= sg->p.offset);
 	}
 
+	if (tmpsg.p.offset == sc->ident->stuff[SMP_SPC_STUFF])
+		printf("Wrapped silo\n");
 
-	assert(smp_segend(sg) <= sc->mediasize);
-
-	sg2 = VTAILQ_FIRST(&sc->segments);
-	if (sg2 != NULL && sg2->p.offset > sc->free_offset) {
-		if (smp_segend(sg) > sg2->p.offset) {
-			printf("Out of space in persistent silo\n");
-			printf("Committing suicide, restart will make space\n");
-			exit (0);
-		}
-		assert(smp_segend(sg) <= sg2->p.offset);
-	}
+	ALLOC_OBJ(sg, SMP_SEG_MAGIC);
+	if (sg == NULL)
+		/* Failed allocation */
+		return;
+	*sg = tmpsg;
+	sg->lru = LRU_Alloc();
+	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
 
 	sg->p.offset = IRNUP(sc, sg->p.offset);
 	sg->p.length = IRNDN(sc, sg->p.length);
@@ -248,6 +250,7 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 
 	Lck_AssertHeld(&sc->mtx);
 
+	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
 	assert(sg == sc->cur_seg);
 	AN(sg->p.offset);
 	sc->cur_seg = NULL;
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:15 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:15 +0200
Subject: [PATCH 08/13] Add an assert that segment rounding doesn't overstep
 our previous allocation calculations.
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-8-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent_silo.c |    1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 0014647..cc4a060 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -218,6 +218,7 @@ smp_new_seg(struct smp_sc *sc)
 
 	sg->p.offset = IRNUP(sc, sg->p.offset);
 	sg->p.length = IRNDN(sc, sg->p.length);
+	assert(sg->p.offset + sg->p.length <= tmpsg.p.offset + tmpsg.p.length);
 	sc->free_offset = sg->p.offset + sg->p.length;
 
 	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:16 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:16 +0200
Subject: [PATCH 09/13] Add a signal_close callback method to stevedores,
 which can be used to signal background threads to stop in preparation
 for the coming close callback.
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-9-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/stevedore.c |    9 +++++++++
 bin/varnishd/storage/storage.h   |    2 ++
 2 files changed, 11 insertions(+)

diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index cd1a316..fa5a683 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -436,6 +436,15 @@ STV_close(void)
 {
 	struct stevedore *stv;
 
+	/* Signal intent to close */
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+		if (stv->signal_close != NULL)
+			stv->signal_close(stv);
+	stv = stv_transient;
+	if (stv->signal_close != NULL)
+		stv->signal_close(stv);
+
+	/* Close each in turn */
 	VTAILQ_FOREACH(stv, &stv_stevedores, list)
 		if (stv->close != NULL)
 			stv->close(stv);
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index aae9186..fd1adef 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -47,6 +47,7 @@ typedef void storage_free_f(struct storage *);
 typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *,
     struct objcore **, unsigned ltot, const struct stv_objsecrets *);
 typedef void storage_close_f(const struct stevedore *);
+typedef void storage_signal_close_f(const struct stevedore *);
 
 /* Prototypes for VCL variable responders */
 #define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
@@ -69,6 +70,7 @@ struct stevedore {
 	storage_free_f		*free;		/* --//-- */
 	storage_close_f		*close;		/* --//-- */
 	storage_allocobj_f	*allocobj;	/* --//-- */
+	storage_signal_close_f	*signal_close;	/* --//-- */
 
 	struct lru		*lru;
 
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:20 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:20 +0200
Subject: [PATCH 13/13] Add consistency checks between the ban lists at startup
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-13-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c |   15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index f38aba6..8f12c2e 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -336,9 +336,18 @@ smp_open(const struct stevedore *st)
 
 	sc->ident = SIGN_DATA(&sc->idn);
 
-	/* We attempt ban1 first, and if that fails, try ban2 */
-	if (smp_open_bans(sc, &sc->ban1))
-		AZ(smp_open_bans(sc, &sc->ban2));
+	/* Check ban lists */
+	if (smp_chk_signspace(&sc->ban1)) {
+		/* Ban list 1 is broken, use ban2 */
+		AZ(smp_chk_signspace(&sc->ban2));
+		smp_copy_signspace(&sc->ban1, &sc->ban2);
+		smp_sync_sign(&sc->ban1.ctx);
+	} else {
+		/* Ban1 is OK, copy to ban2 for consistency */
+		smp_copy_signspace(&sc->ban2, &sc->ban1);
+		smp_sync_sign(&sc->ban2.ctx);
+	}
+	AZ(smp_open_bans(sc, &sc->ban1));
 
 	/* We attempt seg1 first, and if that fails, try seg2 */
 	if (smp_open_segs(sc, &sc->seg1))
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:19 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:19 +0200
Subject: [PATCH 12/13] Also report dropped bans to the stevedores.
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-12-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/cache/cache.h       |    1 +
 bin/varnishd/cache/cache_ban.c   |    6 ++++++
 bin/varnishd/storage/stevedore.c |   10 ++++++++++
 bin/varnishd/storage/storage.h   |    3 +++
 4 files changed, 20 insertions(+)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7ecf954..95fc5e2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1055,6 +1055,7 @@ void STV_open(void);
 void STV_close(void);
 void STV_Freestore(struct object *o);
 void STV_NewBan(const uint8_t *ban, unsigned len);
+void STV_DropBan(const uint8_t *ban, unsigned len);
 
 /* storage_synth.c */
 struct vsb *SMS_Makesynth(struct object *obj);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index f895467..4f0d332 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -814,6 +814,9 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass)
 	do {
 		Lck_Lock(&ban_mtx);
 		b2 = ban_CheckLast();
+		if (b2 != NULL)
+			/* Notify stevedores */
+			STV_DropBan(b2->spec, ban_len(b2->spec));
 		Lck_Unlock(&ban_mtx);
 		if (b2 != NULL)
 			BAN_Free(b2);
@@ -961,6 +964,9 @@ ban_lurker(struct worker *wrk, void *priv)
 			 */
 			Lck_Lock(&ban_mtx);
 			bf = ban_CheckLast();
+			if (bf != NULL)
+				/* Notify stevedores */
+				STV_DropBan(bf->spec, ban_len(bf->spec));
 			Lck_Unlock(&ban_mtx);
 			if (bf != NULL)
 				BAN_Free(bf);
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index c02e5f7..b79f197 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -463,6 +463,16 @@ STV_NewBan(const uint8_t *ban, unsigned len)
 			stv->newban(stv, ban, len);
 }
 
+void
+STV_DropBan(const uint8_t *ban, unsigned len)
+{
+	struct stevedore *stv;
+
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+		if (stv->dropban != NULL)
+			stv->dropban(stv, ban, len);
+}
+
 /*--------------------------------------------------------------------
  * VRT functions for stevedores
  */
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 49cd0f8..740f282 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -50,6 +50,8 @@ typedef void storage_close_f(const struct stevedore *);
 typedef void storage_signal_close_f(const struct stevedore *);
 typedef void storage_newban_f(struct stevedore *, const uint8_t *ban,
     unsigned len);
+typedef void storage_dropban_f(struct stevedore *, const uint8_t *ban,
+    unsigned len);
 
 /* Prototypes for VCL variable responders */
 #define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
@@ -74,6 +76,7 @@ struct stevedore {
 	storage_allocobj_f	*allocobj;	/* --//-- */
 	storage_signal_close_f	*signal_close;	/* --//-- */
 	storage_newban_f	*newban;	/* --//-- */
+	storage_dropban_f	*dropban;	/* --//-- */
 
 	struct lru		*lru;
 
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:17 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:17 +0200
Subject: [PATCH 10/13] smp_thread() stopping
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-10-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c |   43 ++++++++++++++++++++++-------
 bin/varnishd/storage/storage_persistent.h |    2 ++
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 744a5da..6b8764f 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -51,6 +51,7 @@
 #include "vcli_priv.h"
 #include "vend.h"
 #include "vsha256.h"
+#include "vtim.h"
 
 #include "persistent.h"
 #include "storage/storage_persistent.h"
@@ -287,15 +288,24 @@ smp_thread(struct worker *wrk, void *priv)
 	BAN_TailDeref(&sc->tailban);
 	AZ(sc->tailban);
 	printf("Silo completely loaded\n");
-	while (1) {
-		(void)sleep (1);
+
+	/* Housekeeping loop */
+	Lck_Lock(&sc->mtx);
+	while (!(sc->flags & SMP_SC_STOP)) {
 		sg = VTAILQ_FIRST(&sc->segments);
-		if (sg != NULL && sg -> sc->cur_seg && sg->nobj == 0) {
-			Lck_Lock(&sc->mtx);
+		if (sg != NULL && sg != sc->cur_seg && sg->nobj == 0)
 			smp_save_segs(sc);
-			Lck_Unlock(&sc->mtx);
-		}
+
+		Lck_Unlock(&sc->mtx);
+		VTIM_sleep(3.14159265359 - 2);
+		Lck_Lock(&sc->mtx);
 	}
+
+	smp_save_segs(sc);
+
+	Lck_Unlock(&sc->mtx);
+	pthread_exit(0);
+
 	NEEDLESS_RETURN(NULL);
 }
 
@@ -307,7 +317,6 @@ static void
 smp_open(const struct stevedore *st)
 {
 	struct smp_sc	*sc;
-	pthread_t pt;
 
 	ASSERT_CLI();
 
@@ -349,7 +358,7 @@ smp_open(const struct stevedore *st)
 	smp_new_seg(sc);
 
 	/* Start the worker silo worker thread, it will load the objects */
-	WRK_BgThread(&pt, "persistence", smp_thread, sc);
+	WRK_BgThread(&sc->bgthread, "persistence", smp_thread, sc);
 
 	VTAILQ_INSERT_TAIL(&silos, sc, list);
 	Lck_Unlock(&sc->mtx);
@@ -360,7 +369,7 @@ smp_open(const struct stevedore *st)
  */
 
 static void
-smp_close(const struct stevedore *st)
+smp_signal_close(const struct stevedore *st)
 {
 	struct smp_sc	*sc;
 
@@ -371,9 +380,22 @@ smp_close(const struct stevedore *st)
 	if (sc->cur_seg != NULL)
 		smp_close_seg(sc, sc->cur_seg);
 	AZ(sc->cur_seg);
+	sc->flags |= SMP_SC_STOP;
 	Lck_Unlock(&sc->mtx);
+}
+
+static void
+smp_close(const struct stevedore *st)
+{
+	struct smp_sc	*sc;
+	void *status;
+
+	ASSERT_CLI();
+
+	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
 
-	/* XXX: reap thread */
+	pthread_join(sc->bgthread, &status);
+	AZ(status);
 }
 
 /*--------------------------------------------------------------------
@@ -561,6 +583,7 @@ const struct stevedore smp_stevedore = {
 	.alloc	=	smp_alloc,
 	.allocobj =	smp_allocobj,
 	.free	=	smp_free,
+	.signal_close = smp_signal_close,
 };
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index aceb996..cb2efb3 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -104,8 +104,10 @@ struct smp_sc {
 #define SMP_SC_MAGIC		0x7b73af0a
 	struct stevedore	*parent;
 
+	pthread_t		bgthread;
 	unsigned		flags;
 #define SMP_SC_LOADED		(1 << 0)
+#define SMP_SC_STOP		(1 << 1)
 
 	const struct stevedore	*stevedore;
 	int			fd;
-- 
1.7.9.5




From martin at varnish-software.com  Mon Oct  1 10:26:18 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  1 Oct 2012 12:26:18 +0200
Subject: [PATCH 11/13] Generalize the ban reporting to the stevedores using
 their API. This way any stevedore interested in new bans can request to be
 notified (not just the persistent).
In-Reply-To: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
References: <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349087180-11089-11-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/cache/cache.h                |    2 +-
 bin/varnishd/cache/cache_ban.c            |    6 ++++--
 bin/varnishd/storage/stevedore.c          |    9 +++++++++
 bin/varnishd/storage/storage.h            |    3 +++
 bin/varnishd/storage/storage_persistent.c |    7 +++++--
 5 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7533782..7ecf954 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1054,6 +1054,7 @@ void STV_free(struct storage *st);
 void STV_open(void);
 void STV_close(void);
 void STV_Freestore(struct object *o);
+void STV_NewBan(const uint8_t *ban, unsigned len);
 
 /* storage_synth.c */
 struct vsb *SMS_Makesynth(struct object *obj);
@@ -1063,7 +1064,6 @@ void SMS_Init(void);
 /* storage_persistent.c */
 void SMP_Init(void);
 void SMP_Ready(void);
-void SMP_NewBan(const uint8_t *ban, unsigned len);
 
 /*
  * A normal pointer difference is signed, but we never want a negative value
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 2054d3e..f895467 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -413,7 +413,7 @@ BAN_Insert(struct ban *b)
 	else
 		be = NULL;
 
-	SMP_NewBan(b->spec, ln);
+	STV_NewBan(b->spec, ln);	/* Notify stevedores */
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
@@ -595,7 +595,9 @@ BAN_Compile(void)
 
 	ASSERT_CLI();
 
-	SMP_NewBan(ban_magic->spec, ban_len(ban_magic->spec));
+	/* Notify stevedores */
+	STV_NewBan(ban_magic->spec, ban_len(ban_magic->spec));
+
 	ban_start = VTAILQ_FIRST(&ban_head);
 	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index fa5a683..c02e5f7 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -453,6 +453,15 @@ STV_close(void)
 		stv->close(stv);
 }
 
+void
+STV_NewBan(const uint8_t *ban, unsigned len)
+{
+	struct stevedore *stv;
+
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+		if (stv->newban != NULL)
+			stv->newban(stv, ban, len);
+}
 
 /*--------------------------------------------------------------------
  * VRT functions for stevedores
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index fd1adef..49cd0f8 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -48,6 +48,8 @@ typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *,
     struct objcore **, unsigned ltot, const struct stv_objsecrets *);
 typedef void storage_close_f(const struct stevedore *);
 typedef void storage_signal_close_f(const struct stevedore *);
+typedef void storage_newban_f(struct stevedore *, const uint8_t *ban,
+    unsigned len);
 
 /* Prototypes for VCL variable responders */
 #define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
@@ -71,6 +73,7 @@ struct stevedore {
 	storage_close_f		*close;		/* --//-- */
 	storage_allocobj_f	*allocobj;	/* --//-- */
 	storage_signal_close_f	*signal_close;	/* --//-- */
+	storage_newban_f	*newban;	/* --//-- */
 
 	struct lru		*lru;
 
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 6b8764f..f38aba6 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -92,11 +92,13 @@ smp_appendban(struct smp_sc *sc, struct smp_signspace *spc,
 
 /* Trust that cache_ban.c takes care of locking */
 
-void
-SMP_NewBan(const uint8_t *ban, unsigned ln)
+static void
+smp_newban(struct stevedore *stv, const uint8_t *ban, unsigned ln)
 {
 	struct smp_sc *sc;
 
+	(void)stv;
+
 	VTAILQ_FOREACH(sc, &silos, list) {
 		smp_appendban(sc, &sc->ban1, ln, ban);
 		smp_appendban(sc, &sc->ban2, ln, ban);
@@ -584,6 +586,7 @@ const struct stevedore smp_stevedore = {
 	.allocobj =	smp_allocobj,
 	.free	=	smp_free,
 	.signal_close = smp_signal_close,
+	.newban =	smp_newban,
 };
 
 /*--------------------------------------------------------------------
-- 
1.7.9.5




From tfheen at varnish-software.com  Tue Oct  2 06:41:24 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Tue,  2 Oct 2012 08:41:24 +0200
Subject: [PATCH] Don't put POLLHUP i .events
Message-ID: <1349160084-21519-1-git-send-email-tfheen@varnish-software.com>

POLLHUP is always checked for, and is output-only on Linux and OSX, so
avoid putting it in the list of events we check for.
---
 bin/varnishtest/vtc_http.c    |    2 +-
 bin/varnishtest/vtc_varnish.c |   10 +---------
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 0678efd..4f084ad 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1043,7 +1043,7 @@ cmd_http_expect_close(CMD_ARGS)
 	vtc_log(vl, 4, "Expecting close (fd = %d)", hp->fd);
 	while (1) {
 		fds[0].fd = hp->fd;
-		fds[0].events = POLLIN | POLLHUP | POLLERR;
+		fds[0].events = POLLIN | POLLERR;
 		fds[0].revents = 0;
 		i = poll(fds, 1, 1000);
 		if (i == 0)
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 0036720..794080f 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -395,16 +395,8 @@ varnish_launch(struct varnish *v)
 	fd[0].fd = v->cli_fd;
 	fd[0].events = POLLIN;
 	fd[1].fd = v->fds[0];
-	fd[1].events = POLLHUP;
-#ifdef __APPLE__
-	/*
-	 * OSX cannot poll a pipe for POLLHUP only, poll just returns
-	 * zero with no revents.
-	 */
-	i = poll(fd, 1, 10000);
-#else
+	fd[1].events = 0; /* Only care about POLLHUP, which is output-only */
 	i = poll(fd, 2, 10000);
-#endif
 	vtc_log(v->vl, 4, "CLIPOLL %d 0x%x 0x%x",
 	    i, fd[0].revents, fd[1].revents);
 	if (i == 0) {
-- 
1.7.10.4




From phk at phk.freebsd.dk  Tue Oct  9 07:20:16 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 09 Oct 2012 07:20:16 +0000
Subject: [PATCH 01/13] Add a EXP_NukeLRU() function to nuke an entire LRU
 structure at a time.
In-Reply-To: Your message of "Mon, 01 Oct 2012 12:26:08 +0200."
 <1349087180-11089-1-git-send-email-martin@varnish-software.com>
Message-ID: <60269.1349767216@critter.freebsd.dk>

In message <1349087180-11089-1-git-send-email-martin at varnish-software.com>, Mar
tin Blix Grydeland writes:

I may have asked this before, but why not simply make a
local VTAILQ_HEAD on the stack and move everything there in one
iteration ?

Is it to not block the mutex'es too long ?

Poul-Henning

PS: This email is _also_ a sort of test of patchwork :-)

>+#define NUKEBUF 10
>+
>+void
>+EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
>+{
>+	struct objcore *oc;
>+	struct objcore *oc_array[NUKEBUF];
>+	struct object *o;
>+	int i, n;
>+	double t;
>+
>+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
>+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
>+
>+	t = VTIM_real();
>+	Lck_Lock(&lru->mtx);
>+	while (!VTAILQ_EMPTY(&lru->lru_head)) {
>+		Lck_Lock(&exp_mtx);
>+		n = 0;
>+		while (n < NUKEBUF) {
>+			oc = VTAILQ_FIRST(&lru->lru_head);
>+			if (oc == NULL)
>+				break;
>+			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
>+			assert(oc_getlru(oc) == lru);
>+
>+			/* Remove from the LRU and binheap */
>+			VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
>+			assert(oc->timer_idx != BINHEAP_NOIDX);
>+			binheap_delete(exp_heap, oc->timer_idx);
>+			assert(oc->timer_idx == BINHEAP_NOIDX);
>+
>+			oc_array[n++] = oc;
>+			VSC_C_main->n_lru_nuked++;
>+		}




-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From tfheen at varnish-software.com  Wed Oct 10 12:42:09 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Wed, 10 Oct 2012 14:42:09 +0200
Subject: RFC: bans with grace (aka softbans)
Message-ID: <20121010124209.GJ19207@err.no>

Hi all,

The current implementation of bans in Varnish is fine for what it was
originally designed for: getting rid of chunks of your cache, fast and
in a manner that makes sure the matching objects are never seen again.
Over time, we've seen that people use it in many setups where it's less
imperative that the content is not seen again, that is, they want to
have the expressiveness of bans, but they also want grace to apply.

Our suggested solution for this is softbans: It's like a ban, but it
just expires the content by changing the TTL and does not touch any of
the grace or keep counters.

The current ban interface is:

 * ban(ban_expression) and ban_url(regex) in VCL.  
 * ?ban <ban_expression>?, ?ban.url <regex>? in varnishadm.
 * ban.list in varnishadm

Possible interface for adding softbans:

softban(regex); (in VCL)
?softban <regex>? (in varnishadm)

ban_url(URL) is an alias for ban(req.http.url ~ URL). Soft bans does not
have this alias.

Possible interface for listing softbans:

The varnishadm ban.list output is used for diagnostics. A ban.list
currently looks like this:

0x2aab74ae4540 1343650837.638163 ??468 ?obj.http.x-url ~ /article1.html
0x2aab74ae4500 1343650837.582181 ????0 ?obj.http.x-url ~ /article2.html
0x2aab74a9a540 1343649766.052658 ???54G obj.http.x-url ~ /article3.html

It is suggested that an extra ?s? flag is introduced along with the
existing ?G? (Gone) flag in the output. If deemed necessary for
completeness a ?h? flag for hard bans may be introduced as well.

Comments are welcome,
-- 
Tollef Fog Heen
Technical lead, Varnish Software
t: +47 21 98 92 64



From martin at varnish-software.com  Wed Oct 10 14:27:14 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:14 +0200
Subject: [PATCH 01/10] Maintain an offset into the persisted ban list pointing
 to the first non-dropped ban.
Message-ID: <1349879243-30369-1-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c |   37 +++++++++++++++++++++++++++++
 bin/varnishd/storage/storage_persistent.h |    1 +
 2 files changed, 38 insertions(+)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 5f756b7..f5398b9 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -90,6 +90,37 @@ smp_appendban(struct smp_sc *sc, struct smp_signspace *spc,
 	smp_append_signspace(spc, ptr - ptr2);
 }
 
+/*--------------------------------------------------------------------
+ * Update *offset with the offset of the first ban record past ban
+ */
+
+static void
+smp_offset_banlist(struct smp_sc *sc, struct smp_signspace *spc,
+		   uint64_t *offset, const uint8_t *ban, unsigned ln)
+{
+	uint8_t *ptr, *ptr2;
+	uint32_t len;
+
+	(void)sc;
+	ptr = ptr2 = (uint8_t *)SIGNSPACE_DATA(spc) + *offset;
+	assert(ptr <= (uint8_t *)SIGNSPACE_FRONT(spc));
+
+	while (ptr < (uint8_t *)SIGNSPACE_FRONT(spc)) {
+		AZ(memcmp(ptr, "BAN", 4));
+		ptr += 4;
+
+		len = vbe32dec(ptr);
+		ptr += 4;
+
+		ptr += len;
+
+		if (len == ln && !memcmp(ptr - len, ban, len)) {
+			*offset += ptr - ptr2;
+			return;
+		}
+	}
+}
+
 /* Trust that cache_ban.c takes care of locking */
 
 static void
@@ -106,6 +137,12 @@ smp_baninfo(struct stevedore *stv, enum baninfo event,
 			smp_appendban(sc, &sc->ban2, len, ban);
 		}
 		break;
+	case BI_DROP:
+		VTAILQ_FOREACH(sc, &silos, list) {
+			smp_offset_banlist(sc, &sc->ban1, &sc->ban1_offset,
+					   ban, len);
+		}
+		break;
 	default:
 		/* Ignored */
 		break;
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index cb2efb3..c2cbac3 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -139,6 +139,7 @@ struct smp_sc {
 	struct smp_signspace	seg2;
 
 	struct ban		*tailban;
+	uint64_t		ban1_offset;
 
 	struct lock		mtx;
 
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:15 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:15 +0200
Subject: [PATCH 02/10] Add a BAN_Shutdown() routine that is called before
 STV_Close(), and makes sure that the ban lists will not change until we exit.
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-2-git-send-email-martin@varnish-software.com>

This is to give persistent stevedores a chance to clean up their ban
lists without having to worry about the lists changing under them
(normally the ban callbacks are called under the ban mutex, but not so
during exit).

This is achieved by locking the ban mutex on BAN_Shutdown(), and never
releasing it again.
---
 bin/varnishd/cache/cache.h      |    1 +
 bin/varnishd/cache/cache_ban.c  |   14 ++++++++++++++
 bin/varnishd/cache/cache_main.c |    1 +
 3 files changed, 16 insertions(+)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6d4fe86..49effb2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -748,6 +748,7 @@ int BAN_AddTest(struct cli *, struct ban *, const char *, const char *,
 void BAN_Free(struct ban *b);
 void BAN_Insert(struct ban *b);
 void BAN_Init(void);
+void BAN_Shutdown(void);
 void BAN_NewObjCore(struct objcore *oc);
 void BAN_DestroyObj(struct objcore *oc);
 int BAN_CheckObject(struct object *o, struct req *sp);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index aa8bca9..1e7cce8 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1145,3 +1145,17 @@ BAN_Init(void)
 	VSC_C_main->bans_gone++;
 	BAN_Insert(ban_magic);
 }
+
+/*
+ * Take the ban_mtx and return, preventing any BAN functions from succeeding. 
+ *
+ * This is called before STV_close to allow persistent stevedores to
+ * close down without any bans changing (the persistent ban callbacks
+ * are run under the ban_mtx).
+ */
+
+void
+BAN_Shutdown(void)
+{
+	Lck_Lock(&ban_mtx);
+}
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index ac36308..d30f92c 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -230,6 +230,7 @@ child_main(void)
 
 	CLI_Run();
 
+	BAN_Shutdown();
 	STV_close();
 
 	printf("Child dies\n");
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:19 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:19 +0200
Subject: [PATCH 06/10] Signal the smp_thread() by condvar
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-6-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c      |    8 +++++---
 bin/varnishd/storage/storage_persistent.h      |    1 +
 bin/varnishd/storage/storage_persistent_silo.c |    4 ++--
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 0575c11..429cd45 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -429,9 +429,9 @@ smp_thread(struct worker *wrk, void *priv)
 		if (sc->flags & SMP_SC_SYNC)
 			smp_save_segs(sc);
 
-		Lck_Unlock(&sc->mtx);
-		VTIM_sleep(3.14159265359 - 2);
-		Lck_Lock(&sc->mtx);
+		if (!(sc->flags & (SMP_SC_SYNC | SMP_SC_STOP)))
+			/* Wait for something to do */
+			(void)Lck_CondWait(&sc->cond, &sc->mtx, NULL);
 	}
 
 	smp_save_segs(sc);
@@ -455,6 +455,7 @@ smp_open(const struct stevedore *st)
 
 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
 
+	AZ(pthread_cond_init(&sc->cond, NULL));
 	Lck_New(&sc->mtx, lck_smp);
 	Lck_Lock(&sc->mtx);
 
@@ -523,6 +524,7 @@ smp_signal_close(const struct stevedore *st)
 		smp_close_seg(sc, sc->cur_seg);
 	AZ(sc->cur_seg);
 	sc->flags |= SMP_SC_STOP;
+	AZ(pthread_cond_signal(&sc->cond));
 	Lck_Unlock(&sc->mtx);
 }
 
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index 0c4e3a3..e716c51 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -142,6 +142,7 @@ struct smp_sc {
 	struct ban		*tailban;
 	uint64_t		ban1_offset;
 
+	pthread_cond_t		cond;
 	struct lock		mtx;
 
 	/* Cleaner metrics */
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index fb384ee..9dd827e 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -56,6 +56,7 @@ smp_sync_segs(struct smp_sc *sc)
 {
 	Lck_AssertHeld(&sc->mtx);
 	sc->flags |= SMP_SC_SYNC;
+	AZ(pthread_cond_signal(&sc->cond));
 }
 
 /*--------------------------------------------------------------------
@@ -269,7 +270,6 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	sc->free_offset = smp_segend(sg);
 }
 
-
 /*---------------------------------------------------------------------
  */
 
@@ -487,7 +487,7 @@ smp_oc_freeobj(struct objcore *oc)
 
 	if (sg->nobj == 0 && sg == VTAILQ_FIRST(&sg->sc->segments)) {
 		/* Sync segments to remove empty at start */
-		sg->sc->flags |= SMP_SC_SYNC;
+		smp_sync_segs(sg->sc);
 	}
 
 	Lck_Unlock(&sg->sc->mtx);
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:16 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:16 +0200
Subject: [PATCH 03/10] Truncate ban lists on smp_close(), and when full.
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-3-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c |   32 +++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index f5398b9..914de31 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -121,6 +121,33 @@ smp_offset_banlist(struct smp_sc *sc, struct smp_signspace *spc,
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Truncate ban1 eliminating everything up to ban1_offset. Copy to ban2
+ * and sync.
+ */
+
+static void
+smp_compact_banlists(struct smp_sc *sc)
+{
+	uint8_t *ptr, *ptr2;
+
+	if (sc->ban1_offset == 0)
+		return;
+
+	/* Compact ban1 */
+	ptr = (uint8_t *)SIGNSPACE_DATA(&sc->ban1) + sc->ban1_offset;
+	ptr2 = (uint8_t *)SIGNSPACE_FRONT(&sc->ban1);
+	assert(ptr <= ptr2);
+	memmove(SIGNSPACE_DATA(&sc->ban1), ptr, ptr2 - ptr);
+	sc->ban1_offset = 0;
+	smp_trunc_signspace(&sc->ban1, ptr2 - ptr);
+	smp_sync_sign(&sc->ban1.ctx);
+
+	/* Copy to ban2 */
+	smp_copy_signspace(&sc->ban2, &sc->ban1);
+	smp_sync_sign(&sc->ban2.ctx);
+}
+
 /* Trust that cache_ban.c takes care of locking */
 
 static void
@@ -133,6 +160,9 @@ smp_baninfo(struct stevedore *stv, enum baninfo event,
 	switch (event) {
 	case BI_NEW:
 		VTAILQ_FOREACH(sc, &silos, list) {
+			if (SIGNSPACE_FREE(&sc->ban1) < len + 8)
+				smp_compact_banlists(sc);
+
 			smp_appendban(sc, &sc->ban1, len, ban);
 			smp_appendban(sc, &sc->ban2, len, ban);
 		}
@@ -449,6 +479,8 @@ smp_close(const struct stevedore *st)
 
 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
 
+	smp_compact_banlists(sc);
+
 	pthread_join(sc->bgthread, &status);
 	AZ(status);
 }
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:20 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:20 +0200
Subject: [PATCH 07/10] Mark new segments as new,
 and don't write them to the persisted segment table until they are
 closed.
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-7-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c      |    6 ++++--
 bin/varnishd/storage/storage_persistent.h      |    1 +
 bin/varnishd/storage/storage_persistent_silo.c |    2 ++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 429cd45..021c5e2 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -368,8 +368,8 @@ smp_save_segs(struct smp_sc *sc)
 	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
 		if (sg->nobj > 0)
 			break;
-		if (sg == sc->cur_seg)
-			continue;
+		if (sg->flags & SMP_SEG_NEW)
+			break;
 		VTAILQ_REMOVE(&sc->segments, sg, list);
 		LRU_Free(sg->lru);
 		FREE_OBJ(sg);
@@ -386,6 +386,8 @@ smp_save_segs(struct smp_sc *sc)
 	VTAILQ_FOREACH(sg, &sc->segments, list) {
 		assert(sg->p.offset < sc->mediasize);
 		assert(sg->p.offset + sg->p.length <= sc->mediasize);
+		if (sg->flags & SMP_SEG_NEW)
+			break;
 		*ss = sg->p;
 		ss++;
 		length += sizeof *ss;
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index e716c51..cb2fc82 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -87,6 +87,7 @@ struct smp_seg {
 	unsigned		flags;
 #define SMP_SEG_MUSTLOAD	(1 << 0)
 #define SMP_SEG_LOADED		(1 << 1)
+#define SMP_SEG_NEW		(1 << 2)
 
 	uint32_t		nobj;		/* Number of objects */
 	uint32_t		nalloc;		/* Allocations */
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 9dd827e..981d1c6 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -176,6 +176,7 @@ smp_new_seg(struct smp_sc *sc)
 		return;
 	*sg = tmpsg;
 	sg->lru = LRU_Alloc();
+	sg->flags |= SMP_SEG_NEW;
 	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
 
 	sg->p.offset = IRNUP(sc, sg->p.offset);
@@ -266,6 +267,7 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	smp_sync_sign(sg->ctx);
 
 	/* Request sync of segment list */
+	sg->flags &= ~SMP_SEG_NEW;
 	smp_sync_segs(sc);
 	sc->free_offset = smp_segend(sg);
 }
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:17 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:17 +0200
Subject: [PATCH 04/10] Don't always check the sign after smp_append_sign().
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-4-git-send-email-martin@varnish-software.com>

Remove the XXXAZ(smp_chk_sign) after smp_append_sign(), as it causes
heavy unnecessary CPU usage after each sign update. No assertion
reports have come from this test, so there is no reason to expect the
signs not working.
---
 bin/varnishd/storage/storage_persistent_subr.c |    1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index f73ae06..a18f2cc 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -128,7 +128,6 @@ smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len)
 	SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
 	SHA256_Final(sign, &cx);
 	memcpy(SIGN_END(ctx), sign, sizeof sign);
-XXXAZ(smp_chk_sign(ctx));
 }
 
 /*--------------------------------------------------------------------
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:21 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:21 +0200
Subject: [PATCH 08/10] Defer syncing signs inside the segments for later,
 and do it without locks in smp_thread().
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-8-git-send-email-martin@varnish-software.com>

This is to avoid doing long IO operations while holding the mutex.
---
 bin/varnishd/storage/storage_persistent.c      |   26 ++++++++++++--
 bin/varnishd/storage/storage_persistent.h      |    6 +++-
 bin/varnishd/storage/storage_persistent_silo.c |   44 +++++++++++-------------
 3 files changed, 50 insertions(+), 26 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 021c5e2..00bd940 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -375,6 +375,26 @@ smp_save_segs(struct smp_sc *sc)
 		FREE_OBJ(sg);
 	}
 
+	/* Sync the signs of any segments marked as needing it */
+	VTAILQ_FOREACH(sg, &sc->segments, list) {
+		if (sg->flags & SMP_SEG_SYNCSIGNS) {
+			AZ(sg->flags & SMP_SEG_NEW);
+			AN(sg->nalloc); /* Empty segments shouldn't be here */
+
+			/* Make sure they have all been set up */
+			AN(sg->ctx_head.ss);
+			AN(sg->ctx_obj.ss);
+			AN(sg->ctx_tail.ss);
+
+			sg->flags &= ~SMP_SEG_SYNCSIGNS;
+			Lck_Unlock(&sc->mtx);
+			smp_sync_sign(&sg->ctx_head);
+			smp_sync_sign(&sg->ctx_obj);
+			smp_sync_sign(&sg->ctx_tail);
+			Lck_Lock(&sc->mtx);
+		}
+	}
+
 	Lck_Unlock(&sc->mtx);
 	AZ(smp_chk_signspace(&sc->seg1)); /* Page in */
 	smp_reset_signspace(&sc->seg1);
@@ -589,7 +609,8 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
 	if (left < extra + min_size) {
 		if (sc->cur_seg != NULL)
 			smp_close_seg(sc, sc->cur_seg);
-		smp_new_seg(sc);
+		if (sc->cur_seg == NULL)
+			smp_new_seg(sc);
 		if (sc->cur_seg != NULL)
 			left = smp_spaceleft(sc, sc->cur_seg);
 		else
@@ -800,7 +821,8 @@ debug_persistent(struct cli *cli, const char * const * av, void *priv)
 			VTIM_sleep(0.1);
 			Lck_Lock(&sc->mtx);
 		}
-		smp_new_seg(sc);
+		if (sc->cur_seg == NULL)
+			smp_new_seg(sc);
 	} else if (!strcmp(av[3], "dump")) {
 		debug_report_silo(cli, sc, 1);
 	} else {
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index cb2fc82..93ec45e 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -88,6 +88,7 @@ struct smp_seg {
 #define SMP_SEG_MUSTLOAD	(1 << 0)
 #define SMP_SEG_LOADED		(1 << 1)
 #define SMP_SEG_NEW		(1 << 2)
+#define SMP_SEG_SYNCSIGNS	(1 << 3)
 
 	uint32_t		nobj;		/* Number of objects */
 	uint32_t		nalloc;		/* Allocations */
@@ -95,7 +96,10 @@ struct smp_seg {
 
 	/* Only for open segment */
 	struct smp_object	*objs;		/* objdesc array */
-	struct smp_signctx	ctx[1];
+
+	struct smp_signctx	ctx_head;
+	struct smp_signctx	ctx_obj;
+	struct smp_signctx	ctx_tail;
 };
 
 VTAILQ_HEAD(smp_seghead, smp_seg);
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 981d1c6..addb836 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -81,7 +81,6 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 	struct objcore *oc;
 	uint32_t no;
 	double t_now = VTIM_real();
-	struct smp_signctx ctx[1];
 
 	ASSERT_SILO_THREAD(sc);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -92,8 +91,8 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 	AN(sg->p.offset);
 	if (sg->p.objlist == 0)
 		return;
-	smp_def_sign(sc, ctx, sg->p.offset, "SEGHEAD");
-	if (smp_chk_sign(ctx))
+	smp_def_sign(sc, &sg->ctx_head, sg->p.offset, "SEGHEAD");
+	if (smp_chk_sign(&sg->ctx_head))
 		return;
 
 	/* test SEGTAIL */
@@ -186,12 +185,6 @@ smp_new_seg(struct smp_sc *sc)
 
 	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
 
-	/* Neuter the new segment in case there is an old one there */
-	AN(sg->p.offset);
-	smp_def_sign(sc, sg->ctx, sg->p.offset, "SEGHEAD");
-	smp_reset_sign(sg->ctx);
-	smp_sync_sign(sg->ctx);
-
 	/* Set up our allocation points */
 	sc->cur_seg = sg;
 	sc->next_bot = sg->p.offset + IRNUP(sc, SMP_SIGN_SPACE);
@@ -200,6 +193,11 @@ smp_new_seg(struct smp_sc *sc)
 	IASSERTALIGN(sc, sc->next_bot);
 	IASSERTALIGN(sc, sc->next_top);
 	sg->objs = (void*)(sc->base + sc->next_top);
+
+	/* Neuter the new segment in case there is an old one there */
+	AN(sg->p.offset);
+	smp_def_sign(sc, &sg->ctx_head, sg->p.offset, "SEGHEAD");
+	smp_reset_sign(&sg->ctx_head);
 }
 
 /*--------------------------------------------------------------------
@@ -247,29 +245,29 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 		sg->p.length = (sc->next_top - sg->p.offset)
 		     + len + IRNUP(sc, SMP_SIGN_SPACE);
 		(void)smp_spaceleft(sc, sg);	/* for the asserts */
-
 	}
+	sc->free_offset = smp_segend(sg);
 
 	/* Update the segment header */
 	sg->p.objlist = sc->next_top;
 
-	/* Write the (empty) OBJIDX signature */
-	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
-	assert(sc->next_top >= sc->next_bot);
-	smp_def_sign(sc, sg->ctx, sc->next_top, "OBJIDX");
-	smp_reset_sign(sg->ctx);
-	smp_sync_sign(sg->ctx);
+	dst = sc->next_top - IRNUP(sc, SMP_SIGN_SPACE);
+	assert(dst >= sc->next_bot);
 
+	/* Write the (empty) OBJIDX signature */
+	smp_def_sign(sc, &sg->ctx_obj, dst, "OBJIDX");
+	smp_reset_sign(&sg->ctx_obj);
 	/* Write the (empty) SEGTAIL signature */
-	smp_def_sign(sc, sg->ctx,
-	    sg->p.offset + sg->p.length - IRNUP(sc, SMP_SIGN_SPACE), "SEGTAIL");
-	smp_reset_sign(sg->ctx);
-	smp_sync_sign(sg->ctx);
-
-	/* Request sync of segment list */
+	smp_def_sign(sc, &sg->ctx_tail,
+	    sg->p.offset + sg->p.length - IRNUP(sc, SMP_SIGN_SPACE),
+	    "SEGTAIL");
+	smp_reset_sign(&sg->ctx_tail);
+	/* Ask smp_thread() to sync the signs */
+	sg->flags |= SMP_SEG_SYNCSIGNS;
+
+	/* Remove the new flag and request sync of segment list */
 	sg->flags &= ~SMP_SEG_NEW;
 	smp_sync_segs(sc);
-	sc->free_offset = smp_segend(sg);
 }
 
 /*---------------------------------------------------------------------
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:18 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:18 +0200
Subject: [PATCH 05/10] Sync the segment list in smp_thread().
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-5-git-send-email-martin@varnish-software.com>

This allows us to avoid waiting for disk IO while holding the silo
mutex.
---
 bin/varnishd/storage/storage_persistent.c      |   67 +++++++++++++++++++++++-
 bin/varnishd/storage/storage_persistent.h      |    3 +-
 bin/varnishd/storage/storage_persistent_silo.c |   66 ++++++-----------------
 3 files changed, 83 insertions(+), 53 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 914de31..0575c11 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -342,6 +342,64 @@ smp_open_segs(struct smp_sc *sc, struct smp_signspace *spc)
 }
 
 /*--------------------------------------------------------------------
+ * Write the segmentlist back to the silo.
+ *
+ * We write the first copy, sync it synchronously, then write the
+ * second copy and sync it synchronously.
+ *
+ * Provided the kernel doesn't lie, that means we will always have
+ * at least one valid copy on in the silo.
+ */
+
+static void
+smp_save_segs(struct smp_sc *sc)
+{
+	struct smp_segptr *ss;
+	struct smp_seg *sg, *sg2;
+	uint64_t length;
+
+	Lck_AssertHeld(&sc->mtx);
+	sc->flags &= ~SMP_SC_SYNC;
+
+	/*
+	 * Remove empty segments from the front of the list
+	 * before we write the segments to disk.
+	 */
+	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
+		if (sg->nobj > 0)
+			break;
+		if (sg == sc->cur_seg)
+			continue;
+		VTAILQ_REMOVE(&sc->segments, sg, list);
+		LRU_Free(sg->lru);
+		FREE_OBJ(sg);
+	}
+
+	Lck_Unlock(&sc->mtx);
+	AZ(smp_chk_signspace(&sc->seg1)); /* Page in */
+	smp_reset_signspace(&sc->seg1);
+	Lck_Lock(&sc->mtx);
+
+	/* First write to seg1 while holding lock */
+	ss = SIGNSPACE_FRONT(&sc->seg1);
+	length = 0;
+	VTAILQ_FOREACH(sg, &sc->segments, list) {
+		assert(sg->p.offset < sc->mediasize);
+		assert(sg->p.offset + sg->p.length <= sc->mediasize);
+		*ss = sg->p;
+		ss++;
+		length += sizeof *ss;
+	}
+
+	Lck_Unlock(&sc->mtx);
+	smp_append_signspace(&sc->seg1, length);
+	smp_sync_sign(&sc->seg1.ctx); /* Sync without lock */
+	/* Copy seg1 to seg2 */
+	smp_copy_signspace(&sc->seg2, &sc->seg1);
+	smp_sync_sign(&sc->seg2.ctx);
+	Lck_Lock(&sc->mtx);
+}
+/*--------------------------------------------------------------------
  * Silo worker thread
  */
 
@@ -368,8 +426,7 @@ smp_thread(struct worker *wrk, void *priv)
 	/* Housekeeping loop */
 	Lck_Lock(&sc->mtx);
 	while (!(sc->flags & SMP_SC_STOP)) {
-		sg = VTAILQ_FIRST(&sc->segments);
-		if (sg != NULL && sg != sc->cur_seg && sg->nobj == 0)
+		if (sc->flags & SMP_SC_SYNC)
 			smp_save_segs(sc);
 
 		Lck_Unlock(&sc->mtx);
@@ -733,6 +790,12 @@ debug_persistent(struct cli *cli, const char * const * av, void *priv)
 	if (!strcmp(av[3], "sync")) {
 		if (sc->cur_seg != NULL)
 			smp_close_seg(sc, sc->cur_seg);
+		smp_sync_segs(sc);
+		while (sc->flags & SMP_SC_SYNC) {
+			Lck_Unlock(&sc->mtx);
+			VTIM_sleep(0.1);
+			Lck_Lock(&sc->mtx);
+		}
 		smp_new_seg(sc);
 	} else if (!strcmp(av[3], "dump")) {
 		debug_report_silo(cli, sc, 1);
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index c2cbac3..0c4e3a3 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -108,6 +108,7 @@ struct smp_sc {
 	unsigned		flags;
 #define SMP_SC_LOADED		(1 << 0)
 #define SMP_SC_STOP		(1 << 1)
+#define SMP_SC_SYNC		(1 << 2)
 
 	const struct stevedore	*stevedore;
 	int			fd;
@@ -194,7 +195,7 @@ void smp_load_seg(struct worker *, const struct smp_sc *sc, struct smp_seg *sg);
 void smp_new_seg(struct smp_sc *sc);
 void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg);
 void smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx);
-void smp_save_segs(struct smp_sc *sc);
+void smp_sync_segs(struct smp_sc *sc);
 
 /* storage_persistent_subr.c */
 
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 7ca79d6..fb384ee 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -48,59 +48,14 @@
 #include "storage/storage_persistent.h"
 
 /*--------------------------------------------------------------------
- * Write the segmentlist back to the silo.
- *
- * We write the first copy, sync it synchronously, then write the
- * second copy and sync it synchronously.
- *
- * Provided the kernel doesn't lie, that means we will always have
- * at least one valid copy on in the silo.
+ * Signal smp_thread() to sync the segment list to disk
  */
 
-static void
-smp_save_seg(const struct smp_sc *sc, struct smp_signspace *spc)
-{
-	struct smp_segptr *ss;
-	struct smp_seg *sg;
-	uint64_t length;
-
-	Lck_AssertHeld(&sc->mtx);
-	smp_reset_signspace(spc);
-	ss = SIGNSPACE_DATA(spc);
-	length = 0;
-	VTAILQ_FOREACH(sg, &sc->segments, list) {
-		assert(sg->p.offset < sc->mediasize);
-		assert(sg->p.offset + sg->p.length <= sc->mediasize);
-		*ss = sg->p;
-		ss++;
-		length += sizeof *ss;
-	}
-	smp_append_signspace(spc, length);
-	smp_sync_sign(&spc->ctx);
-}
-
 void
-smp_save_segs(struct smp_sc *sc)
+smp_sync_segs(struct smp_sc *sc)
 {
-	struct smp_seg *sg, *sg2;
-
 	Lck_AssertHeld(&sc->mtx);
-
-	/*
-	 * Remove empty segments from the front of the list
-	 * before we write the segments to disk.
-	 */
-	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
-		if (sg->nobj > 0)
-			break;
-		if (sg == sc->cur_seg)
-			continue;
-		VTAILQ_REMOVE(&sc->segments, sg, list);
-		LRU_Free(sg->lru);
-		FREE_OBJ(sg);
-	}
-	smp_save_seg(sc, &sc->seg1);
-	smp_save_seg(sc, &sc->seg2);
+	sc->flags |= SMP_SC_SYNC;
 }
 
 /*--------------------------------------------------------------------
@@ -177,6 +132,12 @@ smp_new_seg(struct smp_sc *sc)
 	AZ(sc->cur_seg);
 	Lck_AssertHeld(&sc->mtx);
 
+	if (sc->flags & SMP_SC_STOP) {
+		/* Housekeeping thread is stopping, don't allow new
+		 * segments as there is noone around to persist it */
+		return;
+	}
+
 	/* XXX: find where it goes in silo */
 
 	memset(&tmpsg, 0, sizeof tmpsg);
@@ -303,8 +264,8 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	smp_reset_sign(sg->ctx);
 	smp_sync_sign(sg->ctx);
 
-	/* Save segment list */
-	smp_save_segs(sc);
+	/* Request sync of segment list */
+	smp_sync_segs(sc);
 	sc->free_offset = smp_segend(sg);
 }
 
@@ -524,6 +485,11 @@ smp_oc_freeobj(struct objcore *oc)
 	sg->nobj--;
 	sg->nfixed--;
 
+	if (sg->nobj == 0 && sg == VTAILQ_FIRST(&sg->sc->segments)) {
+		/* Sync segments to remove empty at start */
+		sg->sc->flags |= SMP_SC_SYNC;
+	}
+
 	Lck_Unlock(&sg->sc->mtx);
 }
 
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:23 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:23 +0200
Subject: [PATCH 10/10] Avoid race condition between segment compacting and
 object lookup.
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-10-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c      |    8 ++++--
 bin/varnishd/storage/storage_persistent.h      |    1 +
 bin/varnishd/storage/storage_persistent_silo.c |   36 +++++++++++++++++++++---
 3 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 4306e4d..a41d52a 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -680,6 +680,8 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
 			(*so)->ptr = 0;;
 			sg->objs = *so;
 			*idx = ++sg->p.lobjlist;
+			/* Add ref early so segment will stick around */
+			sg->nobj++;
 		}
 		(void)smp_spaceleft(sc, sg);	/* for the assert */
 	}
@@ -740,20 +742,20 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	oc->flags |= OC_F_LRUDONTMOVE;
+	smp_init_oc(oc, sg, objidx);
 
 	Lck_Lock(&sc->mtx);
 	sg->nfixed++;
-	sg->nobj++;
+	assert(sg->nobj > 0);	/* Our ref added by smp_allocx() */
 
 	/* We have to do this somewhere, might as well be here... */
+	so = smp_find_so(sg, objidx); /* Might have changed during unlock */
 	assert(sizeof so->hash == DIGEST_LEN);
 	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
 	so->ttl = EXP_Grace(NULL, o);
 	so->ptr = (uint8_t*)o - sc->base;
 	so->ban = BAN_Time(oc->ban);
 
-	smp_init_oc(oc, sg, objidx);
-
 	Lck_Unlock(&sc->mtx);
 	return (o);
 }
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index 9e0642d..0496a95 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -204,6 +204,7 @@ void smp_load_seg(struct worker *, const struct smp_sc *sc, struct smp_seg *sg);
 void smp_new_seg(struct smp_sc *sc);
 void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg);
 void smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx);
+struct smp_object * smp_find_so(const struct smp_seg *sg, unsigned priv2);
 void smp_sync_segs(struct smp_sc *sc);
 
 /* storage_persistent_subr.c */
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index e432302..07b6c9a 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -43,6 +43,7 @@
 #include "hash/hash_slinger.h"
 #include "vsha256.h"
 #include "vtim.h"
+#include "vmb.h"
 
 #include "persistent.h"
 #include "storage/storage_persistent.h"
@@ -267,6 +268,7 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	sg->flags |= SMP_SEG_SYNCSIGNS;
 
 	/* Remove the new flag and request sync of segment list */
+	VMB();			/* See comments in smp_oc_getobj() */
 	sg->flags &= ~SMP_SEG_NEW;
 	smp_sync_segs(sc);
 }
@@ -298,9 +300,11 @@ smp_check_reserve(struct smp_sc *sc)
 }
 
 /*---------------------------------------------------------------------
+ * Find the struct smp_object in the segment's object list by
+ * it's objindex (oc->priv2)
  */
 
-static struct smp_object *
+struct smp_object *
 smp_find_so(const struct smp_seg *sg, unsigned priv2)
 {
 	struct smp_object *so;
@@ -401,16 +405,33 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 	struct storage *st;
 	uint64_t l;
 	int bad;
+	int has_lock;
 
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	/* Some calls are direct, but they should match anyway */
 	assert(oc->methods->getobj == smp_oc_getobj);
 
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	if (ds == NULL)
 		AZ(oc->flags & OC_F_NEEDFIXUP);
 
 	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
+	if (sg->flags & SMP_SEG_NEW) {
+		/* Segment is new and can be closed and compacted at
+		 * any time. We need to keep a lock during access to
+		 * the objlist. */
+		Lck_Lock(&sg->sc->mtx);
+		has_lock = 1;
+	} else {
+		/* Since the NEW flag is removed after the compacting
+		 * and a memory barrier, any compacting should have
+		 * been done with the changes visible to us if we
+		 * can't see the flag. Should be safe to proceed
+		 * without locks. */
+		has_lock = 0;
+	}
 	so = smp_find_so(sg, oc->priv2);
+	AN(so);
+	AN(so->ptr);
 
 	o = (void*)(sg->sc->base + so->ptr);
 	/*
@@ -426,11 +447,17 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 	 * If this flag is not set, it will not be, and the lock is not
 	 * needed to test it.
 	 */
-	if (!(oc->flags & OC_F_NEEDFIXUP))
+	if (!(oc->flags & OC_F_NEEDFIXUP)) {
+		if (has_lock)
+			Lck_Unlock(&sg->sc->mtx);
 		return (o);
+	}
 
 	AN(ds);
-	Lck_Lock(&sg->sc->mtx);
+	if (!has_lock) {
+		Lck_Lock(&sg->sc->mtx);
+		has_lock = 1;
+	}
 	/* Check again, we might have raced. */
 	if (oc->flags & OC_F_NEEDFIXUP) {
 		/* We trust caller to have a refcnt for us */
@@ -457,6 +484,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 		ds->n_vampireobject--;
 		oc->flags &= ~OC_F_NEEDFIXUP;
 	}
+	AN(has_lock);
 	Lck_Unlock(&sg->sc->mtx);
 	EXP_Rearm(o);
 	return (o);
-- 
1.7.9.5




From martin at varnish-software.com  Wed Oct 10 14:27:22 2012
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Wed, 10 Oct 2012 16:27:22 +0200
Subject: [PATCH 09/10] Force-expire segments on low storage in persistent
In-Reply-To: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
Message-ID: <1349879243-30369-9-git-send-email-martin@varnish-software.com>

---
 bin/varnishd/storage/storage_persistent.c      |   47 +++++++++++++++++++++++-
 bin/varnishd/storage/storage_persistent.h      |    5 +++
 bin/varnishd/storage/storage_persistent_mgt.c  |    4 +-
 bin/varnishd/storage/storage_persistent_silo.c |   27 ++++++++++++++
 4 files changed, 79 insertions(+), 4 deletions(-)

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 00bd940..4306e4d 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -371,6 +371,10 @@ smp_save_segs(struct smp_sc *sc)
 		if (sg->flags & SMP_SEG_NEW)
 			break;
 		VTAILQ_REMOVE(&sc->segments, sg, list);
+		if (sg->flags & SMP_SEG_NUKED) {
+			assert(sc->free_pending >= sg->p.length);
+			sc->free_pending -= sg->p.length;
+		}
 		LRU_Free(sg->lru);
 		FREE_OBJ(sg);
 	}
@@ -406,6 +410,10 @@ smp_save_segs(struct smp_sc *sc)
 	VTAILQ_FOREACH(sg, &sc->segments, list) {
 		assert(sg->p.offset < sc->mediasize);
 		assert(sg->p.offset + sg->p.length <= sc->mediasize);
+		if (sg->flags & SMP_SEG_NUKED) {
+			AZ(length);
+			continue;
+		}
 		if (sg->flags & SMP_SEG_NEW)
 			break;
 		*ss = sg->p;
@@ -421,6 +429,39 @@ smp_save_segs(struct smp_sc *sc)
 	smp_sync_sign(&sc->seg2.ctx);
 	Lck_Lock(&sc->mtx);
 }
+
+/*
+ * Raise the free_reserve by nuking segments
+ */
+
+static void
+smp_raise_reserve(struct worker *wrk, struct vsl_log *vsl, struct smp_sc *sc)
+{
+	struct smp_seg *sg;
+
+	Lck_AssertHeld(&sc->mtx);
+
+	VTAILQ_FOREACH(sg, &sc->segments, list) {
+		if (sg == sc->cur_seg)
+			break;
+		if (smp_silospaceleft(sc) + sc->free_pending > sc->free_reserve)
+			break;
+		if (sg->flags & SMP_SEG_NEW)
+			break;
+		if (sg->flags & SMP_SEG_NUKED)
+			continue;
+
+		/* Nuke this segment */
+		Lck_Unlock(&sc->mtx);
+		EXP_NukeLRU(wrk, vsl, sg->lru);
+		Lck_Lock(&sc->mtx);
+		sg->flags |= SMP_SEG_NUKED;
+		sc->free_pending += sg->p.length;
+	}
+	assert(smp_silospaceleft(sc) + sc->free_pending > sc->free_reserve);
+	sc->flags &= ~SMP_SC_LOW;
+}
+
 /*--------------------------------------------------------------------
  * Silo worker thread
  */
@@ -430,10 +471,12 @@ smp_thread(struct worker *wrk, void *priv)
 {
 	struct smp_sc	*sc;
 	struct smp_seg *sg;
+	struct vsl_log vsl;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC);
 	sc->thread = pthread_self();
+	VSL_Setup(&vsl, NULL, 0);
 
 	/* First, load all the objects from all segments */
 	VTAILQ_FOREACH(sg, &sc->segments, list)
@@ -448,10 +491,12 @@ smp_thread(struct worker *wrk, void *priv)
 	/* Housekeeping loop */
 	Lck_Lock(&sc->mtx);
 	while (!(sc->flags & SMP_SC_STOP)) {
+		if (sc->flags & SMP_SC_LOW)
+			smp_raise_reserve(wrk, &vsl, sc);
 		if (sc->flags & SMP_SC_SYNC)
 			smp_save_segs(sc);
 
-		if (!(sc->flags & (SMP_SC_SYNC | SMP_SC_STOP)))
+		if (!(sc->flags & (SMP_SC_LOW | SMP_SC_SYNC | SMP_SC_STOP)))
 			/* Wait for something to do */
 			(void)Lck_CondWait(&sc->cond, &sc->mtx, NULL);
 	}
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index 93ec45e..9e0642d 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -89,6 +89,7 @@ struct smp_seg {
 #define SMP_SEG_LOADED		(1 << 1)
 #define SMP_SEG_NEW		(1 << 2)
 #define SMP_SEG_SYNCSIGNS	(1 << 3)
+#define SMP_SEG_NUKED		(1 << 4)
 
 	uint32_t		nobj;		/* Number of objects */
 	uint32_t		nalloc;		/* Allocations */
@@ -114,6 +115,7 @@ struct smp_sc {
 #define SMP_SC_LOADED		(1 << 0)
 #define SMP_SC_STOP		(1 << 1)
 #define SMP_SC_SYNC		(1 << 2)
+#define SMP_SC_LOW		(1 << 3)
 
 	const struct stevedore	*stevedore;
 	int			fd;
@@ -133,6 +135,7 @@ struct smp_sc {
 	uint64_t		next_top;	/* next alloc address top */
 
 	uint64_t		free_offset;
+	uint64_t		free_pending;
 
 	pthread_t		thread;
 
@@ -224,6 +227,8 @@ void smp_msync(void *addr, size_t length);
 
 void smp_newsilo(struct smp_sc *sc);
 int smp_valid_silo(struct smp_sc *sc);
+uint64_t smp_silospaceleft(struct smp_sc *sc);
+void smp_check_reserve(struct smp_sc *sc);
 
 /*--------------------------------------------------------------------
  * Caculate payload of some stuff
diff --git a/bin/varnishd/storage/storage_persistent_mgt.c b/bin/varnishd/storage/storage_persistent_mgt.c
index c5b1dfd..26c9d36 100644
--- a/bin/varnishd/storage/storage_persistent_mgt.c
+++ b/bin/varnishd/storage/storage_persistent_mgt.c
@@ -114,9 +114,7 @@ smp_metrics(struct smp_sc *sc)
 	fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
 	    sc->aim_nseg, (uintmax_t)sc->aim_segl);
 
-	/*
-	 * How much space in the free reserve pool ?
-	 */
+	/* XXX: Random number larger than 1 */
 	sc->free_reserve = sc->aim_segl * 10;
 
 	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index addb836..e432302 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -184,6 +184,7 @@ smp_new_seg(struct smp_sc *sc)
 	sc->free_offset = sg->p.offset + sg->p.length;
 
 	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
+	smp_check_reserve(sc);
 
 	/* Set up our allocation points */
 	sc->cur_seg = sg;
@@ -270,6 +271,32 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	smp_sync_segs(sc);
 }
 
+uint64_t
+smp_silospaceleft(struct smp_sc *sc)
+{
+	struct smp_seg *sg;
+
+	Lck_AssertHeld(&sc->mtx);
+
+	sg = VTAILQ_FIRST(&sc->segments);
+	if (sg == NULL)
+		return (sc->mediasize - sc->free_offset);
+	if (sg->p.offset < sc->free_offset) {
+		return ((sc->mediasize - sc->free_offset) +
+			(sg->p.offset - sc->ident->stuff[SMP_SPC_STUFF]));
+	}
+	return (sg->p.offset - sc->free_offset);
+}
+
+void
+smp_check_reserve(struct smp_sc *sc)
+{
+	Lck_AssertHeld(&sc->mtx);
+
+	if (smp_silospaceleft(sc) + sc->free_pending < sc->free_reserve)
+		sc->flags |= SMP_SC_LOW;
+}
+
 /*---------------------------------------------------------------------
  */
 
-- 
1.7.9.5




From fgsch at lodoss.net  Thu Oct 11 02:59:12 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 11 Oct 2012 03:59:12 +0100
Subject: [PATCH] Implement std.rfcdate
Message-ID: <20121011035912.cc6531acb4464292dde9d871@lodoss.net>

Hi,

The diff below does:

1. Implements std.rfcdate(), documentation and (brief) test included
2. Correctly treats TIME + DURATION as a TIME as per https://www.varnish-cache.org/docs/trunk/phk/vcl_expr.html
3. Changes now to return a TIME - the old form is available via std.rfcdate(now)

In particular my main motivation behind #3 is something scn mentioned on IRC:

> I had a 15min digression with (now + 0s) and all sorts of tricks to find unixtime in vcl a few weeks back

which I believe it's a bit inconsistent.

The current behaviour:

- `now' by itself will return a date in rfc 1123 format
- `now + duration' will return a time
- there is no way to convert a time other than now to a rfc 1123 format if it's not via inline C

With this diff:

- `now' and `now + duration' returns a time
- `std.rfcdate(now)' and `std.rfcdate(now + duration)' returns the date in rfc 1123 format.

Other possibilities could be:

- Return a date for `now' and `now + duration' and add a way to convert both cases to a time
- Return a date for `now' and `now + duration' and add a new variable to replace now that returns a time

Comments?

f.-

 bin/varnishtest/tests/m00009.vtc       |   20 ++++++++++++++++++++
 doc/sphinx/reference/vmod_std.rst      |   12 ++++++++++++
 lib/libvcl/vcc_expr.c                  |    6 ++++--
 lib/libvmod_std/vmod.vcc               |    1 +
 lib/libvmod_std/vmod_std_conversions.c |    6 ++++++
 5 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/bin/varnishtest/tests/m00009.vtc b/bin/varnishtest/tests/m00009.vtc
new file mode 100644
index 0000000..d0b12d0
--- /dev/null
+++ b/bin/varnishtest/tests/m00009.vtc
@@ -0,0 +1,20 @@
+varnishtest "test vmod_std.rfcdate conversion"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+	sub vcl_deliver {
+		set resp.http.now = std.rfcdate(now);
+		set resp.http.future = std.rfcdate(now + 30m);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index 984331e..814446d 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -138,6 +138,18 @@ Description
 Example
 	if (std.integer(beresp.http.x-foo, 0) > 5) { ... }
 
+rfcdate
+--------
+Prototype
+	rfcdate(TIME t)
+Return value
+	String
+Description
+	Converts the TIME *t* to a string representing the date in
+	RFC 1123 format.
+Example
+	set resp.http.Expire = std.rfcdate(now + 30m);
+
 collect
 -------
 Prototype
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 72c26a9..2d9ebae 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -431,7 +431,7 @@ vcc_expr_tostring(struct expr **e, enum var_type fmt)
 	case IP:	p = "VRT_IP_string(req, \v1)"; break;
 	case BYTES:	p = "VRT_REAL_string(req, \v1)"; break; /* XXX */
 	case REAL:	p = "VRT_REAL_string(req, \v1)"; break;
-	case TIME:	p = "VRT_TIME_string(req, \v1)"; break;
+	case TIME:	p = "VRT_REAL_string(req, \v1)"; break;
 	default:	break;
 	}
 	if (p != NULL) {
@@ -856,7 +856,9 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type fmt)
 			vcc_ErrWhere2(tl, tk, tl->t);
 			return;
 		}
-		if (tk->tok == '+')
+		if (tk->tok == '+' && (*e)->fmt == TIME)
+			*e = vcc_expr_edit(TIME, "(\v1+\v2)", *e, e2);
+		else if (tk->tok == '+')
 			*e = vcc_expr_edit(f2, "(\v1+\v2)", *e, e2);
 		else if (f2 == TIME && e2->fmt == TIME)
 			*e = vcc_expr_edit(DURATION, "(\v1-\v2)", *e, e2);
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 3238164..c2a018a 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -36,3 +36,4 @@ Function STRING fileread(PRIV_CALL, STRING)
 Function DURATION duration(STRING, DURATION)
 Function INT integer(STRING, INT)
 Function VOID collect(HEADER)
+Function STRING rfcdate(TIME)
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index e728611..956f80e 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -117,3 +117,9 @@ vmod_integer(struct req *req, const char *p, long i)
 
 	return (r);
 }
+
+const char * __match_proto__(td_std_rfcdate)
+vmod_rfcdate(struct req *req, double t)
+{
+	return (VRT_TIME_string(req, t));
+}



From phk at phk.freebsd.dk  Thu Oct 11 08:24:48 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Thu, 11 Oct 2012 08:24:48 +0000
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: Your message of "Thu, 11 Oct 2012 03:59:12 +0100."
 <20121011035912.cc6531acb4464292dde9d871@lodoss.net>
Message-ID: <76517.1349943888@critter.freebsd.dk>

In message <20121011035912.cc6531acb4464292dde9d871 at lodoss.net>, "Federico G. S
chwindt" writes:

I'm not sure I understand what you are trying to do here, and I wonder
if you're not doing it backwards ?

Wouldn't a 
	DOUBLE std.unixtime(TIME)
make more sense ?

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From fgsch at lodoss.net  Thu Oct 11 08:54:02 2012
From: fgsch at lodoss.net (Federico Schwindt)
Date: Thu, 11 Oct 2012 09:54:02 +0100
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: <76517.1349943888@critter.freebsd.dk>
References: <20121011035912.cc6531acb4464292dde9d871@lodoss.net>
 <76517.1349943888@critter.freebsd.dk>
Message-ID: <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>

On Thu, Oct 11, 2012 at 9:24 AM, Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
> In message <20121011035912.cc6531acb4464292dde9d871 at lodoss.net>, "Federico G. S
> chwindt" writes:
>
> I'm not sure I understand what you are trying to do here, and I wonder
> if you're not doing it backwards ?
>
> Wouldn't a
>         DOUBLE std.unixtime(TIME)
> make more sense ?

That's why I wrote at the bottom:

- Return a date for `now' and `now + duration' and add a way to
convert both cases to a time

The only reason I went for this approach is that as per # 2:

2. Correctly treats TIME + DURATION as a TIME as per
https://www.varnishcache.org/docs/trunk/phk/vcl_expr.html

f.-



From phk at phk.freebsd.dk  Thu Oct 11 08:56:29 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Thu, 11 Oct 2012 08:56:29 +0000
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: Your message of "Thu, 11 Oct 2012 09:54:02 +0100."
 <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>
Message-ID: <76753.1349945789@critter.freebsd.dk>

In message <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug at mail.gmail.com>
, Federico Schwindt writes:
>On Thu, Oct 11, 2012 at 9:24 AM, Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
>> In message <20121011035912.cc6531acb4464292dde9d871 at lodoss.net>, "Federico G. S
>> chwindt" writes:
>>
>> I'm not sure I understand what you are trying to do here [...]

This is probably my biggest hurdle right now:  What is the problem
we are trying to solve ?


-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From fgsch at lodoss.net  Thu Oct 11 09:03:32 2012
From: fgsch at lodoss.net (Federico Schwindt)
Date: Thu, 11 Oct 2012 10:03:32 +0100
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: <76753.1349945789@critter.freebsd.dk>
References: <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>
 <76753.1349945789@critter.freebsd.dk>
Message-ID: <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg@mail.gmail.com>

On Thu, Oct 11, 2012 at 9:56 AM, Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
> In message <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug at mail.gmail.com>
> , Federico Schwindt writes:
>>On Thu, Oct 11, 2012 at 9:24 AM, Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
>>> In message <20121011035912.cc6531acb4464292dde9d871 at lodoss.net>, "Federico G. S
>>> chwindt" writes:
>>>
>>> I'm not sure I understand what you are trying to do here [...]
>
> This is probably my biggest hurdle right now:  What is the problem
> we are trying to solve ?

1. There is no way to get a date for `now + duration'.
This is something useful and often needed. Currently the only way is
via inline-C.
See https://www.varnish-cache.org/trac/wiki/VCLExampleSetExpires

2. now and now + duration behave differently (string vs duration)
This is for consistency.

3. now + duration returns a duration, not a time as supposed to be.
This is a bug.

f.-



From phk at critter.freebsd.dk  Thu Oct 11 09:05:03 2012
From: phk at critter.freebsd.dk (Poul-Henning Kamp)
Date: Thu, 11 Oct 2012 09:05:03 +0000
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg@mail.gmail.com>
References: <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>
 <76753.1349945789@critter.freebsd.dk>
 <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg@mail.gmail.com>
Message-ID: <76841.1349946303@critter.freebsd.dk>

In message <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg at mail.gmail.com>
, Federico Schwindt writes:

>2. now and now + duration behave differently (string vs duration)
>This is for consistency.

Then this is the problem we should fix.

Probably what we should do is move now() to vmod_std so it gets 
a proper type.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From fgsch at lodoss.net  Thu Oct 11 10:18:43 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 11 Oct 2012 11:18:43 +0100
Subject: [PATCH] TIME + DURATION -> TIME (was Re: [PATCH] Implement
 std.rfcdate)
In-Reply-To: <76841.1349946303@critter.freebsd.dk>
References: <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>
 <76753.1349945789@critter.freebsd.dk>
 <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg@mail.gmail.com>
 <76841.1349946303@critter.freebsd.dk>
Message-ID: <20121011111843.99ae1ed3d271018f7f33d975@lodoss.net>

On Thu, 11 Oct 2012 09:05:03 +0000
Poul-Henning Kamp <phk at critter.freebsd.dk> wrote:

> In message <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg at mail.gmail.com>
> , Federico Schwindt writes:
> 
> >2. now and now + duration behave differently (string vs duration)
> >This is for consistency.
> 
> Then this is the problem we should fix.
> 
> Probably what we should do is move now() to vmod_std so it gets 
> a proper type.

Ok, see below.

f.-

 bin/varnishtest/tests/m00009.vtc       |   22 ++++++++++++++++++++++
 doc/sphinx/reference/vmod_std.rst      |   11 +++++++++++
 lib/libvcl/vcc_expr.c                  |    4 +++-
 lib/libvmod_std/vmod.vcc               |    1 +
 lib/libvmod_std/vmod_std_conversions.c |    6 ++++++
 5 files changed, 43 insertions(+), 1 deletions(-)

diff --git a/bin/varnishtest/tests/m00009.vtc b/bin/varnishtest/tests/m00009.vtc
new file mode 100644
index 0000000..c1f4716
--- /dev/null
+++ b/bin/varnishtest/tests/m00009.vtc
@@ -0,0 +1,22 @@
+varnishtest "test vmod_std.unixtime conversion"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+	sub vcl_deliver {
+		set resp.http.now = now;
+		set resp.http.now_unixtime = std.unixtime(now);
+		set resp.http.future = now + 30m;
+		set resp.http.future_unixtime = std.unixtime(now + 30m);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index 984331e..4215a79 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -138,6 +138,17 @@ Description
 Example
 	if (std.integer(beresp.http.x-foo, 0) > 5) { ... }
 
+unixtime
+--------
+Prototype
+	unixtime(TIME t)
+Return value
+	Real
+Description
+	Converts TIME *t* to a REAL number.
+Example
+	set beresp.http.x-unixtime = std.unixtime(now);
+
 collect
 -------
 Prototype
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 72c26a9..d79608c 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -856,7 +856,9 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type fmt)
 			vcc_ErrWhere2(tl, tk, tl->t);
 			return;
 		}
-		if (tk->tok == '+')
+		if (tk->tok == '+' && (*e)->fmt == TIME)
+			*e = vcc_expr_edit(TIME, "(\v1+\v2)", *e, e2);
+		else if (tk->tok == '+')
 			*e = vcc_expr_edit(f2, "(\v1+\v2)", *e, e2);
 		else if (f2 == TIME && e2->fmt == TIME)
 			*e = vcc_expr_edit(DURATION, "(\v1-\v2)", *e, e2);
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 3238164..4cc9e5a 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -36,3 +36,4 @@ Function STRING fileread(PRIV_CALL, STRING)
 Function DURATION duration(STRING, DURATION)
 Function INT integer(STRING, INT)
 Function VOID collect(HEADER)
+Function REAL unixtime(TIME)
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index e728611..54cc7d8 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -117,3 +117,9 @@ vmod_integer(struct req *req, const char *p, long i)
 
 	return (r);
 }
+
+double __match_proto__(td_std_unixtime)
+vmod_unixtime(struct req *req, double t)
+{
+	return (t);
+}



From phk at phk.freebsd.dk  Thu Oct 11 10:19:06 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Thu, 11 Oct 2012 10:19:06 +0000
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: <76841.1349946303@critter.freebsd.dk>
References: <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>
 <76753.1349945789@critter.freebsd.dk>
 <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg@mail.gmail.com>
 <76841.1349946303@critter.freebsd.dk>
Message-ID: <78074.1349950746@critter.freebsd.dk>

--------
In message <76841.1349946303 at critter.freebsd.dk>, Poul-Henning Kamp writes:


Can you try this patch ?

(NB: copy&pasted, you probably need to apply by hand...)

(PS: and place open a ticket for this bug)

diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 72c26a9..5223392 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -847,6 +847,10 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type
                ERRCHK(tl);
                if (tk->tok == '-' && (*e)->fmt == TIME && e2->fmt == TIME) {
                        /* OK */
+               } else if (tk->tok == '+' &&
+                   (*e)->fmt == TIME && e2->fmt == DURATION) {
+                       f2 = TIME;
+                       /* OK */
                } else if (tk->tok == '-' &&
                    (*e)->fmt == BYTES && e2->fmt == BYTES) {
                        /* OK */


-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From fgsch at lodoss.net  Thu Oct 11 10:35:15 2012
From: fgsch at lodoss.net (Federico Schwindt)
Date: Thu, 11 Oct 2012 11:35:15 +0100
Subject: [PATCH] Implement std.rfcdate
In-Reply-To: <78074.1349950746@critter.freebsd.dk>
References: <CAJV_h0aLpdoSSRvPtQorCdqxTG0nh_hCVSeygio3jQb1wuGnug@mail.gmail.com>
 <76753.1349945789@critter.freebsd.dk>
 <CAJV_h0Y6rOZ79cGdtYY7mOgk8Y9R0k0R5c5vGvnY4nPfhhB+qg@mail.gmail.com>
 <76841.1349946303@critter.freebsd.dk>
 <78074.1349950746@critter.freebsd.dk>
Message-ID: <CAJV_h0bOb6c1aBYbADPGGMvBVy=sJRTgyvAwdfKQf+K9oMFcUQ@mail.gmail.com>

On Thu, Oct 11, 2012 at 11:19 AM, Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
> --------
> In message <76841.1349946303 at critter.freebsd.dk>, Poul-Henning Kamp writes:
>
>
> Can you try this patch ?
>
> (NB: copy&pasted, you probably need to apply by hand...)
>
> (PS: and place open a ticket for this bug)
>
> diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
> index 72c26a9..5223392 100644
> --- a/lib/libvcl/vcc_expr.c
> +++ b/lib/libvcl/vcc_expr.c
> @@ -847,6 +847,10 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type
>                 ERRCHK(tl);
>                 if (tk->tok == '-' && (*e)->fmt == TIME && e2->fmt == TIME) {
>                         /* OK */
> +               } else if (tk->tok == '+' &&
> +                   (*e)->fmt == TIME && e2->fmt == DURATION) {
> +                       f2 = TIME;
> +                       /* OK */
>                 } else if (tk->tok == '-' &&
>                     (*e)->fmt == BYTES && e2->fmt == BYTES) {
>                         /* OK */

Yes, that works.

f.-



From tfheen at varnish-software.com  Mon Oct 15 11:49:44 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 15 Oct 2012 13:49:44 +0200
Subject: [PATCH 1/2] Expose exp->entered via macros
Message-ID: <1350301785-6835-1-git-send-email-tfheen@varnish-software.com>

---
 bin/varnishd/cache/cache.h        |    2 ++
 bin/varnishd/cache/cache_expire.c |    1 +
 2 files changed, 3 insertions(+)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6d4fe86..1c6a833 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -782,9 +782,11 @@ void EXP_Clr(struct exp *e);
 double EXP_Get_ttl(const struct exp *e);
 double EXP_Get_grace(const struct exp *e);
 double EXP_Get_keep(const struct exp *e);
+double EXP_Get_entered(const struct exp *e);
 void EXP_Set_ttl(struct exp *e, double v);
 void EXP_Set_grace(struct exp *e, double v);
 void EXP_Set_keep(struct exp *e, double v);
+void EXP_Set_entered(struct exp *e, double v);
 
 double EXP_Ttl(const struct req *, const struct object*);
 double EXP_Grace(const struct req *, const struct object*);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 80c630f..bc3cb0b 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -102,6 +102,7 @@ EXP_Clr(struct exp *e)
 EXP_ACCESS(ttl, -1., (e->grace = e->keep = -1.))
 EXP_ACCESS(grace, 0., )
 EXP_ACCESS(keep, 0.,)
+EXP_ACCESS(entered, 0.,)
 
 /*--------------------------------------------------------------------
  * Calculate an objects effective keep, grace or ttl time, suitably
-- 
1.7.10.4




From tfheen at varnish-software.com  Mon Oct 15 11:49:45 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 15 Oct 2012 13:49:45 +0200
Subject: [PATCH 2/2] Add softban support
In-Reply-To: <1350301785-6835-1-git-send-email-tfheen@varnish-software.com>
References: <1350301785-6835-1-git-send-email-tfheen@varnish-software.com>
Message-ID: <1350301785-6835-2-git-send-email-tfheen@varnish-software.com>

Soft bans are like normal bans, but they are subject to grace per
usual rules.
---
 bin/varnishd/cache/cache.h       |    1 +
 bin/varnishd/cache/cache_ban.c   |   56 ++++++++++++++++++++++++++++++++----
 bin/varnishd/cache/cache_vrt.c   |    6 ++--
 bin/varnishtest/tests/s00004.vtc |   47 ++++++++++++++++++++++++++++++
 bin/varnishtest/tests/s00005.vtc |   59 ++++++++++++++++++++++++++++++++++++++
 include/vcli.h                   |    7 +++++
 include/vrt.h                    |    4 +--
 lib/libvcl/vcc_action.c          |   25 ++++++++++++++--
 8 files changed, 193 insertions(+), 12 deletions(-)
 create mode 100644 bin/varnishtest/tests/s00004.vtc
 create mode 100644 bin/varnishtest/tests/s00005.vtc

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1c6a833..01d5d07 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -757,6 +757,7 @@ void BAN_Compile(void);
 struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail);
 void BAN_TailDeref(struct ban **ban);
 double BAN_Time(const struct ban *ban);
+void BAN_Set_Soft(struct ban *b, int soft);
 
 /* cache_busyobj.c */
 void VBO_Init(void);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index aa8bca9..9b34f0c 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -42,6 +42,7 @@
  *	8 bytes	- double: timestamp		XXX: Byteorder ?
  *	4 bytes - be32: length
  *	1 byte - flags: 0x01: BAN_F_REQ
+ *			0x02: BAN_F_SOFT
  *	N tests
  * A test have this form:
  *	1 byte - arg (see ban_vars.h col 3 "BAN_ARG_XXX")
@@ -82,6 +83,7 @@ struct ban {
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
+#define BAN_F_SOFT		(1 << 3)	/* soft ban, aka grace */
 #define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
@@ -392,7 +394,11 @@ BAN_Insert(struct ban *b)
 
 	t0 = VTIM_real();
 	memcpy(b->spec, &t0, sizeof t0);
-	b->spec[12] = (b->flags & BAN_F_REQ) ? 1 : 0;
+	b->spec[12] = 0;
+	if (b->flags & BAN_F_REQ)
+		b->spec[12] |= 0x1;
+	if (b->flags & BAN_F_SOFT)
+		b->spec[12] |= 0x2;
 	memcpy(b->spec + 13, VSB_data(b->vsb), ln);
 	ln += 13;
 	vbe32enc(b->spec + 8, ln);
@@ -551,8 +557,10 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 	AN(b2->spec);
 	memcpy(b2->spec, ban, len);
 	b2->flags |= gone;
-	if (ban[12])
+	if (ban[12] & 0x01)
 		b2->flags |= BAN_F_REQ;
+	if (ban[12] & 0x02)
+		b2->flags |= BAN_F_SOFT;
 	if (b == NULL)
 		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
 	else
@@ -587,6 +595,21 @@ BAN_Time(const struct ban *b)
 }
 
 /*--------------------------------------------------------------------
+ * Set/unset the soft flag on a ban
+ */
+
+void
+BAN_Set_Soft(struct ban *b, int soft)
+{
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	if (soft)
+		b->flags |= BAN_F_SOFT;
+	else
+		b->flags &= ~BAN_F_SOFT;
+}
+
+/*--------------------------------------------------------------------
  * All silos have read their bans, ready for action
  */
 
@@ -687,7 +710,7 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	struct ban *b;
 	struct objcore *oc;
 	struct ban * volatile b0;
-	unsigned tests, skipped;
+	unsigned tests, skipped, soft;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	CHECK_OBJ_ORNULL(req_http, HTTP_MAGIC);
@@ -708,6 +731,7 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	 */
 	tests = 0;
 	skipped = 0;
+	soft = 0;
 	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 		if (b->flags & BAN_F_GONE)
@@ -724,15 +748,21 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 			 * be other bans that match, so we soldier on
 			 */
 			skipped++;
-		} else if (ban_evaluate(b->spec, o->http, req_http, &tests))
+		} else if (ban_evaluate(b->spec, o->http, req_http, &tests)) {
+			if (b->flags & BAN_F_SOFT) {
+				soft = 1;
+				continue;
+			}
+			soft = 0; /* Found hard ban */
 			break;
+		}
 	}
 
 	Lck_Lock(&ban_mtx);
 	VSC_C_main->bans_tested++;
 	VSC_C_main->bans_tests_tested += tests;
 
-	if (b == oc->ban && skipped > 0) {
+	if (b == oc->ban && skipped > 0 && soft == 0) {
 		AZ(req_http);
 		Lck_Unlock(&ban_mtx);
 		/*
@@ -751,10 +781,20 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	}
 	Lck_Unlock(&ban_mtx);
 
-	if (b == oc->ban) {	/* not banned */
+	if (soft == 0 && b == oc->ban) {	/* not banned */
 		oc->ban = b0;
 		oc_updatemeta(oc);
 		return (0);
+	} else if (soft) {
+		/* Softban, set ttl to now */
+		EXP_Set_ttl(&o->exp, VTIM_real() -
+			    EXP_Get_entered(&o->exp));
+		oc->ban = b0;
+		oc_updatemeta(oc);
+		/* XXX: no req in lurker */
+		VSLb(vsl, SLT_ExpBan, "%u was softbanned", o->vxid);
+		EXP_Rearm(o);
+		return (2);
 	} else {
 		EXP_Clr(&o->exp);
 		oc->ban = NULL;
@@ -1026,6 +1066,9 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 		VCLI_SetResult(cli, CLIS_CANT);
 		return;
 	}
+	if (strcmp(av[1], "softban") == 0) {
+		b->flags |= BAN_F_SOFT;
+	}
 	for (i = 0; i < narg; i += 4)
 		if (BAN_AddTest(cli, b, av[i + 2], av[i + 3], av[i + 4])) {
 			BAN_Free(b);
@@ -1125,6 +1168,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 static struct cli_proto ban_cmds[] = {
 	{ CLI_BAN_URL,				"", ccf_ban_url },
 	{ CLI_BAN,				"", ccf_ban },
+	{ CLI_SOFTBAN,				"", ccf_ban },
 	{ CLI_BAN_LIST,				"", ccf_ban_list },
 	{ NULL }
 };
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index a2dd8cd..7773869 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -415,7 +415,7 @@ VRT_synth_page(const struct req *req, unsigned flags, const char *str, ...)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_ban(const struct req *req, char *cmds, ...)
+VRT_ban(const struct req *req, int soft, char *cmds, ...)
 {
 	char *a1, *a2, *a3;
 	va_list ap;
@@ -424,6 +424,7 @@ VRT_ban(const struct req *req, char *cmds, ...)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	b = BAN_New();
+	BAN_Set_Soft(b, soft);
 	va_start(ap, cmds);
 	a1 = cmds;
 	good = 0;
@@ -450,7 +451,7 @@ VRT_ban(const struct req *req, char *cmds, ...)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_ban_string(const struct req *req, const char *str)
+VRT_ban_string(const struct req *req, int soft, const char *str)
 {
 	char *a1, *a2, *a3;
 	char **av;
@@ -466,6 +467,7 @@ VRT_ban_string(const struct req *req, const char *str)
 		return;
 	}
 	b = BAN_New();
+	BAN_Set_Soft(b, soft);
 	good = 0;
 	for (i = 1; ;) {
 		a1 = av[i++];
diff --git a/bin/varnishtest/tests/s00004.vtc b/bin/varnishtest/tests/s00004.vtc
new file mode 100644
index 0000000..781d1dc
--- /dev/null
+++ b/bin/varnishtest/tests/s00004.vtc
@@ -0,0 +1,47 @@
+varnishtest "Softbans from CLI"
+
+server s1 {
+	rxreq
+	txresp
+
+	sema r1 sync 3
+
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_hit {
+	    if (obj.ttl <= 0s) {
+	    	    set req.http.grace = "t";
+	    }
+	}
+	sub vcl_fetch { 
+		set beresp.ttl = 1m;
+		set beresp.grace = 1m; 
+	}
+	sub vcl_deliver {
+	    set resp.http.grace = req.http.grace;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	delay 1
+	txreq
+	sema r1 sync 3
+	rxresp
+} -start
+
+varnish v1 -cliok "softban req.url ~ ^"
+delay 1;
+
+client c2 {
+	sema r1 sync 3
+	txreq
+	rxresp
+	expect resp.http.grace == "t"
+	expect resp.status == 200
+} -run
diff --git a/bin/varnishtest/tests/s00005.vtc b/bin/varnishtest/tests/s00005.vtc
new file mode 100644
index 0000000..35d1a8a
--- /dev/null
+++ b/bin/varnishtest/tests/s00005.vtc
@@ -0,0 +1,59 @@
+varnishtest "Softbans from VCL"
+
+server s1 {
+	rxreq
+	txresp
+
+	sema r1 sync 3
+
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+	    if (req.request == "SOFTBAN") {
+	       softban("req.url ~ " + req.url);
+	       error 410;
+	    }
+	}
+
+	sub vcl_hit {
+	    if (obj.ttl <= 0s) {
+	    	    set req.http.grace = "t";
+	    }
+	}
+	sub vcl_fetch { 
+		set beresp.ttl = 1m;
+		set beresp.grace = 1m; 
+	}
+	sub vcl_deliver {
+	    set resp.http.grace = req.http.grace;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	delay 1
+	txreq
+	sema r1 sync 3
+	rxresp
+} -start
+
+delay 1
+client c2 {
+	txreq -req SOFTBAN -url /
+	rxresp
+	expect resp.status == 410
+} -run
+
+
+client c3 {
+	sema r1 sync 3
+	txreq
+	rxresp
+	expect resp.http.grace == "t"
+	expect resp.status == 200
+} -run
diff --git a/include/vcli.h b/include/vcli.h
index 04ac11c..3d79e7b 100644
--- a/include/vcli.h
+++ b/include/vcli.h
@@ -78,6 +78,13 @@
 	"\tList the active bans.",					\
 	0, 0
 
+#define CLI_SOFTBAN							\
+	"softban",							\
+	"softban <field> <operator> <arg> [&& <field> <oper> <arg>]...",	\
+	"\tAll objects where the all the conditions match will be "	\
+	    "marked obsolete, but still eligible for grace.",		\
+	3, UINT_MAX
+
 #define CLI_VCL_LOAD							\
 	"vcl.load",							\
 	"vcl.load <configname> <filename>",				\
diff --git a/include/vrt.h b/include/vrt.h
index 2029f7a..9fed394 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -149,8 +149,8 @@ int VRT_re_match(struct req *, const char *, void *re);
 const char *VRT_regsub(struct req *, int all, const char *,
     void *, const char *);
 
-void VRT_ban(const struct req *, char *, ...);
-void VRT_ban_string(const struct req *, const char *);
+void VRT_ban(const struct req *, int, char *, ...);
+void VRT_ban_string(const struct req *, int, const char *);
 void VRT_purge(struct req *, double ttl, double grace);
 
 void VRT_count(struct req *, unsigned);
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 7eecc33..4458e16 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -178,7 +178,7 @@ parse_ban(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_ban_string(req, ");
+	Fb(tl, 1, "VRT_ban_string(req, 0, ");
 	vcc_Expr(tl, STRING);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");
@@ -197,7 +197,7 @@ parse_ban_url(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_ban(req, \"req.url\", \"~\", ");
+	Fb(tl, 1, "VRT_ban(req, 0, \"req.url\", \"~\", ");
 	vcc_Expr(tl, STRING);
 	ERRCHK(tl);
 	ExpectErr(tl, ')');
@@ -208,6 +208,26 @@ parse_ban_url(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
+parse_softban(struct vcc *tl)
+{
+
+	vcc_NextToken(tl);
+
+	ExpectErr(tl, '(');
+	vcc_NextToken(tl);
+
+	Fb(tl, 1, "VRT_ban_string(req, 1, ");
+	vcc_Expr(tl, STRING);
+	ERRCHK(tl);
+	Fb(tl, 0, ");\n");
+
+	ExpectErr(tl, ')');
+	vcc_NextToken(tl);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
 parse_new_syntax(struct vcc *tl)
 {
 
@@ -320,6 +340,7 @@ static struct action_table {
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "ban",		parse_ban },
 	{ "ban_url",		parse_ban_url },
+	{ "softban",		parse_softban },
 	{ "remove",		parse_unset }, /* backward compatibility */
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
-- 
1.7.10.4




From github at bsdchicks.com  Mon Oct 15 14:09:33 2012
From: github at bsdchicks.com (Rogier 'DocWilco' Mulhuijzen)
Date: Mon, 15 Oct 2012 16:09:33 +0200
Subject: [PATCH] Don't do conditionals for passes, leave that to the backend
Message-ID: <1350310173-14801-1-git-send-email-github@bsdchicks.com>

---
 bin/varnishd/cache/cache_response.c |    5 +++-
 bin/varnishtest/tests/r01206.vtc    |   48 +++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 bin/varnishtest/tests/r01206.vtc

diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index cb152a5..9839102 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -108,6 +108,7 @@ RES_BuildHttp(struct req *req)
 	char time_str[30];
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 
 	http_ClrHeader(req->resp);
 	http_FilterResp(req->obj->http, req->resp, 0);
@@ -122,7 +123,9 @@ RES_BuildHttp(struct req *req)
 	if (req->res_mode & RES_GUNZIP)
 		http_Unset(req->resp, H_Content_Encoding);
 
-	if (req->obj->response == 200
+	if (req->obj->objcore != NULL
+	    && !(req->obj->objcore->flags & OC_F_PASS)
+            && req->obj->response == 200
 	    && req->http->conds && RFC2616_Do_Cond(req)) {
 		req->wantbody = 0;
 		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
diff --git a/bin/varnishtest/tests/r01206.vtc b/bin/varnishtest/tests/r01206.vtc
new file mode 100644
index 0000000..1f8444d
--- /dev/null
+++ b/bin/varnishtest/tests/r01206.vtc
@@ -0,0 +1,48 @@
+varnishtest "Pass shouldn't honor IMS/INM if the backend doesn't"
+
+server s1 {
+	rxreq
+	txresp -bodylen 6
+
+	rxreq
+	txresp -hdr "ETag: 123456789" \
+	    -bodylen 6
+
+	rxreq
+	txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+	    -bodylen 6
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return(pass);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+
+	txreq -hdr "If-None-Match: 123456789"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+
+	txreq -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+} -run
+
+server s1 -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_pass = true;
+	}
+}
+
+client c1 -run
+
-- 
1.7.9.5




From github at bsdchicks.com  Mon Oct 15 14:14:03 2012
From: github at bsdchicks.com (Rogier 'DocWilco' Mulhuijzen)
Date: Mon, 15 Oct 2012 16:14:03 +0200
Subject: [PATCH] Don't do conditionals for passes, leave that to the backend
Message-ID: <1350310443-15182-1-git-send-email-github@bsdchicks.com>

The last one had a whitespace booboo, fixed in this one.

---
 bin/varnishd/cache/cache_response.c |    5 +++-
 bin/varnishtest/tests/r01206.vtc    |   48 +++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 bin/varnishtest/tests/r01206.vtc

diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index cb152a5..3fecd27 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -108,6 +108,7 @@ RES_BuildHttp(struct req *req)
 	char time_str[30];
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 
 	http_ClrHeader(req->resp);
 	http_FilterResp(req->obj->http, req->resp, 0);
@@ -122,7 +123,9 @@ RES_BuildHttp(struct req *req)
 	if (req->res_mode & RES_GUNZIP)
 		http_Unset(req->resp, H_Content_Encoding);
 
-	if (req->obj->response == 200
+	if (req->obj->objcore != NULL
+	    && !(req->obj->objcore->flags & OC_F_PASS)
+	    && req->obj->response == 200
 	    && req->http->conds && RFC2616_Do_Cond(req)) {
 		req->wantbody = 0;
 		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
diff --git a/bin/varnishtest/tests/r01206.vtc b/bin/varnishtest/tests/r01206.vtc
new file mode 100644
index 0000000..1f8444d
--- /dev/null
+++ b/bin/varnishtest/tests/r01206.vtc
@@ -0,0 +1,48 @@
+varnishtest "Pass shouldn't honor IMS/INM if the backend doesn't"
+
+server s1 {
+	rxreq
+	txresp -bodylen 6
+
+	rxreq
+	txresp -hdr "ETag: 123456789" \
+	    -bodylen 6
+
+	rxreq
+	txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+	    -bodylen 6
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return(pass);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+
+	txreq -hdr "If-None-Match: 123456789"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+
+	txreq -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+} -run
+
+server s1 -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_pass = true;
+	}
+}
+
+client c1 -run
+
-- 
1.7.9.5




From fgsch at lodoss.net  Tue Oct 16 08:16:01 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 16 Oct 2012 09:16:01 +0100
Subject: PATCH: Fix ticket 1212
Message-ID: <20121016091601.a9a0a8685168faa76b41b86b@lodoss.net>

 bin/varnishtest/tests/r01212.vtc |    9 +++++++++
 lib/libvcl/vcc_expr.c            |    1 +
 2 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/bin/varnishtest/tests/r01212.vtc b/bin/varnishtest/tests/r01212.vtc
new file mode 100644
index 0000000..b9d489c
--- /dev/null
+++ b/bin/varnishtest/tests/r01212.vtc
@@ -0,0 +1,9 @@
+varnishtest "#1212 - Vmod with HEADER argument given a STRING asserts the VCL compiler"
+
+varnish v1 -badvcl {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+
+	sub vcl_recv {
+		std.collect("foo");
+	}
+}
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 06f93ec..4b2390d 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -582,6 +582,7 @@ vcc_Eval_Func(struct vcc *tl, struct expr **e, const struct symbol *sym)
 				SkipToken(tl, ',');
 		} else if (fmt == HEADER) {
 			const struct var *v;
+			ExpectErr(tl, ID);
 			sym = VCC_FindSymbol(tl, tl->t, SYM_NONE);
 			ERRCHK(tl);
 			SkipToken(tl, ID);



From fgsch at lodoss.net  Tue Oct 16 08:17:27 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 16 Oct 2012 09:17:27 +0100
Subject: PATCH: exercise more VCL expressions
Message-ID: <20121016091727.31fbaf941b587b77c23570a1@lodoss.net>

 bin/varnishtest/tests/v00020.vtc |  110 +++++++++++++++++++++++++++++++++++++-
 1 files changed, 109 insertions(+), 1 deletions(-)

diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 1e36b43..62a7cbc 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -26,7 +26,7 @@ varnish v1 -vcl {
 varnish v1 -badvcl {
 	sub vcl_recv {
 		if (!req.restarts != req.url) {
-			set req.http.foo = "foo" + 3;
+			set req.http.foo = "foo";
 		}
 
 	}
@@ -79,3 +79,111 @@ varnish v1 -badvcl {
 		}
 	}
 }
+
+varnish v1 -vcl {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_recv {
+		set req.http.foo = "foo" + "bar";
+		set req.http.foo = "foo" + 1;
+		set req.http.foo = "foo" + now;
+
+		set req.http.foo = now + 1s;
+		set req.http.foo = now - 1s;
+		set req.http.foo = now - now;
+
+		set req.http.foo = 1 + 1;
+		set req.http.foo = 1 - 1;
+
+		set req.ttl = 1s;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = "foo" - "bar";
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = now + "foo";
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = now + now;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = now + 1;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1 + "foo";
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1 + now;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1 + 1s;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1s;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1s + 1;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1s + now;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = 1s + "foo";
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = "foo" + 1s;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.ttl = 1s + 1;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.ttl = 1s + now;
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.ttl = 1s + "foo";
+	}
+}



From jeremy at thomersonfamily.com  Thu Oct 18 00:58:34 2012
From: jeremy at thomersonfamily.com (Jeremy Thomerson)
Date: Wed, 17 Oct 2012 19:58:34 -0500
Subject: New VMOD - timeutils
Message-ID: <CADVmKVANWDWftD_a1FMxLUTz5q5cq5B-N5aCyrndGeXA0Jx21A@mail.gmail.com>

Devs,

  I just created a new VMOD [0] and would love if some of you could look
over it.  It's my first one.  The primary reason I created it was so that I
could avoid having to have inline C in my VCL as suggested at [1].  I asked
about this a few weeks ago on IRC and was told the best way to contribute a
way of doing this was to create a VMOD.  Ironically, today when I was
getting ready to publish my VMOD, I came across [2] where one of the two
things I needed to accomplish was discussed.  However, my VMOD goes beyond
what was discussed in that patch in order to provide additional
functionality.

  Please let me know what you think.  Also, if you know how to address the
TODO listed at [3], please let me know!  Thanks!

  NOTE: This VMOD has only been tested locally, and not in production yet.
I hope to be putting it into production in the next few weeks if all goes
well.  All feedback welcome.

Jeremy Thomerson

[0] - https://github.com/jthomerson/libvmod-timeutils/
[1] - https://www.varnish-cache.org/trac/wiki/VCLExampleSetExpires?version=1
[2] -
https://www.varnish-cache.org/lists/pipermail/varnish-dev/2012-October/007308.html
[3] -
https://github.com/jthomerson/libvmod-timeutils/blob/8d2921fe02ba778cc37b37aba62d226ee8f9f235/src/tests/expires_from_cache_control.vtc#L33
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121017/d805a08f/attachment.html>

From fgsch at lodoss.net  Thu Oct 18 11:09:56 2012
From: fgsch at lodoss.net (Federico Schwindt)
Date: Thu, 18 Oct 2012 12:09:56 +0100
Subject: New VMOD - timeutils
In-Reply-To: <CADVmKVANWDWftD_a1FMxLUTz5q5cq5B-N5aCyrndGeXA0Jx21A@mail.gmail.com>
References: <CADVmKVANWDWftD_a1FMxLUTz5q5cq5B-N5aCyrndGeXA0Jx21A@mail.gmail.com>
Message-ID: <CAJV_h0a0QzJsdDxk3wWTvtncqRMeYzEjPkJqEvw9TnHoi3mbsg@mail.gmail.com>

On Thu, Oct 18, 2012 at 1:58 AM, Jeremy Thomerson
<jeremy at thomersonfamily.com> wrote:
> Devs,
>
>   I just created a new VMOD [0] and would love if some of you could look
> over it.  It's my first one.  The primary reason I created it was so that I
> could avoid having to have inline C in my VCL as suggested at [1].  I asked
> about this a few weeks ago on IRC and was told the best way to contribute a
> way of doing this was to create a VMOD.  Ironically, today when I was
> getting ready to publish my VMOD, I came across [2] where one of the two
> things I needed to accomplish was discussed.  However, my VMOD goes beyond
> what was discussed in that patch in order to provide additional
> functionality.
> [..]

Hi,

I haven't looked at your vmod to comment but the TIME + DURATION issue
has been properly fixed in commit
cc513fdc6cc098fcdc378af6ec80959a5c744054 so now it's possible to do:

set resp.http.foo = now + 1s;

That will return a RFC date.
[1] could be achieved with:

set resp.http.foo = now + std.duration(regsub(resp.http.cache-control,
".* maxage=([0-9]*).*", "\1s"), 10s);

Cheers,

f.-



From fgsch at lodoss.net  Thu Oct 18 16:27:59 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 18 Oct 2012 17:27:59 +0100
Subject: PATCH: Fix 1216 - Bug:'error': not a valid action in method
 'vcl_deliver'.
Message-ID: <20121018162759.GA25936@nitzer.lodoss.local>


  Remove error from documentation and add missing backend declaration from
the test so it shows the expected error.

  f.-

diff --git a/bin/varnishtest/tests/r01027.vtc b/bin/varnishtest/tests/r01027.vtc
index 6c20b00..7545adc 100644
--- a/bin/varnishtest/tests/r01027.vtc
+++ b/bin/varnishtest/tests/r01027.vtc
@@ -1,6 +1,7 @@
 varnishtest "Test if you can error in vcl_deliver"
 
 varnish v1 -badvcl {
+	backend b { .host = "127.0.0.1"; }
 	sub vcl_deliver {
 		error 201 "ok";
 	}
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 4971315..1c1d8a2 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -512,9 +512,6 @@ vcl_deliver
   deliver
     Deliver the object to the client.
 
-  error code [reason]
-    Return the specified error code to the client and abandon the request.
-
   restart
     Restart the transaction. Increases the restart counter. If the number 
     of restarts is higher than *max_restarts* varnish emits a guru meditation 



From phk at phk.freebsd.dk  Mon Oct 22 09:44:24 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Mon, 22 Oct 2012 09:44:24 +0000
Subject: [PATCH 02/10] Add a BAN_Shutdown() routine that is called before
 STV_Close(), and makes sure that the ban lists will not change until we exit.
In-Reply-To: <1349879243-30369-2-git-send-email-martin@varnish-software.com>
References: <1349879243-30369-1-git-send-email-martin@varnish-software.com>
 <1349879243-30369-2-git-send-email-martin@varnish-software.com>
Message-ID: <15418.1350899064@critter.freebsd.dk>

--------
In message <1349879243-30369-2-git-send-email-martin at varnish-software.com>, Mar
tin Blix Grydeland writes:

I'm not happy with this one, it risks creating a pile-up of workers
which are trying to add bans's from VCL, and it does not shut down
the lurker in any meaningful way.

Add a private flag in cache_ban.c which indicates that a shutdown
is in progress, set it in the BAN_Shutdown() function.

When this flag is set, all attempts to add bans should fail before
trying to grab the mutex.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From lasse.karstensen at gmail.com  Mon Oct 22 11:53:32 2012
From: lasse.karstensen at gmail.com (Lasse Karstensen)
Date: Mon, 22 Oct 2012 13:53:32 +0200
Subject: New VMOD - timeutils
In-Reply-To: <CADVmKVANWDWftD_a1FMxLUTz5q5cq5B-N5aCyrndGeXA0Jx21A@mail.gmail.com>
References: <CADVmKVANWDWftD_a1FMxLUTz5q5cq5B-N5aCyrndGeXA0Jx21A@mail.gmail.com>
Message-ID: <20121022115331.GA30742@sierra.hyse.org>

Jeremy Thomerson:
[..]
>   Please let me know what you think.  Also, if you know how to address the
> TODO listed at [3], please let me know!  Thanks!
>   NOTE: This VMOD has only been tested locally, and not in production yet.
> I hope to be putting it into production in the next few weeks if all goes
> well.  All feedback welcome.

Hi Jeremy.

Thank you for contributing to the Varnish community. 

Since we will be living with 3.0 for a while, I think this vmod makes
sense.

I've sent a pull request for a small patch to make it slightly more
production friendly. Hope you will accept it.

-- 
Lasse Karstensen
Varnish Software AS
http://www.varnish-software.com/



From jeremy at thomersonfamily.com  Mon Oct 22 15:09:40 2012
From: jeremy at thomersonfamily.com (Jeremy Thomerson)
Date: Mon, 22 Oct 2012 10:09:40 -0500
Subject: New VMOD - timeutils
In-Reply-To: <20121022115331.GA30742@sierra.hyse.org>
References: <CADVmKVANWDWftD_a1FMxLUTz5q5cq5B-N5aCyrndGeXA0Jx21A@mail.gmail.com>
 <20121022115331.GA30742@sierra.hyse.org>
Message-ID: <CADVmKVDBkj_dtbA3q6ip2Mhpd-qzUhiVC4QZ5PTsOpyTGvQmKw@mail.gmail.com>

On Mon, Oct 22, 2012 at 6:53 AM, Lasse Karstensen <
lasse.karstensen at gmail.com> wrote:

> Jeremy Thomerson:
> [..]
> >   Please let me know what you think.  Also, if you know how to address
> the
> > TODO listed at [3], please let me know!  Thanks!
> >   NOTE: This VMOD has only been tested locally, and not in production
> yet.
> > I hope to be putting it into production in the next few weeks if all goes
> > well.  All feedback welcome.
>
> Hi Jeremy.
>
> Thank you for contributing to the Varnish community.
>
> Since we will be living with 3.0 for a while, I think this vmod makes
> sense.
>
> I've sent a pull request for a small patch to make it slightly more
> production friendly. Hope you will accept it.
>

Thank you!  I merged it.  With that change, do you think that this is ready
to run on production Varnish servers?  Also, what should I do to make it
easy for others to start using?  I'm assuming:

1 - git tag a 0.1 release
2 - list it in https://www.varnish-cache.org/vmods
3 - provide packaging for various distros (although I'm not sure what most
of the community uses, and I don't have any good experience in creating
packages for most distros)

Thoughts?

Thanks again,
Jeremy Thomerson
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121022/5116b7b3/attachment.html>

From in.vaibhavjain at gmail.com  Fri Oct 26 10:38:24 2012
From: in.vaibhavjain at gmail.com (Vaibhav Jain)
Date: Fri, 26 Oct 2012 16:08:24 +0530
Subject: Varnish 2.x Debugging
Message-ID: <CAGNDeNkvH6ONnZt24C02-ocgwBkE6_W8qMpGK2Rk5v8G6jdBtA@mail.gmail.com>

Hello Developers,

I am using Varnish 2.x on Drupal 6. I want to debug the VCL file, and see
at what point what all information I get so that I may code what I want.
I am at a subroutine vcl_recv, and have a request object "req", but how can
i know what all information does this object contain.
Is there anything like print_r for PHP where I can see all the information
contained in that object either on terminal or in logs.
I have tested varnishlog, but no good, also checked syslog, where i was
able to print static content, but not the complete object.

Also, whatever I have been able to grab from the net, I also have a feeling
that we have some static set of variables, if so, where Can i find the
complete list ?

Awaiting some help ...

-- 
Regards,
Vaibhav Jain
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121026/a433f273/attachment.html>

From slink at schokola.de  Mon Oct 29 17:20:53 2012
From: slink at schokola.de (Nils Goroll)
Date: Mon, 29 Oct 2012 18:20:53 +0100
Subject: [PATCH] Don't assert that privileges exist which have been introduced
 later than Solaris 10 FCS.
Message-ID: <508EBAF5.2020502@schokola.de>

(see also top-level comment: "For privileges which have been added later..:")

c613b135570f87535839e3a94630880d16910f4f has broken varnish for any Solaris OS 
older than onnv_140
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: 0001-Don-t-assert-that-privileges-exist-which-have-been-i.patch
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121029/576339ef/attachment.ksh>

From phk at phk.freebsd.dk  Mon Oct 29 21:02:39 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Mon, 29 Oct 2012 21:02:39 +0000
Subject: [PATCH] Don't assert that privileges exist which have been
 introduced later than Solaris 10 FCS.
In-Reply-To: <508EBAF5.2020502@schokola.de>
References: <508EBAF5.2020502@schokola.de>
Message-ID: <77297.1351544559@critter.freebsd.dk>

--------
In message <508EBAF5.2020502 at schokola.de>, Nils Goroll writes:

Niels, can't we find some way to assert that we get the privs
which are supported ?

I'm sort of paranoid about silent priv-sep issues, because they have
a tendency to become security exploits...


-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From fgsch at lodoss.net  Mon Oct 29 22:09:08 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 29 Oct 2012 22:09:08 +0000
Subject: PATCH: document beresp.http.header
Message-ID: <20121029220908.0ed9471ae743f0fda3922309@lodoss.net>

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1c1d8a2..1ad222f 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -719,6 +719,9 @@ beresp.do_gunzip
   Boolean. Unzip the object before storing it in the cache.  Defaults
   to false.
 
+beresp.http.header
+  The corresponding HTTP header.
+
 beresp.proto
   The HTTP protocol version used the backend replied with.
 




From fgsch at lodoss.net  Mon Oct 29 22:12:47 2012
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 29 Oct 2012 22:12:47 +0000
Subject: PATCH: remove non-ascii characters and don't manually split words.
Message-ID: <20121029221247.370d8dbd98798f0e937443be@lodoss.net>

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1c1d8a2..60daaa3 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -276,7 +276,7 @@ which can later be used to match client addresses:
   }
 
 If an ACL entry specifies a host name which Varnish is unable to
-resolve, it will match any address it is com? pared to.  Consequently,
+resolve, it will match any address it is compared to.  Consequently,
 if it is preceded by a negation mark, it will reject any address it is
 compared to, which may not be what you intended.  If the entry is
 enclosed in parentheses, however, it will simply be ignored.
@@ -407,8 +407,8 @@ vcl_pipe
 vcl_pass
   Called upon entering pass mode.  In this mode, the request is passed
   on to the backend, and the backend's response is passed on to the
-  client, but is not entered into the cache.  Subsequent requests sub?
-  mitted over the same client connection are handled normally.
+  client, but is not entered into the cache.  Subsequent requests
+  submitted over the same client connection are handled normally.
   
   The vcl_pass subroutine may terminate with calling return() with one of
   the following keywords:



From slink at schokola.de  Tue Oct 30 08:13:23 2012
From: slink at schokola.de (Nils Goroll)
Date: Tue, 30 Oct 2012 09:13:23 +0100
Subject: [PATCH] tailored assertions for priv_addset and setppriv
In-Reply-To: <77297.1351544559@critter.freebsd.dk>
References: <508EBAF5.2020502@schokola.de>
 <77297.1351544559@critter.freebsd.dk>
Message-ID: <508F8C23.2000805@schokola.de>

On 10/29/12 10:02 PM, Poul-Henning Kamp wrote:
> Niels, can't we find some way to assert that we get the privs
> which are supported ?

The attached patch asserts that priv_addset either succeeds, or fails with 
EINVAL. I've taken VTCP_Assert as a template.

Consequently, I have added the same check for three setppriv calls.

> I'm sort of paranoid about silent priv-sep issues, because they have
> a tendency to become security exploits...

I second your point in general, but this is not the place where this risk lives:

In mgt_sandbox_solaris_waive, we construct the privilege sets of the privileges 
we intend to use. These are then inverted and the inverse is removed from the 
active privilege sets. If the latter fails, the SETPPRIV macro logs a warning.

So if adding a privilege failed, we end up with less privileges rather than more.

We could, however, consider to assert in SETPPRIV that setppriv(PRIV_OFF, ...) 
succeeds rather than only throwing a warning if it fails, because this implies 
that we may be running with more privileges than we intended to.

The attached patch does not contain this change yet.

Nils
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: 0001-tailored-assertions-for-priv_addset-and-setppriv.patch
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121030/f41b03ed/attachment.ksh>

From phk at phk.freebsd.dk  Tue Oct 30 10:08:27 2012
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 30 Oct 2012 10:08:27 +0000
Subject: [PATCH] tailored assertions for priv_addset and setppriv
In-Reply-To: <508F8C23.2000805@schokola.de>
References: <508EBAF5.2020502@schokola.de>
 <77297.1351544559@critter.freebsd.dk> <508F8C23.2000805@schokola.de>
Message-ID: <5309.1351591707@critter.freebsd.dk>

--------
In message <508F8C23.2000805 at schokola.de>, Nils Goroll writes:

>We could, however, consider to assert in SETPPRIV that setppriv(PRIV_OFF, ...) 
>succeeds rather than only throwing a warning if it fails, because this implies 
>that we may be running with more privileges than we intended to.

I think that is a good idea.  Warnings tend to become "it always says that..."

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From slink at schokola.de  Tue Oct 30 11:05:35 2012
From: slink at schokola.de (Nils Goroll)
Date: Tue, 30 Oct 2012 12:05:35 +0100
Subject: [PATCH] tailored assertions for priv_addset and setppriv
In-Reply-To: <5309.1351591707@critter.freebsd.dk>
References: <508EBAF5.2020502@schokola.de>
 <77297.1351544559@critter.freebsd.dk> <508F8C23.2000805@schokola.de>
 <5309.1351591707@critter.freebsd.dk>
Message-ID: <508FB47F.9010408@schokola.de>



On 10/30/12 11:08 AM, Poul-Henning Kamp wrote:
> I think that is a good idea.  Warnings tend to become "it always says that..."

:) Here we go.
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: 0001-tailored-assertions-for-priv_addset-and-setppriv-as.patch
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121030/704d2c54/attachment.ksh>

From slink at schokola.de  Tue Oct 30 11:19:04 2012
From: slink at schokola.de (Nils Goroll)
Date: Tue, 30 Oct 2012 12:19:04 +0100
Subject: (final?) [PATCH] tailored assertions for priv_addset and setppriv,
 assert that setppriv(PRIV_OFF) succeeds when waiving privileges
In-Reply-To: <508FB47F.9010408@schokola.de>
References: <508EBAF5.2020502@schokola.de>
 <77297.1351544559@critter.freebsd.dk> <508F8C23.2000805@schokola.de>
 <5309.1351591707@critter.freebsd.dk> <508FB47F.9010408@schokola.de>
Message-ID: <508FB7A8.9000306@schokola.de>

code should match comments, if I say we use defines from sys/priv_names.h when 
they existed at OpenSolaris Launch, then I should do just that.

Sorry, I hope this is the final one for today. ;)

Nils
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: 0001-tailored-assertions-for-priv_addset-and-setppriv-as.patch
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-dev/attachments/20121030/11277a70/attachment.ksh>

From Raul.Rangel at disney.com  Tue Oct 30 16:30:56 2012
From: Raul.Rangel at disney.com (Rangel, Raul)
Date: Tue, 30 Oct 2012 09:30:56 -0700
Subject: RFC: new vcl_lookup{} proposal
Message-ID: <D3A8B375599CBC4599AD1B619647B6EF434E9A878F@SM-CALA-VXMB05B.swna.wdpr.disney.com>

This reply is in response to https://www.varnish-cache.org/trac/ticket/1107.

I like the explicit vcl. The less magic the easier it is to understand what is going on. I would also like to add that I love the idea of not having the first client block once the ttl expires. 

For my specific scenario having vcl_lookup is good...

I would like the ability to return a graced object instead of the current response from the backend. Sometimes my app server fails to get data from one of its external data sources. If this happens the app server can tag the request as being "not-ideal". If varnish gets a non-ideal response I want it to return the grace object if it exists otherwise return the non-ideal response.

An example of a non-ideal response would be an ESI that renders a gallery. If the app server fails to connect to the gallery data source it will render an empty gallery instead of returning a 500. Now if varnish has a stale copy then I would rather the client get a stale gallery. If varnish does not have a stale copy I would rather the user get an empty gallery. This way the user still gets the correct HTML and I can try and render the gallery on the client side. This is almost like saint mode but I don't want to mark the backend as sick.

Here is some sample vcl I dreamed up:

sub vcl_lookup {
	if (obj && req.http.X-Serve-Stale) {
		obj.ttl = 2s;
		return (deliver);
	}
}

sub vcl_fetch {
	if (beresp.status == 208) { // 208: The content might not be complete and a stale copy should be used
		set beresp.status = 200;
		if (!req.http.X-Serve-Stale) {
			set req.http.X-Serve-Stale = "yes";
			return (restart);
		}
	}
}

This is not currently possible with vcl_miss because we don't have the grace object. Though having a stale_obj and return(stale) in vcl_fetch might be the ideal way to handle this.

Raul



From tfheen at varnish-software.com  Wed Oct 31 07:24:11 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Wed, 31 Oct 2012 08:24:11 +0100
Subject: PATCH: remove non-ascii characters and don't manually split words.
In-Reply-To: <20121029221247.370d8dbd98798f0e937443be@lodoss.net>
References: <20121029221247.370d8dbd98798f0e937443be@lodoss.net>
Message-ID: <20121031072411.GD11889@err.no>

Thanks, applied

-- 
Tollef Fog Heen
Technical lead | Varnish Software AS
t: +47 21 98 92 64
We Make Websites Fly!



From tfheen at varnish-software.com  Wed Oct 31 07:27:53 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Wed, 31 Oct 2012 08:27:53 +0100
Subject: PATCH: document beresp.http.header
In-Reply-To: <20121029220908.0ed9471ae743f0fda3922309@lodoss.net>
References: <20121029220908.0ed9471ae743f0fda3922309@lodoss.net>
Message-ID: <20121031072753.GF11889@err.no>

Thanks, applied

-- 
Tollef Fog Heen
Technical lead | Varnish Software AS
t: +47 21 98 92 64
We Make Websites Fly!



From tfheen at varnish-software.com  Wed Oct 31 07:27:11 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Wed, 31 Oct 2012 08:27:11 +0100
Subject: Varnish 2.x Debugging
In-Reply-To: <CAGNDeNkvH6ONnZt24C02-ocgwBkE6_W8qMpGK2Rk5v8G6jdBtA@mail.gmail.com>
References: <CAGNDeNkvH6ONnZt24C02-ocgwBkE6_W8qMpGK2Rk5v8G6jdBtA@mail.gmail.com>
Message-ID: <20121031072711.GE11889@err.no>

]] Vaibhav Jain

> Also, whatever I have been able to grab from the net, I also have a feeling
> that we have some static set of variables, if so, where Can i find the
> complete list ?

This is not really a question for -dev, which is about Varnish
development.

You might want to take a look at man vcl.

Cheers,
-- 
Tollef Fog Heen
Technical lead | Varnish Software AS
t: +47 21 98 92 64
We Make Websites Fly!



From tfheen at varnish-software.com  Wed Oct 31 08:31:53 2012
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Wed, 31 Oct 2012 09:31:53 +0100
Subject: [PATCH] Add softban support
In-Reply-To: <1350301785-6835-2-git-send-email-tfheen@varnish-software.com>
References: <1350301785-6835-2-git-send-email-tfheen@varnish-software.com>
Message-ID: <1351672313-16874-1-git-send-email-tfheen@varnish-software.com>

Soft bans are like normal bans, but they are subject to grace per
usual rules.
---
 bin/varnishd/cache/cache.h       |    1 +
 bin/varnishd/cache/cache_ban.c   |   58 +++++++++++++++++++++++++++++++++----
 bin/varnishd/cache/cache_vrt.c   |    6 ++--
 bin/varnishtest/tests/s00004.vtc |   47 ++++++++++++++++++++++++++++++
 bin/varnishtest/tests/s00005.vtc |   59 ++++++++++++++++++++++++++++++++++++++
 include/vcli.h                   |    7 +++++
 include/vrt.h                    |    4 +--
 lib/libvcl/vcc_action.c          |   25 ++++++++++++++--
 8 files changed, 195 insertions(+), 12 deletions(-)
 create mode 100644 bin/varnishtest/tests/s00004.vtc
 create mode 100644 bin/varnishtest/tests/s00005.vtc

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6d4fe86..66ee459 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -757,6 +757,7 @@ void BAN_Compile(void);
 struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail);
 void BAN_TailDeref(struct ban **ban);
 double BAN_Time(const struct ban *ban);
+void BAN_Set_Soft(struct ban *b, int soft);
 
 /* cache_busyobj.c */
 void VBO_Init(void);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index aa8bca9..6270a4a 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -42,6 +42,7 @@
  *	8 bytes	- double: timestamp		XXX: Byteorder ?
  *	4 bytes - be32: length
  *	1 byte - flags: 0x01: BAN_F_REQ
+ *			0x02: BAN_F_SOFT
  *	N tests
  * A test have this form:
  *	1 byte - arg (see ban_vars.h col 3 "BAN_ARG_XXX")
@@ -82,6 +83,7 @@ struct ban {
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
+#define BAN_F_SOFT		(1 << 3)	/* soft ban, aka grace */
 #define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
@@ -392,7 +394,11 @@ BAN_Insert(struct ban *b)
 
 	t0 = VTIM_real();
 	memcpy(b->spec, &t0, sizeof t0);
-	b->spec[12] = (b->flags & BAN_F_REQ) ? 1 : 0;
+	b->spec[12] = 0;
+	if (b->flags & BAN_F_REQ)
+		b->spec[12] |= 0x1;
+	if (b->flags & BAN_F_SOFT)
+		b->spec[12] |= 0x2;
 	memcpy(b->spec + 13, VSB_data(b->vsb), ln);
 	ln += 13;
 	vbe32enc(b->spec + 8, ln);
@@ -551,8 +557,10 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 	AN(b2->spec);
 	memcpy(b2->spec, ban, len);
 	b2->flags |= gone;
-	if (ban[12])
+	if (ban[12] & 0x01)
 		b2->flags |= BAN_F_REQ;
+	if (ban[12] & 0x02)
+		b2->flags |= BAN_F_SOFT;
 	if (b == NULL)
 		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
 	else
@@ -587,6 +595,21 @@ BAN_Time(const struct ban *b)
 }
 
 /*--------------------------------------------------------------------
+ * Set/unset the soft flag on a ban
+ */
+
+void
+BAN_Set_Soft(struct ban *b, int soft)
+{
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	if (soft)
+		b->flags |= BAN_F_SOFT;
+	else
+		b->flags &= ~BAN_F_SOFT;
+}
+
+/*--------------------------------------------------------------------
  * All silos have read their bans, ready for action
  */
 
@@ -687,7 +710,8 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	struct ban *b;
 	struct objcore *oc;
 	struct ban * volatile b0;
-	unsigned tests, skipped;
+	unsigned tests, skipped, soft;
+	double now;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	CHECK_OBJ_ORNULL(req_http, HTTP_MAGIC);
@@ -708,6 +732,7 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	 */
 	tests = 0;
 	skipped = 0;
+	soft = 0;
 	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 		if (b->flags & BAN_F_GONE)
@@ -724,15 +749,21 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 			 * be other bans that match, so we soldier on
 			 */
 			skipped++;
-		} else if (ban_evaluate(b->spec, o->http, req_http, &tests))
+		} else if (ban_evaluate(b->spec, o->http, req_http, &tests)) {
+			if (b->flags & BAN_F_SOFT) {
+				soft = 1;
+				continue;
+			}
+			soft = 0; /* Found hard ban */
 			break;
+		}
 	}
 
 	Lck_Lock(&ban_mtx);
 	VSC_C_main->bans_tested++;
 	VSC_C_main->bans_tests_tested += tests;
 
-	if (b == oc->ban && skipped > 0) {
+	if (b == oc->ban && skipped > 0 && soft == 0) {
 		AZ(req_http);
 		Lck_Unlock(&ban_mtx);
 		/*
@@ -751,10 +782,21 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	}
 	Lck_Unlock(&ban_mtx);
 
-	if (b == oc->ban) {	/* not banned */
+	if (soft == 0 && b == oc->ban) {	/* not banned */
 		oc->ban = b0;
 		oc_updatemeta(oc);
 		return (0);
+	} else if (soft) {
+		now =  VTIM_real();
+		/* Softban, set ttl to now */
+		if (o->exp.entered + o->exp.ttl > now)
+			o->exp.ttl = now - o->exp.entered;
+		oc->ban = b0;
+		oc_updatemeta(oc);
+		/* XXX: no req in lurker */
+		VSLb(vsl, SLT_ExpBan, "%u was softbanned", o->vxid);
+		EXP_Rearm(o);
+		return (2);
 	} else {
 		EXP_Clr(&o->exp);
 		oc->ban = NULL;
@@ -1026,6 +1068,9 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 		VCLI_SetResult(cli, CLIS_CANT);
 		return;
 	}
+	if (strcmp(av[1], "softban") == 0) {
+		b->flags |= BAN_F_SOFT;
+	}
 	for (i = 0; i < narg; i += 4)
 		if (BAN_AddTest(cli, b, av[i + 2], av[i + 3], av[i + 4])) {
 			BAN_Free(b);
@@ -1125,6 +1170,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 static struct cli_proto ban_cmds[] = {
 	{ CLI_BAN_URL,				"", ccf_ban_url },
 	{ CLI_BAN,				"", ccf_ban },
+	{ CLI_SOFTBAN,				"", ccf_ban },
 	{ CLI_BAN_LIST,				"", ccf_ban_list },
 	{ NULL }
 };
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index ed7ae59..70537fc 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -433,7 +433,7 @@ VRT_synth_page(const struct req *req, unsigned flags, const char *str, ...)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_ban(const struct req *req, char *cmds, ...)
+VRT_ban(const struct req *req, int soft, char *cmds, ...)
 {
 	char *a1, *a2, *a3;
 	va_list ap;
@@ -442,6 +442,7 @@ VRT_ban(const struct req *req, char *cmds, ...)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	b = BAN_New();
+	BAN_Set_Soft(b, soft);
 	va_start(ap, cmds);
 	a1 = cmds;
 	good = 0;
@@ -468,7 +469,7 @@ VRT_ban(const struct req *req, char *cmds, ...)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_ban_string(const struct req *req, const char *str)
+VRT_ban_string(const struct req *req, int soft, const char *str)
 {
 	char *a1, *a2, *a3;
 	char **av;
@@ -484,6 +485,7 @@ VRT_ban_string(const struct req *req, const char *str)
 		return;
 	}
 	b = BAN_New();
+	BAN_Set_Soft(b, soft);
 	good = 0;
 	for (i = 1; ;) {
 		a1 = av[i++];
diff --git a/bin/varnishtest/tests/s00004.vtc b/bin/varnishtest/tests/s00004.vtc
new file mode 100644
index 0000000..781d1dc
--- /dev/null
+++ b/bin/varnishtest/tests/s00004.vtc
@@ -0,0 +1,47 @@
+varnishtest "Softbans from CLI"
+
+server s1 {
+	rxreq
+	txresp
+
+	sema r1 sync 3
+
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_hit {
+	    if (obj.ttl <= 0s) {
+	    	    set req.http.grace = "t";
+	    }
+	}
+	sub vcl_fetch { 
+		set beresp.ttl = 1m;
+		set beresp.grace = 1m; 
+	}
+	sub vcl_deliver {
+	    set resp.http.grace = req.http.grace;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	delay 1
+	txreq
+	sema r1 sync 3
+	rxresp
+} -start
+
+varnish v1 -cliok "softban req.url ~ ^"
+delay 1;
+
+client c2 {
+	sema r1 sync 3
+	txreq
+	rxresp
+	expect resp.http.grace == "t"
+	expect resp.status == 200
+} -run
diff --git a/bin/varnishtest/tests/s00005.vtc b/bin/varnishtest/tests/s00005.vtc
new file mode 100644
index 0000000..35d1a8a
--- /dev/null
+++ b/bin/varnishtest/tests/s00005.vtc
@@ -0,0 +1,59 @@
+varnishtest "Softbans from VCL"
+
+server s1 {
+	rxreq
+	txresp
+
+	sema r1 sync 3
+
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+	    if (req.request == "SOFTBAN") {
+	       softban("req.url ~ " + req.url);
+	       error 410;
+	    }
+	}
+
+	sub vcl_hit {
+	    if (obj.ttl <= 0s) {
+	    	    set req.http.grace = "t";
+	    }
+	}
+	sub vcl_fetch { 
+		set beresp.ttl = 1m;
+		set beresp.grace = 1m; 
+	}
+	sub vcl_deliver {
+	    set resp.http.grace = req.http.grace;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	delay 1
+	txreq
+	sema r1 sync 3
+	rxresp
+} -start
+
+delay 1
+client c2 {
+	txreq -req SOFTBAN -url /
+	rxresp
+	expect resp.status == 410
+} -run
+
+
+client c3 {
+	sema r1 sync 3
+	txreq
+	rxresp
+	expect resp.http.grace == "t"
+	expect resp.status == 200
+} -run
diff --git a/include/vcli.h b/include/vcli.h
index 04ac11c..3d79e7b 100644
--- a/include/vcli.h
+++ b/include/vcli.h
@@ -78,6 +78,13 @@
 	"\tList the active bans.",					\
 	0, 0
 
+#define CLI_SOFTBAN							\
+	"softban",							\
+	"softban <field> <operator> <arg> [&& <field> <oper> <arg>]...",	\
+	"\tAll objects where the all the conditions match will be "	\
+	    "marked obsolete, but still eligible for grace.",		\
+	3, UINT_MAX
+
 #define CLI_VCL_LOAD							\
 	"vcl.load",							\
 	"vcl.load <configname> <filename>",				\
diff --git a/include/vrt.h b/include/vrt.h
index c98e085..bb21f93 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -177,8 +177,8 @@ int VRT_re_match(struct req *, const char *, void *re);
 const char *VRT_regsub(struct req *, int all, const char *,
     void *, const char *);
 
-void VRT_ban(const struct req *, char *, ...);
-void VRT_ban_string(const struct req *, const char *);
+void VRT_ban(const struct req *, int, char *, ...);
+void VRT_ban_string(const struct req *, int, const char *);
 void VRT_purge(struct req *, double ttl, double grace);
 
 void VRT_count(struct req *, unsigned);
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index eae6231..3662a39 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -183,7 +183,7 @@ parse_ban(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_ban_string(req, ");
+	Fb(tl, 1, "VRT_ban_string(req, 0, ");
 	vcc_Expr(tl, STRING);
 	ERRCHK(tl);
 	Fb(tl, 0, ");\n");
@@ -202,7 +202,7 @@ parse_ban_url(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_ban(req, \"req.url\", \"~\", ");
+	Fb(tl, 1, "VRT_ban(req, 0, \"req.url\", \"~\", ");
 	vcc_Expr(tl, STRING);
 	ERRCHK(tl);
 	ExpectErr(tl, ')');
@@ -213,6 +213,26 @@ parse_ban_url(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
+parse_softban(struct vcc *tl)
+{
+
+	vcc_NextToken(tl);
+
+	ExpectErr(tl, '(');
+	vcc_NextToken(tl);
+
+	Fb(tl, 1, "VRT_ban_string(req, 1, ");
+	vcc_Expr(tl, STRING);
+	ERRCHK(tl);
+	Fb(tl, 0, ");\n");
+
+	ExpectErr(tl, ')');
+	vcc_NextToken(tl);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
 parse_new_syntax(struct vcc *tl)
 {
 
@@ -325,6 +345,7 @@ static struct action_table {
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "ban",		parse_ban },
 	{ "ban_url",		parse_ban_url },
+	{ "softban",		parse_softban },
 	{ "remove",		parse_unset }, /* backward compatibility */
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
-- 
1.7.10.4