"} obj.status " " obj.response {"

From geoff at varnish-cache.org Mon Aug 1 11:18:08 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:08 +0200 Subject: [experimental-ims] fe7e315 Print 0, not - for zero-byte replies Message-ID: commit fe7e315cb98ffdc2e4f510e43c7b54bc1af840c1 Author: Tollef Fog Heen Date: Mon Jul 11 09:50:09 2011 +0200 Print 0, not - for zero-byte replies Fixes: #949 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 89c1ee1..521d52e 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -546,7 +546,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, case 'b': /* %b */ - fprintf(fo, "%s", lp->df_b ? lp->df_b : "-"); + fprintf(fo, "%s", lp->df_b ? lp->df_b : "0"); break; case 'H': From geoff at varnish-cache.org Mon Aug 1 11:18:08 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:08 +0200 Subject: [experimental-ims] 3a0d628 Don't assert on sleep failure due to signals in VSL library (some utils expect to receive SIGHUP). Message-ID: commit 3a0d628c1e2dff4b7a15253739f7cecdff8ec19c Author: Martin Blix Grydeland Date: Fri Jul 8 16:30:06 2011 +0200 Don't assert on sleep failure due to signals in VSL library (some utils expect to receive SIGHUP). Fixes: #947 diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 2b3b45e..4141c28 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -184,7 +184,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp) if (vsl->flags & F_NON_BLOCKING) return (-1); w += SLEEP_USEC; - AZ(usleep(SLEEP_USEC)); + assert(usleep(SLEEP_USEC) == 0 || errno == EINTR); continue; } *pp = (void*)(uintptr_t)vsl->log_ptr; /* Loose volatile */ From geoff at varnish-cache.org Mon Aug 1 11:18:08 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:08 +0200 Subject: [experimental-ims] c11ac80 Avoid segfaulting if hitmiss or handling hasn't been set yet Message-ID: commit c11ac805b50dcf77aac5d4ba3d210207924250bf Author: Tollef Fog Heen Date: Mon Jul 11 10:14:40 2011 +0200 Avoid segfaulting if hitmiss or handling hasn't been set yet Fall back to "-" if the handling has not been decided yet. Fixes: #950 Fixes: #944 Fixes: #918 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 521d52e..969cc22 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -654,11 +654,11 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, p += 9+15; break; } else if (strncmp(what, "hitmiss}x", 9) == 0) { - fprintf(fo, "%s", lp->df_hitmiss); + fprintf(fo, "%s", (lp->df_hitmiss ? lp->df_hitmiss : "-")); p += 9+8; break; } else if (strncmp(what, "handling}x", 10) == 0) { - fprintf(fo, "%s", lp->df_handling); + fprintf(fo, "%s", (lp->df_handling ? lp->df_handling : "-")); p += 9+9; break; } From geoff at varnish-cache.org Mon Aug 1 11:18:09 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:09 +0200 Subject: [experimental-ims] 17df9a6 Fix compilation errors (set-but-unused variables) Message-ID: commit 17df9a6a0656a4b11487d7b9c9fdafcadde0af91 Author: Tollef Fog Heen Date: Mon Jul 11 10:43:36 2011 +0200 Fix compilation errors (set-but-unused variables) Thanks to new GCC for spotting those. diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index f52eb4b..4788a23 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -265,6 +265,7 @@ res_WriteGunzipObj(struct sess *sp) st->ptr, st->len, obuf, sizeof obuf, &obufl); /* XXX: error check */ + (void)i; } if (obufl) { (void)WRW_Write(sp->wrk, obuf, obufl); diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c index 71a9c81..f7355cb 100644 --- a/bin/varnishreplay/varnishreplay.c +++ b/bin/varnishreplay/varnishreplay.c @@ -494,7 +494,6 @@ replay_thread(void *arg) struct replay_thread *thr = arg; struct message *msg; enum VSL_tag_e tag; - size_t len; char *ptr; const char *next; @@ -504,7 +503,6 @@ replay_thread(void *arg) while ((msg = mailbox_get(&thr->mbox)) != NULL) { tag = msg->tag; - len = msg->len; ptr = msg->ptr; thread_log(2, 0, "%s(%s)", VSL_tags[tag], msg->ptr); @@ -639,14 +637,11 @@ gen_traffic(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len, unsigned spec, const char *ptr, uint64_t bitmap) { struct replay_thread *thr; - const char *end; struct message *msg; (void)priv; (void)bitmap; - end = ptr + len; - if (fd == 0 || !(spec & VSL_S_CLIENT)) return (0); From geoff at varnish-cache.org Mon Aug 1 11:18:09 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:09 +0200 Subject: [experimental-ims] ba20513 Fix up syntax for matching in docs Message-ID: commit ba2051320e581c84026821ed3e86fa3681e294dc Author: Tollef Fog Heen Date: Fri Jul 15 08:39:37 2011 +0200 Fix up syntax for matching in docs diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst index 12617f5..5a1fbac 100644 --- a/doc/sphinx/faq/general.rst +++ b/doc/sphinx/faq/general.rst @@ -86,13 +86,13 @@ The "varnishlog" utility may produce a horrendous amount of output. To be able The ReqStart token will include the client IP address. To see log entries matching this, type:: - $ varnishlog -c -o ReqStart 192.0.2.123 + $ varnishlog -c -m ReqStart:192.0.2.123 To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. At the shell command line, type:: - $ varnishlog -b -o TxHeader 192.0.2.123 + $ varnishlog -b -m TxHeader:192.0.2.123 **How can I rewrite URLS before they are sent to the backend?** diff --git a/doc/sphinx/tutorial/troubleshooting.rst b/doc/sphinx/tutorial/troubleshooting.rst index aeb2786..7ab9435 100644 --- a/doc/sphinx/tutorial/troubleshooting.rst +++ b/doc/sphinx/tutorial/troubleshooting.rst @@ -80,13 +80,13 @@ give you a clue. Since varnishlog logs so much data it might be hard to track the entries down. You can set varnishlog to log all your 503 errors by issuing the following command::: - $ varnishlog -c -o TxStatus 503 + $ varnishlog -c -m TxStatus:503 If the error happened just a short time ago the transaction might still be in the shared memory log segment. To get varnishlog to process the whole shared memory log just add the -d option::: - $ varnishlog -d -c -o TxStatus 503 + $ varnishlog -d -c -m TxStatus:503 Please see the varnishlog man page for elaborations on further filtering capabilities and explanation of the various options. From geoff at varnish-cache.org Mon Aug 1 11:18:09 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:09 +0200 Subject: [experimental-ims] 044141f Remove obsolete bits from varnishlog man page Message-ID: commit 044141f625258b661fdd9e29c3f7c24d1b6d115b Author: Tollef Fog Heen Date: Fri Jul 15 08:41:52 2011 +0200 Remove obsolete bits from varnishlog man page diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst index c0d73c9..be9c9c5 100644 --- a/doc/sphinx/reference/varnishlog.rst +++ b/doc/sphinx/reference/varnishlog.rst @@ -81,10 +81,6 @@ The following options are available: -x tag Exclude log entries with the specified tag. -If the -o option was specified, an additional tag and regex may be -specified to select only requests which generated a log entry with the -given tag whose contents match the given regex. - TAGS ==== The following log entry tags are currently defined: From geoff at varnish-cache.org Mon Aug 1 11:18:09 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:09 +0200 Subject: [experimental-ims] e6a9571 Use vsb for varnishncsa to prevent partial lines on error in format strings Message-ID: commit e6a9571a5810105939c330c53dd18cc5be7feb0e Author: Tollef Fog Heen Date: Fri Jul 15 09:02:05 2011 +0200 Use vsb for varnishncsa to prevent partial lines on error in format strings diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am index f34a0e2..9b652da 100644 --- a/bin/varnishncsa/Makefile.am +++ b/bin/varnishncsa/Makefile.am @@ -13,6 +13,7 @@ varnishncsa_SOURCES = \ $(top_builddir)/lib/libvarnish/assert.c \ $(top_builddir)/lib/libvarnish/flopen.c \ $(top_builddir)/lib/libvarnish/version.c \ + $(top_builddir)/lib/libvarnish/vsb.c \ $(top_builddir)/lib/libvarnish/vpf.c varnishncsa_LDADD = \ diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 969cc22..934fdf3 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -485,6 +485,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, FILE *fo = priv; char *q, tbuf[64]; const char *p; + struct vsb *os; if (fd >= nll) { struct logline **newll = ll; @@ -533,12 +534,12 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, /* We have a complete data set - log a line */ fo = priv; + os = VSB_new_auto(); for (p = format; *p != '\0'; p++) { - if (*p != '%') { - fprintf(fo, "%c", *p); + VSB_putc(os, *p); continue; } p++; @@ -546,29 +547,29 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, case 'b': /* %b */ - fprintf(fo, "%s", lp->df_b ? lp->df_b : "0"); + VSB_cat(os, lp->df_b ? lp->df_b : "0"); break; case 'H': - fprintf(fo, "%s", lp->df_H); + VSB_cat(os, lp->df_H); break; case 'h': if (!lp->df_h && spec & VSL_S_BACKEND) - fprintf(fo, "127.0.0.1"); + VSB_cat(os, "127.0.0.1"); else - fprintf(fo, "%s", lp->df_h ? lp->df_h : "-"); + VSB_cat(os, lp->df_h ? lp->df_h : "-"); break; case 'l': - fprintf(fo, "-"); + VSB_putc(os, '-'); break; case 'm': - fprintf(fo, "%s", lp->df_m); + VSB_cat(os, lp->df_m); break; case 'q': - fprintf(fo, "%s", lp->df_q ? lp->df_q : ""); + VSB_cat(os, lp->df_q ? lp->df_q : ""); break; case 'r': @@ -576,30 +577,32 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, * Fake "%r". This would be a lot easier if Varnish * normalized the request URL. */ - fprintf(fo, "%s ", lp->df_m); + VSB_cat(os, lp->df_m); + VSB_putc(os, ' '); if (lp->df_Host) { if (strncmp(lp->df_Host, "http://", 7) != 0) - fprintf(fo, "http://"); - fprintf(fo, "%s", lp->df_Host); + VSB_cat(os, "http://"); + VSB_cat(os, lp->df_Host); } - fprintf(fo, "%s", lp->df_U); - fprintf(fo, "%s ", lp->df_q ? lp->df_q : ""); - fprintf(fo, "%s", lp->df_H); + VSB_cat(os, lp->df_U); + VSB_cat(os, lp->df_q ? lp->df_q : ""); + VSB_putc(os, ' '); + VSB_cat(os, lp->df_H); break; case 's': /* %s */ - fprintf(fo, "%s", lp->df_s ? lp->df_s : ""); + VSB_cat(os, lp->df_s ? lp->df_s : ""); break; case 't': /* %t */ strftime(tbuf, sizeof tbuf, "[%d/%b/%Y:%T %z]", &lp->df_t); - fprintf(fo, "%s", tbuf); + VSB_cat(os, tbuf); break; case 'U': - fprintf(fo, "%s", lp->df_U); + VSB_cat(os, lp->df_U); break; case 'u': @@ -616,49 +619,45 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, q = strchr(rubuf, ':'); if (q != NULL) *q = '\0'; - fprintf(fo, "%s", rubuf); + VSB_cat(os, rubuf); free(rubuf); } else { - fprintf(fo, "-"); + VSB_putc(os, '-'); } break; case '{': if (strncmp(p, "{Referer}i", 10) == 0) { - fprintf(fo, "%s", - lp->df_Referer ? lp->df_Referer : "-"); + VSB_cat(os, lp->df_Referer ? lp->df_Referer : "-"); p += 9; break; } else if (strncmp(p, "{Host}i", 7) == 0) { - fprintf(fo, "%s", - lp->df_Host ? lp->df_Host : "-"); + VSB_cat(os, lp->df_Host ? lp->df_Host : "-"); p += 6; break; } else if (strncmp(p, "{X-Forwarded-For}i", 18) == 0) { /* %{Referer}i */ - fprintf(fo, "%s", - lp->df_X_Forwarded_For ? lp->df_X_Forwarded_For : "-"); + VSB_cat(os, lp->df_X_Forwarded_For ? lp->df_X_Forwarded_For : "-"); p += 17; break; } else if (strncmp(p, "{User-agent}i", 13) == 0) { /* %{User-agent}i */ - fprintf(fo, "%s", - lp->df_User_agent ? lp->df_User_agent : "-"); + VSB_cat(os, lp->df_User_agent ? lp->df_User_agent : "-"); p += 12; break; } else if (strncmp(p, "{Varnish:", 9) == 0) { /* Scan for what we're looking for */ const char *what = p+9; if (strncmp(what, "time_firstbyte}x", 16) == 0) { - fprintf(fo, "%s", lp->df_ttfb); + VSB_cat(os, lp->df_ttfb); p += 9+15; break; } else if (strncmp(what, "hitmiss}x", 9) == 0) { - fprintf(fo, "%s", (lp->df_hitmiss ? lp->df_hitmiss : "-")); + VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-")); p += 9+8; break; } else if (strncmp(what, "handling}x", 10) == 0) { - fprintf(fo, "%s", (lp->df_handling ? lp->df_handling : "-")); + VSB_cat(os, (lp->df_handling ? lp->df_handling : "-")); p += 9+9; break; } @@ -670,14 +669,16 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, exit(1); } } - fprintf(fo, "\n"); + VSB_putc(os, '\n'); /* flush the stream */ + VSB_finish(os); + fprintf(fo, "%s", VSB_data(os)); fflush(fo); /* clean up */ clean_logline(lp); - + VSB_delete(os); return (reopen); } From geoff at varnish-cache.org Mon Aug 1 11:18:10 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:10 +0200 Subject: [experimental-ims] cbd2874 Handle errors in varnishncsa formats slightly better Message-ID: commit cbd2874256f2ce749e1cf8cbda989bb424bb6c39 Author: Tollef Fog Heen Date: Fri Jul 15 09:03:46 2011 +0200 Handle errors in varnishncsa formats slightly better Print out the rest of the format, changing the error from "Unknown format character: {" to: "Unknown format starting at: %{asdf}x" diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 934fdf3..9178168 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -665,7 +665,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, /* Fall through if we haven't handled something */ /* FALLTHROUGH*/ default: - fprintf(stderr, "Unknown format character: %c\n", *p); + fprintf(stderr, "Unknown format starting at: %s\n", --p); exit(1); } } From geoff at varnish-cache.org Mon Aug 1 11:18:11 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:11 +0200 Subject: [experimental-ims] 3e86cd8 Make it possible to log arbitrary headers in varnishncsa Message-ID: commit 3e86cd8d1e160613c1bcd5def787f4ba8196a9e3 Author: Tollef Fog Heen Date: Fri Jul 15 10:57:16 2011 +0200 Make it possible to log arbitrary headers in varnishncsa diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 9178168..94c81ef 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -54,8 +54,7 @@ * %q Query string * %H Protocol version * - * TODO: - Make it possible to grab any request header - * - Maybe rotate/compress log + * TODO: - Maybe rotate/compress log */ #include "config.h" @@ -72,6 +71,7 @@ #include "vsb.h" #include "vpf.h" +#include "vqueue.h" #include "libvarnish.h" #include "vsl.h" @@ -81,14 +81,16 @@ static volatile sig_atomic_t reopen; +struct hdr { + char *key; + char *value; + VTAILQ_ENTRY(hdr) list; +}; + static struct logline { char *df_H; /* %H, Protocol version */ - char *df_Host; /* %{Host}i */ - char *df_Referer; /* %{Referer}i */ char *df_U; /* %U, URL path */ char *df_q; /* %q, query string */ - char *df_User_agent; /* %{User-agent}i */ - char *df_X_Forwarded_For; /* %{X-Forwarded-For}i */ char *df_b; /* %b, Bytes */ char *df_h; /* %h (host name / IP adress)*/ char *df_m; /* %m, Request method*/ @@ -101,6 +103,7 @@ static struct logline { int active; /* Is log line in an active trans */ int complete; /* Is log line complete */ uint64_t bitmap; /* Bitmap for regex matches */ + VTAILQ_HEAD(, hdr) headers; } **ll; struct VSM_data *vd; @@ -186,23 +189,39 @@ trimline(const char *str, const char *end) return (p); } +static char * +get_header(struct logline *l, const char *name) +{ + struct hdr *h; + VTAILQ_FOREACH(h, &l->headers, list) { + if (strcasecmp(h->key, name) == 0) { + return h->value; + break; + } + } + return NULL; +} + static void clean_logline(struct logline *lp) { + struct hdr *h, *h2; #define freez(x) do { if (x) free(x); x = NULL; } while (0); freez(lp->df_H); - freez(lp->df_Host); - freez(lp->df_Referer); freez(lp->df_U); freez(lp->df_q); - freez(lp->df_User_agent); - freez(lp->df_X_Forwarded_For); freez(lp->df_b); freez(lp->df_h); freez(lp->df_m); freez(lp->df_s); freez(lp->df_u); freez(lp->df_ttfb); + VTAILQ_FOREACH_SAFE(h, &lp->headers, list, h2) { + VTAILQ_REMOVE(&lp->headers, h, list); + freez(h->key); + freez(h->value); + freez(h); + } #undef freez memset(lp, 0, sizeof *lp); } @@ -293,17 +312,22 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, case SLT_TxHeader: if (!lp->active) break; - if (isprefix(ptr, "user-agent:", end, &next)) - lp->df_User_agent = trimline(next, end); - else if (isprefix(ptr, "referer:", end, &next)) - lp->df_Referer = trimline(next, end); - else if (isprefix(ptr, "authorization:", end, &next) && - isprefix(next, "basic", end, &next)) + if (isprefix(ptr, "authorization:", end, &next) && + isprefix(next, "basic", end, &next)) { lp->df_u = trimline(next, end); - else if (isprefix(ptr, "x-forwarded-for:", end, &next)) - lp->df_X_Forwarded_For = trimline(next, end); - else if (isprefix(ptr, "host:", end, &next)) - lp->df_Host = trimline(next, end); + } else { + struct hdr *h; + const char *split; + size_t l; + h = malloc(sizeof(struct hdr)); + AN(h); + split = strchr(ptr, ':'); + AN(split); + l = strlen(split); + h->key = trimline(ptr, split-1); + h->value = trimline(split+1, split+l-1); + VTAILQ_INSERT_HEAD(&lp->headers, h, list); + } break; case SLT_BackendReuse: @@ -394,22 +418,20 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, case SLT_RxHeader: if (!lp->active) break; - if (isprefix(ptr, "user-agent:", end, &next)) { - free(lp->df_User_agent); - lp->df_User_agent = trimline(next, end); - } else if (isprefix(ptr, "referer:", end, &next)) { - free(lp->df_Referer); - lp->df_Referer = trimline(next, end); - } else if (isprefix(ptr, "authorization:", end, &next) && - isprefix(next, "basic", end, &next)) { + if (isprefix(ptr, "authorization:", end, &next) && + isprefix(next, "basic", end, &next)) { free(lp->df_u); lp->df_u = trimline(next, end); - } else if (isprefix(ptr, "x-forwarded-for:", end, &next)) { - free(lp->df_X_Forwarded_For); - lp->df_X_Forwarded_For = trimline(next, end); - } else if (isprefix(ptr, "host:", end, &next)) { - free(lp->df_Host); - lp->df_Host = trimline(next, end); + } else { + struct hdr *h; + const char *split; + h = malloc(sizeof(struct hdr)); + AN(h); + split = strchr(ptr, ':'); + AN(split); + h->key = trimline(ptr, split); + h->value = trimline(split+1, end); + VTAILQ_INSERT_HEAD(&lp->headers, h, list); } break; @@ -579,10 +601,10 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, */ VSB_cat(os, lp->df_m); VSB_putc(os, ' '); - if (lp->df_Host) { - if (strncmp(lp->df_Host, "http://", 7) != 0) + if (get_header(lp, "Host")) { + if (strncmp(get_header(lp, "Host"), "http://", 7) != 0) VSB_cat(os, "http://"); - VSB_cat(os, lp->df_Host); + VSB_cat(os, get_header(lp, "Host")); } VSB_cat(os, lp->df_U); VSB_cat(os, lp->df_q ? lp->df_q : ""); @@ -626,42 +648,44 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, } break; - case '{': - if (strncmp(p, "{Referer}i", 10) == 0) { - VSB_cat(os, lp->df_Referer ? lp->df_Referer : "-"); - p += 9; - break; - } else if (strncmp(p, "{Host}i", 7) == 0) { - VSB_cat(os, lp->df_Host ? lp->df_Host : "-"); - p += 6; - break; - } else if (strncmp(p, "{X-Forwarded-For}i", 18) == 0) { - /* %{Referer}i */ - VSB_cat(os, lp->df_X_Forwarded_For ? lp->df_X_Forwarded_For : "-"); - p += 17; - break; - } else if (strncmp(p, "{User-agent}i", 13) == 0) { - /* %{User-agent}i */ - VSB_cat(os, lp->df_User_agent ? lp->df_User_agent : "-"); - p += 12; + case '{': { + const char *h, *tmp; + char fname[100], type; + tmp = p; + type = 0; + while (*tmp != '\0' && *tmp != '}') + tmp++; + if (*tmp == '}') { + tmp++; + type = *tmp; + memcpy(fname, p+1, tmp-p-2); + } + + switch (type) { + case 'i': + h = get_header(lp, fname); + VSB_cat(os, h ? h : "-"); + p = tmp; break; - } else if (strncmp(p, "{Varnish:", 9) == 0) { - /* Scan for what we're looking for */ - const char *what = p+9; - if (strncmp(what, "time_firstbyte}x", 16) == 0) { + case 'x': + if (strcmp(fname, "Varnish:time_firstbyte") == 0) { VSB_cat(os, lp->df_ttfb); - p += 9+15; - break; - } else if (strncmp(what, "hitmiss}x", 9) == 0) { + p = tmp; + } else if (strcmp(fname, "Varnish:hitmiss") == 0) { VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-")); - p += 9+8; + p = tmp; break; - } else if (strncmp(what, "handling}x", 10) == 0) { + } else if (strcmp(fname, "handling") == 0) { VSB_cat(os, (lp->df_handling ? lp->df_handling : "-")); - p += 9+9; + p = tmp; break; } + default: + fprintf(stderr, "Unknown format starting at: %s\n", --p); + exit(1); } + break; + } /* Fall through if we haven't handled something */ /* FALLTHROUGH*/ default: diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 414e07e..48befed 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -68,8 +68,7 @@ The following options are available: Remote host %{X}i - The contents of header line X. Supported headers are - *Referer*, *Host*, *X-Forwarded-For* and *User-agent*. + The contents of header line X. %l Remote logname (always '-') From geoff at varnish-cache.org Mon Aug 1 11:18:12 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:12 +0200 Subject: [experimental-ims] 041b457 Add support for logging response headers in varnishncsa Message-ID: commit 041b4574b85b3f6b2ec9940d311258b9cbb15357 Author: Tollef Fog Heen Date: Fri Jul 15 11:38:03 2011 +0200 Add support for logging response headers in varnishncsa diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 94c81ef..6aea3d8 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -103,7 +103,8 @@ static struct logline { int active; /* Is log line in an active trans */ int complete; /* Is log line complete */ uint64_t bitmap; /* Bitmap for regex matches */ - VTAILQ_HEAD(, hdr) headers; + VTAILQ_HEAD(, hdr) req_headers; /* Request headers */ + VTAILQ_HEAD(, hdr) resp_headers; /* Response headers */ } **ll; struct VSM_data *vd; @@ -190,10 +191,23 @@ trimline(const char *str, const char *end) } static char * -get_header(struct logline *l, const char *name) +req_header(struct logline *l, const char *name) { struct hdr *h; - VTAILQ_FOREACH(h, &l->headers, list) { + VTAILQ_FOREACH(h, &l->req_headers, list) { + if (strcasecmp(h->key, name) == 0) { + return h->value; + break; + } + } + return NULL; +} + +static char * +resp_header(struct logline *l, const char *name) +{ + struct hdr *h; + VTAILQ_FOREACH(h, &l->resp_headers, list) { if (strcasecmp(h->key, name) == 0) { return h->value; break; @@ -216,8 +230,14 @@ clean_logline(struct logline *lp) freez(lp->df_s); freez(lp->df_u); freez(lp->df_ttfb); - VTAILQ_FOREACH_SAFE(h, &lp->headers, list, h2) { - VTAILQ_REMOVE(&lp->headers, h, list); + VTAILQ_FOREACH_SAFE(h, &lp->req_headers, list, h2) { + VTAILQ_REMOVE(&lp->req_headers, h, list); + freez(h->key); + freez(h->value); + freez(h); + } + VTAILQ_FOREACH_SAFE(h, &lp->resp_headers, list, h2) { + VTAILQ_REMOVE(&lp->resp_headers, h, list); freez(h->key); freez(h->value); freez(h); @@ -326,7 +346,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, l = strlen(split); h->key = trimline(ptr, split-1); h->value = trimline(split+1, split+l-1); - VTAILQ_INSERT_HEAD(&lp->headers, h, list); + VTAILQ_INSERT_HEAD(&lp->req_headers, h, list); } break; @@ -415,10 +435,12 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_s = trimline(ptr, end); break; + case SLT_TxHeader: case SLT_RxHeader: if (!lp->active) break; - if (isprefix(ptr, "authorization:", end, &next) && + if (tag == SLT_RxHeader && + isprefix(ptr, "authorization:", end, &next) && isprefix(next, "basic", end, &next)) { free(lp->df_u); lp->df_u = trimline(next, end); @@ -431,7 +453,10 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, AN(split); h->key = trimline(ptr, split); h->value = trimline(split+1, end); - VTAILQ_INSERT_HEAD(&lp->headers, h, list); + if (tag == SLT_RxHeader) + VTAILQ_INSERT_HEAD(&lp->req_headers, h, list); + else + VTAILQ_INSERT_HEAD(&lp->resp_headers, h, list); } break; @@ -601,10 +626,10 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, */ VSB_cat(os, lp->df_m); VSB_putc(os, ' '); - if (get_header(lp, "Host")) { - if (strncmp(get_header(lp, "Host"), "http://", 7) != 0) + if (req_header(lp, "Host")) { + if (strncmp(req_header(lp, "Host"), "http://", 7) != 0) VSB_cat(os, "http://"); - VSB_cat(os, get_header(lp, "Host")); + VSB_cat(os, req_header(lp, "Host")); } VSB_cat(os, lp->df_U); VSB_cat(os, lp->df_q ? lp->df_q : ""); @@ -659,11 +684,17 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, tmp++; type = *tmp; memcpy(fname, p+1, tmp-p-2); + fname[tmp-p-2] = 0; } switch (type) { case 'i': - h = get_header(lp, fname); + h = req_header(lp, fname); + VSB_cat(os, h ? h : "-"); + p = tmp; + break; + case 'o': + h = resp_header(lp, fname); VSB_cat(os, h ? h : "-"); p = tmp; break; diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 48befed..9ddd26e 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -79,6 +79,9 @@ The following options are available: %q The query string, if no query string exists, an empty string. + %{X}o + The contents of response header line X. + %r The first line of the request From geoff at varnish-cache.org Mon Aug 1 11:18:13 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:13 +0200 Subject: [experimental-ims] 8d88729 Minor clarification about %{X}i Message-ID: commit 8d8872929bd01dc2b54a07a2e699aa6da017c25a Author: Tollef Fog Heen Date: Fri Jul 15 11:38:15 2011 +0200 Minor clarification about %{X}i diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 9ddd26e..65f08dd 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -68,7 +68,7 @@ The following options are available: Remote host %{X}i - The contents of header line X. + The contents of request header line X. %l Remote logname (always '-') From geoff at varnish-cache.org Mon Aug 1 11:18:14 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:14 +0200 Subject: [experimental-ims] 61b0dcf Revert "Print 0, not - for zero-byte replies" Message-ID: commit 61b0dcfb45c972c9319a7208c1aa4a308710568b Author: Tollef Fog Heen Date: Fri Jul 15 11:50:06 2011 +0200 Revert "Print 0, not - for zero-byte replies" "-" is correct according to the Apache LogFormat documentation on http://httpd.apache.org/docs/current/mod/mod_log_config.html This reverts commit fe7e315cb98ffdc2e4f510e43c7b54bc1af840c1. See #949 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 6aea3d8..ce58fdc 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -594,7 +594,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, case 'b': /* %b */ - VSB_cat(os, lp->df_b ? lp->df_b : "0"); + VSB_cat(os, lp->df_b ? lp->df_b : "-"); break; case 'H': From geoff at varnish-cache.org Mon Aug 1 11:18:14 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:14 +0200 Subject: [experimental-ims] 39249ec Fall back to using current time if urandom is not available Message-ID: commit 39249ecefaa8ed19bc261b6c6ae27997cf0a1a63 Author: Tollef Fog Heen Date: Fri Jul 15 12:14:08 2011 +0200 Fall back to using current time if urandom is not available diff --git a/lib/libvarnish/vtmpfile.c b/lib/libvarnish/vtmpfile.c index 60a9c21..327790a 100644 --- a/lib/libvarnish/vtmpfile.c +++ b/lib/libvarnish/vtmpfile.c @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -47,8 +48,12 @@ seed_random(void) unsigned seed; fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) - return (1); + if (fd == -1) { + /* urandom not available, fall back to something + * weaker */ + srandom(time(NULL)); + return (0); + } if (read(fd, &seed, sizeof seed) != sizeof seed) return (1); (void)close(fd); From geoff at varnish-cache.org Mon Aug 1 11:18:15 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:15 +0200 Subject: [experimental-ims] 2061b6e Adjust paths in spec file for trunk builds Message-ID: commit 2061b6e21ee799aeed9091d80c9f8397fece1b81 Author: Tollef Fog Heen Date: Fri Jul 15 12:14:16 2011 +0200 Adjust paths in spec file for trunk builds Switch back from beta2 to trunk meaning RPMs can be built directly from the tarballs again. Fixes: #955 diff --git a/redhat/varnish.spec b/redhat/varnish.spec index 7fc5a7a..d8b6fbb 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -1,12 +1,12 @@ Summary: High-performance HTTP accelerator Name: varnish Version: 3.0.0 -Release: 0.beta2%{?dist} +Release: 0.20110715%{?dist} License: BSD Group: System Environment/Daemons URL: http://www.varnish-cache.org/ #Source0: http://repo.varnish-cache.org/source/%{name}-%{version}.tar.gz -Source0: %{name}-3.0.0-beta2.tar.gz +Source0: %{name}-trunk.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # To build from git, start with a make dist, see redhat/README.redhat # You will need at least automake autoconf libtool python-docutils @@ -72,7 +72,7 @@ Documentation files for %name %prep #%setup -q -%setup -q -n varnish-3.0.0-beta2 +%setup -q -n varnish-trunk mkdir examples cp bin/varnishd/default.vcl etc/zope-plone.vcl examples From geoff at varnish-cache.org Mon Aug 1 11:18:15 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:15 +0200 Subject: [experimental-ims] d7c7c39 Increase http_resp_hdr_len and http_req_hdr_len Message-ID: commit d7c7c39abd8c56a7bce04991c7a8c35b58940dca Author: Tollef Fog Heen Date: Fri Jul 15 12:23:46 2011 +0200 Increase http_resp_hdr_len and http_req_hdr_len We're seeing at least up to 4k in the wild, so increase the limit somewhat. See: #939 diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 26fcb31..3126744 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -522,7 +522,7 @@ static const struct parspec input_parspec[] = { "Maximum length of any HTTP client request header we will " "allow. The limit is inclusive its continuation lines.\n", 0, - "2048", "bytes" }, + "4096", "bytes" }, { "http_req_size", tweak_uint, &master.http_req_size, 256, UINT_MAX, "Maximum number of bytes of HTTP client request we will deal " @@ -538,7 +538,7 @@ static const struct parspec input_parspec[] = { "Maximum length of any HTTP backend response header we will " "allow. The limit is inclusive its continuation lines.\n", 0, - "2048", "bytes" }, + "4096", "bytes" }, { "http_resp_size", tweak_uint, &master.http_resp_size, 256, UINT_MAX, "Maximum number of bytes of HTTP backend resonse we will deal " From geoff at varnish-cache.org Mon Aug 1 11:18:16 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:16 +0200 Subject: [experimental-ims] 0c5337e Change the response code on overly long request lines from 400 to 413 as per discussions in httpbis WG Message-ID: commit 0c5337e2732a94c68cb4161c47f9aaf63d4e6e3c Author: Poul-Henning Kamp Date: Mon Jul 18 10:32:45 2011 +0000 Change the response code on overly long request lines from 400 to 413 as per discussions in httpbis WG diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 5d8050a..15d1b52 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -522,7 +522,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p, if (q - p > htc->maxhdr) { VSC_C_main->losthdr++; WSL(w, SLT_LostHeader, fd, "%.*s", q - p, p); - return (400); + return (413); } /* Empty header = end of headers */ @@ -547,7 +547,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p, } else { VSC_C_main->losthdr++; WSL(w, SLT_LostHeader, fd, "%.*s", q - p, p); - return (400); + return (413); } } return (0); @@ -598,7 +598,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, hp->hd[h2].e = p; if (!Tlen(hp->hd[h2])) - return (400); + return (413); /* Skip SP */ for (; vct_issp(*p); p++) { diff --git a/bin/varnishtest/tests/c00039.vtc b/bin/varnishtest/tests/c00039.vtc index f8bba06..9339b59 100644 --- a/bin/varnishtest/tests/c00039.vtc +++ b/bin/varnishtest/tests/c00039.vtc @@ -23,7 +23,7 @@ client c1 { expect resp.status == 200 txreq -url "/1" -hdr "1...5....0....5....0....5....0....5....0." rxresp - expect resp.status == 400 + expect resp.status == 413 } -run client c1 { @@ -32,7 +32,7 @@ client c1 { expect resp.status == 200 txreq -url "/2" -hdr "1...5....0....5\n ..0....5....0....5....0." rxresp - expect resp.status == 400 + expect resp.status == 413 } -run client c1 { From geoff at varnish-cache.org Mon Aug 1 11:18:16 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:16 +0200 Subject: [experimental-ims] 4172f98 Fix a bug in the entity-replacement code in the ESI-parser. Message-ID: commit 4172f9888077d42d8c0148a353d6940764d46154 Author: Poul-Henning Kamp Date: Mon Jul 18 14:25:00 2011 +0000 Fix a bug in the entity-replacement code in the ESI-parser. Patch by: scoof Fixes #961 diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c index 411db43..afef1fb 100644 --- a/bin/varnishd/cache_esi_parse.c +++ b/bin/varnishd/cache_esi_parse.c @@ -518,7 +518,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what) #define R(w,f,r) \ if (q + w <= p + l && !memcmp(q, f, w)) { \ VSB_printf(vep->vsb, "%c", r); \ - q += l; \ + q += w; \ continue; \ } R(6, "'", '\''); diff --git a/bin/varnishtest/tests/r00961.vtc b/bin/varnishtest/tests/r00961.vtc new file mode 100644 index 0000000..b162379 --- /dev/null +++ b/bin/varnishtest/tests/r00961.vtc @@ -0,0 +1,49 @@ +varnishtest "Test XML 1.0 entity references" + +server s1 { + rxreq + expect req.url == "/" + txresp -body { + + + + + + } + + rxreq + expect req.url == "/t&t" + txresp -body "1" + + rxreq + expect req.url == "/tt" + txresp -body "333" + + rxreq + expect req.url == {/t't} + txresp -body "4444" + + rxreq + expect req.url == {/t"t} + txresp -body "55555" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (pass); + } + sub vcl_fetch { + set beresp.do_esi = true; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 32 +} -run From geoff at varnish-cache.org Mon Aug 1 11:18:16 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:16 +0200 Subject: [experimental-ims] 9f3fbdf Merged conditional backend request feature Message-ID: commit 9f3fbdff886999a6777412828b00f0f8581d5a7c Author: Geoff Simmons Date: Thu Jun 2 21:42:28 2011 +0200 Merged conditional backend request feature diff --git a/bin/varnishtest/tests/i00000.vtc b/bin/varnishtest/tests/i00000.vtc new file mode 100644 index 0000000..773e1e5 --- /dev/null +++ b/bin/varnishtest/tests/i00000.vtc @@ -0,0 +1,498 @@ +# $Id$ + +test "VCL compiler coverage test: conditional refresh extension" + +## stale_obj is r/o in miss, fetch and error, illegal everywhere else + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj = true; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.proto ~ "foo") { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.proto = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.status == 200) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.status = 200; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.response ~ "foo") { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.response = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.hits > 100) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.hits = 100; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.http.Foo ~ "foo") { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.http.Foo = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.ttl > 1m) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.ttl = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.grace > 1m) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.grace = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.lastuse > 1m) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.lastuse = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (stale_obj.keep > 1m) { return(pass); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set stale_obj.keep = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj = true; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.proto ~ "foo") { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.proto = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.status == 200) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.status = 200; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.response ~ "foo") { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.response = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.hits > 100) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.hits = 100; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.http.Foo ~ "foo") { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.http.Foo = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.ttl > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.ttl = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.grace > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.grace = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.lastuse > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.lastuse = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (stale_obj.keep > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set stale_obj.keep = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj = true; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.proto ~ "foo") { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.proto = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.status == 200) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.status = 200; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.response ~ "foo") { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.response = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.hits > 100) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.hits = 100; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.http.Foo ~ "foo") { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.http.Foo = "foo"; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.ttl > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.ttl = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.grace > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.grace = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.lastuse > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.lastuse = 1m; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (stale_obj.keep > 1m) { return(deliver); } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set stale_obj.keep = 1m; } +} + +## keep is r/w for req everywhere +## r/w for beresp in fetch +## r/w for obj in hit and error (like ttl and grace) +## r/o for stale_obj in fetch, miss and error (already tested) + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_recv { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_recv { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pipe { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pipe { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pass { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pass { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hash { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hash { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hit { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hit { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_deliver { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_deliver { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { set req.keep = 1s; } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (req.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (beresp.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set beresp.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hit { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; } + sub vcl_error { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_miss { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_fetch { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_recv { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_recv { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pipe { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pipe { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pass { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_pass { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hash { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_hash { set obj.keep = 1s; } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_deliver { if (obj.keep > 1s) { set req.keep = 1s; } } +} + +varnish v1 -badvcl { + backend b { .host = "127.0.0.1"; } + sub vcl_deliver { set obj.keep = 1s; } +} + +varnish v1 -cliok "param.show default_keep" + +varnish v1 -cliok "param.set default_keep 30s" + +## XXX: This is not caught as an error +#varnish v1 -clierr 106 "param.set default_keep foo" + diff --git a/bin/varnishtest/tests/i00001.vtc b/bin/varnishtest/tests/i00001.vtc new file mode 100644 index 0000000..488305f --- /dev/null +++ b/bin/varnishtest/tests/i00001.vtc @@ -0,0 +1,271 @@ +# $Id$ + +test "Test basic conditional requests to backends" + +## By default (default_keep = default_grace), cond. requests happen during grace + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -hdr "ETag: foo" \ + -body "123456" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + expect req.http.If-None-Match == "foo" + txresp -status 304 +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 0.5s; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 6 + expect resp.bodylen == 6 + expect resp.http.Age == 0 +} -run + +delay 0.6 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## With keep > 0, using both If-Modified-Since and If-None-Match + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -hdr "ETag: foo" \ + -body "123456" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + expect req.http.If-None-Match == "foo" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + expect req.http.If-None-Match == "foo" + txresp -status 200 \ + -hdr "Last-Modified: Wed, 27 Jun 2008 12:00:01 GMT" \ + -hdr "ETag: foo" \ + -body "ABCDEF" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Wed, 27 Jun 2008 12:00:01 GMT" + expect req.http.If-None-Match == "foo" + txresp -status 304 + +} -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_fetch { + set beresp.http.X-Original-Keep = beresp.keep; + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.ETag == "foo" + expect resp.http.content-length == 6 + expect resp.bodylen == 6 + expect resp.http.Age == 0 + expect resp.http.X-Original-Keep == 10.000 +} + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 2 +varnish v1 -expect cond_not_validated == 1 + +server s1 -wait +varnish v1 -stop + +## Just If-Modified-Since + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "123456" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 \ + -hdr "Last-Modified: Wed, 27 Jun 2008 12:00:01 GMT" \ + -body "ABCDEF" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Wed, 27 Jun 2008 12:00:01 GMT" + txresp -status 304 + +} -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + sub vcl_fetch { + set beresp.http.X-Original-Keep = beresp.keep; + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 6 + expect resp.bodylen == 6 + expect resp.http.Age == 0 + expect resp.http.X-Original-Keep == 10.000 +} + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 2 +varnish v1 -expect cond_not_validated == 1 + +server s1 -wait +varnish v1 -stop + +## Just If-None-Match + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "ETag: foo" \ + -body "123456" + + rxreq + expect req.url == "/foo" + expect req.http.If-None-Match == "foo" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-None-Match == "foo" + txresp -status 200 \ + -hdr "ETag: bar" \ + -body "ABCDEF" + + rxreq + expect req.url == "/foo" + expect req.http.If-None-Match == "bar" + txresp -status 304 + +} -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + sub vcl_fetch { + set beresp.http.X-Original-Keep = beresp.keep; + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.ETag == "foo" + expect resp.http.content-length == 6 + expect resp.bodylen == 6 + expect resp.http.Age == 0 + expect resp.http.X-Original-Keep == 10.000 +} + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.ETag == "bar" + expect resp.http.content-length == 6 + expect resp.bodylen == 6 + expect resp.http.Age == 0 + expect resp.http.X-Original-Keep == 10.000 +} + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c2 -run + +delay 1.1 + +client c2 -run + +varnish v1 -expect fetch_304 == 2 +varnish v1 -expect cond_not_validated == 1 diff --git a/bin/varnishtest/tests/i00002.vtc b/bin/varnishtest/tests/i00002.vtc new file mode 100644 index 0000000..3725532 --- /dev/null +++ b/bin/varnishtest/tests/i00002.vtc @@ -0,0 +1,108 @@ +# $Id$ + +test "handling stale_obj in vcl_miss() and vcl_fetch()" + +server s1 { + rxreq + expect req.url == "/foo" + expect req.http.X-VT-Stale == "false" + expect req.http.X-VT-Stale-Status == 503 + expect req.http.X-VT-Stale-Hits == 0 + expect req.http.X-VT-Stale-TTL == -1.000 + expect req.http.X-VT-Stale-Grace == -1.000 + expect req.http.X-VT-Stale-Last-Use == 0.000 + expect req.http.X-VT-Stale-Keep == -1.000 + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "1234567890" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + expect req.http.X-VT-Stale == "true" + expect req.http.X-VT-Stale-Proto == "HTTP/1.1" + expect req.http.X-VT-Stale-Status == 200 + expect req.http.X-VT-Stale-Response == "Ok" + expect req.http.X-VT-Stale-Last-Modified == "Thu, 26 Jun 2008 12:00:01 GMT" + expect req.http.X-VT-Stale-TTL == 0.500 + expect req.http.X-VT-Stale-Grace == 0.500 + expect req.http.X-VT-Stale-Last-Use != 0.000 + expect req.http.X-VT-Stale-Keep == 1.000 + txresp -status 304 +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.grace = 0.5s; + set beresp.ttl = 0.5s; + set beresp.keep = 1s; + + if (stale_obj) { + set beresp.http.X-VT-Stale = "true"; + } + else { + set beresp.http.X-VT-Stale = "false"; + } + set beresp.http.X-VT-Stale-Proto = stale_obj.proto; + set beresp.http.X-VT-Stale-Status = stale_obj.status; + set beresp.http.X-VT-Stale-Response = stale_obj.response; + set beresp.http.X-VT-Stale-Hits = stale_obj.hits; + set beresp.http.X-VT-Stale-Last-Modified = stale_obj.http.Last-Modified; + set beresp.http.X-VT-Stale-TTL = stale_obj.ttl; + set beresp.http.X-VT-Stale-Grace = stale_obj.grace; + set beresp.http.X-VT-Stale-Last-Use = stale_obj.lastuse; + set beresp.http.X-VT-Stale-Keep = stale_obj.keep; + } + + sub vcl_miss { + if (stale_obj) { + set bereq.http.X-VT-Stale = "true"; + } + else { + set bereq.http.X-VT-Stale = "false"; + } + set bereq.http.X-VT-Stale-Proto = stale_obj.proto; + set bereq.http.X-VT-Stale-Status = stale_obj.status; + set bereq.http.X-VT-Stale-Response = stale_obj.response; + set bereq.http.X-VT-Stale-Hits = stale_obj.hits; + set bereq.http.X-VT-Stale-Last-Modified = stale_obj.http.Last-Modified; + set bereq.http.X-VT-Stale-TTL = stale_obj.ttl; + set bereq.http.X-VT-Stale-Grace = stale_obj.grace; + set bereq.http.X-VT-Stale-Last-Use = stale_obj.lastuse; + set bereq.http.X-VT-Stale-Keep = stale_obj.keep; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 10 + expect resp.http.X-VT-Stale == "false" + expect resp.http.X-VT-Stale-Status == 503 + expect resp.http.X-VT-Stale-Hits == 0 + expect resp.http.X-VT-Stale-TTL == -1.000 + expect resp.http.X-VT-Stale-Grace == -1.000 + expect resp.http.X-VT-Stale-Last-Use == 0.000 + expect resp.http.X-VT-Stale-Keep == -1.000 +} -run + +delay 1.1 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 10 + expect resp.http.Content-Length == 10 + expect resp.http.X-VT-Stale == "true" + expect resp.http.X-VT-Stale-Proto == "HTTP/1.1" + expect resp.http.X-VT-Stale-Status == 200 + expect resp.http.X-VT-Stale-Response == "Ok" + expect resp.http.X-VT-Stale-Hits == 0 + expect resp.http.X-VT-Stale-Last-Modified == "Thu, 26 Jun 2008 12:00:01 GMT" + expect resp.http.X-VT-Stale-TTL == 0.500 + expect resp.http.X-VT-Stale-Grace == 0.500 + expect resp.http.X-VT-Stale-Last-Use != 0.000 + expect resp.http.X-VT-Stale-Keep == 1.000 +} -run diff --git a/bin/varnishtest/tests/i00003.vtc b/bin/varnishtest/tests/i00003.vtc new file mode 100644 index 0000000..809b317 --- /dev/null +++ b/bin/varnishtest/tests/i00003.vtc @@ -0,0 +1,146 @@ +# $Id$ + +test "Test some anticipated use cases for conditional backend requests" + +## In vcl_miss(), it is possible to veto a conditional request by removing any +## If-Modified-Since or If-None-Match header. + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -hdr "ETag: foo" \ + -body "abcdefghijklmonpqrstuvwxyz" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + expect req.http.If-None-Match != "foo" + txresp -status 200 -body "abcdefghijklmonpqrstuvwxyz" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } + + sub vcl_miss { + unset bereq.http.If-Modified-Since; + unset bereq.http.If-None-Match; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 26 +} + +client c1 -run + +delay 1.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 0 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## Verify that if a client sends a conditional request to Varnish, then Varnish +## can return a 304 response to the client after it got 304 from the backend + +server s2 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 +} -start + +varnish v1 -vcl { + backend s2 { + .host = "${s2_addr}"; .port = "${s2_port}"; + } + + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } +} -start + +client c2 { + txreq -url "/foo" \ + -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT" + rxresp + expect resp.status == 304 +} + +client c1 -run + +delay 1.1 + +client c2 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s2 -wait +varnish v1 -stop + +## +## If stale_obj has a gzipped body, make sure it interacts correctly with clients +## + +server s2 { + rxreq + expect req.http.accept-encoding == "gzip" + txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" -gzipbody FOO + + rxreq + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 +} -start + +varnish v1 -cliok "param.set http_gzip_support true" -vcl { + backend s2 { + .host = "${s2_addr}"; .port = "${s2_port}"; + } + + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } +} -start + +client c1 { + txreq -hdr "Accept-encoding: gzip;q=0.1" + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.bodylen == "3" +} -run + +delay 1.1 + +client c2 { + txreq + rxresp + expect resp.bodylen == "3" + expect resp.http.content-encoding != "gzip" +} -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 diff --git a/bin/varnishtest/tests/i00004.vtc b/bin/varnishtest/tests/i00004.vtc new file mode 100644 index 0000000..0943f98 --- /dev/null +++ b/bin/varnishtest/tests/i00004.vtc @@ -0,0 +1,201 @@ +# $Id$ + +test "Verify the semantics of keep (timeout for conditional requests)" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "foo bar baz quux" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 -body "foo bar baz quux" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 16 + expect resp.http.Age == 0 +} + +varnish v1 -cli "param.set default_keep 1" + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 2.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## Start a new varnish (now using the default default_keep), +## and manipulate beresp.keep in VCL + +server s1 -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1s; + } +} -start + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 2.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## Manipulate obj.keep in vcl_hit() + +server s1 -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + } + + sub vcl_hit { + set obj.keep = 1s; + } +} -start + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 2.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## Manipulate obj.keep in vcl_error() + +server s1 -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + } + + sub vcl_hit { + error 200 "Ok"; + } + + sub vcl_error { + set obj.keep = 1s; + return(deliver); + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 2.1 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## req.keep sets an upper bound for all *.keeps in a session + +server s1 -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_recv { + set req.keep = 0.5s; + } + + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1s; + } +} -start + +client c1 -run + +delay 1.1 + +client c1 -run + +delay 1.6 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 diff --git a/bin/varnishtest/tests/i00005.vtc b/bin/varnishtest/tests/i00005.vtc new file mode 100644 index 0000000..0bcac4f --- /dev/null +++ b/bin/varnishtest/tests/i00005.vtc @@ -0,0 +1,147 @@ +# $Id$ + +test "Verify interactions of ttl, keep, grace and bans" + +## Banned objects are not used for conditional requests + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "The quick brown fox jumps over the lazy dog." + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 -body "The quick brown fox jumps over the lazy dog." +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.grace = 0s; + set beresp.ttl = 1s; + set beresp.keep = 1m; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 44 +} + +client c1 -run + +delay 1.1 + +varnish v1 -cli "ban.url \"/foo\"" + +client c1 -run + +varnish v1 -expect fetch_304 == 0 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## If keep is set to non-zero, but then ttl is set to 0, +## then the object is not used for conditional requests + +server s1 -start + +varnish v1 -vcl { + backend s2 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_fetch { + set beresp.keep = 1m; + set beresp.ttl = 0s; + } +} -start + +client c1 -run + +client c1 -run + +varnish v1 -expect fetch_304 == 0 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## If ttl is set to 0 but keep is then set to non-zero, +## then the object is used for conditional requests +## First test using beresp in vcl_fetch() + +server s2 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "The quick brown fox jumps over the lazy dog." + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 +} -start + +varnish v1 -vcl { + backend s2 { + .host = "${s2_addr}"; .port = "${s2_port}"; + } + + # NB: return(deliver) necessary here, because default vcl_fetch() returns + # hit_for_pass if beresp.ttl <= 0 + sub vcl_fetch { + set beresp.ttl = 0s; + set beresp.keep = 1m; + return (deliver); + } +} -start + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 44 +} -run + +delay 0.5 + +client c2 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s2 -wait +varnish v1 -stop + +## Verify using obj.ttl in vcl_hit() + +server s2 -start + +varnish v1 -vcl { + backend s2 { + .host = "${s2_addr}"; .port = "${s2_port}"; + } + + sub vcl_hit { + set obj.ttl = 0s; + set obj.keep = 1m; + } +} -start + +client c1 -run + +client c1 -run + +delay 0.5 + +client c1 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 diff --git a/bin/varnishtest/tests/i00006.vtc b/bin/varnishtest/tests/i00006.vtc new file mode 100644 index 0000000..87c6192 --- /dev/null +++ b/bin/varnishtest/tests/i00006.vtc @@ -0,0 +1,156 @@ +# $Id$ + +test "Verify effects of ttl, keep and grace on expiration" + +## Verify that an object's lifetime in the cache is +## obj.ttl + max(obj.grace, obj.keep) +## First with grace < keep + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "fred garply waldo xyzzy" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 -body "fred garply waldo xyzzy" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 0.5s; + set beresp.grace = 0.5s; + set beresp.keep = 1s; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 23 +} -run + +delay 1.1 + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 23 + expect resp.msg == "Ok Not Modified" +} -run + +delay 1.6 + +client c3 { + txreq -url "/foo" + rxresp + expect resp.msg == "Ok" +} -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## Now with keep < grace + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "fred garply waldo xyzzy" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 -body "fred garply waldo xyzzy" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 1s; + set beresp.grace = 1s; + set beresp.keep = 0.5s; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 23 +} -run + +delay 1.1 + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 23 + expect resp.msg == "Ok Not Modified" +} -run + +delay 1.6 + +client c3 { + txreq -url "/foo" + rxresp + expect resp.msg == "Ok" +} -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 + +server s1 -wait +varnish v1 -stop + +## Verify that req.keep sets an upper bound in the session + +server s1 -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; .port = "${s1_port}"; + } + + sub vcl_recv { + set req.keep = 1s; + } + + sub vcl_fetch { + set beresp.ttl = 1s; + set beresp.grace = 0s; + set beresp.keep = 2s; + } +} -start + +client c1 -run + +delay 1.1 + +client c2 -run + +delay 2.1 + +client c3 -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 0 diff --git a/bin/varnishtest/tests/i00007.vtc b/bin/varnishtest/tests/i00007.vtc new file mode 100644 index 0000000..487c0d0 --- /dev/null +++ b/bin/varnishtest/tests/i00007.vtc @@ -0,0 +1,129 @@ +# $Id$ + +test "Passes through responses to backend conditionals to the client if status != 304 or 200" + +# Testing a sample from each of the Nxx status codes + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "Varnish has poked you" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 304 + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 201 -msg "Created" \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "Varnish has poked you" + + # 3xx responses typically do not include Last-Modified or a body + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 301 -msg "Moved Permanently" + + # Restore a cached object with Last-Modified + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "Varnish has poked you" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 400 -msg "Bad Request" \ + -body "Varnish has poked you" + + # Restore a cached object with Last-Modified + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 200 \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "Varnish has poked you" + + rxreq + expect req.url == "/foo" + expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT" + txresp -status 500 -msg "Internal Server Error" \ + -body "Varnish has poked you" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 1s; + set beresp.grace = 0s; + set beresp.keep = 1s; + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 21 +} -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 201 + expect resp.msg == "Created" + expect resp.http.Last-Modified == "Thu, 26 Jun 2008 12:00:01 GMT" + expect resp.bodylen == 21 +} -run + +delay 1.1 + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 301 + expect resp.msg == "Moved Permanently" +} -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 400 + expect resp.msg == "Bad Request" + expect resp.bodylen == 21 +} -run + +delay 1.1 + +client c1 -run + +delay 1.1 + +client c2 { + txreq -url "/foo" + rxresp + expect resp.status == 500 + expect resp.msg == "Internal Server Error" + expect resp.bodylen == 21 +} -run + +varnish v1 -expect fetch_304 == 1 +varnish v1 -expect cond_not_validated == 4 From geoff at varnish-cache.org Mon Aug 1 11:18:16 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:16 +0200 Subject: [experimental-ims] ec2c6ca Merged conditional backend request feature Message-ID: commit ec2c6cac843d99220f13ee638154469b87dddd60 Author: Geoff Simmons Date: Thu Jun 2 21:47:07 2011 +0200 Merged conditional backend request feature diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index a875071..8515c5e 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -596,6 +596,7 @@ struct sess { struct objcore *objcore; struct VCL_conf *vcl; + struct object *stale_obj; /* The busy objhead we sleep on */ struct objhead *hash_objhead; @@ -753,6 +754,12 @@ void http_SetResp(struct http *to, const char *proto, int status, void http_FilterFields(struct worker *w, int fd, struct http *to, const struct http *fm, unsigned how); void http_FilterHeader(const struct sess *sp, unsigned how); + +/* Check if a refresh should be done */ +void http_CheckRefresh(struct sess *sp); +/* Check if we got 304 response */ +void http_Check304(struct sess *sp); + void http_PutProtocol(struct worker *w, int fd, const struct http *to, const char *protocol); void http_PutStatus(struct http *to, int status); diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 23ad92f..6455985 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -558,6 +558,7 @@ cnt_fetch(struct sess *sp) sp->wrk->age = 0; EXP_Clr(&sp->wrk->exp); sp->wrk->exp.ttl = RFC2616_Ttl(sp); + sp->wrk->exp.keep = params->default_keep; /* pass from vclrecv{} has negative TTL */ if (sp->objcore == NULL) @@ -638,7 +639,7 @@ cnt_fetchbody(struct sess *sp) int i; struct http *hp, *hp2; char *b; - unsigned l, nhttp; + unsigned l, nhttp, stale_nhttp; struct vsb *vary = NULL; int varyl = 0, pass; @@ -724,6 +725,10 @@ cnt_fetchbody(struct sess *sp) l = http_EstimateWS(sp->wrk->beresp, pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); + if (sp->stale_obj) { + l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp); + nhttp += stale_nhttp; + } /* Create Vary instructions */ if (sp->objcore != NULL) { @@ -763,6 +768,7 @@ cnt_fetchbody(struct sess *sp) return (0); } CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + sp->obj->exp.keep = sp->wrk->exp.keep; sp->wrk->storage_hint = NULL; @@ -789,11 +795,29 @@ cnt_fetchbody(struct sess *sp) hp2->logtag = HTTP_Obj; http_CopyResp(hp2, hp); - http_FilterFields(sp->wrk, sp->fd, hp2, hp, - pass ? HTTPH_R_PASS : HTTPH_A_INS); - http_CopyHome(sp->wrk, sp->fd, hp2); - - if (http_GetHdr(hp, H_Last_Modified, &b)) + + http_FilterFields(sp->wrk, sp->fd, hp2, hp, + pass ? HTTPH_R_PASS : HTTPH_A_INS); + + /* + * If we found a candidate for conditional backend request, attempt it + * now. If backend responds with 304, http_Check304() merges stale_obj + * into sp->obj, any other response is handled as usual. In either case, + * the stale_obj is no longer needed in the cache, so discard it. + */ + if (sp->stale_obj) { + http_Check304(sp); + if (sp->wrk->beresp->status == 304) + assert(sp->obj->http->status == 200); + EXP_Clr(&sp->stale_obj->exp); + EXP_Rearm(sp->stale_obj); + HSH_Deref(sp->wrk, NULL, &sp->stale_obj); + AZ(sp->stale_obj); + } + http_CopyHome(sp->wrk, sp->fd, hp2); + + if (http_GetHdr(hp, H_Last_Modified, &b) + || http_GetHdr(sp->obj->http, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); else sp->obj->last_modified = floor(sp->wrk->entered); @@ -1089,6 +1113,8 @@ cnt_lookup(struct sess *sp) sp->wrk->stats.cache_hitpass++; WSP(sp, SLT_HitPass, "%u", sp->obj->xid); (void)HSH_Deref(sp->wrk, NULL, &sp->obj); + if (sp->stale_obj != NULL) + (void)HSH_Deref(sp->wrk, NULL, &sp->stale_obj); sp->objcore = NULL; sp->step = STP_PASS; return (0); @@ -1149,6 +1175,13 @@ cnt_miss(struct sess *sp) sp->wrk->connect_timeout = 0; sp->wrk->first_byte_timeout = 0; sp->wrk->between_bytes_timeout = 0; + + /* If a candidate for a conditional backend request was found, + * add If-Modified-Since and/or If-None-Match to the bereq. + */ + if (sp->stale_obj) + http_CheckRefresh(sp); + VCL_miss_method(sp); switch(sp->handling) { case VCL_RET_ERROR: diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c index 1063c6b..b3704b4 100644 --- a/bin/varnishd/cache_fetch.c +++ b/bin/varnishd/cache_fetch.c @@ -498,8 +498,13 @@ FetchBody(struct sess *sp) AN(sp->director); AssertObjPassOrBusy(sp->obj); + /* If we've freshened from another object and got a "Not Modified" + * response, then we have already duped the other object's body. + */ + if (sp->wrk->beresp->status != 304) + AZ(VTAILQ_FIRST(&sp->obj->store)); + AZ(sp->wrk->vgz_rx); - AZ(VTAILQ_FIRST(&sp->obj->store)); switch (sp->wrk->body_status) { case BS_NONE: cls = 0; diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 6fdff15..7f9eacd 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -305,13 +305,16 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) struct objcore *oc; struct objcore *busy_oc, *grace_oc; struct object *o; - double grace_ttl; + struct object *stale_o; /* for freshness check */ + double grace_ttl, stale_ttl; + char *p; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC); AN(hash); + AZ(sp->stale_obj); w = sp->wrk; HSH_Prealloc(sp); @@ -339,7 +342,9 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) assert(oh->refcnt > 0); busy_oc = NULL; grace_oc = NULL; + stale_o = NULL; /* for freshness check */ grace_ttl = NAN; + stale_ttl = NAN; VTAILQ_FOREACH(oc, &oh->objcs, list) { /* Must be at least our own ref + the objcore we examine */ assert(oh->refcnt > 1); @@ -362,7 +367,8 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) o = oc_getobj(sp->wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - if (o->exp.ttl <= 0.) + if (o->exp.ttl <= 0. && o->exp.grace <= 0. + && o->exp.keep <= 0.) continue; if (BAN_CheckObject(o, sp)) continue; @@ -384,6 +390,24 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) grace_ttl = o->entered + o->exp.ttl; } } + + /* At this point we know: + * - o's TTL has elapsed + * - o is not busy or banned, + * - o is not a Vary match. + * The object may be used for a conditional backend request if + * - the keep time has not elapsed, and + * - it has a Last-Modified and/or an ETag header. + * If there are several, use the least expired one. + */ + if (EXP_Keep(sp, o) >= sp->t_req + && (http_GetHdr(o->http, H_Last_Modified, &p) + || http_GetHdr(o->http, H_ETag, &p))) + if (stale_o == NULL || stale_ttl < o->entered + o->exp.ttl) { + stale_o = o; + stale_ttl = o->entered + o->exp.ttl; + } + } /* @@ -451,6 +475,18 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) return (NULL); } + /* If we're not serving a valid or graced object and we saved stale_o, + * it is a candidate for the conditional backend request. */ + AZ(oc && !sp->hash_always_miss); + AZ(busy_oc); + if (stale_o != NULL) { + AZ(stale_o->objcore->flags & OC_F_BUSY); + CHECK_OBJ_NOTNULL(stale_o->objcore, OBJCORE_MAGIC); + Lck_AssertHeld(&oh->mtx); + stale_o->objcore->refcnt++; + sp->stale_obj = stale_o; + } + /* Insert (precreated) objcore in objecthead */ oc = w->nobjcore; w->nobjcore = NULL; diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 15d1b52..c467c31 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -40,6 +40,8 @@ #include "vct.h" #include "cache.h" +#include "hash_slinger.h" +#include "stevedore.h" #define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":"; #include "http_headers.h" @@ -65,6 +67,10 @@ static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = { /*lint -restore */ static enum VSL_tag_e + +void http_FilterMissingFields(struct worker *w, int fd, struct http *to, + const struct http *fm); + http2shmlog(const struct http *hp, int t) { @@ -848,6 +854,36 @@ http_FilterFields(struct worker *w, int fd, struct http *to, } } +/*--------------------------------------------------------------------- + * Same as http_FilterFields but keep any existing hdrs in fm. + * Furthermore, before copy, check if fm already has that hdr, and if so + * do not copy. Used for 304 refresh processing. + */ + +/* XXX: uplex/GS: Also, don't filter according to the "how" bitmap in + * http_headers.h. We only use this to copy from one cached object to + * another, so if a header made into the first object, we want it. + */ + +void +http_FilterMissingFields(struct worker *w, int fd, struct http *to, + const struct http *fm) +{ + unsigned u; + unsigned hdrlen; + + CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) { + if (fm->hd[u].b == NULL) + continue; + hdrlen = strchr(fm->hd[u].b, ':') - fm->hd[u].b; + if (http_findhdr(to, hdrlen, fm->hd[u].b)) + continue; + http_copyheader(w, fd, to, fm, u); + } +} + /*--------------------------------------------------------------------*/ void @@ -869,6 +905,89 @@ http_FilterHeader(const struct sess *sp, unsigned how) http_PrintfHeader(sp->wrk, sp->fd, hp, "X-Varnish: %u", sp->xid); } +/*------------------------------------------------------------------- + * This function checks for sp->freshen_obj. If present, HSH_Lookup() + * found an expired object that qualifies for a refresh check, + * so add the appropriate headers. + */ + +void +http_CheckRefresh(struct sess *sp) +{ + struct object *freshen_obj; + struct http *obj_hp, *bereq_hp; + char *p; + + freshen_obj = sp->stale_obj; + CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC); + bereq_hp = sp->wrk->bereq; + CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC); + obj_hp = freshen_obj->http; + CHECK_OBJ_NOTNULL(obj_hp, HTTP_MAGIC); + + if(http_GetHdr(obj_hp, H_ETag, &p)) + http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-None-Match: %s", p); + + if(http_GetHdr(obj_hp, H_Last_Modified, &p)) + http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-Modified-Since: %s",p); +} + +/*------------------------------------------------------------------- + * Called after fetch and sp->freshen_obj present. Check + * response and handle as needed. + */ + +void +http_Check304(struct sess *sp) +{ + struct object *o, *o_stale; + char *p; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + o_stale = sp->stale_obj; + CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC); + o = sp->obj; + CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + + if (sp->wrk->beresp->status != 304) { + /* + * IMS/INM headers may have been removed in VCL, so only count a + * non-validating response if they were present in the request. + */ + if (http_GetHdr(sp->wrk->bereq, H_If_Modified_Since, &p) + || http_GetHdr(sp->wrk->bereq, H_If_None_Match, &p)) + sp->wrk->stats.cond_not_validated++; + return; + } + + /* + * Copy headers we need from the stale object into the 304 response + */ + http_FilterMissingFields(sp->wrk, sp->fd, sp->obj->http, + sp->stale_obj->http); + + /* + * Dup the stale object's storage in to the new object + * and reset Content-Length from the size of the storage. + */ + STV_dup(sp, o_stale, o); + http_Unset(o->http, H_Content_Length); + http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u", o->len); + + http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified"); + http_SetH(o->http, HTTP_HDR_REQ, "GET"); + http_copyh(o->http, sp->wrk->bereq, HTTP_HDR_URL); + + /* + * XXX: Are we copying all the necessary fields from stale_obj? + * Should we copy o_stale->hits into o->hits? + */ + o->response = 200; + o->gziped = o_stale->gziped; + + AZ(o_stale->objcore->flags & OC_F_BUSY); +} + /*-------------------------------------------------------------------- * This function copies any header fields which reference foreign * storage into our own WS. diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c index 65b7522..07adeac 100644 --- a/bin/varnishd/cache_vrt.c +++ b/bin/varnishd/cache_vrt.c @@ -112,6 +112,10 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where) CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); hp = sp->obj->http; break; + case HDR_STALE_OBJ: + CHECK_OBJ_NOTNULL(sp->stale_obj, OBJECT_MAGIC); + hp = sp->stale_obj->http; + break; default: INCOMPL(); } @@ -126,6 +130,11 @@ VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n) struct http *hp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (where == HDR_STALE_OBJ && sp->stale_obj == NULL) { + WSP(sp, SLT_VCL_error, + "stale_obj does not exist (reading header %s)", n); + return NULL; + } hp = vrt_selecthttp(sp, where); if (!http_GetHdr(hp, n, &p)) return (NULL); diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c index 297151c..a6c6c57 100644 --- a/bin/varnishd/cache_vrt_var.c +++ b/bin/varnishd/cache_vrt_var.c @@ -40,6 +40,9 @@ #include "cache_backend.h" #include "hash_slinger.h" +#define ILLEGAL_R(sess, obj, field) \ +WSP(sess, SLT_VCL_error, "%s does not exist (reading field %s)", obj, field) + static char vrt_hostname[255] = ""; /*--------------------------------------------------------------------*/ @@ -61,7 +64,7 @@ vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld, va_end(ap); } -#define VRT_DO_HDR(obj, hdr, http, fld) \ +#define VRT_DO_HDR_l(obj, hdr, cont, http, fld) \ void \ VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...) \ { \ @@ -69,53 +72,74 @@ VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...) \ \ va_start(ap, p); \ vrt_do_string(sp->wrk, sp->fd, \ - http, fld, #obj "." #hdr, p, ap); \ + cont->http, fld, #obj "." #hdr, p, ap); \ va_end(ap); \ -} \ - \ +} + +#define VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable) \ const char * \ VRT_r_##obj##_##hdr(const struct sess *sp) \ { \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(http, HTTP_MAGIC); \ - return (http->hd[fld].b); \ -} - -VRT_DO_HDR(req, request, sp->http, HTTP_HDR_REQ) -VRT_DO_HDR(req, url, sp->http, HTTP_HDR_URL) -VRT_DO_HDR(req, proto, sp->http, HTTP_HDR_PROTO) -VRT_DO_HDR(bereq, request, sp->wrk->bereq, HTTP_HDR_REQ) -VRT_DO_HDR(bereq, url, sp->wrk->bereq, HTTP_HDR_URL) -VRT_DO_HDR(bereq, proto, sp->wrk->bereq, HTTP_HDR_PROTO) -VRT_DO_HDR(obj, proto, sp->obj->http, HTTP_HDR_PROTO) -VRT_DO_HDR(obj, response, sp->obj->http, HTTP_HDR_RESPONSE) -VRT_DO_HDR(resp, proto, sp->wrk->resp, HTTP_HDR_PROTO) -VRT_DO_HDR(resp, response, sp->wrk->resp, HTTP_HDR_RESPONSE) -VRT_DO_HDR(beresp, proto, sp->wrk->beresp, HTTP_HDR_PROTO) -VRT_DO_HDR(beresp, response, sp->wrk->beresp, HTTP_HDR_RESPONSE) + if (!nullable || cont != NULL) { \ + CHECK_OBJ_NOTNULL(cont->http, HTTP_MAGIC); \ + return (cont->http->hd[fld].b); \ + } \ + ILLEGAL_R(sp, #obj, #hdr); \ + return(NULL); \ +} \ + +#define VRT_DO_HDR(obj, hdr, cont, http, fld, nullable) \ +VRT_DO_HDR_l(obj, hdr, cont, http, fld) \ +VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable) \ + +VRT_DO_HDR(req, request, sp, http, HTTP_HDR_REQ, 0) +VRT_DO_HDR(req, url, sp, http, HTTP_HDR_URL, 0) +VRT_DO_HDR(req, proto, sp, http, HTTP_HDR_PROTO, 0) +VRT_DO_HDR(bereq, request, sp->wrk, bereq, HTTP_HDR_REQ, 0) +VRT_DO_HDR(bereq, url, sp->wrk, bereq, HTTP_HDR_URL, 0) +VRT_DO_HDR(bereq, proto, sp->wrk, bereq, HTTP_HDR_PROTO, 0) +VRT_DO_HDR(obj, proto, sp->obj, http, HTTP_HDR_PROTO, 0) +VRT_DO_HDR(obj, response, sp->obj, http, HTTP_HDR_RESPONSE, 0) +VRT_DO_HDR(resp, proto, sp->wrk, resp, HTTP_HDR_PROTO, 0) +VRT_DO_HDR(resp, response, sp->wrk, resp, HTTP_HDR_RESPONSE, 0) +VRT_DO_HDR(beresp, proto, sp->wrk, beresp, HTTP_HDR_PROTO, 0) +VRT_DO_HDR(beresp, response, sp->wrk, beresp, HTTP_HDR_RESPONSE, 0) +VRT_DO_HDR_r(stale_obj, proto, sp->stale_obj, http, HTTP_HDR_PROTO, 1) +VRT_DO_HDR_r(stale_obj, response, sp->stale_obj, http, HTTP_HDR_RESPONSE, 1) /*--------------------------------------------------------------------*/ -#define VRT_DO_STATUS(obj, http) \ +#define VRT_DO_STATUS_l(obj, cont, http) \ void \ VRT_l_##obj##_status(const struct sess *sp, int num) \ { \ \ assert(num >= 100 && num <= 999); \ - http->status = num; \ -} \ - \ + cont->http->status = num; \ +} + +#define VRT_DO_STATUS_r(obj, cont, http, nullable) \ int \ VRT_r_##obj##_status(const struct sess *sp) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return(http->status); \ + if (nullable && cont == NULL) { \ + ILLEGAL_R(sp, #obj, "status"); \ + return (503); \ + } \ + return(cont->http->status); \ } -VRT_DO_STATUS(obj, sp->obj->http) -VRT_DO_STATUS(beresp, sp->wrk->beresp) -VRT_DO_STATUS(resp, sp->wrk->resp) +#define VRT_DO_STATUS(obj, cont, http, nullable) \ +VRT_DO_STATUS_l(obj, cont, http) \ +VRT_DO_STATUS_r(obj, cont, http, nullable) \ + +VRT_DO_STATUS(obj, sp->obj, http, 0) +VRT_DO_STATUS(beresp, sp->wrk, beresp, 0) +VRT_DO_STATUS(resp, sp->wrk, resp, 0) +VRT_DO_STATUS_r(stale_obj, sp->stale_obj, http, 1) /*--------------------------------------------------------------------*/ @@ -362,37 +386,45 @@ VRT_r_req_restarts(const struct sess *sp) /*--------------------------------------------------------------------*/ -#define VRT_DO_EXP(which, exp, fld, extra) \ - \ -void __match_proto__() \ -VRT_l_##which##_##fld(struct sess *sp, double a) \ -{ \ - \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - EXP_Set_##fld(&exp, a); \ - extra; \ -} \ - \ -double __match_proto__() \ -VRT_r_##which##_##fld(struct sess *sp) \ -{ \ - \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return(EXP_Get_##fld(&exp)); \ -} - -VRT_DO_EXP(req, sp->exp, ttl, ) -VRT_DO_EXP(req, sp->exp, grace, ) -VRT_DO_EXP(req, sp->exp, keep, ) -VRT_DO_EXP(obj, sp->obj->exp, grace, EXP_Rearm(sp->obj)) -VRT_DO_EXP(obj, sp->obj->exp, ttl, - EXP_Rearm(sp->obj); - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->obj->xid, a, sp->t_req)) -VRT_DO_EXP(obj, sp->obj->exp, keep, EXP_Rearm(sp->obj)) -VRT_DO_EXP(beresp, sp->wrk->exp, grace, ) -VRT_DO_EXP(beresp, sp->wrk->exp, ttl, - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req)) -VRT_DO_EXP(beresp, sp->wrk->exp, keep, ) +#define VRT_DO_EXP_l(which, cont, fld, extra) \ +void __match_proto__() \ +VRT_l_##which##_##fld(struct sess *sp, double a) \ +{ \ + \ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + EXP_Set_##fld(&cont->exp, a); \ + extra; \ +} + +#define VRT_DO_EXP_r(which, cont, fld, nullable) \ +double __match_proto__() \ +VRT_r_##which##_##fld(struct sess *sp) \ +{ \ + \ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + if (nullable && cont == NULL) { \ + ILLEGAL_R(sp, #which, #fld); \ + return (-1); \ + } \ + return(EXP_Get_##fld(&cont->exp)); \ +} + +#define VRT_DO_EXP(which, cont, fld, nullable, extra) \ +VRT_DO_EXP_l(which, cont, fld, extra) \ +VRT_DO_EXP_r(which, cont, fld, nullable) \ + +VRT_DO_EXP(req, sp, ttl, 0, ) +VRT_DO_EXP(req, sp, grace, 0, ) +VRT_DO_EXP(req, sp, keep, 0, ) +VRT_DO_EXP(obj, sp->obj, grace, 0, EXP_Rearm(sp->obj)) +VRT_DO_EXP(obj, sp->obj, ttl, 0, EXP_Rearm(sp->obj)) +VRT_DO_EXP(obj, sp->obj, keep, 0, EXP_Rearm(sp->obj)) +VRT_DO_EXP(beresp, sp->wrk, grace, 0, ) +VRT_DO_EXP(beresp, sp->wrk, ttl, 0, ) +VRT_DO_EXP(beresp, sp->wrk, keep, 0, ) +VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 1) +VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 1) +VRT_DO_EXP_r(stale_obj, sp->stale_obj, keep, 1) /*-------------------------------------------------------------------- * req.xid @@ -493,6 +525,8 @@ VRT_r_server_port(struct sess *sp) /*--------------------------------------------------------------------*/ +/* XXX: uplex/GS: a nice macro would eliminate the repetition here ... */ + int VRT_r_obj_hits(const struct sess *sp) { @@ -502,6 +536,19 @@ VRT_r_obj_hits(const struct sess *sp) return (sp->obj->hits); } +int +VRT_r_stale_obj_hits(const struct sess *sp) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (sp->stale_obj == NULL) { + ILLEGAL_R(sp, "stale_obj", "hits"); + return (0); + } + CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC); /* XXX */ + return (sp->stale_obj->hits); +} + double VRT_r_obj_lastuse(const struct sess *sp) { @@ -511,6 +558,19 @@ VRT_r_obj_lastuse(const struct sess *sp) return (TIM_real() - sp->obj->last_use); } +double +VRT_r_stale_obj_lastuse(const struct sess *sp) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (sp->stale_obj == NULL) { + ILLEGAL_R(sp, "stale_obj", "lastuse"); + return (0); + } + CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC); /* XXX */ + return (TIM_real() - sp->stale_obj->last_use); +} + unsigned VRT_r_req_backend_healthy(const struct sess *sp) { @@ -519,3 +579,9 @@ VRT_r_req_backend_healthy(const struct sess *sp) return (VDI_Healthy(sp->director, sp)); } +unsigned +VRT_r_stale_obj(const struct sess *sp) +{ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + return (sp->stale_obj != NULL); +} diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 3126744..0eb67cf 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -582,13 +582,13 @@ static const struct parspec input_parspec[] = { "10", "seconds" }, { "default_keep", tweak_timeout_double, &master.default_keep, 0, UINT_MAX, - "Default keep period. We will keep a useless object " + "Default keep period. We will keep a stale object " "around this long, making it available for conditional " "backend fetches. " "That means that the object will be removed from the " - "cache at the end of ttl+grace+keep.", + "cache at the end of ttl+max(grace,keep).", DELAYED_EFFECT, - "0", "seconds" }, + "10", "seconds" }, { "sess_timeout", tweak_timeout, &master.sess_timeout, 0, 0, "Idle timeout for persistent sessions. " "If a HTTP request has not been received in this many " diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 413e0b6..7c1665e 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -370,6 +370,30 @@ STV_trim(struct storage *st, size_t size) st->stevedore->trim(st, size); } +/* + * Duplicate the object storage (HTML body) from src into target, using a + * stevedore-specific dup method for src's stevedore. + * + * Currently, every method just copies storage from one object to the other, + * but this method of encapsulation opens the path to future techniques of + * sharing storage together with reference counting. + */ +void +STV_dup(const struct sess *sp, struct object *src, struct object *target) +{ + struct stevedore *stv; + + CHECK_OBJ_NOTNULL(src, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(target, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(src->objstore, STORAGE_MAGIC); + CHECK_OBJ_NOTNULL(src->objstore->stevedore, STEVEDORE_MAGIC); + + stv = src->objstore->stevedore; + AN(stv->dup); + + stv->dup(sp, src, target); +} + void STV_free(struct storage *st) { @@ -586,3 +610,24 @@ VRT_Stv_##nm(const char *nm) \ #include "vrt_stv_var.h" #undef VRTSTVVAR + +/* + * Default object store dup just copies the storage. + */ +void +default_dup(const struct sess *sp, struct object *src, struct object *target) +{ + struct storage *st, *st2; + unsigned cl; + + VTAILQ_FOREACH(st2, &src->store, list) { + cl = st2->len; + st = STV_alloc(sp, cl); + XXXAN(st); + assert(st->space >= cl); + VTAILQ_INSERT_TAIL(&target->store, st, list); + st->len = cl; + target->len += cl; + memcpy(st->ptr, st2->ptr, cl); + } +} diff --git a/bin/varnishd/stevedore.h b/bin/varnishd/stevedore.h index 5a5cc4b..1c0d2ee 100644 --- a/bin/varnishd/stevedore.h +++ b/bin/varnishd/stevedore.h @@ -39,6 +39,7 @@ struct stv_objsecrets; typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); +typedef void storage_dup_f(const struct sess *sp, struct object *src, struct object *target); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp, @@ -70,6 +71,7 @@ struct stevedore { storage_open_f *open; /* called by cache process */ storage_alloc_f *alloc; /* --//-- */ storage_trim_f *trim; /* --//-- */ + storage_dup_f *dup; /* --//-- */ storage_free_f *free; /* --//-- */ storage_close_f *close; /* --//-- */ storage_allocobj_f *allocobj; /* --//-- */ @@ -93,6 +95,7 @@ struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot, struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len, struct exp *, unsigned nhttp); struct storage *STV_alloc(const struct sess *sp, size_t size); +void STV_dup(const struct sess *sp, struct object *src, struct object *target); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); void STV_open(void); @@ -117,3 +120,6 @@ extern const struct stevedore smp_stevedore; #ifdef HAVE_LIBUMEM extern const struct stevedore smu_stevedore; #endif + +/* Default dup method */ +void default_dup(const struct sess *sp, struct object *src, struct object *target); diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c index 27cc86f..84e56ce 100644 --- a/bin/varnishd/storage_file.c +++ b/bin/varnishd/storage_file.c @@ -551,6 +551,7 @@ const struct stevedore smf_stevedore = { .alloc = smf_alloc, .trim = smf_trim, .free = smf_free, + .dup = default_dup, }; #ifdef INCLUDE_TEST_DRIVER diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 12a390f..2e9b81f 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -221,4 +221,5 @@ const struct stevedore sma_stevedore = { .trim = sma_trim, .var_free_space = sma_free_space, .var_used_space = sma_used_space, + .dup = default_dup, }; diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c index a4ba598..4cc0eca 100644 --- a/bin/varnishd/storage_persistent.c +++ b/bin/varnishd/storage_persistent.c @@ -567,6 +567,7 @@ const struct stevedore smp_stevedore = { .allocobj = smp_allocobj, .free = smp_free, .trim = smp_trim, + .dup = default_dup, }; /*-------------------------------------------------------------------- diff --git a/bin/varnishd/storage_synth.c b/bin/varnishd/storage_synth.c index 24479e3..3ddab87 100644 --- a/bin/varnishd/storage_synth.c +++ b/bin/varnishd/storage_synth.c @@ -67,6 +67,7 @@ static struct stevedore sms_stevedore = { .magic = STEVEDORE_MAGIC, .name = "synth", .free = sms_free, + .dup = default_dup, }; struct vsb * diff --git a/bin/varnishd/storage_umem.c b/bin/varnishd/storage_umem.c index 6e28a94..474f809 100644 --- a/bin/varnishd/storage_umem.c +++ b/bin/varnishd/storage_umem.c @@ -163,6 +163,7 @@ const struct stevedore smu_stevedore = { .alloc = smu_alloc, .free = smu_free, .trim = smu_trim, + .dup = default_dup, }; #endif /* HAVE_UMEM_H */ diff --git a/include/vrt.h b/include/vrt.h index cfafe75..9d9ce16 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -159,7 +159,7 @@ int VRT_rewrite(const char *, const char *); void VRT_error(struct sess *, unsigned, const char *); int VRT_switch_config(const char *); -enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP }; +enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_STALE_OBJ, HDR_BEREQ, HDR_BERESP }; char *VRT_GetHdr(const struct sess *, enum gethdr_e where, const char *); void VRT_SetHdr(const struct sess *, enum gethdr_e where, const char *, const char *, ...); diff --git a/include/vsc_fields.h b/include/vsc_fields.h index 0289b81..534501a 100644 --- a/include/vsc_fields.h +++ b/include/vsc_fields.h @@ -155,6 +155,8 @@ VSC_F(vmods, uint64_t, 0, 'i', "Loaded VMODs") VSC_F(n_gzip, uint64_t, 0, 'a', "Gzip operations") VSC_F(n_gunzip, uint64_t, 0, 'a', "Gunzip operations") +VSC_F(cond_not_validated, uint64_t, 1, 'a', "Non-validating responses") + #endif /**********************************************************************/ diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py index 5407a51..28375fa 100755 --- a/lib/libvcl/generate.py +++ b/lib/libvcl/generate.py @@ -427,6 +427,66 @@ sp_variables = ( ( ), 'const struct sess *' ), + ('stale_obj', + 'BOOL', + ( 'error', 'miss', 'fetch',), + ( ), + 'const struct sess *' + ), + ('stale_obj.proto', + 'STRING', + ( 'error', 'miss', 'fetch',), + ( ), + 'const struct sess *' + ), + ('stale_obj.status', + 'INT', + ( 'error', 'miss', 'fetch',), + ( ), + 'const struct sess *' + ), + ('stale_obj.response', + 'STRING', + ( 'error', 'miss', 'fetch',), + ( ), + 'const struct sess *' + ), + ('stale_obj.hits', + 'INT', + ( 'error', 'miss', 'fetch',), + ( ), + 'const struct sess *' + ), + ('stale_obj.http.', + 'HDR_STALE_OBJ', + ( 'error', 'miss', 'fetch',), + ( ), # XXX ? + 'const struct sess *' + ), + ('stale_obj.ttl', + 'DURATION', + ( 'error', 'miss', 'fetch',), + ( ), + 'struct sess *' + ), + ('stale_obj.grace', + 'DURATION', + ( 'error', 'miss', 'fetch',), + ( ), + 'struct sess *' + ), + ('stale_obj.lastuse', + 'DURATION', + ( 'error', 'miss', 'fetch',), + ( ), + 'const struct sess *' + ), + ('stale_obj.keep', + 'DURATION', + ( 'error', 'miss', 'fetch',), + ( ), + 'struct sess *' + ), ('resp.proto', 'STRING', ( 'deliver',), From geoff at varnish-cache.org Mon Aug 1 11:18:19 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:19 +0200 Subject: [experimental-ims] 0017c88 Merge conditional backend requests with current trunk Message-ID: commit 0017c88129ded4062b8297b8c53b8c0e652eb65e Author: Geoff Simmons Date: Tue Jun 7 06:59:57 2011 +0200 Merge conditional backend requests with current trunk diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 04110c2..dd1124e 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -70,10 +70,6 @@ void http_FilterMissingFields(struct worker *w, int fd, struct http *to, const struct http *fm); static enum VSL_tag_e - -void http_FilterMissingFields(struct worker *w, int fd, struct http *to, - const struct http *fm); - http2shmlog(const struct http *hp, int t) { From geoff at varnish-cache.org Mon Aug 1 11:18:17 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:17 +0200 Subject: [experimental-ims] cd2eb9c Merge conditional backend requests with current trunk Message-ID: commit cd2eb9c21ecec1f6379cc2baf274156e2f29a6a1 Author: Geoff Simmons Date: Tue Jun 7 06:59:57 2011 +0200 Merge conditional backend requests with current trunk diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 8515c5e..d80dd74 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -701,6 +701,7 @@ void EXP_Set_keep(struct exp *e, double v); double EXP_Ttl(const struct sess *, const struct object*); double EXP_Grace(const struct sess *, const struct object*); +double EXP_Keep(const struct sess *, const struct object*); void EXP_Insert(struct object *o); void EXP_Inject(struct objcore *oc, struct lru *lru, double when); void EXP_Init(void); diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c index c0e79cc..5fc944a 100644 --- a/bin/varnishd/cache_expire.c +++ b/bin/varnishd/cache_expire.c @@ -97,7 +97,7 @@ EXP_ACCESS(keep, 0.,) * by per-session limits. */ -static double +double EXP_Keep(const struct sess *sp, const struct object *o) { double r; diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index c467c31..dd1124e 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -66,11 +66,10 @@ static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = { }; /*lint -restore */ -static enum VSL_tag_e - void http_FilterMissingFields(struct worker *w, int fd, struct http *to, const struct http *fm); +static enum VSL_tag_e http2shmlog(const struct http *hp, int t) { diff --git a/bin/varnishtest/tests/i00000.vtc b/bin/varnishtest/tests/i00000.vtc index 773e1e5..4c4aeb5 100644 --- a/bin/varnishtest/tests/i00000.vtc +++ b/bin/varnishtest/tests/i00000.vtc @@ -1,6 +1,6 @@ # $Id$ -test "VCL compiler coverage test: conditional refresh extension" +varnishtest "VCL compiler coverage test: conditional refresh extension" ## stale_obj is r/o in miss, fetch and error, illegal everywhere else diff --git a/bin/varnishtest/tests/i00001.vtc b/bin/varnishtest/tests/i00001.vtc index 488305f..306cc57 100644 --- a/bin/varnishtest/tests/i00001.vtc +++ b/bin/varnishtest/tests/i00001.vtc @@ -1,6 +1,6 @@ # $Id$ -test "Test basic conditional requests to backends" +varnishtest "Test basic conditional requests to backends" ## By default (default_keep = default_grace), cond. requests happen during grace diff --git a/bin/varnishtest/tests/i00002.vtc b/bin/varnishtest/tests/i00002.vtc index 3725532..adae568 100644 --- a/bin/varnishtest/tests/i00002.vtc +++ b/bin/varnishtest/tests/i00002.vtc @@ -1,6 +1,6 @@ # $Id$ -test "handling stale_obj in vcl_miss() and vcl_fetch()" +varnishtest "handling stale_obj in vcl_miss() and vcl_fetch()" server s1 { rxreq diff --git a/bin/varnishtest/tests/i00003.vtc b/bin/varnishtest/tests/i00003.vtc index 809b317..293e843 100644 --- a/bin/varnishtest/tests/i00003.vtc +++ b/bin/varnishtest/tests/i00003.vtc @@ -1,6 +1,6 @@ # $Id$ -test "Test some anticipated use cases for conditional backend requests" +varnishtest "Test some anticipated use cases for conditional backend requests" ## In vcl_miss(), it is possible to veto a conditional request by removing any ## If-Modified-Since or If-None-Match header. diff --git a/bin/varnishtest/tests/i00004.vtc b/bin/varnishtest/tests/i00004.vtc index 0943f98..bb70176 100644 --- a/bin/varnishtest/tests/i00004.vtc +++ b/bin/varnishtest/tests/i00004.vtc @@ -1,6 +1,6 @@ # $Id$ -test "Verify the semantics of keep (timeout for conditional requests)" +varnishtest "Verify the semantics of keep (timeout for conditional requests)" server s1 { rxreq diff --git a/bin/varnishtest/tests/i00005.vtc b/bin/varnishtest/tests/i00005.vtc index 0bcac4f..983afb2 100644 --- a/bin/varnishtest/tests/i00005.vtc +++ b/bin/varnishtest/tests/i00005.vtc @@ -1,6 +1,6 @@ # $Id$ -test "Verify interactions of ttl, keep, grace and bans" +varnishtest "Verify interactions of ttl, keep, grace and bans" ## Banned objects are not used for conditional requests diff --git a/bin/varnishtest/tests/i00006.vtc b/bin/varnishtest/tests/i00006.vtc index 87c6192..db15078 100644 --- a/bin/varnishtest/tests/i00006.vtc +++ b/bin/varnishtest/tests/i00006.vtc @@ -1,6 +1,6 @@ # $Id$ -test "Verify effects of ttl, keep and grace on expiration" +varnishtest "Verify effects of ttl, keep and grace on expiration" ## Verify that an object's lifetime in the cache is ## obj.ttl + max(obj.grace, obj.keep) diff --git a/bin/varnishtest/tests/i00007.vtc b/bin/varnishtest/tests/i00007.vtc index 487c0d0..4a9b8c0 100644 --- a/bin/varnishtest/tests/i00007.vtc +++ b/bin/varnishtest/tests/i00007.vtc @@ -1,6 +1,6 @@ # $Id$ -test "Passes through responses to backend conditionals to the client if status != 304 or 200" +varnishtest "Passes through responses to backend conditionals to the client if status != 304 or 200" # Testing a sample from each of the Nxx status codes From geoff at varnish-cache.org Mon Aug 1 11:18:19 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:19 +0200 Subject: [experimental-ims] a36f0ed Merge branch 'experimental-ims' of ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims Message-ID: commit a36f0ed4a2ef671fea62dd9b6a32a899fae8d7da Merge: 0017c88 e1406a8 Author: Geoff Simmons Date: Mon Aug 1 13:16:45 2011 +0200 Merge branch 'experimental-ims' of ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims Conflicts: bin/varnishd/cache_vrt_var.c From phk at varnish-cache.org Mon Aug 1 11:18:54 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 01 Aug 2011 13:18:54 +0200 Subject: [master] 2357a95 Fix a classic programmer bug, spotted by the increasingly eagle-eyed LLVM C-compiler. Message-ID: commit 2357a951fcffb0294f93e1488f8b03fec1ca4936 Author: Poul-Henning Kamp Date: Mon Aug 1 11:16:14 2011 +0000 Fix a classic programmer bug, spotted by the increasingly eagle-eyed LLVM C-compiler. You have no idea how much or how long (20+ years!) I have missed having an alternative to GCC. Remember: Mono cultures are particularly bad for you, when they are good for you Kudos to: The LLVM team. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index e0f7c21..2b6e3d4 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -206,7 +206,7 @@ varnish_thread(void *priv) (void)VTCP_nonblocking(v->fds[0]); while (1) { fds = &fd; - memset(fds, 0, sizeof fds); + memset(fds, 0, sizeof *fds); fds->fd = v->fds[0]; fds->events = POLLIN; i = poll(fds, 1, 1000); From phk at varnish-cache.org Mon Aug 1 11:32:33 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 01 Aug 2011 13:32:33 +0200 Subject: [master] 2f10ef7 Another insufficiently wide memset spotted by LLVM. Message-ID: commit 2f10ef7d8a1b44a56d816fcae6ba5b7c339ea273 Author: Poul-Henning Kamp Date: Mon Aug 1 11:20:48 2011 +0000 Another insufficiently wide memset spotted by LLVM. diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c index ea8d6e2..552c0be 100644 --- a/bin/varnishd/storage_persistent_subr.c +++ b/bin/varnishd/storage_persistent_subr.c @@ -67,7 +67,7 @@ smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx, AZ(off & 7); /* Alignment */ assert(strlen(id) < sizeof ctx->ss->ident); - memset(ctx, 0, sizeof ctx); + memset(ctx, 0, sizeof *ctx); ctx->ss = (void*)(sc->base + off); ctx->unique = sc->unique; ctx->id = id; From phk at varnish-cache.org Mon Aug 1 11:32:34 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 01 Aug 2011 13:32:34 +0200 Subject: [master] f96ecd7 And a third case of sizeof missing dereference spotted by LLVM Message-ID: commit f96ecd748277cc9581cdb81c711d678b2986b9bb Author: Poul-Henning Kamp Date: Mon Aug 1 11:32:08 2011 +0000 And a third case of sizeof missing dereference spotted by LLVM diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 15d1b52..2141cc4 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -911,9 +911,11 @@ http_ClrHeader(struct http *to) { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - /* XXX: don't to->f = to->v; it would kill pipelining */ to->nhd = HTTP_HDR_FIRST; - memset(to->hd, 0, sizeof to->hd); + to->status = 0; + to->protover = 0; + to->conds = 0; + memset(to->hd, 0, sizeof *to->hd * to->shd); } /*--------------------------------------------------------------------*/ From geoff at varnish-cache.org Mon Aug 1 11:18:18 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:18 +0200 Subject: [experimental-ims] d41a659 Merged conditional backend request feature Message-ID: commit d41a659d3634499b9c18c6f5788639c7a2ea4a66 Author: Geoff Simmons Date: Thu Jun 2 21:47:07 2011 +0200 Merged conditional backend request feature diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index dd1124e..04110c2 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -70,6 +70,10 @@ void http_FilterMissingFields(struct worker *w, int fd, struct http *to, const struct http *fm); static enum VSL_tag_e + +void http_FilterMissingFields(struct worker *w, int fd, struct http *to, + const struct http *fm); + http2shmlog(const struct http *hp, int t) { diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c index a6c6c57..0a22e5f 100644 --- a/bin/varnishd/cache_vrt_var.c +++ b/bin/varnishd/cache_vrt_var.c @@ -417,10 +417,13 @@ VRT_DO_EXP(req, sp, ttl, 0, ) VRT_DO_EXP(req, sp, grace, 0, ) VRT_DO_EXP(req, sp, keep, 0, ) VRT_DO_EXP(obj, sp->obj, grace, 0, EXP_Rearm(sp->obj)) -VRT_DO_EXP(obj, sp->obj, ttl, 0, EXP_Rearm(sp->obj)) +RT_DO_EXP(obj, sp->obj, ttl, 0, + EXP_Rearm(sp->obj); + WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->obj->xid, a, sp->t_req)) VRT_DO_EXP(obj, sp->obj, keep, 0, EXP_Rearm(sp->obj)) VRT_DO_EXP(beresp, sp->wrk, grace, 0, ) -VRT_DO_EXP(beresp, sp->wrk, ttl, 0, ) +VRT_DO_EXP(beresp, sp->wrk, ttl, 0, + WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req)) VRT_DO_EXP(beresp, sp->wrk, keep, 0, ) VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 1) VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 1) From geoff at varnish-cache.org Mon Aug 1 11:18:18 2011 From: geoff at varnish-cache.org (Geoff Simmons) Date: Mon, 01 Aug 2011 13:18:18 +0200 Subject: [experimental-ims] f1951c4 Seed random() on startup from /dev/urandom so vtmpfile actually returns random file names Message-ID: commit f1951c46026a144ee65b2a7114080ca22866d487 Author: Martin Blix Grydeland Date: Fri Jun 10 09:43:48 2011 +0200 Seed random() on startup from /dev/urandom so vtmpfile actually returns random file names diff --git a/lib/libvarnish/vtmpfile.c b/lib/libvarnish/vtmpfile.c index 327790a..c998fc2 100644 --- a/lib/libvarnish/vtmpfile.c +++ b/lib/libvarnish/vtmpfile.c @@ -48,12 +48,8 @@ seed_random(void) unsigned seed; fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) { - /* urandom not available, fall back to something - * weaker */ - srandom(time(NULL)); - return (0); - } + if (fd == -1) + return (1); if (read(fd, &seed, sizeof seed) != sizeof seed) return (1); (void)close(fd); From tfheen at varnish-cache.org Mon Aug 1 12:06:21 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Mon, 01 Aug 2011 14:06:21 +0200 Subject: [master] 8888447 Make VSL_Open not open the shmlog if run with -r Message-ID: commit 88884473af98bc9a3d32f352726e974e3b995023 Author: Tollef Fog Heen Date: Mon Aug 1 13:53:23 2011 +0200 Make VSL_Open not open the shmlog if run with -r diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 4141c28..b06c4df 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -375,9 +375,11 @@ VSL_Open(struct VSM_data *vd, int diag) vsl = vd->vsl; CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); - i = VSM_Open(vd, diag); - if (i) - return (i); + if (vsl->r_fd == -1) { + i = VSM_Open(vd, diag); + if (i) + return (i); + } if (!vsl->d_opt && vsl->r_fd == -1) { while (*vsl->log_ptr != VSL_ENDMARKER) From tfheen at varnish-cache.org Mon Aug 1 12:06:26 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Mon, 01 Aug 2011 14:06:26 +0200 Subject: [master] 66101b9 Fix up reading of saved log files Message-ID: commit 66101b90620c1ba08b1e60a059d38835c2395ccd Author: Tollef Fog Heen Date: Mon Aug 1 13:53:31 2011 +0200 Fix up reading of saved log files Make sure we compensate for sizeof(int) and the stuff we have already read. Fixes: #848 diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index b06c4df..95e8cae 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -165,7 +165,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp) vsl->rbuflen = l; } i = read(vsl->r_fd, vsl->rbuf + 2, l * 4L - 8L); - if (i != l) + if (i != (l * 4L - 8L)) return (-1); *pp = vsl->rbuf; return (1); From tfheen at varnish-cache.org Mon Aug 1 13:04:32 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Mon, 01 Aug 2011 15:04:32 +0200 Subject: [master] 8d662f9 Only strip out -p to status/killproc for old fedora/RHEL Message-ID: commit 8d662f9c5be529c5f0ab0a3c1da6efda51a62b5d Author: Tollef Fog Heen Date: Mon Aug 1 14:53:25 2011 +0200 Only strip out -p to status/killproc for old fedora/RHEL It seems most RPM-based distros has the -p switch to killproc and status those days, so only blacklist the ones we know about that does not have it. Fixes: #969 diff --git a/redhat/varnish.spec b/redhat/varnish.spec index d8b6fbb..ca282e2 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -110,7 +110,7 @@ EOF tail -n +11 etc/default.vcl >> redhat/default.vcl -%if 0%{?fedora}%{?rhel} == 0 || 0%{?rhel} <= 4 && 0%{?fedora} <= 8 +%if 0%{?fedora}%{?rhel} != 0 && 0%{?rhel} <= 4 && 0%{?fedora} <= 8 # Old style daemon function sed -i 's,--pidfile \$pidfile,,g; s,status -p \$pidfile,status,g; From phk at varnish-cache.org Tue Aug 2 07:48:54 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Tue, 02 Aug 2011 09:48:54 +0200 Subject: [master] 2c74419 Minor polish to the malloc allocator: Message-ID: commit 2c74419d94b00bcb44c1e1cbd41501240b51b807 Author: Poul-Henning Kamp Date: Tue Aug 2 07:48:02 2011 +0000 Minor polish to the malloc allocator: Ensure that stats are correct if allocations fail. Don't rely on the VSC counter for checking the limit. diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 12a390f..56b4daa 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -44,6 +44,7 @@ struct sma_sc { #define SMA_SC_MAGIC 0x1ac8a345 struct lock sma_mtx; size_t sma_max; + size_t sma_alloc; struct VSC_C_sma *stats; }; @@ -59,14 +60,16 @@ static struct storage * sma_alloc(struct stevedore *st, size_t size) { struct sma_sc *sma_sc; - struct sma *sma; + struct sma *sma = NULL; + void *p; CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); Lck_Lock(&sma_sc->sma_mtx); sma_sc->stats->nreq++; - if (sma_sc->stats->nbytes + size > sma_sc->sma_max) + if (sma_sc->sma_alloc + size > sma_sc->sma_max) size = 0; else { + sma_sc->sma_alloc += size; sma_sc->stats->nobj++; sma_sc->stats->nbytes += size; sma_sc->stats->balloc += size; @@ -82,17 +85,26 @@ sma_alloc(struct stevedore *st, size_t size) * performance-wise it would be a catastropy with chunksized * allocations growing another full page, just to accomodate the sma. */ - ALLOC_OBJ(sma, SMA_MAGIC); - if (sma == NULL) - return (NULL); /* XXX: stats suffer */ + + p = malloc(size); + if (p != NULL) { + ALLOC_OBJ(sma, SMA_MAGIC); + if (sma != NULL) + sma->s.ptr = p; + else + free(p); + } + if (sma == NULL) { + Lck_Lock(&sma_sc->sma_mtx); + sma_sc->stats->nobj--; + sma_sc->stats->nbytes -= size; + sma_sc->stats->balloc -= size; + Lck_Unlock(&sma_sc->sma_mtx); + return (NULL); + } sma->sc = sma_sc; sma->sz = size; sma->s.priv = sma; - sma->s.ptr = malloc(size); - if (sma->s.ptr == NULL) { - free(sma); - return (NULL); /* XXX: stats suffer */ - } sma->s.len = 0; sma->s.space = size; #ifdef SENDFILE_WORKS @@ -114,6 +126,7 @@ sma_free(struct storage *s) sma_sc = sma->sc; assert(sma->sz == sma->s.space); Lck_Lock(&sma_sc->sma_mtx); + sma_sc->sma_alloc -= sma->sz; sma_sc->stats->nobj--; sma_sc->stats->nbytes -= sma->sz; sma_sc->stats->bfree += sma->sz; @@ -152,7 +165,7 @@ sma_used_space(const struct stevedore *st) struct sma_sc *sma_sc; CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); - return (sma_sc->stats->nbytes); + return (sma_sc->sma_alloc); } static double @@ -161,7 +174,7 @@ sma_free_space(const struct stevedore *st) struct sma_sc *sma_sc; CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); - return (sma_sc->sma_max - sma_sc->stats->nbytes); + return (sma_sc->sma_max - sma_sc->sma_alloc); } static void @@ -193,10 +206,7 @@ sma_init(struct stevedore *parent, int ac, char * const *av) ARGV_ERR("(-smalloc) size \"%s\": too small, " "did you forget to specify M or G?\n", av[0]); - printf("SMA.%s: max size %ju MB.\n", parent->ident, - u / (1024 * 1024)); sc->sma_max = u; - } static void From phk at varnish-cache.org Tue Aug 2 09:49:36 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Tue, 02 Aug 2011 11:49:36 +0200 Subject: [master] beb0c5b Cap the TTL (to param "shortlived") when we use the Transient storage to avoid dropping an object on out of storage conditions. Message-ID: commit beb0c5b1f4f49d711822e90ca73d69bbed683a71 Author: Poul-Henning Kamp Date: Tue Aug 2 09:48:16 2011 +0000 Cap the TTL (to param "shortlived") when we use the Transient storage to avoid dropping an object on out of storage conditions. I belive this... Fixes #953 Otherwise please reopen. diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 23ad92f..4bac471 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -754,7 +754,8 @@ cnt_fetchbody(struct sess *sp) */ sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l, &sp->wrk->exp, nhttp); - sp->wrk->exp.ttl = params->shortlived; + if (sp->wrk->exp.ttl > params->shortlived) + sp->wrk->exp.ttl = params->shortlived; } if (sp->obj == NULL) { sp->err_code = 503; From tfheen at varnish-cache.org Tue Aug 2 12:06:33 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 02 Aug 2011 14:06:33 +0200 Subject: [master] b5c9ba0 Be more aggressive in getting rid of threads and slower in adding threads, hopefully fixing c00002 Message-ID: commit b5c9ba0e7c087bede4506b948b78755682eeb373 Author: Tollef Fog Heen Date: Tue Aug 2 14:05:15 2011 +0200 Be more aggressive in getting rid of threads and slower in adding threads, hopefully fixing c00002 diff --git a/bin/varnishtest/tests/c00002.vtc b/bin/varnishtest/tests/c00002.vtc index 62eaf17..88862b3 100644 --- a/bin/varnishtest/tests/c00002.vtc +++ b/bin/varnishtest/tests/c00002.vtc @@ -5,7 +5,7 @@ server s1 { txresp -hdr "Connection: close" -body "012345\n" } -start -varnish v1 -arg "-p thread_pool_min=2 -p thread_pool_max=8 -p thread_pools=4 -p thread_pool_purge_delay=10" +varnish v1 -arg "-p thread_pool_min=2 -p thread_pool_max=8 -p thread_pools=4 -p thread_pool_purge_delay=100 -p thread_pool_timeout=1 -p thread_pool_add_delay=100" varnish v1 -vcl+backend {} -start From tfheen at varnish-cache.org Tue Aug 2 12:57:21 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 02 Aug 2011 14:57:21 +0200 Subject: [master] f19fab6 Fix typo in help text Message-ID: commit f19fab60f1de45262588d03ba10bf1c95fff08d1 Author: Tollef Fog Heen Date: Tue Aug 2 14:13:36 2011 +0200 Fix typo in help text diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 8775950..b280022 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -154,7 +154,7 @@ usage(void) fprintf(stderr, FMT, "-l", "Leave /tmp/vtc.* if test fails"); fprintf(stderr, FMT, "-L", "Always leave /tmp/vtc.*"); fprintf(stderr, FMT, "-n iterations", "Run tests this many times"); - fprintf(stderr, FMT, "-q", "Quiet mode: report only failues"); + fprintf(stderr, FMT, "-q", "Quiet mode: report only failures"); fprintf(stderr, FMT, "-t duration", "Time tests out after this long"); fprintf(stderr, FMT, "-v", "Verbose mode: always report test log"); fprintf(stderr, "\n"); From tfheen at varnish-cache.org Tue Aug 2 12:57:21 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 02 Aug 2011 14:57:21 +0200 Subject: [master] a39f3ee Reset bereq http struct on restart from vcl_miss and vcl_pass Message-ID: commit a39f3ee67c0a88bb3b5a0b97a9070bcc6a11f92a Author: Tollef Fog Heen Date: Tue Aug 2 14:55:21 2011 +0200 Reset bereq http struct on restart from vcl_miss and vcl_pass Thanks a lot to David for minimised test case showing the bug. Fixes: #965 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 4bac471..eb0af3e 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -1155,6 +1155,7 @@ cnt_miss(struct sess *sp) case VCL_RET_ERROR: AZ(HSH_Deref(sp->wrk, sp->objcore, NULL)); sp->objcore = NULL; + http_Setup(sp->wrk->bereq, NULL); sp->step = STP_ERROR; return (0); case VCL_RET_PASS: @@ -1223,6 +1224,7 @@ cnt_pass(struct sess *sp) sp->wrk->between_bytes_timeout = 0; VCL_pass_method(sp); if (sp->handling == VCL_RET_ERROR) { + http_Setup(sp->wrk->bereq, NULL); sp->step = STP_ERROR; return (0); } diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc new file mode 100644 index 0000000..f8228a9 --- /dev/null +++ b/bin/varnishtest/tests/r00965.vtc @@ -0,0 +1,52 @@ +varnishtest "restart in vcl_miss #965" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.X-Banned == "check") { remove req.http.X-Banned; } + elseif (req.restarts == 0) { + set req.http.X-Banned = "check"; + if (req.http.x-pass) { + return (pass); + } else { + return (lookup); + } + } + } + + sub vcl_hash { + ## Check if they have a ban in the cache, or if they are going to be banned in cache. + if (req.http.X-Banned) { + hash_data(client.ip); + return (hash); + } + } + + sub vcl_error { + if (obj.status == 988) { return (restart); } + } + + sub vcl_miss { + if (req.http.X-Banned == "check") { error 988 "restarting"; } + } + + sub vcl_pass { + if (req.http.X-Banned == "check") { error 988 "restarting"; } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + txreq + rxresp + expect resp.status == 200 + txreq -hdr "X-Pass: 1" + rxresp + expect resp.status == 200 +} -run From phk at varnish-cache.org Tue Aug 2 13:04:17 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Tue, 02 Aug 2011 15:04:17 +0200 Subject: [master] 59a0967 Slam grace and keep to zero if we fall back to a transient object. Message-ID: commit 59a09675463c3b4aa5a72a4814f788b4d4f2d32f Author: Poul-Henning Kamp Date: Tue Aug 2 13:03:38 2011 +0000 Slam grace and keep to zero if we fall back to a transient object. Reminded by: Tollef Related to: #953 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index eb0af3e..f93a61b 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -756,6 +756,8 @@ cnt_fetchbody(struct sess *sp) &sp->wrk->exp, nhttp); if (sp->wrk->exp.ttl > params->shortlived) sp->wrk->exp.ttl = params->shortlived; + sp->wrk->exp.grace = 0.0; + sp->wrk->exp.keep = 0.0; } if (sp->obj == NULL) { sp->err_code = 503; From martin at varnish-cache.org Wed Aug 3 10:36:11 2011 From: martin at varnish-cache.org (Martin Blix Grydeland) Date: Wed, 03 Aug 2011 12:36:11 +0200 Subject: [master] e93ef71 Free the workers busyobj if any on cleanup Message-ID: commit e93ef7195576db670f0f4f716362b4964e83439b Author: Martin Blix Grydeland Date: Tue Aug 2 23:02:42 2011 +0200 Free the workers busyobj if any on cleanup diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 6fdff15..92ee92f 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -144,6 +144,10 @@ HSH_Cleanup(struct worker *w) free(w->nhashpriv); w->nhashpriv = NULL; } + if (w->nbusyobj != NULL) { + FREE_OBJ(w->nbusyobj); + w->nbusyobj = NULL; + } } void From martin at varnish-cache.org Wed Aug 3 10:36:12 2011 From: martin at varnish-cache.org (Martin Blix Grydeland) Date: Wed, 03 Aug 2011 12:36:12 +0200 Subject: [master] 627964a Make HSH_Deref return the busyobj to the worker for reuse when dereferencing and destroying a busy objcore. Message-ID: commit 627964a38390334a7b584a88277e36395ec5bb8b Author: Martin Blix Grydeland Date: Tue Aug 2 23:04:14 2011 +0200 Make HSH_Deref return the busyobj to the worker for reuse when dereferencing and destroying a busy objcore. Free the busyobj in the case that the worker already has one (potential race condition from the expiry/ban-lurker threads?) diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 92ee92f..0ca8766 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -718,6 +718,16 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo) BAN_DestroyObj(oc); AZ(oc->ban); + if (oc->flags & OC_F_BUSY) { + CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); + if (w->nbusyobj == NULL) + w->nbusyobj = oc->busyobj; + else + FREE_OBJ(oc->busyobj); + oc->busyobj = NULL; + } + AZ(oc->busyobj); + if (oc->methods != NULL) { oc_freeobj(oc); w->stats.n_object--; From kristian at varnish-cache.org Thu Aug 4 14:48:33 2011 From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=) Date: Thu, 04 Aug 2011 16:48:33 +0200 Subject: [master] 53a842b Add test-case for #971 Message-ID: commit 53a842bc2eee26b6cf960f91b0f3b80c81ae499a Author: Kristian Lyngstol Date: Thu Aug 4 16:48:20 2011 +0200 Add test-case for #971 diff --git a/bin/varnishtest/tests/r00971.vtc b/bin/varnishtest/tests/r00971.vtc new file mode 100644 index 0000000..b508cd7 --- /dev/null +++ b/bin/varnishtest/tests/r00971.vtc @@ -0,0 +1,22 @@ + +varnishtest "Test DNS director order" + +varnish v1 -vcl+backend { + + backend test { + .host = "192.168.0.1"; + } + + director foo dns { + { .backend = { .host = "127.0.0.1";} } + } + + sub vcl_recv { + set req.backend = foo; + if (req.http.x-aa) { + set req.backend = test; + } + } + +} -start + From tfheen at varnish-cache.org Fri Aug 5 07:22:21 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Fri, 05 Aug 2011 09:22:21 +0200 Subject: [master] 97ff8c4 Report dlerror if dlopen fails Message-ID: commit 97ff8c4b29a01cb5f3e1c0463a326347f109793b Author: Tollef Fog Heen Date: Fri Aug 5 09:14:23 2011 +0200 Report dlerror if dlopen fails dlopen typically only fails here if the child process does not have access to the build directory and the user runs varnishtest as root (meaning the child setuids to nobody). Report the dlerror and give a hopefully helpful hint to help diagnose the error. Fixes: #959 diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c index 9844413..4a4d230 100644 --- a/bin/varnishd/cache_vrt_vmod.c +++ b/bin/varnishd/cache_vrt_vmod.c @@ -83,7 +83,11 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path) REPLACE(v->path, path); v->hdl = dlopen(v->path, RTLD_NOW | RTLD_LOCAL); - AN(v->hdl); + if (! v->hdl) { + char buf[1024]; + sprintf(buf, "dlopen failed (child process lacks permission?): %.512s", dlerror()); + VAS_Fail(__func__, __FILE__, __LINE__, buf, 0, 0); + } x = dlsym(v->hdl, "Vmod_Name"); AN(x); From phk at varnish-cache.org Mon Aug 8 12:34:17 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 08 Aug 2011 14:34:17 +0200 Subject: [master] 573e86f Don't core dump on negative or execessively large fds in "order mode" Message-ID: commit 573e86f890ab3d96fa9fe7bff3f83b7ab69997ba Author: Poul-Henning Kamp Date: Mon Aug 8 12:33:41 2011 +0000 Don't core dump on negative or execessively large fds in "order mode" XXX: the 64k fd limitation should probably be fixed. diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index e33afba..ed29865 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -97,6 +97,10 @@ h_order(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len, struct VSM_data *vd = priv; + /* XXX: Just ignore any fd not inside the bitmap */ + if (fd >= sizeof bitmap / sizeof bitmap[0]) + return (0); + bitmap[fd] |= bm; type = (spec & VSL_S_CLIENT) ? 'c' : From phk at varnish-cache.org Mon Aug 8 12:35:26 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 08 Aug 2011 14:35:26 +0200 Subject: [master] 8510961 Fix the boundary detection code in vsl.c, slightly more paranoid than DocWilcos proposed fix in #956 Message-ID: commit 85109613095793ac0650615edfc3e84c004b4d30 Author: Poul-Henning Kamp Date: Mon Aug 8 12:34:38 2011 +0000 Fix the boundary detection code in vsl.c, slightly more paranoid than DocWilcos proposed fix in #956 diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 95e8cae..cc1fc0b 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -173,20 +173,30 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp) for (w = 0; w < TIMEOUT_USEC;) { t = *vsl->log_ptr; - if (t == VSL_WRAPMARKER || - (t == VSL_ENDMARKER && vsl->last_seq != vsl->log_start[0])) { + if (t == VSL_WRAPMARKER) { + /* Wrap around not possible at front */ + assert(vsl->log_ptr != vsl->log_start + 1); vsl->log_ptr = vsl->log_start + 1; - vsl->last_seq = vsl->log_start[0]; VRMB(); continue; - } + } if (t == VSL_ENDMARKER) { + if (vsl->log_ptr != vsl->log_start + 1 && + vsl->last_seq != vsl->log_start[0]) { + /* ENDMARKER not at front and seq wrapped */ + vsl->log_ptr = vsl->log_start + 1; + VRMB(); + continue; + } if (vsl->flags & F_NON_BLOCKING) return (-1); w += SLEEP_USEC; assert(usleep(SLEEP_USEC) == 0 || errno == EINTR); continue; } + if (vsl->log_ptr == vsl->log_start + 1) + vsl->last_seq = vsl->log_start[0]; + *pp = (void*)(uintptr_t)vsl->log_ptr; /* Loose volatile */ vsl->log_ptr = VSL_NEXT(vsl->log_ptr); return (1); From tfheen at varnish-cache.org Tue Aug 9 09:06:40 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:40 +0200 Subject: [3.0] c0ec0ef Reference varnish-cli + -n semantics Message-ID: commit c0ec0ef4bebcbcbf4540c2357587e95c80b0b72e Author: Per Buer Date: Fri Apr 29 18:44:29 2011 +0200 Reference varnish-cli + -n semantics diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst index 7d543c2..bfad196 100644 --- a/doc/sphinx/reference/varnishadm.rst +++ b/doc/sphinx/reference/varnishadm.rst @@ -9,7 +9,7 @@ Control a running varnish instance :Author: Cecilie Fritzvold :Author: Per Buer :Date: 2010-05-31 -:Version: 0.2 +:Version: 0.3 :Manual section: 1 SYNOPSIS @@ -21,7 +21,10 @@ DESCRIPTION =========== The varnishadm utility establishes a CLI connection to varnishd either -using -n *name* or using the -T and -S arguments. If -n *name* is the location of the secret file and the address:port is looked up in shared memory. If neither is given varnishadm will look for an instance without a given name. +using -n *name* or using the -T and -S arguments. If -n *name* is +given the location of the secret file and the address:port is looked +up in shared memory. If neither is given varnishadm will look for an +instance without a given name. If a command is given, the command and arguments are sent over the CLI connection and the result returned on stdout. @@ -46,10 +49,13 @@ OPTIONS -n name Connect to the instance of varnishd with this name. -Available commands and parameters are documented in the varnishd(1) -manual page. Additionally, a summary of commands can be obtained by -issuing the *help* command, and a summary of parameters can be -obtained by issuing the *param.show* command. +The syntax and operation of the actual CLI interface is described in +the varnish-cli(7) manual page. Parameteres are described in +varnishd(1) manual page. + +Additionally, a summary of commands can be obtained by issuing the +*help* command, and a summary of parameters can be obtained by issuing +the *param.show* command. EXIT STATUS =========== From tfheen at varnish-cache.org Tue Aug 9 09:06:41 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:41 +0200 Subject: [3.0] 7be3623 Fix compilation errors (set-but-unused variables) Message-ID: commit 7be3623066ac63aeb9df45e3bed3c986ff128214 Author: Tollef Fog Heen Date: Mon Jul 11 10:43:36 2011 +0200 Fix compilation errors (set-but-unused variables) Thanks to new GCC for spotting those. diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index f52eb4b..4788a23 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -265,6 +265,7 @@ res_WriteGunzipObj(struct sess *sp) st->ptr, st->len, obuf, sizeof obuf, &obufl); /* XXX: error check */ + (void)i; } if (obufl) { (void)WRW_Write(sp->wrk, obuf, obufl); diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c index 71a9c81..f7355cb 100644 --- a/bin/varnishreplay/varnishreplay.c +++ b/bin/varnishreplay/varnishreplay.c @@ -494,7 +494,6 @@ replay_thread(void *arg) struct replay_thread *thr = arg; struct message *msg; enum VSL_tag_e tag; - size_t len; char *ptr; const char *next; @@ -504,7 +503,6 @@ replay_thread(void *arg) while ((msg = mailbox_get(&thr->mbox)) != NULL) { tag = msg->tag; - len = msg->len; ptr = msg->ptr; thread_log(2, 0, "%s(%s)", VSL_tags[tag], msg->ptr); @@ -639,14 +637,11 @@ gen_traffic(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len, unsigned spec, const char *ptr, uint64_t bitmap) { struct replay_thread *thr; - const char *end; struct message *msg; (void)priv; (void)bitmap; - end = ptr + len; - if (fd == 0 || !(spec & VSL_S_CLIENT)) return (0); From tfheen at varnish-cache.org Tue Aug 9 09:06:43 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:43 +0200 Subject: [3.0] 48bfc6d reference varnish-cli(7) Message-ID: commit 48bfc6dd83e2a10a23554fc086349839f9a002af Author: Per Buer Date: Fri Apr 29 18:46:08 2011 +0200 reference varnish-cli(7) diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index e6c1798..da7ae6b 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -776,6 +776,7 @@ waiter SEE ALSO ======== +* varnish-cli(7) * varnishlog(1) * varnishhist(1) * varnishncsa(1) From tfheen at varnish-cache.org Tue Aug 9 09:06:44 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:44 +0200 Subject: [3.0] fb1dfce Request specific variables are not available in vcl_init{} and vcl_fini{} Message-ID: commit fb1dfce71a2c05428c428230d7c3676c897e9f02 Author: Poul-Henning Kamp Date: Thu Jun 16 06:25:44 2011 +0000 Request specific variables are not available in vcl_init{} and vcl_fini{} diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py index ffb8a08..5407a51 100755 --- a/lib/libvcl/generate.py +++ b/lib/libvcl/generate.py @@ -98,107 +98,110 @@ returns =( ####################################################################### # Variables available in sessions +# +# 'all' means all methods +# 'proc' means all methods but 'init' and 'fini' sp_variables = ( ('client.ip', 'IP', - ( 'all',), + ( 'proc',), ( ), 'const struct sess *' ), ('client.identity', 'STRING', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'struct sess *' ), ('server.ip', 'IP', - ( 'all',), + ( 'proc',), ( ), 'struct sess *' ), ('server.hostname', 'STRING', - ( 'all',), + ( 'proc',), ( ), 'struct sess *' ), ('server.identity', 'STRING', - ( 'all',), + ( 'proc',), ( ), 'struct sess *' ), ('server.port', 'INT', - ( 'all',), + ( 'proc',), ( ), 'struct sess *' ), ('req.request', 'STRING', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'const struct sess *' ), ('req.url', 'STRING', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'const struct sess *' ), ('req.proto', 'STRING', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'const struct sess *' ), ('req.http.', 'HDR_REQ', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'const struct sess *' ), ('req.backend', 'BACKEND', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'struct sess *' ), ('req.restarts', 'INT', - ( 'all',), + ( 'proc',), ( ), 'const struct sess *' ), ('req.esi_level', 'INT', - ( 'all',), + ( 'proc',), ( ), 'const struct sess *' ), ('req.ttl', 'DURATION', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'struct sess *' ), ('req.grace', 'DURATION', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'struct sess *' ), ('req.keep', 'DURATION', - ( 'all',), - ( 'all',), + ( 'proc',), + ( 'proc',), 'struct sess *' ), ('req.xid', 'STRING', - ( 'all',), + ( 'proc',), ( ), 'struct sess *' ), @@ -210,13 +213,13 @@ sp_variables = ( ), ('req.can_gzip', 'BOOL', - ( 'all',), + ( 'proc',), ( ), 'struct sess *' ), ('req.backend.healthy', 'BOOL', - ( 'all',), + ( 'proc',), ( ), 'const struct sess *' ), @@ -773,6 +776,11 @@ def restrict(fo, spec): return if spec[0] == 'all': spec = vcls + if spec[0] == 'proc': + spec = list() + for i in vcls: + if i != "init" and i != "fini": + spec.append(i) p = "" n = 0 for j in spec: From tfheen at varnish-cache.org Tue Aug 9 09:06:44 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:44 +0200 Subject: [3.0] b96c0af Use file locking on SHMFILE to indicate this file is currently in use Message-ID: commit b96c0af72e3f58f2bc767c387c8df350f9092f1e Author: Martin Blix Grydeland Date: Tue Jun 14 13:09:56 2011 +0200 Use file locking on SHMFILE to indicate this file is currently in use Fixes: #876 diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c index f28bc9b..4fc905c 100644 --- a/bin/varnishd/mgt_shmem.c +++ b/bin/varnishd/mgt_shmem.c @@ -101,6 +101,7 @@ #include "heritage.h" #include "vmb.h" #include "vsm.h" +#include "flopen.h" #ifndef MAP_HASSEMAPHORE #define MAP_HASSEMAPHORE 0 /* XXX Linux */ @@ -125,11 +126,24 @@ vsl_n_check(int fd) struct VSM_head slh; int i; struct stat st; + pid_t pid; AZ(fstat(fd, &st)); if (!S_ISREG(st.st_mode)) ARGV_ERR("\tshmlog: Not a file\n"); + /* Test if the SHMFILE is locked by other Varnish */ + if (fltest(fd, &pid) > 0) { + fprintf(stderr, + "SHMFILE locked by running varnishd master (pid=%jd)\n", + (intmax_t)pid); + fprintf(stderr, + "(Use unique -n arguments if you want multiple " + "instances)\n"); + exit(2); + } + + /* Read owning pid from SHMFILE */ memset(&slh, 0, sizeof slh); /* XXX: for flexelint */ i = read(fd, &slh, sizeof slh); if (i != sizeof slh) @@ -138,15 +152,11 @@ vsl_n_check(int fd) return; if (slh.hdrsize != sizeof slh) return; - if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) { - fprintf(stderr, - "SHMFILE owned by running varnishd master (pid=%jd)\n", - (intmax_t)slh.master_pid); - fprintf(stderr, - "(Use unique -n arguments if you want multiple " - "instances.)\n"); - exit(2); + fprintf(stderr, + "WARNING: Taking over SHMFILE marked as owned by " + "running process (pid=%jd)\n", + (intmax_t)slh.master_pid); } } @@ -161,15 +171,20 @@ vsl_buildnew(const char *fn, unsigned size, int fill) int i; unsigned u; char buf[64*1024]; + int flags; (void)unlink(fn); - vsl_fd = open(fn, O_RDWR | O_CREAT | O_EXCL, 0644); + vsl_fd = flopen(fn, O_RDWR | O_CREAT | O_EXCL | O_NONBLOCK, 0644); if (vsl_fd < 0) { fprintf(stderr, "Could not create %s: %s\n", fn, strerror(errno)); exit (1); } - + flags = fcntl(vsl_fd, F_GETFL); + assert(flags != -1); + flags &= ~O_NONBLOCK; + AZ(fcntl(vsl_fd, F_SETFL, flags)); + memset(&slh, 0, sizeof slh); slh.magic = VSM_HEAD_MAGIC; slh.hdrsize = sizeof slh; @@ -277,7 +292,6 @@ mgt_SHM_Init(const char *l_arg) vsl_n_check(i); (void)close(i); } - (void)close(i); vsl_buildnew(VSM_FILENAME, size, fill); VSM_head = (void *)mmap(NULL, size, diff --git a/include/flopen.h b/include/flopen.h index 4bd0209..e5b6030 100644 --- a/include/flopen.h +++ b/include/flopen.h @@ -31,6 +31,9 @@ #ifndef FLOPEN_H_INCLUDED #define FLOPEN_H_INCLUDED +#include + int flopen(const char *, int, ...); +int fltest(int fd, pid_t *pid); #endif diff --git a/lib/libvarnish/flopen.c b/lib/libvarnish/flopen.c index 43882fe..483d8d6 100644 --- a/lib/libvarnish/flopen.c +++ b/lib/libvarnish/flopen.c @@ -107,3 +107,27 @@ flopen(const char *path, int flags, ...) return (fd); } } + +/* Tests if the given fd is locked through flopen + * If pid is non-NULL, stores the pid of the process holding the lock there + * Returns 1 if the file is locked + * Returns 0 if the file is unlocked + * Returns -1 on error (and errno) + */ +int +fltest(int fd, pid_t *pid) +{ + struct flock lock; + + memset(&lock, 0, sizeof lock); + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + + if (fcntl(fd, F_GETLK, &lock) == -1) + return (-1); + if (lock.l_type == F_UNLCK) + return (0); + if (pid != NULL) + *pid = lock.l_pid; + return (1); +} From tfheen at varnish-cache.org Tue Aug 9 09:06:44 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:44 +0200 Subject: [3.0] 78055b5 Make all three gzip parameter varnish parameters: gzip_level, gzip_window and gzip_memlevel. Message-ID: commit 78055b5654ede873b1d2f484320f9367fb59ef4c Author: Poul-Henning Kamp Date: Wed Jun 22 14:28:36 2011 +0000 Make all three gzip parameter varnish parameters: gzip_level, gzip_window and gzip_memlevel. Still same pessimal values though. diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c index e9ae902..f320a0a 100644 --- a/bin/varnishd/cache_gzip.c +++ b/bin/varnishd/cache_gzip.c @@ -202,8 +202,8 @@ VGZ_NewGzip(struct sess *sp, const char *id) i = deflateInit2(&vg->vz, params->gzip_level, /* Level */ Z_DEFLATED, /* Method */ - 16 + 8, /* Window bits (16=gzip + 15) */ - 1, /* memLevel */ + 16 + params->gzip_window, /* Window bits (16=gzip + 15) */ + params->gzip_memlevel, /* memLevel */ Z_DEFAULT_STRATEGY); if (i != Z_OK) printf("deflateInit2() = %d\n", i); diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h index 16828ed..084d5dc 100644 --- a/bin/varnishd/heritage.h +++ b/bin/varnishd/heritage.h @@ -204,6 +204,8 @@ struct params { unsigned gzip_stack_buffer; unsigned gzip_tmp_space; unsigned gzip_level; + unsigned gzip_window; + unsigned gzip_memlevel; double critbit_cooloff; diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 1caa563..1b996e5 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -887,6 +887,14 @@ static const struct parspec input_parspec[] = { "Gzip compression level: 0=debug, 1=fast, 9=best", 0, "6", ""}, + { "gzip_window", tweak_uint, &master.gzip_window, 8, 15, + "Gzip window size 8=least, 15=most compression", + 0, + "8", ""}, + { "gzip_window", tweak_uint, &master.gzip_memlevel, 1, 9, + "Gzip memory level 1=least, 9=most compression", + 0, + "1", ""}, { "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer, 2048, UINT_MAX, "Size of stack buffer used for gzip processing.\n" From tfheen at varnish-cache.org Tue Aug 9 09:06:44 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:44 +0200 Subject: [3.0] 16cbc36 Beef the gzip parameters up to the usual defaults: window=15 and memlevel 8. Message-ID: commit 16cbc36ad5824ca1729d09467547dcd7537c8927 Author: Poul-Henning Kamp Date: Wed Jun 22 14:48:57 2011 +0000 Beef the gzip parameters up to the usual defaults: window=15 and memlevel 8. Fix testcases which depend on older values. diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 1b996e5..26fcb31 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -888,13 +888,15 @@ static const struct parspec input_parspec[] = { 0, "6", ""}, { "gzip_window", tweak_uint, &master.gzip_window, 8, 15, - "Gzip window size 8=least, 15=most compression", + "Gzip window size 8=least, 15=most compression.\n" + "Memory impact is 8=1k, 9=2k, ... 15=128k.", 0, - "8", ""}, - { "gzip_window", tweak_uint, &master.gzip_memlevel, 1, 9, - "Gzip memory level 1=least, 9=most compression", + "15", ""}, + { "gzip_memlevel", tweak_uint, &master.gzip_memlevel, 1, 9, + "Gzip memory level 1=slow/least, 9=fast/most compression.\n" + "Memory impact is 1=1k, 2=2k, ... 9=256k.", 0, - "1", ""}, + "8", ""}, { "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer, 2048, UINT_MAX, "Size of stack buffer used for gzip processing.\n" diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc index e402c5e..39302b4 100644 --- a/bin/varnishtest/tests/e00022.vtc +++ b/bin/varnishtest/tests/e00022.vtc @@ -26,6 +26,8 @@ varnish v1 -arg "-p sess_workspace=131072 -p thread_pool_stack=262144" -vcl+back varnish v1 -cliok "param.set esi_syntax 0xc" varnish v1 -cliok "param.set http_gzip_support true" varnish v1 -cliok "param.set gzip_tmp_space 1" +varnish v1 -cliok "param.set gzip_window 8" +varnish v1 -cliok "param.set gzip_memlevel 1" client c1 { txreq -hdr "Accept-Encoding: gzip" diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc index 4d713c7..dba75ae 100644 --- a/bin/varnishtest/tests/g00002.vtc +++ b/bin/varnishtest/tests/g00002.vtc @@ -11,7 +11,10 @@ server s1 { txresp -body {

} } -start -varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { +varnish v1 \ + -cliok "param.set http_gzip_support true" \ + -cliok "param.set gzip_memlevel 1" \ + -vcl+backend { sub vcl_fetch { set beresp.do_esi = true; From tfheen at varnish-cache.org Tue Aug 9 09:06:45 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:45 +0200 Subject: [3.0] 5e36c5e Explain what a purge is. Message-ID: commit 5e36c5e0e9bcb203e30e4ba81a34c945aa1ee79b Author: Per Buer Date: Thu Jun 30 10:18:36 2011 +0200 Explain what a purge is. diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst index 99f3f77..bc40691 100644 --- a/doc/sphinx/tutorial/purging.rst +++ b/doc/sphinx/tutorial/purging.rst @@ -17,6 +17,10 @@ bans. First, let me explain the HTTP purges. HTTP Purges =========== +A *purge* is what happens when you pick out an object from the cache +and discard it along with its variants. Usually a purge is invoked +through HTTP with the method PURGE. + An HTTP purge is similar to an HTTP GET request, except that the *method* is PURGE. Actually you can call the method whatever you'd like, but most people refer to this as purging. Squid supports the From tfheen at varnish-cache.org Tue Aug 9 09:06:45 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:45 +0200 Subject: [3.0] 9935d3f do_stream Message-ID: commit 9935d3f4d44eadef240da2d5631d1290db115e3d Author: Per Buer Date: Thu Jun 30 10:23:25 2011 +0200 do_stream diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index b9aaddc..5b0927a 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -661,6 +661,12 @@ The following variables are available after the requested object has been retrieved from the backend, before it is entered into the cache. In other words, they are available in vcl_fetch: +beresp.do_stream + Deliver the object to the client directly without fetching the whole + object into varnish. If this request is pass'ed it will not be + stored in memory. As of Varnish Cache 3.0 the object will marked as busy + as it is delivered so only client can access the object. + beresp.do_esi Boolean. ESI-process the object after fetching it. Defaults to false. Set it to true to parse the object for ESI directives. From tfheen at varnish-cache.org Tue Aug 9 09:06:45 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:45 +0200 Subject: [3.0] 91921da Don't assert on sleep failure due to signals in VSL library (some utils expect to receive SIGHUP). Message-ID: commit 91921da1082fb36d610752077ddb2466259e541c Author: Martin Blix Grydeland Date: Fri Jul 8 16:30:06 2011 +0200 Don't assert on sleep failure due to signals in VSL library (some utils expect to receive SIGHUP). Fixes: #947 diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 2b3b45e..4141c28 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -184,7 +184,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp) if (vsl->flags & F_NON_BLOCKING) return (-1); w += SLEEP_USEC; - AZ(usleep(SLEEP_USEC)); + assert(usleep(SLEEP_USEC) == 0 || errno == EINTR); continue; } *pp = (void*)(uintptr_t)vsl->log_ptr; /* Loose volatile */ From tfheen at varnish-cache.org Tue Aug 9 09:06:45 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:45 +0200 Subject: [3.0] 0b1d542 Avoid segfaulting if hitmiss or handling hasn't been set yet Message-ID: commit 0b1d5427c4de52934f19a4b3f4293ae3293b08b7 Author: Tollef Fog Heen Date: Mon Jul 11 10:14:40 2011 +0200 Avoid segfaulting if hitmiss or handling hasn't been set yet Fall back to "-" if the handling has not been decided yet. Fixes: #950 Fixes: #944 Fixes: #918 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 89c1ee1..17fd8d1 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -654,11 +654,11 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, p += 9+15; break; } else if (strncmp(what, "hitmiss}x", 9) == 0) { - fprintf(fo, "%s", lp->df_hitmiss); + fprintf(fo, "%s", (lp->df_hitmiss ? lp->df_hitmiss : "-")); p += 9+8; break; } else if (strncmp(what, "handling}x", 10) == 0) { - fprintf(fo, "%s", lp->df_handling); + fprintf(fo, "%s", (lp->df_handling ? lp->df_handling : "-")); p += 9+9; break; } From tfheen at varnish-cache.org Tue Aug 9 09:06:46 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:46 +0200 Subject: [3.0] a4c57ec Fix up syntax for matching in docs Message-ID: commit a4c57ecc0cefc81a8c535bf2fd8350b45b5254bf Author: Tollef Fog Heen Date: Fri Jul 15 08:39:37 2011 +0200 Fix up syntax for matching in docs diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst index 12617f5..5a1fbac 100644 --- a/doc/sphinx/faq/general.rst +++ b/doc/sphinx/faq/general.rst @@ -86,13 +86,13 @@ The "varnishlog" utility may produce a horrendous amount of output. To be able The ReqStart token will include the client IP address. To see log entries matching this, type:: - $ varnishlog -c -o ReqStart 192.0.2.123 + $ varnishlog -c -m ReqStart:192.0.2.123 To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. At the shell command line, type:: - $ varnishlog -b -o TxHeader 192.0.2.123 + $ varnishlog -b -m TxHeader:192.0.2.123 **How can I rewrite URLS before they are sent to the backend?** diff --git a/doc/sphinx/tutorial/troubleshooting.rst b/doc/sphinx/tutorial/troubleshooting.rst index aeb2786..7ab9435 100644 --- a/doc/sphinx/tutorial/troubleshooting.rst +++ b/doc/sphinx/tutorial/troubleshooting.rst @@ -80,13 +80,13 @@ give you a clue. Since varnishlog logs so much data it might be hard to track the entries down. You can set varnishlog to log all your 503 errors by issuing the following command::: - $ varnishlog -c -o TxStatus 503 + $ varnishlog -c -m TxStatus:503 If the error happened just a short time ago the transaction might still be in the shared memory log segment. To get varnishlog to process the whole shared memory log just add the -d option::: - $ varnishlog -d -c -o TxStatus 503 + $ varnishlog -d -c -m TxStatus:503 Please see the varnishlog man page for elaborations on further filtering capabilities and explanation of the various options. From tfheen at varnish-cache.org Tue Aug 9 09:06:46 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:46 +0200 Subject: [3.0] 7dabcb5 Remove obsolete bits from varnishlog man page Message-ID: commit 7dabcb5df1c3b11c8e3820e7ccd680c8e495af6f Author: Tollef Fog Heen Date: Fri Jul 15 08:41:52 2011 +0200 Remove obsolete bits from varnishlog man page diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst index c0d73c9..be9c9c5 100644 --- a/doc/sphinx/reference/varnishlog.rst +++ b/doc/sphinx/reference/varnishlog.rst @@ -81,10 +81,6 @@ The following options are available: -x tag Exclude log entries with the specified tag. -If the -o option was specified, an additional tag and regex may be -specified to select only requests which generated a log entry with the -given tag whose contents match the given regex. - TAGS ==== The following log entry tags are currently defined: From tfheen at varnish-cache.org Tue Aug 9 09:06:46 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:46 +0200 Subject: [3.0] f5248e5 Use vsb for varnishncsa to prevent partial lines on error in format strings Message-ID: commit f5248e50e32f2fc5e70763040fd3cfa847718839 Author: Tollef Fog Heen Date: Fri Jul 15 09:02:05 2011 +0200 Use vsb for varnishncsa to prevent partial lines on error in format strings diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am index f34a0e2..9b652da 100644 --- a/bin/varnishncsa/Makefile.am +++ b/bin/varnishncsa/Makefile.am @@ -13,6 +13,7 @@ varnishncsa_SOURCES = \ $(top_builddir)/lib/libvarnish/assert.c \ $(top_builddir)/lib/libvarnish/flopen.c \ $(top_builddir)/lib/libvarnish/version.c \ + $(top_builddir)/lib/libvarnish/vsb.c \ $(top_builddir)/lib/libvarnish/vpf.c varnishncsa_LDADD = \ diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 17fd8d1..c3a7d5b 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -485,6 +485,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, FILE *fo = priv; char *q, tbuf[64]; const char *p; + struct vsb *os; if (fd >= nll) { struct logline **newll = ll; @@ -533,12 +534,12 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, /* We have a complete data set - log a line */ fo = priv; + os = VSB_new_auto(); for (p = format; *p != '\0'; p++) { - if (*p != '%') { - fprintf(fo, "%c", *p); + VSB_putc(os, *p); continue; } p++; @@ -546,29 +547,29 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, case 'b': /* %b */ - fprintf(fo, "%s", lp->df_b ? lp->df_b : "-"); + VSB_cat(os, lp->df_b ? lp->df_b : "-"); break; case 'H': - fprintf(fo, "%s", lp->df_H); + VSB_cat(os, lp->df_H); break; case 'h': if (!lp->df_h && spec & VSL_S_BACKEND) - fprintf(fo, "127.0.0.1"); + VSB_cat(os, "127.0.0.1"); else - fprintf(fo, "%s", lp->df_h ? lp->df_h : "-"); + VSB_cat(os, lp->df_h ? lp->df_h : "-"); break; case 'l': - fprintf(fo, "-"); + VSB_putc(os, '-'); break; case 'm': - fprintf(fo, "%s", lp->df_m); + VSB_cat(os, lp->df_m); break; case 'q': - fprintf(fo, "%s", lp->df_q ? lp->df_q : ""); + VSB_cat(os, lp->df_q ? lp->df_q : ""); break; case 'r': @@ -576,30 +577,32 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, * Fake "%r". This would be a lot easier if Varnish * normalized the request URL. */ - fprintf(fo, "%s ", lp->df_m); + VSB_cat(os, lp->df_m); + VSB_putc(os, ' '); if (lp->df_Host) { if (strncmp(lp->df_Host, "http://", 7) != 0) - fprintf(fo, "http://"); - fprintf(fo, "%s", lp->df_Host); + VSB_cat(os, "http://"); + VSB_cat(os, lp->df_Host); } - fprintf(fo, "%s", lp->df_U); - fprintf(fo, "%s ", lp->df_q ? lp->df_q : ""); - fprintf(fo, "%s", lp->df_H); + VSB_cat(os, lp->df_U); + VSB_cat(os, lp->df_q ? lp->df_q : ""); + VSB_putc(os, ' '); + VSB_cat(os, lp->df_H); break; case 's': /* %s */ - fprintf(fo, "%s", lp->df_s ? lp->df_s : ""); + VSB_cat(os, lp->df_s ? lp->df_s : ""); break; case 't': /* %t */ strftime(tbuf, sizeof tbuf, "[%d/%b/%Y:%T %z]", &lp->df_t); - fprintf(fo, "%s", tbuf); + VSB_cat(os, tbuf); break; case 'U': - fprintf(fo, "%s", lp->df_U); + VSB_cat(os, lp->df_U); break; case 'u': @@ -616,49 +619,45 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, q = strchr(rubuf, ':'); if (q != NULL) *q = '\0'; - fprintf(fo, "%s", rubuf); + VSB_cat(os, rubuf); free(rubuf); } else { - fprintf(fo, "-"); + VSB_putc(os, '-'); } break; case '{': if (strncmp(p, "{Referer}i", 10) == 0) { - fprintf(fo, "%s", - lp->df_Referer ? lp->df_Referer : "-"); + VSB_cat(os, lp->df_Referer ? lp->df_Referer : "-"); p += 9; break; } else if (strncmp(p, "{Host}i", 7) == 0) { - fprintf(fo, "%s", - lp->df_Host ? lp->df_Host : "-"); + VSB_cat(os, lp->df_Host ? lp->df_Host : "-"); p += 6; break; } else if (strncmp(p, "{X-Forwarded-For}i", 18) == 0) { /* %{Referer}i */ - fprintf(fo, "%s", - lp->df_X_Forwarded_For ? lp->df_X_Forwarded_For : "-"); + VSB_cat(os, lp->df_X_Forwarded_For ? lp->df_X_Forwarded_For : "-"); p += 17; break; } else if (strncmp(p, "{User-agent}i", 13) == 0) { /* %{User-agent}i */ - fprintf(fo, "%s", - lp->df_User_agent ? lp->df_User_agent : "-"); + VSB_cat(os, lp->df_User_agent ? lp->df_User_agent : "-"); p += 12; break; } else if (strncmp(p, "{Varnish:", 9) == 0) { /* Scan for what we're looking for */ const char *what = p+9; if (strncmp(what, "time_firstbyte}x", 16) == 0) { - fprintf(fo, "%s", lp->df_ttfb); + VSB_cat(os, lp->df_ttfb); p += 9+15; break; } else if (strncmp(what, "hitmiss}x", 9) == 0) { - fprintf(fo, "%s", (lp->df_hitmiss ? lp->df_hitmiss : "-")); + VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-")); p += 9+8; break; } else if (strncmp(what, "handling}x", 10) == 0) { - fprintf(fo, "%s", (lp->df_handling ? lp->df_handling : "-")); + VSB_cat(os, (lp->df_handling ? lp->df_handling : "-")); p += 9+9; break; } @@ -670,14 +669,16 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, exit(1); } } - fprintf(fo, "\n"); + VSB_putc(os, '\n'); /* flush the stream */ + VSB_finish(os); + fprintf(fo, "%s", VSB_data(os)); fflush(fo); /* clean up */ clean_logline(lp); - + VSB_delete(os); return (reopen); } From tfheen at varnish-cache.org Tue Aug 9 09:06:47 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:47 +0200 Subject: [3.0] a716784 Handle errors in varnishncsa formats slightly better Message-ID: commit a7167844b5274f4d2e0343b17b194ef398926f5a Author: Tollef Fog Heen Date: Fri Jul 15 09:03:46 2011 +0200 Handle errors in varnishncsa formats slightly better Print out the rest of the format, changing the error from "Unknown format character: {" to: "Unknown format starting at: %{asdf}x" diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index c3a7d5b..07582ac 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -665,7 +665,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, /* Fall through if we haven't handled something */ /* FALLTHROUGH*/ default: - fprintf(stderr, "Unknown format character: %c\n", *p); + fprintf(stderr, "Unknown format starting at: %s\n", --p); exit(1); } } From tfheen at varnish-cache.org Tue Aug 9 09:06:47 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:47 +0200 Subject: [3.0] 49c62a5 Make it possible to log arbitrary headers in varnishncsa Message-ID: commit 49c62a56d79ea85857cfa62abe70cb0aca2574bd Author: Tollef Fog Heen Date: Fri Jul 15 10:57:16 2011 +0200 Make it possible to log arbitrary headers in varnishncsa diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 07582ac..f5f7180 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -54,8 +54,7 @@ * %q Query string * %H Protocol version * - * TODO: - Make it possible to grab any request header - * - Maybe rotate/compress log + * TODO: - Maybe rotate/compress log */ #include "config.h" @@ -72,6 +71,7 @@ #include "vsb.h" #include "vpf.h" +#include "vqueue.h" #include "libvarnish.h" #include "vsl.h" @@ -81,14 +81,16 @@ static volatile sig_atomic_t reopen; +struct hdr { + char *key; + char *value; + VTAILQ_ENTRY(hdr) list; +}; + static struct logline { char *df_H; /* %H, Protocol version */ - char *df_Host; /* %{Host}i */ - char *df_Referer; /* %{Referer}i */ char *df_U; /* %U, URL path */ char *df_q; /* %q, query string */ - char *df_User_agent; /* %{User-agent}i */ - char *df_X_Forwarded_For; /* %{X-Forwarded-For}i */ char *df_b; /* %b, Bytes */ char *df_h; /* %h (host name / IP adress)*/ char *df_m; /* %m, Request method*/ @@ -101,6 +103,7 @@ static struct logline { int active; /* Is log line in an active trans */ int complete; /* Is log line complete */ uint64_t bitmap; /* Bitmap for regex matches */ + VTAILQ_HEAD(, hdr) headers; } **ll; struct VSM_data *vd; @@ -186,23 +189,39 @@ trimline(const char *str, const char *end) return (p); } +static char * +get_header(struct logline *l, const char *name) +{ + struct hdr *h; + VTAILQ_FOREACH(h, &l->headers, list) { + if (strcasecmp(h->key, name) == 0) { + return h->value; + break; + } + } + return NULL; +} + static void clean_logline(struct logline *lp) { + struct hdr *h, *h2; #define freez(x) do { if (x) free(x); x = NULL; } while (0); freez(lp->df_H); - freez(lp->df_Host); - freez(lp->df_Referer); freez(lp->df_U); freez(lp->df_q); - freez(lp->df_User_agent); - freez(lp->df_X_Forwarded_For); freez(lp->df_b); freez(lp->df_h); freez(lp->df_m); freez(lp->df_s); freez(lp->df_u); freez(lp->df_ttfb); + VTAILQ_FOREACH_SAFE(h, &lp->headers, list, h2) { + VTAILQ_REMOVE(&lp->headers, h, list); + freez(h->key); + freez(h->value); + freez(h); + } #undef freez memset(lp, 0, sizeof *lp); } @@ -293,17 +312,22 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, case SLT_TxHeader: if (!lp->active) break; - if (isprefix(ptr, "user-agent:", end, &next)) - lp->df_User_agent = trimline(next, end); - else if (isprefix(ptr, "referer:", end, &next)) - lp->df_Referer = trimline(next, end); - else if (isprefix(ptr, "authorization:", end, &next) && - isprefix(next, "basic", end, &next)) + if (isprefix(ptr, "authorization:", end, &next) && + isprefix(next, "basic", end, &next)) { lp->df_u = trimline(next, end); - else if (isprefix(ptr, "x-forwarded-for:", end, &next)) - lp->df_X_Forwarded_For = trimline(next, end); - else if (isprefix(ptr, "host:", end, &next)) - lp->df_Host = trimline(next, end); + } else { + struct hdr *h; + const char *split; + size_t l; + h = malloc(sizeof(struct hdr)); + AN(h); + split = strchr(ptr, ':'); + AN(split); + l = strlen(split); + h->key = trimline(ptr, split-1); + h->value = trimline(split+1, split+l-1); + VTAILQ_INSERT_HEAD(&lp->headers, h, list); + } break; case SLT_BackendReuse: @@ -394,22 +418,20 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, case SLT_RxHeader: if (!lp->active) break; - if (isprefix(ptr, "user-agent:", end, &next)) { - free(lp->df_User_agent); - lp->df_User_agent = trimline(next, end); - } else if (isprefix(ptr, "referer:", end, &next)) { - free(lp->df_Referer); - lp->df_Referer = trimline(next, end); - } else if (isprefix(ptr, "authorization:", end, &next) && - isprefix(next, "basic", end, &next)) { + if (isprefix(ptr, "authorization:", end, &next) && + isprefix(next, "basic", end, &next)) { free(lp->df_u); lp->df_u = trimline(next, end); - } else if (isprefix(ptr, "x-forwarded-for:", end, &next)) { - free(lp->df_X_Forwarded_For); - lp->df_X_Forwarded_For = trimline(next, end); - } else if (isprefix(ptr, "host:", end, &next)) { - free(lp->df_Host); - lp->df_Host = trimline(next, end); + } else { + struct hdr *h; + const char *split; + h = malloc(sizeof(struct hdr)); + AN(h); + split = strchr(ptr, ':'); + AN(split); + h->key = trimline(ptr, split); + h->value = trimline(split+1, end); + VTAILQ_INSERT_HEAD(&lp->headers, h, list); } break; @@ -579,10 +601,10 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, */ VSB_cat(os, lp->df_m); VSB_putc(os, ' '); - if (lp->df_Host) { - if (strncmp(lp->df_Host, "http://", 7) != 0) + if (get_header(lp, "Host")) { + if (strncmp(get_header(lp, "Host"), "http://", 7) != 0) VSB_cat(os, "http://"); - VSB_cat(os, lp->df_Host); + VSB_cat(os, get_header(lp, "Host")); } VSB_cat(os, lp->df_U); VSB_cat(os, lp->df_q ? lp->df_q : ""); @@ -626,42 +648,44 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, } break; - case '{': - if (strncmp(p, "{Referer}i", 10) == 0) { - VSB_cat(os, lp->df_Referer ? lp->df_Referer : "-"); - p += 9; - break; - } else if (strncmp(p, "{Host}i", 7) == 0) { - VSB_cat(os, lp->df_Host ? lp->df_Host : "-"); - p += 6; - break; - } else if (strncmp(p, "{X-Forwarded-For}i", 18) == 0) { - /* %{Referer}i */ - VSB_cat(os, lp->df_X_Forwarded_For ? lp->df_X_Forwarded_For : "-"); - p += 17; - break; - } else if (strncmp(p, "{User-agent}i", 13) == 0) { - /* %{User-agent}i */ - VSB_cat(os, lp->df_User_agent ? lp->df_User_agent : "-"); - p += 12; + case '{': { + const char *h, *tmp; + char fname[100], type; + tmp = p; + type = 0; + while (*tmp != '\0' && *tmp != '}') + tmp++; + if (*tmp == '}') { + tmp++; + type = *tmp; + memcpy(fname, p+1, tmp-p-2); + } + + switch (type) { + case 'i': + h = get_header(lp, fname); + VSB_cat(os, h ? h : "-"); + p = tmp; break; - } else if (strncmp(p, "{Varnish:", 9) == 0) { - /* Scan for what we're looking for */ - const char *what = p+9; - if (strncmp(what, "time_firstbyte}x", 16) == 0) { + case 'x': + if (strcmp(fname, "Varnish:time_firstbyte") == 0) { VSB_cat(os, lp->df_ttfb); - p += 9+15; - break; - } else if (strncmp(what, "hitmiss}x", 9) == 0) { + p = tmp; + } else if (strcmp(fname, "Varnish:hitmiss") == 0) { VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-")); - p += 9+8; + p = tmp; break; - } else if (strncmp(what, "handling}x", 10) == 0) { + } else if (strcmp(fname, "handling") == 0) { VSB_cat(os, (lp->df_handling ? lp->df_handling : "-")); - p += 9+9; + p = tmp; break; } + default: + fprintf(stderr, "Unknown format starting at: %s\n", --p); + exit(1); } + break; + } /* Fall through if we haven't handled something */ /* FALLTHROUGH*/ default: diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 414e07e..48befed 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -68,8 +68,7 @@ The following options are available: Remote host %{X}i - The contents of header line X. Supported headers are - *Referer*, *Host*, *X-Forwarded-For* and *User-agent*. + The contents of header line X. %l Remote logname (always '-') From tfheen at varnish-cache.org Tue Aug 9 09:06:55 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:55 +0200 Subject: [3.0] b312abd Add support for logging response headers in varnishncsa Message-ID: commit b312abd112daede9a736a9567270640d84352640 Author: Tollef Fog Heen Date: Fri Jul 15 11:38:03 2011 +0200 Add support for logging response headers in varnishncsa diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index f5f7180..ce58fdc 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -103,7 +103,8 @@ static struct logline { int active; /* Is log line in an active trans */ int complete; /* Is log line complete */ uint64_t bitmap; /* Bitmap for regex matches */ - VTAILQ_HEAD(, hdr) headers; + VTAILQ_HEAD(, hdr) req_headers; /* Request headers */ + VTAILQ_HEAD(, hdr) resp_headers; /* Response headers */ } **ll; struct VSM_data *vd; @@ -190,10 +191,23 @@ trimline(const char *str, const char *end) } static char * -get_header(struct logline *l, const char *name) +req_header(struct logline *l, const char *name) { struct hdr *h; - VTAILQ_FOREACH(h, &l->headers, list) { + VTAILQ_FOREACH(h, &l->req_headers, list) { + if (strcasecmp(h->key, name) == 0) { + return h->value; + break; + } + } + return NULL; +} + +static char * +resp_header(struct logline *l, const char *name) +{ + struct hdr *h; + VTAILQ_FOREACH(h, &l->resp_headers, list) { if (strcasecmp(h->key, name) == 0) { return h->value; break; @@ -216,8 +230,14 @@ clean_logline(struct logline *lp) freez(lp->df_s); freez(lp->df_u); freez(lp->df_ttfb); - VTAILQ_FOREACH_SAFE(h, &lp->headers, list, h2) { - VTAILQ_REMOVE(&lp->headers, h, list); + VTAILQ_FOREACH_SAFE(h, &lp->req_headers, list, h2) { + VTAILQ_REMOVE(&lp->req_headers, h, list); + freez(h->key); + freez(h->value); + freez(h); + } + VTAILQ_FOREACH_SAFE(h, &lp->resp_headers, list, h2) { + VTAILQ_REMOVE(&lp->resp_headers, h, list); freez(h->key); freez(h->value); freez(h); @@ -326,7 +346,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, l = strlen(split); h->key = trimline(ptr, split-1); h->value = trimline(split+1, split+l-1); - VTAILQ_INSERT_HEAD(&lp->headers, h, list); + VTAILQ_INSERT_HEAD(&lp->req_headers, h, list); } break; @@ -415,10 +435,12 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_s = trimline(ptr, end); break; + case SLT_TxHeader: case SLT_RxHeader: if (!lp->active) break; - if (isprefix(ptr, "authorization:", end, &next) && + if (tag == SLT_RxHeader && + isprefix(ptr, "authorization:", end, &next) && isprefix(next, "basic", end, &next)) { free(lp->df_u); lp->df_u = trimline(next, end); @@ -431,7 +453,10 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, AN(split); h->key = trimline(ptr, split); h->value = trimline(split+1, end); - VTAILQ_INSERT_HEAD(&lp->headers, h, list); + if (tag == SLT_RxHeader) + VTAILQ_INSERT_HEAD(&lp->req_headers, h, list); + else + VTAILQ_INSERT_HEAD(&lp->resp_headers, h, list); } break; @@ -601,10 +626,10 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, */ VSB_cat(os, lp->df_m); VSB_putc(os, ' '); - if (get_header(lp, "Host")) { - if (strncmp(get_header(lp, "Host"), "http://", 7) != 0) + if (req_header(lp, "Host")) { + if (strncmp(req_header(lp, "Host"), "http://", 7) != 0) VSB_cat(os, "http://"); - VSB_cat(os, get_header(lp, "Host")); + VSB_cat(os, req_header(lp, "Host")); } VSB_cat(os, lp->df_U); VSB_cat(os, lp->df_q ? lp->df_q : ""); @@ -659,11 +684,17 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd, tmp++; type = *tmp; memcpy(fname, p+1, tmp-p-2); + fname[tmp-p-2] = 0; } switch (type) { case 'i': - h = get_header(lp, fname); + h = req_header(lp, fname); + VSB_cat(os, h ? h : "-"); + p = tmp; + break; + case 'o': + h = resp_header(lp, fname); VSB_cat(os, h ? h : "-"); p = tmp; break; diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 48befed..9ddd26e 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -79,6 +79,9 @@ The following options are available: %q The query string, if no query string exists, an empty string. + %{X}o + The contents of response header line X. + %r The first line of the request From tfheen at varnish-cache.org Tue Aug 9 09:06:59 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:06:59 +0200 Subject: [3.0] 6645f2a Minor clarification about %{X}i Message-ID: commit 6645f2ad2ba305fe4b3909ffb16934646478e56f Author: Tollef Fog Heen Date: Fri Jul 15 11:38:15 2011 +0200 Minor clarification about %{X}i diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 9ddd26e..65f08dd 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -68,7 +68,7 @@ The following options are available: Remote host %{X}i - The contents of header line X. + The contents of request header line X. %l Remote logname (always '-') From tfheen at varnish-cache.org Tue Aug 9 09:07:02 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:07:02 +0200 Subject: [3.0] 01d04b5 Fall back to using current time if urandom is not available Message-ID: commit 01d04b52baddb915bb074eff95b9a9ef2357f624 Author: Tollef Fog Heen Date: Fri Jul 15 12:14:08 2011 +0200 Fall back to using current time if urandom is not available diff --git a/lib/libvarnish/vtmpfile.c b/lib/libvarnish/vtmpfile.c index 60a9c21..327790a 100644 --- a/lib/libvarnish/vtmpfile.c +++ b/lib/libvarnish/vtmpfile.c @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -47,8 +48,12 @@ seed_random(void) unsigned seed; fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) - return (1); + if (fd == -1) { + /* urandom not available, fall back to something + * weaker */ + srandom(time(NULL)); + return (0); + } if (read(fd, &seed, sizeof seed) != sizeof seed) return (1); (void)close(fd); From tfheen at varnish-cache.org Tue Aug 9 09:07:08 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 11:07:08 +0200 Subject: [3.0] 24eea3f Increase http_resp_hdr_len and http_req_hdr_len Message-ID: commit 24eea3f471dd6ba2bc5d3881691994588faf8fcb Author: Tollef Fog Heen Date: Fri Jul 15 12:23:46 2011 +0200 Increase http_resp_hdr_len and http_req_hdr_len We're seeing at least up to 4k in the wild, so increase the limit somewhat. See: #939 diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 26fcb31..3126744 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -522,7 +522,7 @@ static const struct parspec input_parspec[] = { "Maximum length of any HTTP client request header we will " "allow. The limit is inclusive its continuation lines.\n", 0, - "2048", "bytes" }, + "4096", "bytes" }, { "http_req_size", tweak_uint, &master.http_req_size, 256, UINT_MAX, "Maximum number of bytes of HTTP client request we will deal " @@ -538,7 +538,7 @@ static const struct parspec input_parspec[] = { "Maximum length of any HTTP backend response header we will " "allow. The limit is inclusive its continuation lines.\n", 0, - "2048", "bytes" }, + "4096", "bytes" }, { "http_resp_size", tweak_uint, &master.http_resp_size, 256, UINT_MAX, "Maximum number of bytes of HTTP backend resonse we will deal " From phk at varnish-cache.org Tue Aug 9 10:02:37 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Tue, 09 Aug 2011 12:02:37 +0200 Subject: [master] d9b4655 Overhaul the stevedore statistics, the SMA and SMF interpreted certain fields in different ways. Message-ID: commit d9b4655a7959a32e796c5456d0cb1390ebd7ee6c Author: Poul-Henning Kamp Date: Tue Aug 9 10:01:53 2011 +0000 Overhaul the stevedore statistics, the SMA and SMF interpreted certain fields in different ways. diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c index 27cc86f..ea78526 100644 --- a/bin/varnishd/storage_file.c +++ b/bin/varnishd/storage_file.c @@ -182,9 +182,9 @@ insfree(struct smf_sc *sc, struct smf *sp) b = sp->size / sc->pagesize; if (b >= NBUCKET) { b = NBUCKET - 1; - sc->stats->n_smf_large++; + sc->stats->g_smf_large++; } else { - sc->stats->n_smf_frag++; + sc->stats->g_smf_frag++; } sp->flist = &sc->free[b]; ns = b * sc->pagesize; @@ -212,9 +212,9 @@ remfree(const struct smf_sc *sc, struct smf *sp) b = sp->size / sc->pagesize; if (b >= NBUCKET) { b = NBUCKET - 1; - sc->stats->n_smf_large--; + sc->stats->g_smf_large--; } else { - sc->stats->n_smf_frag--; + sc->stats->g_smf_frag--; } assert(sp->flist == &sc->free[b]); VTAILQ_REMOVE(sp->flist, sp, status); @@ -260,7 +260,7 @@ alloc_smf(struct smf_sc *sc, size_t bytes) /* Split from front */ sp2 = malloc(sizeof *sp2); XXXAN(sp2); - sc->stats->n_smf++; + sc->stats->g_smf++; *sp2 = *sp; sp->offset += bytes; @@ -302,7 +302,7 @@ free_smf(struct smf *sp) VTAILQ_REMOVE(&sc->order, sp2, order); remfree(sc, sp2); free(sp2); - sc->stats->n_smf--; + sc->stats->g_smf--; } sp2 = VTAILQ_PREV(sp, smfhead, order); @@ -314,7 +314,7 @@ free_smf(struct smf *sp) sp2->size += sp->size; VTAILQ_REMOVE(&sc->order, sp, order); free(sp); - sc->stats->n_smf--; + sc->stats->g_smf--; sp = sp2; } @@ -339,7 +339,7 @@ trim_smf(struct smf *sp, size_t bytes) CHECK_OBJ_NOTNULL(sp, SMF_MAGIC); sp2 = malloc(sizeof *sp2); XXXAN(sp2); - sc->stats->n_smf++; + sc->stats->g_smf++; *sp2 = *sp; sp2->size -= bytes; @@ -365,7 +365,7 @@ new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len) XXXAN(sp); sp->magic = SMF_MAGIC; sp->s.magic = STORAGE_MAGIC; - sc->stats->n_smf++; + sc->stats->g_smf++; sp->sc = sc; sp->size = len; @@ -452,7 +452,7 @@ smf_open(const struct stevedore *st) if (sum < MINPAGES * (off_t)getpagesize()) exit (2); - sc->stats->bfree += sc->filesize; + sc->stats->g_space += sc->filesize; } /*--------------------------------------------------------------------*/ @@ -468,16 +468,18 @@ smf_alloc(struct stevedore *st, size_t size) size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); Lck_Lock(&sc->mtx); - sc->stats->nreq++; + sc->stats->c_req++; smf = alloc_smf(sc, size); if (smf == NULL) { + sc->stats->c_fail++; Lck_Unlock(&sc->mtx); return (NULL); } CHECK_OBJ_NOTNULL(smf, SMF_MAGIC); - sc->stats->nobj++; - sc->stats->balloc += smf->size; - sc->stats->bfree -= smf->size; + sc->stats->g_alloc++; + sc->stats->c_bytes += smf->size; + sc->stats->g_bytes += smf->size; + sc->stats->g_space -= smf->size; Lck_Unlock(&sc->mtx); CHECK_OBJ_NOTNULL(&smf->s, STORAGE_MAGIC); /*lint !e774 */ XXXAN(smf); @@ -513,8 +515,9 @@ smf_trim(struct storage *s, size_t size) size &= ~(sc->pagesize - 1); if (smf->size > size) { Lck_Lock(&sc->mtx); - sc->stats->balloc -= (smf->size - size); - sc->stats->bfree += (smf->size - size); + sc->stats->c_freed += (smf->size - size); + sc->stats->g_bytes -= (smf->size - size); + sc->stats->g_space += (smf->size - size); trim_smf(smf, size); assert(smf->size == size); Lck_Unlock(&sc->mtx); @@ -534,9 +537,10 @@ smf_free(struct storage *s) CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC); sc = smf->sc; Lck_Lock(&sc->mtx); - sc->stats->nobj--; - sc->stats->balloc -= smf->size; - sc->stats->bfree += smf->size; + sc->stats->g_alloc--; + sc->stats->c_freed += smf->size; + sc->stats->g_bytes -= smf->size; + sc->stats->g_space += smf->size; free_smf(smf); Lck_Unlock(&sc->mtx); } diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 56b4daa..7035ff0 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -65,14 +65,17 @@ sma_alloc(struct stevedore *st, size_t size) CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); Lck_Lock(&sma_sc->sma_mtx); - sma_sc->stats->nreq++; - if (sma_sc->sma_alloc + size > sma_sc->sma_max) + sma_sc->stats->c_req++; + if (sma_sc->sma_alloc + size > sma_sc->sma_max) { size = 0; - else { + sma_sc->stats->c_fail += size; + } else { sma_sc->sma_alloc += size; - sma_sc->stats->nobj++; - sma_sc->stats->nbytes += size; - sma_sc->stats->balloc += size; + sma_sc->stats->c_bytes += size; + sma_sc->stats->g_alloc++; + sma_sc->stats->g_bytes += size; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space -= size; } Lck_Unlock(&sma_sc->sma_mtx); @@ -96,9 +99,16 @@ sma_alloc(struct stevedore *st, size_t size) } if (sma == NULL) { Lck_Lock(&sma_sc->sma_mtx); - sma_sc->stats->nobj--; - sma_sc->stats->nbytes -= size; - sma_sc->stats->balloc -= size; + /* + * XXX: Not nice to have counters go backwards, but we do + * XXX: Not want to pick up the lock twice just for stats. + */ + sma_sc->stats->c_fail++; + sma_sc->stats->c_bytes -= size; + sma_sc->stats->g_alloc--; + sma_sc->stats->g_bytes -= size; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space += size; Lck_Unlock(&sma_sc->sma_mtx); return (NULL); } @@ -127,9 +137,11 @@ sma_free(struct storage *s) assert(sma->sz == sma->s.space); Lck_Lock(&sma_sc->sma_mtx); sma_sc->sma_alloc -= sma->sz; - sma_sc->stats->nobj--; - sma_sc->stats->nbytes -= sma->sz; - sma_sc->stats->bfree += sma->sz; + sma_sc->stats->g_alloc--; + sma_sc->stats->g_bytes -= sma->sz; + sma_sc->stats->c_freed += sma->sz; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space += sma->sz; Lck_Unlock(&sma_sc->sma_mtx); free(sma->s.ptr); free(sma); @@ -150,8 +162,10 @@ sma_trim(struct storage *s, size_t size) assert(size < sma->sz); if ((p = realloc(sma->s.ptr, size)) != NULL) { Lck_Lock(&sma_sc->sma_mtx); - sma_sc->stats->nbytes -= (sma->sz - size); - sma_sc->stats->bfree += sma->sz - size; + sma_sc->stats->g_bytes -= (sma->sz - size); + sma_sc->stats->c_freed += sma->sz - size; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space += sma->sz - size; sma->sz = size; Lck_Unlock(&sma_sc->sma_mtx); sma->s.ptr = p; @@ -219,6 +233,8 @@ sma_open(const struct stevedore *st) sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats, VSC_CLASS, VSC_TYPE_SMA, st->ident); memset(sma_sc->stats, 0, sizeof *sma_sc->stats); + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space = sma_sc->sma_max; } const struct stevedore sma_stevedore = { diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc index 243de19..54a7ab7 100644 --- a/bin/varnishtest/tests/b00002.vtc +++ b/bin/varnishtest/tests/b00002.vtc @@ -21,7 +21,7 @@ client c1 { delay .1 varnish v1 -expect n_object == 0 -varnish v1 -expect SMA.Transient.nobj == 0 +varnish v1 -expect SMA.Transient.g_alloc == 0 varnish v1 -expect client_conn == 1 varnish v1 -expect client_req == 1 varnish v1 -expect s_sess == 1 diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc index dba75ae..3f2f951 100644 --- a/bin/varnishtest/tests/g00002.vtc +++ b/bin/varnishtest/tests/g00002.vtc @@ -32,9 +32,9 @@ client c1 { } -run # If this fails, the multiple storage allocations did not happen -varnish v1 -expect SMF.s0.nreq != 0 -varnish v1 -expect SMF.s0.nreq != 1 -varnish v1 -expect SMF.s0.nreq != 2 +varnish v1 -expect SMF.s0.c_req != 0 +varnish v1 -expect SMF.s0.c_req != 1 +varnish v1 -expect SMF.s0.c_req != 2 client c1 { # See varnish can gunzip it. diff --git a/include/vsc_fields.h b/include/vsc_fields.h index 0289b81..0952698 100644 --- a/include/vsc_fields.h +++ b/include/vsc_fields.h @@ -28,6 +28,13 @@ * * 3rd argument marks fields for inclusion in the per worker-thread * stats structure. + * + * XXX: We need a much more consistent naming of these fields, this has + * XXX: turned into a major mess, causing trouble already for backends. + * XXX: + * XXX: Please converge on: + * XXX: c_* counter (total bytes ever allocated from sma) + * XXX: g_* gauge (presently allocated bytes from sma) */ /**********************************************************************/ @@ -173,11 +180,13 @@ VSC_F(colls, uint64_t, 0, 'a', "Collisions") */ #if defined(VSC_DO_SMA) || defined (VSC_DO_SMF) -VSC_F(nreq, uint64_t, 0, 'a', "Allocator requests") -VSC_F(nobj, uint64_t, 0, 'i', "Outstanding allocations") -VSC_F(nbytes, uint64_t, 0, 'i', "Outstanding bytes") -VSC_F(balloc, uint64_t, 0, 'i', "Bytes allocated") -VSC_F(bfree, uint64_t, 0, 'i', "Bytes free") +VSC_F(c_req, uint64_t, 0, 'a', "Allocator requests") +VSC_F(c_fail, uint64_t, 0, 'a', "Allocator failures") +VSC_F(c_bytes, uint64_t, 0, 'a', "Bytes allocated") +VSC_F(c_freed, uint64_t, 0, 'a', "Bytes freed") +VSC_F(g_alloc, uint64_t, 0, 'i', "Allocations outstanding") +VSC_F(g_bytes, uint64_t, 0, 'i', "Bytes outstanding") +VSC_F(g_space, uint64_t, 0, 'i', "Bytes available") #endif @@ -190,9 +199,9 @@ VSC_F(bfree, uint64_t, 0, 'i', "Bytes free") /**********************************************************************/ #ifdef VSC_DO_SMF -VSC_F(n_smf, uint64_t, 0, 'i', "N struct smf") -VSC_F(n_smf_frag, uint64_t, 0, 'i', "N small free smf") -VSC_F(n_smf_large, uint64_t, 0, 'i', "N large free smf") +VSC_F(g_smf, uint64_t, 0, 'i', "N struct smf") +VSC_F(g_smf_frag, uint64_t, 0, 'i', "N small free smf") +VSC_F(g_smf_large, uint64_t, 0, 'i', "N large free smf") #endif /**********************************************************************/ From tfheen at varnish-cache.org Tue Aug 9 16:01:40 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 18:01:40 +0200 Subject: [master] f67de46 Markup/typos Message-ID: commit f67de468e735245ff650fbc39050c36c6478b321 Author: Tollef Fog Heen Date: Tue Aug 9 14:35:54 2011 +0200 Markup/typos diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index c6b1722..69b6546 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -34,8 +34,8 @@ statistics bans Bans are filters that are applied to keep Varnish from serving stale content. When you issue a ban Varnish will not serve any - *banned* object from cache, but rather re-fetch it from it's back - end servers. + *banned* object from cache, but rather re-fetch it from its + backend servers. process management You can stop and start the cache (child) process though the diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 5b0927a..a16d7f5 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -112,8 +112,8 @@ These can be set in the declaration like this::: } To mark a backend as unhealthy after number of items have been added -to it's saintmode list .saintmode_threshold can be set to the maximum -list size. Setting a value of 0 disables saintmode checking entirely +to its saintmode list ``.saintmode_threshold`` can be set to the maximum +list size. Setting a value of 0 disables saint mode checking entirely for that backend. The value in the backend declaration overrides the parameter. From tfheen at varnish-cache.org Tue Aug 9 16:01:44 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Tue, 09 Aug 2011 18:01:44 +0200 Subject: [master] 048ad79 Update param docs Message-ID: commit 048ad79c595e6dc557ebd6306c5831f034ff3371 Author: Tollef Fog Heen Date: Tue Aug 9 18:01:33 2011 +0200 Update param docs diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index da7ae6b..0a18546 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -280,7 +280,7 @@ ban_dups ban_lurker_sleep - Units: s - - Default: 0.1 + - Default: 0.01 How long time does the ban lurker thread sleeps between successful attempts to push the last item up the ban list. It always sleeps a second when nothing can be done. A value of zero disables the ban lurker. @@ -291,15 +291,8 @@ between_bytes_timeout Default timeout between bytes when receiving data from backend. We only wait for this many seconds between bytes before giving up. A value of 0 means it will never time out. VCL can override this default value for each backend request and backend request. This parameter does not apply to pipe. -cache_vbcs - - Units: bool - - Default: off - - Flags: experimental - - Cache vbc's or rely on malloc, that's the question. - cc_command - - Default: exec gcc -std=gnu99 -DDIAGNOSTICS -pthread -fpic -shared -Wl,-x -o %o %s + - Default: exec gcc -std=gnu99 -pthread -fpic -shared -Wl,-x -o %o %s - Flags: must_reload Command used for compiling the C source code to a dlopen(3) loadable object. Any occurrence of %s in the string will be replaced with the source file name, and %o will be replaced with the output file name. @@ -326,14 +319,14 @@ clock_skew connect_timeout - Units: s - - Default: 0.4 + - Default: 0.7 Default connection timeout for backend connections. We only try to connect to the backend for this many seconds before giving up. VCL can override this default value for each backend and backend request. critbit_cooloff - Units: s - Default: 180.0 - - Flags: experimental + - Flags: How long time the critbit hasher keeps deleted objheads on the cooloff list. @@ -345,6 +338,13 @@ default_grace Default grace period. We will deliver an object this long after it has expired, provided another thread is attempting to get a new copy. Objects already cached will not be affected by changes made until they are fetched from the backend again. +default_keep + - Units: seconds + - Default: 0 + - Flags: delayed + + Default keep period. We will keep a useless object around this long, making it available for conditional backend fetches. That means that the object will be removed from the cache at the end of ttl+grace+keep. + default_ttl - Units: seconds - Default: 120 @@ -375,12 +375,6 @@ diag_bitmap 0x80000000 - do edge-detection on digest. Use 0x notation and do the bitor in your head :-) -err_ttl - - Units: seconds - - Default: 0 - - The TTL assigned to the synthesized error pages - esi_syntax - Units: bitmap - Default: 0 @@ -397,7 +391,7 @@ expiry_sleep - Units: seconds - Default: 1 - How long the expiry thread sleeps when there is nothing for it to do. Reduce if your expiry thread gets behind. + How long the expiry thread sleeps when there is nothing for it to do. fetch_chunksize - Units: kilobytes @@ -407,6 +401,13 @@ fetch_chunksize The default chunksize used by fetcher. This should be bigger than the majority of objects with short TTLs. Internal limits in the storage_file module makes increases above 128kb a dubious idea. +fetch_maxchunksize + - Units: kilobytes + - Default: 262144 + - Flags: experimental + + The maximum chunksize we attempt to allocate from storage. Making this too large may cause delays and storage fragmentation. + first_byte_timeout - Units: s - Default: 60 @@ -424,6 +425,12 @@ gzip_level Gzip compression level: 0=debug, 1=fast, 9=best +gzip_memlevel + - Default: 8 + + Gzip memory level 1=slow/least, 9=fast/most compression. + Memory impact is 1=1k, 2=2k, ... 9=256k. + gzip_stack_buffer - Units: Bytes - Default: 32768 @@ -443,6 +450,12 @@ gzip_tmp_space 2 - thread workspace If you have much gzip/gunzip activity, it may be an advantage to use workspace for these allocations to reduce malloc activity. Be aware that gzip needs 256+KB and gunzip needs 32+KB of workspace (64+KB if ESI processing). +gzip_window + - Default: 15 + + Gzip window size 8=least, 15=most compression. + Memory impact is 8=1k, 9=2k, ... 15=128k. + http_gzip_support - Units: bool - Default: on @@ -454,25 +467,46 @@ http_gzip_support Clients that do not support gzip will have their Accept-Encoding header removed. For more information on how gzip is implemented please see the chapter on gzip in the Varnish reference. - Note: Enabling gzip on a running Varnish instance using ESI can - yield content where cached, uncompressed pages have compressed ESI - elements. To avoid this, either ban all ESI-related elements before - enabling gzip, or restart Varnish. - -http_headers +http_max_hdr - Units: header lines - Default: 64 - Maximum number of HTTP headers we will deal with. - This space is preallocated in sessions and workthreads only objects allocate only space for the headers they store. + Maximum number of HTTP headers we will deal with in client request or backend reponses. Note that the first line occupies five header fields. + This paramter does not influence storage consumption, objects allocate exact space for the headers they store. http_range_support - Units: bool - - Default: off + - Default: on - Flags: experimental Enable support for HTTP Range headers. +http_req_hdr_len + - Units: bytes + - Default: 4096 + + Maximum length of any HTTP client request header we will allow. The limit is inclusive its continuation lines. + +http_req_size + - Units: bytes + - Default: 32768 + + Maximum number of bytes of HTTP client request we will deal with. This is a limit on all bytes up to the double blank line which ends the HTTP request. + The memory for the request is allocated from the session workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up. + +http_resp_hdr_len + - Units: bytes + - Default: 4096 + + Maximum length of any HTTP backend response header we will allow. The limit is inclusive its continuation lines. + +http_resp_size + - Units: bytes + - Default: 32768 + + Maximum number of bytes of HTTP backend resonse we will deal with. This is a limit on all bytes up to the double blank line which ends the HTTP request. + The memory for the request is allocated from the worker workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up. + listen_address - Default: :80 - Flags: must_restart @@ -489,9 +523,9 @@ listen_depth log_hashstring - Units: bool - - Default: off + - Default: on - Log the hash string to shared memory log. + Log the hash string components to shared memory log. log_local_address - Units: bool @@ -507,8 +541,8 @@ lru_interval Grace period before object moves on LRU list. Objects are only moved to the front of the LRU list if they have not been moved there already inside this timeout period. This reduces the amount of lock operations necessary for LRU list access. -max_esi_includes - - Units: includes +max_esi_depth + - Units: levels - Default: 5 Maximum depth of esi:include processing. @@ -566,10 +600,11 @@ saintmode_threshold send_timeout - Units: seconds - - Default: 600 + - Default: 60 - Flags: delayed - Send timeout for client connections. If no data has been sent to the client in this many seconds, the session is closed. + Send timeout for client connections. If the HTTP response hasn't been transmitted in this many + seconds the session is closed. See setsockopt(2) under SO_SNDTIMEO for more information. sess_timeout @@ -632,8 +667,7 @@ syslog_cli_traffic thread_pool_add_delay - Units: milliseconds - - Default: 20 - - Flags: experimental + - Default: 2 Wait at least this long between creating threads. @@ -671,7 +705,7 @@ thread_pool_max - Default: 500 - Flags: delayed, experimental - The maximum number of worker threads in all pools combined. + The maximum number of worker threads in each pool. Do not set this higher than you have to, since excess worker threads soak up RAM and CPU and generally just get in the way of getting work done. @@ -680,7 +714,7 @@ thread_pool_min - Default: 5 - Flags: delayed, experimental - The minimum number of threads in each worker pool. + The minimum number of worker threads in each pool. Increasing this may help ramp up faster from low load situations where threads have expired. @@ -716,6 +750,13 @@ thread_pool_timeout Minimum is 1 second. +thread_pool_workspace + - Units: bytes + - Default: 65536 + - Flags: delayed + + Bytes of HTTP protocol workspace allocated for worker threads. This space must be big enough for the backend request and responses, and response to the client plus any other memory needs in the VCL code.Minimum is 1024 bytes. + thread_pools - Units: pools - Default: 2 From phk at varnish-cache.org Wed Aug 10 07:56:34 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 09:56:34 +0200 Subject: [master] b40bb36 Don't zap the size until we have updated statistics Message-ID: commit b40bb3667c01f3625d237f98848d34501e1c9fb2 Author: Poul-Henning Kamp Date: Wed Aug 10 07:50:13 2011 +0000 Don't zap the size until we have updated statistics diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 7035ff0..178973a 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -67,8 +67,8 @@ sma_alloc(struct stevedore *st, size_t size) Lck_Lock(&sma_sc->sma_mtx); sma_sc->stats->c_req++; if (sma_sc->sma_alloc + size > sma_sc->sma_max) { - size = 0; sma_sc->stats->c_fail += size; + size = 0; } else { sma_sc->sma_alloc += size; sma_sc->stats->c_bytes += size; From phk at varnish-cache.org Wed Aug 10 07:56:36 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 09:56:36 +0200 Subject: [master] c1a6faa Move 'age' and 'entered' into struct exp. Message-ID: commit c1a6faaf55ac0d5c4fee8bbc0b9281f5c1118592 Author: Poul-Henning Kamp Date: Wed Aug 10 07:52:02 2011 +0000 Move 'age' and 'entered' into struct exp. Simplify the move from sp->wrk to sp->obj accordingly. Add a diagram that explains how ttl, grace & keep related to each other. Fix VCL's obj.ttl variable to appear to be relative to "now" rather than obj->entered. (#956) Also log SLT_TTL when we change obj.grace and obj.keep Make SLT_TTL always have the same format: XID "RFC" or "VCL" obj.ttl obj.grace obj.keep obj.entered obj.age In addition "RFC" has: obj.date obj.expires obj.max-age Fixes #956 Thanks to: DocWilco diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index a875071..2d04724 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -242,6 +242,8 @@ struct exp { double ttl; double grace; double keep; + double age; + double entered; }; /*--------------------------------------------------------------------*/ @@ -309,8 +311,6 @@ struct worker { struct http *beresp; struct http *resp; - double age; - double entered; struct exp exp; /* This is only here so VRT can find it */ @@ -507,8 +507,6 @@ struct object { ssize_t len; - double age; - double entered; struct exp exp; double last_modified; diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index f93a61b..beb3d55 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -428,7 +428,7 @@ cnt_error(struct sess *sp) } AN(sp->obj); sp->obj->xid = sp->xid; - sp->obj->entered = sp->t_req; + sp->obj->exp.entered = sp->t_req; } else { /* XXX: Null the headers ? */ } @@ -554,9 +554,8 @@ cnt_fetch(struct sess *sp) /* * What does RFC2616 think about TTL ? */ - sp->wrk->entered = TIM_real(); - sp->wrk->age = 0; EXP_Clr(&sp->wrk->exp); + sp->wrk->exp.entered = TIM_real(); sp->wrk->exp.ttl = RFC2616_Ttl(sp); /* pass from vclrecv{} has negative TTL */ @@ -782,8 +781,6 @@ cnt_fetchbody(struct sess *sp) sp->obj->xid = sp->xid; sp->obj->response = sp->err_code; - sp->obj->age = sp->wrk->age; - sp->obj->entered = sp->wrk->entered; WS_Assert(sp->obj->ws_o); /* Filter into object */ @@ -799,7 +796,7 @@ cnt_fetchbody(struct sess *sp) if (http_GetHdr(hp, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); else - sp->obj->last_modified = floor(sp->wrk->entered); + sp->obj->last_modified = floor(sp->wrk->exp.entered); assert(WRW_IsReleased(sp->wrk)); diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c index c0e79cc..7822bf6 100644 --- a/bin/varnishd/cache_expire.c +++ b/bin/varnishd/cache_expire.c @@ -35,6 +35,18 @@ * * We hold a single object reference for both data structures. * + * An attempted overview: + * + * EXP_Ttl() EXP_Grace() EXP_Keep() + * | | | + * entered v v | + * | +--------------->+ | + * v | grace | + * +---------------------->+ | + * ttl | v + * +---------------------------->+ + * keep + * */ #include "config.h" @@ -68,6 +80,8 @@ EXP_Clr(struct exp *e) e->ttl = -1; e->grace = -1; e->keep = -1; + e->age = 0; + e->entered = 0; } #define EXP_ACCESS(fld, low_val, extra) \ @@ -93,8 +107,8 @@ EXP_ACCESS(grace, 0., ) EXP_ACCESS(keep, 0.,) /*-------------------------------------------------------------------- - * Calculate when an object is out of ttl or grace, possibly constrained - * by per-session limits. + * Calculate an objects effective keep, grace or ttl time, suitably + * adjusted for defaults and by per-session limits. */ static double @@ -131,7 +145,7 @@ EXP_Ttl(const struct sess *sp, const struct object *o) r = o->exp.ttl; if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r) r = sp->exp.ttl; - return (o->entered + r); + return (o->exp.entered + r); } /*-------------------------------------------------------------------- @@ -214,8 +228,8 @@ EXP_Insert(struct object *o) AssertObjBusy(o); HSH_Ref(oc); - assert(o->entered != 0 && !isnan(o->entered)); - o->last_lru = o->entered; + assert(o->exp.entered != 0 && !isnan(o->exp.entered)); + o->last_lru = o->exp.entered; lru = oc_getlru(oc); CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 0ca8766..739fee0 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -383,9 +383,9 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) */ if (EXP_Grace(sp, o) >= sp->t_req) { if (grace_oc == NULL || - grace_ttl < o->entered + o->exp.ttl) { + grace_ttl < o->exp.entered + o->exp.ttl) { grace_oc = oc; - grace_ttl = o->entered + o->exp.ttl; + grace_ttl = o->exp.entered + o->exp.ttl; } } } diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index 4788a23..11f7fb6 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -226,7 +226,7 @@ RES_BuildHttp(struct sess *sp) http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Age: %.0f", - sp->obj->age + sp->t_resp - sp->obj->entered); + sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered); http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish"); http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s", sp->doclose ? "close" : "keep-alive"); diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c index 297151c..1ee011b 100644 --- a/bin/varnishd/cache_vrt_var.c +++ b/bin/varnishd/cache_vrt_var.c @@ -360,15 +360,20 @@ VRT_r_req_restarts(const struct sess *sp) return (sp->restarts); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * NB: TTL is relative to when object was created, whereas grace and + * keep are relative to ttl. + */ -#define VRT_DO_EXP(which, exp, fld, extra) \ +#define VRT_DO_EXP(which, exp, fld, offset, extra) \ \ void __match_proto__() \ VRT_l_##which##_##fld(struct sess *sp, double a) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + if (a > 0.) \ + a += offset; \ EXP_Set_##fld(&exp, a); \ extra; \ } \ @@ -378,21 +383,37 @@ VRT_r_##which##_##fld(struct sess *sp) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return(EXP_Get_##fld(&exp)); \ -} - -VRT_DO_EXP(req, sp->exp, ttl, ) -VRT_DO_EXP(req, sp->exp, grace, ) -VRT_DO_EXP(req, sp->exp, keep, ) -VRT_DO_EXP(obj, sp->obj->exp, grace, EXP_Rearm(sp->obj)) -VRT_DO_EXP(obj, sp->obj->exp, ttl, - EXP_Rearm(sp->obj); - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->obj->xid, a, sp->t_req)) -VRT_DO_EXP(obj, sp->obj->exp, keep, EXP_Rearm(sp->obj)) -VRT_DO_EXP(beresp, sp->wrk->exp, grace, ) -VRT_DO_EXP(beresp, sp->wrk->exp, ttl, - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req)) -VRT_DO_EXP(beresp, sp->wrk->exp, keep, ) + return(EXP_Get_##fld(&exp) - offset); \ +} + +static void +vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e) +{ + WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", + xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep, + sp->t_req, e->age + (sp->t_req - e->entered)); +} + +VRT_DO_EXP(req, sp->exp, ttl, 0, ) +VRT_DO_EXP(req, sp->exp, grace, 0, ) +VRT_DO_EXP(req, sp->exp, keep, 0, ) + +VRT_DO_EXP(obj, sp->obj->exp, grace, 0, + EXP_Rearm(sp->obj); + vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);) +VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered), + EXP_Rearm(sp->obj); + vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);) +VRT_DO_EXP(obj, sp->obj->exp, keep, 0, + EXP_Rearm(sp->obj); + vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);) + +VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0, + vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);) +VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0, + vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);) +VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0, + vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);) /*-------------------------------------------------------------------- * req.xid diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c index 7b2525f..69637ea 100644 --- a/bin/varnishd/rfc2616.c +++ b/bin/varnishd/rfc2616.c @@ -76,7 +76,7 @@ RFC2616_Ttl(const struct sess *sp) hp = sp->wrk->beresp; - assert(sp->wrk->entered != 0.0 && !isnan(sp->wrk->entered)); + assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered)); /* If all else fails, cache using default ttl */ ttl = params->default_ttl; @@ -116,7 +116,7 @@ RFC2616_Ttl(const struct sess *sp) max_age = strtoul(p, NULL, 0); if (http_GetHdr(hp, H_Age, &p)) { age = strtoul(p, NULL, 0); - sp->wrk->age = age; + sp->wrk->exp.age = age; } if (age > max_age) @@ -145,16 +145,16 @@ RFC2616_Ttl(const struct sess *sp) } if (h_date == 0 || - fabs(h_date - sp->wrk->entered) < params->clock_skew) { + fabs(h_date - sp->wrk->exp.entered) < params->clock_skew) { /* * If we have no Date: header or if it is * sufficiently close to our clock we will * trust Expires: relative to our own clock. */ - if (h_expires < sp->wrk->entered) + if (h_expires < sp->wrk->exp.entered) ttl = 0; else - ttl = h_expires - sp->wrk->entered; + ttl = h_expires - sp->wrk->exp.entered; break; } else { /* @@ -168,8 +168,10 @@ RFC2616_Ttl(const struct sess *sp) } /* calculated TTL, Our time, Date, Expires, max-age, age */ - WSP(sp, SLT_TTL, "%u RFC %g %.0f %.0f %.0f %u %u", sp->xid, - ttl, sp->wrk->entered, h_date, h_expires, max_age, age); + WSP(sp, SLT_TTL, + "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u %u", + sp->xid, ttl, -1. -1., sp->wrk->exp.entered, sp->wrk->exp.age, + h_date, h_expires, max_age); return (ttl); } diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 413e0b6..9a0b671 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -36,7 +36,6 @@ #include #include #include -#include #include "cache.h" #include "stevedore.h" @@ -248,7 +247,6 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot, http_Setup(o->http, o->ws_o); o->http->magic = HTTP_MAGIC; - o->entered = NAN; o->exp = *soc->exp; VTAILQ_INIT(&o->store); sp->wrk->stats.n_object++; diff --git a/bin/varnishtest/tests/r00956.vtc b/bin/varnishtest/tests/r00956.vtc new file mode 100644 index 0000000..c98996d --- /dev/null +++ b/bin/varnishtest/tests/r00956.vtc @@ -0,0 +1,49 @@ +varnishtest "obj.ttl relative/absolute" + +server s1 { + rxreq + txresp -hdr "Cache-Control: max-age=23" -hdr "Age: 4" -bodylen 40 +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 10s; + set req.http.foo = beresp.ttl; + set req.http.bar = "xxx"; + } + sub vcl_hit { + set req.http.foo = obj.ttl; + set obj.ttl = 7s; + set obj.grace = 120s; + set obj.keep = 1h; + set req.http.bar = obj.ttl; + } + sub vcl_deliver { + set resp.http.foo = req.http.foo; + set resp.http.bar = req.http.bar; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 40 + expect resp.http.foo == 10.000 + expect resp.http.bar == "xxx" + + delay 2 + txreq + rxresp + expect resp.bodylen == 40 + # XXX: should be: < 8 + expect resp.http.foo != 10.000 + expect resp.http.bar == 7.000 + + delay 2 + txreq + rxresp + expect resp.bodylen == 40 + # XXX: should be: < 5 + expect resp.http.foo != 7.000 + expect resp.http.bar == 7.000 +} -run From phk at varnish-cache.org Wed Aug 10 09:07:25 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 11:07:25 +0200 Subject: [master] 9765e50 Fiddle the "RFC" SLT_TTL record into submission. Message-ID: commit 9765e5026d9572267b3774979ccfe31d77cf624c Author: Poul-Henning Kamp Date: Wed Aug 10 09:06:58 2011 +0000 Fiddle the "RFC" SLT_TTL record into submission. Document the TTL record. diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c index 69637ea..045eacb 100644 --- a/bin/varnishd/rfc2616.c +++ b/bin/varnishd/rfc2616.c @@ -89,6 +89,16 @@ RFC2616_Ttl(const struct sess *sp) * We do not support ranges yet, so 206 is out. */ + if (http_GetHdr(hp, H_Age, &p)) { + age = strtoul(p, NULL, 0); + sp->wrk->exp.age = age; + } + if (http_GetHdr(hp, H_Expires, &p)) + h_expires = TIM_parse(p); + + if (http_GetHdr(hp, H_Date, &p)) + h_date = TIM_parse(p); + switch (sp->err_code) { default: sp->wrk->exp.ttl = -1.; @@ -114,10 +124,6 @@ RFC2616_Ttl(const struct sess *sp) max_age = 0; else max_age = strtoul(p, NULL, 0); - if (http_GetHdr(hp, H_Age, &p)) { - age = strtoul(p, NULL, 0); - sp->wrk->exp.age = age; - } if (age > max_age) ttl = 0; @@ -126,17 +132,10 @@ RFC2616_Ttl(const struct sess *sp) break; } - /* Next look for absolute specifications from backend */ - - if (http_GetHdr(hp, H_Expires, &p)) - h_expires = TIM_parse(p); - /* No expire header, fall back to default */ if (h_expires == 0) break; - if (http_GetHdr(hp, H_Date, &p)) - h_date = TIM_parse(p); /* If backend told us it is expired already, don't cache. */ if (h_expires < h_date) { @@ -169,8 +168,8 @@ RFC2616_Ttl(const struct sess *sp) /* calculated TTL, Our time, Date, Expires, max-age, age */ WSP(sp, SLT_TTL, - "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u %u", - sp->xid, ttl, -1. -1., sp->wrk->exp.entered, sp->wrk->exp.age, + "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", + sp->xid, ttl, -1., -1., sp->wrk->exp.entered, sp->wrk->exp.age, h_date, h_expires, max_age); return (ttl); diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index ac1026a..e9f2ff3 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -4,6 +4,38 @@ Shared Memory Logging ===================== +TTL records +~~~~~~~~~~~ + +A TTL record is emitted whenever the ttl, grace or keep values for an +object is set. + +The format is:: + + %u %s %d %d %d %d %d [ %d %u %u ] + | | | | | | | | | | + | | | | | | | | | +- Max-Age from Cache-Control header + | | | | | | | | +---- Expires header + | | | | | | | +------- Date header + | | | | | | +------------ Age (incl Age: header value) + | | | | | +--------------- Reference time for TTL + | | | | +------------------ Keep + | | | +--------------------- Grace + | | +------------------------ TTL + | +--------------------------- "RFC" or "VCL" + +------------------------------ object XID + +The last three fields are only present in "RFC" headers. + +Examples:: + + 1001 RFC 19 -1 -1 1312966109 4 0 0 23 + 1001 VCL 10 -1 -1 1312966109 4 + 1001 VCL 7 -1 -1 1312966111 6 + 1001 VCL 7 120 -1 1312966111 6 + 1001 VCL 7 120 3600 1312966111 6 + 1001 VCL 12 120 3600 1312966113 8 + Gzip records ~~~~~~~~~~~~ @@ -14,7 +46,6 @@ gunziped, will run into many of these. The format is:: - %c %c %c %d %d %d %d %d | | | | | | | | | | | | | | | +- Bit length of compressed data @@ -26,7 +57,7 @@ The format is:: | +------------------- 'F' = Fetch, 'D' = Deliver +---------------------- 'G' = Gzip, 'U' = Gunzip, 'u' = Gunzip-test -Which in practice could look like:: +Examples:: U F E 182 159 80 80 1392 G F E 159 173 80 1304 1314 From tfheen at varnish-cache.org Wed Aug 10 09:27:57 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 10 Aug 2011 11:27:57 +0200 Subject: [master] 9f00f52 Fix typo Message-ID: commit 9f00f529cd18f97c61d39e46d72ca23dbf6c1f1c Author: Tollef Fog Heen Date: Wed Aug 10 11:26:04 2011 +0200 Fix typo diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index d0e8ade..5cb5ad7 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -57,7 +57,7 @@ becomes ``beresp.cacheable`` is gone ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0`` +``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0s`` returns are now done with the ``return()`` function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From tfheen at varnish-cache.org Wed Aug 10 09:32:25 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 10 Aug 2011 11:32:25 +0200 Subject: [master] d6a5687 Document do_esi Message-ID: commit d6a5687f01bd09d96a868820c11beaf0c518ee5d Author: Tollef Fog Heen Date: Wed Aug 10 11:31:37 2011 +0200 Document do_esi Put esi ? do_esi in upgrade checklist, fix markup typo. Fixes #974 diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 5cb5ad7..1a31733 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -78,7 +78,7 @@ becomes ``req.hash`` is replaced with ``hash_data()`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You no longer append to the hash with +=, so +You no longer append to the hash with ``+=``, so set req.hash += req.url; @@ -86,6 +86,17 @@ becomes hash_data(req.url); +``esi`` is replaced with ``beresp.do_esi`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You no longer enable ESI with ``esi``, so + + esi; + +in ``vcl_fetch`` becomes + + set beresp.do_esi = true; + ``pass`` in ``vcl_fetch`` renamed to ``hit_for_pass`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From phk at varnish-cache.org Wed Aug 10 12:18:51 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 14:18:51 +0200 Subject: [master] bcc71b9 Add a couple of asserts to make sure we only do ims/range on 200. Message-ID: commit bcc71b944736d4ab76960cf3b05ec58655bea451 Author: Poul-Henning Kamp Date: Wed Aug 10 12:04:58 2011 +0000 Add a couple of asserts to make sure we only do ims/range on 200. diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index 11f7fb6..0c306df 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -47,6 +47,7 @@ res_do_304(struct sess *sp) char lm[64]; char *p; + assert(sp->obj->response == 200); http_ClrHeader(sp->wrk->resp); sp->wrk->resp->logtag = HTTP_Tx; http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified"); @@ -127,7 +128,7 @@ res_dorange(struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) { ssize_t low, high, has_low; - (void)sp; + assert(sp->obj->response == 200); if (strncmp(r, "bytes=", 6)) return; r += 6; From phk at varnish-cache.org Wed Aug 10 12:18:51 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 14:18:51 +0200 Subject: [master] 409308d Move the function which checks if we can do condiitional (304) delivery into the RFC policy module where it belongs. Message-ID: commit 409308d1e7c59251d5b5fceeb2f4f45cc304a43f Author: Poul-Henning Kamp Date: Wed Aug 10 12:15:42 2011 +0000 Move the function which checks if we can do condiitional (304) delivery into the RFC policy module where it belongs. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 2d04724..209dfc0 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -937,6 +937,7 @@ unsigned WS_Free(const struct ws *ws); double RFC2616_Ttl(const struct sess *sp); enum body_status RFC2616_Body(const struct sess *sp); unsigned RFC2616_Req_Gzip(const struct sess *sp); +int RFC2616_Do_Cond(const struct sess *sp); /* storage_synth.c */ struct vsb *SMS_Makesynth(struct object *obj); diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index 0c306df..3cd7911 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -86,45 +86,8 @@ res_do_304(struct sess *sp) /*--------------------------------------------------------------------*/ -static int -res_do_conds(struct sess *sp) -{ - char *p, *e; - double ims; - int do_cond = 0; - - /* RFC 2616 13.3.4 states we need to match both ETag - and If-Modified-Since if present*/ - - if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) { - if (!sp->obj->last_modified) - return (0); - ims = TIM_parse(p); - if (ims > sp->t_req) /* [RFC2616 14.25] */ - return (0); - if (sp->obj->last_modified > ims) - return (0); - do_cond = 1; - } - - if (http_GetHdr(sp->http, H_If_None_Match, &p) && - http_GetHdr(sp->obj->http, H_ETag, &e)) { - if (strcmp(p,e) != 0) - return (0); - do_cond = 1; - } - - if (do_cond == 1) { - res_do_304(sp); - return (1); - } - return (0); -} - -/*--------------------------------------------------------------------*/ - static void -res_dorange(struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) +res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) { ssize_t low, high, has_low; @@ -196,8 +159,10 @@ RES_BuildHttp(struct sess *sp) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - if (sp->obj->response == 200 && sp->http->conds && res_do_conds(sp)) + if (sp->obj->response == 200 && sp->http->conds && RFC2616_Do_Cond(sp)) { + res_do_304(sp); return; + } http_ClrHeader(sp->wrk->resp); sp->wrk->resp->logtag = HTTP_Tx; diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c index 045eacb..eb01850 100644 --- a/bin/varnishd/rfc2616.c +++ b/bin/varnishd/rfc2616.c @@ -306,3 +306,36 @@ RFC2616_Req_Gzip(const struct sess *sp) /* Bad client, no gzip. */ return (0); } + +/*--------------------------------------------------------------------*/ + +int +RFC2616_Do_Cond(const struct sess *sp) +{ + char *p, *e; + double ims; + int do_cond = 0; + + /* RFC 2616 13.3.4 states we need to match both ETag + and If-Modified-Since if present*/ + + if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) { + if (!sp->obj->last_modified) + return (0); + ims = TIM_parse(p); + if (ims > sp->t_req) /* [RFC2616 14.25] */ + return (0); + if (sp->obj->last_modified > ims) + return (0); + do_cond = 1; + } + + if (http_GetHdr(sp->http, H_If_None_Match, &p) && + http_GetHdr(sp->obj->http, H_ETag, &e)) { + if (strcmp(p,e) != 0) + return (0); + do_cond = 1; + } + + return (do_cond); +} From phk at varnish-cache.org Wed Aug 10 12:18:51 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 14:18:51 +0200 Subject: [master] d9f5779 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit d9f57799fc84558960716f5879004139329c5886 Merge: 409308d d6a5687 Author: Poul-Henning Kamp Date: Wed Aug 10 12:18:48 2011 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at varnish-cache.org Wed Aug 10 13:34:43 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 15:34:43 +0200 Subject: [master] 5c8bb04 Don't panic if we can both do conditional (IMS) and stream, prefer IMS, it is less work and less data to transmit. Message-ID: commit 5c8bb0448dcf2d3b4e091ff4e7a285f578bb3f38 Author: Poul-Henning Kamp Date: Wed Aug 10 13:15:07 2011 +0000 Don't panic if we can both do conditional (IMS) and stream, prefer IMS, it is less work and less data to transmit. Fixes #972 Thanks to: Martin diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index beb3d55..f5d2957 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -800,6 +800,16 @@ cnt_fetchbody(struct sess *sp) assert(WRW_IsReleased(sp->wrk)); + /* + * If we can deliver a 304 reply, we don't bother streaming. + * Notice that vcl_deliver{} could still nuke the headers + * that allow the 304, in which case we return 200 non-stream. + */ + if (sp->obj->response == 200 && + sp->http->conds && + RFC2616_Do_Cond(sp)) + sp->wrk->do_stream = 0; + if (sp->wrk->do_stream) { sp->step = STP_PREPRESP; return (0); From phk at varnish-cache.org Wed Aug 10 13:34:43 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 15:34:43 +0200 Subject: [master] 029e95a Test case for #972 Message-ID: commit 029e95a7afcb30a76d9908fc952b0b16b24fd8b7 Author: Poul-Henning Kamp Date: Wed Aug 10 13:18:36 2011 +0000 Test case for #972 Written by: Martin diff --git a/bin/varnishtest/tests/r00972.vtc b/bin/varnishtest/tests/r00972.vtc new file mode 100644 index 0000000..ed3a916 --- /dev/null +++ b/bin/varnishtest/tests/r00972.vtc @@ -0,0 +1,21 @@ +varnishtest "Test conditional delivery and do_stream" + +server s1 { + rxreq + txresp -hdr "ETag: foo" -body "11111\n" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq -hdr "If-None-Match: foo" + rxresp + expect resp.status == 304 + expect resp.http.etag == "foo" + expect resp.bodylen == 0 +} -run + From phk at varnish-cache.org Wed Aug 10 13:34:43 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2011 15:34:43 +0200 Subject: [master] 624ec16 Move the 304/ims processing into the object delivery code, where we also have the 206/range handling. Message-ID: commit 624ec16be8ea0d452d92e0723629b046d56a7d73 Author: Poul-Henning Kamp Date: Wed Aug 10 13:33:25 2011 +0000 Move the 304/ims processing into the object delivery code, where we also have the 206/range handling. Instead of building the 304 response from scratch, dumb down the 200 response accordingly. This implement the policy consideration of #970 led to. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 209dfc0..ef8435c 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -889,7 +889,7 @@ void WSL_Flush(struct worker *w, int overflow); #endif /* cache_response.c */ -void RES_BuildHttp(struct sess *sp); +void RES_BuildHttp(const struct sess *sp); void RES_WriteObj(struct sess *sp); void RES_StreamStart(struct sess *sp); void RES_StreamEnd(struct sess *sp); diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index 3cd7911..17d1525 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -42,51 +42,6 @@ /*--------------------------------------------------------------------*/ static void -res_do_304(struct sess *sp) -{ - char lm[64]; - char *p; - - assert(sp->obj->response == 200); - http_ClrHeader(sp->wrk->resp); - sp->wrk->resp->logtag = HTTP_Tx; - http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified"); - TIM_format(sp->t_req, lm); - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", lm); - http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish"); - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "X-Varnish: %u", sp->xid); - if (sp->obj->last_modified) { - TIM_format(sp->obj->last_modified, lm); - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "Last-Modified: %s", lm); - } - - /* http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5 */ - if (http_GetHdr(sp->obj->http, H_Cache_Control, &p)) - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "Cache-Control: %s", p); - if (http_GetHdr(sp->obj->http, H_Content_Location, &p)) - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "Content-Location: %s", p); - if (http_GetHdr(sp->obj->http, H_ETag, &p)) - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "ETag: %s", p); - if (http_GetHdr(sp->obj->http, H_Expires, &p)) - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "Expires: %s", p); - if (http_GetHdr(sp->obj->http, H_Vary, &p)) - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, - "Vary: %s", p); - - http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s", - sp->doclose ? "close" : "keep-alive"); - sp->wantbody = 0; -} - -/*--------------------------------------------------------------------*/ - -static void res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) { ssize_t low, high, has_low; @@ -153,16 +108,12 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) /*--------------------------------------------------------------------*/ void -RES_BuildHttp(struct sess *sp) +RES_BuildHttp(const struct sess *sp) { char time_str[30]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - if (sp->obj->response == 200 && sp->http->conds && RFC2616_Do_Cond(sp)) { - res_do_304(sp); - return; - } http_ClrHeader(sp->wrk->resp); sp->wrk->resp->logtag = HTTP_Tx; @@ -297,7 +248,10 @@ res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high) assert(u == sp->obj->len); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Deliver an object. + * Attempt optimizations like 304 and 206 here. + */ void RES_WriteObj(struct sess *sp) @@ -309,6 +263,15 @@ RES_WriteObj(struct sess *sp) WRW_Reserve(sp->wrk, &sp->fd); + if (sp->obj->response == 200 && + sp->http->conds && + RFC2616_Do_Cond(sp)) { + sp->wantbody = 0; + http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified"); + http_Unset(sp->wrk->resp, H_Content_Length); + http_Unset(sp->wrk->resp, H_Transfer_Encoding); + } + /* * If nothing special planned, we can attempt Range support */ @@ -339,7 +302,7 @@ RES_WriteObj(struct sess *sp) WRW_Chunked(sp->wrk); if (!sp->wantbody) { - /* This was a HEAD request */ + /* This was a HEAD or conditional request */ } else if (sp->obj->len == 0) { /* Nothing to do here */ } else if (sp->wrk->res_mode & RES_ESI) { From phk at varnish-cache.org Thu Aug 11 08:40:24 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Thu, 11 Aug 2011 10:40:24 +0200 Subject: [master] 12a3f19 Polish up the unit-test case for the binheap, looking for #967 Message-ID: commit 12a3f19d8d2bbbde3797e621fc6a46e1c4f5cd1c Author: Poul-Henning Kamp Date: Thu Aug 11 08:40:01 2011 +0000 Polish up the unit-test case for the binheap, looking for #967 diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c index e6a00e7..6eb454b 100644 --- a/lib/libvarnish/binary_heap.c +++ b/lib/libvarnish/binary_heap.c @@ -28,8 +28,9 @@ * * Implementation of a binary heap API * - * We use a malloc(3)/realloc(3) array to store the pointers using the - * classical FORTRAN strategy. + * See also: + * http://portal.acm.org/citation.cfm?doid=1785414.1785434 + * (or: http://queue.acm.org/detail.cfm?id=1814327) */ #include "config.h" @@ -392,25 +393,45 @@ binheap_reorder(const struct binheap *bh, unsigned idx) #ifdef TEST_DRIVER /* Test driver -------------------------------------------------------*/ #include +#include + +static void +vasfail(const char *func, const char *file, int line, + const char *cond, int err, int xxx) +{ + fprintf(stderr, "PANIC: %s %s %d %s %d %d\n", + func, file, line, cond, err, xxx); + abort(); +} + +vas_f *VAS_Fail = vasfail; struct foo { + unsigned magic; +#define FOO_MAGIC 0x23239823 unsigned idx; unsigned key; + unsigned n; }; +#if 1 #define M 31011091 /* Number of operations */ -#define N 10313102 /* Number of items */ +#define N 10313102 /* Number of items */ +#else +#define M 3401 /* Number of operations */ +#define N 1131 /* Number of items */ +#endif #define R -1 /* Random modulus */ -struct foo ff[N]; +struct foo *ff[N]; static int cmp(void *priv, void *a, void *b) { struct foo *fa, *fb; - fa = a; - fb = b; + CAST_OBJ_NOTNULL(fa, a, FOO_MAGIC); + CAST_OBJ_NOTNULL(fb, b, FOO_MAGIC); return (fa->key < fb->key); } @@ -419,12 +440,12 @@ update(void *priv, void *a, unsigned u) { struct foo *fa; - fa = a; + CAST_OBJ_NOTNULL(fa, a, FOO_MAGIC); fa->idx = u; } void -chk(struct binheap *bh) +chk2(struct binheap *bh) { unsigned u, v; struct foo *fa, *fb; @@ -441,7 +462,7 @@ int main(int argc, char **argv) { struct binheap *bh; - unsigned u, v, lr; + unsigned u, v, lr, n; struct foo *fp; if (0) { @@ -452,58 +473,80 @@ main(int argc, char **argv) } bh = binheap_new(NULL, cmp, update); - /* First insert our N elements */ - for (u = 0; u < N; u++) { - lr = random() % R; - ff[u].key = lr; - binheap_insert(bh, &ff[u]); - - fp = binheap_root(bh); - assert(fp->idx == 1); - assert(fp->key <= lr); - } - fprintf(stderr, "%d inserts OK\n", N); - /* For M cycles, pick the root, insert new */ - for (u = 0; u < M; u++) { - fp = binheap_root(bh); - assert(fp->idx == 1); - - /* It cannot possibly be larger than the last value we added */ - assert(fp->key <= lr); - binheap_delete(bh, fp->idx); - - lr = random() % R; - fp->key = lr; - binheap_insert(bh, fp); - } - fprintf(stderr, "%d replacements OK\n", M); - /* The remove everything */ - lr = 0; - for (u = 0; u < N; u++) { - fp = binheap_root(bh); - assert(fp->idx == 1); - assert(fp->key >= lr); - lr = fp->key; - binheap_delete(bh, fp->idx); - } - fprintf(stderr, "%d removes OK\n", N); - - for (u = 0; u < M; u++) { - v = random() % N; - if (ff[v].idx > 0) { - assert(ff[v].idx != 0); - binheap_delete(bh, ff[v].idx); - assert(ff[v].idx == 0); - } else { - assert(ff[v].idx == 0); - ff[v].key = random() % R; - binheap_insert(bh, &ff[v]); - assert(ff[v].idx != 0); + while (1) { + /* First insert our N elements */ + for (u = 0; u < N; u++) { + lr = random() % R; + ALLOC_OBJ(ff[u], FOO_MAGIC); + assert(ff[u] != NULL); + ff[u]->key = lr; + ff[u]->n = u; + binheap_insert(bh, ff[u]); + + fp = binheap_root(bh); + assert(fp->idx == 1); + assert(fp->key <= lr); + } + fprintf(stderr, "%d inserts OK\n", N); + /* For M cycles, pick the root, insert new */ + for (u = 0; u < M; u++) { + fp = binheap_root(bh); + CHECK_OBJ_NOTNULL(fp, FOO_MAGIC); + assert(fp->idx == 1); + + /* It cannot possibly be larger than the last value we added */ + assert(fp->key <= lr); + binheap_delete(bh, fp->idx); + + + n = fp->n; + ALLOC_OBJ(ff[n], FOO_MAGIC); + assert(ff[n] != NULL); + FREE_OBJ(fp); + fp = ff[n]; + fp->n = n; + + lr = random() % R; + fp->key = lr; + binheap_insert(bh, fp); + } + fprintf(stderr, "%d replacements OK\n", M); + /* The remove everything */ + lr = 0; + for (u = 0; u < N; u++) { + fp = binheap_root(bh); + CHECK_OBJ_NOTNULL(fp, FOO_MAGIC); + assert(fp->idx == 1); + assert(fp->key >= lr); + lr = fp->key; + binheap_delete(bh, fp->idx); + ff[fp->n] = NULL; + FREE_OBJ(fp); + } + fprintf(stderr, "%d removes OK\n", N); + + for (u = 0; u < M; u++) { + v = random() % N; + if (ff[v] != NULL) { + CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC); + assert(ff[v]->idx != 0); + binheap_delete(bh, ff[v]->idx); + assert(ff[v]->idx == 0); + FREE_OBJ(ff[v]); + ff[v] = NULL; + } else { + ALLOC_OBJ(ff[v], FOO_MAGIC); + assert(ff[v] != NULL); + ff[v]->key = random() % R; + binheap_insert(bh, ff[v]); + CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC); + assert(ff[v]->idx != 0); + } + if (0) + chk2(bh); } - if (0) - chk(bh); + fprintf(stderr, "%d updates OK\n", M); } - fprintf(stderr, "%d updates OK\n", M); return (0); } #endif From phk at varnish-cache.org Thu Aug 11 11:01:08 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Thu, 11 Aug 2011 13:01:08 +0200 Subject: [master] a6ddafc Clamp rather than overflow on child indexes when we get to the end of the UINT_MAX items we can support. Message-ID: commit a6ddafc0e87bcec71f016b8cae77b78641e617d7 Author: Poul-Henning Kamp Date: Thu Aug 11 10:59:34 2011 +0000 Clamp rather than overflow on child indexes when we get to the end of the UINT_MAX items we can support. Found by adding a lot of asserts and brute force testing, both of which I have left in. Fixes #967 May also be relevant to #827 diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c index 7822bf6..97e5734 100644 --- a/bin/varnishd/cache_expire.c +++ b/bin/varnishd/cache_expire.c @@ -182,6 +182,8 @@ exp_insert(struct objcore *oc, struct lru *lru) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); + Lck_AssertHeld(&lru->mtx); + Lck_AssertHeld(&exp_mtx); assert(oc->timer_idx == BINHEAP_NOIDX); binheap_insert(exp_heap, oc); assert(oc->timer_idx != BINHEAP_NOIDX); From phk at varnish-cache.org Thu Aug 11 11:26:21 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Thu, 11 Aug 2011 13:26:21 +0200 Subject: [master] 726d93f Duh! Git committing from the wrong directory doesn't do what you expect: Message-ID: commit 726d93ff5a815774d7a3b4a23dcba2efb3c08ca7 Author: Poul-Henning Kamp Date: Thu Aug 11 11:25:41 2011 +0000 Duh! Git committing from the wrong directory doesn't do what you expect: Clamp rather than overflow on child indexes when we get to the end of the UINT_MAX items we can support. Found by adding a lot of asserts and brute force testing, both of which I have left in. Fixes #967 May also be relevant to #827 diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c index 6eb454b..c94cbfd 100644 --- a/lib/libvarnish/binary_heap.c +++ b/lib/libvarnish/binary_heap.c @@ -37,6 +37,7 @@ #include #include +#include #include "binary_heap.h" #include "libvarnish.h" @@ -97,6 +98,7 @@ parent(const struct binheap *bh, unsigned u) unsigned po; unsigned v; + assert(u != UINT_MAX); po = u & bh->page_mask; if (u < bh->page_size || po > 3) { @@ -114,6 +116,7 @@ parent(const struct binheap *bh, unsigned u) static void child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b) { + uintmax_t uu; if (u > bh->page_mask && (u & (bh->page_mask - 1)) == 0) { /* First two elements are magical except on the first page */ @@ -123,16 +126,32 @@ child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b) *a = (u & ~bh->page_mask) >> 1; *a |= u & (bh->page_mask >> 1); *a += 1; - *a <<= bh->page_shift; - *b = *a + 1; + uu = (uintmax_t)*a << bh->page_shift; + *a = uu; + if (*a == uu) { + *b = *a + 1; + } else { + /* + * An unsigned is not big enough: clamp instead + * of truncating. We do not support adding + * more than UINT_MAX elements anyway, so this + * is without consequence. + */ + *a = UINT_MAX; + *b = UINT_MAX; + } } else { /* The rest is as usual, only inside the page */ *a = u + (u & bh->page_mask); *b = *a + 1; } #ifdef PARANOIA - assert(parent(bh, *a) == u); - assert(parent(bh, *b) == u); + assert(*a > 0); + assert(*b > 0); + if (*a != UINT_MAX) { + assert(parent(bh, *a) == u); + assert(parent(bh, *b) == u); + } #endif } @@ -215,12 +234,12 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f) static void binheap_update(const struct binheap *bh, unsigned u) { + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); assert(u < bh->next); assert(A(bh, u) != NULL); - if (bh->update == NULL) - return; - bh->update(bh->priv, A(bh, u), u); + if (bh->update != NULL) + bh->update(bh->priv, A(bh, u), u); } static void @@ -228,9 +247,12 @@ binhead_swap(const struct binheap *bh, unsigned u, unsigned v) { void *p; + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); assert(u < bh->next); + assert(A(bh, u) != NULL); assert(v < bh->next); + assert(A(bh, v) != NULL); p = A(bh, u); A(bh, u) = A(bh, v); A(bh, v) = p; @@ -243,9 +265,17 @@ binheap_trickleup(const struct binheap *bh, unsigned u) { unsigned v; - assert(bh->magic == BINHEAP_MAGIC); + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); + assert(u < bh->next); + assert(A(bh, u) != NULL); + while (u > ROOT_IDX) { + assert(u < bh->next); + assert(A(bh, u) != NULL); v = parent(bh, u); + assert(v < u); + assert(v < bh->next); + assert(A(bh, v) != NULL); if (!bh->cmp(bh->priv, A(bh, u), A(bh, v))) break; binhead_swap(bh, u, v); @@ -254,20 +284,37 @@ binheap_trickleup(const struct binheap *bh, unsigned u) return (u); } -static void +static unsigned binheap_trickledown(const struct binheap *bh, unsigned u) { unsigned v1, v2; + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); + assert(u < bh->next); + assert(A(bh, u) != NULL); + while (1) { + assert(u < bh->next); + assert(A(bh, u) != NULL); child(bh, u, &v1, &v2); + assert(v1 > 0); + assert(v2 > 0); + assert(v1 <= v2); + if (v1 >= bh->next) - return; - if (v2 < bh->next && bh->cmp(bh->priv, A(bh, v2), A(bh, v1))) - v1 = v2; + return (u); + + assert(A(bh, v1) != NULL); + if (v1 != v2 && v2 < bh->next) { + assert(A(bh, v2) != NULL); + if (bh->cmp(bh->priv, A(bh, v2), A(bh, v1))) + v1 = v2; + } + assert(v1 < bh->next); + assert(A(bh, v1) != NULL); if (bh->cmp(bh->priv, A(bh, u), A(bh, v1))) - return; + return (u); binhead_swap(bh, u, v1); u = v1; } @@ -283,10 +330,13 @@ binheap_insert(struct binheap *bh, void *p) assert(bh->length >= bh->next); if (bh->length == bh->next) binheap_addrow(bh); + assert(bh->length > bh->next); u = bh->next++; A(bh, u) = p; binheap_update(bh, u); (void)binheap_trickleup(bh, u); + assert(u < bh->next); + assert(A(bh, u) != NULL); } @@ -332,11 +382,11 @@ binheap_root(const struct binheap *bh) * N{height}-1 upward trickles. * * When we fill the hole with the tail object, the worst case is - * that it trickles all the way down to become the tail object - * again. - * In other words worst case is N{height} downward trickles. - * But there is a pretty decent chance that it does not make - * it all the way down. + * that it trickles all the way up to of this half-tree, or down + * to become the tail object again. + * + * In other words worst case is N{height} up or downward trickles. + * But there is a decent chance that it does not make it all the way. */ void @@ -358,7 +408,13 @@ binheap_delete(struct binheap *bh, unsigned idx) A(bh, bh->next) = NULL; binheap_update(bh, idx); idx = binheap_trickleup(bh, idx); - binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); + idx = binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); /* * We keep a hysteresis of one full row before we start to @@ -387,7 +443,13 @@ binheap_reorder(const struct binheap *bh, unsigned idx) assert(idx > 0); assert(A(bh, idx) != NULL); idx = binheap_trickleup(bh, idx); - binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); + idx = binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); } #ifdef TEST_DRIVER @@ -416,7 +478,7 @@ struct foo { #if 1 #define M 31011091 /* Number of operations */ -#define N 10313102 /* Number of items */ +#define N 17313102 /* Number of items */ #else #define M 3401 /* Number of operations */ #define N 1131 /* Number of items */ @@ -472,6 +534,11 @@ main(int argc, char **argv) srandom(u); } bh = binheap_new(NULL, cmp, update); + for (n = 2; n; n += n) { + child(bh, n - 1, &u, &v); + child(bh, n, &u, &v); + child(bh, n + 1, &u, &v); + } while (1) { /* First insert our N elements */ @@ -530,10 +597,15 @@ main(int argc, char **argv) if (ff[v] != NULL) { CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC); assert(ff[v]->idx != 0); - binheap_delete(bh, ff[v]->idx); - assert(ff[v]->idx == 0); - FREE_OBJ(ff[v]); - ff[v] = NULL; + if (ff[v]->key & 1) { + binheap_delete(bh, ff[v]->idx); + assert(ff[v]->idx == BINHEAP_NOIDX); + FREE_OBJ(ff[v]); + ff[v] = NULL; + } else { + ff[v]->key = random() % R; + binheap_reorder(bh, ff[v]->idx); + } } else { ALLOC_OBJ(ff[v], FOO_MAGIC); assert(ff[v] != NULL); From phk at varnish-cache.org Thu Aug 11 14:12:30 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Thu, 11 Aug 2011 16:12:30 +0200 Subject: [master] e8a63f4 Try to read the silo signature to find the correct address to map the silo into VM. If this fails or we get garbage, the silo will be cleared. Message-ID: commit e8a63f4d5e687cf3a04d34f696c661446d6c8d25 Author: Poul-Henning Kamp Date: Thu Aug 11 14:11:42 2011 +0000 Try to read the silo signature to find the correct address to map the silo into VM. If this fails or we get garbage, the silo will be cleared. Fixes #962 diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c index 0681264..a22cc59 100644 --- a/bin/varnishd/storage_persistent_mgt.c +++ b/bin/varnishd/storage_persistent_mgt.c @@ -120,6 +120,8 @@ void smp_mgt_init(struct stevedore *parent, int ac, char * const *av) { struct smp_sc *sc; + struct smp_sign sgn; + void *target; int i; ASSERT_MGT(); @@ -158,7 +160,15 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) AZ(ftruncate(sc->fd, sc->mediasize)); - sc->base = mmap(NULL, sc->mediasize, PROT_READ|PROT_WRITE, + /* Try to determine correct mmap address */ + i = read(sc->fd, &sgn, sizeof sgn); + assert(i == sizeof sgn); + if (!strcmp(sgn.ident, "SILO")) + target = (void*)sgn.mapped; + else + target = NULL; + + sc->base = mmap(target, sc->mediasize, PROT_READ|PROT_WRITE, MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0); if (sc->base == MAP_FAILED) @@ -169,8 +179,11 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) sc->ident = SIGN_DATA(&sc->idn); i = smp_valid_silo(sc); - if (i) + if (i) { + printf("Warning SILO (%s) not reloaded (reason=%d)\n", + sc->filename, i); smp_newsilo(sc); + } AZ(smp_valid_silo(sc)); smp_metrics(sc); diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c index 552c0be..6fad388 100644 --- a/bin/varnishd/storage_persistent_subr.c +++ b/bin/varnishd/storage_persistent_subr.c @@ -238,26 +238,27 @@ smp_valid_silo(struct smp_sc *sc) assert(strlen(SMP_IDENT_STRING) < sizeof si->ident); - if (smp_chk_sign(&sc->idn)) - return (1); + i = smp_chk_sign(&sc->idn); + if (i) + return (i); si = sc->ident; if (strcmp(si->ident, SMP_IDENT_STRING)) - return (2); + return (12); if (si->byte_order != 0x12345678) - return (3); + return (13); if (si->size != sizeof *si) - return (4); + return (14); if (si->major_version != 2) - return (5); + return (15); if (si->mediasize != sc->mediasize) - return (7); + return (17); if (si->granularity != sc->granularity) - return (8); + return (18); if (si->align < sizeof(void*)) - return (9); + return (19); if (!PWR2(si->align)) - return (10); + return (20); sc->align = si->align; sc->unique = si->unique; diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc new file mode 100644 index 0000000..66f447b --- /dev/null +++ b/bin/varnishtest/tests/r00962.vtc @@ -0,0 +1,47 @@ +varnishtest "Test address remapping" + +server s1 { + rxreq + txresp +} -start + +shell "rm -f ${tmpdir}/_.per?" + +varnish v1 \ + -arg "-pdiag_bitmap=0x20000" \ + -storage "-spersistent,${tmpdir}/_.per1,10m -spersistent,${tmpdir}/_.per2,10m" \ + -vcl+backend { + sub vcl_fetch { + set beresp.storage = "s0"; + } +} -start + +varnish v1 -stop + +varnish v1 -start + +client c1 { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" +} -run + +varnish v1 -cliok "storage.list" +varnish v1 -cliok "debug.persistent s0 dump" +varnish v1 -cliok "debug.persistent s0 sync" +varnish v1 -stop + +varnish v2 \ + -arg "-pdiag_bitmap=0x20000" \ + -storage "-spersistent,${tmpdir}/_.per2,10m -spersistent,${tmpdir}/_.per1,10m" \ + -vcl+backend { } -start + +client c1 -connect ${v2_sock} { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" +} -run + +# shell "rm -f /tmp/__v1/_.per" From phk at varnish-cache.org Mon Aug 15 07:29:54 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 09:29:54 +0200 Subject: [master] bc2a0bd ifdef the OFOF development tool Message-ID: commit bc2a0bd07058013d56fd53237b1ddd8012fb8e89 Author: Poul-Henning Kamp Date: Mon Aug 15 07:28:52 2011 +0000 ifdef the OFOF development tool diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c index 60e63bf..a3d1851 100644 --- a/bin/varnishd/cache_cli.c +++ b/bin/varnishd/cache_cli.c @@ -140,6 +140,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) SZOF(struct vbc); SZOF(struct VSC_C_main); SZOF(struct lock); +#if 0 #define OFOF(foo, bar) { foo __foo; VCLI_Out(cli, \ "%-30s = 0x%4zx @ 0x%4zx\n", \ #foo "." #bar, sizeof(__foo.bar), offsetof(foo, bar)); } @@ -199,6 +200,8 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) OFOF(struct object, esidata); OFOF(struct object, last_use); #endif +#undef OFOF +#endif } /*--------------------------------------------------------------------*/ From phk at varnish-cache.org Mon Aug 15 07:29:55 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 09:29:55 +0200 Subject: [master] cd55687 Constify Message-ID: commit cd55687a540fabb5ad37a28e4cd87fef0ade8a18 Author: Poul-Henning Kamp Date: Mon Aug 15 07:29:25 2011 +0000 Constify diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h index c065218..d2de06e 100644 --- a/bin/varnishd/cache_backend.h +++ b/bin/varnishd/cache_backend.h @@ -147,7 +147,7 @@ void VBE_DropRefLocked(struct backend *b); /* cache_backend_poll.c */ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr); void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p); -void VBP_Use(struct backend *b, const struct vrt_backend_probe const *p); +void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p); /* Init functions for directors */ typedef void dir_init_f(struct cli *, struct director **, int , const void*); diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c index d7b5268..e504f0c 100644 --- a/bin/varnishd/cache_backend_poll.c +++ b/bin/varnishd/cache_backend_poll.c @@ -510,7 +510,7 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, const char *hos } void -VBP_Use(struct backend *b, const struct vrt_backend_probe *p) +VBP_Use(const struct backend *b, const struct vrt_backend_probe *p) { struct vbp_target *vt; struct vbp_vcl *vcl; From phk at varnish-cache.org Mon Aug 15 07:29:56 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 09:29:56 +0200 Subject: [master] 94c870b We always allocate an object entirely inside one stevedore now. Message-ID: commit 94c870bf1c6eeb78b1a25866dcb0ab541afc453e Author: Poul-Henning Kamp Date: Mon Aug 15 07:29:37 2011 +0000 We always allocate an object entirely inside one stevedore now. diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 9a0b671..533874e 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -151,20 +151,15 @@ static struct storage * stv_alloc(const struct sess *sp, size_t size) { struct storage *st; - struct stevedore *stv = NULL; + struct stevedore *stv; unsigned fail = 0; /* - * Always try the stevedore which allocated the object in order to - * not needlessly split an object across multiple stevedores. + * Always use the stevedore which allocated the object in order to + * keep an object inside the same stevedore. */ - if (sp->obj != NULL) { - CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); - stv = sp->obj->objstore->stevedore; - } else { - INCOMPL(); - stv = stv_transient; - } + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + stv = sp->obj->objstore->stevedore; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); if (size > (size_t)(params->fetch_maxchunksize) << 10) From kristian at varnish-cache.org Mon Aug 15 08:29:37 2011 From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=) Date: Mon, 15 Aug 2011 10:29:37 +0200 Subject: [master] b231cf2 Fix the test for #971 Message-ID: commit b231cf2bf2e949cc2baf516aa8fa2d15710d2979 Author: Kristian Lyngstol Date: Mon Aug 15 10:19:14 2011 +0200 Fix the test for #971 Ironically, this does NOT fix #971 (see if you can handle this, trac). (WIP) diff --git a/bin/varnishtest/tests/r00971.vtc b/bin/varnishtest/tests/r00971.vtc index b508cd7..f8b57f0 100644 --- a/bin/varnishtest/tests/r00971.vtc +++ b/bin/varnishtest/tests/r00971.vtc @@ -9,6 +9,9 @@ varnish v1 -vcl+backend { director foo dns { { .backend = { .host = "127.0.0.1";} } + .list = { + "192.168.0.0"/24; + } } sub vcl_recv { From phk at varnish-cache.org Mon Aug 15 08:39:13 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 10:39:13 +0200 Subject: [master] 369b5b6 Prompted by HTTPbis and FlexeLint in unison, go over the nhttp and response code stuff and fully incorporate the fact that they are both 16 unsigned now. Message-ID: commit 369b5b6b751375acb3d9b55381f331507f92ca0c Author: Poul-Henning Kamp Date: Mon Aug 15 08:37:38 2011 +0000 Prompted by HTTPbis and FlexeLint in unison, go over the nhttp and response code stuff and fully incorporate the fact that they are both 16 unsigned now. No functional changes. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index ef8435c..0b5ad8e 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -739,21 +739,21 @@ int VGZ_WrwGunzip(const struct sess *, struct vgz *, const void *ibuf, /* cache_http.c */ unsigned HTTP_estimate(unsigned nhttp); void HTTP_Copy(struct http *to, const struct http * const fm); -struct http *HTTP_create(void *p, unsigned nhttp); +struct http *HTTP_create(void *p, uint16_t nhttp); const char *http_StatusMessage(unsigned); -unsigned http_EstimateWS(const struct http *fm, unsigned how, unsigned *nhd); +unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd); void HTTP_Init(void); void http_ClrHeader(struct http *to); unsigned http_Write(struct worker *w, const struct http *hp, int resp); void http_CopyResp(struct http *to, const struct http *fm); -void http_SetResp(struct http *to, const char *proto, int status, +void http_SetResp(struct http *to, const char *proto, uint16_t status, const char *response); void http_FilterFields(struct worker *w, int fd, struct http *to, const struct http *fm, unsigned how); void http_FilterHeader(const struct sess *sp, unsigned how); void http_PutProtocol(struct worker *w, int fd, const struct http *to, const char *protocol); -void http_PutStatus(struct http *to, int status); +void http_PutStatus(struct http *to, uint16_t status); void http_PutResponse(struct worker *w, int fd, const struct http *to, const char *response); void http_PrintfHeader(struct worker *w, int fd, struct http *to, @@ -768,11 +768,11 @@ int http_GetHdrData(const struct http *hp, const char *hdr, int http_GetHdrField(const struct http *hp, const char *hdr, const char *field, char **ptr); double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field); -int http_GetStatus(const struct http *hp); +uint16_t http_GetStatus(const struct http *hp); const char *http_GetReq(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); -int http_DissectRequest(struct sess *sp); -int http_DissectResponse(struct worker *w, const struct http_conn *htc, +uint16_t http_DissectRequest(struct sess *sp); +uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc, struct http *sp); const char *http_DoConnection(const struct http *hp); void http_CopyHome(struct worker *w, int fd, const struct http *hp); diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index f5d2957..42d4b3b 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -417,10 +417,10 @@ cnt_error(struct sess *sp) /* XXX: 1024 is a pure guess */ EXP_Clr(&w->exp); sp->obj = STV_NewObject(sp, NULL, 1024, &w->exp, - params->http_max_hdr); + (uint16_t)params->http_max_hdr); if (sp->obj == NULL) sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, - 1024, &w->exp, params->http_max_hdr); + 1024, &w->exp, (uint16_t)params->http_max_hdr); if (sp->obj == NULL) { sp->doclose = "Out of objects"; sp->step = STP_DONE; @@ -637,7 +637,8 @@ cnt_fetchbody(struct sess *sp) int i; struct http *hp, *hp2; char *b; - unsigned l, nhttp; + uint16_t nhttp; + unsigned l; struct vsb *vary = NULL; int varyl = 0, pass; @@ -1411,7 +1412,7 @@ DOT start -> recv [style=bold,color=green] static int cnt_start(struct sess *sp) { - int done; + uint16_t done; char *p; const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; @@ -1439,7 +1440,7 @@ cnt_start(struct sess *sp) done = http_DissectRequest(sp); /* If we could not even parse the request, just close */ - if (done < 0) { + if (done == 400) { sp->step = STP_DONE; vca_close_session(sp, "junk"); return (0); diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 2141cc4..7fe0a8a 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -118,7 +118,7 @@ HTTP_estimate(unsigned nhttp) } struct http * -HTTP_create(void *p, unsigned nhttp) +HTTP_create(void *p, uint16_t nhttp) { struct http *hp; @@ -135,7 +135,7 @@ HTTP_create(void *p, unsigned nhttp) void http_Setup(struct http *hp, struct ws *ws) { - unsigned shd; + uint16_t shd; txt *hd; unsigned char *hdf; @@ -425,6 +425,7 @@ http_DoConnection(const struct http *hp) return (NULL); } ret = NULL; + AN(p); for (; *p; p++) { if (vct_issp(*p)) continue; @@ -463,7 +464,7 @@ http_HdrIs(const struct http *hp, const char *hdr, const char *val) /*--------------------------------------------------------------------*/ -int +uint16_t http_GetStatus(const struct http *hp) { @@ -483,7 +484,7 @@ http_GetReq(const struct http *hp) * Detect conditionals (headers which start with '^[Ii][Ff]-') */ -static int +static uint16_t http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p, const struct http_conn *htc) { @@ -557,7 +558,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p, * Deal with first line of HTTP protocol message. */ -static int +static uint16_t http_splitline(struct worker *w, int fd, struct http *hp, const struct http_conn *htc, int h1, int h2, int h3) { @@ -577,7 +578,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, q = p; for (; !vct_issp(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } hp->hd[h1].b = q; hp->hd[h1].e = p; @@ -585,14 +586,14 @@ http_splitline(struct worker *w, int fd, struct http *hp, /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } /* Second field cannot contain LWS or CTL */ q = p; for (; !vct_islws(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } hp->hd[h2].b = q; hp->hd[h2].e = p; @@ -603,7 +604,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } /* Third field is optional and cannot contain CTL */ @@ -611,7 +612,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, if (!vct_iscrlf(*p)) { for (; !vct_iscrlf(*p); p++) if (!vct_issep(*p) && vct_isctl(*p)) - return (-1); + return (400); } hp->hd[h3].b = q; hp->hd[h3].e = p; @@ -650,12 +651,12 @@ http_ProtoVer(struct http *hp) /*--------------------------------------------------------------------*/ -int +uint16_t http_DissectRequest(struct sess *sp) { struct http_conn *htc; struct http *hp; - int i; + uint16_t retval; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); htc = sp->htc; @@ -665,23 +666,26 @@ http_DissectRequest(struct sess *sp) hp->logtag = HTTP_Rx; - i = http_splitline(sp->wrk, sp->fd, hp, htc, + retval = http_splitline(sp->wrk, sp->fd, hp, htc, HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO); - if (i != 0) { + if (retval != 0) { WSPR(sp, SLT_HttpGarbage, htc->rxbuf); - return (i); + return (retval); } http_ProtoVer(hp); - return (i); + return (retval); } /*--------------------------------------------------------------------*/ -int +uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc, struct http *hp) { - int i = 0; + int j; + uint16_t retval = 0; + char *p; + CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -689,23 +693,33 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc, if (http_splitline(w, htc->fd, hp, htc, HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE)) - i = 503; + retval = 503; - if (i == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7)) - i = 503; + if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7)) + retval = 503; - if (i == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3) - i = 503; + if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3) + retval = 503; - if (i == 0) { - hp->status = strtoul(hp->hd[HTTP_HDR_STATUS].b, NULL, 10); - if (hp->status < 100 || hp->status > 999) - i = 503; + if (retval == 0) { + hp->status = 0; + p = hp->hd[HTTP_HDR_STATUS].b; + for (j = 100; j != 0; j /= 10) { + if (!vct_isdigit(*p)) { + retval = 503; + break; + } + hp->status += (uint16_t)(j * (*p - '0')); + p++; + } + if (*p != '\0') + retval = 503; } - if (i != 0) { + if (retval != 0) { WSLR(w, SLT_HttpGarbage, htc->fd, htc->rxbuf); - hp->status = i; + assert(retval >= 100 && retval <= 999); + hp->status = retval; } else { http_ProtoVer(hp); } @@ -718,7 +732,7 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc, hp->hd[HTTP_HDR_RESPONSE].e = strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0'); } - return (i); + return (retval); } /*--------------------------------------------------------------------*/ @@ -763,12 +777,13 @@ http_CopyResp(struct http *to, const struct http *fm) } void -http_SetResp(struct http *to, const char *proto, int status, +http_SetResp(struct http *to, const char *proto, uint16_t status, const char *response) { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); http_SetH(to, HTTP_HDR_PROTO, proto); + assert(status >= 100 && status <= 999); to->status = status; http_SetH(to, HTTP_HDR_RESPONSE, response); } @@ -798,7 +813,7 @@ http_copyheader(struct worker *w, int fd, struct http *to, */ unsigned -http_EstimateWS(const struct http *fm, unsigned how, unsigned *nhd) +http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd) { unsigned u, l; @@ -967,10 +982,10 @@ http_PutProtocol(struct worker *w, int fd, const struct http *to, } void -http_PutStatus(struct http *to, int status) +http_PutStatus(struct http *to, uint16_t status) { - assert(status >= 0 && status <= 999); + assert(status >= 100 && status <= 999); to->status = status; } @@ -1011,7 +1026,7 @@ http_PrintfHeader(struct worker *w, int fd, struct http *to, void http_Unset(struct http *hp, const char *hdr) { - unsigned u, v; + uint16_t u, v; for (v = u = HTTP_HDR_FIRST; u < hp->nhd; u++) { if (hp->hd[u].b == NULL) diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c index 603ba83..0875e07 100644 --- a/bin/varnishd/cache_pool.c +++ b/bin/varnishd/cache_pool.c @@ -116,7 +116,7 @@ WRK_SumStat(struct worker *w) static void * wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace, - unsigned nhttp, unsigned http_space, unsigned siov) + uint16_t nhttp, unsigned http_space, unsigned siov) { struct worker *w, ww; uint32_t wlog[shm_workspace / 4]; @@ -218,12 +218,13 @@ static void * wrk_thread(void *priv) { struct wq *qp; - unsigned nhttp; + uint16_t nhttp; unsigned siov; CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC); + assert(params->http_max_hdr <= 65535); /* We need to snapshot these two for consistency */ - nhttp = params->http_max_hdr; + nhttp = (uint16_t)params->http_max_hdr; siov = nhttp * 2; if (siov > IOV_MAX) siov = IOV_MAX; diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c index 87e05c7..af422a6 100644 --- a/bin/varnishd/cache_session.c +++ b/bin/varnishd/cache_session.c @@ -101,8 +101,8 @@ ses_sm_alloc(void) { struct sessmem *sm; unsigned char *p, *q; - volatile unsigned nws; - volatile unsigned nhttp; + unsigned nws; + uint16_t nhttp; unsigned l, hl; if (VSC_C_main->n_sess_mem >= params->max_sess) @@ -113,7 +113,7 @@ ses_sm_alloc(void) * view of the value. */ nws = params->sess_workspace; - nhttp = params->http_max_hdr; + nhttp = (uint16_t)params->http_max_hdr; hl = HTTP_estimate(nhttp); l = sizeof *sm + nws + 2 * hl; p = malloc(l); diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c index 65b7522..c17c361 100644 --- a/bin/varnishd/cache_vrt.c +++ b/bin/varnishd/cache_vrt.c @@ -61,7 +61,9 @@ VRT_error(struct sess *sp, unsigned code, const char *reason) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ? reason : "(null)"); - sp->err_code = code ? code : 503; + if (code < 100 || code > 999) + code = 503; + sp->err_code = (uint16_t)code; sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code); } diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c index 1ee011b..2054953 100644 --- a/bin/varnishd/cache_vrt_var.c +++ b/bin/varnishd/cache_vrt_var.c @@ -102,7 +102,7 @@ VRT_l_##obj##_status(const struct sess *sp, int num) \ { \ \ assert(num >= 100 && num <= 999); \ - http->status = num; \ + http->status = (uint16_t)num; \ } \ \ int \ diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 3126744..7bfa707 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -549,7 +549,7 @@ static const struct parspec input_parspec[] = { "how much of that the request is allowed to take up.", 0, "32768", "bytes" }, - { "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, UINT_MAX, + { "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, 65535, "Maximum number of HTTP headers we will deal with in " "client request or backend reponses. " "Note that the first line occupies five header fields.\n" diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 533874e..1dad892 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -199,7 +199,7 @@ stv_alloc(const struct sess *sp, size_t size) struct stv_objsecrets { unsigned magic; #define STV_OBJ_SECRETES_MAGIC 0x78c87247 - unsigned nhttp; + uint16_t nhttp; unsigned lhttp; unsigned wsl; struct exp *exp; @@ -294,7 +294,7 @@ stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot, struct object * STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep, - unsigned nhttp) + uint16_t nhttp) { struct object *o; struct stevedore *stv; diff --git a/bin/varnishd/stevedore.h b/bin/varnishd/stevedore.h index 5a5cc4b..0ccf834 100644 --- a/bin/varnishd/stevedore.h +++ b/bin/varnishd/stevedore.h @@ -91,7 +91,7 @@ struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot, const struct stv_objsecrets *soc); struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len, - struct exp *, unsigned nhttp); + struct exp *, uint16_t nhttp); struct storage *STV_alloc(const struct sess *sp, size_t size); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); From phk at varnish-cache.org Mon Aug 15 08:39:14 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 10:39:14 +0200 Subject: [master] e13e923 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit e13e923d6738bc588e44bb5757821a8c880a599f Merge: 369b5b6 b231cf2 Author: Poul-Henning Kamp Date: Mon Aug 15 08:39:09 2011 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From kristian at varnish-cache.org Mon Aug 15 09:17:20 2011 From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=) Date: Mon, 15 Aug 2011 11:17:20 +0200 Subject: [master] 0e3c3ed Merge branch 'master' of git+ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 0e3c3ed0cbe726dd3a45c13d9946ff2b20a54134 Merge: 1695f2a e13e923 Author: Kristian Lyngstol Date: Mon Aug 15 11:16:54 2011 +0200 Merge branch 'master' of git+ssh://git.varnish-cache.org/git/varnish-cache From kristian at varnish-cache.org Mon Aug 15 09:17:20 2011 From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=) Date: Mon, 15 Aug 2011 11:17:20 +0200 Subject: [master] 1695f2a Move r00971.vtc to tests.disabled/ and add README Message-ID: commit 1695f2abb7823ef2a52079331f6e3f9951d55cc6 Author: Kristian Lyngstol Date: Mon Aug 15 11:16:15 2011 +0200 Move r00971.vtc to tests.disabled/ and add README diff --git a/bin/varnishtest/tests.disabled/README b/bin/varnishtest/tests.disabled/README new file mode 100644 index 0000000..7b44514 --- /dev/null +++ b/bin/varnishtest/tests.disabled/README @@ -0,0 +1,5 @@ +Tests in this directory are not executed automatically on 'make check'. + +The main reason for a test to live here is when the underlying problem +isn't fixed yet, but a test case exists. This avoids breaking the rest of +the 'make check' tests while a fix is being produced. diff --git a/bin/varnishtest/tests.disabled/r00971.vtc b/bin/varnishtest/tests.disabled/r00971.vtc new file mode 100644 index 0000000..f8b57f0 --- /dev/null +++ b/bin/varnishtest/tests.disabled/r00971.vtc @@ -0,0 +1,25 @@ + +varnishtest "Test DNS director order" + +varnish v1 -vcl+backend { + + backend test { + .host = "192.168.0.1"; + } + + director foo dns { + { .backend = { .host = "127.0.0.1";} } + .list = { + "192.168.0.0"/24; + } + } + + sub vcl_recv { + set req.backend = foo; + if (req.http.x-aa) { + set req.backend = test; + } + } + +} -start + diff --git a/bin/varnishtest/tests/r00971.vtc b/bin/varnishtest/tests/r00971.vtc deleted file mode 100644 index f8b57f0..0000000 --- a/bin/varnishtest/tests/r00971.vtc +++ /dev/null @@ -1,25 +0,0 @@ - -varnishtest "Test DNS director order" - -varnish v1 -vcl+backend { - - backend test { - .host = "192.168.0.1"; - } - - director foo dns { - { .backend = { .host = "127.0.0.1";} } - .list = { - "192.168.0.0"/24; - } - } - - sub vcl_recv { - set req.backend = foo; - if (req.http.x-aa) { - set req.backend = test; - } - } - -} -start - From kristian at varnish-cache.org Mon Aug 15 12:00:17 2011 From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=) Date: Mon, 15 Aug 2011 14:00:17 +0200 Subject: [master] 3dffeb9 Add (disabled) test-case for #977 Message-ID: commit 3dffeb9e491d2811667af29931dbb43b71292fc0 Author: Kristian Lyngstol Date: Mon Aug 15 13:59:58 2011 +0200 Add (disabled) test-case for #977 diff --git a/bin/varnishtest/tests.disabled/r00977.vtc b/bin/varnishtest/tests.disabled/r00977.vtc new file mode 100644 index 0000000..e6c61da --- /dev/null +++ b/bin/varnishtest/tests.disabled/r00977.vtc @@ -0,0 +1,28 @@ + +varnishtest "Test proper fallbacks of client director" + +server s1 -repeat 1 { + rxreq + txresp -status 200 +} -start + +varnish v1 -vcl+backend { + director foo client{ + .retries = 5; + { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } + { .backend = s1; .weight = 1;} + } + sub vcl_recv { + set req.backend = foo; + set client.identity = "44.452"; + return (pass); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -expect backend_fail == 1 From perbu at varnish-cache.org Mon Aug 15 12:52:47 2011 From: perbu at varnish-cache.org (Per Buer) Date: Mon, 15 Aug 2011 14:52:47 +0200 Subject: [master] 65f4cf6 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 65f4cf6486c8572e0d948e5b1471340112b8ce6a Merge: 0b23a0f 3dffeb9 Author: Per Buer Date: Mon Aug 15 14:11:54 2011 +0200 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From perbu at varnish-cache.org Mon Aug 15 12:52:47 2011 From: perbu at varnish-cache.org (Per Buer) Date: Mon, 15 Aug 2011 14:52:47 +0200 Subject: [master] 0b23a0f document the transient stevedore Message-ID: commit 0b23a0f083865f75905bf66dde80fc4ac6458463 Author: Per Buer Date: Mon Aug 15 14:11:50 2011 +0200 document the transient stevedore diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index da7ae6b..7813f0b 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -208,6 +208,10 @@ file[,path[,size[,granularity]]] persistence[XXX] New, shiny, better. +Transient[,size] + Storage for transient (short lived) objects. By default this is + unlimited. This storage backend behaves just like the malloc backend and takes the same options. + Management Interface -------------------- From perbu at varnish-cache.org Mon Aug 15 12:52:47 2011 From: perbu at varnish-cache.org (Per Buer) Date: Mon, 15 Aug 2011 14:52:47 +0200 Subject: [master] 47e277b Add emphasis to explain when bans are checked against Message-ID: commit 47e277b389d5c5be1b8f808d7582570a5366d473 Author: Per Buer Date: Mon Aug 15 14:46:13 2011 +0200 Add emphasis to explain when bans are checked against diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst index bc40691..873a687 100644 --- a/doc/sphinx/tutorial/purging.rst +++ b/doc/sphinx/tutorial/purging.rst @@ -89,7 +89,7 @@ they could issue:: Quite powerful, really. Bans are checked when we hit an object in the cache, but before we -deliver it. An object is only checked against newer bans. If you have +deliver it. *An object is only checked against newer bans*. If you have a lot of objects with long TTL in your cache you should be aware of a potential performance impact of having many bans. From perbu at varnish-cache.org Mon Aug 15 12:52:47 2011 From: perbu at varnish-cache.org (Per Buer) Date: Mon, 15 Aug 2011 14:52:47 +0200 Subject: [master] 9bef407 more on bans, the lurker Message-ID: commit 9bef407d3f19337a804cf54505c6953f01e52957 Author: Per Buer Date: Mon Aug 15 14:51:43 2011 +0200 more on bans, the lurker diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst index 873a687..ce1d945 100644 --- a/doc/sphinx/tutorial/purging.rst +++ b/doc/sphinx/tutorial/purging.rst @@ -89,9 +89,18 @@ they could issue:: Quite powerful, really. Bans are checked when we hit an object in the cache, but before we -deliver it. *An object is only checked against newer bans*. If you have -a lot of objects with long TTL in your cache you should be aware of a -potential performance impact of having many bans. +deliver it. *An object is only checked against newer bans*. + +Bans that only match against beresp.* are also processed by a +background worker threads called the *ban lurker*. The ban lurker will +walk the heap and try to match objects and will evict the matching +objects. How aggressive the ban lurker is can be controlled by the +parameter ban_lurker_sleep. + +Bans that are older then the oldest objects in the cache are discarded +without evaluation. If you have a lot of objects with long TTL, that +are seldom accessed you might accumulate a lot of bans. This might +impact CPU usage and thereby performance. You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL:: From perbu at varnish-cache.org Mon Aug 15 12:52:48 2011 From: perbu at varnish-cache.org (Per Buer) Date: Mon, 15 Aug 2011 14:52:48 +0200 Subject: [master] 6b39846 the basics on persistent storage. Explained the semantics quickly so people understand why not all objects survive a crash Message-ID: commit 6b39846d472bd2d633df7744b0d3118f710d26b2 Author: Per Buer Date: Mon Aug 15 14:52:43 2011 +0200 the basics on persistent storage. Explained the semantics quickly so people understand why not all objects survive a crash diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 3396651..162b7c5 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -205,8 +205,32 @@ file[,path[,size[,granularity]]] The default size is the VM page size. The size should be reduced if you have many small objects. -persistence[XXX] - New, shiny, better. +persistent,path,size {experimental} + + Persistent storage. Varnish will store objects in a file in a + manner that will secure the survival of *most* of the objects in + the event of a planned or unplanned shutdown of Varnish. + + The path parameter specifies the path to the backing file. If + the file doesn't exist Varnish will create it. + + The size parameter specifies the size of the backing file. The + size is assumed to be in bytes, unless followed by one of the + following suffixes: + + K, k The size is expressed in kibibytes. + + M, m The size is expressed in mebibytes. + + G, g The size is expressed in gibibytes. + + T, t The size is expressed in tebibytes. + + Varnish will split the file into logical *silos* and write to + the silos in the manner of a circular buffer. Only one silo will + be kept open at any given point in time. Full silos are + *sealed*. When Varnish starts after a shutdown it will discard + the content of any silo that isn't sealed. Transient[,size] Storage for transient (short lived) objects. By default this is From perbu at varnish-cache.org Mon Aug 15 14:02:46 2011 From: perbu at varnish-cache.org (Per Buer) Date: Mon, 15 Aug 2011 16:02:46 +0200 Subject: [master] 6a59dac more on transient. didn't get it rigth the first time. Thanks to scoof for pointing it out. Message-ID: commit 6a59dacccb1fc402c5d1cd703c048835ba400c38 Author: Per Buer Date: Mon Aug 15 16:02:44 2011 +0200 more on transient. didn't get it rigth the first time. Thanks to scoof for pointing it out. diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 162b7c5..df6b87f 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -87,9 +87,11 @@ OPTIONS -S file Path to a file containing a secret used for authorizing access to the management port. --s type[,options] +-s [name=]type[,options] Use the specified storage backend. See Storage Types for a list of supported storage - types. This option can be used multiple times to specify multiple storage files. + types. This option can be used multiple times to specify multiple storage files. You + can name the different backends. Varnish will then reference that backend with the + given name in logs, statistics, etc. -T address[:port] Offer a management interface on the specified address and port. See Management @@ -232,10 +234,12 @@ persistent,path,size {experimental} *sealed*. When Varnish starts after a shutdown it will discard the content of any silo that isn't sealed. -Transient[,size] - Storage for transient (short lived) objects. By default this is - unlimited. This storage backend behaves just like the malloc backend and takes the same options. - +Transient Storage +----------------- + + If you name any of your storage backend "Transient" it will be + used for transient (short lived) objects. By default Varnish + would use an unlimited malloc backend for this. Management Interface -------------------- From phk at varnish-cache.org Mon Aug 15 15:02:25 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 17:02:25 +0200 Subject: [master] 0caee0a My recent 'discover correct -spersistent mmap address' fix broke 32 bit systems. Message-ID: commit 0caee0aa29e8d18cae4e9d66b0cf8b459c056499 Author: Poul-Henning Kamp Date: Mon Aug 15 15:01:39 2011 +0000 My recent 'discover correct -spersistent mmap address' fix broke 32 bit systems. Spotted by: Kristian diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c index a22cc59..c4f2191 100644 --- a/bin/varnishd/storage_persistent_mgt.c +++ b/bin/varnishd/storage_persistent_mgt.c @@ -164,7 +164,7 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) i = read(sc->fd, &sgn, sizeof sgn); assert(i == sizeof sgn); if (!strcmp(sgn.ident, "SILO")) - target = (void*)sgn.mapped; + target = (void*)(uintptr_t)sgn.mapped; else target = NULL; From phk at varnish-cache.org Mon Aug 15 19:48:47 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 21:48:47 +0200 Subject: [master] 149c4d4 Implement a consistent retry policy in the random/client/hash director: Message-ID: commit 149c4d4f6e256deb1ed145cf22849e2bd03ab5b7 Author: Poul-Henning Kamp Date: Mon Aug 15 19:45:58 2011 +0000 Implement a consistent retry policy in the random/client/hash director: If the first (policy-chosen) backend fails to get us a connection, retry a random backend (still according to their weight) until retries are exhausted. Kristian sent a proof of concept patch, I just cleaned it up and made it compile. Thanks to: Kristian Fixes #977 diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c index f85d905..c2b74dd 100644 --- a/bin/varnishd/cache_dir_random.c +++ b/bin/varnishd/cache_dir_random.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2010 Varnish Software AS + * Copyright (c) 2006-2011 Varnish Software AS * All rights reserved. * * Author: Poul-Henning Kamp @@ -26,17 +26,22 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This code is shared between the random and hash directors, because they - * share the same properties and most of the same selection logic. + * This code is shared between the random, client and hash directors, because + * they share the same properties and most of the same selection logic. * - * The random director picks a backend on random, according to weight, - * from the healty subset of backends. + * The random director picks a backend on random. + * + * The hash director picks based on the hash from vcl_hash{} + * + * The client director picks based on client identity or IP-address + * + * In all cases, the choice is by weight of the healthy subset of + * configured backends. + * + * Failures to get a connection are retried, here all three policies + * fall back to a deterministically random choice, by weight in the + * healthy subset. * - * The hash director first tries to locate the "canonical" backend from - * the full set, according to weight, and if it is healthy selects it. - * If the canonical backend is not healthy, we pick a backend according - * to weight from the healthy subset. That way only traffic to unhealthy - * backends gets redistributed. */ #include "config.h" @@ -46,6 +51,7 @@ #include #include +#include #include #include #include @@ -77,114 +83,117 @@ struct vdi_random { unsigned nhosts; }; -static struct vbc * -vdi_random_getfd(const struct director *d, struct sess *sp) +/* + * Applies sha256 using the given context and input/length, and returns + * a double in the range [0...1[ based on the hash. + */ +static double +vdi_random_sha(const char *input, ssize_t len) { - int i, k; - struct vdi_random *vs; - double r, s1; - unsigned u = 0; - struct vbc *vbe; - struct director *d2; struct SHA256Context ctx; - uint8_t sign[SHA256_LEN], *hp = NULL; + uint8_t sign[SHA256_LEN]; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC); + AN(input); + SHA256_Init(&ctx); + SHA256_Update(&ctx, input, len); + SHA256_Final(sign, &ctx); + return (vle32dec(sign) / exp2(32)); +} + +/* + * Sets up the initial seed for picking a backend according to policy. + */ +static double +vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp) +{ + const char *p; + double retval; - if (vs->criteria == c_client) { - /* - * Hash the client IP# ascii representation, rather than - * rely on the raw IP# being a good hash distributor, since - * experience shows this not to be the case. - * We do not hash the port number, to make everybody behind - * a given NAT gateway fetch from the same backend. - */ - SHA256_Init(&ctx); - AN(sp->addr); + switch (vs->criteria) { + case c_client: if (sp->client_identity != NULL) - SHA256_Update(&ctx, sp->client_identity, - strlen(sp->client_identity)); + p = sp->client_identity; else - SHA256_Update(&ctx, sp->addr, strlen(sp->addr)); - SHA256_Final(sign, &ctx); - hp = sign; + p = sp->addr; + retval = vdi_random_sha(p, strlen(p)); + break; + case c_hash: + AN(sp->digest); + retval = vle32dec(sp->digest) / exp2(32); + break; + case c_random: + default: + retval = random() / exp2(31); + break; } - if (vs->criteria == c_hash) { - /* - * Reuse the hash-string, the objective here is to fetch the - * same object on the same backend all the time - */ - hp = sp->digest; + return (retval); +} + +/* + * Find the healthy backend corresponding to the weight r [0...1[ + */ +static struct vbc * +vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r) +{ + double w[vs->nhosts]; + int i; + double s1; + + assert(r >= 0.0 && r < 1.0); + + memset(w, 0, sizeof w); + /* Sum up the weights of healty backends */ + s1 = 0.0; + for (i = 0; i < vs->nhosts; i++) { + if (VDI_Healthy(vs->hosts[i].backend, sp)) + w[i] = vs->hosts[i].weight; + s1 += w[i]; } - /* - * If we are hashing, first try to hit our "canonical backend" - * If that fails, we fall through, and select a weighted backend - * amongst the healthy set. - */ - if (vs->criteria != c_random) { - AN(hp); - u = vle32dec(hp); - r = u / 4294967296.0; - assert(r >= 0.0 && r < 1.0); - r *= vs->tot_weight; - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - s1 += vs->hosts[i].weight; - if (r >= s1) - continue; - d2 = vs->hosts[i].backend; - if (!VDI_Healthy(d2, sp)) - break; - vbe = VDI_GetFd(d2, sp); - if (vbe != NULL) - return (vbe); - break; - } + if (s1 == 0.0) + return (NULL); + + r *= s1; + s1 = 0.0; + for (i = 0; i < vs->nhosts; i++) { + s1 += w[i]; + if (r < s1) + return(VDI_GetFd(vs->hosts[i].backend, sp)); } + return (NULL); +} - for (k = 0; k < vs->retries; ) { - /* Sum up the weights of healty backends */ - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - d2 = vs->hosts[i].backend; - /* XXX: cache result of healty to avoid double work */ - if (VDI_Healthy(d2, sp)) - s1 += vs->hosts[i].weight; - } - - if (s1 == 0.0) - return (NULL); - - if (vs->criteria != c_random) { - r = u / 4294967296.0; - } else { - /* Pick a random threshold in that interval */ - r = random() / 2147483648.0; /* 2^31 */ - } - assert(r >= 0.0 && r < 1.0); - r *= s1; - - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - d2 = vs->hosts[i].backend; - if (!VDI_Healthy(d2, sp)) - continue; - s1 += vs->hosts[i].weight; - if (r >= s1) - continue; - vbe = VDI_GetFd(d2, sp); - if (vbe != NULL) - return (vbe); - break; - } - k++; +/* + * Try the specified number of times to get a backend. + * First one according to policy, after that, deterministically + * random by rehashing the key. + */ +static struct vbc * +vdi_random_getfd(const struct director *d, struct sess *sp) +{ + int k; + struct vdi_random *vs; + double r; + struct vbc *vbe; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC); + + r = vdi_random_init_seed(vs, sp); + + for (k = 0; k < vs->retries; k++) { + vbe = vdi_random_pick_one(sp, vs, r); + if (vbe != NULL) + return (vbe); + r = vdi_random_sha((void *)&r, sizeof(r)); } return (NULL); } +/* + * Healthy if just a single backend is... + */ static unsigned vdi_random_healthy(const struct director *d, const struct sess *sp) { diff --git a/bin/varnishtest/tests.disabled/r00977.vtc b/bin/varnishtest/tests.disabled/r00977.vtc deleted file mode 100644 index e6c61da..0000000 --- a/bin/varnishtest/tests.disabled/r00977.vtc +++ /dev/null @@ -1,28 +0,0 @@ - -varnishtest "Test proper fallbacks of client director" - -server s1 -repeat 1 { - rxreq - txresp -status 200 -} -start - -varnish v1 -vcl+backend { - director foo client{ - .retries = 5; - { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } - { .backend = s1; .weight = 1;} - } - sub vcl_recv { - set req.backend = foo; - set client.identity = "44.452"; - return (pass); - } -} -start - -client c1 { - txreq - rxresp - expect resp.status == 200 -} -run - -varnish v1 -expect backend_fail == 1 diff --git a/bin/varnishtest/tests/r00977.vtc b/bin/varnishtest/tests/r00977.vtc new file mode 100644 index 0000000..d15b55d --- /dev/null +++ b/bin/varnishtest/tests/r00977.vtc @@ -0,0 +1,46 @@ + +varnishtest "Test proper fallbacks of client director" + +server s1 { + rxreq + txresp -status 200 + accept + rxreq + txresp -status 200 +} -start + +varnish v1 -vcl+backend { + director foo client{ + .retries = 5; + { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } + { .backend = s1; .weight = 1;} + } + director bar client{ + .retries = 1; + { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } + { .backend = s1; .weight = 1;} + } + sub vcl_recv { + if (req.url ~ "/one") { + set req.backend = foo; + } else { + set req.backend = bar; + } + # Carefully chosen seed that'll give us bad backend on + # first try and good on second. + set client.identity = "1.4"; + return (pass); + } +} -start + +client c1 { + txreq -url "/one" + rxresp + expect resp.status == 200 + + txreq -url "/two" + rxresp + expect resp.status == 503 +} -run + +varnish v1 -expect backend_fail == 2 From phk at varnish-cache.org Mon Aug 15 20:48:38 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 22:48:38 +0200 Subject: [master] b38682b Make sure the entire waiting list is rushed when an object goes non-busy. Not sure what I thought when I changed it last, but it was clearly not smart thinking. Message-ID: commit b38682bb5aa5d36c856a634e121b15c0994c4fa6 Author: Poul-Henning Kamp Date: Mon Aug 15 20:47:21 2011 +0000 Make sure the entire waiting list is rushed when an object goes non-busy. Not sure what I thought when I changed it last, but it was clearly not smart thinking. Spotted by: Martin Test case by: Martin Fixes #963 diff --git a/bin/varnishtest/tests/r00963.vtc b/bin/varnishtest/tests/r00963.vtc new file mode 100644 index 0000000..6e8fc1d --- /dev/null +++ b/bin/varnishtest/tests/r00963.vtc @@ -0,0 +1,45 @@ +varnishtest "Test hsh_rush" + +server s1 { + rxreq + sema r1 sync 5 + txresp -bodylen 10 +} -start + +varnish v1 -vcl+backend { +} -start + +varnish v1 -cliok "param.set rush_exponent 2" + +client c1 { + txreq + sema r1 sync 5 + rxresp + expect resp.bodylen == 10 +} -start + +client c2 { + txreq + sema r1 sync 5 + rxresp + expect resp.bodylen == 10 +} -start + +client c3 { + txreq + sema r1 sync 5 + rxresp + expect resp.bodylen == 10 +} -start + +client c4 { + txreq + sema r1 sync 5 + rxresp + expect resp.bodylen == 10 +} -start + +client c1 -wait +client c2 -wait +client c3 -wait +client c4 -wait From phk at varnish-cache.org Mon Aug 15 21:53:50 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2011 23:53:50 +0200 Subject: [master] 466c078 Oops, didn't get this bit of the #963 fix in. Message-ID: commit 466c078f04bb871cc7edc7b5d2844259ca5c9483 Author: Poul-Henning Kamp Date: Mon Aug 15 21:53:31 2011 +0000 Oops, didn't get this bit of the #963 fix in. diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 739fee0..39afde6 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -490,8 +490,6 @@ hsh_rush(struct objhead *oh) CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_AssertHeld(&oh->mtx); wl = oh->waitinglist; - if (wl == NULL) - return; CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC); for (u = 0; u < params->rush_exponent; u++) { sp = VTAILQ_FIRST(&wl->list); @@ -632,7 +630,8 @@ HSH_Unbusy(const struct sess *sp) AZ(sp->wrk->nbusyobj); sp->wrk->nbusyobj = oc->busyobj; oc->busyobj = NULL; - hsh_rush(oh); + if (oh->waitinglist != NULL) + hsh_rush(oh); AN(oc->ban); Lck_Unlock(&oh->mtx); assert(oc_getobj(sp->wrk, oc) == o); @@ -709,7 +708,7 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo) /* Must have an object */ AN(oc->methods); } - if (oc->flags & OC_F_BUSY) + if (oh->waitinglist != NULL) hsh_rush(oh); Lck_Unlock(&oh->mtx); if (r != 0) From phk at varnish-cache.org Wed Aug 17 07:10:07 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 09:10:07 +0200 Subject: [master] 298c7c3 We can not use a lenght based response when we transform (gzip/gunzip) we stream and backend didn't send c-l. Message-ID: commit 298c7c382b731dbde290ce32d3199d076a4d845d Author: Poul-Henning Kamp Date: Wed Aug 17 07:08:11 2011 +0000 We can not use a lenght based response when we transform (gzip/gunzip) we stream and backend didn't send c-l. Fixes #980 Testcase by: Martin diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 42d4b3b..672e1dc 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -166,8 +166,8 @@ cnt_prepresp(struct sess *sp) sp->wrk->res_mode = 0; - if (!sp->wrk->do_stream || - (sp->wrk->h_content_length != NULL && !sp->wrk->do_gunzip)) + if ((sp->wrk->h_content_length != NULL || !sp->wrk->do_stream) && + !sp->wrk->do_gzip && !sp->wrk->do_gunzip) sp->wrk->res_mode |= RES_LEN; if (!sp->disable_esi && sp->obj->esidata != NULL) { diff --git a/bin/varnishtest/tests/r00980.vtc b/bin/varnishtest/tests/r00980.vtc new file mode 100644 index 0000000..9591786 --- /dev/null +++ b/bin/varnishtest/tests/r00980.vtc @@ -0,0 +1,29 @@ +varnishtest "r00980 test gzip on fetch with content_length and do_stream" + +server s1 { + rxreq + expect req.url == "/foobar" + expect req.http.accept-encoding == "gzip" + txresp -bodylen 43 +} -start + +varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { + + sub vcl_fetch { + set beresp.do_gzip = true; + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq -url /foobar -hdr "Accept-Encoding: gzip" + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.bodylen == 43 + + txreq -url /foobar + rxresp + expect resp.http.content-encoding == "resp.http.content-encoding" + expect resp.bodylen == 43 +} -run From phk at varnish-cache.org Wed Aug 17 07:16:31 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 09:16:31 +0200 Subject: [master] eab8652 restart in vcl_deliver{} would crash in vcl_fetch{} due to missing cleanup. Message-ID: commit eab8652f3f2c29275aa2236e5c967d0d1d07de24 Author: Poul-Henning Kamp Date: Wed Aug 17 07:15:30 2011 +0000 restart in vcl_deliver{} would crash in vcl_fetch{} due to missing cleanup. Found & Fixed by: Martin Fixes #979 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 672e1dc..47f3720 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -233,6 +233,7 @@ cnt_prepresp(struct sess *sp) AZ(sp->obj); sp->restarts++; sp->director = NULL; + sp->wrk->h_content_length = NULL; http_Setup(sp->wrk->bereq, NULL); http_Setup(sp->wrk->beresp, NULL); http_Setup(sp->wrk->resp, NULL); diff --git a/bin/varnishtest/tests/r00979.vtc b/bin/varnishtest/tests/r00979.vtc new file mode 100644 index 0000000..bc72efc --- /dev/null +++ b/bin/varnishtest/tests/r00979.vtc @@ -0,0 +1,29 @@ +varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver" + +server s1 { + rxreq + txresp -status 200 -body "1" + expect_close + + accept + rxreq + txresp -status 200 -body "11" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = true; + } + sub vcl_deliver { + if (req.restarts == 0) { + return (restart); + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 2 +} -run From phk at varnish-cache.org Wed Aug 17 07:24:52 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 09:24:52 +0200 Subject: [master] 81010e4 Cleare do_stream on all esi objects, including included objects. Message-ID: commit 81010e415ca34634c01db5d6245c224e2e538f70 Author: Poul-Henning Kamp Date: Wed Aug 17 07:24:13 2011 +0000 Cleare do_stream on all esi objects, including included objects. Found & Fixed by: Martin Fixes #978 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 47f3720..93bfc75 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -718,7 +718,7 @@ cnt_fetchbody(struct sess *sp) else if (sp->wrk->is_gzip) sp->wrk->vfp = &vfp_testgzip; - if (sp->wrk->do_esi) + if (sp->wrk->do_esi || sp->esi_level > 0) sp->wrk->do_stream = 0; if (!sp->wantbody) sp->wrk->do_stream = 0; diff --git a/bin/varnishtest/tests/r00978.vtc b/bin/varnishtest/tests/r00978.vtc new file mode 100644 index 0000000..39b89cb --- /dev/null +++ b/bin/varnishtest/tests/r00978.vtc @@ -0,0 +1,40 @@ +varnishtest "r00978.vtc Test esi_level > 0 and do_stream" + +server s1 { + rxreq + expect req.url == "/" + txresp -body { + + Before include + + After include + } + rxreq + expect req.url == "/body1" + txresp -body { + Included file + } +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.esi_level > 0) { + set req.url = req.url + req.esi_level; + } + } + sub vcl_fetch { + if (req.url == "/") { + set beresp.do_esi = true; + } + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 65 + expect resp.status == 200 +} -run + +varnish v1 -expect esi_errors == 0 From perbu at varnish-cache.org Wed Aug 17 08:18:19 2011 From: perbu at varnish-cache.org (Per Buer) Date: Wed, 17 Aug 2011 10:18:19 +0200 Subject: [master] 4c2945d hit_for_pass in vcl_fetch Message-ID: commit 4c2945dbcf1a8372ebed900f6998436b5d550199 Author: Per Buer Date: Wed Aug 17 10:10:07 2011 +0200 hit_for_pass in vcl_fetch diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index a16d7f5..c193e08 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -485,8 +485,12 @@ vcl_fetch error code [reason] Return the specified error code to the client and abandon the request. - pass - Switch to pass mode. Control will eventually pass to vcl_pass. + hit_for_pass + Pass in fetch. This will create a hit_for_pass object. Note that + the TTL for the hit_for_pass object will be set to what the + current value of beresp.ttl. Control will be handled to + vcl_deliver on the current request, but subsequent requests will + go directly to vcl_pass based on the hit_for_pass object. restart Restart the transaction. Increases the restart counter. If the number From perbu at varnish-cache.org Wed Aug 17 08:18:20 2011 From: perbu at varnish-cache.org (Per Buer) Date: Wed, 17 Aug 2011 10:18:20 +0200 Subject: [master] 8766504 std.log missing () Message-ID: commit 876650490605d218315357be23fea83a279aff11 Author: Per Buer Date: Wed Aug 17 10:18:15 2011 +0200 std.log missing () diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 1a31733..2afbf64 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -21,7 +21,7 @@ String concatenation did not have an operator previously, but this has now been becomes import std; - std.log "log something"; + std.log("log something"); You only need to import std once. From perbu at varnish-cache.org Wed Aug 17 08:18:20 2011 From: perbu at varnish-cache.org (Per Buer) Date: Wed, 17 Aug 2011 10:18:20 +0200 Subject: [master] 2bedcf2 remove cruft about -o Message-ID: commit 2bedcf2f99d98c376ee6d38df9dc004043a732e0 Author: Per Buer Date: Wed Aug 17 10:10:38 2011 +0200 remove cruft about -o diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst index be9c9c5..d8cd200 100644 --- a/doc/sphinx/reference/varnishlog.rst +++ b/doc/sphinx/reference/varnishlog.rst @@ -52,10 +52,10 @@ The following options are available: -k num Only show the first num log records. -m tag:regex only list transactions where tag matches regex. Multiple - -m options are AND-ed together. Can not be combined with -O + -m options are AND-ed together. --n Specifies the name of the varnishd instance to get logs from. If -n is not - specified, the host name is used. +-n Specifies the name of the varnishd instance to get logs from. If + -n is not specified, the host name is used. -o Ignored for compatibility with earlier versions. From phk at varnish-cache.org Wed Aug 17 08:34:20 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 10:34:20 +0200 Subject: [master] 38415a2 Add a "sendhex" verb Message-ID: commit 38415a2ae565fa70dbb56a3cc5043fd2edbb9179 Author: Poul-Henning Kamp Date: Wed Aug 17 08:31:18 2011 +0000 Add a "sendhex" verb diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 55228d5..366a5dd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -904,6 +904,47 @@ cmd_http_send(CMD_ARGS) } /********************************************************************** + * Send a hex string + */ + +static void +cmd_http_sendhex(CMD_ARGS) +{ + struct http *hp; + char buf[3], *q; + uint8_t *p; + int i, j, l; + + (void)cmd; + (void)vl; + CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); + AN(av[1]); + AZ(av[2]); + l = strlen(av[1]) / 2; + p = malloc(l); + AN(p); + q = av[1]; + for (i = 0; i < l; i++) { + while (vct_issp(*q)) + q++; + if (*q == '\0') + break; + memcpy(buf, q, 2); + q += 2; + buf[2] = '\0'; + if (!vct_ishex(buf[0]) || !vct_ishex(buf[1])) + vtc_log(hp->vl, 0, "Illegal Hex char \"%c%c\"", + buf[0], buf[1]); + p[i] = strtoul(buf, NULL, 16); + } + vtc_hexdump(hp->vl, 4, "sendhex", (void*)p, i); + j = write(hp->fd, p, i); + assert(j == i); + free(p); + +} + +/********************************************************************** * Send a string as chunked encoding */ @@ -1080,6 +1121,7 @@ static const struct cmds http_cmds[] = { { "gunzip", cmd_http_gunzip_body }, { "expect", cmd_http_expect }, { "send", cmd_http_send }, + { "sendhex", cmd_http_sendhex }, { "chunked", cmd_http_chunked }, { "chunkedlen", cmd_http_chunkedlen }, { "delay", cmd_delay }, diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index f0841c0..18f295a 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -227,8 +227,10 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha { int nl = 1; unsigned l; + double tx; CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC); + tx = TIM_mono() - t0; assert(len >= 0); assert(lvl < NLEAD); AZ(pthread_mutex_lock(&vl->mtx)); @@ -236,8 +238,8 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha if (pfx == NULL) pfx = ""; if (str == NULL) - VSB_printf(vl->vsb, "%s %-4s %s(null)\n", - lead[lvl], vl->id, pfx); + VSB_printf(vl->vsb, "%s %-4s %4.1f %s| (null)", + lead[lvl], vl->id, tx, pfx); else { for (l = 0; l < len; l++, str++) { if (l > 512) { @@ -245,8 +247,8 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha break; } if (nl) { - VSB_printf(vl->vsb, "%s %-4s %s| ", - lead[lvl], vl->id, pfx); + VSB_printf(vl->vsb, "%s %-4s %4.1f %s| ", + lead[lvl], vl->id, tx, pfx); nl = 0; } VSB_printf(vl->vsb, " %02x", *str); From phk at varnish-cache.org Wed Aug 17 08:34:20 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 10:34:20 +0200 Subject: [master] a8481d0 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit a8481d09784d86ce8e2ac0cf033cf2540f00eee2 Merge: 38415a2 8766504 Author: Poul-Henning Kamp Date: Wed Aug 17 08:34:18 2011 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From perbu at varnish-cache.org Wed Aug 17 09:07:16 2011 From: perbu at varnish-cache.org (Per Buer) Date: Wed, 17 Aug 2011 11:07:16 +0200 Subject: [master] f4084bb pulled one of the last changes on -m (removed -O by accident) Message-ID: commit f4084bb487fb6e520bf5432ee4dcc1c3470240cc Author: Per Buer Date: Wed Aug 17 11:06:23 2011 +0200 pulled one of the last changes on -m (removed -O by accident) diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst index d8cd200..9e6ada9 100644 --- a/doc/sphinx/reference/varnishlog.rst +++ b/doc/sphinx/reference/varnishlog.rst @@ -52,7 +52,8 @@ The following options are available: -k num Only show the first num log records. -m tag:regex only list transactions where tag matches regex. Multiple - -m options are AND-ed together. + -m options are AND-ed together. Can not be combined with + -O. -n Specifies the name of the varnishd instance to get logs from. If -n is not specified, the host name is used. From perbu at varnish-cache.org Wed Aug 17 09:07:16 2011 From: perbu at varnish-cache.org (Per Buer) Date: Wed, 17 Aug 2011 11:07:16 +0200 Subject: [master] e416c09 added () to EXAMPLES for std.log Message-ID: commit e416c09b8d1ae1bee6af6ce249392557640fb23a Author: Per Buer Date: Wed Aug 17 11:06:41 2011 +0200 added () to EXAMPLES for std.log diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index c193e08..281b653 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -834,7 +834,7 @@ based on the request URL::: sub vcl_fetch { if (beresp.ttl < 120s) { - std.log "Adjusting TTL"; + std.log("Adjusting TTL"); set beresp.ttl = 120s; } } From perbu at varnish-cache.org Wed Aug 17 09:07:16 2011 From: perbu at varnish-cache.org (Per Buer) Date: Wed, 17 Aug 2011 11:07:16 +0200 Subject: [master] d4681a6 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit d4681a6f83ff5a683cfb504085b0b7143c88d8cc Merge: e416c09 a8481d0 Author: Per Buer Date: Wed Aug 17 11:07:14 2011 +0200 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From tfheen at varnish-cache.org Wed Aug 17 09:25:22 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:22 +0200 Subject: [3.0] ef07e1d Pass priv2 member instead of objcore pointer, thats all we need. Message-ID: commit ef07e1ddbc3cd6642e9ddc5f6168b1d5399e896c Author: Poul-Henning Kamp Date: Mon Jun 20 07:38:19 2011 +0000 Pass priv2 member instead of objcore pointer, thats all we need. diff --git a/bin/varnishd/storage_persistent_silo.c b/bin/varnishd/storage_persistent_silo.c index d4ccf1e..a1eb98b 100644 --- a/bin/varnishd/storage_persistent_silo.c +++ b/bin/varnishd/storage_persistent_silo.c @@ -308,15 +308,13 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg) */ static struct smp_object * -smp_find_so(const struct smp_seg *sg, const struct objcore *oc) +smp_find_so(const struct smp_seg *sg, unsigned priv2) { struct smp_object *so; - unsigned smp_idx; - smp_idx = oc->priv2; - assert(smp_idx > 0); - assert(smp_idx <= sg->p.lobjlist); - so = &sg->objs[sg->p.lobjlist - smp_idx]; + assert(priv2 > 0); + assert(priv2 <= sg->p.lobjlist); + so = &sg->objs[sg->p.lobjlist - priv2]; return (so); } @@ -390,7 +388,7 @@ smp_oc_getobj(struct worker *wrk, struct objcore *oc) AZ(oc->flags & OC_F_NEEDFIXUP); CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC); - so = smp_find_so(sg, oc); + so = smp_find_so(sg, oc->priv2); o = (void*)(sg->sc->base + so->ptr); /* @@ -456,7 +454,7 @@ smp_oc_updatemeta(struct objcore *oc) CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC); CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC); - so = smp_find_so(sg, oc); + so = smp_find_so(sg, oc->priv2); mttl = EXP_Grace(NULL, o); @@ -481,7 +479,7 @@ smp_oc_freeobj(struct objcore *oc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC); - so = smp_find_so(sg, oc); + so = smp_find_so(sg, oc->priv2); Lck_Lock(&sg->sc->mtx); so->ttl = 0; From tfheen at varnish-cache.org Wed Aug 17 09:25:22 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:22 +0200 Subject: [3.0] d5ee593 Optimize the objhead layout to avoid packing on 64bit systems. Message-ID: commit d5ee593a20f6cd5b09a1a1ec41635522202c1d5f Author: Poul-Henning Kamp Date: Mon Jun 20 08:09:48 2011 +0000 Optimize the objhead layout to avoid packing on 64bit systems. diff --git a/bin/varnishd/hash_slinger.h b/bin/varnishd/hash_slinger.h index b9430c7..2c142cf 100644 --- a/bin/varnishd/hash_slinger.h +++ b/bin/varnishd/hash_slinger.h @@ -75,8 +75,8 @@ struct objhead { unsigned magic; #define OBJHEAD_MAGIC 0x1b96615d - struct lock mtx; int refcnt; + struct lock mtx; VTAILQ_HEAD(,objcore) objcs; unsigned char digest[DIGEST_LEN]; struct waitinglist *waitinglist; From tfheen at varnish-cache.org Wed Aug 17 09:25:22 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:22 +0200 Subject: [3.0] c20bb7d Reduce the http.protover field from double to uint8_t for space reasons. Message-ID: commit c20bb7dacb6dc0d34bda1ff6dc15279d20ade8fa Author: Poul-Henning Kamp Date: Mon Jun 20 08:33:35 2011 +0000 Reduce the http.protover field from double to uint8_t for space reasons. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index f3558ff..0fe1271 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -171,7 +171,7 @@ struct http { unsigned char conds; /* If-* headers present */ enum httpwhence logtag; int status; - double protover; + uint8_t protover; unsigned shd; /* Size of hd space */ txt *hd; diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 4375344..4a17c56 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -200,7 +200,7 @@ cnt_prepresp(struct sess *sp) sp->wrk->res_mode |= RES_LEN; else if (!sp->wantbody) { /* Nothing */ - } else if (sp->http->protover >= 1.1) { + } else if (sp->http->protover >= 11) { sp->wrk->res_mode |= RES_CHUNKED; } else { sp->wrk->res_mode |= RES_EOF; diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 8df780a..5d8050a 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -420,7 +420,7 @@ http_DoConnection(const struct http *hp) unsigned u; if (!http_GetHdr(hp, H_Connection, &p)) { - if (hp->protover < 1.1) + if (hp->protover < 11) return ("not HTTP/1.1"); return (NULL); } @@ -640,11 +640,11 @@ http_ProtoVer(struct http *hp) { if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.0")) - hp->protover = 1.0; + hp->protover = 10; else if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) - hp->protover = 1.1; + hp->protover = 11; else - hp->protover = 0.9; + hp->protover = 9; } diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c index 83ebd71..7b2525f 100644 --- a/bin/varnishd/rfc2616.c +++ b/bin/varnishd/rfc2616.c @@ -186,7 +186,7 @@ RFC2616_Body(const struct sess *sp) hp = sp->wrk->beresp; - if (hp->protover < 1.1 && !http_HdrIs(hp, H_Connection, "keep-alive")) + if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive")) sp->wrk->do_close = 1; else if (http_HdrIs(hp, H_Connection, "close")) sp->wrk->do_close = 1; @@ -262,7 +262,7 @@ RFC2616_Body(const struct sess *sp) return (BS_EOF); } - if (hp->protover < 1.1) { + if (hp->protover < 11) { /* * With no Connection header, assume EOF. */ From tfheen at varnish-cache.org Wed Aug 17 09:25:22 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:22 +0200 Subject: [3.0] ece423d reduce some fieldwidths and pack struct http to avoid packing. Message-ID: commit ece423dc74827fbd6fe0022cf9c055bcda745544 Author: Poul-Henning Kamp Date: Mon Jun 20 08:39:17 2011 +0000 reduce some fieldwidths and pack struct http to avoid packing. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 0fe1271..48d1456 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -166,18 +166,17 @@ struct http { unsigned magic; #define HTTP_MAGIC 0x6428b5c9 - struct ws *ws; - - unsigned char conds; /* If-* headers present */ enum httpwhence logtag; - int status; - uint8_t protover; - unsigned shd; /* Size of hd space */ + struct ws *ws; txt *hd; unsigned char *hdf; #define HDF_FILTER (1 << 0) /* Filtered by Connection */ - unsigned nhd; /* Next free hd */ + uint16_t shd; /* Size of hd space */ + uint16_t nhd; /* Next free hd */ + uint16_t status; + uint8_t protover; + uint8_t conds; /* If-* headers present */ }; /*-------------------------------------------------------------------- From tfheen at varnish-cache.org Wed Aug 17 09:25:23 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:23 +0200 Subject: [3.0] 7392101 Pack struct ws better to avoid padding. Message-ID: commit 73921013ac8ec03f3174de715e0c83c8019abe43 Author: Poul-Henning Kamp Date: Mon Jun 20 09:16:20 2011 +0000 Pack struct ws better to avoid padding. Repack struct object, reducing a couple of field widths along the way. This shaves 16 bytes of struct object. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 48d1456..f8dedb1 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -143,12 +143,12 @@ enum step { struct ws { unsigned magic; #define WS_MAGIC 0x35fac554 + unsigned overflow; /* workspace overflowed */ const char *id; /* identity */ char *s; /* (S)tart of buffer */ char *f; /* (F)ree pointer */ char *r; /* (R)eserved length */ char *e; /* (E)nd of buffer */ - int overflow; /* workspace overflowed */ }; /*-------------------------------------------------------------------- @@ -479,12 +479,13 @@ struct object { struct objcore *objcore; struct ws ws_o[1]; - unsigned char *vary; - unsigned response; + uint8_t *vary; + unsigned hits; + uint16_t response; /* XXX: make bitmap */ - unsigned gziped; + uint8_t gziped; /* Bit positions in the gzip stream */ ssize_t gzip_start; ssize_t gzip_last; @@ -507,7 +508,6 @@ struct object { double last_use; - int hits; }; /* -------------------------------------------------------------------*/ @@ -566,7 +566,7 @@ struct sess { unsigned handling; unsigned char sendbody; unsigned char wantbody; - int err_code; + uint16_t err_code; const char *err_reason; VTAILQ_ENTRY(sess) list; diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c index 387dffb..2d3066c 100644 --- a/bin/varnishd/cache_cli.c +++ b/bin/varnishd/cache_cli.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -124,7 +125,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) (void)priv; #define SZOF(foo) VCLI_Out(cli, \ - "sizeof(%s) = %zd = 0x%zx\n", #foo, sizeof(foo), sizeof(foo)); + "sizeof(%s) = %zd = 0x%zx\n", #foo, sizeof(foo), sizeof(foo)) SZOF(struct ws); SZOF(struct http); SZOF(struct http_conn); @@ -139,6 +140,63 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) SZOF(struct vbc); SZOF(struct VSC_C_main); SZOF(struct lock); +#define OFOF(foo, bar) { foo __foo; VCLI_Out(cli, \ + "%-30s = 0x%4zx @ 0x%4zx\n", \ + #foo "." #bar, sizeof(__foo.bar), offsetof(foo, bar)); } +#if 0 + OFOF(struct objhead, magic); + OFOF(struct objhead, refcnt); + OFOF(struct objhead, mtx); + OFOF(struct objhead, objcs); + OFOF(struct objhead, digest); + OFOF(struct objhead, waitinglist); + OFOF(struct objhead, _u); +#endif +#if 0 + OFOF(struct http, magic); + OFOF(struct http, logtag); + OFOF(struct http, ws); + OFOF(struct http, hd); + OFOF(struct http, hdf); + OFOF(struct http, shd); + OFOF(struct http, nhd); + OFOF(struct http, status); + OFOF(struct http, protover); + OFOF(struct http, conds); +#endif +#if 0 + OFOF(struct storage, magic); + OFOF(struct storage, fd); + OFOF(struct storage, where); + OFOF(struct storage, list); + OFOF(struct storage, stevedore); + OFOF(struct storage, priv); + OFOF(struct storage, ptr); + OFOF(struct storage, len); + OFOF(struct storage, space); +#endif + OFOF(struct object, magic); + OFOF(struct object, xid); + OFOF(struct object, objstore); + OFOF(struct object, objcore); + OFOF(struct object, ws_o); + OFOF(struct object, vary); + OFOF(struct object, hits); + OFOF(struct object, response); + OFOF(struct object, gziped); + OFOF(struct object, gzip_start); + OFOF(struct object, gzip_last); + OFOF(struct object, gzip_stop); + OFOF(struct object, len); + OFOF(struct object, age); + OFOF(struct object, entered); + OFOF(struct object, exp); + OFOF(struct object, last_modified); + OFOF(struct object, last_lru); + OFOF(struct object, http); + OFOF(struct object, store); + OFOF(struct object, esidata); + OFOF(struct object, last_use); } /*--------------------------------------------------------------------*/ From tfheen at varnish-cache.org Wed Aug 17 09:25:23 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:23 +0200 Subject: [3.0] f9c460c ifdef offsetoff output in debug.sizeof Message-ID: commit f9c460c4e69398fa2fc2b78b4c0520bf173f0e48 Author: Poul-Henning Kamp Date: Mon Jun 20 09:28:41 2011 +0000 ifdef offsetoff output in debug.sizeof diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c index 2d3066c..60e63bf 100644 --- a/bin/varnishd/cache_cli.c +++ b/bin/varnishd/cache_cli.c @@ -175,6 +175,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) OFOF(struct storage, len); OFOF(struct storage, space); #endif +#if 0 OFOF(struct object, magic); OFOF(struct object, xid); OFOF(struct object, objstore); @@ -197,6 +198,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) OFOF(struct object, store); OFOF(struct object, esidata); OFOF(struct object, last_use); +#endif } /*--------------------------------------------------------------------*/ From tfheen at varnish-cache.org Wed Aug 17 09:25:23 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:23 +0200 Subject: [3.0] 22c0814 Move the length field to the front of entries in the vary string. Message-ID: commit 22c0814c4e9f795d0962c69a7acdf6b36df1705a Author: Poul-Henning Kamp Date: Tue Jun 21 09:38:55 2011 +0000 Move the length field to the front of entries in the vary string. diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c index 1300a20..c8578d1 100644 --- a/bin/varnishd/cache_vary.c +++ b/bin/varnishd/cache_vary.c @@ -41,13 +41,13 @@ * The vary matching string has the following format: * * Sequence of: { + * \ Length of header contents. + * / * \ *

\ Same format as argument to http_GetHdr() * ':' / * '\0' / - * \ Length of header contents. - * / - *

Only present if length != 0xffff + *

> Only present if length != 0xffff * } * '\0' */ @@ -97,9 +97,6 @@ VRY_Create(const struct sess *sp, const struct http *hp) (char)(1 + (q - p)), (int)(q - p), p, 0); AZ(VSB_finish(sbh)); - /* Append to vary matching string */ - VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh)); - if (http_GetHdr(sp->http, VSB_data(sbh), &h)) { /* Trim leading and trailing space */ while (isspace(*h)) @@ -110,12 +107,15 @@ VRY_Create(const struct sess *sp, const struct http *hp) /* Encode two byte length and contents */ l = e - h; assert(!(l & ~0xffff)); - VSB_printf(sb, "%c%c", (unsigned)l >> 8, l & 0xff); - VSB_bcat(sb, h, e - h); } else { - /* Mark as "not present" */ - VSB_printf(sb, "%c%c", 0xff, 0xff); + e = h; + l = 0xffff; } + VSB_printf(sb, "%c%c", (unsigned)l >> 8, l & 0xff); + /* Append to vary matching string */ + VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh)); + if (e != h) + VSB_bcat(sb, h, e - h); while (isspace(*q)) q++; @@ -125,7 +125,7 @@ VRY_Create(const struct sess *sp, const struct http *hp) p = q; } /* Terminate vary matching string */ - VSB_printf(sb, "%c", 0); + VSB_printf(sb, "%c%c%c", 0xff, 0xff, 0); VSB_delete(sbh); AZ(VSB_finish(sb)); @@ -133,12 +133,16 @@ VRY_Create(const struct sess *sp, const struct http *hp) } int -VRY_Match(const struct sess *sp, const unsigned char *vary) +VRY_Match(const struct sess *sp, const uint8_t *vary) { char *h, *e; int i, l, lh; - while (*vary) { + while (1) { + l = vary[0] * 256 + vary[1]; + vary += 2; + if (!*vary) + break; if (params->http_gzip_support && !strcasecmp(H_Accept_Encoding, (const char*)vary)) { @@ -152,8 +156,6 @@ VRY_Match(const struct sess *sp, const unsigned char *vary) * setting of http_gzip_support. */ vary += *vary + 2; - l = vary[0] * 256 + vary[1]; - vary += 2; if (l != 0xffff) vary += l; continue; @@ -162,10 +164,6 @@ VRY_Match(const struct sess *sp, const unsigned char *vary) i = http_GetHdr(sp->http, (const char*)vary, &h); vary += *vary + 2; - /* Expected length of header (or 0xffff) */ - l = vary[0] * 256 + vary[1]; - vary += 2; - /* Fail if we have the header, but shouldn't */ if (i && l == 0xffff) return (0); From tfheen at varnish-cache.org Wed Aug 17 09:25:24 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:24 +0200 Subject: [3.0] b6e6fe9 Use VCT instead of . Trust HTTP_GetHdr() to remove leading white space. Message-ID: commit b6e6fe93b55f3c58c648e9006fccbea0eeacba2d Author: Poul-Henning Kamp Date: Tue Jun 21 09:46:35 2011 +0000 Use VCT instead of . Trust HTTP_GetHdr() to remove leading white space. diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c index c8578d1..8eab103 100644 --- a/bin/varnishd/cache_vary.c +++ b/bin/varnishd/cache_vary.c @@ -54,11 +54,11 @@ #include "config.h" -#include #include #include #include "cache.h" +#include "vct.h" struct vsb * VRY_Create(const struct sess *sp, const struct http *hp) @@ -86,9 +86,9 @@ VRY_Create(const struct sess *sp, const struct http *hp) for (p = v; *p; p++) { /* Find next header-name */ - if (isspace(*p)) + if (vct_issp(*p)) continue; - for (q = p; *q && !isspace(*q) && *q != ','; q++) + for (q = p; *q && !vct_issp(*q) && *q != ','; q++) continue; /* Build a header-matching string out of it */ @@ -98,11 +98,10 @@ VRY_Create(const struct sess *sp, const struct http *hp) AZ(VSB_finish(sbh)); if (http_GetHdr(sp->http, VSB_data(sbh), &h)) { - /* Trim leading and trailing space */ - while (isspace(*h)) - h++; + AZ(vct_issp(*h)); + /* Trim trailing space */ e = strchr(h, '\0'); - while (e > h && isspace(e[-1])) + while (e > h && vct_issp(e[-1])) e--; /* Encode two byte length and contents */ l = e - h; @@ -117,7 +116,7 @@ VRY_Create(const struct sess *sp, const struct http *hp) if (e != h) VSB_bcat(sb, h, e - h); - while (isspace(*q)) + while (vct_issp(*q)) q++; if (*q == '\0') break; @@ -175,11 +174,9 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) if (!i) continue; - /* Trim leading & trailing space */ - while (isspace(*h)) - h++; + /* Trim trailing space */ e = strchr(h, '\0'); - while (e > h && isspace(e[-1])) + while (e > h && vct_issp(e[-1])) e--; /* Fail if wrong length */ From tfheen at varnish-cache.org Wed Aug 17 09:25:24 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:24 +0200 Subject: [3.0] 26c0049 Overhaul the Vary matching code to make it faster, by assuming (but do not trusting) that all objects under an objhdr have the same Vary string. Message-ID: commit 26c00497dc2f7fbbc67a2989e77af4a2ec978181 Author: Poul-Henning Kamp Date: Tue Jun 21 11:55:07 2011 +0000 Overhaul the Vary matching code to make it faster, by assuming (but do not trusting) that all objects under an objhdr have the same Vary string. Build a vary string for the request on the wrk->ws as we do the comparison and use it as a cache for subsequent comparisons. This will also be first step of sky's "pre-vary" diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index f8dedb1..22fa5ff 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -298,7 +298,11 @@ struct worker { uint32_t *wlb, *wlp, *wle; unsigned wlr; + /* Lookup stuff */ struct SHA256Context *sha256ctx; + uint8_t *vary_b; + uint8_t *vary_l; + uint8_t *vary_e; struct http_conn htc[1]; struct ws ws[1]; @@ -880,7 +884,7 @@ void RES_StreamPoll(const struct sess *sp); /* cache_vary.c */ struct vsb *VRY_Create(const struct sess *sp, const struct http *hp); -int VRY_Match(const struct sess *sp, const unsigned char *vary); +int VRY_Match(const struct sess *sp, const uint8_t *vary); /* cache_vcl.c */ void VCL_Init(void); diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 2a6dcc5..0324b57 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -330,6 +330,13 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) } CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + AZ(sp->wrk->vary_b); + AZ(sp->wrk->vary_l); + AZ(sp->wrk->vary_e); + WS_Reserve(sp->wrk->ws, 0); + sp->wrk->vary_b = (void*)sp->wrk->ws->f; + sp->wrk->vary_e = (void*)sp->wrk->ws->r; + sp->wrk->vary_b[2] = '\0'; Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); busy_oc = NULL; @@ -374,6 +381,11 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) } } + WS_ReleaseP(sp->wrk->ws, (void*)sp->wrk->vary_b); + sp->wrk->vary_b = NULL; + sp->wrk->vary_l = NULL; + sp->wrk->vary_e = NULL; + /* * If we have seen a busy object or the backend is unhealthy, and * we have an object in grace, use it, if req.grace is also diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c index 8eab103..27abf74 100644 --- a/bin/varnishd/cache_vary.c +++ b/bin/varnishd/cache_vary.c @@ -58,6 +58,7 @@ #include #include "cache.h" +#include "vend.h" #include "vct.h" struct vsb * @@ -131,63 +132,112 @@ VRY_Create(const struct sess *sp, const struct http *hp) return(sb); } +/* + * Find length of a vary entry + */ +static unsigned +vry_len(const uint8_t *p) +{ + unsigned l = vbe16dec(p); + + return (2 + p[2] + 2 + (l == 0xffff ? 0 : l)); +} + +/* + * Compare two vary entries + */ +static int +vry_cmp(const uint8_t * const *v1, uint8_t * const *v2) +{ + unsigned retval = 0; + + if (!memcmp(*v1, *v2, vry_len(*v1))) { + /* Same same */ + retval = 0; + } else if (memcmp((*v1) + 2, (*v2) + 2, (*v1)[2] + 2)) { + /* Different header */ + retval = 1; + } else if (params->http_gzip_support && + !strcasecmp(H_Accept_Encoding, (const char*)((*v1)+2))) { + /* + * If we do gzip processing, we do not vary on Accept-Encoding, + * because we want everybody to get the gzip'ed object, and + * varnish will gunzip as necessary. We implement the skip at + * check time, rather than create time, so that object in + * persistent storage can be used with either setting of + * http_gzip_support. + */ + retval = 0; + } else { + /* Same header, different content */ + retval = 2; + } + return (retval); +} + int VRY_Match(const struct sess *sp, const uint8_t *vary) { + uint8_t *vsp = sp->wrk->vary_b; char *h, *e; - int i, l, lh; - - while (1) { - l = vary[0] * 256 + vary[1]; - vary += 2; - if (!*vary) - break; - - if (params->http_gzip_support && - !strcasecmp(H_Accept_Encoding, (const char*)vary)) { - /* - * If we do gzip processing, we do not vary on - * Accept-Encoding, because we want everybody to - * get the gzip'ed object, and varnish will gunzip - * as necessary. We implement the skip at check - * time, rather than create time, so that object - * in persistent storage can be used with either - * setting of http_gzip_support. - */ - vary += *vary + 2; - if (l != 0xffff) - vary += l; - continue; + unsigned lh, ln; + int i, retval = 1, oflo = 0; + + AN(vsp); + while (vary[2]) { + i = vry_cmp(&vary, &vsp); + if (i == 1) { + /* Build a new entry */ + + i = http_GetHdr(sp->http, (const char*)(vary+2), &h); + if (i) { + /* Trim trailing space */ + e = strchr(h, '\0'); + while (e > h && vct_issp(e[-1])) + e--; + lh = e - h; + assert(lh < 0xffff); + } else { + e = h = NULL; + lh = 0xffff; + } + + /* Length of the entire new vary entry */ + ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh); + if (vsp + ln >= sp->wrk->vary_e) { + vsp = sp->wrk->vary_b; + oflo = 1; + } + + /* We MUST have space for one entry */ + assert(vsp + ln < sp->wrk->vary_e); + + vbe16enc(vsp, (uint16_t)lh); + memcpy(vsp + 2, vary + 2, vary[2] + 2); + if (h != NULL && e != NULL) + memcpy(vsp + 2 + vsp[2] + 2, h, e - h); + + i = vry_cmp(&vary, &vsp); + assert(i != 1); /* hdr must be the same now */ } - /* Look for header */ - i = http_GetHdr(sp->http, (const char*)vary, &h); - vary += *vary + 2; - - /* Fail if we have the header, but shouldn't */ - if (i && l == 0xffff) - return (0); - /* or if we don't when we should */ - if (l != 0xffff && !i) - return (0); - - /* Nothing to match */ - if (!i) - continue; - - /* Trim trailing space */ - e = strchr(h, '\0'); - while (e > h && vct_issp(e[-1])) - e--; - - /* Fail if wrong length */ - lh = e - h; - if (lh != l) - return (0); + if (i != 0) + retval = 0; + vsp += vry_len(vsp); + vary += vry_len(vary); + } + if (vsp + 3 > sp->wrk->vary_e) + oflo = 1; - /* or if wrong contents */ - if (memcmp(h, vary, l)) - return (0); - vary += l; + if (oflo) { + /* XXX: Should log this */ + vsp = sp->wrk->vary_b; } - return (1); + vsp[0] = 0xff; + vsp[1] = 0xff; + vsp[2] = 0; + if (oflo) + sp->wrk->vary_l = NULL; + else + sp->wrk->vary_l = vsp + 3; + return (retval); } From tfheen at varnish-cache.org Wed Aug 17 09:25:27 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:27 +0200 Subject: [3.0] 1a43c37 Move the setup/teardown of requests vary prediction space from cache_hash to cache_center. Message-ID: commit 1a43c37d1a103b8b01ea2700b2239bc1a80ff434 Author: Poul-Henning Kamp Date: Wed Jun 22 08:34:46 2011 +0000 Move the setup/teardown of requests vary prediction space from cache_hash to cache_center. diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 4a17c56..5b55d58 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -1028,13 +1028,28 @@ cnt_lookup(struct sess *sp) struct objcore *oc; struct object *o; struct objhead *oh; + struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); + wrk = sp->wrk; + AZ(wrk->vary_b); + AZ(wrk->vary_l); + AZ(wrk->vary_e); + (void)WS_Reserve(wrk->ws, 0); + wrk->vary_b = (void*)wrk->ws->f; + wrk->vary_e = (void*)wrk->ws->r; + wrk->vary_b[2] = '\0'; + oc = HSH_Lookup(sp, &oh); + WS_Release(wrk->ws, 0); + wrk->vary_b = NULL; + wrk->vary_l = NULL; + wrk->vary_e = NULL; + if (oc == NULL) { /* * We lost the session to a busy object, disembark the diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 0324b57..2a6dcc5 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -330,13 +330,6 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) } CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - AZ(sp->wrk->vary_b); - AZ(sp->wrk->vary_l); - AZ(sp->wrk->vary_e); - WS_Reserve(sp->wrk->ws, 0); - sp->wrk->vary_b = (void*)sp->wrk->ws->f; - sp->wrk->vary_e = (void*)sp->wrk->ws->r; - sp->wrk->vary_b[2] = '\0'; Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); busy_oc = NULL; @@ -381,11 +374,6 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) } } - WS_ReleaseP(sp->wrk->ws, (void*)sp->wrk->vary_b); - sp->wrk->vary_b = NULL; - sp->wrk->vary_l = NULL; - sp->wrk->vary_e = NULL; - /* * If we have seen a busy object or the backend is unhealthy, and * we have an object in grace, use it, if req.grace is also From tfheen at varnish-cache.org Wed Aug 17 09:25:25 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:25 +0200 Subject: [3.0] 8c6ce17 Don't overwrite reference results unless told to Message-ID: commit 8c6ce17cf9bda9c9fc2f35cc96e3bbe77ae9af32 Author: Poul-Henning Kamp Date: Wed Jun 22 08:31:56 2011 +0000 Don't overwrite reference results unless told to diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index d163374..d87fe66 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -20,4 +20,6 @@ if [ -f _.fl.old ] ; then diff -u _.fl.old _.fl fi -mv _.fl _.fl.old +if [ "x$1" = "x-ok" ] ; then + mv _.fl _.fl.old +fi From tfheen at varnish-cache.org Wed Aug 17 09:25:27 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:27 +0200 Subject: [3.0] a5e2100 Make sure we neuter the next entry after the one we just created to prevent old junk on the workspace from being taken for valid. Message-ID: commit a5e2100e65c8a75165d16244b9fddab2333b2edd Author: Poul-Henning Kamp Date: Wed Jun 22 09:29:46 2011 +0000 Make sure we neuter the next entry after the one we just created to prevent old junk on the workspace from being taken for valid. diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c index 27abf74..01816f6 100644 --- a/bin/varnishd/cache_vary.c +++ b/bin/varnishd/cache_vary.c @@ -209,13 +209,19 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) oflo = 1; } - /* We MUST have space for one entry */ - assert(vsp + ln < sp->wrk->vary_e); + /* + * We MUST have space for one entry and the end marker + * after it, which prevents old junk from confusing us + */ + assert(vsp + ln + 2 < sp->wrk->vary_e); vbe16enc(vsp, (uint16_t)lh); memcpy(vsp + 2, vary + 2, vary[2] + 2); - if (h != NULL && e != NULL) + if (h != NULL && e != NULL) { memcpy(vsp + 2 + vsp[2] + 2, h, e - h); + vsp[2 + vary[2] + 2 + (e - h) + 2] = '\0'; + } else + vsp[2 + vary[2] + 2 + 2] = '\0'; i = vry_cmp(&vary, &vsp); assert(i != 1); /* hdr must be the same now */ From tfheen at varnish-cache.org Wed Aug 17 09:25:28 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:28 +0200 Subject: [3.0] 8f451fa Move request vary string from wrk to sess Message-ID: commit 8f451fa73c3fee45f4c20531541cefc31571756f Author: Poul-Henning Kamp Date: Wed Jun 22 10:39:47 2011 +0000 Move request vary string from wrk to sess diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 22fa5ff..1d3db55 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -300,9 +300,6 @@ struct worker { /* Lookup stuff */ struct SHA256Context *sha256ctx; - uint8_t *vary_b; - uint8_t *vary_l; - uint8_t *vary_e; struct http_conn htc[1]; struct ws ws[1]; @@ -554,6 +551,11 @@ struct sess { unsigned char digest[DIGEST_LEN]; + /* Built Vary string */ + uint8_t *vary_b; + uint8_t *vary_l; + uint8_t *vary_e; + struct http_conn htc[1]; /* Timestamps, all on TIM_real() timescale */ @@ -884,7 +886,7 @@ void RES_StreamPoll(const struct sess *sp); /* cache_vary.c */ struct vsb *VRY_Create(const struct sess *sp, const struct http *hp); -int VRY_Match(const struct sess *sp, const uint8_t *vary); +int VRY_Match(struct sess *sp, const uint8_t *vary); /* cache_vcl.c */ void VCL_Init(void); diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 5b55d58..23ad92f 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -1028,28 +1028,23 @@ cnt_lookup(struct sess *sp) struct objcore *oc; struct object *o; struct objhead *oh; - struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); - - wrk = sp->wrk; - AZ(wrk->vary_b); - AZ(wrk->vary_l); - AZ(wrk->vary_e); - (void)WS_Reserve(wrk->ws, 0); - wrk->vary_b = (void*)wrk->ws->f; - wrk->vary_e = (void*)wrk->ws->r; - wrk->vary_b[2] = '\0'; + if (sp->hash_objhead == NULL) { + /* Not a waiting list return */ + AZ(sp->vary_b); + AZ(sp->vary_l); + AZ(sp->vary_e); + (void)WS_Reserve(sp->ws, 0); + sp->vary_b = (void*)sp->ws->f; + sp->vary_e = (void*)sp->ws->r; + sp->vary_b[2] = '\0'; + } oc = HSH_Lookup(sp, &oh); - WS_Release(wrk->ws, 0); - wrk->vary_b = NULL; - wrk->vary_l = NULL; - wrk->vary_e = NULL; - if (oc == NULL) { /* * We lost the session to a busy object, disembark the @@ -1060,6 +1055,7 @@ cnt_lookup(struct sess *sp) return (1); } + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); @@ -1067,6 +1063,14 @@ cnt_lookup(struct sess *sp) if (oc->flags & OC_F_BUSY) { sp->wrk->stats.cache_miss++; + if (sp->vary_l != NULL) + WS_ReleaseP(sp->ws, (void*)sp->vary_l); + else + WS_Release(sp->ws, 0); + sp->vary_b = NULL; + sp->vary_l = NULL; + sp->vary_e = NULL; + sp->objcore = oc; sp->step = STP_MISS; return (0); @@ -1076,6 +1080,11 @@ cnt_lookup(struct sess *sp) CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); sp->obj = o; + WS_Release(sp->ws, 0); + sp->vary_b = NULL; + sp->vary_l = NULL; + sp->vary_e = NULL; + if (oc->flags & OC_F_PASS) { sp->wrk->stats.cache_hitpass++; WSP(sp, SLT_HitPass, "%u", sp->obj->xid); diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c index 01816f6..89e1675 100644 --- a/bin/varnishd/cache_vary.c +++ b/bin/varnishd/cache_vary.c @@ -176,9 +176,9 @@ vry_cmp(const uint8_t * const *v1, uint8_t * const *v2) } int -VRY_Match(const struct sess *sp, const uint8_t *vary) +VRY_Match(struct sess *sp, const uint8_t *vary) { - uint8_t *vsp = sp->wrk->vary_b; + uint8_t *vsp = sp->vary_b; char *h, *e; unsigned lh, ln; int i, retval = 1, oflo = 0; @@ -204,8 +204,8 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) /* Length of the entire new vary entry */ ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh); - if (vsp + ln >= sp->wrk->vary_e) { - vsp = sp->wrk->vary_b; + if (vsp + ln >= sp->vary_e) { + vsp = sp->vary_b; oflo = 1; } @@ -213,7 +213,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) * We MUST have space for one entry and the end marker * after it, which prevents old junk from confusing us */ - assert(vsp + ln + 2 < sp->wrk->vary_e); + assert(vsp + ln + 2 < sp->vary_e); vbe16enc(vsp, (uint16_t)lh); memcpy(vsp + 2, vary + 2, vary[2] + 2); @@ -231,19 +231,19 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) vsp += vry_len(vsp); vary += vry_len(vary); } - if (vsp + 3 > sp->wrk->vary_e) + if (vsp + 3 > sp->vary_e) oflo = 1; if (oflo) { /* XXX: Should log this */ - vsp = sp->wrk->vary_b; + vsp = sp->vary_b; } vsp[0] = 0xff; vsp[1] = 0xff; vsp[2] = 0; if (oflo) - sp->wrk->vary_l = NULL; + sp->vary_l = NULL; else - sp->wrk->vary_l = vsp + 3; + sp->vary_l = vsp + 3; return (retval); } From tfheen at varnish-cache.org Wed Aug 17 09:25:29 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:29 +0200 Subject: [3.0] 03fe1a4 Don't force a rerun to save reference if we just ran it. Message-ID: commit 03fe1a49c5416af5419b2f648ee6c237d4815597 Author: Poul-Henning Kamp Date: Wed Jun 22 10:52:47 2011 +0000 Don't force a rerun to save reference if we just ran it. diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index d87fe66..690e445 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -1,5 +1,11 @@ #!/bin/sh +if [ "x$1" = "x-ok" -a -f _.fl ] ; then + echo "Saved as reference" + mv _.fl _.fl.old + exit 0 +fi + flexelint \ ../flint.lnt \ flint.lnt \ @@ -21,5 +27,6 @@ if [ -f _.fl.old ] ; then fi if [ "x$1" = "x-ok" ] ; then + echo "Saved as reference" mv _.fl _.fl.old fi From tfheen at varnish-cache.org Wed Aug 17 09:25:28 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:28 +0200 Subject: [3.0] 0459355 Remove unused/unreferenced stuff. Message-ID: commit 0459355a0af86dbc81bfe4f6e9608eb3d33b1063 Author: Poul-Henning Kamp Date: Wed Jun 22 10:51:08 2011 +0000 Remove unused/unreferenced stuff. Not detected by: FlexeLint 9.00f Detected by: FlexeLint 9.00g diff --git a/include/libvarnish.h b/include/libvarnish.h index 317dc9e..157a64e 100644 --- a/include/libvarnish.h +++ b/include/libvarnish.h @@ -104,8 +104,6 @@ int vtmpfile(char *); char *vreadfile(const char *pfx, const char *fn, ssize_t *sz); char *vreadfd(int fd, ssize_t *sz); -const char* VCS_Version(void); - /* Safe printf into a fixed-size buffer */ #define bprintf(buf, fmt, ...) \ do { \ diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c index 3198ce1..f90c1b1 100644 --- a/lib/libvcl/vcc_backend.c +++ b/lib/libvcl/vcc_backend.c @@ -211,44 +211,6 @@ Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port) } /*-------------------------------------------------------------------- - * When a new VCL is loaded, it is likely to contain backend declarations - * identical to other loaded VCL programs, and we want to reuse the state - * of those in order to not have to relearn statistics, DNS etc. - * - * This function emits a space separated text-string of the tokens which - * define a given backend which can be used to determine "identical backend" - * in that context. - */ - -void -vcc_EmitBeIdent(const struct vcc *tl, struct vsb *v, - int serial, const struct token *first, const struct token *last) -{ - - assert(first != last); - VSB_printf(v, "\t.ident ="); - if (serial >= 0) { - VSB_printf(v, "\n\t \"%.*s %.*s [%d] \"", - PF(tl->t_policy), PF(tl->t_dir), serial); - } else { - VSB_printf(v, "\n\t \"%.*s %.*s \"", - PF(tl->t_policy), PF(tl->t_dir)); - } - while (1) { - if (first->dec != NULL) - VSB_printf(v, "\n\t \"\\\"\" %.*s \"\\\" \"", - PF(first)); - else - VSB_printf(v, "\n\t \"%.*s \"", PF(first)); - if (first == last) - break; - first = VTAILQ_NEXT(first, list); - AN(first); - } - VSB_printf(v, ",\n"); -} - -/*-------------------------------------------------------------------- * Parse a backend probe specification */ diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h index c90d225..b8c8454 100644 --- a/lib/libvcl/vcc_compile.h +++ b/lib/libvcl/vcc_compile.h @@ -210,8 +210,6 @@ void vcc_IsField(struct vcc *tl, struct token **t, struct fld_spec *fs); void vcc_FieldsOk(struct vcc *tl, const struct fld_spec *fs); void Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port); -void vcc_EmitBeIdent(const struct vcc *tl, struct vsb *v, - int serial, const struct token *first, const struct token *last); /* vcc_compile.c */ extern struct method method_tab[]; @@ -245,7 +243,6 @@ parsedirector_f vcc_ParseRoundRobinDirector; void vcc_RTimeVal(struct vcc *tl, double *); void vcc_TimeVal(struct vcc *tl, double *); unsigned vcc_UintVal(struct vcc *tl); -double vcc_DoubleVal(struct vcc *tl); void vcc_Expr(struct vcc *tl, enum var_type typ); void vcc_Expr_Call(struct vcc *tl, const struct symbol *sym); void vcc_Expr_Init(struct vcc *tl); @@ -267,8 +264,6 @@ sym_wildcard_t vcc_Stv_Wildcard; /* vcc_string.c */ char *vcc_regexp(struct vcc *tl); -int vcc_StringVal(struct vcc *tl); -void vcc_ExpectedStringval(struct vcc *tl); /* vcc_symb.c */ struct symbol *VCC_AddSymbolStr(struct vcc *tl, const char *name, enum symkind); @@ -303,8 +298,6 @@ void vcc_AddToken(struct vcc *tl, unsigned tok, const char *b, sym_wildcard_t vcc_Var_Wildcard; const struct var *vcc_FindVar(struct vcc *tl, const struct token *t, int wr_access, const char *use); -void vcc_VarVal(struct vcc *tl, const struct var *vp, - const struct token *vt); /* vcc_vmod.c */ void vcc_ParseImport(struct vcc *tl); diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c index 82a8aba..9a504e5 100644 --- a/lib/libvcl/vcc_expr.c +++ b/lib/libvcl/vcc_expr.c @@ -145,7 +145,7 @@ vcc_NumVal(struct vcc *tl, double *d, int *frac) vcc_NextToken(tl); } -double +static double vcc_DoubleVal(struct vcc *tl) { double d; diff --git a/lib/libvcl/vcc_var.c b/lib/libvcl/vcc_var.c index 94a5fb9..3871dbc 100644 --- a/lib/libvcl/vcc_var.c +++ b/lib/libvcl/vcc_var.c @@ -121,29 +121,3 @@ vcc_FindVar(struct vcc *tl, const struct token *t, int wr_access, vcc_ErrWhere(tl, t); return (NULL); } - -/*--------------------------------------------------------------------*/ - -void -vcc_VarVal(struct vcc *tl, const struct var *vp, const struct token *vt) -{ - double d; - - if (vp->fmt == TIME) { - vcc_TimeVal(tl, &d); - ERRCHK(tl); - Fb(tl, 0, "%g", d); - } else if (vp->fmt == DURATION) { - vcc_RTimeVal(tl, &d); - ERRCHK(tl); - Fb(tl, 0, "%g", d); - } else if (vp->fmt == INT) { - Fb(tl, 0, "%u", vcc_UintVal(tl)); - } else { - AN(vt); - VSB_printf(tl->sb, - "Variable has incompatible type.\n"); - vcc_ErrWhere(tl, vt); - return; - } -} From tfheen at varnish-cache.org Wed Aug 17 09:25:29 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:29 +0200 Subject: [3.0] e3deca9 Enforce that oc_getobj() does not work on BUSY objcores. Message-ID: commit e3deca91c38dd4c521bdf8110db23e00066b8479 Author: Poul-Henning Kamp Date: Mon Jun 27 11:12:42 2011 +0000 Enforce that oc_getobj() does not work on BUSY objcores. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index 1d3db55..eb7077c 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -434,6 +434,7 @@ oc_getobj(struct worker *wrk, struct objcore *oc) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + AZ(oc->flags & OC_F_BUSY); AN(oc->methods); AN(oc->methods->getobj); return (oc->methods->getobj(wrk, oc)); diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 2a6dcc5..d5c5ebe 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -594,7 +594,6 @@ HSH_Unbusy(const struct sess *sp) AssertObjBusy(o); AN(oc->ban); - assert(oc_getobj(sp->wrk, oc) == o); assert(oc->refcnt > 0); assert(oh->refcnt > 0); if (o->ws_o->overflow) @@ -613,6 +612,7 @@ HSH_Unbusy(const struct sess *sp) hsh_rush(oh); AN(oc->ban); Lck_Unlock(&oh->mtx); + assert(oc_getobj(sp->wrk, oc) == o); } void From tfheen at varnish-cache.org Wed Aug 17 09:25:30 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:30 +0200 Subject: [3.0] 35dc396 Introduce the "busyobj" structure which is only valid for busy objects. Message-ID: commit 35dc396455141833cbfe7ed4b2a63a10e6687bfb Author: Poul-Henning Kamp Date: Mon Jun 27 12:11:41 2011 +0000 Introduce the "busyobj" structure which is only valid for busy objects. Use it to hold the derived vary string for the request (if any) and skip busy objects with non-matching Vary during hash lookup. Idea by: sky diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index eb7077c..a875071 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -97,6 +97,7 @@ struct director; struct object; struct objhead; struct objcore; +struct busyobj; struct storage; struct workreq; struct vrt_backend; @@ -281,6 +282,7 @@ struct worker { struct objhead *nobjhead; struct objcore *nobjcore; struct waitinglist *nwaitinglist; + struct busyobj *nbusyobj; void *nhashpriv; struct dstat stats; @@ -416,6 +418,7 @@ struct objcore { void *priv; unsigned priv2; struct objhead *objhead; + struct busyobj *busyobj; double timer_when; unsigned flags; #define OC_F_BUSY (1<<1) @@ -470,7 +473,16 @@ oc_getlru(const struct objcore *oc) return (oc->methods->getlru(oc)); } +/* Busy Object structure ---------------------------------------------*/ + +struct busyobj { + unsigned magic; +#define BUSYOBJ_MAGIC 0x23b95567 + uint8_t *vary; +}; + /* Object structure --------------------------------------------------*/ + VTAILQ_HEAD(storagehead, storage); struct object { diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index d5c5ebe..6fdff15 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -111,6 +111,11 @@ HSH_Prealloc(const struct sess *sp) } CHECK_OBJ_NOTNULL(w->nwaitinglist, WAITINGLIST_MAGIC); + if (w->nbusyobj == NULL) { + ALLOC_OBJ(w->nbusyobj, BUSYOBJ_MAGIC); + XXXAN(w->nbusyobj); + } + if (hash->prep != NULL) hash->prep(sp); } @@ -342,8 +347,15 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) assert(oc->objhead == oh); if (oc->flags & OC_F_BUSY) { - if (!sp->hash_ignore_busy) - busy_oc = oc; + CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); + if (sp->hash_ignore_busy) + continue; + + if (oc->busyobj->vary != NULL && + !VRY_Match(sp, oc->busyobj->vary)) + continue; + + busy_oc = oc; continue; } @@ -445,6 +457,10 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) AN(oc->flags & OC_F_BUSY); oc->refcnt = 1; + w->nbusyobj->vary = sp->vary_b; + oc->busyobj = w->nbusyobj; + w->nbusyobj = NULL; + /* * Busy objects go on the tail, so they will not trip up searches. * HSH_Unbusy() will move them to the front. @@ -609,6 +625,9 @@ HSH_Unbusy(const struct sess *sp) VTAILQ_REMOVE(&oh->objcs, oc, list); VTAILQ_INSERT_HEAD(&oh->objcs, oc, list); oc->flags &= ~OC_F_BUSY; + AZ(sp->wrk->nbusyobj); + sp->wrk->nbusyobj = oc->busyobj; + oc->busyobj = NULL; hsh_rush(oh); AN(oc->ban); Lck_Unlock(&oh->mtx); diff --git a/bin/varnishtest/tests/c00043.vtc b/bin/varnishtest/tests/c00043.vtc new file mode 100644 index 0000000..46cc4e1 --- /dev/null +++ b/bin/varnishtest/tests/c00043.vtc @@ -0,0 +1,44 @@ +varnishtest "predictive vary" + + +server s1 { + rxreq + txresp -hdr "Vary: foo" -bodylen 1 + rxreq + sema r2 sync 2 + sema r1 sync 2 + txresp -hdr "Vary: foo" -bodylen 2 +} -start + +server s2 { + rxreq + txresp -hdr "Vary: foo" -bodylen 3 +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.bar) { + set req.backend = s2; + } + } +} -start + +client c1 { + txreq -hdr "Foo: vary1" + rxresp + expect resp.bodylen == 1 + txreq -hdr "Foo: vary2" + rxresp + expect resp.bodylen == 2 +} -start + +client c2 { + sema r2 sync 2 + txreq -hdr "Foo: vary3" -hdr "bar: yes" + rxresp + sema r1 sync 2 + expect resp.bodylen == 3 +} -start + +client c1 -wait +client c2 -wait From tfheen at varnish-cache.org Wed Aug 17 09:25:31 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:31 +0200 Subject: [3.0] 58d36cb Fix a compiler fault if two IP comparisons were on the same line of source code. Message-ID: commit 58d36cb328f41f1ad3c5a1cc8a6bedbfa873b4b4 Author: Poul-Henning Kamp Date: Thu Jun 30 08:30:40 2011 +0000 Fix a compiler fault if two IP comparisons were on the same line of source code. This is substantially DocWilcos fix, but instead of adding yet another unique numbering variable, I have collapsed the once we have into a single one. Fixes #948 diff --git a/bin/varnishtest/tests/r00948.vtc b/bin/varnishtest/tests/r00948.vtc new file mode 100644 index 0000000..559cbbc --- /dev/null +++ b/bin/varnishtest/tests/r00948.vtc @@ -0,0 +1,30 @@ +varnishtest "anon acl numbering" + + +server s1 { + rxreq + expect req.http.foo1 == "Match" + expect req.http.foo2 == "Match" + txresp -bodylen 40 +} -start + +varnish v1 -vcl+backend { + +sub vcl_recv { + if (client.ip == "${s1_addr}" || client.ip == "${bad_ip}") { + set req.http.foo1 = "Match"; + } + if (client.ip == "${bad_ip}" || client.ip == "${s1_addr}") { + set req.http.foo2 = "Match"; + } +} + +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 40 +} -run + + diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c index c506c78..30c9eea 100644 --- a/lib/libvcl/vcc_acl.c +++ b/lib/libvcl/vcc_acl.c @@ -454,7 +454,7 @@ vcc_Acl_Hack(struct vcc *tl, char *b) VTAILQ_INIT(&tl->acl); tcond = tl->t->tok; vcc_NextToken(tl); - bprintf(acln, "%u", tl->cnt); + bprintf(acln, "%u", tl->unique++); vcc_acl_entry(tl); vcc_acl_emit(tl, acln, 1); sprintf(b, "%smatch_acl_anon_%s(sp, \v1)", diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c index f90c1b1..babfe01 100644 --- a/lib/libvcl/vcc_backend.c +++ b/lib/libvcl/vcc_backend.c @@ -83,7 +83,7 @@ emit_sockaddr(struct vcc *tl, void *sa, unsigned sal) AN(sal); assert(sal < 256); Fh(tl, 0, "\nstatic const unsigned char sockaddr%u[%d] = {\n", - tl->nsockaddr, sal + 1); + tl->unique, sal + 1); Fh(tl, 0, " %3u, /* Length */\n", sal); u = sa; for (len = 0; len nsockaddr++); + return (tl->unique++); } /*-------------------------------------------------------------------- diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h index b8c8454..ec7caed 100644 --- a/lib/libvcl/vcc_compile.h +++ b/lib/libvcl/vcc_compile.h @@ -162,8 +162,7 @@ struct vcc { struct token *t_dir; struct token *t_policy; - unsigned recnt; - unsigned nsockaddr; + unsigned unique; unsigned nvmodpriv; unsigned err_unref; diff --git a/lib/libvcl/vcc_string.c b/lib/libvcl/vcc_string.c index 06e7277..ed91c35 100644 --- a/lib/libvcl/vcc_string.c +++ b/lib/libvcl/vcc_string.c @@ -63,7 +63,7 @@ vcc_regexp(struct vcc *tl) return (NULL); } VRE_free(&t); - sprintf(buf, "VGC_re_%u", tl->recnt++); + sprintf(buf, "VGC_re_%u", tl->unique++); p = TlAlloc(tl, strlen(buf) + 1); strcpy(p, buf); From tfheen at varnish-cache.org Wed Aug 17 09:25:32 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:32 +0200 Subject: [3.0] c426c9b Split registration and selection of backend poll functions into two different functions. Message-ID: commit c426c9bded4211345e0fd8921d2de0fbc4e0d8fd Author: Poul-Henning Kamp Date: Thu Jun 30 11:06:51 2011 +0000 Split registration and selection of backend poll functions into two different functions. I have not managed to write a vtc case for this one, but I am pretty sure this: Fixes: #945 diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c index 8c13112..3d6a405 100644 --- a/bin/varnishd/cache_backend.c +++ b/bin/varnishd/cache_backend.c @@ -430,7 +430,7 @@ VBE_UseHealth(const struct director *vdi) CAST_OBJ_NOTNULL(vs, vdi->priv, VDI_SIMPLE_MAGIC); if (vs->vrt->probe == NULL) return; - VBP_Start(vs->backend, vs->vrt->probe, vs->vrt->hosthdr); + VBP_Use(vs->backend, vs->vrt->probe); } /*-------------------------------------------------------------------- @@ -475,7 +475,8 @@ vdi_simple_fini(const struct director *d) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); - VBP_Stop(vs->backend, vs->vrt->probe); + if (vs->vrt->probe != NULL) + VBP_Remove(vs->backend, vs->vrt->probe); VBE_DropRefVcl(vs->backend); free(vs->dir.vcl_name); vs->dir.magic = 0; @@ -506,8 +507,8 @@ VRT_init_dir_simple(struct cli *cli, struct director **bp, int idx, vs->vrt = t; vs->backend = VBE_AddBackend(cli, t); - if (vs->backend->probe == NULL) - VBP_Start(vs->backend, vs->vrt->probe, vs->vrt->hosthdr); + if (vs->vrt->probe != NULL) + VBP_Insert(vs->backend, vs->vrt->probe, vs->vrt->hosthdr); bp[idx] = &vs->dir; } diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h index cb707ba..c065218 100644 --- a/bin/varnishd/cache_backend.h +++ b/bin/varnishd/cache_backend.h @@ -145,8 +145,9 @@ void VBE_DropRefVcl(struct backend *); void VBE_DropRefLocked(struct backend *b); /* cache_backend_poll.c */ -void VBP_Start(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr); -void VBP_Stop(struct backend *b, struct vrt_backend_probe const *p); +void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr); +void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p); +void VBP_Use(struct backend *b, const struct vrt_backend_probe const *p); /* Init functions for directors */ typedef void dir_init_f(struct cli *, struct director **, int , const void*); diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c index d1acf2a..d7b5268 100644 --- a/bin/varnishd/cache_backend_poll.c +++ b/bin/varnishd/cache_backend_poll.c @@ -463,11 +463,11 @@ vbp_new_vcl(const struct vrt_backend_probe *p, const char *hosthdr) } /*-------------------------------------------------------------------- - * Start/Stop called from cache_backend.c + * Insert/Remove/Use called from cache_backend.c */ void -VBP_Start(struct backend *b, const struct vrt_backend_probe *p, const char *hosthdr) +VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, const char *hosthdr) { struct vbp_target *vt; struct vbp_vcl *vcl; @@ -475,9 +475,7 @@ VBP_Start(struct backend *b, const struct vrt_backend_probe *p, const char *host unsigned u; ASSERT_CLI(); - - if (p == NULL) - return; + AN(p); if (b->probe == NULL) { ALLOC_OBJ(vt, VBP_TARGET_MAGIC); @@ -493,21 +491,12 @@ VBP_Start(struct backend *b, const struct vrt_backend_probe *p, const char *host vt = b->probe; } - VTAILQ_FOREACH(vcl, &vt->vcls, list) { - if (vcl->probep != p) - continue; - - AZ(startthread); - Lck_Lock(&vbp_mtx); - VTAILQ_REMOVE(&vt->vcls, vcl, list); - VTAILQ_INSERT_HEAD(&vt->vcls, vcl, list); - Lck_Unlock(&vbp_mtx); - return; - } + VTAILQ_FOREACH(vcl, &vt->vcls, list) + assert (vcl->probep != p); vcl = vbp_new_vcl(p, hosthdr); Lck_Lock(&vbp_mtx); - VTAILQ_INSERT_HEAD(&vt->vcls, vcl, list); + VTAILQ_INSERT_TAIL(&vt->vcls, vcl, list); Lck_Unlock(&vbp_mtx); if (startthread) { @@ -521,21 +510,42 @@ VBP_Start(struct backend *b, const struct vrt_backend_probe *p, const char *host } void -VBP_Stop(struct backend *b, struct vrt_backend_probe const *p) +VBP_Use(struct backend *b, const struct vrt_backend_probe *p) { struct vbp_target *vt; struct vbp_vcl *vcl; - void *ret; ASSERT_CLI(); + AN(p); + CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC); + AN(b->probe); + vt = b->probe; - if (p == NULL) + VTAILQ_FOREACH(vcl, &vt->vcls, list) { + if (vcl->probep != p) + continue; + + Lck_Lock(&vbp_mtx); + VTAILQ_REMOVE(&vt->vcls, vcl, list); + VTAILQ_INSERT_HEAD(&vt->vcls, vcl, list); + Lck_Unlock(&vbp_mtx); return; + } +} - CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC); +void +VBP_Remove(struct backend *b, struct vrt_backend_probe const *p) +{ + struct vbp_target *vt; + struct vbp_vcl *vcl; + void *ret; + ASSERT_CLI(); + AN(p); + CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC); AN(b->probe); vt = b->probe; + VTAILQ_FOREACH(vcl, &vt->vcls, list) if (vcl->probep == p) break; From tfheen at varnish-cache.org Wed Aug 17 09:25:59 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:59 +0200 Subject: [3.0] 01e4580 Minor polish to the malloc allocator: Message-ID: commit 01e45806107a3974f1f7607ac725f79df28b518f Author: Poul-Henning Kamp Date: Tue Aug 2 07:48:02 2011 +0000 Minor polish to the malloc allocator: Ensure that stats are correct if allocations fail. Don't rely on the VSC counter for checking the limit. diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 12a390f..56b4daa 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -44,6 +44,7 @@ struct sma_sc { #define SMA_SC_MAGIC 0x1ac8a345 struct lock sma_mtx; size_t sma_max; + size_t sma_alloc; struct VSC_C_sma *stats; }; @@ -59,14 +60,16 @@ static struct storage * sma_alloc(struct stevedore *st, size_t size) { struct sma_sc *sma_sc; - struct sma *sma; + struct sma *sma = NULL; + void *p; CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); Lck_Lock(&sma_sc->sma_mtx); sma_sc->stats->nreq++; - if (sma_sc->stats->nbytes + size > sma_sc->sma_max) + if (sma_sc->sma_alloc + size > sma_sc->sma_max) size = 0; else { + sma_sc->sma_alloc += size; sma_sc->stats->nobj++; sma_sc->stats->nbytes += size; sma_sc->stats->balloc += size; @@ -82,17 +85,26 @@ sma_alloc(struct stevedore *st, size_t size) * performance-wise it would be a catastropy with chunksized * allocations growing another full page, just to accomodate the sma. */ - ALLOC_OBJ(sma, SMA_MAGIC); - if (sma == NULL) - return (NULL); /* XXX: stats suffer */ + + p = malloc(size); + if (p != NULL) { + ALLOC_OBJ(sma, SMA_MAGIC); + if (sma != NULL) + sma->s.ptr = p; + else + free(p); + } + if (sma == NULL) { + Lck_Lock(&sma_sc->sma_mtx); + sma_sc->stats->nobj--; + sma_sc->stats->nbytes -= size; + sma_sc->stats->balloc -= size; + Lck_Unlock(&sma_sc->sma_mtx); + return (NULL); + } sma->sc = sma_sc; sma->sz = size; sma->s.priv = sma; - sma->s.ptr = malloc(size); - if (sma->s.ptr == NULL) { - free(sma); - return (NULL); /* XXX: stats suffer */ - } sma->s.len = 0; sma->s.space = size; #ifdef SENDFILE_WORKS @@ -114,6 +126,7 @@ sma_free(struct storage *s) sma_sc = sma->sc; assert(sma->sz == sma->s.space); Lck_Lock(&sma_sc->sma_mtx); + sma_sc->sma_alloc -= sma->sz; sma_sc->stats->nobj--; sma_sc->stats->nbytes -= sma->sz; sma_sc->stats->bfree += sma->sz; @@ -152,7 +165,7 @@ sma_used_space(const struct stevedore *st) struct sma_sc *sma_sc; CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); - return (sma_sc->stats->nbytes); + return (sma_sc->sma_alloc); } static double @@ -161,7 +174,7 @@ sma_free_space(const struct stevedore *st) struct sma_sc *sma_sc; CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); - return (sma_sc->sma_max - sma_sc->stats->nbytes); + return (sma_sc->sma_max - sma_sc->sma_alloc); } static void @@ -193,10 +206,7 @@ sma_init(struct stevedore *parent, int ac, char * const *av) ARGV_ERR("(-smalloc) size \"%s\": too small, " "did you forget to specify M or G?\n", av[0]); - printf("SMA.%s: max size %ju MB.\n", parent->ident, - u / (1024 * 1024)); sc->sma_max = u; - } static void From tfheen at varnish-cache.org Wed Aug 17 09:26:12 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:12 +0200 Subject: [3.0] 85a7216 Be more aggressive in getting rid of threads and slower in adding threads, hopefully fixing c00002 Message-ID: commit 85a7216c1460c1c4c3e7b5374c5ec13afa6236a8 Author: Tollef Fog Heen Date: Tue Aug 2 14:05:15 2011 +0200 Be more aggressive in getting rid of threads and slower in adding threads, hopefully fixing c00002 diff --git a/bin/varnishtest/tests/c00002.vtc b/bin/varnishtest/tests/c00002.vtc index 62eaf17..88862b3 100644 --- a/bin/varnishtest/tests/c00002.vtc +++ b/bin/varnishtest/tests/c00002.vtc @@ -5,7 +5,7 @@ server s1 { txresp -hdr "Connection: close" -body "012345\n" } -start -varnish v1 -arg "-p thread_pool_min=2 -p thread_pool_max=8 -p thread_pools=4 -p thread_pool_purge_delay=10" +varnish v1 -arg "-p thread_pool_min=2 -p thread_pool_max=8 -p thread_pools=4 -p thread_pool_purge_delay=100 -p thread_pool_timeout=1 -p thread_pool_add_delay=100" varnish v1 -vcl+backend {} -start From tfheen at varnish-cache.org Wed Aug 17 09:26:15 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:15 +0200 Subject: [3.0] 105ee6c Fix typo in help text Message-ID: commit 105ee6c0fc2308bbf3f9cdbb22fb1a7eb48a1abc Author: Tollef Fog Heen Date: Tue Aug 2 14:13:36 2011 +0200 Fix typo in help text diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 8775950..b280022 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -154,7 +154,7 @@ usage(void) fprintf(stderr, FMT, "-l", "Leave /tmp/vtc.* if test fails"); fprintf(stderr, FMT, "-L", "Always leave /tmp/vtc.*"); fprintf(stderr, FMT, "-n iterations", "Run tests this many times"); - fprintf(stderr, FMT, "-q", "Quiet mode: report only failues"); + fprintf(stderr, FMT, "-q", "Quiet mode: report only failures"); fprintf(stderr, FMT, "-t duration", "Time tests out after this long"); fprintf(stderr, FMT, "-v", "Verbose mode: always report test log"); fprintf(stderr, "\n"); From tfheen at varnish-cache.org Wed Aug 17 09:26:20 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:20 +0200 Subject: [3.0] a65cb2e Reset bereq http struct on restart from vcl_miss and vcl_pass Message-ID: commit a65cb2e4dc616761b663f7f73520359539485f70 Author: Tollef Fog Heen Date: Tue Aug 2 14:55:21 2011 +0200 Reset bereq http struct on restart from vcl_miss and vcl_pass Thanks a lot to David for minimised test case showing the bug. Fixes: #965 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 4bac471..eb0af3e 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -1155,6 +1155,7 @@ cnt_miss(struct sess *sp) case VCL_RET_ERROR: AZ(HSH_Deref(sp->wrk, sp->objcore, NULL)); sp->objcore = NULL; + http_Setup(sp->wrk->bereq, NULL); sp->step = STP_ERROR; return (0); case VCL_RET_PASS: @@ -1223,6 +1224,7 @@ cnt_pass(struct sess *sp) sp->wrk->between_bytes_timeout = 0; VCL_pass_method(sp); if (sp->handling == VCL_RET_ERROR) { + http_Setup(sp->wrk->bereq, NULL); sp->step = STP_ERROR; return (0); } diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc new file mode 100644 index 0000000..f8228a9 --- /dev/null +++ b/bin/varnishtest/tests/r00965.vtc @@ -0,0 +1,52 @@ +varnishtest "restart in vcl_miss #965" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.X-Banned == "check") { remove req.http.X-Banned; } + elseif (req.restarts == 0) { + set req.http.X-Banned = "check"; + if (req.http.x-pass) { + return (pass); + } else { + return (lookup); + } + } + } + + sub vcl_hash { + ## Check if they have a ban in the cache, or if they are going to be banned in cache. + if (req.http.X-Banned) { + hash_data(client.ip); + return (hash); + } + } + + sub vcl_error { + if (obj.status == 988) { return (restart); } + } + + sub vcl_miss { + if (req.http.X-Banned == "check") { error 988 "restarting"; } + } + + sub vcl_pass { + if (req.http.X-Banned == "check") { error 988 "restarting"; } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + txreq + rxresp + expect resp.status == 200 + txreq -hdr "X-Pass: 1" + rxresp + expect resp.status == 200 +} -run From tfheen at varnish-cache.org Wed Aug 17 09:26:27 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:27 +0200 Subject: [3.0] 088ad84 Slam grace and keep to zero if we fall back to a transient object. Message-ID: commit 088ad84e70171b36aa2669b9643fe9a70036505e Author: Poul-Henning Kamp Date: Tue Aug 2 13:03:38 2011 +0000 Slam grace and keep to zero if we fall back to a transient object. Reminded by: Tollef Related to: #953 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index eb0af3e..f93a61b 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -756,6 +756,8 @@ cnt_fetchbody(struct sess *sp) &sp->wrk->exp, nhttp); if (sp->wrk->exp.ttl > params->shortlived) sp->wrk->exp.ttl = params->shortlived; + sp->wrk->exp.grace = 0.0; + sp->wrk->exp.keep = 0.0; } if (sp->obj == NULL) { sp->err_code = 503; From tfheen at varnish-cache.org Wed Aug 17 09:26:58 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:58 +0200 Subject: [3.0] 132ad32 Markup/typos Message-ID: commit 132ad322ea5fbe3779b0c5178f070b3c137f6f79 Author: Tollef Fog Heen Date: Tue Aug 9 14:35:54 2011 +0200 Markup/typos diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index c6b1722..69b6546 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -34,8 +34,8 @@ statistics bans Bans are filters that are applied to keep Varnish from serving stale content. When you issue a ban Varnish will not serve any - *banned* object from cache, but rather re-fetch it from it's back - end servers. + *banned* object from cache, but rather re-fetch it from its + backend servers. process management You can stop and start the cache (child) process though the diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 5b0927a..a16d7f5 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -112,8 +112,8 @@ These can be set in the declaration like this::: } To mark a backend as unhealthy after number of items have been added -to it's saintmode list .saintmode_threshold can be set to the maximum -list size. Setting a value of 0 disables saintmode checking entirely +to its saintmode list ``.saintmode_threshold`` can be set to the maximum +list size. Setting a value of 0 disables saint mode checking entirely for that backend. The value in the backend declaration overrides the parameter. From tfheen at varnish-cache.org Wed Aug 17 09:27:06 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:06 +0200 Subject: [3.0] f7a2d82 Don't zap the size until we have updated statistics Message-ID: commit f7a2d8272eadda0b2ab3c1de3202d4aa12b02727 Author: Poul-Henning Kamp Date: Wed Aug 10 07:50:13 2011 +0000 Don't zap the size until we have updated statistics diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 7035ff0..178973a 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -67,8 +67,8 @@ sma_alloc(struct stevedore *st, size_t size) Lck_Lock(&sma_sc->sma_mtx); sma_sc->stats->c_req++; if (sma_sc->sma_alloc + size > sma_sc->sma_max) { - size = 0; sma_sc->stats->c_fail += size; + size = 0; } else { sma_sc->sma_alloc += size; sma_sc->stats->c_bytes += size; From tfheen at varnish-cache.org Wed Aug 17 09:27:16 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:16 +0200 Subject: [3.0] 400e1bc Oops, didn't get this bit of the #963 fix in. Message-ID: commit 400e1bc06e065f2712bd1913598a911f5282a9cc Author: Poul-Henning Kamp Date: Mon Aug 15 21:53:31 2011 +0000 Oops, didn't get this bit of the #963 fix in. diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 0ca8766..8789865 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -490,8 +490,6 @@ hsh_rush(struct objhead *oh) CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_AssertHeld(&oh->mtx); wl = oh->waitinglist; - if (wl == NULL) - return; CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC); for (u = 0; u < params->rush_exponent; u++) { sp = VTAILQ_FIRST(&wl->list); @@ -632,7 +630,8 @@ HSH_Unbusy(const struct sess *sp) AZ(sp->wrk->nbusyobj); sp->wrk->nbusyobj = oc->busyobj; oc->busyobj = NULL; - hsh_rush(oh); + if (oh->waitinglist != NULL) + hsh_rush(oh); AN(oc->ban); Lck_Unlock(&oh->mtx); assert(oc_getobj(sp->wrk, oc) == o); @@ -709,7 +708,7 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo) /* Must have an object */ AN(oc->methods); } - if (oc->flags & OC_F_BUSY) + if (oh->waitinglist != NULL) hsh_rush(oh); Lck_Unlock(&oh->mtx); if (r != 0) From tfheen at varnish-cache.org Wed Aug 17 09:27:21 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:21 +0200 Subject: [3.0] f900240 Move 'age' and 'entered' into struct exp. Message-ID: commit f900240736f0b6c782644d0600cd639829d24da7 Author: Poul-Henning Kamp Date: Wed Aug 10 07:52:02 2011 +0000 Move 'age' and 'entered' into struct exp. Simplify the move from sp->wrk to sp->obj accordingly. Add a diagram that explains how ttl, grace & keep related to each other. Fix VCL's obj.ttl variable to appear to be relative to "now" rather than obj->entered. (#956) Also log SLT_TTL when we change obj.grace and obj.keep Make SLT_TTL always have the same format: XID "RFC" or "VCL" obj.ttl obj.grace obj.keep obj.entered obj.age In addition "RFC" has: obj.date obj.expires obj.max-age Fixes #956 Thanks to: DocWilco diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index a875071..2d04724 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -242,6 +242,8 @@ struct exp { double ttl; double grace; double keep; + double age; + double entered; }; /*--------------------------------------------------------------------*/ @@ -309,8 +311,6 @@ struct worker { struct http *beresp; struct http *resp; - double age; - double entered; struct exp exp; /* This is only here so VRT can find it */ @@ -507,8 +507,6 @@ struct object { ssize_t len; - double age; - double entered; struct exp exp; double last_modified; diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index f93a61b..beb3d55 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -428,7 +428,7 @@ cnt_error(struct sess *sp) } AN(sp->obj); sp->obj->xid = sp->xid; - sp->obj->entered = sp->t_req; + sp->obj->exp.entered = sp->t_req; } else { /* XXX: Null the headers ? */ } @@ -554,9 +554,8 @@ cnt_fetch(struct sess *sp) /* * What does RFC2616 think about TTL ? */ - sp->wrk->entered = TIM_real(); - sp->wrk->age = 0; EXP_Clr(&sp->wrk->exp); + sp->wrk->exp.entered = TIM_real(); sp->wrk->exp.ttl = RFC2616_Ttl(sp); /* pass from vclrecv{} has negative TTL */ @@ -782,8 +781,6 @@ cnt_fetchbody(struct sess *sp) sp->obj->xid = sp->xid; sp->obj->response = sp->err_code; - sp->obj->age = sp->wrk->age; - sp->obj->entered = sp->wrk->entered; WS_Assert(sp->obj->ws_o); /* Filter into object */ @@ -799,7 +796,7 @@ cnt_fetchbody(struct sess *sp) if (http_GetHdr(hp, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); else - sp->obj->last_modified = floor(sp->wrk->entered); + sp->obj->last_modified = floor(sp->wrk->exp.entered); assert(WRW_IsReleased(sp->wrk)); diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c index c0e79cc..7822bf6 100644 --- a/bin/varnishd/cache_expire.c +++ b/bin/varnishd/cache_expire.c @@ -35,6 +35,18 @@ * * We hold a single object reference for both data structures. * + * An attempted overview: + * + * EXP_Ttl() EXP_Grace() EXP_Keep() + * | | | + * entered v v | + * | +--------------->+ | + * v | grace | + * +---------------------->+ | + * ttl | v + * +---------------------------->+ + * keep + * */ #include "config.h" @@ -68,6 +80,8 @@ EXP_Clr(struct exp *e) e->ttl = -1; e->grace = -1; e->keep = -1; + e->age = 0; + e->entered = 0; } #define EXP_ACCESS(fld, low_val, extra) \ @@ -93,8 +107,8 @@ EXP_ACCESS(grace, 0., ) EXP_ACCESS(keep, 0.,) /*-------------------------------------------------------------------- - * Calculate when an object is out of ttl or grace, possibly constrained - * by per-session limits. + * Calculate an objects effective keep, grace or ttl time, suitably + * adjusted for defaults and by per-session limits. */ static double @@ -131,7 +145,7 @@ EXP_Ttl(const struct sess *sp, const struct object *o) r = o->exp.ttl; if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r) r = sp->exp.ttl; - return (o->entered + r); + return (o->exp.entered + r); } /*-------------------------------------------------------------------- @@ -214,8 +228,8 @@ EXP_Insert(struct object *o) AssertObjBusy(o); HSH_Ref(oc); - assert(o->entered != 0 && !isnan(o->entered)); - o->last_lru = o->entered; + assert(o->exp.entered != 0 && !isnan(o->exp.entered)); + o->last_lru = o->exp.entered; lru = oc_getlru(oc); CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c index 8789865..39afde6 100644 --- a/bin/varnishd/cache_hash.c +++ b/bin/varnishd/cache_hash.c @@ -383,9 +383,9 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) */ if (EXP_Grace(sp, o) >= sp->t_req) { if (grace_oc == NULL || - grace_ttl < o->entered + o->exp.ttl) { + grace_ttl < o->exp.entered + o->exp.ttl) { grace_oc = oc; - grace_ttl = o->entered + o->exp.ttl; + grace_ttl = o->exp.entered + o->exp.ttl; } } } diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index 4788a23..11f7fb6 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -226,7 +226,7 @@ RES_BuildHttp(struct sess *sp) http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Age: %.0f", - sp->obj->age + sp->t_resp - sp->obj->entered); + sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered); http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish"); http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s", sp->doclose ? "close" : "keep-alive"); diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c index 297151c..1ee011b 100644 --- a/bin/varnishd/cache_vrt_var.c +++ b/bin/varnishd/cache_vrt_var.c @@ -360,15 +360,20 @@ VRT_r_req_restarts(const struct sess *sp) return (sp->restarts); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * NB: TTL is relative to when object was created, whereas grace and + * keep are relative to ttl. + */ -#define VRT_DO_EXP(which, exp, fld, extra) \ +#define VRT_DO_EXP(which, exp, fld, offset, extra) \ \ void __match_proto__() \ VRT_l_##which##_##fld(struct sess *sp, double a) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + if (a > 0.) \ + a += offset; \ EXP_Set_##fld(&exp, a); \ extra; \ } \ @@ -378,21 +383,37 @@ VRT_r_##which##_##fld(struct sess *sp) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return(EXP_Get_##fld(&exp)); \ -} - -VRT_DO_EXP(req, sp->exp, ttl, ) -VRT_DO_EXP(req, sp->exp, grace, ) -VRT_DO_EXP(req, sp->exp, keep, ) -VRT_DO_EXP(obj, sp->obj->exp, grace, EXP_Rearm(sp->obj)) -VRT_DO_EXP(obj, sp->obj->exp, ttl, - EXP_Rearm(sp->obj); - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->obj->xid, a, sp->t_req)) -VRT_DO_EXP(obj, sp->obj->exp, keep, EXP_Rearm(sp->obj)) -VRT_DO_EXP(beresp, sp->wrk->exp, grace, ) -VRT_DO_EXP(beresp, sp->wrk->exp, ttl, - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req)) -VRT_DO_EXP(beresp, sp->wrk->exp, keep, ) + return(EXP_Get_##fld(&exp) - offset); \ +} + +static void +vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e) +{ + WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", + xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep, + sp->t_req, e->age + (sp->t_req - e->entered)); +} + +VRT_DO_EXP(req, sp->exp, ttl, 0, ) +VRT_DO_EXP(req, sp->exp, grace, 0, ) +VRT_DO_EXP(req, sp->exp, keep, 0, ) + +VRT_DO_EXP(obj, sp->obj->exp, grace, 0, + EXP_Rearm(sp->obj); + vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);) +VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered), + EXP_Rearm(sp->obj); + vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);) +VRT_DO_EXP(obj, sp->obj->exp, keep, 0, + EXP_Rearm(sp->obj); + vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);) + +VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0, + vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);) +VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0, + vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);) +VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0, + vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);) /*-------------------------------------------------------------------- * req.xid diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c index 7b2525f..69637ea 100644 --- a/bin/varnishd/rfc2616.c +++ b/bin/varnishd/rfc2616.c @@ -76,7 +76,7 @@ RFC2616_Ttl(const struct sess *sp) hp = sp->wrk->beresp; - assert(sp->wrk->entered != 0.0 && !isnan(sp->wrk->entered)); + assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered)); /* If all else fails, cache using default ttl */ ttl = params->default_ttl; @@ -116,7 +116,7 @@ RFC2616_Ttl(const struct sess *sp) max_age = strtoul(p, NULL, 0); if (http_GetHdr(hp, H_Age, &p)) { age = strtoul(p, NULL, 0); - sp->wrk->age = age; + sp->wrk->exp.age = age; } if (age > max_age) @@ -145,16 +145,16 @@ RFC2616_Ttl(const struct sess *sp) } if (h_date == 0 || - fabs(h_date - sp->wrk->entered) < params->clock_skew) { + fabs(h_date - sp->wrk->exp.entered) < params->clock_skew) { /* * If we have no Date: header or if it is * sufficiently close to our clock we will * trust Expires: relative to our own clock. */ - if (h_expires < sp->wrk->entered) + if (h_expires < sp->wrk->exp.entered) ttl = 0; else - ttl = h_expires - sp->wrk->entered; + ttl = h_expires - sp->wrk->exp.entered; break; } else { /* @@ -168,8 +168,10 @@ RFC2616_Ttl(const struct sess *sp) } /* calculated TTL, Our time, Date, Expires, max-age, age */ - WSP(sp, SLT_TTL, "%u RFC %g %.0f %.0f %.0f %u %u", sp->xid, - ttl, sp->wrk->entered, h_date, h_expires, max_age, age); + WSP(sp, SLT_TTL, + "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u %u", + sp->xid, ttl, -1. -1., sp->wrk->exp.entered, sp->wrk->exp.age, + h_date, h_expires, max_age); return (ttl); } diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 413e0b6..9a0b671 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -36,7 +36,6 @@ #include #include #include -#include #include "cache.h" #include "stevedore.h" @@ -248,7 +247,6 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot, http_Setup(o->http, o->ws_o); o->http->magic = HTTP_MAGIC; - o->entered = NAN; o->exp = *soc->exp; VTAILQ_INIT(&o->store); sp->wrk->stats.n_object++; diff --git a/bin/varnishtest/tests/r00956.vtc b/bin/varnishtest/tests/r00956.vtc new file mode 100644 index 0000000..c98996d --- /dev/null +++ b/bin/varnishtest/tests/r00956.vtc @@ -0,0 +1,49 @@ +varnishtest "obj.ttl relative/absolute" + +server s1 { + rxreq + txresp -hdr "Cache-Control: max-age=23" -hdr "Age: 4" -bodylen 40 +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 10s; + set req.http.foo = beresp.ttl; + set req.http.bar = "xxx"; + } + sub vcl_hit { + set req.http.foo = obj.ttl; + set obj.ttl = 7s; + set obj.grace = 120s; + set obj.keep = 1h; + set req.http.bar = obj.ttl; + } + sub vcl_deliver { + set resp.http.foo = req.http.foo; + set resp.http.bar = req.http.bar; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 40 + expect resp.http.foo == 10.000 + expect resp.http.bar == "xxx" + + delay 2 + txreq + rxresp + expect resp.bodylen == 40 + # XXX: should be: < 8 + expect resp.http.foo != 10.000 + expect resp.http.bar == 7.000 + + delay 2 + txreq + rxresp + expect resp.bodylen == 40 + # XXX: should be: < 5 + expect resp.http.foo != 7.000 + expect resp.http.bar == 7.000 +} -run From tfheen at varnish-cache.org Wed Aug 17 09:27:24 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:24 +0200 Subject: [3.0] b26529d Fiddle the "RFC" SLT_TTL record into submission. Message-ID: commit b26529dc43ce882a639cfcbb47d2094e18f4c753 Author: Poul-Henning Kamp Date: Wed Aug 10 09:06:58 2011 +0000 Fiddle the "RFC" SLT_TTL record into submission. Document the TTL record. diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c index 69637ea..045eacb 100644 --- a/bin/varnishd/rfc2616.c +++ b/bin/varnishd/rfc2616.c @@ -89,6 +89,16 @@ RFC2616_Ttl(const struct sess *sp) * We do not support ranges yet, so 206 is out. */ + if (http_GetHdr(hp, H_Age, &p)) { + age = strtoul(p, NULL, 0); + sp->wrk->exp.age = age; + } + if (http_GetHdr(hp, H_Expires, &p)) + h_expires = TIM_parse(p); + + if (http_GetHdr(hp, H_Date, &p)) + h_date = TIM_parse(p); + switch (sp->err_code) { default: sp->wrk->exp.ttl = -1.; @@ -114,10 +124,6 @@ RFC2616_Ttl(const struct sess *sp) max_age = 0; else max_age = strtoul(p, NULL, 0); - if (http_GetHdr(hp, H_Age, &p)) { - age = strtoul(p, NULL, 0); - sp->wrk->exp.age = age; - } if (age > max_age) ttl = 0; @@ -126,17 +132,10 @@ RFC2616_Ttl(const struct sess *sp) break; } - /* Next look for absolute specifications from backend */ - - if (http_GetHdr(hp, H_Expires, &p)) - h_expires = TIM_parse(p); - /* No expire header, fall back to default */ if (h_expires == 0) break; - if (http_GetHdr(hp, H_Date, &p)) - h_date = TIM_parse(p); /* If backend told us it is expired already, don't cache. */ if (h_expires < h_date) { @@ -169,8 +168,8 @@ RFC2616_Ttl(const struct sess *sp) /* calculated TTL, Our time, Date, Expires, max-age, age */ WSP(sp, SLT_TTL, - "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u %u", - sp->xid, ttl, -1. -1., sp->wrk->exp.entered, sp->wrk->exp.age, + "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", + sp->xid, ttl, -1., -1., sp->wrk->exp.entered, sp->wrk->exp.age, h_date, h_expires, max_age); return (ttl); diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index ac1026a..e9f2ff3 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -4,6 +4,38 @@ Shared Memory Logging ===================== +TTL records +~~~~~~~~~~~ + +A TTL record is emitted whenever the ttl, grace or keep values for an +object is set. + +The format is:: + + %u %s %d %d %d %d %d [ %d %u %u ] + | | | | | | | | | | + | | | | | | | | | +- Max-Age from Cache-Control header + | | | | | | | | +---- Expires header + | | | | | | | +------- Date header + | | | | | | +------------ Age (incl Age: header value) + | | | | | +--------------- Reference time for TTL + | | | | +------------------ Keep + | | | +--------------------- Grace + | | +------------------------ TTL + | +--------------------------- "RFC" or "VCL" + +------------------------------ object XID + +The last three fields are only present in "RFC" headers. + +Examples:: + + 1001 RFC 19 -1 -1 1312966109 4 0 0 23 + 1001 VCL 10 -1 -1 1312966109 4 + 1001 VCL 7 -1 -1 1312966111 6 + 1001 VCL 7 120 -1 1312966111 6 + 1001 VCL 7 120 3600 1312966111 6 + 1001 VCL 12 120 3600 1312966113 8 + Gzip records ~~~~~~~~~~~~ @@ -14,7 +46,6 @@ gunziped, will run into many of these. The format is:: - %c %c %c %d %d %d %d %d | | | | | | | | | | | | | | | +- Bit length of compressed data @@ -26,7 +57,7 @@ The format is:: | +------------------- 'F' = Fetch, 'D' = Deliver +---------------------- 'G' = Gzip, 'U' = Gunzip, 'u' = Gunzip-test -Which in practice could look like:: +Examples:: U F E 182 159 80 80 1392 G F E 159 173 80 1304 1314 From tfheen at varnish-cache.org Wed Aug 17 09:27:28 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:28 +0200 Subject: [3.0] cbd6a3f Fix typo Message-ID: commit cbd6a3fa55e0919db34657a2447f798500c17e55 Author: Tollef Fog Heen Date: Wed Aug 10 11:26:04 2011 +0200 Fix typo diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index d0e8ade..5cb5ad7 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -57,7 +57,7 @@ becomes ``beresp.cacheable`` is gone ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0`` +``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0s`` returns are now done with the ``return()`` function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From tfheen at varnish-cache.org Wed Aug 17 09:27:32 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:32 +0200 Subject: [3.0] f7fd7ef Document do_esi Message-ID: commit f7fd7ef0dbad50c74db9775200deebf3e8dd9f86 Author: Tollef Fog Heen Date: Wed Aug 10 11:31:37 2011 +0200 Document do_esi Put esi ? do_esi in upgrade checklist, fix markup typo. Fixes #974 diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 5cb5ad7..1a31733 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -78,7 +78,7 @@ becomes ``req.hash`` is replaced with ``hash_data()`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You no longer append to the hash with +=, so +You no longer append to the hash with ``+=``, so set req.hash += req.url; @@ -86,6 +86,17 @@ becomes hash_data(req.url); +``esi`` is replaced with ``beresp.do_esi`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You no longer enable ESI with ``esi``, so + + esi; + +in ``vcl_fetch`` becomes + + set beresp.do_esi = true; + ``pass`` in ``vcl_fetch`` renamed to ``hit_for_pass`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From tfheen at varnish-cache.org Wed Aug 17 09:27:41 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:41 +0200 Subject: [3.0] af52295 Add a couple of asserts to make sure we only do ims/range on 200. Message-ID: commit af52295ecea890c2a9d456d65e390d1e23255828 Author: Poul-Henning Kamp Date: Wed Aug 10 12:04:58 2011 +0000 Add a couple of asserts to make sure we only do ims/range on 200. diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c index 11f7fb6..0c306df 100644 --- a/bin/varnishd/cache_response.c +++ b/bin/varnishd/cache_response.c @@ -47,6 +47,7 @@ res_do_304(struct sess *sp) char lm[64]; char *p; + assert(sp->obj->response == 200); http_ClrHeader(sp->wrk->resp); sp->wrk->resp->logtag = HTTP_Tx; http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified"); @@ -127,7 +128,7 @@ res_dorange(struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) { ssize_t low, high, has_low; - (void)sp; + assert(sp->obj->response == 200); if (strncmp(r, "bytes=", 6)) return; r += 6; From tfheen at varnish-cache.org Wed Aug 17 09:28:14 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:14 +0200 Subject: [3.0] a958b02 Polish up the unit-test case for the binheap, looking for #967 Message-ID: commit a958b02f06172418d417de2c5b2ffa7342173339 Author: Poul-Henning Kamp Date: Thu Aug 11 08:40:01 2011 +0000 Polish up the unit-test case for the binheap, looking for #967 diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c index e6a00e7..6eb454b 100644 --- a/lib/libvarnish/binary_heap.c +++ b/lib/libvarnish/binary_heap.c @@ -28,8 +28,9 @@ * * Implementation of a binary heap API * - * We use a malloc(3)/realloc(3) array to store the pointers using the - * classical FORTRAN strategy. + * See also: + * http://portal.acm.org/citation.cfm?doid=1785414.1785434 + * (or: http://queue.acm.org/detail.cfm?id=1814327) */ #include "config.h" @@ -392,25 +393,45 @@ binheap_reorder(const struct binheap *bh, unsigned idx) #ifdef TEST_DRIVER /* Test driver -------------------------------------------------------*/ #include +#include + +static void +vasfail(const char *func, const char *file, int line, + const char *cond, int err, int xxx) +{ + fprintf(stderr, "PANIC: %s %s %d %s %d %d\n", + func, file, line, cond, err, xxx); + abort(); +} + +vas_f *VAS_Fail = vasfail; struct foo { + unsigned magic; +#define FOO_MAGIC 0x23239823 unsigned idx; unsigned key; + unsigned n; }; +#if 1 #define M 31011091 /* Number of operations */ -#define N 10313102 /* Number of items */ +#define N 10313102 /* Number of items */ +#else +#define M 3401 /* Number of operations */ +#define N 1131 /* Number of items */ +#endif #define R -1 /* Random modulus */ -struct foo ff[N]; +struct foo *ff[N]; static int cmp(void *priv, void *a, void *b) { struct foo *fa, *fb; - fa = a; - fb = b; + CAST_OBJ_NOTNULL(fa, a, FOO_MAGIC); + CAST_OBJ_NOTNULL(fb, b, FOO_MAGIC); return (fa->key < fb->key); } @@ -419,12 +440,12 @@ update(void *priv, void *a, unsigned u) { struct foo *fa; - fa = a; + CAST_OBJ_NOTNULL(fa, a, FOO_MAGIC); fa->idx = u; } void -chk(struct binheap *bh) +chk2(struct binheap *bh) { unsigned u, v; struct foo *fa, *fb; @@ -441,7 +462,7 @@ int main(int argc, char **argv) { struct binheap *bh; - unsigned u, v, lr; + unsigned u, v, lr, n; struct foo *fp; if (0) { @@ -452,58 +473,80 @@ main(int argc, char **argv) } bh = binheap_new(NULL, cmp, update); - /* First insert our N elements */ - for (u = 0; u < N; u++) { - lr = random() % R; - ff[u].key = lr; - binheap_insert(bh, &ff[u]); - - fp = binheap_root(bh); - assert(fp->idx == 1); - assert(fp->key <= lr); - } - fprintf(stderr, "%d inserts OK\n", N); - /* For M cycles, pick the root, insert new */ - for (u = 0; u < M; u++) { - fp = binheap_root(bh); - assert(fp->idx == 1); - - /* It cannot possibly be larger than the last value we added */ - assert(fp->key <= lr); - binheap_delete(bh, fp->idx); - - lr = random() % R; - fp->key = lr; - binheap_insert(bh, fp); - } - fprintf(stderr, "%d replacements OK\n", M); - /* The remove everything */ - lr = 0; - for (u = 0; u < N; u++) { - fp = binheap_root(bh); - assert(fp->idx == 1); - assert(fp->key >= lr); - lr = fp->key; - binheap_delete(bh, fp->idx); - } - fprintf(stderr, "%d removes OK\n", N); - - for (u = 0; u < M; u++) { - v = random() % N; - if (ff[v].idx > 0) { - assert(ff[v].idx != 0); - binheap_delete(bh, ff[v].idx); - assert(ff[v].idx == 0); - } else { - assert(ff[v].idx == 0); - ff[v].key = random() % R; - binheap_insert(bh, &ff[v]); - assert(ff[v].idx != 0); + while (1) { + /* First insert our N elements */ + for (u = 0; u < N; u++) { + lr = random() % R; + ALLOC_OBJ(ff[u], FOO_MAGIC); + assert(ff[u] != NULL); + ff[u]->key = lr; + ff[u]->n = u; + binheap_insert(bh, ff[u]); + + fp = binheap_root(bh); + assert(fp->idx == 1); + assert(fp->key <= lr); + } + fprintf(stderr, "%d inserts OK\n", N); + /* For M cycles, pick the root, insert new */ + for (u = 0; u < M; u++) { + fp = binheap_root(bh); + CHECK_OBJ_NOTNULL(fp, FOO_MAGIC); + assert(fp->idx == 1); + + /* It cannot possibly be larger than the last value we added */ + assert(fp->key <= lr); + binheap_delete(bh, fp->idx); + + + n = fp->n; + ALLOC_OBJ(ff[n], FOO_MAGIC); + assert(ff[n] != NULL); + FREE_OBJ(fp); + fp = ff[n]; + fp->n = n; + + lr = random() % R; + fp->key = lr; + binheap_insert(bh, fp); + } + fprintf(stderr, "%d replacements OK\n", M); + /* The remove everything */ + lr = 0; + for (u = 0; u < N; u++) { + fp = binheap_root(bh); + CHECK_OBJ_NOTNULL(fp, FOO_MAGIC); + assert(fp->idx == 1); + assert(fp->key >= lr); + lr = fp->key; + binheap_delete(bh, fp->idx); + ff[fp->n] = NULL; + FREE_OBJ(fp); + } + fprintf(stderr, "%d removes OK\n", N); + + for (u = 0; u < M; u++) { + v = random() % N; + if (ff[v] != NULL) { + CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC); + assert(ff[v]->idx != 0); + binheap_delete(bh, ff[v]->idx); + assert(ff[v]->idx == 0); + FREE_OBJ(ff[v]); + ff[v] = NULL; + } else { + ALLOC_OBJ(ff[v], FOO_MAGIC); + assert(ff[v] != NULL); + ff[v]->key = random() % R; + binheap_insert(bh, ff[v]); + CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC); + assert(ff[v]->idx != 0); + } + if (0) + chk2(bh); } - if (0) - chk(bh); + fprintf(stderr, "%d updates OK\n", M); } - fprintf(stderr, "%d updates OK\n", M); return (0); } #endif From tfheen at varnish-cache.org Wed Aug 17 09:28:19 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:19 +0200 Subject: [3.0] add1d82 Clamp rather than overflow on child indexes when we get to the end of the UINT_MAX items we can support. Message-ID: commit add1d82bb439055c752c479a1badd320ea378df9 Author: Poul-Henning Kamp Date: Wed Aug 17 08:43:55 2011 +0200 Clamp rather than overflow on child indexes when we get to the end of the UINT_MAX items we can support. Found by adding a lot of asserts and brute force testing, both of which I have left in. Fixes #967 May also be relevant to #827 (cherry picked from commit 726d93ff5a815774d7a3b4a23dcba2efb3c08ca7 and a6ddafc0e87bcec71f016b8cae77b78641e617d7) diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c index 7822bf6..97e5734 100644 --- a/bin/varnishd/cache_expire.c +++ b/bin/varnishd/cache_expire.c @@ -182,6 +182,8 @@ exp_insert(struct objcore *oc, struct lru *lru) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); + Lck_AssertHeld(&lru->mtx); + Lck_AssertHeld(&exp_mtx); assert(oc->timer_idx == BINHEAP_NOIDX); binheap_insert(exp_heap, oc); assert(oc->timer_idx != BINHEAP_NOIDX); diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c index 6eb454b..c94cbfd 100644 --- a/lib/libvarnish/binary_heap.c +++ b/lib/libvarnish/binary_heap.c @@ -37,6 +37,7 @@ #include #include +#include #include "binary_heap.h" #include "libvarnish.h" @@ -97,6 +98,7 @@ parent(const struct binheap *bh, unsigned u) unsigned po; unsigned v; + assert(u != UINT_MAX); po = u & bh->page_mask; if (u < bh->page_size || po > 3) { @@ -114,6 +116,7 @@ parent(const struct binheap *bh, unsigned u) static void child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b) { + uintmax_t uu; if (u > bh->page_mask && (u & (bh->page_mask - 1)) == 0) { /* First two elements are magical except on the first page */ @@ -123,16 +126,32 @@ child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b) *a = (u & ~bh->page_mask) >> 1; *a |= u & (bh->page_mask >> 1); *a += 1; - *a <<= bh->page_shift; - *b = *a + 1; + uu = (uintmax_t)*a << bh->page_shift; + *a = uu; + if (*a == uu) { + *b = *a + 1; + } else { + /* + * An unsigned is not big enough: clamp instead + * of truncating. We do not support adding + * more than UINT_MAX elements anyway, so this + * is without consequence. + */ + *a = UINT_MAX; + *b = UINT_MAX; + } } else { /* The rest is as usual, only inside the page */ *a = u + (u & bh->page_mask); *b = *a + 1; } #ifdef PARANOIA - assert(parent(bh, *a) == u); - assert(parent(bh, *b) == u); + assert(*a > 0); + assert(*b > 0); + if (*a != UINT_MAX) { + assert(parent(bh, *a) == u); + assert(parent(bh, *b) == u); + } #endif } @@ -215,12 +234,12 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f) static void binheap_update(const struct binheap *bh, unsigned u) { + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); assert(u < bh->next); assert(A(bh, u) != NULL); - if (bh->update == NULL) - return; - bh->update(bh->priv, A(bh, u), u); + if (bh->update != NULL) + bh->update(bh->priv, A(bh, u), u); } static void @@ -228,9 +247,12 @@ binhead_swap(const struct binheap *bh, unsigned u, unsigned v) { void *p; + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); assert(u < bh->next); + assert(A(bh, u) != NULL); assert(v < bh->next); + assert(A(bh, v) != NULL); p = A(bh, u); A(bh, u) = A(bh, v); A(bh, v) = p; @@ -243,9 +265,17 @@ binheap_trickleup(const struct binheap *bh, unsigned u) { unsigned v; - assert(bh->magic == BINHEAP_MAGIC); + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); + assert(u < bh->next); + assert(A(bh, u) != NULL); + while (u > ROOT_IDX) { + assert(u < bh->next); + assert(A(bh, u) != NULL); v = parent(bh, u); + assert(v < u); + assert(v < bh->next); + assert(A(bh, v) != NULL); if (!bh->cmp(bh->priv, A(bh, u), A(bh, v))) break; binhead_swap(bh, u, v); @@ -254,20 +284,37 @@ binheap_trickleup(const struct binheap *bh, unsigned u) return (u); } -static void +static unsigned binheap_trickledown(const struct binheap *bh, unsigned u) { unsigned v1, v2; + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); + assert(u < bh->next); + assert(A(bh, u) != NULL); + while (1) { + assert(u < bh->next); + assert(A(bh, u) != NULL); child(bh, u, &v1, &v2); + assert(v1 > 0); + assert(v2 > 0); + assert(v1 <= v2); + if (v1 >= bh->next) - return; - if (v2 < bh->next && bh->cmp(bh->priv, A(bh, v2), A(bh, v1))) - v1 = v2; + return (u); + + assert(A(bh, v1) != NULL); + if (v1 != v2 && v2 < bh->next) { + assert(A(bh, v2) != NULL); + if (bh->cmp(bh->priv, A(bh, v2), A(bh, v1))) + v1 = v2; + } + assert(v1 < bh->next); + assert(A(bh, v1) != NULL); if (bh->cmp(bh->priv, A(bh, u), A(bh, v1))) - return; + return (u); binhead_swap(bh, u, v1); u = v1; } @@ -283,10 +330,13 @@ binheap_insert(struct binheap *bh, void *p) assert(bh->length >= bh->next); if (bh->length == bh->next) binheap_addrow(bh); + assert(bh->length > bh->next); u = bh->next++; A(bh, u) = p; binheap_update(bh, u); (void)binheap_trickleup(bh, u); + assert(u < bh->next); + assert(A(bh, u) != NULL); } @@ -332,11 +382,11 @@ binheap_root(const struct binheap *bh) * N{height}-1 upward trickles. * * When we fill the hole with the tail object, the worst case is - * that it trickles all the way down to become the tail object - * again. - * In other words worst case is N{height} downward trickles. - * But there is a pretty decent chance that it does not make - * it all the way down. + * that it trickles all the way up to of this half-tree, or down + * to become the tail object again. + * + * In other words worst case is N{height} up or downward trickles. + * But there is a decent chance that it does not make it all the way. */ void @@ -358,7 +408,13 @@ binheap_delete(struct binheap *bh, unsigned idx) A(bh, bh->next) = NULL; binheap_update(bh, idx); idx = binheap_trickleup(bh, idx); - binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); + idx = binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); /* * We keep a hysteresis of one full row before we start to @@ -387,7 +443,13 @@ binheap_reorder(const struct binheap *bh, unsigned idx) assert(idx > 0); assert(A(bh, idx) != NULL); idx = binheap_trickleup(bh, idx); - binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); + idx = binheap_trickledown(bh, idx); + assert(idx < bh->next); + assert(idx > 0); + assert(A(bh, idx) != NULL); } #ifdef TEST_DRIVER @@ -416,7 +478,7 @@ struct foo { #if 1 #define M 31011091 /* Number of operations */ -#define N 10313102 /* Number of items */ +#define N 17313102 /* Number of items */ #else #define M 3401 /* Number of operations */ #define N 1131 /* Number of items */ @@ -472,6 +534,11 @@ main(int argc, char **argv) srandom(u); } bh = binheap_new(NULL, cmp, update); + for (n = 2; n; n += n) { + child(bh, n - 1, &u, &v); + child(bh, n, &u, &v); + child(bh, n + 1, &u, &v); + } while (1) { /* First insert our N elements */ @@ -530,10 +597,15 @@ main(int argc, char **argv) if (ff[v] != NULL) { CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC); assert(ff[v]->idx != 0); - binheap_delete(bh, ff[v]->idx); - assert(ff[v]->idx == 0); - FREE_OBJ(ff[v]); - ff[v] = NULL; + if (ff[v]->key & 1) { + binheap_delete(bh, ff[v]->idx); + assert(ff[v]->idx == BINHEAP_NOIDX); + FREE_OBJ(ff[v]); + ff[v] = NULL; + } else { + ff[v]->key = random() % R; + binheap_reorder(bh, ff[v]->idx); + } } else { ALLOC_OBJ(ff[v], FOO_MAGIC); assert(ff[v] != NULL); From tfheen at varnish-cache.org Wed Aug 17 09:28:23 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:23 +0200 Subject: [3.0] ae07042 ifdef the OFOF development tool Message-ID: commit ae07042e814379f27dad2b52dc4f9203829e4571 Author: Poul-Henning Kamp Date: Mon Aug 15 07:28:52 2011 +0000 ifdef the OFOF development tool diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c index 60e63bf..a3d1851 100644 --- a/bin/varnishd/cache_cli.c +++ b/bin/varnishd/cache_cli.c @@ -140,6 +140,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) SZOF(struct vbc); SZOF(struct VSC_C_main); SZOF(struct lock); +#if 0 #define OFOF(foo, bar) { foo __foo; VCLI_Out(cli, \ "%-30s = 0x%4zx @ 0x%4zx\n", \ #foo "." #bar, sizeof(__foo.bar), offsetof(foo, bar)); } @@ -199,6 +200,8 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv) OFOF(struct object, esidata); OFOF(struct object, last_use); #endif +#undef OFOF +#endif } /*--------------------------------------------------------------------*/ From tfheen at varnish-cache.org Wed Aug 17 09:28:28 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:28 +0200 Subject: [3.0] 6cba43b Try to read the silo signature to find the correct address to map the silo into VM. If this fails or we get garbage, the silo will be cleared. Message-ID: commit 6cba43b4b61ee694d16cc7a1706ac41ec706173f Author: Poul-Henning Kamp Date: Thu Aug 11 14:11:42 2011 +0000 Try to read the silo signature to find the correct address to map the silo into VM. If this fails or we get garbage, the silo will be cleared. Fixes #962 diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c index 0681264..a22cc59 100644 --- a/bin/varnishd/storage_persistent_mgt.c +++ b/bin/varnishd/storage_persistent_mgt.c @@ -120,6 +120,8 @@ void smp_mgt_init(struct stevedore *parent, int ac, char * const *av) { struct smp_sc *sc; + struct smp_sign sgn; + void *target; int i; ASSERT_MGT(); @@ -158,7 +160,15 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) AZ(ftruncate(sc->fd, sc->mediasize)); - sc->base = mmap(NULL, sc->mediasize, PROT_READ|PROT_WRITE, + /* Try to determine correct mmap address */ + i = read(sc->fd, &sgn, sizeof sgn); + assert(i == sizeof sgn); + if (!strcmp(sgn.ident, "SILO")) + target = (void*)sgn.mapped; + else + target = NULL; + + sc->base = mmap(target, sc->mediasize, PROT_READ|PROT_WRITE, MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0); if (sc->base == MAP_FAILED) @@ -169,8 +179,11 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) sc->ident = SIGN_DATA(&sc->idn); i = smp_valid_silo(sc); - if (i) + if (i) { + printf("Warning SILO (%s) not reloaded (reason=%d)\n", + sc->filename, i); smp_newsilo(sc); + } AZ(smp_valid_silo(sc)); smp_metrics(sc); diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c index 552c0be..6fad388 100644 --- a/bin/varnishd/storage_persistent_subr.c +++ b/bin/varnishd/storage_persistent_subr.c @@ -238,26 +238,27 @@ smp_valid_silo(struct smp_sc *sc) assert(strlen(SMP_IDENT_STRING) < sizeof si->ident); - if (smp_chk_sign(&sc->idn)) - return (1); + i = smp_chk_sign(&sc->idn); + if (i) + return (i); si = sc->ident; if (strcmp(si->ident, SMP_IDENT_STRING)) - return (2); + return (12); if (si->byte_order != 0x12345678) - return (3); + return (13); if (si->size != sizeof *si) - return (4); + return (14); if (si->major_version != 2) - return (5); + return (15); if (si->mediasize != sc->mediasize) - return (7); + return (17); if (si->granularity != sc->granularity) - return (8); + return (18); if (si->align < sizeof(void*)) - return (9); + return (19); if (!PWR2(si->align)) - return (10); + return (20); sc->align = si->align; sc->unique = si->unique; diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc new file mode 100644 index 0000000..66f447b --- /dev/null +++ b/bin/varnishtest/tests/r00962.vtc @@ -0,0 +1,47 @@ +varnishtest "Test address remapping" + +server s1 { + rxreq + txresp +} -start + +shell "rm -f ${tmpdir}/_.per?" + +varnish v1 \ + -arg "-pdiag_bitmap=0x20000" \ + -storage "-spersistent,${tmpdir}/_.per1,10m -spersistent,${tmpdir}/_.per2,10m" \ + -vcl+backend { + sub vcl_fetch { + set beresp.storage = "s0"; + } +} -start + +varnish v1 -stop + +varnish v1 -start + +client c1 { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" +} -run + +varnish v1 -cliok "storage.list" +varnish v1 -cliok "debug.persistent s0 dump" +varnish v1 -cliok "debug.persistent s0 sync" +varnish v1 -stop + +varnish v2 \ + -arg "-pdiag_bitmap=0x20000" \ + -storage "-spersistent,${tmpdir}/_.per2,10m -spersistent,${tmpdir}/_.per1,10m" \ + -vcl+backend { } -start + +client c1 -connect ${v2_sock} { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" +} -run + +# shell "rm -f /tmp/__v1/_.per" From tfheen at varnish-cache.org Wed Aug 17 09:28:46 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:46 +0200 Subject: [3.0] 9703116 Prompted by HTTPbis and FlexeLint in unison, go over the nhttp and response code stuff and fully incorporate the fact that they are both 16 unsigned now. Message-ID: commit 970311694e70d2d67d0227238f200f2150b7958a Author: Poul-Henning Kamp Date: Mon Aug 15 08:37:38 2011 +0000 Prompted by HTTPbis and FlexeLint in unison, go over the nhttp and response code stuff and fully incorporate the fact that they are both 16 unsigned now. No functional changes. diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h index ef8435c..0b5ad8e 100644 --- a/bin/varnishd/cache.h +++ b/bin/varnishd/cache.h @@ -739,21 +739,21 @@ int VGZ_WrwGunzip(const struct sess *, struct vgz *, const void *ibuf, /* cache_http.c */ unsigned HTTP_estimate(unsigned nhttp); void HTTP_Copy(struct http *to, const struct http * const fm); -struct http *HTTP_create(void *p, unsigned nhttp); +struct http *HTTP_create(void *p, uint16_t nhttp); const char *http_StatusMessage(unsigned); -unsigned http_EstimateWS(const struct http *fm, unsigned how, unsigned *nhd); +unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd); void HTTP_Init(void); void http_ClrHeader(struct http *to); unsigned http_Write(struct worker *w, const struct http *hp, int resp); void http_CopyResp(struct http *to, const struct http *fm); -void http_SetResp(struct http *to, const char *proto, int status, +void http_SetResp(struct http *to, const char *proto, uint16_t status, const char *response); void http_FilterFields(struct worker *w, int fd, struct http *to, const struct http *fm, unsigned how); void http_FilterHeader(const struct sess *sp, unsigned how); void http_PutProtocol(struct worker *w, int fd, const struct http *to, const char *protocol); -void http_PutStatus(struct http *to, int status); +void http_PutStatus(struct http *to, uint16_t status); void http_PutResponse(struct worker *w, int fd, const struct http *to, const char *response); void http_PrintfHeader(struct worker *w, int fd, struct http *to, @@ -768,11 +768,11 @@ int http_GetHdrData(const struct http *hp, const char *hdr, int http_GetHdrField(const struct http *hp, const char *hdr, const char *field, char **ptr); double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field); -int http_GetStatus(const struct http *hp); +uint16_t http_GetStatus(const struct http *hp); const char *http_GetReq(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); -int http_DissectRequest(struct sess *sp); -int http_DissectResponse(struct worker *w, const struct http_conn *htc, +uint16_t http_DissectRequest(struct sess *sp); +uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc, struct http *sp); const char *http_DoConnection(const struct http *hp); void http_CopyHome(struct worker *w, int fd, const struct http *hp); diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index f5d2957..42d4b3b 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -417,10 +417,10 @@ cnt_error(struct sess *sp) /* XXX: 1024 is a pure guess */ EXP_Clr(&w->exp); sp->obj = STV_NewObject(sp, NULL, 1024, &w->exp, - params->http_max_hdr); + (uint16_t)params->http_max_hdr); if (sp->obj == NULL) sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, - 1024, &w->exp, params->http_max_hdr); + 1024, &w->exp, (uint16_t)params->http_max_hdr); if (sp->obj == NULL) { sp->doclose = "Out of objects"; sp->step = STP_DONE; @@ -637,7 +637,8 @@ cnt_fetchbody(struct sess *sp) int i; struct http *hp, *hp2; char *b; - unsigned l, nhttp; + uint16_t nhttp; + unsigned l; struct vsb *vary = NULL; int varyl = 0, pass; @@ -1411,7 +1412,7 @@ DOT start -> recv [style=bold,color=green] static int cnt_start(struct sess *sp) { - int done; + uint16_t done; char *p; const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; @@ -1439,7 +1440,7 @@ cnt_start(struct sess *sp) done = http_DissectRequest(sp); /* If we could not even parse the request, just close */ - if (done < 0) { + if (done == 400) { sp->step = STP_DONE; vca_close_session(sp, "junk"); return (0); diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c index 2141cc4..7fe0a8a 100644 --- a/bin/varnishd/cache_http.c +++ b/bin/varnishd/cache_http.c @@ -118,7 +118,7 @@ HTTP_estimate(unsigned nhttp) } struct http * -HTTP_create(void *p, unsigned nhttp) +HTTP_create(void *p, uint16_t nhttp) { struct http *hp; @@ -135,7 +135,7 @@ HTTP_create(void *p, unsigned nhttp) void http_Setup(struct http *hp, struct ws *ws) { - unsigned shd; + uint16_t shd; txt *hd; unsigned char *hdf; @@ -425,6 +425,7 @@ http_DoConnection(const struct http *hp) return (NULL); } ret = NULL; + AN(p); for (; *p; p++) { if (vct_issp(*p)) continue; @@ -463,7 +464,7 @@ http_HdrIs(const struct http *hp, const char *hdr, const char *val) /*--------------------------------------------------------------------*/ -int +uint16_t http_GetStatus(const struct http *hp) { @@ -483,7 +484,7 @@ http_GetReq(const struct http *hp) * Detect conditionals (headers which start with '^[Ii][Ff]-') */ -static int +static uint16_t http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p, const struct http_conn *htc) { @@ -557,7 +558,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p, * Deal with first line of HTTP protocol message. */ -static int +static uint16_t http_splitline(struct worker *w, int fd, struct http *hp, const struct http_conn *htc, int h1, int h2, int h3) { @@ -577,7 +578,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, q = p; for (; !vct_issp(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } hp->hd[h1].b = q; hp->hd[h1].e = p; @@ -585,14 +586,14 @@ http_splitline(struct worker *w, int fd, struct http *hp, /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } /* Second field cannot contain LWS or CTL */ q = p; for (; !vct_islws(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } hp->hd[h2].b = q; hp->hd[h2].e = p; @@ -603,7 +604,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (-1); + return (400); } /* Third field is optional and cannot contain CTL */ @@ -611,7 +612,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, if (!vct_iscrlf(*p)) { for (; !vct_iscrlf(*p); p++) if (!vct_issep(*p) && vct_isctl(*p)) - return (-1); + return (400); } hp->hd[h3].b = q; hp->hd[h3].e = p; @@ -650,12 +651,12 @@ http_ProtoVer(struct http *hp) /*--------------------------------------------------------------------*/ -int +uint16_t http_DissectRequest(struct sess *sp) { struct http_conn *htc; struct http *hp; - int i; + uint16_t retval; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); htc = sp->htc; @@ -665,23 +666,26 @@ http_DissectRequest(struct sess *sp) hp->logtag = HTTP_Rx; - i = http_splitline(sp->wrk, sp->fd, hp, htc, + retval = http_splitline(sp->wrk, sp->fd, hp, htc, HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO); - if (i != 0) { + if (retval != 0) { WSPR(sp, SLT_HttpGarbage, htc->rxbuf); - return (i); + return (retval); } http_ProtoVer(hp); - return (i); + return (retval); } /*--------------------------------------------------------------------*/ -int +uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc, struct http *hp) { - int i = 0; + int j; + uint16_t retval = 0; + char *p; + CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -689,23 +693,33 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc, if (http_splitline(w, htc->fd, hp, htc, HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE)) - i = 503; + retval = 503; - if (i == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7)) - i = 503; + if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7)) + retval = 503; - if (i == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3) - i = 503; + if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3) + retval = 503; - if (i == 0) { - hp->status = strtoul(hp->hd[HTTP_HDR_STATUS].b, NULL, 10); - if (hp->status < 100 || hp->status > 999) - i = 503; + if (retval == 0) { + hp->status = 0; + p = hp->hd[HTTP_HDR_STATUS].b; + for (j = 100; j != 0; j /= 10) { + if (!vct_isdigit(*p)) { + retval = 503; + break; + } + hp->status += (uint16_t)(j * (*p - '0')); + p++; + } + if (*p != '\0') + retval = 503; } - if (i != 0) { + if (retval != 0) { WSLR(w, SLT_HttpGarbage, htc->fd, htc->rxbuf); - hp->status = i; + assert(retval >= 100 && retval <= 999); + hp->status = retval; } else { http_ProtoVer(hp); } @@ -718,7 +732,7 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc, hp->hd[HTTP_HDR_RESPONSE].e = strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0'); } - return (i); + return (retval); } /*--------------------------------------------------------------------*/ @@ -763,12 +777,13 @@ http_CopyResp(struct http *to, const struct http *fm) } void -http_SetResp(struct http *to, const char *proto, int status, +http_SetResp(struct http *to, const char *proto, uint16_t status, const char *response) { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); http_SetH(to, HTTP_HDR_PROTO, proto); + assert(status >= 100 && status <= 999); to->status = status; http_SetH(to, HTTP_HDR_RESPONSE, response); } @@ -798,7 +813,7 @@ http_copyheader(struct worker *w, int fd, struct http *to, */ unsigned -http_EstimateWS(const struct http *fm, unsigned how, unsigned *nhd) +http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd) { unsigned u, l; @@ -967,10 +982,10 @@ http_PutProtocol(struct worker *w, int fd, const struct http *to, } void -http_PutStatus(struct http *to, int status) +http_PutStatus(struct http *to, uint16_t status) { - assert(status >= 0 && status <= 999); + assert(status >= 100 && status <= 999); to->status = status; } @@ -1011,7 +1026,7 @@ http_PrintfHeader(struct worker *w, int fd, struct http *to, void http_Unset(struct http *hp, const char *hdr) { - unsigned u, v; + uint16_t u, v; for (v = u = HTTP_HDR_FIRST; u < hp->nhd; u++) { if (hp->hd[u].b == NULL) diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c index 603ba83..0875e07 100644 --- a/bin/varnishd/cache_pool.c +++ b/bin/varnishd/cache_pool.c @@ -116,7 +116,7 @@ WRK_SumStat(struct worker *w) static void * wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace, - unsigned nhttp, unsigned http_space, unsigned siov) + uint16_t nhttp, unsigned http_space, unsigned siov) { struct worker *w, ww; uint32_t wlog[shm_workspace / 4]; @@ -218,12 +218,13 @@ static void * wrk_thread(void *priv) { struct wq *qp; - unsigned nhttp; + uint16_t nhttp; unsigned siov; CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC); + assert(params->http_max_hdr <= 65535); /* We need to snapshot these two for consistency */ - nhttp = params->http_max_hdr; + nhttp = (uint16_t)params->http_max_hdr; siov = nhttp * 2; if (siov > IOV_MAX) siov = IOV_MAX; diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c index 87e05c7..af422a6 100644 --- a/bin/varnishd/cache_session.c +++ b/bin/varnishd/cache_session.c @@ -101,8 +101,8 @@ ses_sm_alloc(void) { struct sessmem *sm; unsigned char *p, *q; - volatile unsigned nws; - volatile unsigned nhttp; + unsigned nws; + uint16_t nhttp; unsigned l, hl; if (VSC_C_main->n_sess_mem >= params->max_sess) @@ -113,7 +113,7 @@ ses_sm_alloc(void) * view of the value. */ nws = params->sess_workspace; - nhttp = params->http_max_hdr; + nhttp = (uint16_t)params->http_max_hdr; hl = HTTP_estimate(nhttp); l = sizeof *sm + nws + 2 * hl; p = malloc(l); diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c index 65b7522..c17c361 100644 --- a/bin/varnishd/cache_vrt.c +++ b/bin/varnishd/cache_vrt.c @@ -61,7 +61,9 @@ VRT_error(struct sess *sp, unsigned code, const char *reason) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ? reason : "(null)"); - sp->err_code = code ? code : 503; + if (code < 100 || code > 999) + code = 503; + sp->err_code = (uint16_t)code; sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code); } diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c index 1ee011b..2054953 100644 --- a/bin/varnishd/cache_vrt_var.c +++ b/bin/varnishd/cache_vrt_var.c @@ -102,7 +102,7 @@ VRT_l_##obj##_status(const struct sess *sp, int num) \ { \ \ assert(num >= 100 && num <= 999); \ - http->status = num; \ + http->status = (uint16_t)num; \ } \ \ int \ diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c index 3126744..7bfa707 100644 --- a/bin/varnishd/mgt_param.c +++ b/bin/varnishd/mgt_param.c @@ -549,7 +549,7 @@ static const struct parspec input_parspec[] = { "how much of that the request is allowed to take up.", 0, "32768", "bytes" }, - { "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, UINT_MAX, + { "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, 65535, "Maximum number of HTTP headers we will deal with in " "client request or backend reponses. " "Note that the first line occupies five header fields.\n" diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 533874e..1dad892 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -199,7 +199,7 @@ stv_alloc(const struct sess *sp, size_t size) struct stv_objsecrets { unsigned magic; #define STV_OBJ_SECRETES_MAGIC 0x78c87247 - unsigned nhttp; + uint16_t nhttp; unsigned lhttp; unsigned wsl; struct exp *exp; @@ -294,7 +294,7 @@ stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot, struct object * STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep, - unsigned nhttp) + uint16_t nhttp) { struct object *o; struct stevedore *stv; diff --git a/bin/varnishd/stevedore.h b/bin/varnishd/stevedore.h index 5a5cc4b..0ccf834 100644 --- a/bin/varnishd/stevedore.h +++ b/bin/varnishd/stevedore.h @@ -91,7 +91,7 @@ struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot, const struct stv_objsecrets *soc); struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len, - struct exp *, unsigned nhttp); + struct exp *, uint16_t nhttp); struct storage *STV_alloc(const struct sess *sp, size_t size); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); From tfheen at varnish-cache.org Wed Aug 17 09:28:58 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:58 +0200 Subject: [3.0] 2502e03 Implement a consistent retry policy in the random/client/hash director: Message-ID: commit 2502e03a748ec9ba097276fc34fd2de4d6b782eb Author: Poul-Henning Kamp Date: Mon Aug 15 19:45:58 2011 +0000 Implement a consistent retry policy in the random/client/hash director: If the first (policy-chosen) backend fails to get us a connection, retry a random backend (still according to their weight) until retries are exhausted. Kristian sent a proof of concept patch, I just cleaned it up and made it compile. Thanks to: Kristian Fixes #977 diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c index f85d905..c2b74dd 100644 --- a/bin/varnishd/cache_dir_random.c +++ b/bin/varnishd/cache_dir_random.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2010 Varnish Software AS + * Copyright (c) 2006-2011 Varnish Software AS * All rights reserved. * * Author: Poul-Henning Kamp @@ -26,17 +26,22 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This code is shared between the random and hash directors, because they - * share the same properties and most of the same selection logic. + * This code is shared between the random, client and hash directors, because + * they share the same properties and most of the same selection logic. * - * The random director picks a backend on random, according to weight, - * from the healty subset of backends. + * The random director picks a backend on random. + * + * The hash director picks based on the hash from vcl_hash{} + * + * The client director picks based on client identity or IP-address + * + * In all cases, the choice is by weight of the healthy subset of + * configured backends. + * + * Failures to get a connection are retried, here all three policies + * fall back to a deterministically random choice, by weight in the + * healthy subset. * - * The hash director first tries to locate the "canonical" backend from - * the full set, according to weight, and if it is healthy selects it. - * If the canonical backend is not healthy, we pick a backend according - * to weight from the healthy subset. That way only traffic to unhealthy - * backends gets redistributed. */ #include "config.h" @@ -46,6 +51,7 @@ #include #include +#include #include #include #include @@ -77,114 +83,117 @@ struct vdi_random { unsigned nhosts; }; -static struct vbc * -vdi_random_getfd(const struct director *d, struct sess *sp) +/* + * Applies sha256 using the given context and input/length, and returns + * a double in the range [0...1[ based on the hash. + */ +static double +vdi_random_sha(const char *input, ssize_t len) { - int i, k; - struct vdi_random *vs; - double r, s1; - unsigned u = 0; - struct vbc *vbe; - struct director *d2; struct SHA256Context ctx; - uint8_t sign[SHA256_LEN], *hp = NULL; + uint8_t sign[SHA256_LEN]; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC); + AN(input); + SHA256_Init(&ctx); + SHA256_Update(&ctx, input, len); + SHA256_Final(sign, &ctx); + return (vle32dec(sign) / exp2(32)); +} + +/* + * Sets up the initial seed for picking a backend according to policy. + */ +static double +vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp) +{ + const char *p; + double retval; - if (vs->criteria == c_client) { - /* - * Hash the client IP# ascii representation, rather than - * rely on the raw IP# being a good hash distributor, since - * experience shows this not to be the case. - * We do not hash the port number, to make everybody behind - * a given NAT gateway fetch from the same backend. - */ - SHA256_Init(&ctx); - AN(sp->addr); + switch (vs->criteria) { + case c_client: if (sp->client_identity != NULL) - SHA256_Update(&ctx, sp->client_identity, - strlen(sp->client_identity)); + p = sp->client_identity; else - SHA256_Update(&ctx, sp->addr, strlen(sp->addr)); - SHA256_Final(sign, &ctx); - hp = sign; + p = sp->addr; + retval = vdi_random_sha(p, strlen(p)); + break; + case c_hash: + AN(sp->digest); + retval = vle32dec(sp->digest) / exp2(32); + break; + case c_random: + default: + retval = random() / exp2(31); + break; } - if (vs->criteria == c_hash) { - /* - * Reuse the hash-string, the objective here is to fetch the - * same object on the same backend all the time - */ - hp = sp->digest; + return (retval); +} + +/* + * Find the healthy backend corresponding to the weight r [0...1[ + */ +static struct vbc * +vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r) +{ + double w[vs->nhosts]; + int i; + double s1; + + assert(r >= 0.0 && r < 1.0); + + memset(w, 0, sizeof w); + /* Sum up the weights of healty backends */ + s1 = 0.0; + for (i = 0; i < vs->nhosts; i++) { + if (VDI_Healthy(vs->hosts[i].backend, sp)) + w[i] = vs->hosts[i].weight; + s1 += w[i]; } - /* - * If we are hashing, first try to hit our "canonical backend" - * If that fails, we fall through, and select a weighted backend - * amongst the healthy set. - */ - if (vs->criteria != c_random) { - AN(hp); - u = vle32dec(hp); - r = u / 4294967296.0; - assert(r >= 0.0 && r < 1.0); - r *= vs->tot_weight; - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - s1 += vs->hosts[i].weight; - if (r >= s1) - continue; - d2 = vs->hosts[i].backend; - if (!VDI_Healthy(d2, sp)) - break; - vbe = VDI_GetFd(d2, sp); - if (vbe != NULL) - return (vbe); - break; - } + if (s1 == 0.0) + return (NULL); + + r *= s1; + s1 = 0.0; + for (i = 0; i < vs->nhosts; i++) { + s1 += w[i]; + if (r < s1) + return(VDI_GetFd(vs->hosts[i].backend, sp)); } + return (NULL); +} - for (k = 0; k < vs->retries; ) { - /* Sum up the weights of healty backends */ - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - d2 = vs->hosts[i].backend; - /* XXX: cache result of healty to avoid double work */ - if (VDI_Healthy(d2, sp)) - s1 += vs->hosts[i].weight; - } - - if (s1 == 0.0) - return (NULL); - - if (vs->criteria != c_random) { - r = u / 4294967296.0; - } else { - /* Pick a random threshold in that interval */ - r = random() / 2147483648.0; /* 2^31 */ - } - assert(r >= 0.0 && r < 1.0); - r *= s1; - - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - d2 = vs->hosts[i].backend; - if (!VDI_Healthy(d2, sp)) - continue; - s1 += vs->hosts[i].weight; - if (r >= s1) - continue; - vbe = VDI_GetFd(d2, sp); - if (vbe != NULL) - return (vbe); - break; - } - k++; +/* + * Try the specified number of times to get a backend. + * First one according to policy, after that, deterministically + * random by rehashing the key. + */ +static struct vbc * +vdi_random_getfd(const struct director *d, struct sess *sp) +{ + int k; + struct vdi_random *vs; + double r; + struct vbc *vbe; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC); + + r = vdi_random_init_seed(vs, sp); + + for (k = 0; k < vs->retries; k++) { + vbe = vdi_random_pick_one(sp, vs, r); + if (vbe != NULL) + return (vbe); + r = vdi_random_sha((void *)&r, sizeof(r)); } return (NULL); } +/* + * Healthy if just a single backend is... + */ static unsigned vdi_random_healthy(const struct director *d, const struct sess *sp) { diff --git a/bin/varnishtest/tests.disabled/r00977.vtc b/bin/varnishtest/tests.disabled/r00977.vtc deleted file mode 100644 index e6c61da..0000000 --- a/bin/varnishtest/tests.disabled/r00977.vtc +++ /dev/null @@ -1,28 +0,0 @@ - -varnishtest "Test proper fallbacks of client director" - -server s1 -repeat 1 { - rxreq - txresp -status 200 -} -start - -varnish v1 -vcl+backend { - director foo client{ - .retries = 5; - { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } - { .backend = s1; .weight = 1;} - } - sub vcl_recv { - set req.backend = foo; - set client.identity = "44.452"; - return (pass); - } -} -start - -client c1 { - txreq - rxresp - expect resp.status == 200 -} -run - -varnish v1 -expect backend_fail == 1 diff --git a/bin/varnishtest/tests/r00977.vtc b/bin/varnishtest/tests/r00977.vtc new file mode 100644 index 0000000..d15b55d --- /dev/null +++ b/bin/varnishtest/tests/r00977.vtc @@ -0,0 +1,46 @@ + +varnishtest "Test proper fallbacks of client director" + +server s1 { + rxreq + txresp -status 200 + accept + rxreq + txresp -status 200 +} -start + +varnish v1 -vcl+backend { + director foo client{ + .retries = 5; + { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } + { .backend = s1; .weight = 1;} + } + director bar client{ + .retries = 1; + { .backend = { .host = "${bad_ip}"; .port = "9090"; } .weight = 1; } + { .backend = s1; .weight = 1;} + } + sub vcl_recv { + if (req.url ~ "/one") { + set req.backend = foo; + } else { + set req.backend = bar; + } + # Carefully chosen seed that'll give us bad backend on + # first try and good on second. + set client.identity = "1.4"; + return (pass); + } +} -start + +client c1 { + txreq -url "/one" + rxresp + expect resp.status == 200 + + txreq -url "/two" + rxresp + expect resp.status == 503 +} -run + +varnish v1 -expect backend_fail == 2 From tfheen at varnish-cache.org Wed Aug 17 09:28:59 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:59 +0200 Subject: [3.0] 0006ddd We can not use a lenght based response when we transform (gzip/gunzip) we stream and backend didn't send c-l. Message-ID: commit 0006ddd162326b732e9ebd09f0af403fde460e41 Author: Poul-Henning Kamp Date: Wed Aug 17 07:08:11 2011 +0000 We can not use a lenght based response when we transform (gzip/gunzip) we stream and backend didn't send c-l. Fixes #980 Testcase by: Martin diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 42d4b3b..672e1dc 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -166,8 +166,8 @@ cnt_prepresp(struct sess *sp) sp->wrk->res_mode = 0; - if (!sp->wrk->do_stream || - (sp->wrk->h_content_length != NULL && !sp->wrk->do_gunzip)) + if ((sp->wrk->h_content_length != NULL || !sp->wrk->do_stream) && + !sp->wrk->do_gzip && !sp->wrk->do_gunzip) sp->wrk->res_mode |= RES_LEN; if (!sp->disable_esi && sp->obj->esidata != NULL) { diff --git a/bin/varnishtest/tests/r00980.vtc b/bin/varnishtest/tests/r00980.vtc new file mode 100644 index 0000000..9591786 --- /dev/null +++ b/bin/varnishtest/tests/r00980.vtc @@ -0,0 +1,29 @@ +varnishtest "r00980 test gzip on fetch with content_length and do_stream" + +server s1 { + rxreq + expect req.url == "/foobar" + expect req.http.accept-encoding == "gzip" + txresp -bodylen 43 +} -start + +varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { + + sub vcl_fetch { + set beresp.do_gzip = true; + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq -url /foobar -hdr "Accept-Encoding: gzip" + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.bodylen == 43 + + txreq -url /foobar + rxresp + expect resp.http.content-encoding == "resp.http.content-encoding" + expect resp.bodylen == 43 +} -run From tfheen at varnish-cache.org Wed Aug 17 09:29:00 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:29:00 +0200 Subject: [3.0] 6f7c8a7 restart in vcl_deliver{} would crash in vcl_fetch{} due to missing cleanup. Message-ID: commit 6f7c8a774de891661756df523ec0f60f1709f8ea Author: Poul-Henning Kamp Date: Wed Aug 17 07:15:30 2011 +0000 restart in vcl_deliver{} would crash in vcl_fetch{} due to missing cleanup. Found & Fixed by: Martin Fixes #979 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 672e1dc..47f3720 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -233,6 +233,7 @@ cnt_prepresp(struct sess *sp) AZ(sp->obj); sp->restarts++; sp->director = NULL; + sp->wrk->h_content_length = NULL; http_Setup(sp->wrk->bereq, NULL); http_Setup(sp->wrk->beresp, NULL); http_Setup(sp->wrk->resp, NULL); diff --git a/bin/varnishtest/tests/r00979.vtc b/bin/varnishtest/tests/r00979.vtc new file mode 100644 index 0000000..bc72efc --- /dev/null +++ b/bin/varnishtest/tests/r00979.vtc @@ -0,0 +1,29 @@ +varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver" + +server s1 { + rxreq + txresp -status 200 -body "1" + expect_close + + accept + rxreq + txresp -status 200 -body "11" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = true; + } + sub vcl_deliver { + if (req.restarts == 0) { + return (restart); + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 2 +} -run From tfheen at varnish-cache.org Wed Aug 17 09:29:02 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:29:02 +0200 Subject: [3.0] be6cd99 hit_for_pass in vcl_fetch Message-ID: commit be6cd99791ae3b99f389e4c49e2ae6bf5322bacd Author: Per Buer Date: Wed Aug 17 10:10:07 2011 +0200 hit_for_pass in vcl_fetch diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index a16d7f5..c193e08 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -485,8 +485,12 @@ vcl_fetch error code [reason] Return the specified error code to the client and abandon the request. - pass - Switch to pass mode. Control will eventually pass to vcl_pass. + hit_for_pass + Pass in fetch. This will create a hit_for_pass object. Note that + the TTL for the hit_for_pass object will be set to what the + current value of beresp.ttl. Control will be handled to + vcl_deliver on the current request, but subsequent requests will + go directly to vcl_pass based on the hit_for_pass object. restart Restart the transaction. Increases the restart counter. If the number From tfheen at varnish-cache.org Wed Aug 17 09:29:02 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:29:02 +0200 Subject: [3.0] 6aaefe2 std.log missing () Message-ID: commit 6aaefe23a1912cff9c273adbcf57da548ec923c6 Author: Per Buer Date: Wed Aug 17 10:18:15 2011 +0200 std.log missing () diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 1a31733..2afbf64 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -21,7 +21,7 @@ String concatenation did not have an operator previously, but this has now been becomes import std; - std.log "log something"; + std.log("log something"); You only need to import std once. From tfheen at varnish-cache.org Wed Aug 17 09:29:03 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:29:03 +0200 Subject: [3.0] b6580d8 Add a "sendhex" verb Message-ID: commit b6580d8359161c428f7c3cd4fa98a97632c58cb8 Author: Poul-Henning Kamp Date: Wed Aug 17 08:31:18 2011 +0000 Add a "sendhex" verb diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 55228d5..366a5dd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -904,6 +904,47 @@ cmd_http_send(CMD_ARGS) } /********************************************************************** + * Send a hex string + */ + +static void +cmd_http_sendhex(CMD_ARGS) +{ + struct http *hp; + char buf[3], *q; + uint8_t *p; + int i, j, l; + + (void)cmd; + (void)vl; + CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); + AN(av[1]); + AZ(av[2]); + l = strlen(av[1]) / 2; + p = malloc(l); + AN(p); + q = av[1]; + for (i = 0; i < l; i++) { + while (vct_issp(*q)) + q++; + if (*q == '\0') + break; + memcpy(buf, q, 2); + q += 2; + buf[2] = '\0'; + if (!vct_ishex(buf[0]) || !vct_ishex(buf[1])) + vtc_log(hp->vl, 0, "Illegal Hex char \"%c%c\"", + buf[0], buf[1]); + p[i] = strtoul(buf, NULL, 16); + } + vtc_hexdump(hp->vl, 4, "sendhex", (void*)p, i); + j = write(hp->fd, p, i); + assert(j == i); + free(p); + +} + +/********************************************************************** * Send a string as chunked encoding */ @@ -1080,6 +1121,7 @@ static const struct cmds http_cmds[] = { { "gunzip", cmd_http_gunzip_body }, { "expect", cmd_http_expect }, { "send", cmd_http_send }, + { "sendhex", cmd_http_sendhex }, { "chunked", cmd_http_chunked }, { "chunkedlen", cmd_http_chunkedlen }, { "delay", cmd_delay }, diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index f0841c0..18f295a 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -227,8 +227,10 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha { int nl = 1; unsigned l; + double tx; CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC); + tx = TIM_mono() - t0; assert(len >= 0); assert(lvl < NLEAD); AZ(pthread_mutex_lock(&vl->mtx)); @@ -236,8 +238,8 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha if (pfx == NULL) pfx = ""; if (str == NULL) - VSB_printf(vl->vsb, "%s %-4s %s(null)\n", - lead[lvl], vl->id, pfx); + VSB_printf(vl->vsb, "%s %-4s %4.1f %s| (null)", + lead[lvl], vl->id, tx, pfx); else { for (l = 0; l < len; l++, str++) { if (l > 512) { @@ -245,8 +247,8 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha break; } if (nl) { - VSB_printf(vl->vsb, "%s %-4s %s| ", - lead[lvl], vl->id, pfx); + VSB_printf(vl->vsb, "%s %-4s %4.1f %s| ", + lead[lvl], vl->id, tx, pfx); nl = 0; } VSB_printf(vl->vsb, " %02x", *str); From phk at varnish-cache.org Wed Aug 17 09:34:23 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 11:34:23 +0200 Subject: [master] b54440f If the backend used chunked encoding and sent junk after the gzip data, the thread would go into a spin. Message-ID: commit b54440ffda04ffa093e21de578249d85e648c05d Author: Poul-Henning Kamp Date: Wed Aug 17 09:33:10 2011 +0000 If the backend used chunked encoding and sent junk after the gzip data, the thread would go into a spin. Fixes #942 diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c index f320a0a..40a8bdf 100644 --- a/bin/varnishd/cache_gzip.c +++ b/bin/varnishd/cache_gzip.c @@ -302,7 +302,7 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen) return (VGZ_END); if (i == Z_BUF_ERROR) return (VGZ_STUCK); -printf("INFLATE=%d (%s)\n", i, vg->vz.msg); + VSL(SLT_Debug, 0, "Unknown INFLATE=%d (%s)\n", i, vg->vz.msg); return (VGZ_ERROR); } @@ -629,6 +629,10 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes) while (!VGZ_IbufEmpty(vg)) { VGZ_Obuf(vg, obuf, sizeof obuf); i = VGZ_Gunzip(vg, &dp, &dl); + if (i == VGZ_END && !VGZ_IbufEmpty(vg)) { + WSP(sp, SLT_FetchError, "Junk after gzip data"); + return (-1); + } if (i != VGZ_OK && i != VGZ_END) { WSP(sp, SLT_FetchError, "Invalid Gzip data: %s", vg->vz.msg); diff --git a/bin/varnishtest/tests/r00942.vtc b/bin/varnishtest/tests/r00942.vtc new file mode 100644 index 0000000..0df8eb2 --- /dev/null +++ b/bin/varnishtest/tests/r00942.vtc @@ -0,0 +1,38 @@ +varnishtest "#942 junk after gzip from backend" + +server s1 { + rxreq + txresp -nolen \ + -hdr "Content-Encoding: gzip" \ + -hdr "Transfer-Encoding: Chunked" + send "14\r\n" + # An empty gzip file: + sendhex "1f8b" + sendhex "08" + sendhex "00" + sendhex "00000000" + sendhex "00" + sendhex "03" + sendhex "0300" + sendhex "00000000" + sendhex "00000000" + send "\r\n" + + chunked "FOOBAR" + + chunkedlen 0 + +} -start + +varnish v1 \ + -arg {-p diag_bitmap=0x00010000} \ + -vcl+backend {} + +varnish v1 -start + +client c1 { + txreq + rxresp + expect resp.status == 503 +} -run + From phk at varnish-cache.org Wed Aug 17 09:34:23 2011 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Wed, 17 Aug 2011 11:34:23 +0200 Subject: [master] 6ba82cb Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 6ba82cb2369d20bbc78ba7c2aea77654144ece30 Merge: b54440f d4681a6 Author: Poul-Henning Kamp Date: Wed Aug 17 09:34:04 2011 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From tfheen at varnish-cache.org Wed Aug 17 09:28:58 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:58 +0200 Subject: [3.0] 693a89b My recent 'discover correct -spersistent mmap address' fix broke 32 bit systems. Message-ID: commit 693a89beea7fd63b572811eb5575084f6a3336fd Author: Poul-Henning Kamp Date: Mon Aug 15 15:01:39 2011 +0000 My recent 'discover correct -spersistent mmap address' fix broke 32 bit systems. Spotted by: Kristian diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c index a22cc59..c4f2191 100644 --- a/bin/varnishd/storage_persistent_mgt.c +++ b/bin/varnishd/storage_persistent_mgt.c @@ -164,7 +164,7 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) i = read(sc->fd, &sgn, sizeof sgn); assert(i == sizeof sgn); if (!strcmp(sgn.ident, "SILO")) - target = (void*)sgn.mapped; + target = (void*)(uintptr_t)sgn.mapped; else target = NULL; From tfheen at varnish-cache.org Wed Aug 17 09:26:54 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:54 +0200 Subject: [3.0] 9cc413e Overhaul the stevedore statistics, the SMA and SMF interpreted certain fields in different ways. Message-ID: commit 9cc413ea93d7c11d0680fca04326ebf01b905128 Author: Poul-Henning Kamp Date: Tue Aug 9 10:01:53 2011 +0000 Overhaul the stevedore statistics, the SMA and SMF interpreted certain fields in different ways. diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c index 27cc86f..ea78526 100644 --- a/bin/varnishd/storage_file.c +++ b/bin/varnishd/storage_file.c @@ -182,9 +182,9 @@ insfree(struct smf_sc *sc, struct smf *sp) b = sp->size / sc->pagesize; if (b >= NBUCKET) { b = NBUCKET - 1; - sc->stats->n_smf_large++; + sc->stats->g_smf_large++; } else { - sc->stats->n_smf_frag++; + sc->stats->g_smf_frag++; } sp->flist = &sc->free[b]; ns = b * sc->pagesize; @@ -212,9 +212,9 @@ remfree(const struct smf_sc *sc, struct smf *sp) b = sp->size / sc->pagesize; if (b >= NBUCKET) { b = NBUCKET - 1; - sc->stats->n_smf_large--; + sc->stats->g_smf_large--; } else { - sc->stats->n_smf_frag--; + sc->stats->g_smf_frag--; } assert(sp->flist == &sc->free[b]); VTAILQ_REMOVE(sp->flist, sp, status); @@ -260,7 +260,7 @@ alloc_smf(struct smf_sc *sc, size_t bytes) /* Split from front */ sp2 = malloc(sizeof *sp2); XXXAN(sp2); - sc->stats->n_smf++; + sc->stats->g_smf++; *sp2 = *sp; sp->offset += bytes; @@ -302,7 +302,7 @@ free_smf(struct smf *sp) VTAILQ_REMOVE(&sc->order, sp2, order); remfree(sc, sp2); free(sp2); - sc->stats->n_smf--; + sc->stats->g_smf--; } sp2 = VTAILQ_PREV(sp, smfhead, order); @@ -314,7 +314,7 @@ free_smf(struct smf *sp) sp2->size += sp->size; VTAILQ_REMOVE(&sc->order, sp, order); free(sp); - sc->stats->n_smf--; + sc->stats->g_smf--; sp = sp2; } @@ -339,7 +339,7 @@ trim_smf(struct smf *sp, size_t bytes) CHECK_OBJ_NOTNULL(sp, SMF_MAGIC); sp2 = malloc(sizeof *sp2); XXXAN(sp2); - sc->stats->n_smf++; + sc->stats->g_smf++; *sp2 = *sp; sp2->size -= bytes; @@ -365,7 +365,7 @@ new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len) XXXAN(sp); sp->magic = SMF_MAGIC; sp->s.magic = STORAGE_MAGIC; - sc->stats->n_smf++; + sc->stats->g_smf++; sp->sc = sc; sp->size = len; @@ -452,7 +452,7 @@ smf_open(const struct stevedore *st) if (sum < MINPAGES * (off_t)getpagesize()) exit (2); - sc->stats->bfree += sc->filesize; + sc->stats->g_space += sc->filesize; } /*--------------------------------------------------------------------*/ @@ -468,16 +468,18 @@ smf_alloc(struct stevedore *st, size_t size) size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); Lck_Lock(&sc->mtx); - sc->stats->nreq++; + sc->stats->c_req++; smf = alloc_smf(sc, size); if (smf == NULL) { + sc->stats->c_fail++; Lck_Unlock(&sc->mtx); return (NULL); } CHECK_OBJ_NOTNULL(smf, SMF_MAGIC); - sc->stats->nobj++; - sc->stats->balloc += smf->size; - sc->stats->bfree -= smf->size; + sc->stats->g_alloc++; + sc->stats->c_bytes += smf->size; + sc->stats->g_bytes += smf->size; + sc->stats->g_space -= smf->size; Lck_Unlock(&sc->mtx); CHECK_OBJ_NOTNULL(&smf->s, STORAGE_MAGIC); /*lint !e774 */ XXXAN(smf); @@ -513,8 +515,9 @@ smf_trim(struct storage *s, size_t size) size &= ~(sc->pagesize - 1); if (smf->size > size) { Lck_Lock(&sc->mtx); - sc->stats->balloc -= (smf->size - size); - sc->stats->bfree += (smf->size - size); + sc->stats->c_freed += (smf->size - size); + sc->stats->g_bytes -= (smf->size - size); + sc->stats->g_space += (smf->size - size); trim_smf(smf, size); assert(smf->size == size); Lck_Unlock(&sc->mtx); @@ -534,9 +537,10 @@ smf_free(struct storage *s) CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC); sc = smf->sc; Lck_Lock(&sc->mtx); - sc->stats->nobj--; - sc->stats->balloc -= smf->size; - sc->stats->bfree += smf->size; + sc->stats->g_alloc--; + sc->stats->c_freed += smf->size; + sc->stats->g_bytes -= smf->size; + sc->stats->g_space += smf->size; free_smf(smf); Lck_Unlock(&sc->mtx); } diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c index 56b4daa..7035ff0 100644 --- a/bin/varnishd/storage_malloc.c +++ b/bin/varnishd/storage_malloc.c @@ -65,14 +65,17 @@ sma_alloc(struct stevedore *st, size_t size) CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); Lck_Lock(&sma_sc->sma_mtx); - sma_sc->stats->nreq++; - if (sma_sc->sma_alloc + size > sma_sc->sma_max) + sma_sc->stats->c_req++; + if (sma_sc->sma_alloc + size > sma_sc->sma_max) { size = 0; - else { + sma_sc->stats->c_fail += size; + } else { sma_sc->sma_alloc += size; - sma_sc->stats->nobj++; - sma_sc->stats->nbytes += size; - sma_sc->stats->balloc += size; + sma_sc->stats->c_bytes += size; + sma_sc->stats->g_alloc++; + sma_sc->stats->g_bytes += size; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space -= size; } Lck_Unlock(&sma_sc->sma_mtx); @@ -96,9 +99,16 @@ sma_alloc(struct stevedore *st, size_t size) } if (sma == NULL) { Lck_Lock(&sma_sc->sma_mtx); - sma_sc->stats->nobj--; - sma_sc->stats->nbytes -= size; - sma_sc->stats->balloc -= size; + /* + * XXX: Not nice to have counters go backwards, but we do + * XXX: Not want to pick up the lock twice just for stats. + */ + sma_sc->stats->c_fail++; + sma_sc->stats->c_bytes -= size; + sma_sc->stats->g_alloc--; + sma_sc->stats->g_bytes -= size; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space += size; Lck_Unlock(&sma_sc->sma_mtx); return (NULL); } @@ -127,9 +137,11 @@ sma_free(struct storage *s) assert(sma->sz == sma->s.space); Lck_Lock(&sma_sc->sma_mtx); sma_sc->sma_alloc -= sma->sz; - sma_sc->stats->nobj--; - sma_sc->stats->nbytes -= sma->sz; - sma_sc->stats->bfree += sma->sz; + sma_sc->stats->g_alloc--; + sma_sc->stats->g_bytes -= sma->sz; + sma_sc->stats->c_freed += sma->sz; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space += sma->sz; Lck_Unlock(&sma_sc->sma_mtx); free(sma->s.ptr); free(sma); @@ -150,8 +162,10 @@ sma_trim(struct storage *s, size_t size) assert(size < sma->sz); if ((p = realloc(sma->s.ptr, size)) != NULL) { Lck_Lock(&sma_sc->sma_mtx); - sma_sc->stats->nbytes -= (sma->sz - size); - sma_sc->stats->bfree += sma->sz - size; + sma_sc->stats->g_bytes -= (sma->sz - size); + sma_sc->stats->c_freed += sma->sz - size; + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space += sma->sz - size; sma->sz = size; Lck_Unlock(&sma_sc->sma_mtx); sma->s.ptr = p; @@ -219,6 +233,8 @@ sma_open(const struct stevedore *st) sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats, VSC_CLASS, VSC_TYPE_SMA, st->ident); memset(sma_sc->stats, 0, sizeof *sma_sc->stats); + if (sma_sc->sma_max != SIZE_MAX) + sma_sc->stats->g_space = sma_sc->sma_max; } const struct stevedore sma_stevedore = { diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc index 243de19..54a7ab7 100644 --- a/bin/varnishtest/tests/b00002.vtc +++ b/bin/varnishtest/tests/b00002.vtc @@ -21,7 +21,7 @@ client c1 { delay .1 varnish v1 -expect n_object == 0 -varnish v1 -expect SMA.Transient.nobj == 0 +varnish v1 -expect SMA.Transient.g_alloc == 0 varnish v1 -expect client_conn == 1 varnish v1 -expect client_req == 1 varnish v1 -expect s_sess == 1 diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc index dba75ae..3f2f951 100644 --- a/bin/varnishtest/tests/g00002.vtc +++ b/bin/varnishtest/tests/g00002.vtc @@ -32,9 +32,9 @@ client c1 { } -run # If this fails, the multiple storage allocations did not happen -varnish v1 -expect SMF.s0.nreq != 0 -varnish v1 -expect SMF.s0.nreq != 1 -varnish v1 -expect SMF.s0.nreq != 2 +varnish v1 -expect SMF.s0.c_req != 0 +varnish v1 -expect SMF.s0.c_req != 1 +varnish v1 -expect SMF.s0.c_req != 2 client c1 { # See varnish can gunzip it. diff --git a/include/vsc_fields.h b/include/vsc_fields.h index 0289b81..0952698 100644 --- a/include/vsc_fields.h +++ b/include/vsc_fields.h @@ -28,6 +28,13 @@ * * 3rd argument marks fields for inclusion in the per worker-thread * stats structure. + * + * XXX: We need a much more consistent naming of these fields, this has + * XXX: turned into a major mess, causing trouble already for backends. + * XXX: + * XXX: Please converge on: + * XXX: c_* counter (total bytes ever allocated from sma) + * XXX: g_* gauge (presently allocated bytes from sma) */ /**********************************************************************/ @@ -173,11 +180,13 @@ VSC_F(colls, uint64_t, 0, 'a', "Collisions") */ #if defined(VSC_DO_SMA) || defined (VSC_DO_SMF) -VSC_F(nreq, uint64_t, 0, 'a', "Allocator requests") -VSC_F(nobj, uint64_t, 0, 'i', "Outstanding allocations") -VSC_F(nbytes, uint64_t, 0, 'i', "Outstanding bytes") -VSC_F(balloc, uint64_t, 0, 'i', "Bytes allocated") -VSC_F(bfree, uint64_t, 0, 'i', "Bytes free") +VSC_F(c_req, uint64_t, 0, 'a', "Allocator requests") +VSC_F(c_fail, uint64_t, 0, 'a', "Allocator failures") +VSC_F(c_bytes, uint64_t, 0, 'a', "Bytes allocated") +VSC_F(c_freed, uint64_t, 0, 'a', "Bytes freed") +VSC_F(g_alloc, uint64_t, 0, 'i', "Allocations outstanding") +VSC_F(g_bytes, uint64_t, 0, 'i', "Bytes outstanding") +VSC_F(g_space, uint64_t, 0, 'i', "Bytes available") #endif @@ -190,9 +199,9 @@ VSC_F(bfree, uint64_t, 0, 'i', "Bytes free") /**********************************************************************/ #ifdef VSC_DO_SMF -VSC_F(n_smf, uint64_t, 0, 'i', "N struct smf") -VSC_F(n_smf_frag, uint64_t, 0, 'i', "N small free smf") -VSC_F(n_smf_large, uint64_t, 0, 'i', "N large free smf") +VSC_F(g_smf, uint64_t, 0, 'i', "N struct smf") +VSC_F(g_smf_frag, uint64_t, 0, 'i', "N small free smf") +VSC_F(g_smf_large, uint64_t, 0, 'i', "N large free smf") #endif /**********************************************************************/ From tfheen at varnish-cache.org Wed Aug 17 09:26:06 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:26:06 +0200 Subject: [3.0] 24e4a2f Cap the TTL (to param "shortlived") when we use the Transient storage to avoid dropping an object on out of storage conditions. Message-ID: commit 24e4a2f51a329563188dc5c4cbb58337f0d09aef Author: Poul-Henning Kamp Date: Tue Aug 2 09:48:16 2011 +0000 Cap the TTL (to param "shortlived") when we use the Transient storage to avoid dropping an object on out of storage conditions. I belive this... Fixes #953 Otherwise please reopen. diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 23ad92f..4bac471 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -754,7 +754,8 @@ cnt_fetchbody(struct sess *sp) */ sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l, &sp->wrk->exp, nhttp); - sp->wrk->exp.ttl = params->shortlived; + if (sp->wrk->exp.ttl > params->shortlived) + sp->wrk->exp.ttl = params->shortlived; } if (sp->obj == NULL) { sp->err_code = 503; From tfheen at varnish-cache.org Wed Aug 17 09:25:34 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:34 +0200 Subject: [3.0] 879ca02 Another insufficiently wide memset spotted by LLVM. Message-ID: commit 879ca02cf57e306616518e9983fe017b8e2ae61f Author: Poul-Henning Kamp Date: Mon Aug 1 11:20:48 2011 +0000 Another insufficiently wide memset spotted by LLVM. diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c index ea8d6e2..552c0be 100644 --- a/bin/varnishd/storage_persistent_subr.c +++ b/bin/varnishd/storage_persistent_subr.c @@ -67,7 +67,7 @@ smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx, AZ(off & 7); /* Alignment */ assert(strlen(id) < sizeof ctx->ss->ident); - memset(ctx, 0, sizeof ctx); + memset(ctx, 0, sizeof *ctx); ctx->ss = (void*)(sc->base + off); ctx->unique = sc->unique; ctx->id = id; From tfheen at varnish-cache.org Wed Aug 17 09:25:34 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:25:34 +0200 Subject: [3.0] 4dd65b9 Fix a classic programmer bug, spotted by the increasingly eagle-eyed LLVM C-compiler. Message-ID: commit 4dd65b9c30d47bac6a9f17e875f9442c395ea0ea Author: Poul-Henning Kamp Date: Mon Aug 1 11:16:14 2011 +0000 Fix a classic programmer bug, spotted by the increasingly eagle-eyed LLVM C-compiler. You have no idea how much or how long (20+ years!) I have missed having an alternative to GCC. Remember: Mono cultures are particularly bad for you, when they are good for you Kudos to: The LLVM team. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index e0f7c21..2b6e3d4 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -206,7 +206,7 @@ varnish_thread(void *priv) (void)VTCP_nonblocking(v->fds[0]); while (1) { fds = &fd; - memset(fds, 0, sizeof fds); + memset(fds, 0, sizeof *fds); fds->fd = v->fds[0]; fds->events = POLLIN; i = poll(fds, 1, 1000); From tfheen at varnish-cache.org Wed Aug 17 09:28:57 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:57 +0200 Subject: [3.0] a03088c the basics on persistent storage. Explained the semantics quickly so people understand why not all objects survive a crash Message-ID: commit a03088c86bec3416287652622bfd0659433a5385 Author: Per Buer Date: Mon Aug 15 14:52:43 2011 +0200 the basics on persistent storage. Explained the semantics quickly so people understand why not all objects survive a crash diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 1e00d0c..df6b87f 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -207,8 +207,32 @@ file[,path[,size[,granularity]]] The default size is the VM page size. The size should be reduced if you have many small objects. -persistence[XXX] - New, shiny, better. +persistent,path,size {experimental} + + Persistent storage. Varnish will store objects in a file in a + manner that will secure the survival of *most* of the objects in + the event of a planned or unplanned shutdown of Varnish. + + The path parameter specifies the path to the backing file. If + the file doesn't exist Varnish will create it. + + The size parameter specifies the size of the backing file. The + size is assumed to be in bytes, unless followed by one of the + following suffixes: + + K, k The size is expressed in kibibytes. + + M, m The size is expressed in mebibytes. + + G, g The size is expressed in gibibytes. + + T, t The size is expressed in tebibytes. + + Varnish will split the file into logical *silos* and write to + the silos in the manner of a circular buffer. Only one silo will + be kept open at any given point in time. Full silos are + *sealed*. When Varnish starts after a shutdown it will discard + the content of any silo that isn't sealed. Transient Storage ----------------- From tfheen at varnish-cache.org Wed Aug 17 09:28:39 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:39 +0200 Subject: [3.0] 6b9fa15 We always allocate an object entirely inside one stevedore now. Message-ID: commit 6b9fa157dc5e3151753b066b3daf4926d846119c Author: Poul-Henning Kamp Date: Mon Aug 15 07:29:37 2011 +0000 We always allocate an object entirely inside one stevedore now. diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c index 9a0b671..533874e 100644 --- a/bin/varnishd/stevedore.c +++ b/bin/varnishd/stevedore.c @@ -151,20 +151,15 @@ static struct storage * stv_alloc(const struct sess *sp, size_t size) { struct storage *st; - struct stevedore *stv = NULL; + struct stevedore *stv; unsigned fail = 0; /* - * Always try the stevedore which allocated the object in order to - * not needlessly split an object across multiple stevedores. + * Always use the stevedore which allocated the object in order to + * keep an object inside the same stevedore. */ - if (sp->obj != NULL) { - CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); - stv = sp->obj->objstore->stevedore; - } else { - INCOMPL(); - stv = stv_transient; - } + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + stv = sp->obj->objstore->stevedore; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); if (size > (size_t)(params->fetch_maxchunksize) << 10) From tfheen at varnish-cache.org Wed Aug 17 09:27:54 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:27:54 +0200 Subject: [3.0] f9612d2 Test case for #972 Message-ID: commit f9612d26e6d343a117204dccd5c348217684b8f0 Author: Poul-Henning Kamp Date: Wed Aug 10 13:18:36 2011 +0000 Test case for #972 Written by: Martin diff --git a/bin/varnishtest/tests/r00972.vtc b/bin/varnishtest/tests/r00972.vtc new file mode 100644 index 0000000..ed3a916 --- /dev/null +++ b/bin/varnishtest/tests/r00972.vtc @@ -0,0 +1,21 @@ +varnishtest "Test conditional delivery and do_stream" + +server s1 { + rxreq + txresp -hdr "ETag: foo" -body "11111\n" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq -hdr "If-None-Match: foo" + rxresp + expect resp.status == 304 + expect resp.http.etag == "foo" + expect resp.bodylen == 0 +} -run + From tfheen at varnish-cache.org Wed Aug 17 09:29:01 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:29:01 +0200 Subject: [3.0] 0f8805f Cleare do_stream on all esi objects, including included objects. Message-ID: commit 0f8805f5e56f4d3312721c0cec7a3b1fa371be5f Author: Poul-Henning Kamp Date: Wed Aug 17 07:24:13 2011 +0000 Cleare do_stream on all esi objects, including included objects. Found & Fixed by: Martin Fixes #978 diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c index 47f3720..93bfc75 100644 --- a/bin/varnishd/cache_center.c +++ b/bin/varnishd/cache_center.c @@ -718,7 +718,7 @@ cnt_fetchbody(struct sess *sp) else if (sp->wrk->is_gzip) sp->wrk->vfp = &vfp_testgzip; - if (sp->wrk->do_esi) + if (sp->wrk->do_esi || sp->esi_level > 0) sp->wrk->do_stream = 0; if (!sp->wantbody) sp->wrk->do_stream = 0; diff --git a/bin/varnishtest/tests/r00978.vtc b/bin/varnishtest/tests/r00978.vtc new file mode 100644 index 0000000..39b89cb --- /dev/null +++ b/bin/varnishtest/tests/r00978.vtc @@ -0,0 +1,40 @@ +varnishtest "r00978.vtc Test esi_level > 0 and do_stream" + +server s1 { + rxreq + expect req.url == "/" + txresp -body { + + Before include + + After include + } + rxreq + expect req.url == "/body1" + txresp -body { + Included file + } +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.esi_level > 0) { + set req.url = req.url + req.esi_level; + } + } + sub vcl_fetch { + if (req.url == "/") { + set beresp.do_esi = true; + } + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 65 + expect resp.status == 200 +} -run + +varnish v1 -expect esi_errors == 0 From tfheen at varnish-cache.org Wed Aug 17 09:28:33 2011 From: tfheen at varnish-cache.org (Tollef Fog Heen) Date: Wed, 17 Aug 2011 11:28:33 +0200 Subject: [3.0] 1142de7 Constify Message-ID:

Error "} obj.status " " obj.response {"

Error "} + obj.status + " " + obj.response + {"

Guru Meditation:

Error "} obj.status " " obj.response {"

Error "} + obj.status + " " + obj.response + {"

Guru Meditation:

Error "} obj.status " " obj.response {"

Error "} + obj.status + " " + obj.response + {"

Guru Meditation: