From phk at varnish-cache.org Mon May 3 09:49:32 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 3 May 2010 11:49:32 +0200 Subject: r4753 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-03 11:49:32 +0200 (Mon, 03 May 2010) New Revision: 4753 Modified: trunk/varnish-cache/bin/varnishtest/vtc.c trunk/varnish-cache/bin/varnishtest/vtc.h trunk/varnish-cache/bin/varnishtest/vtc_client.c trunk/varnish-cache/bin/varnishtest/vtc_http.c trunk/varnish-cache/bin/varnishtest/vtc_log.c trunk/varnish-cache/bin/varnishtest/vtc_server.c Log: Add a '-L' flag for running varnishtest in "Learner mode", this will be used for the examples in the tutorial. The argument is the base port TCP number for the examples. Servers listen on ports starting 10 above the base. Server -listen oand client -connect arguments are numeric offsets to the base TCP port number. Polish the log-message output a bit, both with and without -L. Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2010-04-30 08:16:11 UTC (rev 4752) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-03 09:49:32 UTC (rev 4753) @@ -64,6 +64,8 @@ static pthread_mutex_t vtc_mtx; static pthread_cond_t vtc_cond; +int vtc_learn = 0; + /********************************************************************** * Macro facility */ @@ -117,9 +119,11 @@ va_end(ap); m->val = strdup(buf2); AN(m->val); - vtc_log(vl, 4, "macro def %s=%s", name, m->val); + if (!vtc_learn) + vtc_log(vl, 4, "macro def %s=%s", name, m->val); } else if (m != NULL) { - vtc_log(vl, 4, "macro undef %s", name); + if (!vtc_learn) + vtc_log(vl, 4, "macro undef %s", name); VTAILQ_REMOVE(¯o_list, m, list); free(m->name); free(m->val); @@ -305,8 +309,11 @@ for (cp = cmd; cp->name != NULL; cp++) if (!strcmp(token_s[0], cp->name)) break; - if (cp->name == NULL) + if (cp->name == NULL) { vtc_log(vl, 0, "Unknown command: \"%s\"", token_s[0]); + return; + } + vtc_log(vl, 3, "%s", token_s[0]); assert(cp->cmd != NULL); cp->cmd(token_s, priv, cmd, vl); @@ -618,8 +625,19 @@ vtc_loginit(); vltop = vtc_logopen("top"); AN(vltop); - while ((ch = getopt(argc, argv, "n:qt:v")) != -1) { + while ((ch = getopt(argc, argv, "L:n:qt:v")) != -1) { switch (ch) { + case 'L': + /* XXX: append "/tutorial" to default search path */ + vtc_learn = strtoul(optarg, NULL, 0); + if (vtc_learn > 65000) { + fprintf(stderr, + "-L argument must be 1...65000\n"); + exit(1); + } + vtc_verbosity += 2; + dur = 60 * 3; /* seconds */ + break; case 'n': ntest = strtoul(optarg, NULL, 0); break; Modified: trunk/varnish-cache/bin/varnishtest/vtc.h =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.h 2010-04-30 08:16:11 UTC (rev 4752) +++ trunk/varnish-cache/bin/varnishtest/vtc.h 2010-05-03 09:49:32 UTC (rev 4753) @@ -54,6 +54,7 @@ cmd_f cmd_varnish; cmd_f cmd_sema; +extern int vtc_learn; /* Non-zero in Learners mode */ extern int vtc_verbosity; extern int vtc_error; /* Error, bail out */ extern int vtc_stop; /* Abandon current test, no error */ Modified: trunk/varnish-cache/bin/varnishtest/vtc_client.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_client.c 2010-04-30 08:16:11 UTC (rev 4752) +++ trunk/varnish-cache/bin/varnishtest/vtc_client.c 2010-05-03 09:49:32 UTC (rev 4753) @@ -58,7 +58,7 @@ char *spec; - char *connect; + char connect[256]; unsigned repeat; @@ -84,7 +84,7 @@ char *p; CAST_OBJ_NOTNULL(c, priv, CLIENT_MAGIC); - AN(c->connect); + AN(*c->connect); p = strdup(c->connect); vsb = macro_expand(p); @@ -106,9 +106,9 @@ if (fd < 0) vtc_log(c->vl, 0, "Failed to open %s", vsb_data(vsb)); assert(fd >= 0); - vtc_log(vl, 3, "Connected to %s fd is %d", vsb_data(vsb), fd); + vtc_log(vl, 3, "connected fd %d", fd); http_process(vl, c->spec, fd, -1); - vtc_log(vl, 3, "Closing fd %d", fd); + vtc_log(vl, 3, "closing fd %d", fd); TCP_close(&fd); } vtc_log(vl, 2, "Ending"); @@ -135,7 +135,10 @@ if (*c->name != 'c') vtc_log(c->vl, 0, "Client name must start with 'c'"); - REPLACE(c->connect, "${v1_sock}"); + if (vtc_learn) + bprintf(c->connect, "127.0.0.1:%d", vtc_learn); + else + bprintf(c->connect, "%s", "${v1_sock}"); VTAILQ_INSERT_TAIL(&clients, c, list); return (c); } @@ -152,7 +155,6 @@ vtc_logclose(c->vl); free(c->spec); free(c->name); - free(c->connect); /* XXX: MEMLEAK (?)*/ FREE_OBJ(c); } @@ -180,7 +182,8 @@ void *res; CHECK_OBJ_NOTNULL(c, CLIENT_MAGIC); - vtc_log(c->vl, 2, "Waiting for client"); + if (!vtc_learn) + vtc_log(c->vl, 2, "Waiting for client"); AZ(pthread_join(c->tp, &res)); if (res != NULL) vtc_log(c->vl, 0, "Client returned \"%s\"", (char *)res); @@ -238,7 +241,11 @@ if (vtc_error) break; if (!strcmp(*av, "-connect")) { - REPLACE(c->connect, av[1]); + if (vtc_learn) + bprintf(c->connect, "127.0.0.1:%d", + vtc_learn + atoi(av[1])); + else + bprintf(c->connect, "%s", av[1]); av++; continue; } Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_http.c 2010-04-30 08:16:11 UTC (rev 4752) +++ trunk/varnish-cache/bin/varnishtest/vtc_http.c 2010-05-03 09:49:32 UTC (rev 4753) @@ -483,7 +483,6 @@ else vtc_log(hp->vl, 0, "Unknown http rxresp spec: %s\n", *av); - vtc_log(hp->vl, 3, "rxresp"); http_rxhdr(hp); http_splitheader(hp, 0); if (!has_obj) @@ -596,7 +595,6 @@ for(; *av != NULL; av++) vtc_log(hp->vl, 0, "Unknown http rxreq spec: %s\n", *av); - vtc_log(hp->vl, 3, "rxreq"); http_rxhdr(hp); http_splitheader(hp, 1); http_swallow_body(hp, hp->req, 0); @@ -617,7 +615,6 @@ for(; *av != NULL; av++) vtc_log(hp->vl, 0, "Unknown http rxreq spec: %s\n", *av); - vtc_log(hp->vl, 3, "rxhdrs"); http_rxhdr(hp); http_splitheader(hp, 1); } @@ -636,7 +633,6 @@ for(; *av != NULL; av++) vtc_log(hp->vl, 0, "Unknown http rxreq spec: %s\n", *av); - vtc_log(hp->vl, 3, "rxbody"); http_swallow_body(hp, hp->req, 0); vtc_log(hp->vl, 4, "bodylen = %s", hp->bodylen); } @@ -887,7 +883,10 @@ ALLOC_OBJ(hp, HTTP_MAGIC); AN(hp); hp->fd = sock; - hp->timeout = 3000; + if (vtc_learn) + hp->timeout = 120 * 1000; + else + hp->timeout = 3000; hp->nrxbuf = 640*1024; hp->vsb = vsb_newauto(); hp->rxbuf = malloc(hp->nrxbuf); /* XXX */ Modified: trunk/varnish-cache/bin/varnishtest/vtc_log.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_log.c 2010-04-30 08:16:11 UTC (rev 4752) +++ trunk/varnish-cache/bin/varnishtest/vtc_log.c 2010-05-03 09:49:32 UTC (rev 4753) @@ -113,10 +113,10 @@ static const char * const lead[] = { "----", - "# ", - "## ", - "### ", - "####" + "* ", + "** ", + "*** ", + "****" }; #define NLEAD (sizeof(lead)/sizeof(lead[0])) Modified: trunk/varnish-cache/bin/varnishtest/vtc_server.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_server.c 2010-04-30 08:16:11 UTC (rev 4752) +++ trunk/varnish-cache/bin/varnishtest/vtc_server.c 2010-05-03 09:49:32 UTC (rev 4753) @@ -63,7 +63,7 @@ int depth; int sock; - char *listen; + char listen[256]; struct vss_addr **vss_addr; char *addr; char *port; @@ -104,6 +104,7 @@ fd = accept(s->sock, addr, &l); if (fd < 0) vtc_log(vl, 0, "Accepted failed: %s", strerror(errno)); + vtc_log(vl, 3, "accepted fd %d", fd); http_process(vl, s->spec, fd, s->sock); vtc_log(vl, 3, "shutting fd %d", fd); assert((shutdown(fd, SHUT_WR) == 0) @@ -135,7 +136,10 @@ if (*s->name != 's') vtc_log(s->vl, 0, "Server name must start with 's'"); - REPLACE(s->listen, "127.0.0.1:0"); + if (vtc_learn) + bprintf(s->listen, "127.0.0.1:%d", vtc_learn + 10); + else + bprintf(s->listen, "127.0.0.1:%d", 0); AZ(VSS_parse(s->listen, &s->addr, &s->port)); s->repeat = 1; s->depth = 1; @@ -154,7 +158,6 @@ CHECK_OBJ_NOTNULL(s, SERVER_MAGIC); vtc_logclose(s->vl); - free(s->listen); free(s->name); /* XXX: MEMLEAK (?) (VSS ??) */ FREE_OBJ(s); @@ -170,7 +173,8 @@ int naddr; CHECK_OBJ_NOTNULL(s, SERVER_MAGIC); - vtc_log(s->vl, 2, "Starting server"); + if (!vtc_learn) + vtc_log(s->vl, 2, "Starting server"); if (s->sock < 0) { naddr = VSS_resolve(s->addr, s->port, &s->vss_addr); if (naddr != 1) @@ -189,7 +193,8 @@ if (!strcmp(s->port, "0")) REPLACE(s->port, s->aport); } - vtc_log(s->vl, 1, "Listen on %s:%s", s->addr, s->port); + if (!vtc_learn) + vtc_log(s->vl, 1, "Listen on %s:%s", s->addr, s->port); s->run = 1; AZ(pthread_create(&s->tp, NULL, server_thread, s)); } @@ -204,7 +209,8 @@ void *res; CHECK_OBJ_NOTNULL(s, SERVER_MAGIC); - vtc_log(s->vl, 2, "Waiting for server"); + if (!vtc_learn) + vtc_log(s->vl, 2, "Waiting for server"); AZ(pthread_join(s->tp, &res)); if (res != NULL && !vtc_stop) vtc_log(s->vl, 0, "Server returned \"%p\"", @@ -293,7 +299,11 @@ continue; } if (!strcmp(*av, "-listen")) { - REPLACE(s->listen, av[1]); + if (vtc_learn) + bprintf(s->listen, "127.0.0.1:%d", + vtc_learn + 10 + atoi(av[1])); + else + bprintf(s->listen, "%s", av[1]); AZ(VSS_parse(s->listen, &s->addr, &s->port)); av++; continue; From phk at varnish-cache.org Mon May 3 09:50:15 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 3 May 2010 11:50:15 +0200 Subject: r4754 - in trunk/varnish-cache/doc/sphinx/tutorial: . vtc_files Message-ID: Author: phk Date: 2010-05-03 11:50:13 +0200 (Mon, 03 May 2010) New Revision: 4754 Added: trunk/varnish-cache/doc/sphinx/tutorial/intro.rst trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/ trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C001.vtc trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S001.vtc Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst Log: Add the first tutorial example, based on the new -L flag to varnishtest. Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-03 09:49:32 UTC (rev 4753) +++ trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-03 09:50:13 UTC (rev 4754) @@ -7,33 +7,11 @@ Welcome to the Varnish Tutorial, we hope this will help you get to know and understand Varnish. -Most tutorials are written in "subject-order", as the old Peanuts -strip goes:: +.. toctree:: - Jogging: A Handbook - Author: S. Noopy - Chapter 1: Left foot - It was a dark and stormy night... + intro.rst + tut001.rst -This is great when the reader has no choice, or nothing better to do, but -read the entire document before starting. - -We have taken the other approach: "breadth-first", because experience -has shown us that Varnish users wants to get things running, and then -polish up things later on. - -With that in mind, we have written the tutorial so you can break off, -as Calvin tells Ms. Wormwood, "when my brain is full for today", and -come back later and learn more. - -That also means that right from the start, we will have several -things going on in parallel and you will need at least four, sometimes -more, terminal windows at the same time, to run the examples. - - -//todo// First simple example (pending varnishtest support) - - .. todo:: starting varnish with -d, seeing a transaction go through explain varnishlog output for a miss and a hit Added: trunk/varnish-cache/doc/sphinx/tutorial/intro.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/intro.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/intro.rst 2010-05-03 09:50:13 UTC (rev 4754) @@ -0,0 +1,51 @@ +%%%%%%%%%%%% +Introduction +%%%%%%%%%%%% + +Most tutorials are written in "subject-order", as the old Peanuts +strip goes:: + + Jogging: A Handbook + Author: S. Noopy + Chapter 1: Left foot + It was a dark and stormy night... + +This is great when the reader has no choice, and nothing better to +do, but read the entire document before starting. + +We have taken the other approach: "breadth-first", because experience +has shown us that Varnish users wants to get things running, and then +polish up things later on. + +With that in mind, we have written the tutorial so you can break off, +as Calvin tells Ms. Wormwood, "when my brain is full for today", and +come back later and learn more. + +That also means that right from the start, we will have several +things going on in parallel and you will need at least four, sometimes +more, terminal windows at the same time, to run the examples. + +A word about TCP ports +---------------------- + +We have subverted our custom built regression test tool, a program +called ```varnishtest``` to help you get through these examples, +without having to waste too much time setting up webservers as +backends or browsers as clients, to drive the examples. + +But there is one complication we can not escape: TCP port numbers. + +Each of the backends we simulate and the varnishd instances we run +needs a TCP port number to listen to and it is your job to find them, +because we have no idea what servers are running on your computer +nor what TCP ports they use. + +To make this as easy as possible, we have implemented a ```-L +number``` argument to all the varnish programs, which puts them in +"Learner" mode, and in all the examples we have used 20000 as +the number, because on most systems the middle of the range +(1000...65000) is usually not used. + +If these ports are in use on your system (is your colleague also running +the Varnish tutorial ?) simply pick another number and use that +instead of 20000. Added: trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst 2010-05-03 09:50:13 UTC (rev 4754) @@ -0,0 +1,78 @@ +.. _TUT001: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +TUT001: Let's see if varnishtest is working +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +In the first window, start this command:: + + varnishtest -L20000 S001.vtc + +Then in the second window, run this one:: + + varnishtest -L20000 C001.vtc + +If things work as they should, both programs spit out 5 lines, looking +like this in the first window:: + + * top TEST S001.vtc starting + ** s1 Started on 127.0.0.1:20012 + ** s1 Ending + * top RESETTING after S001.vtc + * top TEST S001.vtc completed + +and like this in the second window:: + + * top TEST C001.vtc starting + ** c1 Starting client + ** c1 Ending + * top RESETTING after C001.vtc + * top TEST C001.vtc completed + +If that did not work, please consult the XXX: troubleshooting section. + +Now, try again, but this time run the client with a couple of ``-v`` +arguments:: + + varnishtest -vv -L20000 C001.vtc + +Now the output contains a lot mode detail:: + + * top TEST C001.vtc starting + *** top client + ** c1 Starting client + *** c1 Connect to 127.0.0.1:20012 + *** c1 connected fd 3 + *** c1 txreq + **** c1 txreq| GET / HTTP/1.1\r\n + **** c1 txreq| \r\n + *** c1 rxresp + **** c1 rxhdr| HTTP/1.1 200 Ok\r\n + **** c1 rxhdr| Foo: bar\r\n + **** c1 rxhdr| Content-Length: 12\r\n + **** c1 rxhdr| \r\n + **** c1 http[ 0] | HTTP/1.1 + **** c1 http[ 1] | 200 + **** c1 http[ 2] | Ok + **** c1 http[ 3] | Foo: bar + **** c1 http[ 4] | Content-Length: 12 + **** c1 body| Hello World! + **** c1 bodylen = 12 + *** c1 closing fd 3 + ** c1 Ending + * top RESETTING after C001.vtc + * top TEST C001.vtc completed + +First the client does a ``txreq`` -- "transmit request", and you can +see the HTTP request it sends to the server, a plain boring "GET". + +Then it does a ``rxresp`` -- "receive response", and we can see the +HTTP response we get back, including the HTTP object body. + +Now try again, this time running the server in the first window with +``-vv``. + +If things do not work the way you expect, adding those ``-v`` options +is a good place to start debugging. + +Next we put a Varnish cache between the server and the client: _TUT002 Added: trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C001.vtc =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C001.vtc (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C001.vtc 2010-05-03 09:50:13 UTC (rev 4754) @@ -0,0 +1,6 @@ +# $Id$ + +client c1 -connect 12 { + txreq + rxresp +} -run Added: trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S001.vtc =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S001.vtc (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S001.vtc 2010-05-03 09:50:13 UTC (rev 4754) @@ -0,0 +1,6 @@ +# $Id$ + +server s1 -listen 2 { + rxreq + txresp -hdr "Foo: bar" -body "Hello World!" +} -start -wait From phk at varnish-cache.org Mon May 3 11:49:15 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 3 May 2010 13:49:15 +0200 Subject: r4755 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-03 13:49:15 +0200 (Mon, 03 May 2010) New Revision: 4755 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Syslog the platform we are running on. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-03 09:50:13 UTC (rev 4754) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-03 11:49:15 UTC (rev 4755) @@ -680,7 +680,8 @@ fprintf(stderr, "NOTE: Could not write PID file\n"); if (d_flag) - fprintf(stderr, "Varnish on %s\n", vsb_data(vident) + 1); + fprintf(stderr, "Platform: %s\n", vsb_data(vident) + 1); + syslog(LOG_NOTICE, "Platform: %s\n", vsb_data(vident) + 1); /* Do this again after debugstunt and daemon has run */ mgt_pid = getpid(); From phk at varnish-cache.org Tue May 4 13:55:35 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 4 May 2010 15:55:35 +0200 Subject: r4756 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-04 15:55:35 +0200 (Tue, 04 May 2010) New Revision: 4756 Modified: trunk/varnish-cache/bin/varnishtest/vtc_log.c Log: Only dump up to 512 bytes. Modified: trunk/varnish-cache/bin/varnishtest/vtc_log.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_log.c 2010-05-03 11:49:15 UTC (rev 4755) +++ trunk/varnish-cache/bin/varnishtest/vtc_log.c 2010-05-04 13:55:35 UTC (rev 4756) @@ -172,6 +172,7 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str) { int nl = 1; + unsigned l; CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC); assert(lvl < NLEAD); @@ -183,7 +184,12 @@ vsb_printf(vl->vsb, "%s %-4s %s(null)\n", lead[lvl], vl->id, pfx); else + l = 0; for(; *str != '\0'; str++) { + if (++l > 512) { + vsb_printf(vl->vsb, "..."); + break; + } if (nl) { vsb_printf(vl->vsb, "%s %-4s %s| ", lead[lvl], vl->id, pfx); From phk at varnish-cache.org Tue May 4 13:56:00 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 4 May 2010 15:56:00 +0200 Subject: r4757 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-04 15:56:00 +0200 (Tue, 04 May 2010) New Revision: 4757 Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c Log: Support for sending chunked synth bodie Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_http.c 2010-05-04 13:55:35 UTC (rev 4756) +++ trunk/varnish-cache/bin/varnishtest/vtc_http.c 2010-05-04 13:56:00 UTC (rev 4757) @@ -744,6 +744,37 @@ http_write(hp, 4, "chunked"); } +static void +cmd_http_chunkedlen(CMD_ARGS) +{ + unsigned len; + unsigned u, v; + char buf[16384]; + struct http *hp; + + (void)cmd; + (void)vl; + CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); + AN(av[1]); + AZ(av[2]); + vsb_clear(hp->vsb); + + len = atoi(av[1]); + + for (u = 0; u < sizeof buf; u++) + buf[u] = (u & 7) + '0'; + + for (u = 0; u < len; u += 16384) { + v = len - u; + if (v > sizeof buf) + v = sizeof buf; + vsb_printf(hp->vsb, "%x%s", v, nl); + vsb_printf(hp->vsb, "%*.*s%s", v, v, buf, nl); + } + vsb_printf(hp->vsb, "%x%s", 0, nl); + http_write(hp, 4, "chunked"); +} + /********************************************************************** * set the timeout */ @@ -865,6 +896,7 @@ { "expect", cmd_http_expect }, { "send", cmd_http_send }, { "chunked", cmd_http_chunked }, + { "chunkedlen", cmd_http_chunkedlen }, { "delay", cmd_delay }, { "sema", cmd_sema }, { "expect_close", cmd_http_expect_close }, From phk at varnish-cache.org Tue May 4 14:19:06 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 4 May 2010 16:19:06 +0200 Subject: r4758 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: Author: phk Date: 2010-05-04 16:19:05 +0200 (Tue, 04 May 2010) New Revision: 4758 Added: trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_response.c Log: Fix one of those "nothing can possibly go wrong" bugs that are so typical of "lets just try to get this into the release" features. The resent addition of experimental Range: header support, broke a cornercase in normal content delivery. If an object was delivered from the backend using chunked encoding, and was larger than the storage segment size (default: 128k) this bug may bite. The effect of the bug is that up to storage segment worth of junk may be appended to the transmitted object. This is mostly harmless, because the Content-Length header will make the browser do the right thing, but certain load-balancers will go cross-eyed and act really weird at the TCP level, spewing interesting RST packets to the client. This bug is only in 2.1.1, not in 2.1.0. Fixes: #694 Isolated by: ay Fool who did this: phk Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2010-05-04 13:56:00 UTC (rev 4757) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2010-05-04 14:19:05 UTC (rev 4758) @@ -325,7 +325,7 @@ } if (ptr + len > high) /* Chop tail of segment off */ - len = 1 + high - low; + len = 1 + high - ptr; ptr += len; Added: trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc 2010-05-04 14:19:05 UTC (rev 4758) @@ -0,0 +1,24 @@ +# $Id$ + +test "Wrong calculation of last storage segment length" + +server s1 { + rxreq + send "HTTP/1.1 200 Ok\r\n" + send "Transfer-encoding: chunked\r\n" + send "\r\n" + # This is chunksize (128k) + 2 to force to chunks to be allocated + chunkedlen 131074 +} -start + +varnish v1 -vcl+backend { + sub vcl_deliver { + unset resp.http.content-length; + } +} -start + +client c1 { + txreq -proto HTTP/1.0 + rxresp + expect resp.bodylen == 131074 +} -run From phk at varnish-cache.org Tue May 4 21:04:14 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 4 May 2010 23:04:14 +0200 Subject: r4759 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-04 23:04:13 +0200 (Tue, 04 May 2010) New Revision: 4759 Added: trunk/varnish-cache/lib/libvarnish/vin.c Removed: trunk/varnish-cache/bin/varnishd/instance.c trunk/varnish-cache/lib/libvarnishapi/instance.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/shmlog.h trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnish/Makefile.am trunk/varnish-cache/lib/libvarnishapi/Makefile.am trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Cleanup the -n argument to name/dir/shmfilename munging and put it in a single sourcefile, which appears in both libvarnish and libvarnishapi. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2010-05-04 21:04:13 UTC (rev 4759) @@ -42,7 +42,6 @@ hash_classic.c \ hash_critbit.c \ hash_simple_list.c \ - instance.c \ mgt_child.c \ mgt_cli.c \ mgt_param.c \ Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2010-05-04 21:04:13 UTC (rev 4759) @@ -62,7 +62,7 @@ /* Hash method */ struct hash_slinger *hash; - char name[1024]; + char *name; char identity[1024]; }; @@ -211,6 +211,3 @@ extern struct heritage heritage; void child_main(void); - -int varnish_instance(const char *n_arg, char *name, size_t namelen, - char *dir, size_t dirlen); Deleted: trunk/varnish-cache/bin/varnishd/instance.c =================================================================== --- trunk/varnish-cache/bin/varnishd/instance.c 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/bin/varnishd/instance.c 2010-05-04 21:04:13 UTC (rev 4759) @@ -1 +0,0 @@ -link ../../lib/libvarnishapi/instance.c \ No newline at end of file Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-04 21:04:13 UTC (rev 4759) @@ -422,7 +422,7 @@ char *p, *vcl = NULL; struct cli cli[1]; struct pidfh *pfh = NULL; - char dirname[1024]; + char *dirname; /* * Start out by closing all unwanted file descriptors we might @@ -615,8 +615,7 @@ } } - if (varnish_instance(n_arg, heritage.name, sizeof heritage.name, - dirname, sizeof dirname) != 0) { + if (vin_n_arg(n_arg, &heritage.name, &dirname, NULL) != 0) { fprintf(stderr, "Invalid instance name: %s\n", strerror(errno)); exit(1); Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/include/shmlog.h 2010-05-04 21:04:13 UTC (rev 4759) @@ -109,4 +109,7 @@ SLT_WRAPMARKER = 255 }; +/* This function lives in both libvarnish and libvarnishapi */ +int vin_n_arg(const char *n_arg, char **name, char **dir, char **vsl); + #endif Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-04 21:04:13 UTC (rev 4759) @@ -60,7 +60,4 @@ const char *VSL_Name(void); extern const char *VSL_tags[256]; -/* instance.c */ -int varnish_instance(const char *n_arg, char *name, size_t namelen, char *dir, - size_t dirlen); #endif Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2010-05-04 21:04:13 UTC (rev 4759) @@ -22,6 +22,7 @@ vct.c \ version.c \ vev.c \ + vin.c \ vlu.c \ vpf.c \ vre.c \ @@ -30,6 +31,7 @@ vss.c \ vtmpfile.c +libvarnish_la_CFLAGS = -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' libvarnish_la_LIBADD = ${RT_LIBS} ${NET_LIBS} ${LIBM} @PCRE_LIBS@ DISTCLEANFILES = svn_version.c Added: trunk/varnish-cache/lib/libvarnish/vin.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vin.c (rev 0) +++ trunk/varnish-cache/lib/libvarnish/vin.c 2010-05-04 21:04:13 UTC (rev 4759) @@ -0,0 +1,102 @@ +/*- + * Copyright (c) 2007-2009 Linpro AS + * All rights reserved. + * + * Author: Dag-Erling Sm?rgrav + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * XXX: NB: also used in libvarnishapi + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id: instance.c 4093 2009-06-06 15:56:17Z des $") + +#include +#include +#include +#include +#include + +#include "libvarnish.h" +#include "shmlog.h" + +int +vin_n_arg(const char *n_arg, char **name, char **dir, char **vsl) +{ + char nm[PATH_MAX]; + char dn[PATH_MAX]; + + + /* First: determine the name */ + + if (n_arg == NULL || *n_arg == '\0') { + if (gethostname(nm, sizeof nm) != 0) + return (-1); + } else if (strlen(n_arg) >= sizeof nm) { + /* preliminary length check to avoid overflowing nm */ + errno = ENAMETOOLONG; + return (-1); + } else + bprintf(nm, "%s", n_arg); + + + /* Second: find the directory name */ + + if (*nm == '/') + strcpy(dn, nm); + else if (strlen(VARNISH_STATE_DIR) + 1 + strlen(nm) >= sizeof dn){ + /* preliminary length check to avoid overflowing dm */ + errno = ENAMETOOLONG; + return (-1); + } else { + bprintf(dn, "%s/%s", VARNISH_STATE_DIR, nm); + } + + /* Definitive length check */ + if (strlen(dn) + 1 + strlen(SHMLOG_FILENAME) >= sizeof dn) { + errno = ENAMETOOLONG; + return (-1); + } + + strcat(dn, "/"); + + if (name != NULL) { + *name = strdup(nm); + if (*name == NULL) + return (-1); + } + if (dir != NULL) { + *dir = strdup(dn); + if (*dir == NULL) + return (-1); + } + if (vsl != NULL) { + bprintf(nm, "%s%s", dn, SHMLOG_FILENAME); + *vsl = strdup(nm); + if (*vsl == NULL) + return (-1); + } + return (0); +} Modified: trunk/varnish-cache/lib/libvarnishapi/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2010-05-04 21:04:13 UTC (rev 4759) @@ -7,8 +7,8 @@ libvarnishapi_la_LDFLAGS = -version-info 1:0:0 libvarnishapi_la_SOURCES = \ + ../libvarnish/vin.c \ base64.c \ - instance.c \ shmlog.c libvarnishapi_la_CFLAGS = \ Deleted: trunk/varnish-cache/lib/libvarnishapi/instance.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/instance.c 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/lib/libvarnishapi/instance.c 2010-05-04 21:04:13 UTC (rev 4759) @@ -1,67 +0,0 @@ -/*- - * Copyright (c) 2007-2009 Linpro AS - * All rights reserved. - * - * Author: Dag-Erling Sm?rgrav - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include "svnid.h" -SVNID("$Id$") - -#include -#include -#include - -#include "varnishapi.h" - -int -varnish_instance(const char *n_arg, - char *name, size_t namelen, char *dir, size_t dirlen) -{ - size_t len; - - if (n_arg == NULL) { - if (gethostname(name, namelen) != 0) - return (-1); - } else { - len = snprintf(name, namelen, "%s", n_arg); - if (len >= namelen) { - errno = ENAMETOOLONG; - return (-1); - } - } - - if (*name == '/') - len = snprintf(dir, dirlen, "%s", name); - else - len = snprintf(dir, dirlen, "%s/%s", VARNISH_STATE_DIR, name); - - if (len >= dirlen) { - errno = ENAMETOOLONG; - return (-1); - } - return (0); -} Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-04 14:19:05 UTC (rev 4758) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-04 21:04:13 UTC (rev 4759) @@ -127,21 +127,17 @@ { int i; struct shmloghead slh; - char dirname[PATH_MAX], logname[PATH_MAX]; + char *logname; if (vsl_lh != NULL) return (0); - if (varnish_instance(varnish_name, vsl_name, - sizeof vsl_name, dirname, sizeof dirname) != 0) { + if (vin_n_arg(varnish_name, NULL, NULL, &logname)) { fprintf(stderr, "Invalid instance name: %s\n", strerror(errno)); return (1); } - /* XXX check overflow */ - snprintf(logname, sizeof logname, "%s/%s", dirname, SHMLOG_FILENAME); - vsl_fd = open(logname, O_RDONLY); if (vsl_fd < 0) { fprintf(stderr, "Cannot open %s: %s\n", From tfheen at varnish-cache.org Wed May 5 07:18:04 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 09:18:04 +0200 Subject: r4760 - in branches/2.1: . varnish-cache/bin/varnishd varnish-cache/bin/varnishtest/tests varnish-cache/include varnish-cache/lib/libvarnish varnish-cache/lib/libvcl Message-ID: Author: tfheen Date: 2010-05-05 09:18:04 +0200 (Wed, 05 May 2010) New Revision: 4760 Modified: branches/2.1/ branches/2.1/varnish-cache/bin/varnishd/cache_backend.h branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c branches/2.1/varnish-cache/bin/varnishd/vparam.h branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00002.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc branches/2.1/varnish-cache/include/vct.h branches/2.1/varnish-cache/include/vev.h branches/2.1/varnish-cache/lib/libvarnish/tcp.c branches/2.1/varnish-cache/lib/libvarnish/vev.c branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c Log: Merge r4731: Don't rely on CNN's CDN architecture, use a FQDN that we control. Property changes on: branches/2.1 ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/bin/varnishd/vparam.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/v00002.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/v00002.vtc 2010-05-04 21:04:13 UTC (rev 4759) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/v00002.vtc 2010-05-05 07:18:04 UTC (rev 4760) @@ -114,7 +114,7 @@ varnish v1 -badvcl { /* too many IP numbers */ - backend b1 { .host = "cnn.com"; } + backend b1 { .host = "v00002.freebsd.dk"; } } varnish v1 -badvcl { Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/include/vct.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/include/vev.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/tcp.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/vev.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 Property changes on: branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719 + /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 From tfheen at varnish-cache.org Wed May 5 07:40:15 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 09:40:15 +0200 Subject: r4761 - in branches/2.1: . varnish-cache/bin/varnishd varnish-cache/bin/varnishtest varnish-cache/bin/varnishtest/tests varnish-cache/include varnish-cache/lib/libvarnish varnish-cache/lib/libvcl Message-ID: Author: tfheen Date: 2010-05-05 09:40:13 +0200 (Wed, 05 May 2010) New Revision: 4761 Modified: branches/2.1/ branches/2.1/varnish-cache/bin/varnishd/cache_backend.h branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c branches/2.1/varnish-cache/bin/varnishd/vparam.h branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc branches/2.1/varnish-cache/bin/varnishtest/vtc_http.c branches/2.1/varnish-cache/include/vct.h branches/2.1/varnish-cache/include/vev.h branches/2.1/varnish-cache/lib/libvarnish/tcp.c branches/2.1/varnish-cache/lib/libvarnish/vev.c branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c Log: Merge r4757: Support for sending chunked synth bodie Property changes on: branches/2.1 ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishd/vparam.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Modified: branches/2.1/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/vtc_http.c 2010-05-05 07:18:04 UTC (rev 4760) +++ branches/2.1/varnish-cache/bin/varnishtest/vtc_http.c 2010-05-05 07:40:13 UTC (rev 4761) @@ -747,6 +747,37 @@ http_write(hp, 4, "chunked"); } +static void +cmd_http_chunkedlen(CMD_ARGS) +{ + unsigned len; + unsigned u, v; + char buf[16384]; + struct http *hp; + + (void)cmd; + (void)vl; + CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); + AN(av[1]); + AZ(av[2]); + vsb_clear(hp->vsb); + + len = atoi(av[1]); + + for (u = 0; u < sizeof buf; u++) + buf[u] = (u & 7) + '0'; + + for (u = 0; u < len; u += 16384) { + v = len - u; + if (v > sizeof buf) + v = sizeof buf; + vsb_printf(hp->vsb, "%x%s", v, nl); + vsb_printf(hp->vsb, "%*.*s%s", v, v, buf, nl); + } + vsb_printf(hp->vsb, "%x%s", 0, nl); + http_write(hp, 4, "chunked"); +} + /********************************************************************** * set the timeout */ @@ -868,6 +899,7 @@ { "expect", cmd_http_expect }, { "send", cmd_http_send }, { "chunked", cmd_http_chunked }, + { "chunkedlen", cmd_http_chunkedlen }, { "delay", cmd_delay }, { "sema", cmd_sema }, { "expect_close", cmd_http_expect_close }, Property changes on: branches/2.1/varnish-cache/include/vct.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/include/vev.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/tcp.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/vev.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 Property changes on: branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731 + /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 From phk at varnish-cache.org Wed May 5 07:48:10 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 5 May 2010 09:48:10 +0200 Subject: r4762 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-05 09:48:10 +0200 (Wed, 05 May 2010) New Revision: 4762 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c Log: Assert that we have an object, in case people use inline-C to access obj.http.* Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-05 07:40:13 UTC (rev 4761) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-05 07:48:10 UTC (rev 4762) @@ -115,6 +115,7 @@ hp = sp->wrk->resp; break; case HDR_OBJ: + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); hp = sp->obj->http; break; default: From tfheen at varnish-cache.org Wed May 5 07:48:19 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 09:48:19 +0200 Subject: r4763 - in branches/2.1: . varnish-cache/bin/varnishd varnish-cache/bin/varnishtest/tests varnish-cache/include varnish-cache/lib/libvarnish varnish-cache/lib/libvcl Message-ID: Author: tfheen Date: 2010-05-05 09:48:18 +0200 (Wed, 05 May 2010) New Revision: 4763 Added: branches/2.1/varnish-cache/bin/varnishtest/tests/r00694.vtc Modified: branches/2.1/ branches/2.1/varnish-cache/bin/varnishd/cache_backend.h branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c branches/2.1/varnish-cache/bin/varnishd/cache_response.c branches/2.1/varnish-cache/bin/varnishd/vparam.h branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc branches/2.1/varnish-cache/include/vct.h branches/2.1/varnish-cache/include/vev.h branches/2.1/varnish-cache/lib/libvarnish/tcp.c branches/2.1/varnish-cache/lib/libvarnish/vev.c branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c Log: Merge r4758: Fix trailing garbage for large objects Fix one of those "nothing can possibly go wrong" bugs that are so typical of "lets just try to get this into the release" features. The resent addition of experimental Range: header support, broke a cornercase in normal content delivery. If an object was delivered from the backend using chunked encoding, and was larger than the storage segment size (default: 128k) this bug may bite. The effect of the bug is that up to storage segment worth of junk may be appended to the transmitted object. This is mostly harmless, because the Content-Length header will make the browser do the right thing, but certain load-balancers will go cross-eyed and act really weird at the TCP level, spewing interesting RST packets to the client. This bug is only in 2.1.1, not in 2.1.0. Fixes: #694 Isolated by: ay Fool who did this: phk Property changes on: branches/2.1 ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Modified: branches/2.1/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- branches/2.1/varnish-cache/bin/varnishd/cache_response.c 2010-05-05 07:48:10 UTC (rev 4762) +++ branches/2.1/varnish-cache/bin/varnishd/cache_response.c 2010-05-05 07:48:18 UTC (rev 4763) @@ -325,7 +325,7 @@ } if (ptr + len > high) /* Chop tail of segment off */ - len = 1 + high - low; + len = 1 + high - ptr; ptr += len; Property changes on: branches/2.1/varnish-cache/bin/varnishd/vparam.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Copied: branches/2.1/varnish-cache/bin/varnishtest/tests/r00694.vtc (from rev 4758, trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc) =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/r00694.vtc (rev 0) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/r00694.vtc 2010-05-05 07:48:18 UTC (rev 4763) @@ -0,0 +1,24 @@ +# $Id$ + +test "Wrong calculation of last storage segment length" + +server s1 { + rxreq + send "HTTP/1.1 200 Ok\r\n" + send "Transfer-encoding: chunked\r\n" + send "\r\n" + # This is chunksize (128k) + 2 to force to chunks to be allocated + chunkedlen 131074 +} -start + +varnish v1 -vcl+backend { + sub vcl_deliver { + unset resp.http.content-length; + } +} -start + +client c1 { + txreq -proto HTTP/1.0 + rxresp + expect resp.bodylen == 131074 +} -run Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/include/vct.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/include/vev.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/tcp.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/vev.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 Property changes on: branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757 + /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 From tfheen at varnish-cache.org Wed May 5 08:02:47 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 10:02:47 +0200 Subject: r4764 - in branches/2.1: . varnish-cache/bin/varnishd varnish-cache/bin/varnishtest/tests varnish-cache/include varnish-cache/lib/libvarnish varnish-cache/lib/libvcl Message-ID: Author: tfheen Date: 2010-05-05 10:02:47 +0200 (Wed, 05 May 2010) New Revision: 4764 Modified: branches/2.1/ branches/2.1/varnish-cache/bin/varnishd/cache_backend.h branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c branches/2.1/varnish-cache/bin/varnishd/cache_vrt.c branches/2.1/varnish-cache/bin/varnishd/vparam.h branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc branches/2.1/varnish-cache/include/vct.h branches/2.1/varnish-cache/include/vev.h branches/2.1/varnish-cache/lib/libvarnish/tcp.c branches/2.1/varnish-cache/lib/libvarnish/vev.c branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c Log: Merge r4762: Assert that we have an object, in case people use inline-C to access obj.http.* Property changes on: branches/2.1 ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Modified: branches/2.1/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- branches/2.1/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-05 07:48:18 UTC (rev 4763) +++ branches/2.1/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-05 08:02:47 UTC (rev 4764) @@ -115,6 +115,7 @@ hp = sp->wrk->resp; break; case HDR_OBJ: + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); hp = sp->obj->http; break; default: Property changes on: branches/2.1/varnish-cache/bin/varnishd/vparam.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/include/vct.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/include/vev.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/tcp.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/vev.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758 + /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 From phk at varnish-cache.org Wed May 5 08:08:34 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 5 May 2010 10:08:34 +0200 Subject: r4765 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-05 10:08:34 +0200 (Wed, 05 May 2010) New Revision: 4765 Modified: trunk/varnish-cache/bin/varnishtest/vtc_varnish.c Log: Don't assert varnisttest if vcl fails to compile, fail normally. Modified: trunk/varnish-cache/bin/varnishtest/vtc_varnish.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-05 08:02:47 UTC (rev 4764) +++ trunk/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-05 08:08:34 UTC (rev 4765) @@ -574,9 +574,10 @@ AZ(vsb_overflowed(vsb)); u = varnish_ask_cli(v, vsb_data(vsb), NULL); - if (u != CLIS_OK) + if (u != CLIS_OK) { vtc_log(v->vl, 0, "FAIL VCL does not compile"); - assert(u == CLIS_OK); + return; + } vsb_clear(vsb); vsb_printf(vsb, "vcl.use vcl%d", v->vcl_nbr); vsb_finish(vsb); From tfheen at varnish-cache.org Wed May 5 08:15:50 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 10:15:50 +0200 Subject: r4766 - branches/2.1/varnish-cache/doc Message-ID: Author: tfheen Date: 2010-05-05 10:15:50 +0200 (Wed, 05 May 2010) New Revision: 4766 Added: branches/2.1/varnish-cache/doc/changes-2.1.1-2.1.2.xml branches/2.1/varnish-cache/doc/changes-2.1.2.xml Modified: branches/2.1/varnish-cache/doc/Makefile.am Log: Document 2.1.2 changes Modified: branches/2.1/varnish-cache/doc/Makefile.am =================================================================== --- branches/2.1/varnish-cache/doc/Makefile.am 2010-05-05 08:08:34 UTC (rev 4765) +++ branches/2.1/varnish-cache/doc/Makefile.am 2010-05-05 08:15:50 UTC (rev 4766) @@ -1,6 +1,7 @@ # $Id$ CHANGELOGS = \ + changes-2.1.2.html \ changes-2.1.1.html \ changes-2.1.0.html \ changes-2.0.6.html \ @@ -16,6 +17,7 @@ changes-1.0.4.html XML = \ + changes-2.1.1-2.1.2.xml \ changes-2.1.0-2.1.1.xml \ changes-2.0.6-2.1.0.xml \ changes-2.0.5-2.0.6.xml \ Added: branches/2.1/varnish-cache/doc/changes-2.1.1-2.1.2.xml =================================================================== --- branches/2.1/varnish-cache/doc/changes-2.1.1-2.1.2.xml (rev 0) +++ branches/2.1/varnish-cache/doc/changes-2.1.1-2.1.2.xml 2010-05-05 08:15:50 UTC (rev 4766) @@ -0,0 +1,19 @@ + + +]> + + + + varnishd + + + When adding Range support for 2.1.1, we + accidentially introduced a bug which would append garbage to + objects larger than the chunk size, by default 128k. Browsers + would do the right thing due to Content-Length, but some load + balancers would get very confused. + + + + Added: branches/2.1/varnish-cache/doc/changes-2.1.2.xml =================================================================== --- branches/2.1/varnish-cache/doc/changes-2.1.2.xml (rev 0) +++ branches/2.1/varnish-cache/doc/changes-2.1.2.xml 2010-05-05 08:15:50 UTC (rev 4766) @@ -0,0 +1,12 @@ + + + +]> + + + Varnish + 2.1.2 + + + From phk at varnish-cache.org Wed May 5 08:16:32 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 5 May 2010 10:16:32 +0200 Subject: r4767 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-05 10:16:31 +0200 (Wed, 05 May 2010) New Revision: 4767 Modified: trunk/varnish-cache/bin/varnishtest/vtc.c trunk/varnish-cache/bin/varnishtest/vtc.h trunk/varnish-cache/bin/varnishtest/vtc_client.c Log: Give a sensible message on attempting to expand an undefined macro Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-05 08:15:50 UTC (rev 4766) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-05 08:16:31 UTC (rev 4767) @@ -149,23 +149,23 @@ } struct vsb * -macro_expand(const char *name) +macro_expand(struct vtclog *vl, const char *text) { struct vsb *vsb; - char *p, *q; + char *p, *q, *m; vsb = vsb_newauto(); AN(vsb); - while (*name != '\0') { - p = strstr(name, "${"); + while (*text != '\0') { + p = strstr(text, "${"); if (p == NULL) { - vsb_cat(vsb, name); + vsb_cat(vsb, text); break; } - vsb_bcat(vsb, name, p - name); + vsb_bcat(vsb, text, p - text); q = strchr(p, '}'); if (q == NULL) { - vsb_cat(vsb, name); + vsb_cat(vsb, text); break; } assert(p[0] == '$'); @@ -173,8 +173,13 @@ assert(q[0] == '}'); p += 2; *q = '\0'; - vsb_cat(vsb, macro_get(p)); - name = q + 1; + m = macro_get(p); + if (m == NULL) { + vtc_log(vl, 0, "Macro ${%s} not found", p); + return (NULL); + } + vsb_printf(vsb, "{MACRO NOT FOUND: %s}", p); + text = q + 1; } vsb_finish(vsb); return (vsb); @@ -301,7 +306,7 @@ *token_e[tn] = '\0'; /*lint !e771 */ if (NULL == strstr(token_s[tn], "${")) continue; - token_exp[tn] = macro_expand(token_s[tn]); + token_exp[tn] = macro_expand(vl, token_s[tn]); token_s[tn] = vsb_data(token_exp[tn]); token_e[tn] = strchr(token_s[tn], '\0'); } Modified: trunk/varnish-cache/bin/varnishtest/vtc.h =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.h 2010-05-05 08:15:50 UTC (rev 4766) +++ trunk/varnish-cache/bin/varnishtest/vtc.h 2010-05-05 08:16:31 UTC (rev 4767) @@ -78,4 +78,4 @@ void macro_def(struct vtclog *vl, const char *instance, const char *name, const char *fmt, ...); -struct vsb *macro_expand(const char *name); +struct vsb *macro_expand(struct vtclog *vl, const char *text); Modified: trunk/varnish-cache/bin/varnishtest/vtc_client.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_client.c 2010-05-05 08:15:50 UTC (rev 4766) +++ trunk/varnish-cache/bin/varnishtest/vtc_client.c 2010-05-05 08:16:31 UTC (rev 4767) @@ -86,12 +86,13 @@ CAST_OBJ_NOTNULL(c, priv, CLIENT_MAGIC); AN(*c->connect); + vl = vtc_logopen(c->name); + p = strdup(c->connect); - vsb = macro_expand(p); + AN(p); + vsb = macro_expand(vl, p); AN(vsb); - vl = vtc_logopen(c->name); - if (c->repeat == 0) c->repeat = 1; if (c->repeat != 1) From phk at varnish-cache.org Wed May 5 08:20:01 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 5 May 2010 10:20:01 +0200 Subject: r4768 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-05 10:20:01 +0200 (Wed, 05 May 2010) New Revision: 4768 Modified: trunk/varnish-cache/bin/varnishtest/vtc.c Log: Return in main-thread if macro expansion fails Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-05 08:16:31 UTC (rev 4767) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-05 08:20:01 UTC (rev 4768) @@ -307,6 +307,8 @@ if (NULL == strstr(token_s[tn], "${")) continue; token_exp[tn] = macro_expand(vl, token_s[tn]); + if (vtc_error) + return; token_s[tn] = vsb_data(token_exp[tn]); token_e[tn] = strchr(token_s[tn], '\0'); } From phk at varnish-cache.org Wed May 5 08:28:04 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 5 May 2010 10:28:04 +0200 Subject: r4769 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-05 10:28:04 +0200 (Wed, 05 May 2010) New Revision: 4769 Modified: trunk/varnish-cache/bin/varnishtest/vtc.c Log: Typo: of course we want to expand the macro when we find it. Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-05 08:20:01 UTC (rev 4768) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-05 08:28:04 UTC (rev 4769) @@ -178,7 +178,7 @@ vtc_log(vl, 0, "Macro ${%s} not found", p); return (NULL); } - vsb_printf(vsb, "{MACRO NOT FOUND: %s}", p); + vsb_printf(vsb, "%s", m); text = q + 1; } vsb_finish(vsb); From tfheen at varnish-cache.org Wed May 5 08:38:16 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 10:38:16 +0200 Subject: r4770 - in branches/2.1: . varnish-cache/bin/varnishd varnish-cache/bin/varnishtest varnish-cache/bin/varnishtest/tests varnish-cache/include varnish-cache/lib/libvarnish varnish-cache/lib/libvcl Message-ID: Author: tfheen Date: 2010-05-05 10:38:16 +0200 (Wed, 05 May 2010) New Revision: 4770 Modified: branches/2.1/ branches/2.1/varnish-cache/bin/varnishd/cache_backend.h branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c branches/2.1/varnish-cache/bin/varnishd/vparam.h branches/2.1/varnish-cache/bin/varnishtest/tests/b00000.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00000.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00001.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00002.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00003.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00004.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00005.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/p00006.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00010.vtc branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc branches/2.1/varnish-cache/bin/varnishtest/vtc_varnish.c branches/2.1/varnish-cache/include/vct.h branches/2.1/varnish-cache/include/vev.h branches/2.1/varnish-cache/lib/libvarnish/tcp.c branches/2.1/varnish-cache/lib/libvarnish/vev.c branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c Log: Merge r4750: Add -storage spec to varnishtest Introduce a -storage spec, and default it -sfile,,10m to avoid mapping more VM than a 32bit machine can cope with, if /tmp is huge. Property changes on: branches/2.1 ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishd/cache_backend.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishd/cache_backend_cfg.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishd/vparam.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishd/vparam.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/b00000.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/b00000.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/b00000.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -9,7 +9,7 @@ txresp -body "012345\n" } -start -varnish v1 -arg "-smalloc,1m" -vcl+backend {} -start +varnish v1 -storage "-smalloc,1m" -vcl+backend {} -start varnish v1 -cliok "param.set diag_bitmap 0x2" Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/c00019.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00000.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00000.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00000.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -11,7 +11,8 @@ varnish v1 \ -arg "-pdiag_bitmap=0x20000" \ - -arg "-spersistent,${tmpdir}/_.per,10m" -vcl+backend { } -start + -storage "-spersistent,${tmpdir}/_.per,10m" \ + -vcl+backend { } -start varnish v1 -stop Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00001.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00001.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00001.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -11,7 +11,8 @@ varnish v1 \ -arg "-pdiag_bitmap=0x20000" \ - -arg "-spersistent,${tmpdir}/_.per,10m" -vcl+backend { } -start + -storage "-spersistent,${tmpdir}/_.per,10m" \ + -vcl+backend { } -start client c1 { txreq -url "/" Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00002.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00002.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00002.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -11,8 +11,8 @@ varnish v1 \ -arg "-pdiag_bitmap=0x20000" \ - -arg "-spersistent,${tmpdir}/_.per1,10m" \ - -arg "-spersistent,${tmpdir}/_.per2,10m" \ + -storage "-spersistent,${tmpdir}/_.per1,10m" \ + -storage "-spersistent,${tmpdir}/_.per2,10m" \ -vcl+backend { } -start client c1 { Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00003.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00003.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00003.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -11,7 +11,7 @@ varnish v1 \ -arg "-pdiag_bitmap=0x20000" \ - -arg "-spersistent,${tmpdir}/_.per,10m" \ + -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { } -start varnish v1 -cliok purge.list Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00004.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00004.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00004.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -13,7 +13,7 @@ varnish v1 \ -arg "-pdiag_bitmap=0x20000" \ - -arg "-spersistent,${tmpdir}/_.per,10m" \ + -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { } -start client c1 { Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00005.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00005.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00005.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -11,7 +11,7 @@ varnish v1 \ -arg "-pdiag_bitmap=0x30000" \ - -arg "-spersistent,${tmpdir}/_.per,10m" \ + -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { sub vcl_fetch { set beresp.ttl = 3s; Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/p00006.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/p00006.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/p00006.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -13,7 +13,7 @@ varnish v1 \ - -arg "-spersistent,${tmpdir}/_.per,10m" \ + -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { } -start client c1 { Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00325.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishtest/tests/r00325.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/r00416.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishtest/tests/r00416.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Modified: branches/2.1/varnish-cache/bin/varnishtest/tests/v00010.vtc =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/tests/v00010.vtc 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/tests/v00010.vtc 2010-05-05 08:38:16 UTC (rev 4770) @@ -13,7 +13,7 @@ txresp -hdr "Foo: foo" -body "abcdef\n" } -start -varnish v1 -arg "-smalloc,1m" -vcl+backend { +varnish v1 -storage "-smalloc,1m" -vcl+backend { sub vcl_fetch { if (beresp.http.panic) { Property changes on: branches/2.1/varnish-cache/bin/varnishtest/tests/v00011.vtc ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Modified: branches/2.1/varnish-cache/bin/varnishtest/vtc_varnish.c =================================================================== --- branches/2.1/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-05 08:28:04 UTC (rev 4769) +++ branches/2.1/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-05 08:38:16 UTC (rev 4770) @@ -33,6 +33,7 @@ #include +#include #include #include #include @@ -68,6 +69,8 @@ struct varnish_stats *stats; + struct vsb *storage; + struct vsb *args; int fds[4]; pid_t pid; @@ -171,9 +174,15 @@ vtc_log(v->vl, 0, "Varnish name must start with 'v'"); v->args = vsb_newauto(); + + v->storage = vsb_newauto(); + vsb_printf(v->storage, "-sfile,%s,10M", v->workdir); + vsb_finish(v->storage); + v->cli_fd = -1; VTAILQ_INSERT_TAIL(&varnishes, v, list); + return (v); } @@ -269,6 +278,7 @@ vsb_printf(vsb, " -S %s/_S", v->workdir); vsb_printf(vsb, " -M %s:%s", abuf, pbuf); vsb_printf(vsb, " -P %s/varnishd.pid", v->workdir); + vsb_printf(vsb, " %s", vsb_data(v->storage)); vsb_printf(vsb, " %s", vsb_data(v->args)); vsb_finish(vsb); AZ(vsb_overflowed(vsb)); @@ -663,6 +673,13 @@ for (; *av != NULL; av++) { if (vtc_error) break; + if (!strcmp(*av, "-storage")) { + vsb_clear(v->storage); + vsb_cat(v->storage, av[1]); + vsb_finish(v->storage); + av++; + continue; + } if (!strcmp(*av, "-arg")) { AN(av[1]); AZ(v->pid); Property changes on: branches/2.1/varnish-cache/include/vct.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/include/vct.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/include/vev.h ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/include/vev.h:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/tcp.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/lib/libvarnish/tcp.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/lib/libvarnish/vev.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/lib/libvarnish/vev.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 Property changes on: branches/2.1/varnish-cache/lib/libvcl/vcc_dir_random.c ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4757-4758,4762 + /trunk/varnish-cache/lib/libvcl/vcc_dir_random.c:4637,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 From tfheen at varnish-cache.org Wed May 5 08:48:56 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 10:48:56 +0200 Subject: r4771 - branches/2.1/varnish-cache/redhat Message-ID: Author: tfheen Date: 2010-05-05 10:48:56 +0200 (Wed, 05 May 2010) New Revision: 4771 Modified: branches/2.1/varnish-cache/redhat/varnish.spec Log: Update spec file to 2.1.2 Modified: branches/2.1/varnish-cache/redhat/varnish.spec =================================================================== --- branches/2.1/varnish-cache/redhat/varnish.spec 2010-05-05 08:38:16 UTC (rev 4770) +++ branches/2.1/varnish-cache/redhat/varnish.spec 2010-05-05 08:48:56 UTC (rev 4771) @@ -1,7 +1,7 @@ Summary: High-performance HTTP accelerator Name: varnish -Version: 2.1.1 -Release: 0.svn20100421%{?dist} +Version: 2.1.2 +Release: 1 License: BSD Group: System Environment/Daemons URL: http://www.varnish-cache.org/ @@ -88,6 +88,12 @@ %build +# No pkgconfig/libpcre.pc in rhel4 +%if 0%{?rhel} == 4 + export PCRE_CFLAGS=`pcre-config --cflags` + export PCRE_LIBS=`pcre-config --libs` +%endif + # Remove "--disable static" if you want to build static libraries # jemalloc is not compatible with Red Hat's ppc* RHEL5 kernel koji server :-( %ifarch ppc64 ppc @@ -251,6 +257,17 @@ %postun libs -p /sbin/ldconfig %changelog +* Wed May 05 2010 Tollef Fog Heen - 2.1.2-1 +- New upstream release +- Remove patches merged upstream + +* Tue Apr 27 2010 Ingvar Hagelund - 2.1.1-1 +- New upstream release +- Added a fix for missing pkgconfig/libpcre.pc on rhel4 +- Added a patch from trunk making the rpm buildable on lowspec + build hosts (like Red Hat's ppc build farm nodes) +- Removed patches that are merged upstream + * Wed Apr 14 2010 Ingvar Hagelund - 2.1.0-2 - Added a patch from svn that fixes changes-2.0.6-2.1.0.xml From tfheen at varnish-cache.org Wed May 5 08:50:41 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 10:50:41 +0200 Subject: r4772 - branches/2.1/varnish-cache Message-ID: Author: tfheen Date: 2010-05-05 10:50:41 +0200 (Wed, 05 May 2010) New Revision: 4772 Modified: branches/2.1/varnish-cache/configure.ac Log: Bump version number in configure.ac Modified: branches/2.1/varnish-cache/configure.ac =================================================================== --- branches/2.1/varnish-cache/configure.ac 2010-05-05 08:48:56 UTC (rev 4771) +++ branches/2.1/varnish-cache/configure.ac 2010-05-05 08:50:41 UTC (rev 4772) @@ -5,7 +5,7 @@ Copyright (c) 2006-2010 Redpill Linpro AS Copyright (c) 2010 Varnish Software AS]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [2.1.1], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [2.1.2], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/varnishapi.h) AM_CONFIG_HEADER(config.h) From tfheen at varnish-cache.org Wed May 5 09:34:20 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Wed, 5 May 2010 11:34:20 +0200 Subject: r4773 - in tags: . varnish-2.1.2 Message-ID: Author: tfheen Date: 2010-05-05 11:34:20 +0200 (Wed, 05 May 2010) New Revision: 4773 Added: tags/varnish-2.1.2/ Log: Release 2.1.2 Property changes on: tags/varnish-2.1.2 ___________________________________________________________________ Added: svn:ignore + .deps Makefile Makefile.in aclocal.m4 autom4te.cache compile config.guess config.h config.h.in config.log config.status config.sub configure depcomp install-sh missing stamp-h1 varnish Added: svn:mergeinfo + /trunk:4637,4640,4643-4645,4647-4650,4654-4670,4686,4689-4690,4700,4712,4715-4719,4731,4750,4757-4758,4762 From phk at varnish-cache.org Thu May 6 08:26:50 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 10:26:50 +0200 Subject: r4774 - in trunk/varnish-cache: bin/varnishhist bin/varnishlog bin/varnishncsa bin/varnishreplay bin/varnishsizes bin/varnishtop include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-06 10:26:49 +0200 (Thu, 06 May 2010) New Revision: 4774 Modified: trunk/varnish-cache/bin/varnishhist/varnishhist.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/bin/varnishncsa/varnishncsa.c trunk/varnish-cache/bin/varnishreplay/varnishreplay.c trunk/varnish-cache/bin/varnishsizes/varnishsizes.c trunk/varnish-cache/bin/varnishtop/varnishtop.c trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Centralize the '-n' option handling for the log tools in libvarnishapi. Modified: trunk/varnish-cache/bin/varnishhist/varnishhist.c =================================================================== --- trunk/varnish-cache/bin/varnishhist/varnishhist.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/bin/varnishhist/varnishhist.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -320,15 +320,11 @@ { int o; struct VSL_data *vd; - const char *n_arg = NULL; vd = VSL_New(); - while ((o = getopt(argc, argv, VSL_ARGS "n:Vw:")) != -1) { + while ((o = getopt(argc, argv, VSL_ARGS "Vw:")) != -1) { switch (o) { - case 'n': - n_arg = optarg; - break; case 'V': varnish_version("varnishhist"); exit(0); @@ -342,7 +338,7 @@ } } - if (VSL_OpenLog(vd, n_arg)) + if (VSL_OpenLog(vd)) exit(1); log_ten = log(10.0); Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -311,7 +311,6 @@ { int c; int a_flag = 0, D_flag = 0, o_flag = 0, u_flag = 0; - const char *n_arg = NULL; const char *P_arg = NULL; const char *w_arg = NULL; struct pidfh *pfh = NULL; @@ -319,7 +318,7 @@ vd = VSL_New(); - while ((c = getopt(argc, argv, VSL_ARGS "aDon:P:uVw:")) != -1) { + while ((c = getopt(argc, argv, VSL_ARGS "aDoP:uVw:")) != -1) { switch (c) { case 'a': a_flag = 1; @@ -335,9 +334,6 @@ case 'D': D_flag = 1; break; - case 'n': - n_arg = optarg; - break; case 'o': o_flag = 1; break; @@ -363,7 +359,7 @@ if (o_flag && w_arg != NULL) usage(); - if (VSL_OpenLog(vd, n_arg)) + if (VSL_OpenLog(vd)) exit(1); if (P_arg && (pfh = vpf_open(P_arg, 0644, NULL)) == NULL) { Modified: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c =================================================================== --- trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -521,7 +521,6 @@ { int c; int a_flag = 0, D_flag = 0; - const char *n_arg = NULL; const char *P_arg = NULL; const char *w_arg = NULL; struct pidfh *pfh = NULL; @@ -530,7 +529,7 @@ vd = VSL_New(); - while ((c = getopt(argc, argv, VSL_ARGS "aDn:P:Vw:f")) != -1) { + while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:f")) != -1) { switch (c) { case 'a': a_flag = 1; @@ -541,9 +540,6 @@ case 'D': D_flag = 1; break; - case 'n': - n_arg = optarg; - break; case 'P': P_arg = optarg; break; @@ -569,7 +565,7 @@ VSL_Arg(vd, 'c', optarg); - if (VSL_OpenLog(vd, n_arg)) + if (VSL_OpenLog(vd)) exit(1); if (P_arg && (pfh = vpf_open(P_arg, 0644, NULL)) == NULL) { Modified: trunk/varnish-cache/bin/varnishreplay/varnishreplay.c =================================================================== --- trunk/varnish-cache/bin/varnishreplay/varnishreplay.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/bin/varnishreplay/varnishreplay.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -724,7 +724,7 @@ debug = 0; VSL_Arg(vd, 'c', NULL); - while ((c = getopt(argc, argv, "a:Dr:")) != -1) { + while ((c = getopt(argc, argv, "a:Dr:n:")) != -1) { switch (c) { case 'a': address = optarg; @@ -743,7 +743,7 @@ usage(); } - if (VSL_OpenLog(vd, NULL)) + if (VSL_OpenLog(vd)) exit(1); addr_info = init_connection(address); Modified: trunk/varnish-cache/bin/varnishsizes/varnishsizes.c =================================================================== --- trunk/varnish-cache/bin/varnishsizes/varnishsizes.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/bin/varnishsizes/varnishsizes.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -321,15 +321,11 @@ { int o; struct VSL_data *vd; - const char *n_arg = NULL; vd = VSL_New(); - while ((o = getopt(argc, argv, VSL_ARGS "n:Vw:")) != -1) { + while ((o = getopt(argc, argv, VSL_ARGS "Vw:")) != -1) { switch (o) { - case 'n': - n_arg = optarg; - break; case 'V': varnish_version("varnishsizes"); exit(0); @@ -343,7 +339,7 @@ } } - if (VSL_OpenLog(vd, n_arg)) + if (VSL_OpenLog(vd)) exit(1); log_ten = log(10.0); Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -302,20 +302,16 @@ main(int argc, char **argv) { struct VSL_data *vd; - const char *n_arg = NULL; int o, once = 0; vd = VSL_New(); - while ((o = getopt(argc, argv, VSL_ARGS "1fn:V")) != -1) { + while ((o = getopt(argc, argv, VSL_ARGS "1fV")) != -1) { switch (o) { case '1': VSL_Arg(vd, 'd', NULL); once = 1; break; - case 'n': - n_arg = optarg; - break; case 'f': f_flag = 1; break; @@ -329,7 +325,7 @@ } } - if (VSL_OpenLog(vd, n_arg)) + if (VSL_OpenLog(vd)) exit (1); if (once) { Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-06 08:26:49 UTC (rev 4774) @@ -43,14 +43,14 @@ unsigned len, unsigned spec, const char *ptr); #define VSL_S_CLIENT (1 << 0) #define VSL_S_BACKEND (1 << 1) -#define VSL_ARGS "bCcdI:i:k:r:s:X:x:" +#define VSL_ARGS "bCcdI:i:k:n:r:s:X:x:" #define VSL_USAGE "[-bCcd] [-i tag] [-I regexp] [-k keep]" \ " [-r file] [-s skip] [-X regexp] [-x tag]" vsl_handler VSL_H_Print; struct VSL_data; struct VSL_data *VSL_New(void); void VSL_Select(struct VSL_data *vd, unsigned tag); -int VSL_OpenLog(struct VSL_data *vd, const char *varnish_name); +int VSL_OpenLog(struct VSL_data *vd); void VSL_NonBlocking(struct VSL_data *vd, int nb); int VSL_Dispatch(struct VSL_data *vd, vsl_handler *func, void *priv); int VSL_NextLog(struct VSL_data *lh, unsigned char **pp); Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-05 09:34:20 UTC (rev 4773) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 08:26:49 UTC (rev 4774) @@ -69,6 +69,7 @@ unsigned rbuflen; unsigned char *rbuf; + char *n_opt; int b_opt; int c_opt; int d_opt; @@ -201,7 +202,7 @@ /*--------------------------------------------------------------------*/ int -VSL_OpenLog(struct VSL_data *vd, const char *varnish_name) +VSL_OpenLog(struct VSL_data *vd) { unsigned char *p; @@ -209,7 +210,7 @@ if (vd->fd != -1) return (0); - if (vsl_shmem_map(varnish_name)) + if (vsl_shmem_map(vd->n_opt)) return (-1); vd->head = vsl_lh; @@ -562,6 +563,11 @@ vd->flags |= F_NON_BLOCKING; return (1); case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); + case 'n': + free(vd->n_opt); + vd->n_opt = strdup(opt); + assert(vd->n_opt != NULL); + return (1); case 'r': return (vsl_r_arg(vd, opt)); case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); case 'C': vd->regflags = VRE_CASELESS; return (1); From phk at varnish-cache.org Thu May 6 09:00:15 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 11:00:15 +0200 Subject: r4775 - in trunk/varnish-cache: bin/varnishd bin/varnishhist bin/varnishlog bin/varnishncsa bin/varnishsizes bin/varnishstat bin/varnishtest bin/varnishtop include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-06 11:00:15 +0200 (Thu, 06 May 2010) New Revision: 4775 Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishhist/varnishhist.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/bin/varnishncsa/varnishncsa.c trunk/varnish-cache/bin/varnishsizes/varnishsizes.c trunk/varnish-cache/bin/varnishstat/varnishstat.c trunk/varnish-cache/bin/varnishtest/vtc_varnish.c trunk/varnish-cache/bin/varnishtop/varnishtop.c trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/include/vbm.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Overhaul the libvarnishapi: - make struct VSL_data the uniform handle. - eliminate global variables (this allows a program to have multiple shmlogs open at the same time). - add a VSL_Delete() to clean up This breaks libvarnishapi compatibility with the 2.x series. Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -540,6 +540,7 @@ child_state = CH_STOPPED; } else if (child_state == CH_STOPPING) child_state = CH_STOPPED; + return (0); } @@ -615,6 +616,9 @@ REPORT(LOG_ERR, "vev_schedule() = %d", i); REPORT0(LOG_ERR, "manager dies"); + + /* XXX: quench compiler warning about unused func */ + vbit_destroy(NULL); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishhist/varnishhist.c =================================================================== --- trunk/varnish-cache/bin/varnishhist/varnishhist.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishhist/varnishhist.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -96,7 +96,7 @@ }; static void -update(void) +update(struct VSL_data *vd) { int w = COLS / HIST_RANGE; int n = w * HIST_RANGE; @@ -116,7 +116,7 @@ mvprintw(LINES - 1, w * i, "|1e%d", j); } - mvprintw(0, 0, "%*s", COLS - 1, VSL_Name()); + mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd)); /* count our flock */ for (i = 0; i < n; ++i) @@ -257,7 +257,7 @@ erase(); for (;;) { pthread_mutex_lock(&mtx); - update(); + update(vd); pthread_mutex_unlock(&mtx); timeout(delay * 1000); @@ -323,7 +323,7 @@ vd = VSL_New(); - while ((o = getopt(argc, argv, VSL_ARGS "Vw:")) != -1) { + while ((o = getopt(argc, argv, VSL_LOG_ARGS "Vw:")) != -1) { switch (o) { case 'V': varnish_version("varnishhist"); Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -318,7 +318,7 @@ vd = VSL_New(); - while ((c = getopt(argc, argv, VSL_ARGS "aDoP:uVw:")) != -1) { + while ((c = getopt(argc, argv, VSL_LOG_ARGS "aDoP:uVw:")) != -1) { switch (c) { case 'a': a_flag = 1; Modified: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c =================================================================== --- trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -529,7 +529,7 @@ vd = VSL_New(); - while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:f")) != -1) { + while ((c = getopt(argc, argv, VSL_LOG_ARGS "aDP:Vw:f")) != -1) { switch (c) { case 'a': a_flag = 1; Modified: trunk/varnish-cache/bin/varnishsizes/varnishsizes.c =================================================================== --- trunk/varnish-cache/bin/varnishsizes/varnishsizes.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishsizes/varnishsizes.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -96,7 +96,7 @@ }; static void -update(void) +update(struct VSL_data *vd) { int w = COLS / HIST_RANGE; int n = w * HIST_RANGE; @@ -116,7 +116,7 @@ mvprintw(LINES - 1, w * i, "|1e%d", j); } - mvprintw(0, 0, "%*s", COLS - 1, VSL_Name()); + mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd)); /* count our flock */ for (i = 0; i < n; ++i) @@ -258,7 +258,7 @@ erase(); for (;;) { pthread_mutex_lock(&mtx); - update(); + update(vd); pthread_mutex_unlock(&mtx); timeout(delay * 1000); @@ -324,7 +324,7 @@ vd = VSL_New(); - while ((o = getopt(argc, argv, VSL_ARGS "Vw:")) != -1) { + while ((o = getopt(argc, argv, VSL_LOG_ARGS "Vw:")) != -1) { switch (o) { case 'V': varnish_version("varnishsizes"); Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -94,7 +94,7 @@ } static void -do_curses(struct varnish_stats *VSL_stats, int delay, const char *fields) +do_curses(struct VSL_data *vd, struct varnish_stats *VSL_stats, int delay, const char *fields) { struct varnish_stats copy; struct varnish_stats seen; @@ -129,7 +129,7 @@ rt = VSL_stats->uptime; up = rt; - mvprintw(0, 0, "%*s", COLS - 1, VSL_Name()); + mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd)); mvprintw(0, 0, "%d+%02d:%02d:%02d", rt / 86400, (rt % 86400) / 3600, (rt % 3600) / 60, rt % 60); @@ -347,12 +347,14 @@ main(int argc, char **argv) { int c; + struct VSL_data *vd; struct varnish_stats *VSL_stats; int delay = 1, once = 0, xml = 0; - const char *n_arg = NULL; const char *fields = NULL; - while ((c = getopt(argc, argv, "1f:ln:Vw:x")) != -1) { + vd = VSL_New(); + + while ((c = getopt(argc, argv, VSL_STAT_ARGS "1f:lVw:x")) != -1) { switch (c) { case '1': once = 1; @@ -363,9 +365,6 @@ case 'l': list_fields(); exit(0); - case 'n': - n_arg = optarg; - break; case 'V': varnish_version("varnishstat"); exit(0); @@ -376,11 +375,13 @@ xml = 1; break; default: + if (VSL_Arg(vd, c, optarg) > 0) + break; usage(); } } - if ((VSL_stats = VSL_OpenStats(n_arg)) == NULL) + if ((VSL_stats = VSL_OpenStats(vd)) == NULL) exit(1); if (fields != NULL && !valid_fields(fields)) { @@ -393,7 +394,7 @@ else if (once) do_once(VSL_stats, fields); else - do_curses(VSL_stats, delay, fields); + do_curses(vd, VSL_stats, delay, fields); exit(0); } Modified: trunk/varnish-cache/bin/varnishtest/vtc_varnish.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -80,6 +80,8 @@ int cli_fd; int vcl_nbr; char *workdir; + + struct VSL_data *vd; }; static VTAILQ_HEAD(, varnish) varnishes = @@ -198,7 +200,7 @@ vtc_logclose(v->vl); free(v->name); free(v->workdir); - VSL_Close(); + VSL_Delete(v->vd); /* * We do not delete the workdir, it may contain stuff people @@ -259,6 +261,8 @@ enum cli_status_e u; char *r; + v->vd = VSL_New(); + /* Create listener socket */ nap = VSS_resolve("127.0.0.1", "0", &ap); AN(nap); @@ -359,8 +363,9 @@ free(r); if (v->stats != NULL) - VSL_Close(); - v->stats = VSL_OpenStats(v->workdir); + VSL_Close(v->vd); + VSL_Arg(v->vd, 'n', v->workdir); + v->stats = VSL_OpenStats(v->vd); } /********************************************************************** Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -137,7 +137,7 @@ } static void -update(void) +update(struct VSL_data *vd) { struct top *tp, *tp2; int l, len; @@ -152,7 +152,7 @@ erase(); l = 1; - mvprintw(0, 0, "%*s", COLS - 1, VSL_Name()); + mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd)); mvprintw(0, 0, "list length %u", ntop); VTAILQ_FOREACH_SAFE(tp, &top_head, list, tp2) { if (++l < LINES) { @@ -228,7 +228,7 @@ erase(); for (;;) { pthread_mutex_lock(&mtx); - update(); + update(vd); pthread_mutex_unlock(&mtx); timeout(1000); @@ -306,7 +306,7 @@ vd = VSL_New(); - while ((o = getopt(argc, argv, VSL_ARGS "1fV")) != -1) { + while ((o = getopt(argc, argv, VSL_LOG_ARGS "1fV")) != -1) { switch (o) { case '1': VSL_Arg(vd, 'd', NULL); Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-06 09:00:15 UTC (rev 4775) @@ -43,7 +43,8 @@ unsigned len, unsigned spec, const char *ptr); #define VSL_S_CLIENT (1 << 0) #define VSL_S_BACKEND (1 << 1) -#define VSL_ARGS "bCcdI:i:k:n:r:s:X:x:" +#define VSL_LOG_ARGS "bCcdI:i:k:n:r:s:X:x:" +#define VSL_STAT_ARGS "n:" #define VSL_USAGE "[-bCcd] [-i tag] [-I regexp] [-k keep]" \ " [-r file] [-s skip] [-X regexp] [-x tag]" vsl_handler VSL_H_Print; @@ -55,9 +56,10 @@ int VSL_Dispatch(struct VSL_data *vd, vsl_handler *func, void *priv); int VSL_NextLog(struct VSL_data *lh, unsigned char **pp); int VSL_Arg(struct VSL_data *vd, int arg, const char *opt); -void VSL_Close(void); -struct varnish_stats *VSL_OpenStats(const char *varnish_name); -const char *VSL_Name(void); +void VSL_Close(struct VSL_data *vd); +void VSL_Delete(struct VSL_data *vd); +struct varnish_stats *VSL_OpenStats(struct VSL_data *vd); +const char *VSL_Name(struct VSL_data *vd); extern const char *VSL_tags[256]; #endif Modified: trunk/varnish-cache/include/vbm.h =================================================================== --- trunk/varnish-cache/include/vbm.h 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/include/vbm.h 2010-05-06 09:00:15 UTC (rev 4775) @@ -75,6 +75,16 @@ return (vb); } +static void +vbit_destroy(struct vbitmap *vb) +{ + + if (vb == NULL) + return; + free(vb->bits); + free(vb); +} + static inline void vbit_set(struct vbitmap *vb, unsigned bit) { Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 08:26:49 UTC (rev 4774) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 09:00:15 UTC (rev 4775) @@ -65,7 +65,7 @@ unsigned char *ptr; /* for -r option */ - int fd; + int r_fd; unsigned rbuflen; unsigned char *rbuf; @@ -101,16 +101,15 @@ unsigned long skip; unsigned long keep; + + int vsl_fd; + struct shmloghead *vsl_lh; }; #ifndef MAP_HASSEMAPHORE #define MAP_HASSEMAPHORE 0 /* XXX Linux */ #endif -static int vsl_fd; -static struct shmloghead *vsl_lh; -static char vsl_name[PATH_MAX]; - static int vsl_nextlog(struct VSL_data *vd, unsigned char **pp); /*--------------------------------------------------------------------*/ @@ -124,28 +123,28 @@ /*--------------------------------------------------------------------*/ static int -vsl_shmem_map(const char *varnish_name) +vsl_shmem_map(struct VSL_data *vd) { int i; struct shmloghead slh; char *logname; - if (vsl_lh != NULL) + if (vd->vsl_lh != NULL) return (0); - if (vin_n_arg(varnish_name, NULL, NULL, &logname)) { + if (vin_n_arg(vd->n_opt, NULL, NULL, &logname)) { fprintf(stderr, "Invalid instance name: %s\n", strerror(errno)); return (1); } - vsl_fd = open(logname, O_RDONLY); - if (vsl_fd < 0) { + vd->vsl_fd = open(logname, O_RDONLY); + if (vd->vsl_fd < 0) { fprintf(stderr, "Cannot open %s: %s\n", logname, strerror(errno)); return (1); } - i = read(vsl_fd, &slh, sizeof slh); + i = read(vd->vsl_fd, &slh, sizeof slh); if (i != sizeof slh) { fprintf(stderr, "Cannot read %s: %s\n", logname, strerror(errno)); @@ -157,9 +156,9 @@ return (1); } - vsl_lh = (void *)mmap(NULL, slh.size + sizeof slh, - PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vsl_fd, 0); - if (vsl_lh == MAP_FAILED) { + vd->vsl_lh = (void *)mmap(NULL, slh.size + sizeof slh, + PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsl_fd, 0); + if (vd->vsl_lh == MAP_FAILED) { fprintf(stderr, "Cannot mmap %s: %s\n", logname, strerror(errno)); return (1); @@ -178,20 +177,43 @@ assert(vd != NULL); vd->regflags = 0; vd->magic = VSL_MAGIC; - vd->fd = -1; + vd->vsl_fd = -1; + + + /* XXX: Allocate only if log access */ vd->vbm_client = vbit_init(4096); vd->vbm_backend = vbit_init(4096); vd->vbm_supress = vbit_init(256); vd->vbm_select = vbit_init(256); + + vd->r_fd = -1; + /* XXX: Allocate only if -r option given ? */ vd->rbuflen = SHMLOG_NEXTTAG + 256; vd->rbuf = malloc(vd->rbuflen); assert(vd->rbuf != NULL); + return (vd); } /*--------------------------------------------------------------------*/ void +VSL_Delete(struct VSL_data *vd) +{ + + VSL_Close(vd); + vbit_destroy(vd->vbm_client); + vbit_destroy(vd->vbm_backend); + vbit_destroy(vd->vbm_supress); + vbit_destroy(vd->vbm_select); + free(vd->n_opt); + free(vd->rbuf); + free(vd); +} + +/*--------------------------------------------------------------------*/ + +void VSL_Select(struct VSL_data *vd, unsigned tag) { @@ -207,18 +229,18 @@ unsigned char *p; CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (vd->fd != -1) + if (vd->r_fd != -1) return (0); - if (vsl_shmem_map(vd->n_opt)) + if (vsl_shmem_map(vd)) return (-1); - vd->head = vsl_lh; - vd->logstart = (unsigned char *)vsl_lh + vsl_lh->start; - vd->logend = vd->logstart + vsl_lh->size; + vd->head = vd->vsl_lh; + vd->logstart = (unsigned char *)vd->vsl_lh + vd->vsl_lh->start; + vd->logend = vd->logstart + vd->vsl_lh->size; vd->ptr = vd->logstart; - if (!vd->d_opt && vd->fd == -1) { + if (!vd->d_opt && vd->r_fd == -1) { for (p = vd->ptr; *p != SLT_ENDMARKER; ) p += SHMLOG_LEN(p) + SHMLOG_NEXTTAG; vd->ptr = p; @@ -247,9 +269,9 @@ int i; CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (vd->fd != -1) { + if (vd->r_fd != -1) { assert(vd->rbuflen >= SHMLOG_DATA); - i = read(vd->fd, vd->rbuf, SHMLOG_DATA); + i = read(vd->r_fd, vd->rbuf, SHMLOG_DATA); if (i != SHMLOG_DATA) return (-1); l = SHMLOG_LEN(vd->rbuf) + SHMLOG_NEXTTAG; @@ -260,7 +282,7 @@ vd->rbuflen = l; } l = SHMLOG_LEN(vd->rbuf) + 1; - i = read(vd->fd, vd->rbuf + SHMLOG_DATA, l); + i = read(vd->r_fd, vd->rbuf + SHMLOG_DATA, l); if (i != l) return (-1); *pp = vd->rbuf; @@ -421,10 +443,10 @@ CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (!strcmp(opt, "-")) - vd->fd = STDIN_FILENO; + vd->r_fd = STDIN_FILENO; else - vd->fd = open(opt, O_RDONLY); - if (vd->fd < 0) { + vd->r_fd = open(opt, O_RDONLY); + if (vd->r_fd < 0) { perror(opt); return (-1); } @@ -579,29 +601,30 @@ } struct varnish_stats * -VSL_OpenStats(const char *varnish_name) +VSL_OpenStats(struct VSL_data *vd) { - if (vsl_shmem_map(varnish_name)) + if (vsl_shmem_map(vd)) return (NULL); - return (&vsl_lh->stats); + return (&vd->vsl_lh->stats); } void -VSL_Close(void) +VSL_Close(struct VSL_data *vd) { - if (vsl_lh == NULL) + if (vd->vsl_lh == NULL) return; - assert(0 == munmap((void*)vsl_lh, vsl_lh->size + sizeof *vsl_lh)); - vsl_lh = NULL; - assert(vsl_fd >= 0); - assert(0 == close(vsl_fd)); - vsl_fd = -1; + assert(0 == munmap((void*)vd->vsl_lh, + vd->vsl_lh->size + sizeof *vd->vsl_lh)); + vd->vsl_lh = NULL; + assert(vd->vsl_fd >= 0); + assert(0 == close(vd->vsl_fd)); + vd->vsl_fd = -1; } const char * -VSL_Name(void) +VSL_Name(struct VSL_data *vd) { - return (vsl_name); + return (vd->n_opt); } From phk at varnish-cache.org Thu May 6 09:10:36 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 11:10:36 +0200 Subject: r4776 - in trunk/varnish-cache: include lib/libvarnish lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-06 11:10:36 +0200 (Thu, 06 May 2010) New Revision: 4776 Modified: trunk/varnish-cache/include/shmlog.h trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnish/vin.c trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Implement the -L(earn) option in all the log/stats tools via libvarnishapi Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-06 09:00:15 UTC (rev 4775) +++ trunk/varnish-cache/include/shmlog.h 2010-05-06 09:10:36 UTC (rev 4776) @@ -111,5 +111,9 @@ /* This function lives in both libvarnish and libvarnishapi */ int vin_n_arg(const char *n_arg, char **name, char **dir, char **vsl); +char *vin_L_arg(unsigned L_arg); +#define VIN_L_LOW 1024 +#define VIN_L_HIGH 65000 +#define VIN_L_OK(a) (a >= VIN_L_LOW && a <= VIN_L_HIGH) #endif Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-06 09:00:15 UTC (rev 4775) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-06 09:10:36 UTC (rev 4776) @@ -43,8 +43,8 @@ unsigned len, unsigned spec, const char *ptr); #define VSL_S_CLIENT (1 << 0) #define VSL_S_BACKEND (1 << 1) -#define VSL_LOG_ARGS "bCcdI:i:k:n:r:s:X:x:" -#define VSL_STAT_ARGS "n:" +#define VSL_LOG_ARGS "bCcdI:i:k:L:n:r:s:X:x:" +#define VSL_STAT_ARGS "L:n:" #define VSL_USAGE "[-bCcd] [-i tag] [-I regexp] [-k keep]" \ " [-r file] [-s skip] [-X regexp] [-x tag]" vsl_handler VSL_H_Print; Modified: trunk/varnish-cache/lib/libvarnish/vin.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vin.c 2010-05-06 09:00:15 UTC (rev 4775) +++ trunk/varnish-cache/lib/libvarnish/vin.c 2010-05-06 09:10:36 UTC (rev 4776) @@ -100,3 +100,13 @@ } return (0); } + +char * +vin_L_arg(unsigned L_arg) +{ + char p[PATH_MAX]; + + assert(VIN_L_OK(L_arg)); + bprintf(p, "/tmp/varnish_L_%u", L_arg); + return (strdup(p)); +} Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 09:00:15 UTC (rev 4775) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 09:10:36 UTC (rev 4776) @@ -69,6 +69,7 @@ unsigned rbuflen; unsigned char *rbuf; + unsigned L_opt; char *n_opt; int b_opt; int c_opt; @@ -585,16 +586,27 @@ vd->flags |= F_NON_BLOCKING; return (1); case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); + case 'k': return (vsl_k_arg(vd, opt)); case 'n': free(vd->n_opt); vd->n_opt = strdup(opt); assert(vd->n_opt != NULL); return (1); case 'r': return (vsl_r_arg(vd, opt)); + case 's': return (vsl_s_arg(vd, opt)); case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); case 'C': vd->regflags = VRE_CASELESS; return (1); - case 's': return (vsl_s_arg(vd, opt)); - case 'k': return (vsl_k_arg(vd, opt)); + case 'L': + vd->L_opt = strtoul(opt, NULL, 0); + if (vd->L_opt < 1024 || vd->L_opt > 65000) { + fprintf(stderr, + "Illegal -L option must be [1024...65000]\n"); + exit (1); + } + free(vd->n_opt); + vd->n_opt = vin_L_arg(vd->L_opt); + assert(vd->n_opt != NULL); + return (1); default: return (0); } From phk at varnish-cache.org Thu May 6 09:39:23 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 11:39:23 +0200 Subject: r4777 - in trunk/varnish-cache: bin/varnishd include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-06 11:39:23 +0200 (Thu, 06 May 2010) New Revision: 4777 Modified: trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/shmlog.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Add -L option to varnishd, it will: - enable -d - set listen address to ($L) - set telnet address to ($L + 1) - if no storage configured, use "-smalloc, 1m" - set -n to /tmp/varnish_L_$L - set ping_interval to zero Still missing: VCC magic to hack backends port number. Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-06 09:10:36 UTC (rev 4776) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-06 09:39:23 UTC (rev 4777) @@ -50,6 +50,7 @@ struct vsb; extern struct vsb *vident; int Symbol_Lookup(struct vsb *vsb, void *ptr); +extern unsigned L_arg; #define TRUST_ME(ptr) ((void*)(uintptr_t)(ptr)) Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 09:10:36 UTC (rev 4776) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 09:39:23 UTC (rev 4777) @@ -84,6 +84,7 @@ struct vev_base *mgt_evb; int exit_status = 0; struct vsb *vident; +unsigned L_arg = 0; static void build_vident(void) @@ -423,6 +424,7 @@ struct cli cli[1]; struct pidfh *pfh = NULL; char *dirname; + char tmpbuf[128]; /* * Start out by closing all unwanted file descriptors we might @@ -486,7 +488,7 @@ cli_check(cli); while ((o = getopt(argc, argv, - "a:b:Cdf:Fg:h:i:l:M:n:P:p:S:s:T:t:u:Vx:w:")) != -1) + "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:S:s:T:t:u:Vx:w:")) != -1) switch (o) { case 'a': MCF_ParamSet(cli, "listen_address", optarg); @@ -519,6 +521,20 @@ case 'l': l_arg = optarg; break; + case 'L': + L_arg = strtoul(optarg, NULL, 0); + if (!VIN_L_OK(L_arg)) { + fprintf(stderr, "%s\n", VIN_L_MSG); + exit (1); + } + d_flag++; + n_arg = vin_L_arg(L_arg); + MCF_ParamSet(cli, "ping_interval", "0"); + + bprintf(tmpbuf, "127.0.0.1:%u", L_arg); + MCF_ParamSet(cli, "listen_address", tmpbuf); + + break; case 'M': M_arg = optarg; break; @@ -575,6 +591,12 @@ argc -= optind; argv += optind; + if (L_arg) { + /* Learner mode */ + if (!s_arg_given) + setup_storage("malloc,1m"); + } + if (argc != 0) { fprintf(stderr, "Too many arguments (%s...)\n", argv[0]); usage(); @@ -696,6 +718,10 @@ mgt_cli_master(M_arg); if (T_arg != NULL) mgt_cli_telnet(T_arg); + else if (L_arg > 0) { + bprintf(tmpbuf, "127.0.0.1:%u", L_arg + 1); + mgt_cli_telnet(tmpbuf); + } MGT_Run(); Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-06 09:10:36 UTC (rev 4776) +++ trunk/varnish-cache/include/shmlog.h 2010-05-06 09:39:23 UTC (rev 4777) @@ -115,5 +115,6 @@ #define VIN_L_LOW 1024 #define VIN_L_HIGH 65000 #define VIN_L_OK(a) (a >= VIN_L_LOW && a <= VIN_L_HIGH) +#define VIN_L_MSG "-L argument must be [1024...65000]" #endif Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 09:10:36 UTC (rev 4776) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-06 09:39:23 UTC (rev 4777) @@ -599,8 +599,7 @@ case 'L': vd->L_opt = strtoul(opt, NULL, 0); if (vd->L_opt < 1024 || vd->L_opt > 65000) { - fprintf(stderr, - "Illegal -L option must be [1024...65000]\n"); + fprintf(stderr, "%s\n", VIN_L_MSG); exit (1); } free(vd->n_opt); From phk at varnish-cache.org Thu May 6 10:06:07 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 12:06:07 +0200 Subject: r4778 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: Author: phk Date: 2010-05-06 12:06:06 +0200 (Thu, 06 May 2010) New Revision: 4778 Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/libvcl.h trunk/varnish-cache/lib/libvcl/vcc_backend.c trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h Log: Tell the VCC if we are in -L(earn) mode. In Learn mode, a backend port can be specified as backend fs { .host = "127.0.0.1"; .port = Learn(2); } which complements the varnishtest server::-listen directive: server s1 -listen 2 { rxreq txresp -hdr "Foo: bar" -body "Hello World!" } -start -wait Obviously, the 127.0.0.1 is mandatory for this to work. Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_vcc.c 2010-05-06 09:39:23 UTC (rev 4777) +++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c 2010-05-06 10:06:06 UTC (rev 4778) @@ -447,7 +447,7 @@ mgt_vcc_init(void) { - VCC_InitCompile(default_vcl); + VCC_InitCompile(default_vcl, L_arg); AZ(atexit(mgt_vcc_atexit)); } Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 09:39:23 UTC (rev 4777) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 10:06:06 UTC (rev 4778) @@ -469,8 +469,6 @@ VTAILQ_INIT(&heritage.socks); - mgt_vcc_init(); - MCF_ParamInit(cli); if (sizeof(void *) < 8) { @@ -591,6 +589,8 @@ argc -= optind; argv += optind; + mgt_vcc_init(); + if (L_arg) { /* Learner mode */ if (!s_arg_given) Modified: trunk/varnish-cache/include/libvcl.h =================================================================== --- trunk/varnish-cache/include/libvcl.h 2010-05-06 09:39:23 UTC (rev 4777) +++ trunk/varnish-cache/include/libvcl.h 2010-05-06 10:06:06 UTC (rev 4778) @@ -30,6 +30,6 @@ */ char *VCC_Compile(struct vsb *sb, const char *b, const char *e); -void VCC_InitCompile(const char *default_vcl); +void VCC_InitCompile(const char *default_vcl, unsigned L_arg); const char *VCC_Return_Name(unsigned action); Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_backend.c 2010-05-06 09:39:23 UTC (rev 4777) +++ trunk/varnish-cache/lib/libvcl/vcc_backend.c 2010-05-06 10:06:06 UTC (rev 4778) @@ -393,7 +393,9 @@ struct fld_spec *fs; struct vsb *vsb; unsigned u; + unsigned tL_port = 0; double t; + char tmpbuf[20]; Fh(tl, 1, "\n#define VGC_backend_%s %d\n", vgcname, tl->ndirector); @@ -447,10 +449,39 @@ vcc_NextToken(tl); SkipToken(tl, ';'); } else if (vcc_IdIs(t_field, "port")) { - ExpectErr(tl, CSTR); - assert(tl->t->dec != NULL); - t_port = tl->t; - vcc_NextToken(tl); + if (Learn_mode) { + ExpectErr(tl, ID); + if (!vcc_IdIs(tl->t, "Learn")) { + vsb_printf(tl->sb, + "Expected \"Learn\"\n"); + vcc_ErrToken(tl, tl->t); + vcc_ErrWhere(tl, tl->t); + return; + } + vcc_NextToken(tl); + + ExpectErr(tl, '('); + vcc_NextToken(tl); + + ExpectErr(tl, CNUM); + t_port = tl->t; + tL_port = vcc_UintVal(tl); + if (tL_port > 9) { + vsb_printf(tl->sb, + "Learn port > 9\n"); + vcc_ErrToken(tl, t_port); + vcc_ErrWhere(tl, t_port); + return; + } + + ExpectErr(tl, ')'); + vcc_NextToken(tl); + } else { + ExpectErr(tl, CSTR); + assert(tl->t->dec != NULL); + t_port = tl->t; + vcc_NextToken(tl); + } SkipToken(tl, ';'); } else if (vcc_IdIs(t_field, "host_header")) { ExpectErr(tl, CSTR); @@ -519,7 +550,11 @@ } /* Check that the portname makes sense */ - if (t_port != NULL) { + if (Learn_mode && t_port != NULL && t_port->tok == CNUM) { + assert(tL_port); + bprintf(tmpbuf, "%u", Learn_mode + 10 + tL_port); + Emit_Sockaddr(tl, t_host, tmpbuf); + } else if (t_port != NULL) { ep = CheckHostPort("127.0.0.1", t_port->dec); if (ep != NULL) { vsb_printf(tl->sb, Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2010-05-06 09:39:23 UTC (rev 4777) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2010-05-06 10:06:06 UTC (rev 4778) @@ -86,6 +86,8 @@ static const char *vcc_default_vcl_b, *vcc_default_vcl_e; +unsigned Learn_mode = 0; + /*--------------------------------------------------------------------*/ static void @@ -673,9 +675,10 @@ */ void -VCC_InitCompile(const char *default_vcl) +VCC_InitCompile(const char *default_vcl, unsigned L_arg) { + Learn_mode = L_arg; vcc_default_vcl_b = default_vcl; vcc_default_vcl_e = strchr(default_vcl, '\0'); assert(vcc_default_vcl_e != NULL); Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2010-05-06 09:39:23 UTC (rev 4777) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2010-05-06 10:06:06 UTC (rev 4778) @@ -163,6 +163,7 @@ /* vcc_compile.c */ extern struct method method_tab[]; +extern unsigned Learn_mode; /* * H -> Header, before the C code * C -> C-code From phk at varnish-cache.org Thu May 6 11:02:23 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 13:02:23 +0200 Subject: r4779 - in trunk/varnish-cache: bin/varnishd lib/libvcl Message-ID: Author: phk Date: 2010-05-06 13:02:23 +0200 (Thu, 06 May 2010) New Revision: 4779 Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/lib/libvcl/vcc_backend.c Log: Make varnishd -L mode work with or without -b arguments Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_vcc.c 2010-05-06 10:06:06 UTC (rev 4778) +++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c 2010-05-06 11:02:23 UTC (rev 4779) @@ -335,7 +335,16 @@ /* XXX: annotate vcl with -b/-f arg so people know where it came from */ (void)f_arg; - if (b_arg != NULL) { + + if (L_arg && b_arg == NULL) { + bprintf(buf, + "backend default {\n" + " .host = \"127.0.0.1\";\n" + " .port = %s;\n" + "}\n", "Learn(0)"); + vcl = strdup(buf); + AN(vcl); + } else if (b_arg != NULL) { AZ(vcl); /* * XXX: should do a "HEAD /" on the -b argument to see that Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 10:06:06 UTC (rev 4778) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 11:02:23 UTC (rev 4779) @@ -675,7 +675,7 @@ exit(1); } - if (b_arg != NULL || f_arg != NULL) + if (b_arg != NULL || f_arg != NULL || L_arg > 0) if (mgt_vcc_default(b_arg, f_arg, vcl, C_flag)) exit (2); Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_backend.c 2010-05-06 10:06:06 UTC (rev 4778) +++ trunk/varnish-cache/lib/libvcl/vcc_backend.c 2010-05-06 11:02:23 UTC (rev 4779) @@ -551,7 +551,6 @@ /* Check that the portname makes sense */ if (Learn_mode && t_port != NULL && t_port->tok == CNUM) { - assert(tL_port); bprintf(tmpbuf, "%u", Learn_mode + 10 + tL_port); Emit_Sockaddr(tl, t_host, tmpbuf); } else if (t_port != NULL) { From phk at varnish-cache.org Thu May 6 11:04:49 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 13:04:49 +0200 Subject: r4780 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-06 13:04:49 +0200 (Thu, 06 May 2010) New Revision: 4780 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Only configure -smalloc in -L mode. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 11:02:23 UTC (rev 4779) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-06 11:04:49 UTC (rev 4780) @@ -593,8 +593,10 @@ if (L_arg) { /* Learner mode */ - if (!s_arg_given) + if (!s_arg_given) { setup_storage("malloc,1m"); + s_arg_given = 1; + } } if (argc != 0) { From phk at varnish-cache.org Thu May 6 11:21:13 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 6 May 2010 13:21:13 +0200 Subject: r4781 - in trunk/varnish-cache/doc/sphinx/tutorial: . vtc_files Message-ID: Author: phk Date: 2010-05-06 13:21:13 +0200 (Thu, 06 May 2010) New Revision: 4781 Added: trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C002.vtc trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S002.vtc Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst Log: Add the first real example of running varnishd Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-06 11:04:49 UTC (rev 4780) +++ trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-06 11:21:13 UTC (rev 4781) @@ -11,6 +11,7 @@ intro.rst tut001.rst + tut002.rst .. todo:: starting varnish with -d, seeing a transaction go through Added: trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst 2010-05-06 11:21:13 UTC (rev 4781) @@ -0,0 +1,92 @@ +.. _TUT002: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +TUT002: Caching an object with varnish +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +Now it is time to loose your viginity and cache an object with ``varnishd``. + +In the first window, start this command:: + + varnishd -L20000 -t30 + +You should see something like this:: + + storage_malloc: max size 1 MB. + Using old SHMFILE + Platform: FreeBSD,9.0-CURRENT,amd64,-smalloc,-hcritbit + 200 193 + ----------------------------- + Varnish HTTP accelerator CLI. + ----------------------------- + Type 'help' for command list. + Type 'quit' to close CLI session. + Type 'start' to launch worker process. + +We will explain this stuff later, for now just type ``start`` and you should +see something like:: + + child (88590) Started + 200 0 + + Child (88590) said + Child (88590) said Child starts + +Next, start a backend in a different window:: + + varnishtest -L20000 S002.vtc + +And finally, send a request from a third window:: + + varnishtest -L20000 -vv C002.vtc + +You will notice that both the backend and client windows both do +their thing, and exit back to the shell prompt. + +In the client window you will have a line like:: + + **** c1 http[ 6] | X-Varnish: 17443679 + +This is Varnish telling you that it was involved. (The exact number will +be different for you, it is just a transaction-id.) + +Now, try running *only* the client command again:: + + varnishtest -L20000 -vv C002.vtc + +Tada! You have just received a cache-hit from varnishd. + +This time the ``X-Varnish`` line will have two numbers:: + + **** c1 http[ 6] | X-Varnish: 17443680 17443679 + +The first number is the XID for this request, the second one is the +XID that brought this object into varnishd's cache, it matches the +number you saw above. + +If you run the client again, you will see:: + + **** c1 http[ 6] | X-Varnish: 17443681 17443679 + +You can keep running the client and you will get hits, but +30 seconds after the object was put in the cache, ``varnishd`` +will expire it, and then you will get varnish "Guru meditation" +message, because the backend does not respond:: + + **** c1 body| \n + **** c1 body| 503 Service Unavailable\n + **** c1 body| \n + **** c1 body| \n + **** c1 body|

Error 503 Service Unavailable

\n + **** c1 body|

Service Unavailable

\n + **** c1 body|

Guru Meditation:

\n + **** c1 body|

XID: 1685367940

\n + **** c1 body|
\n + +If you start the backend again: + + varnishtest -L20000 S002.vtc + +Then you can fetch the object for another 30 seconds. + + Added: trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C002.vtc =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C002.vtc (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/C002.vtc 2010-05-06 11:21:13 UTC (rev 4781) @@ -0,0 +1,6 @@ +# $Id$ + +client c1 { + txreq + rxresp +} -run Added: trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S002.vtc =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S002.vtc (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/S002.vtc 2010-05-06 11:21:13 UTC (rev 4781) @@ -0,0 +1,6 @@ +# $Id$ + +server s1 { + rxreq + txresp -hdr "Foo: bar" -body "Hello World!" +} -start -wait From andersb at varnish-cache.org Thu May 6 13:03:51 2010 From: andersb at varnish-cache.org (andersb at varnish-cache.org) Date: Thu, 6 May 2010 15:03:51 +0200 Subject: r4782 - trunk/varnish-cache/doc/sphinx/installation Message-ID: Author: andersb Date: 2010-05-06 15:03:50 +0200 (Thu, 06 May 2010) New Revision: 4782 Modified: trunk/varnish-cache/doc/sphinx/installation/install.rst Log: Added different distros to binary install in installastion documentation Modified: trunk/varnish-cache/doc/sphinx/installation/install.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-06 11:21:13 UTC (rev 4781) +++ trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-06 13:03:50 UTC (rev 4782) @@ -17,11 +17,28 @@ be done with with the systems package manager, typical examples being: -FreeBSD (from source) - ``cd /usr/ports/varnish && make install clean`` -FreeBSD (binary package) - ``pkg_add -r varnish`` +**FreeBSD** + FreeBSD (from source) + ``cd /usr/ports/varnish && make install clean`` + FreeBSD (binary package) + ``pkg_add -r varnish`` + +**CentOS/RedHat 5.4 - RPM files** + + We try to keep the lastest version available as prebuildt RPMs (el4 & el5) on `SourceForge `_. + Varnish is included in the `EPEL `_ repository. **BUT** unfortunatly we had a syntax change in Varnish 2.0.6->2.1.X. This means that we can not update Varnish in `EPEL `_ so the latest version there is Varnish 2.0.6. In the future (EPEL6) we should be available with Varnish 2.1.X or higher. + +**Debian/Ubuntu - DEB files** + + Varnish is distributed to the *unstable* repository of Debian. You should be able to get a hold of the lastest version there. + + Ubuntu syncronize the *unstable* Debian repository. See `Ubuntu Packages `_. + +**Other systems** + + You are probably best of compiling your own code. See `Compiling Varnish from source`_. + If that worked for you, you can skip the rest of this document for now, and and start reading the much more interesting :ref:`Tutorial` instead. From tfheen at varnish-cache.org Fri May 7 09:23:29 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Fri, 7 May 2010 11:23:29 +0200 Subject: r4783 - trunk/varnish-cache Message-ID: Author: tfheen Date: 2010-05-07 11:23:29 +0200 (Fri, 07 May 2010) New Revision: 4783 Modified: trunk/varnish-cache/README Log: Update URLs and email addresses Modified: trunk/varnish-cache/README =================================================================== --- trunk/varnish-cache/README 2010-05-06 13:03:50 UTC (rev 4782) +++ trunk/varnish-cache/README 2010-05-07 09:23:29 UTC (rev 4783) @@ -1,14 +1,10 @@ This is the Varnish high-performance HTTP accelerator. Documentation and additional information about Varnish is available on -the following web sites: +http://www.varnish-cache.org/ - http://www.varnish-cache.org/ Official web site - - http://varnish.projects.linpro.no/ Developer site and wiki - Technical questions about Varnish and this release should be addressed -to . +to . Questions about commercial support and services related to Varnish -should be addressed to . +should be addressed to . From andersb at varnish-cache.org Sat May 8 13:10:11 2010 From: andersb at varnish-cache.org (andersb at varnish-cache.org) Date: Sat, 8 May 2010 15:10:11 +0200 Subject: r4784 - trunk/varnish-cache/doc/sphinx/faq Message-ID: Author: andersb Date: 2010-05-08 15:10:09 +0200 (Sat, 08 May 2010) New Revision: 4784 Modified: trunk/varnish-cache/doc/sphinx/faq/index.rst Log: Started some work on the FAQ. Split into different sections and ported some of the old faq from wikisyntax into ReST.\nNo real plan behind the split, just adding sections that seem useful to have as I go through the old FAQ. Modified: trunk/varnish-cache/doc/sphinx/faq/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/index.rst 2010-05-07 09:23:29 UTC (rev 4783) +++ trunk/varnish-cache/doc/sphinx/faq/index.rst 2010-05-08 13:10:09 UTC (rev 4784) @@ -1,484 +1,21 @@ - .. _faq: -Frequently Asked Varnish Questions -================================== +%%%%%%%%%%%%%%%%%%%%%%%%%% +Frequently asked questions +%%%%%%%%%%%%%%%%%%%%%%%%%% -Varnish does... ---------------- +The most frequently asked questions about Varnish in different contexts. -... Varnish does not cache things, all requests hit the backend - The number one cause is cookies, the ```default.vcl``` will - not cache anything if the request has a ```Cookie:``` header - or the if the backend sends a ```Set-Cookie:``` header. +.. toctree:: - Number two cause is authentication, same thing. + general.rst + http.rst + configuration.rst + logging.rst + old-faq.rst -How... ------- - -... How much RAM/Disk do I need for Varnish ? - - That depends on pretty much everything. - - I think our best current guidance is that you go for a cost-effective - RAM configuration, something like 1-16GB, and a SSD disk. - - Unless you positively know that you will need it, there is - little point in spendng a fortune on a hand-sewn motherboard - that can fit several TB in special RAM blocks, rivetet together - by leftover watch-makers in Switzerland. - - On the other hand, if you plot your traffic in Gb/s, you probably - need all the RAM you can afford/get. - -... How can I limit Varnish to use less RAM ? - - You can not. Varnish operates in Virtual Memory and it is up to the - kernel to decide which process gets to use how much RAM to map the - virtual address-space of the process. - - -Where... --------- - -... Can I find varnish for my operating system ? - - We know that Varnish has been packaged for Debian, Ubuntu, RHEL, - Centos, (Open)SuSE, Gentoo and FreeBSD, possibly more. Check whatever - packagemanager you use. - -Can I... --------- - -... Can I use Varnish as a client-side proxy ? - - No. Varnish needs all backends configured in the VCL. Look at squid - instead. - -... Can I run Varnish on a 32bit system ? - - Yes, recently somebody even claimed to run Varnish on his N900 mobile - phone recently, but if you have the choice, go 64 bit from the start. - - Varnish is written to use Virtual Memory and on a 32bit system that - really cramps your style, and you will have trouble configuring more - than 2 GB of storage. - -... Can I run Varnish on the same system as Apache ? - - Yes, and many people do that with good success. - - There will be competition for resources, but Apache is not particular - good at using RAM effectively and Varnish is, so this synergy usually - more than compensates for the competition. - -... Can I run multiple Varnish on the same system ? - - Yes, as long as you give them different TCP ports and different ```-n``` - arguments, you will be fine. - - -... Can I cache multiple vhosts with one Varnish ? - - Yes, that works right out of the box. - -... Can I see what is cached in Varnish ? - - That is not possible for several reasons. A command to list - all the contents of a Varnish cache with millions of objects would - bring your Varnish to a standstill while it traverses the index. - - Besides, the output is a lot less useful than you might think. - -... Can I use Varnish to do HTTPS ? - - Not at present, and while we keep an eye on this, there are no - current plans to add HTTPS support, until we can find a way where - it adds significant value, relative to running a stand-alone - HTTPS proxy such as ngnix or pound. - -... Can Varnish load balance between multiple backends ? - - Yes, you need VCL code like this:: - - director foobar round-robin { - { .backend = { .host = "www1.example.com; .port = "http"; } } - { .backend = { .host = "www2.example.com; .port = "http"; } } - } - - sub vcl_recv { - set req.backend = foobar; - } - - (XXX: reference to docs, once written) - -Why ... -------- - -... Why are regular expressions case-sensitive ? - - Some HTTP headers, such as ```Host:``` and ```Location:``` - contain FQDN's which by definition is not case-sensitive. Other - HTTP headers are case-sensitive, most notably the URLs. Therefore - a "one size fits all" solution is not possible. - - In previous releases, we used the POSIX regular expressions - supplied with the operating system, and decided, because the - most common use of regexps were on ```Host:``` headers, that - they should not be case-sensitive. - - From version 2.1.0 and forward, we use PCRE regular expressions, - where it *is* possible to control case-sensitivity in the - individual regular expressions, so we decided that it would - probably confuse people if we made the default case-insentive. - (We promise not to change our minds about this again.) - - To make a PCRE regex case insensitive, put ```(?i)``` at the start:: - - if (req.http.host ~ "?iexample.com$") { - ... - } - - See the [http://www.pcre.org/pcre.txt PCRE man pages] for more information. - - -... Why does the ```Via:``` header say 1.1 in Varnish 2.1.x ? - - The number in the ```Via:``` header is the HTTP protocol version - supported/applied, not the softwares version number. - -... Why did you call it *Varnish* ? - - Long story, but basically the instigator of Varnish spent a long - time staring at an art-poster with the word "Vernisage" and ended - up checking it in a dictionary, which gives the following three - meanings of the word: - - r.v. var?nished, var?nish?ing, var?nish?es - - 1. To cover with varnish. - 2. To give a smooth and glossy finish to. - 3. To give a deceptively attractive appearance to; gloss over. - - The three point describes happens to your backend system when you - put Varnish in front of it. - .. todo:: + [V] - Split FAQ up in different sections -The old Trac FAQ, still to be reformattet: ------------------------------------------- - -:: - - === Does Varnish support compression? === - - This is a simple question with a complicated answer; see [wiki:FAQ/Compression]. - - === Where can I find the log files? === - - Varnish does not log to a file, but to shared memory log. Use the varnishlog utility to print the shared memory log or varnishncsa to present it in the Apache / NCSA "combined" log format. - - === What is the purpose of the X-Varnish HTTP header? === - - The X-Varnish HTTP header allows you to find the correct log-entries for the transaction. For a cache hit, X-Varnish will contain both the ID of the current request and the ID of the request that populated the cache. It makes debugging Varnish a lot easier. - - == Configuration == - - == VCL == - - === How do I load VCL file while Varnish is running? === - - 1. Place the VCL file on the server - 1. Telnet into the managment port. - 1. do a "vcl.load " in managment interface. is whatever you would like to call your new configuration. - 1. do a "vcl.use " to start using your new config. - - - === Does Varnish require the system to have a C compiler? === - - Yes. The VCL compiler generates C source as output, and uses the systems C-compiler to compile that into a shared library. If there is no C compiler, Varnish will not work. - - === ... Isn't that security problem? === - - The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer. - - === What is a VCL file? === - - VCL is an acronym for Varnish Configuration Language. In a VCL file, you configure how Varnish should behave. Sample VCL files will be included in this Wiki at a later stage. - - === Where is the documentation on VCL? === - - Please see "man 7 vcl". - - === Should I use ''pipe'' or ''pass'' in my VCL code? What is the difference? === - - When varnish does a ''pass'' it acts like a normal HTTP proxy. It - reads the request and pushes it onto the backend. The next HTTP - request can then be handled like any other. - - ''pipe'' is only used when Varnish for some reason can't handle the - ''pass''. ''pipe'' reads the request, pushes in onty the backend - _only_ pushes bytes back and forth, with no other actions taken. - - Since most HTTP clients do pipeline several requests into one - connection this might give you an undesirable result - as every - subsequent request will reuse the existing ''pipe''. - - Varnish versions prior to 2.0 does not support handling a request body - with ''pass'' mode, so in those releases ''pipe'' is required for - correct handling. - - In 2.0 and later, ''pass'' will handle the request body correctly. - - If you get 503 errors when making a request which is ''pass''ed, make sure - that you're specifying the backend before returning from vcl_recv with ''pass''. - - - === Are regular expressions case sensitive or not? Can I change it? === - - In 2.1 and newer, regular expressions are case sensitive by default. In earlier versions, they were case insensitive. - - To change this for a single regex in 2.1, use "(?i)" at the start. See the [http://www.pcre.org/pcre.txt PCRE man pages] for more information. - - == How do I... == - - === How can I force a refresh on a object cached by varnish? === - - Refreshing is often called [http://dictionary.reference.com/browse/PURGE purging] a document. You can purge at least 2 different ways in Varnish: - - 1. From the command line you can write: - - {{{ - url.purge ^/$ - }}} - - to purge your '''/''' document. As you might see url.purge takes an - [http://en.wikipedia.org/wiki/Regular_expression regular expression] - as its argument. Hence the !^ and $ at the front and end. If the !^ is ommited, all the documents ending in a / in the cache would be deleted. - - So to delete all the documents in the cache, write: - - {{{ - url.purge .* - }}} - - at the command line. - - 2. HTTP PURGE - - VCL code to allow HTTP PURGE [wiki:VCLExamples is to be found here]. Note that this method does not support wildcard purging. - - === How can I debug the requests of a single client? === - - The "varnishlog" utility may produce a horrendous amount of output. To be able debug our own traffic can be useful. - - The ReqStart token will include the client IP address. To see log entries matching this, type: - - {{{ - $ varnishlog -c -o ReqStart 192.0.2.123 - }}} - - To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. - - At the shell command line, type: - {{{ - $ varnishlog -b -o TxHeader 192.0.2.123 - }}} - - === How can I rewrite URLS before they are sent to the backend? === - - You can use the "regsub()" function to do this. Here's an example for zope, to rewrite URL's for the virtualhostmonster: - - {{{ - if (req.http.host ~ "^(www.)?example.com") { - set req.url = regsub(req.url, "^", "/VirtualHostBase/http/example.com:80/Sites/example.com/VirtualHostRoot"); - } - - }}} - - === I have a site with many hostnames, how do I keep them from multiplying the cache? === - - You can do this by normalizing the "Host" header for all your hostnames. Here's a VCL example: - - {{{ - if (req.http.host ~ "^(www.)?example.com") { - set req.http.host = "example.com"; - } - }}} - - === How can I log the client IP address on the backend? === - - All I see is the IP address of the varnish server. How can I log the client IP address? - - We will need to add the IP address to a header used for the backend request, and configure the backend to log the content of this header instead of the address of the connecting client (which is the varnish server). - - Varnish configuration: - {{{ - sub vcl_recv { - # Add a unique header containing the client address - remove req.http.X-Forwarded-For; - set req.http.X-Forwarded-For = client.ip; - # [...] - } - }}} - - For the apache configuration, we copy the "combined" log format to a new one we call "varnishcombined", for instance, and change the client IP field to use the content of the variable we set in the varnish configuration: - {{{ - LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" varnishcombined - }}} - - And so, in our virtualhost, you need to specify this format instead of "combined" (or "common", or whatever else you use) - {{{ - - ServerName www.example.com - # [...] - CustomLog /var/log/apache2/www.example.com/access.log varnishcombined - # [...] - - }}} - - The [http://www.openinfo.co.uk/apache/index.html mod_extract_forwarded Apache module] might also be useful. - - === How do I add a HTTP header? === - - To add a HTTP header, unless you want to add something about the client/request, it is best done in vcl_fetch as this means it will only be processed every time the object is fetched: - - {{{ - sub vcl_fetch { - # Add a unique header containing the cache servers IP address: - remove obj.http.X-Varnish-IP; - set obj.http.X-Varnish-IP = server.ip; - # Another header: - set obj.http.Foo = "bar"; - } - }}} - - === How do I do to alter the request going to the backend? === - You can use the ''bereq'' object for altering requests going to the backend but from my experience you can only 'set' values to it. - So, if you need to change the requested URL, '''this doesn't work''': - - {{{ - sub vcl_miss { - set bereq.url = regsub(bereq.url,"stream/","/"); - fetch; - } - }}} - - Because you cannot read from bereq.url (in the value part of the assignment). You will get: - {{{ - mgt_run_cc(): failed to load compiled VCL program: - ./vcl.1P9zoqAU.o: undefined symbol: VRT_r_bereq_url - VCL compilation failed - }}} - - Instead, you have to use '''req.url''': - - {{{ - sub vcl_miss { - set bereq.url = regsub(req.url,"stream/","/"); - fetch; - } - }}} - - === How do I force the backend to send Vary headers? === - - We have anectdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers. - - It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header. - - The following should be sufficient for most cases: - - {{{ - Accept: */* - Accept-Language: * - Accept-Charset: * - Accept-Encoding: identity - }}} - - Note that Accept-Encoding can not be set to *, as the backend might then send back a compressed response which the client would be unable to process. - - This can of course be implemented in VCL. - - === How can I customize the error messages that Varnish returns? === - - A custom error page can be generated by adding a vcl_error to your configuration file. The default error page looks like this: - - {{{ - sub vcl_error { - set obj.http.Content-Type = "text/html; charset=utf-8"; - - synthetic {" - - - - - "} obj.status " " obj.response {" - - -

Error "} obj.status " " obj.response {"

-

"} obj.response {"

-

Guru Meditation:

-

XID: "} req.xid {"

-
Varnish
- - - "}; - deliver; - } - }}} - - === How do I instruct varnish to ignore the query parameters and only cache one instance of an object? === - - This can be achieved by removing the query parameters using a regexp: - - {{{ - sub vcl_recv { - set req.url = regsub(req.url, "\?.*", ""); - } - }}} - - === Do you have any examples? === - - Many varnish users have contributed [wiki:VCLExamples examples] from their VCLs to solve common problems. A ready made recipe to address your question may be included. - - == Troubleshooting == - - === Why does it look like Varnish sends all requests to the backend? I thought it was a cache? === - - There are 2 common reasons for this: - 1. The object's '''ttl expired'''. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s). - 2. Your site uses '''cookies''': - * By default, varnish will not cache ''responses'' from the backend that come with a '''Set-Cookie''': header. - * By default, varnish will not serve ''requests'' with a '''Cookie:''' header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely. - - === Why am I getting a cache hit, but a request is still going to my backend? === - - Varnish has a feature called ''hit for pass'', which is used when Varnish gets a response from the backend and finds out it cannot be cached. In such cases, Varnish will create a cache object that records that fact, so that the next request goes directly to "pass". - See the entry above for common cases where a backend returns a non-cacheable object. See this [wiki:VCLExampleDefault graphical overview] of how the Varnish request cycle works. - - Since Varnish bundles multiple requests for the same URL to the backend, a common case where a client will get a ''hit for pass'' is: - * Client 1 requests url /foo - * Client 2..N request url /foo - * Varnish tasks a worker to fetch /foo for Client 1 - * Client 2..N are now queued pending response from the worker - * Worker returns object to varnish which turns out to be non-cacheable. - * Client 2..N are now given the ''hit for pass'' object instructing them to go to the backend - - The ''hit for pass'' object will stay cached for the duration of it's ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the ''hit for pass'' object exists. - The [wiki:StatsExplained varnishstat program] can tell you how many ''hit for pass'' objects varnish has served. You can lower the ttl for such an object if '''you are sure this is needed''', using the following logic: - - {{{ - sub vcl_fetch { - if (!obj.cacheable) { - # Limit the lifetime of all 'hit for pass' objects to 10 seconds - obj.ttl = 10s; - pass; - } - } - - }}} - From andersb at varnish-cache.org Sat May 8 13:11:56 2010 From: andersb at varnish-cache.org (andersb at varnish-cache.org) Date: Sat, 8 May 2010 15:11:56 +0200 Subject: r4785 - trunk/varnish-cache/doc/sphinx/faq Message-ID: Author: andersb Date: 2010-05-08 15:11:55 +0200 (Sat, 08 May 2010) New Revision: 4785 Added: trunk/varnish-cache/doc/sphinx/faq/configuration.rst trunk/varnish-cache/doc/sphinx/faq/general.rst trunk/varnish-cache/doc/sphinx/faq/http.rst trunk/varnish-cache/doc/sphinx/faq/logging.rst trunk/varnish-cache/doc/sphinx/faq/old-faq.rst Log: Started some work on the FAQ. Split into different sections and ported some of the old faq from wikisyntax into ReST.\nNo real plan behind the split, just adding sections that seem useful to have as I go through the old FAQ. Added: trunk/varnish-cache/doc/sphinx/faq/configuration.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/configuration.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/faq/configuration.rst 2010-05-08 13:11:55 UTC (rev 4785) @@ -0,0 +1,25 @@ +%%%%%%%%%%%%%%% +Configuration +%%%%%%%%%%%%%%% + +VCL +=== + +**What is VCL?** + +VCL is an acronym for Varnish Configuration Language. In a VCL file, you configure how Varnish should behave. Sample VCL files will be included in this Wiki at a later stage. + +**Where is the documentation on VCL?** + +We are working on documenting VCL. The `WIKI `_ contains some examples. + +Please also see "man 7 vcl". + + +**How do I load VCL file while Varnish is running?** + +* Place the VCL file on the server +* Telnet into the managment port. +* do a "vcl.load " in managment interface. is whatever you would like to call your new configuration. +* do a "vcl.use " to start using your new config. + Added: trunk/varnish-cache/doc/sphinx/faq/general.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/general.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-08 13:11:55 UTC (rev 4785) @@ -0,0 +1,184 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +General questions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +What is Varnish? +================ + +Varnish is a state-of-the-art, high-performance `web accelerator `_. It uses the advanced features in Linux 2.6, FreeBSD 6/7 and Solaris 10 to achieve its high performance. + +Some of the features include + +* A modern design +* VCL - a very flexible configuration language +* Load balancing with health checking of backends +* Partial support for ESI +* URL rewriting +* Graceful handling of "dead" backends + +Features to come (Experimental): + +* Support for Ranged headers +* Support for persistent cache + + +Varnish is free software and is licenced under a modified BSD licence. Please read the introduction to get started with Varnish. + + +How... +====== + +... How much RAM/Disk do I need for Varnish ? + + That depends on pretty much everything. + + I think our best current guidance is that you go for a cost-effective + RAM configuration, something like 1-16GB, and a SSD disk. + + Unless you positively know that you will need it, there is + little point in spendng a fortune on a hand-sewn motherboard + that can fit several TB in special RAM blocks, rivetet together + by leftover watch-makers in Switzerland. + + On the other hand, if you plot your traffic in Gb/s, you probably + need all the RAM you can afford/get. + +... How can I limit Varnish to use less RAM ? + + You can not. Varnish operates in Virtual Memory and it is up to the + kernel to decide which process gets to use how much RAM to map the + virtual address-space of the process. + + +Where... +======== + +... Can I find varnish for my operating system ? + + We know that Varnish has been packaged for Debian, Ubuntu, RHEL, + Centos, (Open)SuSE, Gentoo and FreeBSD, possibly more. Check whatever + packagemanager you use. + +Can I... +======== + +... Can I use Varnish as a client-side proxy ? + + No. Varnish needs all backends configured in the VCL. Look at squid + instead. + +... Can I run Varnish on a 32bit system ? + + Yes, recently somebody even claimed to run Varnish on his N900 mobile + phone recently, but if you have the choice, go 64 bit from the start. + + Varnish is written to use Virtual Memory and on a 32bit system that + really cramps your style, and you will have trouble configuring more + than 2 GB of storage. + +... Can I run Varnish on the same system as Apache ? + + Yes, and many people do that with good success. + + There will be competition for resources, but Apache is not particular + good at using RAM effectively and Varnish is, so this synergy usually + more than compensates for the competition. + +... Can I run multiple Varnish on the same system ? + + Yes, as long as you give them different TCP ports and different ```-n``` + arguments, you will be fine. + + +... Can I cache multiple vhosts with one Varnish ? + + Yes, that works right out of the box. + +... Can I see what is cached in Varnish ? + + That is not possible for several reasons. A command to list + all the contents of a Varnish cache with millions of objects would + bring your Varnish to a standstill while it traverses the index. + + Besides, the output is a lot less useful than you might think. + +... Can I use Varnish to do HTTPS ? + + Not at present, and while we keep an eye on this, there are no + current plans to add HTTPS support, until we can find a way where + it adds significant value, relative to running a stand-alone + HTTPS proxy such as ngnix or pound. + +... Can Varnish load balance between multiple backends ? + + Yes, you need VCL code like this:: + + director foobar round-robin { + { .backend = { .host = "www1.example.com; .port = "http"; } } + { .backend = { .host = "www2.example.com; .port = "http"; } } + } + + sub vcl_recv { + set req.backend = foobar; + } + + (XXX: reference to docs, once written) + +Why ... +======= + +... Varnish does not cache things, all requests hit the backend + + The number one cause is cookies, the ```default.vcl``` will + not cache anything if the request has a ```Cookie:``` header + or the if the backend sends a ```Set-Cookie:``` header. + + Number two cause is authentication, same thing. + +... Why are regular expressions case-sensitive ? + + Some HTTP headers, such as ```Host:``` and ```Location:``` + contain FQDN's which by definition is not case-sensitive. Other + HTTP headers are case-sensitive, most notably the URLs. Therefore + a "one size fits all" solution is not possible. + + In previous releases, we used the POSIX regular expressions + supplied with the operating system, and decided, because the + most common use of regexps were on ```Host:``` headers, that + they should not be case-sensitive. + + From version 2.1.0 and forward, we use PCRE regular expressions, + where it *is* possible to control case-sensitivity in the + individual regular expressions, so we decided that it would + probably confuse people if we made the default case-insentive. + (We promise not to change our minds about this again.) + + To make a PCRE regex case insensitive, put ```(?i)``` at the start:: + + if (req.http.host ~ "?iexample.com$") { + ... + } + + See the [http://www.pcre.org/pcre.txt PCRE man pages] for more information. + + +... Why does the ```Via:``` header say 1.1 in Varnish 2.1.x ? + + The number in the ```Via:``` header is the HTTP protocol version + supported/applied, not the softwares version number. + +... Why did you call it *Varnish* ? + + Long story, but basically the instigator of Varnish spent a long + time staring at an art-poster with the word "Vernisage" and ended + up checking it in a dictionary, which gives the following three + meanings of the word: + + r.v. var?nished, var?nish?ing, var?nish?es + + 1. To cover with varnish. + 2. To give a smooth and glossy finish to. + 3. To give a deceptively attractive appearance to; gloss over. + + The three point describes happens to your backend system when you + put Varnish in front of it. Added: trunk/varnish-cache/doc/sphinx/faq/http.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/http.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/faq/http.rst 2010-05-08 13:11:55 UTC (rev 4785) @@ -0,0 +1,55 @@ +%%%%%%%%%%% +HTTP +%%%%%%%%%%% + +**What is the purpose of the X-Varnish HTTP header?** + +The X-Varnish HTTP header allows you to find the correct log-entries for the transaction. For a cache hit, X-Varnish will contain both the ID of the current request and the ID of the request that populated the cache. It makes debugging Varnish a lot easier. + +**Does Varnish support compression?** + +This is a simple question with a complicated answer; see `WIKI `_. + +**How do I add a HTTP header?** + +To add a HTTP header, unless you want to add something about the client/request, it is best done in vcl_fetch as this means it will only be processed every time the object is fetched:: + + sub vcl_fetch { + # Add a unique header containing the cache servers IP address: + remove obj.http.X-Varnish-IP; + set obj.http.X-Varnish-IP = server.ip; + # Another header: + set obj.http.Foo = "bar"; + } + +**How can I log the client IP address on the backend?** + +All I see is the IP address of the varnish server. How can I log the client IP address? + +We will need to add the IP address to a header used for the backend request, and configure the backend to log the content of this header instead of the address of the connecting client (which is the varnish server). + +Varnish configuration:: + + sub vcl_recv { + # Add a unique header containing the client address + remove req.http.X-Forwarded-For; + set req.http.X-Forwarded-For = client.ip; + # [...] + } + +For the apache configuration, we copy the "combined" log format to a new one we call "varnishcombined", for instance, and change the client IP field to use the content of the variable we set in the varnish configuration:: + + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" varnishcombined + +And so, in our virtualhost, you need to specify this format instead of "combined" (or "common", or whatever else you use):: + + + ServerName www.example.com + # [...] + CustomLog /var/log/apache2/www.example.com/access.log varnishcombined + # [...] + + +The [http://www.openinfo.co.uk/apache/index.html mod_extract_forwarded Apache module] might also be useful. + + Added: trunk/varnish-cache/doc/sphinx/faq/logging.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/logging.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/faq/logging.rst 2010-05-08 13:11:55 UTC (rev 4785) @@ -0,0 +1,8 @@ +%%%%%%%%%%%%%%%%%%% +Logging +%%%%%%%%%%%%%%%%%%% + +**Where can I find the log files?** + +Varnish does not log to a file, but to shared memory log. Use the varnishlog utility to print the shared memory log or varnishncsa to present it in the Apache / NCSA "combined" log format. + Added: trunk/varnish-cache/doc/sphinx/faq/old-faq.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/old-faq.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/faq/old-faq.rst 2010-05-08 13:11:55 UTC (rev 4785) @@ -0,0 +1,239 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +The old Trac FAQ, still to be reformattet: +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +:: + + + + === Does Varnish require the system to have a C compiler? === + + Yes. The VCL compiler generates C source as output, and uses the systems C-compiler to compile that into a shared library. If there is no C compiler, Varnish will not work. + + === ... Isn't that security problem? === + + The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer. + + === Should I use ''pipe'' or ''pass'' in my VCL code? What is the difference? === + + When varnish does a ''pass'' it acts like a normal HTTP proxy. It + reads the request and pushes it onto the backend. The next HTTP + request can then be handled like any other. + + ''pipe'' is only used when Varnish for some reason can't handle the + ''pass''. ''pipe'' reads the request, pushes in onty the backend + _only_ pushes bytes back and forth, with no other actions taken. + + Since most HTTP clients do pipeline several requests into one + connection this might give you an undesirable result - as every + subsequent request will reuse the existing ''pipe''. + + Varnish versions prior to 2.0 does not support handling a request body + with ''pass'' mode, so in those releases ''pipe'' is required for + correct handling. + + In 2.0 and later, ''pass'' will handle the request body correctly. + + If you get 503 errors when making a request which is ''pass''ed, make sure + that you're specifying the backend before returning from vcl_recv with ''pass''. + + + === Are regular expressions case sensitive or not? Can I change it? === + + In 2.1 and newer, regular expressions are case sensitive by default. In earlier versions, they were case insensitive. + + To change this for a single regex in 2.1, use "(?i)" at the start. See the [http://www.pcre.org/pcre.txt PCRE man pages] for more information. + + == How do I... == + + === How can I force a refresh on a object cached by varnish? === + + Refreshing is often called [http://dictionary.reference.com/browse/PURGE purging] a document. You can purge at least 2 different ways in Varnish: + + 1. From the command line you can write: + + {{{ + url.purge ^/$ + }}} + + to purge your '''/''' document. As you might see url.purge takes an + [http://en.wikipedia.org/wiki/Regular_expression regular expression] + as its argument. Hence the !^ and $ at the front and end. If the !^ is ommited, all the documents ending in a / in the cache would be deleted. + + So to delete all the documents in the cache, write: + + {{{ + url.purge .* + }}} + + at the command line. + + 2. HTTP PURGE + + VCL code to allow HTTP PURGE [wiki:VCLExamples is to be found here]. Note that this method does not support wildcard purging. + + === How can I debug the requests of a single client? === + + The "varnishlog" utility may produce a horrendous amount of output. To be able debug our own traffic can be useful. + + The ReqStart token will include the client IP address. To see log entries matching this, type: + + {{{ + $ varnishlog -c -o ReqStart 192.0.2.123 + }}} + + To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. + + At the shell command line, type: + {{{ + $ varnishlog -b -o TxHeader 192.0.2.123 + }}} + + === How can I rewrite URLS before they are sent to the backend? === + + You can use the "regsub()" function to do this. Here's an example for zope, to rewrite URL's for the virtualhostmonster: + + {{{ + if (req.http.host ~ "^(www.)?example.com") { + set req.url = regsub(req.url, "^", "/VirtualHostBase/http/example.com:80/Sites/example.com/VirtualHostRoot"); + } + + }}} + + === I have a site with many hostnames, how do I keep them from multiplying the cache? === + + You can do this by normalizing the "Host" header for all your hostnames. Here's a VCL example: + + {{{ + if (req.http.host ~ "^(www.)?example.com") { + set req.http.host = "example.com"; + } + }}} + + + === How do I do to alter the request going to the backend? === + You can use the ''bereq'' object for altering requests going to the backend but from my experience you can only 'set' values to it. + So, if you need to change the requested URL, '''this doesn't work''': + + {{{ + sub vcl_miss { + set bereq.url = regsub(bereq.url,"stream/","/"); + fetch; + } + }}} + + Because you cannot read from bereq.url (in the value part of the assignment). You will get: + {{{ + mgt_run_cc(): failed to load compiled VCL program: + ./vcl.1P9zoqAU.o: undefined symbol: VRT_r_bereq_url + VCL compilation failed + }}} + + Instead, you have to use '''req.url''': + + {{{ + sub vcl_miss { + set bereq.url = regsub(req.url,"stream/","/"); + fetch; + } + }}} + + === How do I force the backend to send Vary headers? === + + We have anectdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers. + + It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header. + + The following should be sufficient for most cases: + + {{{ + Accept: */* + Accept-Language: * + Accept-Charset: * + Accept-Encoding: identity + }}} + + Note that Accept-Encoding can not be set to *, as the backend might then send back a compressed response which the client would be unable to process. + + This can of course be implemented in VCL. + + === How can I customize the error messages that Varnish returns? === + + A custom error page can be generated by adding a vcl_error to your configuration file. The default error page looks like this: + + {{{ + sub vcl_error { + set obj.http.Content-Type = "text/html; charset=utf-8"; + + synthetic {" + + + + + "} obj.status " " obj.response {" + + +

Error "} obj.status " " obj.response {"

+

"} obj.response {"

+

Guru Meditation:

+

XID: "} req.xid {"

+
Varnish
+ + + "}; + deliver; + } + }}} + + === How do I instruct varnish to ignore the query parameters and only cache one instance of an object? === + + This can be achieved by removing the query parameters using a regexp: + + {{{ + sub vcl_recv { + set req.url = regsub(req.url, "\?.*", ""); + } + }}} + + === Do you have any examples? === + + Many varnish users have contributed [wiki:VCLExamples examples] from their VCLs to solve common problems. A ready made recipe to address your question may be included. + + == Troubleshooting == + + === Why does it look like Varnish sends all requests to the backend? I thought it was a cache? === + + There are 2 common reasons for this: + 1. The object's '''ttl expired'''. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s). + 2. Your site uses '''cookies''': + * By default, varnish will not cache ''responses'' from the backend that come with a '''Set-Cookie''': header. + * By default, varnish will not serve ''requests'' with a '''Cookie:''' header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely. + + === Why am I getting a cache hit, but a request is still going to my backend? === + + Varnish has a feature called ''hit for pass'', which is used when Varnish gets a response from the backend and finds out it cannot be cached. In such cases, Varnish will create a cache object that records that fact, so that the next request goes directly to "pass". + See the entry above for common cases where a backend returns a non-cacheable object. See this [wiki:VCLExampleDefault graphical overview] of how the Varnish request cycle works. + + Since Varnish bundles multiple requests for the same URL to the backend, a common case where a client will get a ''hit for pass'' is: + * Client 1 requests url /foo + * Client 2..N request url /foo + * Varnish tasks a worker to fetch /foo for Client 1 + * Client 2..N are now queued pending response from the worker + * Worker returns object to varnish which turns out to be non-cacheable. + * Client 2..N are now given the ''hit for pass'' object instructing them to go to the backend + + The ''hit for pass'' object will stay cached for the duration of it's ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the ''hit for pass'' object exists. + The [wiki:StatsExplained varnishstat program] can tell you how many ''hit for pass'' objects varnish has served. You can lower the ttl for such an object if '''you are sure this is needed''', using the following logic: + + {{{ + sub vcl_fetch { + if (!obj.cacheable) { + # Limit the lifetime of all 'hit for pass' objects to 10 seconds + obj.ttl = 10s; + pass; + } + } + + }}} + From andersb at varnish-cache.org Sun May 9 18:47:59 2010 From: andersb at varnish-cache.org (andersb at varnish-cache.org) Date: Sun, 9 May 2010 20:47:59 +0200 Subject: r4786 - in trunk/varnish-cache/doc/sphinx: faq installation Message-ID: Author: andersb Date: 2010-05-09 20:47:59 +0200 (Sun, 09 May 2010) New Revision: 4786 Modified: trunk/varnish-cache/doc/sphinx/faq/configuration.rst trunk/varnish-cache/doc/sphinx/faq/general.rst trunk/varnish-cache/doc/sphinx/installation/install.rst Log: Cleaned up some code in the general section. Used the ** marker for bolding the questions. Added a internal ref to the installation doc so we could use best-practice referencing. Added som stuff to FAQ/configuration. Modified: trunk/varnish-cache/doc/sphinx/faq/configuration.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/configuration.rst 2010-05-08 13:11:55 UTC (rev 4785) +++ trunk/varnish-cache/doc/sphinx/faq/configuration.rst 2010-05-09 18:47:59 UTC (rev 4786) @@ -13,7 +13,7 @@ We are working on documenting VCL. The `WIKI `_ contains some examples. -Please also see "man 7 vcl". +Please also see ``man 7 vcl``. **How do I load VCL file while Varnish is running?** Modified: trunk/varnish-cache/doc/sphinx/faq/general.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-08 13:11:55 UTC (rev 4785) +++ trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-09 18:47:59 UTC (rev 4786) @@ -28,90 +28,90 @@ How... ====== -... How much RAM/Disk do I need for Varnish ? +**How much RAM/Disk do I need for Varnish?** - That depends on pretty much everything. +That depends on pretty much everything. - I think our best current guidance is that you go for a cost-effective - RAM configuration, something like 1-16GB, and a SSD disk. +I think our best current guidance is that you go for a cost-effective +RAM configuration, something like 1-16GB, and a SSD disk. - Unless you positively know that you will need it, there is - little point in spendng a fortune on a hand-sewn motherboard - that can fit several TB in special RAM blocks, rivetet together - by leftover watch-makers in Switzerland. +Unless you positively know that you will need it, there is +little point in spendng a fortune on a hand-sewn motherboard +that can fit several TB in special RAM blocks, rivetet together +by leftover watch-makers in Switzerland. - On the other hand, if you plot your traffic in Gb/s, you probably - need all the RAM you can afford/get. +On the other hand, if you plot your traffic in Gb/s, you probably +need all the RAM you can afford/get. -... How can I limit Varnish to use less RAM ? +**How can I limit Varnish to use less RAM?** - You can not. Varnish operates in Virtual Memory and it is up to the - kernel to decide which process gets to use how much RAM to map the - virtual address-space of the process. +You can not. Varnish operates in Virtual Memory and it is up to the +kernel to decide which process gets to use how much RAM to map the +virtual address-space of the process. Where... ======== -... Can I find varnish for my operating system ? +**Can I find varnish for my operating system?** - We know that Varnish has been packaged for Debian, Ubuntu, RHEL, - Centos, (Open)SuSE, Gentoo and FreeBSD, possibly more. Check whatever - packagemanager you use. +We know that Varnish has been packaged for Debian, Ubuntu, RHEL, +Centos, (Open)SuSE, Gentoo and FreeBSD, possibly more. Check whatever +packagemanager you use. Or read :ref:`Installing Varnish on your computer `. Can I... ======== -... Can I use Varnish as a client-side proxy ? +**Can I use Varnish as a client-side proxy?** - No. Varnish needs all backends configured in the VCL. Look at squid - instead. +No. Varnish needs all backends configured in the VCL. Look at squid +instead. -... Can I run Varnish on a 32bit system ? +**Can I run Varnish on a 32bit system?** - Yes, recently somebody even claimed to run Varnish on his N900 mobile - phone recently, but if you have the choice, go 64 bit from the start. +Yes, recently somebody even claimed to run Varnish on his N900 mobile +phone recently, but if you have the choice, go 64 bit from the start. - Varnish is written to use Virtual Memory and on a 32bit system that - really cramps your style, and you will have trouble configuring more - than 2 GB of storage. +Varnish is written to use Virtual Memory and on a 32bit system that +really cramps your style, and you will have trouble configuring more +than 2 GB of storage. -... Can I run Varnish on the same system as Apache ? +**Can I run Varnish on the same system as Apache?** - Yes, and many people do that with good success. +Yes, and many people do that with good success. - There will be competition for resources, but Apache is not particular - good at using RAM effectively and Varnish is, so this synergy usually - more than compensates for the competition. +There will be competition for resources, but Apache is not particular +good at using RAM effectively and Varnish is, so this synergy usually +more than compensates for the competition. -... Can I run multiple Varnish on the same system ? +**Can I run multiple Varnish on the same system?** - Yes, as long as you give them different TCP ports and different ```-n``` - arguments, you will be fine. +Yes, as long as you give them different TCP ports and different ``-n`` +arguments, you will be fine. -... Can I cache multiple vhosts with one Varnish ? +**Can I cache multiple vhosts with one Varnish?** - Yes, that works right out of the box. +Yes, that works right out of the box. -... Can I see what is cached in Varnish ? +**Can I see what is cached in Varnish?** - That is not possible for several reasons. A command to list - all the contents of a Varnish cache with millions of objects would - bring your Varnish to a standstill while it traverses the index. +That is not possible for several reasons. A command to list +all the contents of a Varnish cache with millions of objects would +bring your Varnish to a standstill while it traverses the index. - Besides, the output is a lot less useful than you might think. +Besides, the output is a lot less useful than you might think. -... Can I use Varnish to do HTTPS ? +**Can I use Varnish to do HTTPS?** - Not at present, and while we keep an eye on this, there are no - current plans to add HTTPS support, until we can find a way where - it adds significant value, relative to running a stand-alone - HTTPS proxy such as ngnix or pound. +Not at present, and while we keep an eye on this, there are no +current plans to add HTTPS support, until we can find a way where +it adds significant value, relative to running a stand-alone +HTTPS proxy such as ngnix or pound. -... Can Varnish load balance between multiple backends ? +**Can Varnish load balance between multiple backends?** - Yes, you need VCL code like this:: +Yes, you need VCL code like this:: director foobar round-robin { { .backend = { .host = "www1.example.com; .port = "http"; } } @@ -127,58 +127,72 @@ Why ... ======= -... Varnish does not cache things, all requests hit the backend +**Why does it look like Varnish sends all requests to the backend? I thought it was a cache?** - The number one cause is cookies, the ```default.vcl``` will - not cache anything if the request has a ```Cookie:``` header - or the if the backend sends a ```Set-Cookie:``` header. +There are 2 common reasons for this: + 1. The object's ``ttl expired``. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s). + 2. Your site uses ``cookies``: + * By default, varnish will not cache ``responses`` from the backend that come with a ``Set-Cookie``: header. + * By default, varnish will not serve ``requests`` with a ``Cookie:`` header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely. - Number two cause is authentication, same thing. -... Why are regular expressions case-sensitive ? +**Why are regular expressions case-sensitive?** - Some HTTP headers, such as ```Host:``` and ```Location:``` - contain FQDN's which by definition is not case-sensitive. Other - HTTP headers are case-sensitive, most notably the URLs. Therefore - a "one size fits all" solution is not possible. +Some HTTP headers, such as ``Host:`` and ``Location:`` +contain FQDN's which by definition is not case-sensitive. Other +HTTP headers are case-sensitive, most notably the URLs. Therefore +a "one size fits all" solution is not possible. - In previous releases, we used the POSIX regular expressions - supplied with the operating system, and decided, because the - most common use of regexps were on ```Host:``` headers, that - they should not be case-sensitive. +In previous releases, we used the POSIX regular expressions +supplied with the operating system, and decided, because the +most common use of regexps were on ```Host:``` headers, that +they should not be case-sensitive. - From version 2.1.0 and forward, we use PCRE regular expressions, - where it *is* possible to control case-sensitivity in the - individual regular expressions, so we decided that it would - probably confuse people if we made the default case-insentive. - (We promise not to change our minds about this again.) +From version 2.1.0 and forward, we use PCRE regular expressions, +where it *is* possible to control case-sensitivity in the +individual regular expressions, so we decided that it would +probably confuse people if we made the default case-insentive. +(We promise not to change our minds about this again.) - To make a PCRE regex case insensitive, put ```(?i)``` at the start:: +To make a PCRE regex case insensitive, put ``(?i)`` at the start:: if (req.http.host ~ "?iexample.com$") { ... } - See the [http://www.pcre.org/pcre.txt PCRE man pages] for more information. +See the `PCRE man pages `_ for more information. -... Why does the ```Via:``` header say 1.1 in Varnish 2.1.x ? +**Why does the ``Via:`` header say 1.1 in Varnish 2.1.x?** - The number in the ```Via:``` header is the HTTP protocol version - supported/applied, not the softwares version number. +The number in the ``Via:`` header is the HTTP protocol version +supported/applied, not the softwares version number. -... Why did you call it *Varnish* ? +**Why did you call it *Varnish*?** - Long story, but basically the instigator of Varnish spent a long - time staring at an art-poster with the word "Vernisage" and ended - up checking it in a dictionary, which gives the following three - meanings of the word: +Long story, but basically the instigator of Varnish spent a long +time staring at an art-poster with the word "Vernisage" and ended +up checking it in a dictionary, which gives the following three +meanings of the word: - r.v. var?nished, var?nish?ing, var?nish?es +r.v. var?nished, var?nish?ing, var?nish?es 1. To cover with varnish. 2. To give a smooth and glossy finish to. 3. To give a deceptively attractive appearance to; gloss over. - The three point describes happens to your backend system when you - put Varnish in front of it. +The three point describes happens to your backend system when you +put Varnish in front of it. + +How... +====== + +**How do I instruct varnish to ignore the query parameters and only cache one instance of an object?** + +This can be achieved by removing the query parameters using a regexp:: + + sub vcl_recv { + set req.url = regsub(req.url, "\?.*", ""); + } + + Modified: trunk/varnish-cache/doc/sphinx/installation/install.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-08 13:11:55 UTC (rev 4785) +++ trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-09 18:47:59 UTC (rev 4786) @@ -1,3 +1,5 @@ +.. _install-doc: + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Installing Varnish on your computer %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -27,6 +29,7 @@ **CentOS/RedHat 5.4 - RPM files** We try to keep the lastest version available as prebuildt RPMs (el4 & el5) on `SourceForge `_. + Varnish is included in the `EPEL `_ repository. **BUT** unfortunatly we had a syntax change in Varnish 2.0.6->2.1.X. This means that we can not update Varnish in `EPEL `_ so the latest version there is Varnish 2.0.6. In the future (EPEL6) we should be available with Varnish 2.1.X or higher. **Debian/Ubuntu - DEB files** From andersb at varnish-cache.org Sun May 9 23:15:20 2010 From: andersb at varnish-cache.org (andersb at varnish-cache.org) Date: Mon, 10 May 2010 01:15:20 +0200 Subject: r4787 - trunk/varnish-cache/doc/sphinx/faq Message-ID: Author: andersb Date: 2010-05-10 01:15:19 +0200 (Mon, 10 May 2010) New Revision: 4787 Modified: trunk/varnish-cache/doc/sphinx/faq/configuration.rst trunk/varnish-cache/doc/sphinx/faq/general.rst trunk/varnish-cache/doc/sphinx/faq/old-faq.rst Log: More clean-up. Modified: trunk/varnish-cache/doc/sphinx/faq/configuration.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/configuration.rst 2010-05-09 18:47:59 UTC (rev 4786) +++ trunk/varnish-cache/doc/sphinx/faq/configuration.rst 2010-05-09 23:15:19 UTC (rev 4787) @@ -2,6 +2,8 @@ Configuration %%%%%%%%%%%%%%% +.. _faq-vcl: + VCL === @@ -23,3 +25,28 @@ * do a "vcl.load " in managment interface. is whatever you would like to call your new configuration. * do a "vcl.use " to start using your new config. +**Should I use ''pipe'' or ''pass'' in my VCL code? What is the difference?** + +When varnish does a ``pass`` it acts like a normal HTTP proxy. It +reads the request and pushes it onto the backend. The next HTTP +request can then be handled like any other. + +``pipe`` is only used when Varnish for some reason can't handle the +``pass``. ``pipe`` reads the request, pushes in onty the backend +_only_ pushes bytes back and forth, with no other actions taken. + +Since most HTTP clients do pipeline several requests into one +connection this might give you an undesirable result - as every +subsequent request will reuse the existing ``pipe``. + +Varnish versions prior to 2.0 does not support handling a request body +with ``pass`` mode, so in those releases ``pipe`` is required for +correct handling. + +In 2.0 and later, ``pass`` will handle the request body correctly. + +If you get 503 errors when making a request which is ``pass`` ed, make sure +that you're specifying the backend before returning from vcl_recv with ``pass``. + + + Modified: trunk/varnish-cache/doc/sphinx/faq/general.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-09 18:47:59 UTC (rev 4786) +++ trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-09 23:15:19 UTC (rev 4787) @@ -49,7 +49,53 @@ kernel to decide which process gets to use how much RAM to map the virtual address-space of the process. +**How do I instruct varnish to ignore the query parameters and only cache one instance of an object?** +This can be achieved by removing the query parameters using a regexp:: + + sub vcl_recv { + set req.url = regsub(req.url, "\?.*", ""); + } + +**How can I force a refresh on a object cached by varnish?** + +Refreshing is often called `purging `_ a document. You can purge at least 2 different ways in Varnish: + +1. Command line + + From the command line you can write:: + + url.purge ^/$ + + to purge your / document. As you might see url.purge takes an `regular expression `_ + as its argument. Hence the ``^`` and ``$`` at the front and end. If the ``^`` is ommited, all the documents ending in a ``/`` in the cache would be deleted. + + So to delete all the documents in the cache, write:: + + url.purge .* + + at the command line. + +2. HTTP PURGE + + VCL code to allow HTTP PURGE is to be found `here `_. Note that this method does not support wildcard purging. + +**How can I debug the requests of a single client?** + +The "varnishlog" utility may produce a horrendous amount of output. To be able debug our own traffic can be useful. + +The ReqStart token will include the client IP address. To see log entries matching this, type:: + + $ varnishlog -c -o ReqStart 192.0.2.123 + +To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. + +At the shell command line, type:: + + $ varnishlog -b -o TxHeader 192.0.2.123 + + + Where... ======== @@ -162,7 +208,15 @@ See the `PCRE man pages `_ for more information. +**Are regular expressions case sensitive or not? Can I change it?** +In 2.1 and newer, regular expressions are case sensitive by default. In earlier versions, they were case insensitive. + +To change this for a single regex in 2.1, use ``(?i)`` at the start. + +See the `PCRE man pages `_ for more information. + + **Why does the ``Via:`` header say 1.1 in Varnish 2.1.x?** The number in the ``Via:`` header is the HTTP protocol version @@ -184,15 +238,10 @@ The three point describes happens to your backend system when you put Varnish in front of it. -How... -====== +**Why does Varnish require the system to have a C compiler?** -**How do I instruct varnish to ignore the query parameters and only cache one instance of an object?** +The :ref:`VCL ` compiler generates C source as output (your config file), and uses the systems C-compiler to compile that into a shared library. If there is no C compiler, Varnish will not work. -This can be achieved by removing the query parameters using a regexp:: +**Isn't that security problem?** - sub vcl_recv { - set req.url = regsub(req.url, "\?.*", ""); - } - - +The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer. Modified: trunk/varnish-cache/doc/sphinx/faq/old-faq.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/old-faq.rst 2010-05-09 18:47:59 UTC (rev 4786) +++ trunk/varnish-cache/doc/sphinx/faq/old-faq.rst 2010-05-09 23:15:19 UTC (rev 4787) @@ -6,89 +6,8 @@ - === Does Varnish require the system to have a C compiler? === - - Yes. The VCL compiler generates C source as output, and uses the systems C-compiler to compile that into a shared library. If there is no C compiler, Varnish will not work. - - === ... Isn't that security problem? === - - The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer. - - === Should I use ''pipe'' or ''pass'' in my VCL code? What is the difference? === - - When varnish does a ''pass'' it acts like a normal HTTP proxy. It - reads the request and pushes it onto the backend. The next HTTP - request can then be handled like any other. - - ''pipe'' is only used when Varnish for some reason can't handle the - ''pass''. ''pipe'' reads the request, pushes in onty the backend - _only_ pushes bytes back and forth, with no other actions taken. - - Since most HTTP clients do pipeline several requests into one - connection this might give you an undesirable result - as every - subsequent request will reuse the existing ''pipe''. - - Varnish versions prior to 2.0 does not support handling a request body - with ''pass'' mode, so in those releases ''pipe'' is required for - correct handling. - - In 2.0 and later, ''pass'' will handle the request body correctly. - - If you get 503 errors when making a request which is ''pass''ed, make sure - that you're specifying the backend before returning from vcl_recv with ''pass''. - - - === Are regular expressions case sensitive or not? Can I change it? === - - In 2.1 and newer, regular expressions are case sensitive by default. In earlier versions, they were case insensitive. - - To change this for a single regex in 2.1, use "(?i)" at the start. See the [http://www.pcre.org/pcre.txt PCRE man pages] for more information. - == How do I... == - === How can I force a refresh on a object cached by varnish? === - - Refreshing is often called [http://dictionary.reference.com/browse/PURGE purging] a document. You can purge at least 2 different ways in Varnish: - - 1. From the command line you can write: - - {{{ - url.purge ^/$ - }}} - - to purge your '''/''' document. As you might see url.purge takes an - [http://en.wikipedia.org/wiki/Regular_expression regular expression] - as its argument. Hence the !^ and $ at the front and end. If the !^ is ommited, all the documents ending in a / in the cache would be deleted. - - So to delete all the documents in the cache, write: - - {{{ - url.purge .* - }}} - - at the command line. - - 2. HTTP PURGE - - VCL code to allow HTTP PURGE [wiki:VCLExamples is to be found here]. Note that this method does not support wildcard purging. - - === How can I debug the requests of a single client? === - - The "varnishlog" utility may produce a horrendous amount of output. To be able debug our own traffic can be useful. - - The ReqStart token will include the client IP address. To see log entries matching this, type: - - {{{ - $ varnishlog -c -o ReqStart 192.0.2.123 - }}} - - To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. - - At the shell command line, type: - {{{ - $ varnishlog -b -o TxHeader 192.0.2.123 - }}} - === How can I rewrite URLS before they are sent to the backend? === You can use the "regsub()" function to do this. Here's an example for zope, to rewrite URL's for the virtualhostmonster: From tfheen at varnish-cache.org Mon May 10 11:11:41 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Mon, 10 May 2010 13:11:41 +0200 Subject: r4788 - in trunk/varnish-cache: bin/varnishtest/tests lib/libvcl Message-ID: Author: tfheen Date: 2010-05-10 13:11:41 +0200 (Mon, 10 May 2010) New Revision: 4788 Modified: trunk/varnish-cache/bin/varnishtest/tests/b00030.vtc trunk/varnish-cache/lib/libvcl/vcc_string.c Log: Add missing bits for string version of "now" Modified: trunk/varnish-cache/bin/varnishtest/tests/b00030.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/b00030.vtc 2010-05-09 23:15:19 UTC (rev 4787) +++ trunk/varnish-cache/bin/varnishtest/tests/b00030.vtc 2010-05-10 11:11:41 UTC (rev 4788) @@ -1,4 +1,6 @@ # $Id$ +# We can't test the value of x-timestamp, but this should fail +# if we can't set the header at all. test "Test formatting of timestamps" @@ -11,6 +13,9 @@ sub vcl_recv { error 200 req.grace; } + sub vcl_error { + set obj.http.x-timestamp = now; + } } -start client c1 { Modified: trunk/varnish-cache/lib/libvcl/vcc_string.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_string.c 2010-05-09 23:15:19 UTC (rev 4787) +++ trunk/varnish-cache/lib/libvcl/vcc_string.c 2010-05-10 11:11:41 UTC (rev 4788) @@ -153,6 +153,11 @@ return (vcc_regsub(tl, 0)); if (tl->t->tok == ID && vcc_IdIs(tl->t, "regsuball")) return (vcc_regsub(tl, 1)); + if (tl->t->tok == ID && vcc_IdIs(tl->t, "now")) { + Fb(tl, 0, "VRT_time_string(sp, VRT_r_now(sp))"); + vcc_NextToken(tl); + return 1; + } if (tl->t->tok == VAR) { vp = vcc_FindVar(tl, tl->t, vcc_vars, 0, "read"); if (tl->err) From andersb at varnish-cache.org Mon May 10 19:32:20 2010 From: andersb at varnish-cache.org (andersb at varnish-cache.org) Date: Mon, 10 May 2010 21:32:20 +0200 Subject: r4789 - trunk/varnish-cache/doc/sphinx/faq Message-ID: Author: andersb Date: 2010-05-10 21:32:20 +0200 (Mon, 10 May 2010) New Revision: 4789 Removed: trunk/varnish-cache/doc/sphinx/faq/old-faq.rst Modified: trunk/varnish-cache/doc/sphinx/faq/general.rst trunk/varnish-cache/doc/sphinx/faq/index.rst Log: reStructure'ized the old Trac faq and sorted the stuff into their temporary sections. Deleted old-faq.rst from filesystem and updated the index.rst. Now it's time to qualitycheck the content and remove old examples and outdated info. Modified: trunk/varnish-cache/doc/sphinx/faq/general.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-10 11:11:41 UTC (rev 4788) +++ trunk/varnish-cache/doc/sphinx/faq/general.rst 2010-05-10 19:32:20 UTC (rev 4789) @@ -94,8 +94,83 @@ $ varnishlog -b -o TxHeader 192.0.2.123 +**How can I rewrite URLS before they are sent to the backend?** +You can use the ``regsub()`` function to do this. Here's an example for zope, to rewrite URL's for the virtualhostmonster:: + if (req.http.host ~ "^(www.)?example.com") { + set req.url = regsub(req.url, "^", "/VirtualHostBase/http/example.com:80/Sites/example.com/VirtualHostRoot"); + } + +**I have a site with many hostnames, how do I keep them from multiplying the cache?** + +You can do this by normalizing the ``Host`` header for all your hostnames. Here's a VCL example:: + + if (req.http.host ~ "^(www.)?example.com") { + set req.http.host = "example.com"; + } + +**How do I do to alter the request going to the backend?** +You can use the ``bereq`` object for altering requests going to the backend, but you can only 'set' values to it. Therefore use req.url to build the request:: + + sub vcl_miss { + set bereq.url = regsub(req.url,"stream/","/"); + fetch; + } + +**How do I force the backend to send Vary headers?** + +We have anectdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers. + +It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header. + +The following should be sufficient for most cases:: + + Accept: */* + Accept-Language: * + Accept-Charset: * + Accept-Encoding: identity + +Note that Accept-Encoding can not be set to ``*``, as the backend might then send back a compressed response which the client would be unable to process. + +This can of course be implemented in VCL. + +**How can I customize the error messages that Varnish returns?** + +A custom error page can be generated by adding a ``vcl_error`` to your configuration file. The default error page looks like this:: + + sub vcl_error { + set obj.http.Content-Type = "text/html; charset=utf-8"; + + synthetic {" + + + + + "} obj.status " " obj.response {" + + +

Error "} obj.status " " obj.response {"

+

"} obj.response {"

+

Guru Meditation:

+

XID: "} req.xid {"

+
Varnish
+ + + "}; + deliver; + } + +**How do I instruct varnish to ignore the query parameters and only cache one instance of an object?** + +This can be achieved by removing the query parameters using a regexp:: + + sub vcl_recv { + set req.url = regsub(req.url, "\?.*", ""); + } + + Where... ======== @@ -245,3 +320,30 @@ **Isn't that security problem?** The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer. + +Troubleshooting +=============== + +**Why am I getting a cache hit, but a request is still going to my backend?** + +Varnish has a feature called **hit for pass**, which is used when Varnish gets a response from the backend and finds out it cannot be cached. In such cases, Varnish will create a cache object that records that fact, so that the next request goes directly to "pass". + + Since Varnish bundles multiple requests for the same URL to the backend, a common case where a client will get a **hit for pass** is: + * Client 1 requests url /foo + * Client 2..N request url /foo + * Varnish tasks a worker to fetch /foo for Client 1 + * Client 2..N are now queued pending response from the worker + * Worker returns object to varnish which turns out to be non-cacheable. + * Client 2..N are now given the **hit for pass** object instructing them to go to the backend + +The **hit for pass** object will stay cached for the duration of it's ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the **hit for pass** object exists. +The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. You can lower the ttl for such an object if you are sure this is needed, using the following logic:: + + sub vcl_fetch { + if (!obj.cacheable) { + # Limit the lifetime of all 'hit for pass' objects to 10 seconds + obj.ttl = 10s; + pass; + } + } + Modified: trunk/varnish-cache/doc/sphinx/faq/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/index.rst 2010-05-10 11:11:41 UTC (rev 4788) +++ trunk/varnish-cache/doc/sphinx/faq/index.rst 2010-05-10 19:32:20 UTC (rev 4789) @@ -13,9 +13,8 @@ http.rst configuration.rst logging.rst - old-faq.rst .. todo:: - [V] - Split FAQ up in different sections + [V] - Now that the FAQ has been all reStructureized, we need to qualitycheck the content, update old info, and delete unrelevant info. Deleted: trunk/varnish-cache/doc/sphinx/faq/old-faq.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/faq/old-faq.rst 2010-05-10 11:11:41 UTC (rev 4788) +++ trunk/varnish-cache/doc/sphinx/faq/old-faq.rst 2010-05-10 19:32:20 UTC (rev 4789) @@ -1,158 +0,0 @@ -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -The old Trac FAQ, still to be reformattet: -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -:: - - - - == How do I... == - - === How can I rewrite URLS before they are sent to the backend? === - - You can use the "regsub()" function to do this. Here's an example for zope, to rewrite URL's for the virtualhostmonster: - - {{{ - if (req.http.host ~ "^(www.)?example.com") { - set req.url = regsub(req.url, "^", "/VirtualHostBase/http/example.com:80/Sites/example.com/VirtualHostRoot"); - } - - }}} - - === I have a site with many hostnames, how do I keep them from multiplying the cache? === - - You can do this by normalizing the "Host" header for all your hostnames. Here's a VCL example: - - {{{ - if (req.http.host ~ "^(www.)?example.com") { - set req.http.host = "example.com"; - } - }}} - - - === How do I do to alter the request going to the backend? === - You can use the ''bereq'' object for altering requests going to the backend but from my experience you can only 'set' values to it. - So, if you need to change the requested URL, '''this doesn't work''': - - {{{ - sub vcl_miss { - set bereq.url = regsub(bereq.url,"stream/","/"); - fetch; - } - }}} - - Because you cannot read from bereq.url (in the value part of the assignment). You will get: - {{{ - mgt_run_cc(): failed to load compiled VCL program: - ./vcl.1P9zoqAU.o: undefined symbol: VRT_r_bereq_url - VCL compilation failed - }}} - - Instead, you have to use '''req.url''': - - {{{ - sub vcl_miss { - set bereq.url = regsub(req.url,"stream/","/"); - fetch; - } - }}} - - === How do I force the backend to send Vary headers? === - - We have anectdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers. - - It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header. - - The following should be sufficient for most cases: - - {{{ - Accept: */* - Accept-Language: * - Accept-Charset: * - Accept-Encoding: identity - }}} - - Note that Accept-Encoding can not be set to *, as the backend might then send back a compressed response which the client would be unable to process. - - This can of course be implemented in VCL. - - === How can I customize the error messages that Varnish returns? === - - A custom error page can be generated by adding a vcl_error to your configuration file. The default error page looks like this: - - {{{ - sub vcl_error { - set obj.http.Content-Type = "text/html; charset=utf-8"; - - synthetic {" - - - - - "} obj.status " " obj.response {" - - -

Error "} obj.status " " obj.response {"

-

"} obj.response {"

-

Guru Meditation:

-

XID: "} req.xid {"

-
Varnish
- - - "}; - deliver; - } - }}} - - === How do I instruct varnish to ignore the query parameters and only cache one instance of an object? === - - This can be achieved by removing the query parameters using a regexp: - - {{{ - sub vcl_recv { - set req.url = regsub(req.url, "\?.*", ""); - } - }}} - - === Do you have any examples? === - - Many varnish users have contributed [wiki:VCLExamples examples] from their VCLs to solve common problems. A ready made recipe to address your question may be included. - - == Troubleshooting == - - === Why does it look like Varnish sends all requests to the backend? I thought it was a cache? === - - There are 2 common reasons for this: - 1. The object's '''ttl expired'''. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s). - 2. Your site uses '''cookies''': - * By default, varnish will not cache ''responses'' from the backend that come with a '''Set-Cookie''': header. - * By default, varnish will not serve ''requests'' with a '''Cookie:''' header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely. - - === Why am I getting a cache hit, but a request is still going to my backend? === - - Varnish has a feature called ''hit for pass'', which is used when Varnish gets a response from the backend and finds out it cannot be cached. In such cases, Varnish will create a cache object that records that fact, so that the next request goes directly to "pass". - See the entry above for common cases where a backend returns a non-cacheable object. See this [wiki:VCLExampleDefault graphical overview] of how the Varnish request cycle works. - - Since Varnish bundles multiple requests for the same URL to the backend, a common case where a client will get a ''hit for pass'' is: - * Client 1 requests url /foo - * Client 2..N request url /foo - * Varnish tasks a worker to fetch /foo for Client 1 - * Client 2..N are now queued pending response from the worker - * Worker returns object to varnish which turns out to be non-cacheable. - * Client 2..N are now given the ''hit for pass'' object instructing them to go to the backend - - The ''hit for pass'' object will stay cached for the duration of it's ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the ''hit for pass'' object exists. - The [wiki:StatsExplained varnishstat program] can tell you how many ''hit for pass'' objects varnish has served. You can lower the ttl for such an object if '''you are sure this is needed''', using the following logic: - - {{{ - sub vcl_fetch { - if (!obj.cacheable) { - # Limit the lifetime of all 'hit for pass' objects to 10 seconds - obj.ttl = 10s; - pass; - } - } - - }}} - From phk at varnish-cache.org Mon May 10 19:57:57 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 10 May 2010 21:57:57 +0200 Subject: r4790 - in trunk/varnish-cache/doc/sphinx: . installation phk Message-ID: Author: phk Date: 2010-05-10 21:57:57 +0200 (Mon, 10 May 2010) New Revision: 4790 Added: trunk/varnish-cache/doc/sphinx/phk/ trunk/varnish-cache/doc/sphinx/phk/autocrap.rst trunk/varnish-cache/doc/sphinx/phk/index.rst trunk/varnish-cache/doc/sphinx/phk/sphinx.rst Modified: trunk/varnish-cache/doc/sphinx/index.rst trunk/varnish-cache/doc/sphinx/installation/install.rst Log: Move my rants into their own sandbox Modified: trunk/varnish-cache/doc/sphinx/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/index.rst 2010-05-10 19:32:20 UTC (rev 4789) +++ trunk/varnish-cache/doc/sphinx/index.rst 2010-05-10 19:57:57 UTC (rev 4790) @@ -6,10 +6,11 @@ Welcome to Varnish's documentation! =================================== -We are making a fresh start on the documentation, our previous attempts -have utterly failed, but my discovery of Sphinx_/reStructuredText_ as tools -for writing documentation gives me hope that we have finally found a -platform, that we can kick ourselves to actually use. +Arnold's Laws of Documentation: + (1) If it should exist, it doesn't. + (2) If it does exist, it's out of date. + (3) Only documentation for useless programs transcends the + first two laws. Contents: @@ -20,6 +21,7 @@ tutorial/index.rst reference/index.rst faq/index.rst + phk/index.rst glossary/index.rst Indices and tables @@ -28,82 +30,3 @@ * :ref:`genindex` * :ref:`modindex` * :ref:`search` - -Why Sphinx_ and reStructuredText_ ? -=================================== - -The first school of thought on documentation, is the one we subscribe -to in Varnish right now: "Documentation schmocumentation..." It does -not work for anybody. - -The second school is the "Write a {La}TeX document" school, where -the documentation is seen as a stand alone product, which is produced -independently. This works great for PDF output, and sucks royally -for HTML and TXT output. - -The third school is the "Literate programming" school, which abandons -readability of *both* the program source code *and* the documentation -source, which seems to be one of the best access protections -one can put on the source code of either. - -The fourth school is the "DoxyGen" school, which lets a program -collect a mindless list of hyperlinked variable, procedure, class -and filenames, and call that "documentation". - -And the fifth school is anything that uses a fileformat that -cannot be put into a version control system, because it is -binary and non-diff'able. It doesn't matter if it is -OpenOffice, LyX or Word, a non-diffable doc source is a no go -with programmers. - -Quite frankly, none of these works very well in practice. - -One of the very central issues, is that writing documentation must -not become a big and clear context-switch from programming. That -precludes special graphical editors, browser-based (wiki!) formats -etc. - -Yes, if you write documentation for half your workday, that works, -but if you write code most of your workday, that does not work. -Trust me on this, I have 25 years of experience avoiding using such -tools. - -I found one project which has thought radically about the problem, -and their reasoning is interesting, and quite attractive to me: - -#. TXT files are the lingua franca of computers, even if - you are logged with TELNET using IP over Avian Carriers - (Which is more widespread in Norway than you would think) - you can read documentation in a .TXT format. - -#. TXT is the most restrictive typographical format, so - rather than trying to neuter a high-level format into .TXT, - it is smarter to make the .TXT the source, and reinterpret - it structurally into the more capable formats. - -In other words: we are talking about the ReStructuredText_ of the -Python project, as wrapped by the Sphinx_ project. - -Unless there is something I have totally failed to spot, that is -going to be the new documentation platform in Varnish. - -Take a peek at the Python docs, and try pressing the "show source" -link at the bottom of the left menu: - -(link to random python doc page:) - - http://docs.python.org/py3k/reference/expressions.html - -Dependency wise, that means you can edit docs with no special -tools, you need python+docutils+sphinx to format HTML and a LaTex -(pdflatex ?) to produce PDFs, something I only expect to happen -on the project server on a regular basis. - -I can live with that, I might even rewrite the VCC scripts -from Tcl to Python in that case. - -Poul-Henning, 2010-04-11 - - -.. _Sphinx: http://sphinx.pocoo.org/ -.. _reStructuredText: http://docutils.sourceforge.net/rst.html Modified: trunk/varnish-cache/doc/sphinx/installation/install.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-10 19:32:20 UTC (rev 4789) +++ trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-10 19:57:57 UTC (rev 4790) @@ -98,81 +98,3 @@ make install .. _SubVersion: http://subversion.tigris.org/ - - -Did you call them *autocrap* tools ? -==================================== - -Yes, in fact I did, because they are the worst possible non-solution -to a self-inflicted problem. - -Back in the 1980'ies, the numerous mini- and micro-computer companies -all jumped on the UNIX band-wagon, because it gave them an operating -system for their hardware, but they also tried to "distinguish" themselves -from the competitors, by "adding value". - -That "value" was incompatibility. - -You never knew where they put stuff, what arguments the compiler needed -to behave sensibly, or for that matter, if there were a compiler to begin -with. - -So some deranged imagination, came up with the idea of the ``configure`` -script, which sniffed at your system and set up a ``Makefile`` that would -work. - -Writing configure scripts was hard work, for one thing you needed a ton -of different systems to test them on, so copy&paste became the order of -the day. - -Then some even more deranged imagination, came up with the idea of -writing a script for writing configure scripts, and in an amazing -and daring attempt at the "all time most deranged" crown, used an -obscure and insufferable macro-processor called ``m4`` for the -implementation. - -Now, as it transpires, writing the specification for the configure -producing macros was tedious, so somebody wrote a tool to... - -...do you detect the pattern here ? - -Now, if the result of all this crap, was that I could write my source-code -and tell a tool where the files were, and not only assume, but actually -*trust* that things would just work out, then I could live with it. - -But as it transpires, that is not the case. For one thing, all the -autocrap tools add another layer of version-madness you need to get -right before you can even think about compiling the source code. - -Second, it doesn't actually work, you still have to do the hard work -and figure out the right way to explain to the autocrap tools what -you are trying to do and how to do it, only you have to do so in -a language which is used to produce M4 macro invocations etc. etc. - -In the meantime, the UNIX diversity has shrunk from 50+ significantly -different dialects to just a handful: Linux, \*BSD, Solaris and AIX -and the autocrap tools have become part of the portability problem, -rather than part of the solution. - -Amongst the silly activites of the autocrap generated configure script -in Varnish are: - -* Looks for ANSI-C header files (show me a system later - than 1995 without them ?) - -* Existence and support for POSIX mandated symlinks, (which - are not used by Varnish btw.) - -* Tests, 19 different ways, that the compiler is not a relic from - SYS III days. (Find me just one SYS III running computer with - an ethernet interface ?) - -* Checks if the ISO-C and POSIX mandated ``cos()`` function exists - in ``libm`` (No, I have no idea either...) - -&c. &c. &c. - -Some day when I have the time, I will rip out all the autocrap stuff -and replace it with a 5 line shellscript that calls ``uname -s``. - -Poul-Henning, 2010-04-20 Added: trunk/varnish-cache/doc/sphinx/phk/autocrap.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/phk/autocrap.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/phk/autocrap.rst 2010-05-10 19:57:57 UTC (rev 4790) @@ -0,0 +1,79 @@ +.. _phk_autocrap: + +==================================== +Did you call them *autocrap* tools ? +==================================== + +Yes, in fact I did, because they are the worst possible non-solution +to a self-inflicted problem. + +Back in the 1980'ies, the numerous mini- and micro-computer companies +all jumped on the UNIX band-wagon, because it gave them an operating +system for their hardware, but they also tried to "distinguish" themselves +from the competitors, by "adding value". + +That "value" was incompatibility. + +You never knew where they put stuff, what arguments the compiler needed +to behave sensibly, or for that matter, if there were a compiler to begin +with. + +So some deranged imagination, came up with the idea of the ``configure`` +script, which sniffed at your system and set up a ``Makefile`` that would +work. + +Writing configure scripts was hard work, for one thing you needed a ton +of different systems to test them on, so copy&paste became the order of +the day. + +Then some even more deranged imagination, came up with the idea of +writing a script for writing configure scripts, and in an amazing +and daring attempt at the "all time most deranged" crown, used an +obscure and insufferable macro-processor called ``m4`` for the +implementation. + +Now, as it transpires, writing the specification for the configure +producing macros was tedious, so somebody wrote a tool to... + +...do you detect the pattern here ? + +Now, if the result of all this crap, was that I could write my source-code +and tell a tool where the files were, and not only assume, but actually +*trust* that things would just work out, then I could live with it. + +But as it transpires, that is not the case. For one thing, all the +autocrap tools add another layer of version-madness you need to get +right before you can even think about compiling the source code. + +Second, it doesn't actually work, you still have to do the hard work +and figure out the right way to explain to the autocrap tools what +you are trying to do and how to do it, only you have to do so in +a language which is used to produce M4 macro invocations etc. etc. + +In the meantime, the UNIX diversity has shrunk from 50+ significantly +different dialects to just a handful: Linux, \*BSD, Solaris and AIX +and the autocrap tools have become part of the portability problem, +rather than part of the solution. + +Amongst the silly activites of the autocrap generated configure script +in Varnish are: + +* Looks for ANSI-C header files (show me a system later + than 1995 without them ?) + +* Existence and support for POSIX mandated symlinks, (which + are not used by Varnish btw.) + +* Tests, 19 different ways, that the compiler is not a relic from + SYS III days. (Find me just one SYS III running computer with + an ethernet interface ?) + +* Checks if the ISO-C and POSIX mandated ``cos()`` function exists + in ``libm`` (No, I have no idea either...) + +&c. &c. &c. + +Some day when I have the time, I will rip out all the autocrap stuff +and replace it with a 5 line shellscript that calls ``uname -s``. + +Poul-Henning, 2010-04-20 Added: trunk/varnish-cache/doc/sphinx/phk/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/phk/index.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/phk/index.rst 2010-05-10 19:57:57 UTC (rev 4790) @@ -0,0 +1,13 @@ +.. _phk: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Poul-Hennings random outbursts +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +You may or may not want to know what Poul-Henning think. + +.. toctree:: + + sphinx.rst + autocrap.rst + Added: trunk/varnish-cache/doc/sphinx/phk/sphinx.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/phk/sphinx.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/phk/sphinx.rst 2010-05-10 19:57:57 UTC (rev 4790) @@ -0,0 +1,81 @@ +.. _phk_sphinx: + +=================================== +Why Sphinx_ and reStructuredText_ ? +=================================== + +The first school of thought on documentation, is the one we subscribe +to in Varnish right now: "Documentation schmocumentation..." It does +not work for anybody. + +The second school is the "Write a {La}TeX document" school, where +the documentation is seen as a stand alone product, which is produced +independently. This works great for PDF output, and sucks royally +for HTML and TXT output. + +The third school is the "Literate programming" school, which abandons +readability of *both* the program source code *and* the documentation +source, which seems to be one of the best access protections +one can put on the source code of either. + +The fourth school is the "DoxyGen" school, which lets a program +collect a mindless list of hyperlinked variable, procedure, class +and filenames, and call that "documentation". + +And the fifth school is anything that uses a fileformat that +cannot be put into a version control system, because it is +binary and non-diff'able. It doesn't matter if it is +OpenOffice, LyX or Word, a non-diffable doc source is a no go +with programmers. + +Quite frankly, none of these works very well in practice. + +One of the very central issues, is that writing documentation must +not become a big and clear context-switch from programming. That +precludes special graphical editors, browser-based (wiki!) formats +etc. + +Yes, if you write documentation for half your workday, that works, +but if you write code most of your workday, that does not work. +Trust me on this, I have 25 years of experience avoiding using such +tools. + +I found one project which has thought radically about the problem, +and their reasoning is interesting, and quite attractive to me: + +#. TXT files are the lingua franca of computers, even if + you are logged with TELNET using IP over Avian Carriers + (Which is more widespread in Norway than you would think) + you can read documentation in a .TXT format. + +#. TXT is the most restrictive typographical format, so + rather than trying to neuter a high-level format into .TXT, + it is smarter to make the .TXT the source, and reinterpret + it structurally into the more capable formats. + +In other words: we are talking about the ReStructuredText_ of the +Python project, as wrapped by the Sphinx_ project. + +Unless there is something I have totally failed to spot, that is +going to be the new documentation platform in Varnish. + +Take a peek at the Python docs, and try pressing the "show source" +link at the bottom of the left menu: + +(link to random python doc page:) + + http://docs.python.org/py3k/reference/expressions.html + +Dependency wise, that means you can edit docs with no special +tools, you need python+docutils+sphinx to format HTML and a LaTex +(pdflatex ?) to produce PDFs, something I only expect to happen +on the project server on a regular basis. + +I can live with that, I might even rewrite the VCC scripts +from Tcl to Python in that case. + +Poul-Henning, 2010-04-11 + + +.. _Sphinx: http://sphinx.pocoo.org/ +.. _reStructuredText: http://docutils.sourceforge.net/rst.html From tfheen at varnish-cache.org Fri May 14 09:50:02 2010 From: tfheen at varnish-cache.org (tfheen at varnish-cache.org) Date: Fri, 14 May 2010 11:50:02 +0200 Subject: r4791 - trunk/varnish-cache/lib/libvarnish Message-ID: Author: tfheen Date: 2010-05-14 11:50:01 +0200 (Fri, 14 May 2010) New Revision: 4791 Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am Log: Quieten git/svn/make a bit harder. Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2010-05-10 19:57:57 UTC (rev 4790) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2010-05-14 09:50:01 UTC (rev 4791) @@ -36,7 +36,7 @@ DISTCLEANFILES = svn_version.c svn_version.c: FORCE - V="$$(git log -n 1 --pretty=format:%h 2>/dev/null || LANG=C svnversion -n $(top_srcdir))" \ + V="$$(sh -c "git log -n 1 --pretty=format:%h" 2>/dev/null || LANG=C svnversion -n $(top_srcdir))" \ H="$$(head -n 1 svn_version.c 2>/dev/null || true)"; \ [ "$$V" = "exported" ] && [ -e svn_version.c ] && exit 0 ; \ if [ "/* $$V */" != "$$H" ]; then \ From perbu at varnish-cache.org Sat May 15 11:07:58 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Sat, 15 May 2010 13:07:58 +0200 Subject: r4792 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-15 13:07:58 +0200 (Sat, 15 May 2010) New Revision: 4792 Added: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst trunk/varnish-cache/doc/sphinx/tutorial/backend_servers.rst trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/logging.rst trunk/varnish-cache/doc/sphinx/tutorial/putting_varnish_on_port_80.rst trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst Removed: trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst trunk/varnish-cache/doc/sphinx/tutorial/vtc_files/ Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst trunk/varnish-cache/doc/sphinx/tutorial/intro.rst Log: New tutorial Added: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,148 @@ +Advanced Backend configuration +---------------- + +At some point you might need Varnish to cache content from several +servers. You might want Varnish to map all the URL into one single +host or not. There are lot of options. + +Lets say we need to introduce a Java application into out PHP web +site. Lets say our Java application should handle URL beginning with +/java/. + +We manage to get the thing up and running on port 8000. Now, lets have +a look a default.vcl. + +backend default { + .host = "127.0.0.1"; + .port = "8080"; +} + +We add a new backend. + +backend java { + .host = "127.0.0.1"; + .port = "8000"; +} + +Now we need tell where to send the difference URL. Lets look at vcl_recv: + + +sub vcl_recv { + if (req.url ~ "^/java/") { + set req.backend = java; + } else { + set req.backend = default. + } +} + +It's quite simple, really. Lets stop and think about this for a +moment. As you can see you can define how you choose backends based on +really arbitrary data. You want to send mobile devices to a different +backend? No problem. if (req.User-agent ~ /mobile/) .... should do the +trick. + +Directors +~~~~~~~~~~ + +You can also group several backend into a group of backends. These +groups are called directors. This will give you increased performance +and resillience. You can define several backends and group them +together in a director.:: + + backend server1 { + .host = "192.168.0.10"; + } +:: + backend server2{ + .host = "192.168.0.10"; + } + + +Now we create the director.:: + director example_director round-robin { + { + .backend = server1; + } + # server2 + { + .backend = server2; + } + # foo + } + + +This director is a round-robin director. This means the director will +distribute the incomming requests on a round-robin basis. There is +also a *random* director which distributes requests in a, you guessed +it, random fashion. + +But what if one of your servers goes down? Can Varnish direct all the +requests to the healthy server? Sure it can. This is where the Health +Checks come into play. + +Health checks +~~~~~~~~~~~~ + +Lets set up a director with two backends and health checks. First lets +define the backends.:: + + backend server1 { + .host = "server1.example.com"; + .probe = { + .url = "/"; + .interval = 5s; + .timeout = 1 s; + .window = 5; + .threshold = 3; + } + } +:: + backend server2 { + .host = "server2.example.com"; + .probe = { + .url = "/"; + .interval = 5s; + .timeout = 1 s; + .window = 5; + .threshold = 3; + } + } + +Whats new here is the probe. Varnish will check the health of each +backend with a probe. The options are + +url + What URL should varnish request. + +interval + How often should we poll + +timeout + What is the timeout of the probe + +window + Varnish will maintain a *sliding window* of the results. Here the + window has five checks. + +threshold + How many of the .window last polls must be good for the backend to be declared healthy. + +XXX: Ref to reference guide. + +Now we define the director.:: + +director example_director round-robin { + { + .backend = server1; + } + # server2 + { + .backend = server2; + } + + } + +You use this director just as you would use any other director or +backend. Varnish will not send traffic to hosts that are marked as +unhealty. + Added: trunk/varnish-cache/doc/sphinx/tutorial/backend_servers.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/backend_servers.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/backend_servers.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,39 @@ +.. _tutorial-backend_servers: + +Backend servers +--------------- + +Varnish has a concept of "backend" or "origin" servers. A backend +server is the server providing the content Varnish will accelerate. + +Our first task is to tell Varnish where it can find its content. Start +your favorite text editor and open the varnish default configuration +file. If you installed from source this is +/usr/local/etc/varnish/default.vcl, if you installed from a package it +is probably /etc/varnish/default.vcl. + +Somewhere in the top there will be a section that looks a bit like this.:: + + # backend default { + # .host = "127.0.0.1"; + # .port = "8080"; + # } + +We comment in this bit of text and change the port setting from 8080 +to 80, making the text look like.:: + + backend default { + .host = "127.0.0.1"; + .port = "80"; + } + +Now, this piece of configuration defines a backend in Varnish called +*default*. When Varnish needs to get content from this backend it will +connect to port 80 on localhost (127.0.0.1). + +Varnish can have several backends defined and can you can even join +several backends together into clusters of backends for load balancing +purposes. + +Now that we have the basic Varnish configuration done, let us start up +Varnish on port 8080 so we can do some fundamental testing on it. Added: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,36 @@ +Achieving a high hitrate +======================== + +Now that Varnish is up and running, and you can access your web +application through Varnish we probably need to do some changes to +either the configuration or the application so you'll get a high +hitrate in varnish. + +HTTP Headers +------------ + +Cache-control +~~~~~~~~~~~~~ + +Cookies +~~~~~~~ + +Vary +~~~~ + +Authentication +~~~~~~~~~~~~~~ + +Normalizing your namespace +-------------------------- + +.. _tutorial-increasing_your_hitrate-purging: +Purging +------- + + +HTTP Purges +~~~~~~~~~~~ + +Bans +~~~~ Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-14 09:50:01 UTC (rev 4791) +++ trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -1,17 +1,21 @@ -.. _Tutorial: +.. _tutorial-index: %%%%%%%%%%%%%%%% Varnish Tutorial %%%%%%%%%%%%%%%% -Welcome to the Varnish Tutorial, we hope this will help you get to -know and understand Varnish. - .. toctree:: intro.rst - tut001.rst - tut002.rst + backend_servers.rst + starting_varnish.rst + logging.rst + putting_varnish_on_port_80.rst + vcl.rst + statistics.rst + increasing_your_hitrate.rst + advanced_backend_servers.rst + troubleshooting.rst .. todo:: starting varnish with -d, seeing a transaction go through Modified: trunk/varnish-cache/doc/sphinx/tutorial/intro.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/intro.rst 2010-05-14 09:50:01 UTC (rev 4791) +++ trunk/varnish-cache/doc/sphinx/tutorial/intro.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -2,50 +2,20 @@ Introduction %%%%%%%%%%%% -Most tutorials are written in "subject-order", as the old Peanuts -strip goes:: +Varnish is a web accelerator. It is installed in frond of your web +application and it caches the reponses, making your web site run Varnish +is fast, flexible and easy to use. - Jogging: A Handbook - Author: S. Noopy - Chapter 1: Left foot - It was a dark and stormy night... +This tutorial does not go through every bit of functionality Varnish +has. It will give you a good overview of what Varnish does and how it +is done. -This is great when the reader has no choice, and nothing better to -do, but read the entire document before starting. +We assume you have a web server and a web application up and running +and that you want to accelerate this application with Varnish. -We have taken the other approach: "breadth-first", because experience -has shown us that Varnish users wants to get things running, and then -polish up things later on. +Furthermore we assume you have read the :ref:`Installation` and that +it is installed with the default configuration. -With that in mind, we have written the tutorial so you can break off, -as Calvin tells Ms. Wormwood, "when my brain is full for today", and -come back later and learn more. -That also means that right from the start, we will have several -things going on in parallel and you will need at least four, sometimes -more, terminal windows at the same time, to run the examples. -A word about TCP ports ----------------------- -We have subverted our custom built regression test tool, a program -called ```varnishtest``` to help you get through these examples, -without having to waste too much time setting up webservers as -backends or browsers as clients, to drive the examples. - -But there is one complication we can not escape: TCP port numbers. - -Each of the backends we simulate and the varnishd instances we run -needs a TCP port number to listen to and it is your job to find them, -because we have no idea what servers are running on your computer -nor what TCP ports they use. - -To make this as easy as possible, we have implemented a ```-L -number``` argument to all the varnish programs, which puts them in -"Learner" mode, and in all the examples we have used 20000 as -the number, because on most systems the middle of the range -(1000...65000) is usually not used. - -If these ports are in use on your system (is your colleague also running -the Varnish tutorial ?) simply pick another number and use that -instead of 20000. Added: trunk/varnish-cache/doc/sphinx/tutorial/logging.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/logging.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/logging.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,76 @@ + +Logging in Varnish +------------------ + +One of the really nice features in Varnish is how logging +works. Instead of logging to normal log file Varnish logs to a shared +memory segment. When the end of the segment is reached we start over, +overwriting old data. This is much, much faster then logging to a file +and it doesn't require disk space. + +The flip side is that is you forget to have program actually write the +logs to disk they will disappear. + +varnishlog is one of the programs you can use to look at what Varnish +is logging. Varnishlog gives you the raw logs, everything that is +written to the logs. There are other clients as well, we'll show you +these later. + +In the terminal window you started varnish now type *varnishlog* and +press enter. + +You'll see lines like these scrolling slowly by.:: + + 0 CLI - Rd ping + 0 CLI - Wr 200 PONG 1273698726 1.0 + +These is the Varnish master process checking up on the caching process +to see that everything is OK. + +Now go to the browser and reload the page displaying your web +app. You'll see lines like these.:: + + 11 SessionOpen c 127.0.0.1 58912 0.0.0.0:8080 + 11 ReqStart c 127.0.0.1 58912 595005213 + 11 RxRequest c GET + 11 RxURL c / + 11 RxProtocol c HTTP/1.1 + 11 RxHeader c Host: localhost:8080 + 11 RxHeader c Connection: keep-alive + +The first column is an arbitrary number, it defines the request. Lines +with the same number are part of the same HTTP transaction. The second +column is the *tag* of the log message. All log entries are tagged +with a tag indicating what sort of activity is beeing logged. Tags +starting with Rx indicate Varnish is recieving data and Tx indicates +sending data. + +The third column tell us whether this is is data comming or going to +the client (c) or to/from the backend (b). The forth column is the +data being logged. + +Now, you can filter quite a bit with varnishlog. The basics option you +want to know are: + +-b + Only show log lines from traffic going between Varnish and the backend + servers. This will be useful when we want to optimize cache hit rates. + +-c + Same as -b but for client side traffic. + +-i tag + Only show lines with a certain tag. "varnishlog -i SessionOpen" + will only give you new sessions. + +-I Regex + Filter the data through a regex and only show the matching lines. To + show all cookie headers coming from the clients: + ``$ varnishlog -c -i RxHeader -I Cookie`` + +-o + Group log entries by request ID. + + +Now that Varnish seem to work OK its time to put Varnish on port 80 +while we tune it. Added: trunk/varnish-cache/doc/sphinx/tutorial/putting_varnish_on_port_80.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/putting_varnish_on_port_80.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/putting_varnish_on_port_80.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,20 @@ + +Put Varnish on port 80 +---------------------- + +If your application works OK we can now switch the ports so Varnish +will listen to port 80. Kill varnish.:: + + # pkill varnishd + +and stop your web server. Edit the configuration for your web server +and make it bind to port 8080 instead of 80. Now open the Varnish +default.vcl and change the port of the default backend to 8080. + +Start up your web server and then start varnish.:: + + # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 + +We're removed the -a option. Now Varnish will bind to the http port as +it is its default. Now try your web application and see if it works +OK. Added: trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,42 @@ +.. _tutorial-starting_varnish: +Starting Varnish +---------------- + +I assume varnishd is in your path. You might want to run ``pkill +varnishd`` to make sure Varnish isn't running. Become root and type: + +``# varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080`` + +I added a few options, lets go through them: + +``-f /usr/local/etc/varnish/default.vcl`` + The -f options specifies what configuration varnishd should use. + +``-s malloc,1G`` + The -s options chooses the storage type Varnish should use for + storing its content. I used the type *malloc*, which just uses memory + for storage. There are other backends as well, described in + :ref:tutorial-storage. 1G specifies how much memory should be allocated + - one gigabyte. + +``-T 127.0.0.1:2000`` + Varnish has a buildt in text-based administration + interface. Activating the interface makes Varnish manageble without + stopping it. You can specify what interface the management interface + should listen to. Make sure you don't expose the management interface + to the world as you can easily gain root access to a system via the + Varnish management interace. I recommend tieing it to localhost. If + you have users on your system that you don't fully trust use firewall + rules to restrict access to the interace to root only. + +``-a 0.0.0.0:8080`` + I specify that I want Varnish to listen on port 8080 for incomming + HTTP requests. For a production environment you would probably make + Varnish listen on port 80, which is the default. + +Now you have Varnish running. Let us make sure that it works +properly. Use your browser to go to http://192.168.2.2:8080/ - you +should now see your web application running there. + +Lets make sure that Varnish really does do something to your web +site. To do that we'll take a look at the logs. Added: trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,54 @@ +.. _tutorial-statistics: + + +Statistics +---------- + +Now that your varnish is up and running lets have a look at how it is +doing. There are several tools that can help. + +varnishtop +========== + +The varnishtop utility reads the shared memory logs and presents a +continuously updated list of the most commonly occurring log entries. + +With suitable filtering using the -I, -i, -X and -x options, it can be +used to display a ranking of requested documents, clients, user +agents, or any other information which is recorded in the log. + +XXX Show some nice examples here. + +varnishhist +=========== + +The varnishhist utility reads varnishd(1) shared memory logs and +presents a continuously updated histogram showing the distribution of +the last N requests by their processing. The value of N and the +vertical scale are displayed in the top left corner. The horizontal +scale is logarithmic. Hits are marked with a pipe character ("|"), +and misses are marked with a hash character ("#"). + + +varnishsizes +============ + +Varnishsizes does the same as varnishhist, except it shows the size of +the objects and not the time take to complete the request. This gives +you a good overview of how big the objects you are serving are. + + +varnishstat +=========== + +Varnish has lots of counters. We count misses, hits, information about +the storage, threads created, deleted objects. Just about +everything. varnishstat will dump these counters. This is useful when +tuning varnish. + +There are programs that can poll varnishstat regularly and make nice +graphs of these counters. One such program is Munin. Munin can be +found at http://munin-monitoring.org/ . There is a plugin for munin in +the varnish source code. + + Added: trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,50 @@ +Troubleshooting Varnish +======================= + + +When Varnish won't start +----------------------- + +Sometimes Varnish wont start. There is a pletphora of reasons why +Varnish wont start on your machine. We've seen everything from wrong +permissions on /dev/null to other processses blocking the ports. + +Starting Varnish in debug mode to see what is going on. + +Try to start varnish by:: + + # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080 -d + +Notice the -d option. It will give you some more information on what +is going on. Let us see how Varnish will react to something else +listening on its port.:: + + # varnishd -n foo -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000 -a 0.0.0.0:8080 -d + storage_malloc: max size 1024 MB. + Using old SHMFILE + Platform: Linux,2.6.32-21-generic,i686,-smalloc,-hcritbit + 200 193 + ----------------------------- + Varnish HTTP accelerator CLI. + ----------------------------- + Type 'help' for command list. + Type 'quit' to close CLI session. + Type 'start' to launch worker process. + +Now Varnish is running. Only the master process is running, in debug +mode the cache does not start. Now you're on the console. You can +instruct the master process to start the cache by issuing "start".:: + + start + bind(): Address already in use + 300 22 + Could not open sockets + +And here we have our problem. Something else is bound to the HTTP port +of Varnish. + + +Varnish is crashing +------------------- + +When varnish goes bust. Deleted: trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst 2010-05-14 09:50:01 UTC (rev 4791) +++ trunk/varnish-cache/doc/sphinx/tutorial/tut001.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -1,78 +0,0 @@ -.. _TUT001: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -TUT001: Let's see if varnishtest is working -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -In the first window, start this command:: - - varnishtest -L20000 S001.vtc - -Then in the second window, run this one:: - - varnishtest -L20000 C001.vtc - -If things work as they should, both programs spit out 5 lines, looking -like this in the first window:: - - * top TEST S001.vtc starting - ** s1 Started on 127.0.0.1:20012 - ** s1 Ending - * top RESETTING after S001.vtc - * top TEST S001.vtc completed - -and like this in the second window:: - - * top TEST C001.vtc starting - ** c1 Starting client - ** c1 Ending - * top RESETTING after C001.vtc - * top TEST C001.vtc completed - -If that did not work, please consult the XXX: troubleshooting section. - -Now, try again, but this time run the client with a couple of ``-v`` -arguments:: - - varnishtest -vv -L20000 C001.vtc - -Now the output contains a lot mode detail:: - - * top TEST C001.vtc starting - *** top client - ** c1 Starting client - *** c1 Connect to 127.0.0.1:20012 - *** c1 connected fd 3 - *** c1 txreq - **** c1 txreq| GET / HTTP/1.1\r\n - **** c1 txreq| \r\n - *** c1 rxresp - **** c1 rxhdr| HTTP/1.1 200 Ok\r\n - **** c1 rxhdr| Foo: bar\r\n - **** c1 rxhdr| Content-Length: 12\r\n - **** c1 rxhdr| \r\n - **** c1 http[ 0] | HTTP/1.1 - **** c1 http[ 1] | 200 - **** c1 http[ 2] | Ok - **** c1 http[ 3] | Foo: bar - **** c1 http[ 4] | Content-Length: 12 - **** c1 body| Hello World! - **** c1 bodylen = 12 - *** c1 closing fd 3 - ** c1 Ending - * top RESETTING after C001.vtc - * top TEST C001.vtc completed - -First the client does a ``txreq`` -- "transmit request", and you can -see the HTTP request it sends to the server, a plain boring "GET". - -Then it does a ``rxresp`` -- "receive response", and we can see the -HTTP response we get back, including the HTTP object body. - -Now try again, this time running the server in the first window with -``-vv``. - -If things do not work the way you expect, adding those ``-v`` options -is a good place to start debugging. - -Next we put a Varnish cache between the server and the client: _TUT002 Deleted: trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst 2010-05-14 09:50:01 UTC (rev 4791) +++ trunk/varnish-cache/doc/sphinx/tutorial/tut002.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -1,92 +0,0 @@ -.. _TUT002: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -TUT002: Caching an object with varnish -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -Now it is time to loose your viginity and cache an object with ``varnishd``. - -In the first window, start this command:: - - varnishd -L20000 -t30 - -You should see something like this:: - - storage_malloc: max size 1 MB. - Using old SHMFILE - Platform: FreeBSD,9.0-CURRENT,amd64,-smalloc,-hcritbit - 200 193 - ----------------------------- - Varnish HTTP accelerator CLI. - ----------------------------- - Type 'help' for command list. - Type 'quit' to close CLI session. - Type 'start' to launch worker process. - -We will explain this stuff later, for now just type ``start`` and you should -see something like:: - - child (88590) Started - 200 0 - - Child (88590) said - Child (88590) said Child starts - -Next, start a backend in a different window:: - - varnishtest -L20000 S002.vtc - -And finally, send a request from a third window:: - - varnishtest -L20000 -vv C002.vtc - -You will notice that both the backend and client windows both do -their thing, and exit back to the shell prompt. - -In the client window you will have a line like:: - - **** c1 http[ 6] | X-Varnish: 17443679 - -This is Varnish telling you that it was involved. (The exact number will -be different for you, it is just a transaction-id.) - -Now, try running *only* the client command again:: - - varnishtest -L20000 -vv C002.vtc - -Tada! You have just received a cache-hit from varnishd. - -This time the ``X-Varnish`` line will have two numbers:: - - **** c1 http[ 6] | X-Varnish: 17443680 17443679 - -The first number is the XID for this request, the second one is the -XID that brought this object into varnishd's cache, it matches the -number you saw above. - -If you run the client again, you will see:: - - **** c1 http[ 6] | X-Varnish: 17443681 17443679 - -You can keep running the client and you will get hits, but -30 seconds after the object was put in the cache, ``varnishd`` -will expire it, and then you will get varnish "Guru meditation" -message, because the backend does not respond:: - - **** c1 body| \n - **** c1 body| 503 Service Unavailable\n - **** c1 body| \n - **** c1 body| \n - **** c1 body|

Error 503 Service Unavailable

\n - **** c1 body|

Service Unavailable

\n - **** c1 body|

Guru Meditation:

\n - **** c1 body|

XID: 1685367940

\n - **** c1 body|
\n - -If you start the backend again: - - varnishtest -L20000 S002.vtc - -Then you can fetch the object for another 30 seconds. - - Added: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-15 11:07:58 UTC (rev 4792) @@ -0,0 +1,64 @@ +Varnish Configuration Language - VCL +============================== + +How ordinary configuration files work +--------------------------------- + +Varnish has a really neat configuration system. Most other systems use +configuration directives, where you basically turn on and off a bunch +of switches. + +A very common thing to do in Varnish is to override the cache headers +from our backend. Lets see how this looks in Squid, which has a +standard configuration.:: + + refresh_pattern ^http://images. 3600 20% 7200 + refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 + refresh_pattern -i (/\.jpg) 1800 10% 3600 override-expire + refresh_pattern . 0 20% 4320 + +If you are familiar with squid that probably made sense to you. But +lets point out a few weaknesses with this model. + +1) It's not intuitive. You can guess what the options mean, and you + can (and should) document it in your configuration file. + +2) Which rules have precedence? Does the last rule to match stick? Or + the first? Or does Squid try to combine all the matching rules. I + actually don't know. + +Enter VCL +--------- + +Now enter Varnish. Varnish takes your configuration file and +translates it to C code, then runs it through a compiler and loads +it. When requests come along varnish just executes the relevant +subroutines of the configuration at the relevant times. + +Varnish will execute these subroutines of code at different stages of +its work. Since its code it's execute line by line and precedence +isn't a problem. + +99% of all the changes you'll need to do will be done in two of these +subroutines. + +vcl_recv +~~~~~~~~ + +vcl_recv (yes, we're skimpy with characters, it's Unix) is called at +the beginning of a request, after the complete request has been +received and parsed. Its purpose is to decide whether or not to serve +the request, how to do it, and, if applicable, which backend to use. + +In vcl_recv you can also alter the request, dropping cookies, rewrite +headers. + + +vcl_fetch +~~~~~~~~~ + +vcl_fetch is called *after* a document has been successfully retrieved +from the backend. Normal tasks her are to alter the response headers, +trigger ESI processing, try alternate backend servers in case the +request failed. + From perbu at varnish-cache.org Sat May 15 21:50:13 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Sat, 15 May 2010 23:50:13 +0200 Subject: r4793 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-15 23:50:12 +0200 (Sat, 15 May 2010) New Revision: 4793 Modified: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/logging.rst trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst Log: cleaned up rst. added more text. Modified: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -1,5 +1,5 @@ Advanced Backend configuration ----------------- +============================== At some point you might need Varnish to cache content from several servers. You might want Varnish to map all the URL into one single @@ -10,39 +10,38 @@ /java/. We manage to get the thing up and running on port 8000. Now, lets have -a look a default.vcl. +a look a default.vcl.:: -backend default { - .host = "127.0.0.1"; - .port = "8080"; -} + backend default { + .host = "127.0.0.1"; + .port = "8080"; + } -We add a new backend. +We add a new backend.:: -backend java { - .host = "127.0.0.1"; - .port = "8000"; -} + backend java { + .host = "127.0.0.1"; + .port = "8000"; + } -Now we need tell where to send the difference URL. Lets look at vcl_recv: +Now we need tell where to send the difference URL. Lets look at vcl_recv.:: + sub vcl_recv { + if (req.url ~ "^/java/") { + set req.backend = java; + } else { + set req.backend = default. + } + } -sub vcl_recv { - if (req.url ~ "^/java/") { - set req.backend = java; - } else { - set req.backend = default. - } -} - It's quite simple, really. Lets stop and think about this for a moment. As you can see you can define how you choose backends based on really arbitrary data. You want to send mobile devices to a different backend? No problem. if (req.User-agent ~ /mobile/) .... should do the trick. -Directors -~~~~~~~~~~ +Directors +--------- You can also group several backend into a group of backends. These groups are called directors. This will give you increased performance @@ -52,13 +51,12 @@ backend server1 { .host = "192.168.0.10"; } -:: backend server2{ .host = "192.168.0.10"; } - Now we create the director.:: + director example_director round-robin { { .backend = server1; @@ -81,7 +79,7 @@ Checks come into play. Health checks -~~~~~~~~~~~~ +------------- Lets set up a director with two backends and health checks. First lets define the backends.:: @@ -96,8 +94,7 @@ .threshold = 3; } } -:: - backend server2 { + backend server2 { .host = "server2.example.com"; .probe = { .url = "/"; @@ -131,7 +128,7 @@ Now we define the director.:: -director example_director round-robin { + director example_director round-robin { { .backend = server1; } @@ -145,4 +142,3 @@ You use this director just as you would use any other director or backend. Varnish will not send traffic to hosts that are marked as unhealty. - Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -1,3 +1,5 @@ +.. _tutorial-increasing_your_hitrate: + Achieving a high hitrate ======================== @@ -2,18 +4,141 @@ Now that Varnish is up and running, and you can access your web -application through Varnish we probably need to do some changes to -either the configuration or the application so you'll get a high -hitrate in varnish. +application through Varnish. Unless your application is specifically +written to work behind a web accelerator you'll probably need to do +some changes to either the configuration or the application in order +to get a high hitrate in Varnish. +Note that you need a tool to see what HTTP headers fly between you and +the web server. If you have Varnish the easiest might be to use +varnishlog, but sometimes a separate tool makes sense. Here are the +ones I use. + +lwp-request +~~~~~~~~~~~ + +lwp-request is part of The World-Wide Web library for Perl. It's +couple of really basic programs that can execute a HTTP request and +give you the result. I use two programs, GET and HEAD. + +vg.no was the first site to use Varnish and the people running Varnish +there are quite clueful. So its interesting to look at their HTTP +Headers. Lets send a GET requst for their home page.:: + + $ GET -H 'Host: vg.no' -Used http://vg.no/ + GET http://www.vg.no/ + User-Agent: Rickzilla 1.0 + + 200 OK + Cache-Control: must-revalidate + Refresh: 600 + Title: VG Nett - Forsiden - VG Nett + X-Age: 463 + X-Cache: HIT + X-Rick-Would-Never: Let you down + X-VG-Jobb: http://www.finn.no/finn/job/fulltime/result?keyword=vg+multimedia Merk:HeaderNinja + X-VG-Korken: http://www.youtube.com/watch?v=Fcj8CnD5188 + X-VG-WebCache: joanie + X-VG-WebServer: leon + +OK. Let me explain what it does. GET usually send off HTTP 0.9 +requests, which lack the Host header. So I add a Host header with the +-H option. -U print request headers, -s prints response status -e +prints repsonse headers and -d discards the actual content. We dont +really care about the content, only the headers. + +As you can see VG ads quite a bit of information in their headers. A +lot of this information is added in their VCL. + +So, to check whether a site sets cookies for a specific URL just do +``GET -Used http://example.com/ |grep Set-Cookie`` + +Firefox plugins +~~~~~~~~~~~~~~~ + +There are also a couple of great plugins for Firefox. Both *Live HTTP +Headers* and *Firebug* can show you what headers are beeing sent and +recieved. + HTTP Headers ------------ +Varnish considers itself part of the actual webserver, since its under +your control. The role of *surrogate origin cache* is not really well +defined by the IETF so RFC 2616 doesn't always tell us what we should +do + Cache-control ~~~~~~~~~~~~~ +The Cache-Control instructs caches how to handle the content. Varnish +cares about the *max-age* parameter and uses it to calculate the TTL +for an object. + +"Cache-Control: nocache" is ignored. See +:ref:`tutorial-increasing_your_hitrate-pragma:` for an example on how +to implement support. + Cookies ~~~~~~~ +Varnish will not cache a object comming from the backend with a +Set-Cookie header present. Also, if the client sends a Cookie header, +Varnish will bypass the cache and go directly to the backend. + +This can be overly conservative. A lot of sites use Google Analytics +(GA) to analyse their traffic. GA sets a cookie to track you. This +cookie is used by the client side java script and is therefore of no +interest to the server. + +For a lot of web application it makes sense to completly disregard the +cookies unless you are accessing a special part of the web site. This +VCL snipplet in vcl_recv will disregard cookies unless you are +accessing /admin/.:: + + if ( !( req.url ~ ^/admin/) ) { + unset http.Cookie; + } + +Quite simple. If, however, you need to do something more complicated, +like removing one out of several cookies, things get +difficult. Unfornunatly Varnish doesn't have good tools for +manipulating the Cookies. We have to use regular expressions to do the +work. Let me show you an example where we remove everything the the cookies named COOKIE1 and COOKIE2 and you can marvel at it.:: + + sub vcl_recv { + if (req.http.Cookie) { + set req.http.Cookie = ";" req.http.Cookie; + set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); + set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1="); + set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); + set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", ""); + + if (req.http.Cookie == "") { + remove req.http.Cookie; + } + } + +The example is taken from the Varnish Wiki, where you can find other +scary examples of what can be done i VCL. + Vary ~~~~ +The Vary header is sent by the web server to indicate what makes a +HTTP object Vary. + +.. _tutorial-increasing_your_hitrate-pragma: + +Pragma +~~~~~~ + + +HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this +header. You could easly add support for this header in VCL. + +In vcl_fetch:: + + if (beresp.http.Pragma ~ "nocache") { + pass; + } + Authentication @@ -25,6 +150,7 @@ -------------------------- .. _tutorial-increasing_your_hitrate-purging: + Purging ------- @@ -34,3 +160,6 @@ Bans ~~~~ + + + Modified: trunk/varnish-cache/doc/sphinx/tutorial/logging.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/logging.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/logging.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -61,7 +61,8 @@ -i tag Only show lines with a certain tag. "varnishlog -i SessionOpen" - will only give you new sessions. + will only give you new sessions. Note that the tags are case + insensitive. -I Regex Filter the data through a regex and only show the matching lines. To Modified: trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/starting_varnish.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -1,4 +1,5 @@ .. _tutorial-starting_varnish: + Starting Varnish ---------------- @@ -38,5 +39,12 @@ properly. Use your browser to go to http://192.168.2.2:8080/ - you should now see your web application running there. +Whether or not the application actually goes faster when run through +Varnish depends on a few factors. If you application uses cookies for +every session (a lot of PHP and Java applications seem to send a +session cookie if it is needed or not) or if it uses authentication +chances are Varnish won't do much caching. Ignore that for the moment, +we come back to that in :ref:`tutorial-increasing_your_hitrate`. + Lets make sure that Varnish really does do something to your web site. To do that we'll take a look at the logs. Modified: trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -17,7 +17,11 @@ used to display a ranking of requested documents, clients, user agents, or any other information which is recorded in the log. -XXX Show some nice examples here. +``varnishtop -i rxurl`` will show you what URLs are beeing asked for +by the client. ``varnishtop -i txurl`` will show you what your backend +is beeing asked the most. ``varnishtop -i RxHeader -I +Accept-Encoding`` will show the most popular Accept-Encoding header +the client are sendning you. varnishhist =========== @@ -51,4 +55,3 @@ found at http://munin-monitoring.org/ . There is a plugin for munin in the varnish source code. - Modified: trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -3,7 +3,7 @@ When Varnish won't start ------------------------ +------------------------ Sometimes Varnish wont start. There is a pletphora of reasons why Varnish wont start on your machine. We've seen everything from wrong Modified: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-15 11:07:58 UTC (rev 4792) +++ trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-15 21:50:12 UTC (rev 4793) @@ -1,8 +1,8 @@ Varnish Configuration Language - VCL -============================== +==================================== How ordinary configuration files work ---------------------------------- +------------------------------------- Varnish has a really neat configuration system. Most other systems use configuration directives, where you basically turn on and off a bunch From phk at varnish-cache.org Mon May 17 07:36:04 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 09:36:04 +0200 Subject: r4794 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 09:36:03 +0200 (Mon, 17 May 2010) New Revision: 4794 Added: trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/mgt_shmem.c Removed: trunk/varnish-cache/bin/varnishd/shmlog.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/common.h Log: Split shmlog.c into mgt_shmem.c and cache_shmlog.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2010-05-15 21:50:12 UTC (rev 4793) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2010-05-17 07:36:03 UTC (rev 4794) @@ -33,6 +33,7 @@ cache_pool.c \ cache_response.c \ cache_session.c \ + cache_shmlog.c \ cache_vary.c \ cache_vcl.c \ cache_vrt.c \ @@ -46,9 +47,9 @@ mgt_cli.c \ mgt_param.c \ mgt_pool.c \ + mgt_shmem.c \ mgt_vcc.c \ rfc2616.c \ - shmlog.c \ stevedore.c \ storage_file.c \ storage_malloc.c \ Copied: trunk/varnish-cache/bin/varnishd/cache_shmlog.c (from rev 4793, trunk/varnish-cache/bin/varnishd/shmlog.c) =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c (rev 0) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-17 07:36:03 UTC (rev 4794) @@ -0,0 +1,300 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id$") + +#include +#include +#include + +#include "shmlog.h" +#include "cache.h" + +static pthread_mutex_t vsl_mtx; + +#define LOCKSHM(foo) \ + do { \ + if (pthread_mutex_trylock(foo)) { \ + AZ(pthread_mutex_lock(foo)); \ + VSL_stats->shm_cont++; \ + } \ + } while (0); + +#define UNLOCKSHM(foo) AZ(pthread_mutex_unlock(foo)) + +static void +vsl_wrap(void) +{ + + assert(loghead->magic == SHMLOGHEAD_MAGIC); + *logstart = SLT_ENDMARKER; + logstart[loghead->ptr] = SLT_WRAPMARKER; + loghead->ptr = 0; + VSL_stats->shm_cycles++; + assert(loghead->magic == SHMLOGHEAD_MAGIC); +} + +static void +vsl_hdr(enum shmlogtag tag, unsigned char *p, unsigned len, unsigned id) +{ + + assert(loghead->magic == SHMLOGHEAD_MAGIC); + assert(len < 0x10000); + p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff; + p[__SHMLOG_LEN_LOW] = len & 0xff; + p[__SHMLOG_ID_HIGH] = (id >> 24) & 0xff; + p[__SHMLOG_ID_MEDHIGH] = (id >> 16) & 0xff; + p[__SHMLOG_ID_MEDLOW] = (id >> 8) & 0xff; + p[__SHMLOG_ID_LOW] = id & 0xff; + p[SHMLOG_DATA + len] = '\0'; + p[SHMLOG_NEXTTAG + len] = SLT_ENDMARKER; + /* XXX: Write barrier here */ + p[SHMLOG_TAG] = tag; +} + +/*-------------------------------------------------------------------- + * This variant copies a byte-range directly to the log, without + * taking the detour over sprintf() + */ + +static void +VSLR(enum shmlogtag tag, int id, txt t) +{ + unsigned char *p; + unsigned l, mlen; + + Tcheck(t); + mlen = params->shm_reclen; + + /* Truncate */ + l = Tlen(t); + if (l > mlen) { + l = mlen; + t.e = t.b + l; + } + + /* Only hold the lock while we find our space */ + LOCKSHM(&vsl_mtx); + VSL_stats->shm_writes++; + VSL_stats->shm_records++; + assert(loghead->ptr < loghead->size); + + /* Wrap if necessary */ + if (loghead->ptr + SHMLOG_NEXTTAG + l + 1 >= loghead->size) + vsl_wrap(); + p = logstart + loghead->ptr; + loghead->ptr += SHMLOG_NEXTTAG + l; + assert(loghead->ptr < loghead->size); + UNLOCKSHM(&vsl_mtx); + + memcpy(p + SHMLOG_DATA, t.b, l); + vsl_hdr(tag, p, l, id); +} + +/*--------------------------------------------------------------------*/ + +void +VSL(enum shmlogtag tag, int id, const char *fmt, ...) +{ + va_list ap; + unsigned char *p; + unsigned n, mlen; + txt t; + + AN(fmt); + va_start(ap, fmt); + mlen = params->shm_reclen; + + if (strchr(fmt, '%') == NULL) { + t.b = TRUST_ME(fmt); + t.e = strchr(t.b, '\0'); + VSLR(tag, id, t); + } else { + LOCKSHM(&vsl_mtx); + VSL_stats->shm_writes++; + VSL_stats->shm_records++; + assert(loghead->ptr < loghead->size); + + /* Wrap if we cannot fit a full size record */ + if (loghead->ptr + SHMLOG_NEXTTAG + mlen + 1 >= loghead->size) + vsl_wrap(); + + p = logstart + loghead->ptr; + /* +1 for the NUL */ + n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); + if (n > mlen) + n = mlen; /* we truncate long fields */ + vsl_hdr(tag, p, n, id); + loghead->ptr += SHMLOG_NEXTTAG + n; + assert(loghead->ptr < loghead->size); + UNLOCKSHM(&vsl_mtx); + } + va_end(ap); +} + +/*--------------------------------------------------------------------*/ + +void +WSL_Flush(struct worker *w, int overflow) +{ + unsigned char *p; + unsigned l; + + l = pdiff(w->wlb, w->wlp); + if (l == 0) + return; + LOCKSHM(&vsl_mtx); + VSL_stats->shm_flushes += overflow; + VSL_stats->shm_writes++; + VSL_stats->shm_records += w->wlr; + if (loghead->ptr + l + 1 >= loghead->size) + vsl_wrap(); + p = logstart + loghead->ptr; + p[l] = SLT_ENDMARKER; + loghead->ptr += l; + assert(loghead->ptr < loghead->size); + UNLOCKSHM(&vsl_mtx); + memcpy(p + 1, w->wlb + 1, l - 1); + /* XXX: memory barrier here */ + p[0] = w->wlb[0]; + w->wlp = w->wlb; + w->wlr = 0; +} + +/*--------------------------------------------------------------------*/ + +void +WSLR(struct worker *w, enum shmlogtag tag, int id, txt t) +{ + unsigned char *p; + unsigned l, mlen; + + Tcheck(t); + mlen = params->shm_reclen; + + /* Truncate */ + l = Tlen(t); + if (l > mlen) { + l = mlen; + t.e = t.b + l; + } + + assert(w->wlp < w->wle); + + /* Wrap if necessary */ + if (w->wlp + SHMLOG_NEXTTAG + l + 1 >= w->wle) + WSL_Flush(w, 1); + p = w->wlp; + w->wlp += SHMLOG_NEXTTAG + l; + assert(w->wlp < w->wle); + memcpy(p + SHMLOG_DATA, t.b, l); + vsl_hdr(tag, p, l, id); + w->wlr++; + if (params->diag_bitmap & 0x10000) + WSL_Flush(w, 0); +} + +/*--------------------------------------------------------------------*/ + +void +WSL(struct worker *w, enum shmlogtag tag, int id, const char *fmt, ...) +{ + va_list ap; + unsigned char *p; + unsigned n, mlen; + txt t; + + AN(fmt); + va_start(ap, fmt); + mlen = params->shm_reclen; + + if (strchr(fmt, '%') == NULL) { + t.b = TRUST_ME(fmt); + t.e = strchr(t.b, '\0'); + WSLR(w, tag, id, t); + } else { + assert(w->wlp < w->wle); + + /* Wrap if we cannot fit a full size record */ + if (w->wlp + SHMLOG_NEXTTAG + mlen + 1 >= w->wle) + WSL_Flush(w, 1); + + p = w->wlp; + /* +1 for the NUL */ + n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); + if (n > mlen) + n = mlen; /* we truncate long fields */ + vsl_hdr(tag, p, n, id); + w->wlp += SHMLOG_NEXTTAG + n; + assert(w->wlp < w->wle); + w->wlr++; + } + va_end(ap); + if (params->diag_bitmap & 0x10000) + WSL_Flush(w, 0); +} + +/*--------------------------------------------------------------------*/ + +void +VSL_Panic(int *len, char **ptr) +{ + static char a[1] = { '\0' }; + + AN(len); + AN(ptr); + if (loghead->magic == SHMLOGHEAD_MAGIC) { + assert(loghead->hdrsize == sizeof *loghead); + *len = sizeof(loghead->panicstr); + *ptr = loghead->panicstr; + } else { + *len = 0; + *ptr = a; + } +} + +/*--------------------------------------------------------------------*/ + +void +VSL_Init(void) +{ + + assert(loghead->magic == SHMLOGHEAD_MAGIC); + assert(loghead->hdrsize == sizeof *loghead); + /* XXX more check sanity of loghead ? */ + logstart = (unsigned char *)loghead + loghead->start; + AZ(pthread_mutex_init(&vsl_mtx, NULL)); + loghead->starttime = TIM_real(); + loghead->panicstr[0] = '\0'; + memset(VSL_stats, 0, sizeof *VSL_stats); + loghead->child_pid = getpid(); +} Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-15 21:50:12 UTC (rev 4793) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 07:36:03 UTC (rev 4794) @@ -41,10 +41,12 @@ /* shmlog.c */ void VSL_Panic(int *len, char **ptr); -/* shmlog.c */ +/* mgt_shmem.c */ void VSL_MgtInit(const char *fn, unsigned size); void VSL_MgtPid(void); extern struct varnish_stats *VSL_stats; +extern struct shmloghead *loghead; +extern unsigned char *logstart; /* varnishd.c */ struct vsb; Copied: trunk/varnish-cache/bin/varnishd/mgt_shmem.c (from rev 4793, trunk/varnish-cache/bin/varnishd/shmlog.c) =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c (rev 0) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 07:36:03 UTC (rev 4794) @@ -0,0 +1,162 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id$") + +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "mgt.h" +#include "heritage.h" + +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + +#ifndef MAP_NOSYNC +#define MAP_NOSYNC 0 /* XXX Linux */ +#endif + +struct varnish_stats *VSL_stats; +struct shmloghead *loghead; +unsigned char *logstart; + + +/*--------------------------------------------------------------------*/ + +static int +vsl_goodold(int fd) +{ + struct shmloghead slh; + int i; + + memset(&slh, 0, sizeof slh); /* XXX: for flexelint */ + i = read(fd, &slh, sizeof slh); + if (i != sizeof slh) + return (0); + if (slh.magic != SHMLOGHEAD_MAGIC) + return (0); + if (slh.hdrsize != sizeof slh) + return (0); + if (slh.start != sizeof slh + sizeof *params) + return (0); + + if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) { + fprintf(stderr, + "SHMFILE owned by running varnishd master (pid=%jd)\n", + (intmax_t)slh.master_pid); + fprintf(stderr, + "(Use unique -n arguments if you want multiple " + "instances.)\n"); + exit(2); + } + + if (slh.child_pid != 0 && !kill(slh.child_pid, 0)) { + fprintf(stderr, + "SHMFILE used by orphan varnishd child process (pid=%jd)\n", + (intmax_t)slh.child_pid); + fprintf(stderr, "(We assume that process is busy dying.)\n"); + return (0); + } + + /* XXX more checks */ + heritage.vsl_size = slh.size + slh.start; + return (1); +} + +static void +vsl_buildnew(const char *fn, unsigned size) +{ + struct shmloghead slh; + int i; + + (void)unlink(fn); + heritage.vsl_fd = open(fn, O_RDWR | O_CREAT | O_EXCL, 0644); + if (heritage.vsl_fd < 0) { + fprintf(stderr, "Could not open %s: %s\n", + fn, strerror(errno)); + exit (1); + } + + memset(&slh, 0, sizeof slh); + slh.magic = SHMLOGHEAD_MAGIC; + slh.hdrsize = sizeof slh; + slh.size = size; + slh.ptr = 0; + slh.start = sizeof slh + sizeof *params; + i = write(heritage.vsl_fd, &slh, sizeof slh); + xxxassert(i == sizeof slh); + heritage.vsl_size = slh.start + size; + AZ(ftruncate(heritage.vsl_fd, (off_t)heritage.vsl_size)); +} + +void +VSL_MgtInit(const char *fn, unsigned size) +{ + int i; + struct params *pp; + + i = open(fn, O_RDWR, 0644); + if (i >= 0 && vsl_goodold(i)) { + fprintf(stderr, "Using old SHMFILE\n"); + heritage.vsl_fd = i; + } else { + fprintf(stderr, "Creating new SHMFILE\n"); + (void)close(i); + vsl_buildnew(fn, size); + } + + loghead = (void *)mmap(NULL, heritage.vsl_size, + PROT_READ|PROT_WRITE, + MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, + heritage.vsl_fd, 0); + loghead->master_pid = getpid(); + xxxassert(loghead != MAP_FAILED); + (void)mlock((void*)loghead, heritage.vsl_size); + VSL_stats = &loghead->stats; + pp = (void *)(loghead + 1); + *pp = *params; + params = pp; +} + +void +VSL_MgtPid(void) +{ + + loghead->master_pid = getpid(); +} Deleted: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2010-05-15 21:50:12 UTC (rev 4793) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2010-05-17 07:36:03 UTC (rev 4794) @@ -1,422 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2010 Redpill Linpro AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include "svnid.h" -SVNID("$Id$") - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "shmlog.h" -#include "cache.h" - -#define LOCKSHM(foo) \ - do { \ - if (pthread_mutex_trylock(foo)) { \ - AZ(pthread_mutex_lock(foo)); \ - VSL_stats->shm_cont++; \ - } \ - } while (0); - -#define UNLOCKSHM(foo) AZ(pthread_mutex_unlock(foo)) - -#ifndef MAP_HASSEMAPHORE -#define MAP_HASSEMAPHORE 0 /* XXX Linux */ -#endif - -#ifndef MAP_NOSYNC -#define MAP_NOSYNC 0 /* XXX Linux */ -#endif - -struct varnish_stats *VSL_stats; -static struct shmloghead *loghead; -static unsigned char *logstart; -static pthread_mutex_t vsl_mtx; - - -static void -vsl_wrap(void) -{ - - assert(loghead->magic == SHMLOGHEAD_MAGIC); - *logstart = SLT_ENDMARKER; - logstart[loghead->ptr] = SLT_WRAPMARKER; - loghead->ptr = 0; - VSL_stats->shm_cycles++; - assert(loghead->magic == SHMLOGHEAD_MAGIC); -} - -static void -vsl_hdr(enum shmlogtag tag, unsigned char *p, unsigned len, unsigned id) -{ - - assert(loghead->magic == SHMLOGHEAD_MAGIC); - assert(len < 0x10000); - p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff; - p[__SHMLOG_LEN_LOW] = len & 0xff; - p[__SHMLOG_ID_HIGH] = (id >> 24) & 0xff; - p[__SHMLOG_ID_MEDHIGH] = (id >> 16) & 0xff; - p[__SHMLOG_ID_MEDLOW] = (id >> 8) & 0xff; - p[__SHMLOG_ID_LOW] = id & 0xff; - p[SHMLOG_DATA + len] = '\0'; - p[SHMLOG_NEXTTAG + len] = SLT_ENDMARKER; - /* XXX: Write barrier here */ - p[SHMLOG_TAG] = tag; -} - -/*-------------------------------------------------------------------- - * This variant copies a byte-range directly to the log, without - * taking the detour over sprintf() - */ - -static void -VSLR(enum shmlogtag tag, int id, txt t) -{ - unsigned char *p; - unsigned l, mlen; - - Tcheck(t); - mlen = params->shm_reclen; - - /* Truncate */ - l = Tlen(t); - if (l > mlen) { - l = mlen; - t.e = t.b + l; - } - - /* Only hold the lock while we find our space */ - LOCKSHM(&vsl_mtx); - VSL_stats->shm_writes++; - VSL_stats->shm_records++; - assert(loghead->ptr < loghead->size); - - /* Wrap if necessary */ - if (loghead->ptr + SHMLOG_NEXTTAG + l + 1 >= loghead->size) - vsl_wrap(); - p = logstart + loghead->ptr; - loghead->ptr += SHMLOG_NEXTTAG + l; - assert(loghead->ptr < loghead->size); - UNLOCKSHM(&vsl_mtx); - - memcpy(p + SHMLOG_DATA, t.b, l); - vsl_hdr(tag, p, l, id); -} - -/*--------------------------------------------------------------------*/ - -void -VSL(enum shmlogtag tag, int id, const char *fmt, ...) -{ - va_list ap; - unsigned char *p; - unsigned n, mlen; - txt t; - - AN(fmt); - va_start(ap, fmt); - mlen = params->shm_reclen; - - if (strchr(fmt, '%') == NULL) { - t.b = TRUST_ME(fmt); - t.e = strchr(t.b, '\0'); - VSLR(tag, id, t); - } else { - LOCKSHM(&vsl_mtx); - VSL_stats->shm_writes++; - VSL_stats->shm_records++; - assert(loghead->ptr < loghead->size); - - /* Wrap if we cannot fit a full size record */ - if (loghead->ptr + SHMLOG_NEXTTAG + mlen + 1 >= loghead->size) - vsl_wrap(); - - p = logstart + loghead->ptr; - /* +1 for the NUL */ - n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); - if (n > mlen) - n = mlen; /* we truncate long fields */ - vsl_hdr(tag, p, n, id); - loghead->ptr += SHMLOG_NEXTTAG + n; - assert(loghead->ptr < loghead->size); - UNLOCKSHM(&vsl_mtx); - } - va_end(ap); -} - -/*--------------------------------------------------------------------*/ - -void -WSL_Flush(struct worker *w, int overflow) -{ - unsigned char *p; - unsigned l; - - l = pdiff(w->wlb, w->wlp); - if (l == 0) - return; - LOCKSHM(&vsl_mtx); - VSL_stats->shm_flushes += overflow; - VSL_stats->shm_writes++; - VSL_stats->shm_records += w->wlr; - if (loghead->ptr + l + 1 >= loghead->size) - vsl_wrap(); - p = logstart + loghead->ptr; - p[l] = SLT_ENDMARKER; - loghead->ptr += l; - assert(loghead->ptr < loghead->size); - UNLOCKSHM(&vsl_mtx); - memcpy(p + 1, w->wlb + 1, l - 1); - /* XXX: memory barrier here */ - p[0] = w->wlb[0]; - w->wlp = w->wlb; - w->wlr = 0; -} - -/*--------------------------------------------------------------------*/ - -void -WSLR(struct worker *w, enum shmlogtag tag, int id, txt t) -{ - unsigned char *p; - unsigned l, mlen; - - Tcheck(t); - mlen = params->shm_reclen; - - /* Truncate */ - l = Tlen(t); - if (l > mlen) { - l = mlen; - t.e = t.b + l; - } - - assert(w->wlp < w->wle); - - /* Wrap if necessary */ - if (w->wlp + SHMLOG_NEXTTAG + l + 1 >= w->wle) - WSL_Flush(w, 1); - p = w->wlp; - w->wlp += SHMLOG_NEXTTAG + l; - assert(w->wlp < w->wle); - memcpy(p + SHMLOG_DATA, t.b, l); - vsl_hdr(tag, p, l, id); - w->wlr++; - if (params->diag_bitmap & 0x10000) - WSL_Flush(w, 0); -} - -/*--------------------------------------------------------------------*/ - -void -WSL(struct worker *w, enum shmlogtag tag, int id, const char *fmt, ...) -{ - va_list ap; - unsigned char *p; - unsigned n, mlen; - txt t; - - AN(fmt); - va_start(ap, fmt); - mlen = params->shm_reclen; - - if (strchr(fmt, '%') == NULL) { - t.b = TRUST_ME(fmt); - t.e = strchr(t.b, '\0'); - WSLR(w, tag, id, t); - } else { - assert(w->wlp < w->wle); - - /* Wrap if we cannot fit a full size record */ - if (w->wlp + SHMLOG_NEXTTAG + mlen + 1 >= w->wle) - WSL_Flush(w, 1); - - p = w->wlp; - /* +1 for the NUL */ - n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); - if (n > mlen) - n = mlen; /* we truncate long fields */ - vsl_hdr(tag, p, n, id); - w->wlp += SHMLOG_NEXTTAG + n; - assert(w->wlp < w->wle); - w->wlr++; - } - va_end(ap); - if (params->diag_bitmap & 0x10000) - WSL_Flush(w, 0); -} - -/*--------------------------------------------------------------------*/ - -void -VSL_Panic(int *len, char **ptr) -{ - static char a[1] = { '\0' }; - - AN(len); - AN(ptr); - if (loghead->magic == SHMLOGHEAD_MAGIC) { - assert(loghead->hdrsize == sizeof *loghead); - *len = sizeof(loghead->panicstr); - *ptr = loghead->panicstr; - } else { - *len = 0; - *ptr = a; - } -} - -/*--------------------------------------------------------------------*/ - -void -VSL_Init(void) -{ - - assert(loghead->magic == SHMLOGHEAD_MAGIC); - assert(loghead->hdrsize == sizeof *loghead); - /* XXX more check sanity of loghead ? */ - logstart = (unsigned char *)loghead + loghead->start; - AZ(pthread_mutex_init(&vsl_mtx, NULL)); - loghead->starttime = TIM_real(); - loghead->panicstr[0] = '\0'; - memset(VSL_stats, 0, sizeof *VSL_stats); - loghead->child_pid = getpid(); -} - -/*--------------------------------------------------------------------*/ - -static int -vsl_goodold(int fd) -{ - struct shmloghead slh; - int i; - - memset(&slh, 0, sizeof slh); /* XXX: for flexelint */ - i = read(fd, &slh, sizeof slh); - if (i != sizeof slh) - return (0); - if (slh.magic != SHMLOGHEAD_MAGIC) - return (0); - if (slh.hdrsize != sizeof slh) - return (0); - if (slh.start != sizeof slh + sizeof *params) - return (0); - - if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) { - fprintf(stderr, - "SHMFILE owned by running varnishd master (pid=%jd)\n", - (intmax_t)slh.master_pid); - fprintf(stderr, - "(Use unique -n arguments if you want multiple " - "instances.)\n"); - exit(2); - } - - if (slh.child_pid != 0 && !kill(slh.child_pid, 0)) { - fprintf(stderr, - "SHMFILE used by orphan varnishd child process (pid=%jd)\n", - (intmax_t)slh.child_pid); - fprintf(stderr, "(We assume that process is busy dying.)\n"); - return (0); - } - - /* XXX more checks */ - heritage.vsl_size = slh.size + slh.start; - return (1); -} - -static void -vsl_buildnew(const char *fn, unsigned size) -{ - struct shmloghead slh; - int i; - - (void)unlink(fn); - heritage.vsl_fd = open(fn, O_RDWR | O_CREAT | O_EXCL, 0644); - if (heritage.vsl_fd < 0) { - fprintf(stderr, "Could not open %s: %s\n", - fn, strerror(errno)); - exit (1); - } - - memset(&slh, 0, sizeof slh); - slh.magic = SHMLOGHEAD_MAGIC; - slh.hdrsize = sizeof slh; - slh.size = size; - slh.ptr = 0; - slh.start = sizeof slh + sizeof *params; - i = write(heritage.vsl_fd, &slh, sizeof slh); - xxxassert(i == sizeof slh); - heritage.vsl_size = slh.start + size; - AZ(ftruncate(heritage.vsl_fd, (off_t)heritage.vsl_size)); -} - -void -VSL_MgtInit(const char *fn, unsigned size) -{ - int i; - struct params *pp; - - i = open(fn, O_RDWR, 0644); - if (i >= 0 && vsl_goodold(i)) { - fprintf(stderr, "Using old SHMFILE\n"); - heritage.vsl_fd = i; - } else { - fprintf(stderr, "Creating new SHMFILE\n"); - (void)close(i); - vsl_buildnew(fn, size); - } - - loghead = (void *)mmap(NULL, heritage.vsl_size, - PROT_READ|PROT_WRITE, - MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, - heritage.vsl_fd, 0); - loghead->master_pid = getpid(); - xxxassert(loghead != MAP_FAILED); - (void)mlock((void*)loghead, heritage.vsl_size); - VSL_stats = &loghead->stats; - pp = (void *)(loghead + 1); - *pp = *params; - params = pp; -} - -void -VSL_MgtPid(void) -{ - - loghead->master_pid = getpid(); -} From phk at varnish-cache.org Mon May 17 08:11:44 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 10:11:44 +0200 Subject: r4795 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 10:11:43 +0200 (Mon, 17 May 2010) New Revision: 4795 Modified: trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Move shm-related arg processing to mgt_shmem.c Apply our naming system Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 07:36:03 UTC (rev 4794) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 08:11:43 UTC (rev 4795) @@ -42,8 +42,6 @@ void VSL_Panic(int *len, char **ptr); /* mgt_shmem.c */ -void VSL_MgtInit(const char *fn, unsigned size); -void VSL_MgtPid(void); extern struct varnish_stats *VSL_stats; extern struct shmloghead *loghead; extern unsigned char *logstart; Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-17 07:36:03 UTC (rev 4794) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-17 08:11:43 UTC (rev 4795) @@ -68,6 +68,10 @@ void MCF_DumpMdoc(void); #endif +/* mgt_shmem.c */ +void mgt_SHM_Init(const char *fn, const char *arg); +void mgt_SHM_Pid(void); + /* mgt_vcc.c */ void mgt_vcc_init(void); int mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag); Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 07:36:03 UTC (rev 4794) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 08:11:43 UTC (rev 4795) @@ -126,11 +126,24 @@ } void -VSL_MgtInit(const char *fn, unsigned size) +mgt_SHM_Init(const char *fn, const char *l_arg) { int i; struct params *pp; + const char *arg_default = "80m"; + const char *q; + uintmax_t size; + if (l_arg == NULL) + l_arg = arg_default; + + q = str2bytes(l_arg, &size, 0); + if (q != NULL) { + fprintf(stderr, "Parameter error:\n"); + fprintf(stderr, "\t-l ...: %s\n", q); + exit (1); + } + i = open(fn, O_RDWR, 0644); if (i >= 0 && vsl_goodold(i)) { fprintf(stderr, "Using old SHMFILE\n"); @@ -155,7 +168,7 @@ } void -VSL_MgtPid(void) +mgt_SHM_Pid(void) { loghead->master_pid = getpid(); Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-17 07:36:03 UTC (rev 4794) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-17 08:11:43 UTC (rev 4795) @@ -409,9 +409,7 @@ const char *b_arg = NULL; const char *f_arg = NULL; const char *i_arg = NULL; - const char *l_arg = "80m"; - uintmax_t l_size; - const char *q; + const char *l_arg = NULL; /* default in mgt_shmem.c */ const char *h_arg = "critbit"; const char *M_arg = NULL; const char *n_arg = NULL; @@ -604,13 +602,6 @@ usage(); } - q = str2bytes(l_arg, &l_size, 0); - if (q != NULL) { - fprintf(stderr, "Parameter error:\n"); - fprintf(stderr, "\t-l ...: %s\n", q); - exit (1); - } - /* XXX: we can have multiple CLI actions above, is this enough ? */ if (cli[0].result != CLIS_OK) { fprintf(stderr, "Parameter errors:\n"); @@ -689,7 +680,7 @@ setup_hash(h_arg); - VSL_MgtInit(SHMLOG_FILENAME, l_size); + mgt_SHM_Init(SHMLOG_FILENAME, l_arg); vsb_finish(vident); AZ(vsb_overflowed(vident)); @@ -697,7 +688,7 @@ if (!d_flag && !F_flag) AZ(varnish_daemon(1, 0)); - VSL_MgtPid(); + mgt_SHM_Pid(); if (pfh != NULL && vpf_write(pfh)) fprintf(stderr, "NOTE: Could not write PID file\n"); From phk at varnish-cache.org Mon May 17 08:40:10 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 10:40:10 +0200 Subject: r4796 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 10:40:08 +0200 (Mon, 17 May 2010) New Revision: 4796 Modified: trunk/varnish-cache/bin/varnishd/common.h Log: Add preliminary macro for memory barriers Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 08:11:43 UTC (rev 4795) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 08:40:08 UTC (rev 4796) @@ -70,3 +70,18 @@ }; #define NEEDLESS_RETURN(foo) return (foo) + +/********************************************************************** + * Guess what: There is no POSIX standard for memory barriers. + * XXX: Please try to find the minimal #ifdef to use here, rely on OS + * supplied facilities if at all possible, to avoid descending into the + * full cpu/compiler explosion. + */ + +#ifdef __FreeBSD__ +#include +#define MEMORY_BARRIER() mb() +#else +#warn "MEMORY_BARRIER() is expensive" +#define MEMORY_BARRIER() close(-1) +#endif From phk at varnish-cache.org Mon May 17 08:48:29 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 10:48:29 +0200 Subject: r4797 - in trunk/varnish-cache/bin: varnishd varnishtest Message-ID: Author: phk Date: 2010-05-17 10:48:28 +0200 (Mon, 17 May 2010) New Revision: 4797 Modified: trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/bin/varnishtest/vtc_varnish.c Log: Move vsl handles out of heritage, they do no good there. Change -l argument processing, to take up to three parts: -l,, is size of space for VSL records. default is 80m is size of space for other allocations (more later) default is 1m is '+' or '-' to control preallocation of the shmfile, default to on (+), which reduces risk of fragmentation. Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2010-05-17 08:40:08 UTC (rev 4796) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2010-05-17 08:48:28 UTC (rev 4797) @@ -55,10 +55,6 @@ struct listen_sock_head socks; unsigned nsocks; - /* Share memory log fd and size (incl header) */ - int vsl_fd; - unsigned vsl_size; - /* Hash method */ struct hash_slinger *hash; Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 08:40:08 UTC (rev 4796) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 08:48:28 UTC (rev 4797) @@ -56,6 +56,8 @@ struct shmloghead *loghead; unsigned char *logstart; +static int vsl_fd = -1; +static unsigned vsl_size; /*--------------------------------------------------------------------*/ @@ -95,19 +97,21 @@ } /* XXX more checks */ - heritage.vsl_size = slh.size + slh.start; + vsl_size = slh.size + slh.start; return (1); } static void -vsl_buildnew(const char *fn, unsigned size) +vsl_buildnew(const char *fn, unsigned size, int fill) { struct shmloghead slh; int i; + unsigned u; + char buf[64*1024]; (void)unlink(fn); - heritage.vsl_fd = open(fn, O_RDWR | O_CREAT | O_EXCL, 0644); - if (heritage.vsl_fd < 0) { + vsl_fd = open(fn, O_RDWR | O_CREAT | O_EXCL, 0644); + if (vsl_fd < 0) { fprintf(stderr, "Could not open %s: %s\n", fn, strerror(errno)); exit (1); @@ -119,48 +123,108 @@ slh.size = size; slh.ptr = 0; slh.start = sizeof slh + sizeof *params; - i = write(heritage.vsl_fd, &slh, sizeof slh); + i = write(vsl_fd, &slh, sizeof slh); xxxassert(i == sizeof slh); - heritage.vsl_size = slh.start + size; - AZ(ftruncate(heritage.vsl_fd, (off_t)heritage.vsl_size)); + vsl_size = slh.start + size; + + if (fill) { + memset(buf, 0, sizeof buf); + for (u = sizeof slh; u < size; ) { + i = write(vsl_fd, buf, sizeof buf); + if (i <= 0) { + fprintf(stderr, "Write error %s: %s\n", + fn, strerror(errno)); + exit (1); + } + u += i; + } + } + + AZ(ftruncate(vsl_fd, (off_t)vsl_size)); } void mgt_SHM_Init(const char *fn, const char *l_arg) { - int i; + int i, fill; struct params *pp; - const char *arg_default = "80m"; const char *q; - uintmax_t size; + uintmax_t size, s1, s2; + char **av, **ap; if (l_arg == NULL) - l_arg = arg_default; + l_arg = ""; - q = str2bytes(l_arg, &size, 0); - if (q != NULL) { - fprintf(stderr, "Parameter error:\n"); - fprintf(stderr, "\t-l ...: %s\n", q); - exit (1); + av = ParseArgv(l_arg, ARGV_COMMA); + AN(av); + if (av[0] != NULL) + ARGV_ERR("\t-l ...: %s", av[0]); + + printf("<%s> <%s> <%s>\n", av[1], av[2], av[3]); + + ap = av + 1; + + /* Size of SHMLOG */ + if (*ap != NULL && **ap != '\0') { + q = str2bytes(*ap, &s1, 0); + if (q != NULL) + ARGV_ERR("\t-l[1] ...: %s\n", q); + } else { + s1 = 80 * 1024 * 1024; } + if (*ap != NULL) + ap++; + /* Size of space for other stuff */ + if (*ap != NULL && **ap != '\0') { + q = str2bytes(*ap, &s2, 0); + if (q != NULL) + ARGV_ERR("\t-l[2] ...: %s\n", q); + } else { + s2 = 1024 * 1024; + } + if (*ap != NULL) + ap++; + + /* Fill or not ? */ + if (*ap != NULL) { + if (*ap == '\0') + fill = 1; + else if (!strcmp(*ap, "-")) + fill = 0; + else if (!strcmp(*ap, "+")) + fill = 1; + else + ARGV_ERR("\t-l[3] ...: Must be \"-\" or \"+\"\n"); + ap++; + } else { + fill = 1; + } + + FreeArgv(av); + + size = s1 + s2; + + if (av[2] == NULL) + q = str2bytes(av[2], &size, 0); + i = open(fn, O_RDWR, 0644); if (i >= 0 && vsl_goodold(i)) { fprintf(stderr, "Using old SHMFILE\n"); - heritage.vsl_fd = i; + vsl_fd = i; } else { fprintf(stderr, "Creating new SHMFILE\n"); (void)close(i); - vsl_buildnew(fn, size); + vsl_buildnew(fn, size, fill); } - loghead = (void *)mmap(NULL, heritage.vsl_size, + loghead = (void *)mmap(NULL, vsl_size, PROT_READ|PROT_WRITE, MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, - heritage.vsl_fd, 0); + vsl_fd, 0); loghead->master_pid = getpid(); xxxassert(loghead != MAP_FAILED); - (void)mlock((void*)loghead, heritage.vsl_size); + (void)mlock((void*)loghead, vsl_size); VSL_stats = &loghead->stats; pp = (void *)(loghead + 1); *pp = *params; Modified: trunk/varnish-cache/bin/varnishtest/vtc_varnish.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-17 08:40:08 UTC (rev 4796) +++ trunk/varnish-cache/bin/varnishtest/vtc_varnish.c 2010-05-17 08:48:28 UTC (rev 4797) @@ -276,6 +276,7 @@ AN(vsb); vsb_printf(vsb, "cd ../varnishd &&"); vsb_printf(vsb, " ./varnishd -d -d -n %s", v->workdir); + vsb_printf(vsb, " -l 10m,1m,-"); vsb_printf(vsb, " -p auto_restart=off"); vsb_printf(vsb, " -p syslog_cli_traffic=off"); vsb_printf(vsb, " -a '%s'", "127.0.0.1:0"); From phk at varnish-cache.org Mon May 17 08:51:09 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 10:51:09 +0200 Subject: r4798 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 10:51:09 +0200 (Mon, 17 May 2010) New Revision: 4798 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Update usage with new -l definition. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-17 08:48:28 UTC (rev 4797) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-17 08:51:09 UTC (rev 4798) @@ -210,7 +210,10 @@ fprintf(stderr, FMT, "", " -h classic [default]"); fprintf(stderr, FMT, "", " -h classic,"); fprintf(stderr, FMT, "-i identity", "Identity of varnish instance"); - fprintf(stderr, FMT, "-l bytesize", "Size of shared memory log"); + fprintf(stderr, FMT, "-l shl,free,fill", "Size of shared memory file"); + fprintf(stderr, FMT, "", " shl: space for SHL records [80m]"); + fprintf(stderr, FMT, "", " free: space for other allocations [1m]"); + fprintf(stderr, FMT, "", " fill: prefill new file [+]"); fprintf(stderr, FMT, "-M address:port", "CLI-master to connect to."); fprintf(stderr, FMT, "-n dir", "varnishd working directory"); fprintf(stderr, FMT, "-P file", "PID file"); From phk at varnish-cache.org Mon May 17 09:58:26 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 11:58:26 +0200 Subject: r4799 - in trunk/varnish-cache: bin/varnishd include Message-ID: Author: phk Date: 2010-05-17 11:58:26 +0200 (Mon, 17 May 2010) New Revision: 4799 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/include/shmlog.h Log: Add a pool of shm we can allocate from. Allocate the params from it. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 08:51:09 UTC (rev 4798) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 09:58:26 UTC (rev 4799) @@ -39,6 +39,7 @@ #include #include #include +#include #include "shmlog.h" #include "mgt.h" @@ -57,16 +58,71 @@ unsigned char *logstart; static int vsl_fd = -1; -static unsigned vsl_size; /*--------------------------------------------------------------------*/ +static void * +mgt_SHM_Alloc(unsigned size, const char *type, const char *ident) +{ + struct shmalloc *sha, *sha2; + unsigned seq; + + /* Round up to pointersize */ + size += sizeof(sha) - 1; + size &= ~(sizeof(sha) - 1); + + size += sizeof *sha; + sha = &loghead->head; + while (1) { + CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); + + if (strcmp(sha->type, "Free")) { + sha = (void*)((uintptr_t)sha + sha->len); + continue; + } + assert(size <= sha->len); + + sha2 = (void*)((uintptr_t)sha + size); + + seq = loghead->alloc_seq; + loghead->alloc_seq = 0; + MEMORY_BARRIER(); + + memset(sha2, 0, sizeof *sha2); + sha2->magic = SHMALLOC_MAGIC; + sha2->len = sha->len - size; + bprintf(sha2->type, "%s", "Free"); + MEMORY_BARRIER(); + + sha->len = size; + bprintf(sha->type, "%s", type); + bprintf(sha->ident, "%s", ident); + MEMORY_BARRIER(); + + loghead->alloc_seq = seq++; + MEMORY_BARRIER(); + + return ((void*)(sha + 1)); + + } + return (NULL); +} + +/*--------------------------------------------------------------------*/ + static int -vsl_goodold(int fd) +vsl_goodold(int fd, unsigned size, unsigned s2) { struct shmloghead slh; int i; + struct stat st; + AZ(fstat(fd, &st)); + if (!S_ISREG(st.st_mode)) + ARGV_ERR("\t-l ...: Not a file\n"); + if (st.st_size != size) + return (0); + memset(&slh, 0, sizeof slh); /* XXX: for flexelint */ i = read(fd, &slh, sizeof slh); if (i != sizeof slh) @@ -75,7 +131,7 @@ return (0); if (slh.hdrsize != sizeof slh) return (0); - if (slh.start != sizeof slh + sizeof *params) + if (slh.start != s2) return (0); if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) { @@ -95,14 +151,11 @@ fprintf(stderr, "(We assume that process is busy dying.)\n"); return (0); } - - /* XXX more checks */ - vsl_size = slh.size + slh.start; return (1); } static void -vsl_buildnew(const char *fn, unsigned size, int fill) +vsl_buildnew(const char *fn, unsigned size, int fill, unsigned s2) { struct shmloghead slh; int i; @@ -122,10 +175,9 @@ slh.hdrsize = sizeof slh; slh.size = size; slh.ptr = 0; - slh.start = sizeof slh + sizeof *params; + slh.start = s2; i = write(vsl_fd, &slh, sizeof slh); xxxassert(i == sizeof slh); - vsl_size = slh.start + size; if (fill) { memset(buf, 0, sizeof buf); @@ -140,7 +192,7 @@ } } - AZ(ftruncate(vsl_fd, (off_t)vsl_size)); + AZ(ftruncate(vsl_fd, (off_t)size)); } void @@ -149,7 +201,7 @@ int i, fill; struct params *pp; const char *q; - uintmax_t size, s1, s2; + uintmax_t size, s1, s2, ps; char **av, **ap; if (l_arg == NULL) @@ -204,31 +256,48 @@ FreeArgv(av); size = s1 + s2; + ps = getpagesize(); + size += ps - 1; + size &= ~(ps - 1); if (av[2] == NULL) q = str2bytes(av[2], &size, 0); i = open(fn, O_RDWR, 0644); - if (i >= 0 && vsl_goodold(i)) { + if (i >= 0 && vsl_goodold(i, size, s2)) { fprintf(stderr, "Using old SHMFILE\n"); vsl_fd = i; } else { fprintf(stderr, "Creating new SHMFILE\n"); (void)close(i); - vsl_buildnew(fn, size, fill); + vsl_buildnew(fn, size, fill, s2); } - loghead = (void *)mmap(NULL, vsl_size, + loghead = (void *)mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, vsl_fd, 0); loghead->master_pid = getpid(); xxxassert(loghead != MAP_FAILED); - (void)mlock((void*)loghead, vsl_size); + (void)mlock((void*)loghead, size); VSL_stats = &loghead->stats; - pp = (void *)(loghead + 1); + + /* Initialize pool */ + loghead->alloc_seq = 0; + MEMORY_BARRIER(); + + memset(&loghead->head, 0, sizeof loghead->head); + loghead->head.magic = SHMALLOC_MAGIC; + loghead->head.len = s2 - sizeof *loghead; + bprintf(loghead->head.type, "%s", "Free"); + MEMORY_BARRIER(); + + pp = mgt_SHM_Alloc(sizeof *pp, "Params", ""); *pp = *params; params = pp; + + loghead->alloc_seq = random(); + MEMORY_BARRIER(); } void Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-17 08:51:09 UTC (rev 4798) +++ trunk/varnish-cache/include/shmlog.h 2010-05-17 09:58:26 UTC (rev 4799) @@ -43,8 +43,20 @@ #include "stats.h" +/* + * This structure describes each allocation from the shmlog + */ + +struct shmalloc { +#define SHMALLOC_MAGIC 0x43907b6e /* From /dev/random */ + unsigned magic; + unsigned len; + char type[8]; + char ident[8]; +}; + struct shmloghead { -#define SHMLOGHEAD_MAGIC 4185512499U /* From /dev/random */ +#define SHMLOGHEAD_MAGIC 4185512500U /* From /dev/random */ unsigned magic; unsigned hdrsize; @@ -69,6 +81,10 @@ /* Panic message buffer */ char panicstr[64 * 1024]; + + unsigned alloc_seq; + /* Must be last element */ + struct shmalloc head; }; /* From phk at varnish-cache.org Mon May 17 10:11:15 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 12:11:15 +0200 Subject: r4800 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 12:11:15 +0200 (Mon, 17 May 2010) New Revision: 4800 Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/mgt_child.c Log: Simplify the panic string processing Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_panic.c 2010-05-17 09:58:26 UTC (rev 4799) +++ trunk/varnish-cache/bin/varnishd/cache_panic.c 2010-05-17 10:11:15 UTC (rev 4800) @@ -44,6 +44,7 @@ #include #endif #include "cache.h" +#include "shmlog.h" #include "cache_backend.h" #include "vcl.h" #include "libvcl.h" @@ -57,7 +58,6 @@ * (gdb) printf "%s", panicstr */ -char panicstr[65536]; static struct vsb vsps, *vsp; /*--------------------------------------------------------------------*/ @@ -294,7 +294,6 @@ pan_ic(const char *func, const char *file, int line, const char *cond, int err, int xxx) { - int l; char *p; const char *q; const struct sess *sp; @@ -341,22 +340,19 @@ } vsb_printf(vsp, "\n"); vsb_bcat(vsp, "", 1); /* NUL termination */ - VSL_Panic(&l, &p); - if (l < sizeof(panicstr)) - l = sizeof(panicstr); - memcpy(p, panicstr, l); + if (params->diag_bitmap & 0x4000) - (void)fputs(panicstr, stderr); + (void)fputs(loghead->panicstr, stderr); #ifdef HAVE_ABORT2 if (params->diag_bitmap & 0x8000) { void *arg[1]; - for (p = panicstr; *p; p++) + for (p = loghead->panicstr; *p; p++) if (*p == '\n') *p = ' '; - arg[0] = panicstr; - abort2(panicstr, 1, arg); + arg[0] = loghead->panicstr; + abort2(loghead->panicstr, 1, arg); } #endif if (params->diag_bitmap & 0x1000) @@ -373,5 +369,6 @@ lbv_assert = pan_ic; vsp = &vsps; - AN(vsb_new(vsp, panicstr, sizeof panicstr, VSB_FIXEDLEN)); + AN(vsb_new(vsp, loghead->panicstr, sizeof loghead->panicstr, + VSB_FIXEDLEN)); } Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-17 09:58:26 UTC (rev 4799) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-17 10:11:15 UTC (rev 4800) @@ -266,25 +266,6 @@ /*--------------------------------------------------------------------*/ void -VSL_Panic(int *len, char **ptr) -{ - static char a[1] = { '\0' }; - - AN(len); - AN(ptr); - if (loghead->magic == SHMLOGHEAD_MAGIC) { - assert(loghead->hdrsize == sizeof *loghead); - *len = sizeof(loghead->panicstr); - *ptr = loghead->panicstr; - } else { - *len = 0; - *ptr = a; - } -} - -/*--------------------------------------------------------------------*/ - -void VSL_Init(void) { Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 09:58:26 UTC (rev 4799) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 10:11:15 UTC (rev 4800) @@ -38,9 +38,6 @@ /* cache_acceptor.c */ void VCA_tweak_waiter(struct cli *cli, const char *arg); -/* shmlog.c */ -void VSL_Panic(int *len, char **ptr); - /* mgt_shmem.c */ extern struct varnish_stats *VSL_stats; extern struct shmloghead *loghead; Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2010-05-17 09:58:26 UTC (rev 4799) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2010-05-17 10:11:15 UTC (rev 4800) @@ -59,6 +59,7 @@ #endif #include "mgt.h" +#include "shmlog.h" #include "heritage.h" #include "cli.h" #include "cli_priv.h" @@ -459,13 +460,11 @@ static void mgt_report_panic(pid_t r) { - int l; - char *p; - VSL_Panic(&l, &p); - if (*p == '\0') + if (loghead->panicstr[0] == '\0') return; - REPORT(LOG_ERR, "Child (%jd) Panic message: %s", (intmax_t)r, p); + REPORT(LOG_ERR, "Child (%jd) Panic message: %s", + (intmax_t)r, loghead->panicstr); } /*--------------------------------------------------------------------*/ From phk at varnish-cache.org Mon May 17 10:24:12 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 12:24:12 +0200 Subject: r4801 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 12:24:12 +0200 (Mon, 17 May 2010) New Revision: 4801 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c Log: Minor polishing Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 10:11:15 UTC (rev 4800) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 10:24:12 UTC (rev 4801) @@ -240,7 +240,7 @@ /* Fill or not ? */ if (*ap != NULL) { - if (*ap == '\0') + if (**ap == '\0') fill = 1; else if (!strcmp(*ap, "-")) fill = 0; @@ -253,6 +253,9 @@ fill = 1; } + if (*ap != NULL) + ARGV_ERR("\t-l ...: Too many sub-args\n"); + FreeArgv(av); size = s1 + s2; @@ -260,9 +263,6 @@ size += ps - 1; size &= ~(ps - 1); - if (av[2] == NULL) - q = str2bytes(av[2], &size, 0); - i = open(fn, O_RDWR, 0644); if (i >= 0 && vsl_goodold(i, size, s2)) { fprintf(stderr, "Using old SHMFILE\n"); From phk at varnish-cache.org Mon May 17 12:30:54 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 14:30:54 +0200 Subject: r4802 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 14:30:54 +0200 (Mon, 17 May 2010) New Revision: 4802 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c Log: Do pid checks before compat checks. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 10:24:12 UTC (rev 4801) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 12:30:54 UTC (rev 4802) @@ -108,7 +108,10 @@ return (NULL); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Try to reuse an existing shmem file, but try to not disturb another + * varnishd using the file. + */ static int vsl_goodold(int fd, unsigned size, unsigned s2) @@ -119,9 +122,7 @@ AZ(fstat(fd, &st)); if (!S_ISREG(st.st_mode)) - ARGV_ERR("\t-l ...: Not a file\n"); - if (st.st_size != size) - return (0); + ARGV_ERR("\tshmlog: Not a file\n"); memset(&slh, 0, sizeof slh); /* XXX: for flexelint */ i = read(fd, &slh, sizeof slh); @@ -131,8 +132,6 @@ return (0); if (slh.hdrsize != sizeof slh) return (0); - if (slh.start != s2) - return (0); if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) { fprintf(stderr, @@ -151,6 +150,15 @@ fprintf(stderr, "(We assume that process is busy dying.)\n"); return (0); } + + /* Sanity checks */ + + if (slh.start != s2) + return (0); + + if (st.st_size != size) + return (0); + return (1); } From phk at varnish-cache.org Mon May 17 13:22:53 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 15:22:53 +0200 Subject: r4803 - in trunk/varnish-cache: bin/varnishd include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-17 15:22:53 +0200 (Mon, 17 May 2010) New Revision: 4803 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/include/shmlog.h trunk/varnish-cache/include/stats.h trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Move the stats into an allocated chunk of shmem. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 12:30:54 UTC (rev 4802) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 13:22:53 UTC (rev 4803) @@ -77,7 +77,7 @@ CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); if (strcmp(sha->type, "Free")) { - sha = (void*)((uintptr_t)sha + sha->len); + sha = SHA_NEXT(sha); continue; } assert(size <= sha->len); @@ -288,7 +288,6 @@ loghead->master_pid = getpid(); xxxassert(loghead != MAP_FAILED); (void)mlock((void*)loghead, size); - VSL_stats = &loghead->stats; /* Initialize pool */ loghead->alloc_seq = 0; @@ -300,7 +299,11 @@ bprintf(loghead->head.type, "%s", "Free"); MEMORY_BARRIER(); + VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_STAT_TYPE, ""); + AN(VSL_stats); + pp = mgt_SHM_Alloc(sizeof *pp, "Params", ""); + AN(pp); *pp = *params; params = pp; Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-17 12:30:54 UTC (rev 4802) +++ trunk/varnish-cache/include/shmlog.h 2010-05-17 13:22:53 UTC (rev 4803) @@ -55,6 +55,9 @@ char ident[8]; }; +#define SHA_NEXT(sha) ((void*)((uintptr_t)(sha) + (sha)->len)) +#define SHA_PTR(sha) ((void*)((uintptr_t)((sha) + 1))) + struct shmloghead { #define SHMLOGHEAD_MAGIC 4185512500U /* From /dev/random */ unsigned magic; @@ -77,8 +80,6 @@ /* Current write position relative to the beginning of start */ unsigned ptr; - struct varnish_stats stats; - /* Panic message buffer */ char panicstr[64 * 1024]; Modified: trunk/varnish-cache/include/stats.h =================================================================== --- trunk/varnish-cache/include/stats.h 2010-05-17 12:30:54 UTC (rev 4802) +++ trunk/varnish-cache/include/stats.h 2010-05-17 13:22:53 UTC (rev 4803) @@ -31,6 +31,8 @@ #include +#define VSL_STAT_TYPE "Stats" + struct varnish_stats { #define MAC_STAT(n, t, l, f, e) t n; #include "stat_field.h" Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-17 12:30:54 UTC (rev 4802) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-17 13:22:53 UTC (rev 4803) @@ -57,6 +57,7 @@ int VSL_NextLog(struct VSL_data *lh, unsigned char **pp); int VSL_Arg(struct VSL_data *vd, int arg, const char *opt); void VSL_Close(struct VSL_data *vd); +int VSL_Open(struct VSL_data *vd); void VSL_Delete(struct VSL_data *vd); struct varnish_stats *VSL_OpenStats(struct VSL_data *vd); const char *VSL_Name(struct VSL_data *vd); Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-17 12:30:54 UTC (rev 4802) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-17 13:22:53 UTC (rev 4803) @@ -123,8 +123,8 @@ /*--------------------------------------------------------------------*/ -static int -vsl_shmem_map(struct VSL_data *vd) +int +VSL_Open(struct VSL_data *vd) { int i; struct shmloghead slh; @@ -180,7 +180,6 @@ vd->magic = VSL_MAGIC; vd->vsl_fd = -1; - /* XXX: Allocate only if log access */ vd->vbm_client = vbit_init(4096); vd->vbm_backend = vbit_init(4096); @@ -233,7 +232,7 @@ if (vd->r_fd != -1) return (0); - if (vsl_shmem_map(vd)) + if (VSL_Open(vd)) return (-1); vd->head = vd->vsl_lh; @@ -611,13 +610,36 @@ } } +/*--------------------------------------------------------------------*/ + +static +struct shmalloc * +vsl_find_alloc(struct VSL_data *vd, const char *type, const char *ident) +{ + struct shmalloc *sha; + + assert (vd->vsl_lh != NULL); + for(sha = &vd->vsl_lh->head; ; sha = SHA_NEXT(sha)) { + CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); + if (strcmp(sha->type, type)) + continue; + if (ident != NULL && strcmp(sha->ident, ident)) + continue; + return (sha); + } + return (NULL); +} + struct varnish_stats * VSL_OpenStats(struct VSL_data *vd) { + struct shmalloc *sha; - if (vsl_shmem_map(vd)) + if (VSL_Open(vd)) return (NULL); - return (&vd->vsl_lh->stats); + sha = vsl_find_alloc(vd, VSL_STAT_TYPE, ""); + assert(sha != NULL); + return (SHA_PTR(sha)); } void From phk at varnish-cache.org Mon May 17 15:27:51 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 17:27:51 +0200 Subject: r4804 - trunk/varnish-cache/lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-17 17:27:51 +0200 (Mon, 17 May 2010) New Revision: 4804 Added: trunk/varnish-cache/lib/libvarnishapi/vsl.c trunk/varnish-cache/lib/libvarnishapi/vsl.h trunk/varnish-cache/lib/libvarnishapi/vsl_arg.c trunk/varnish-cache/lib/libvarnishapi/vsl_log.c Removed: trunk/varnish-cache/lib/libvarnishapi/shmlog.c Modified: trunk/varnish-cache/lib/libvarnishapi/Makefile.am Log: Split shmlog.c into more appropriately named vsl*[.ch] files Modified: trunk/varnish-cache/lib/libvarnishapi/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2010-05-17 13:22:53 UTC (rev 4803) +++ trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2010-05-17 15:27:51 UTC (rev 4804) @@ -9,7 +9,9 @@ libvarnishapi_la_SOURCES = \ ../libvarnish/vin.c \ base64.c \ - shmlog.c + vsl.c \ + vsl_arg.c \ + vsl_log.c libvarnishapi_la_CFLAGS = \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' Deleted: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-17 13:22:53 UTC (rev 4803) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2010-05-17 15:27:51 UTC (rev 4804) @@ -1,663 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2010 Redpill Linpro AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include "svnid.h" -SVNID("$Id$") - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "shmlog.h" -#include "vre.h" -#include "vbm.h" -#include "miniobj.h" -#include "varnishapi.h" - -/* Parameters */ -#define SLEEP_USEC (50*1000) -#define TIMEOUT_USEC (5*1000*1000) - -struct VSL_data { - unsigned magic; -#define VSL_MAGIC 0x6e3bd69b - - struct shmloghead *head; - unsigned char *logstart; - unsigned char *logend; - unsigned char *ptr; - - /* for -r option */ - int r_fd; - unsigned rbuflen; - unsigned char *rbuf; - - unsigned L_opt; - char *n_opt; - int b_opt; - int c_opt; - int d_opt; - - unsigned flags; -#define F_SEEN_IX (1 << 0) -#define F_NON_BLOCKING (1 << 1) - - /* - * These two bitmaps mark fd's as belonging to client or backend - * transactions respectively. - */ - struct vbitmap *vbm_client; - struct vbitmap *vbm_backend; - - /* - * Bit map of programatically selected tags, that cannot be suppressed. - * This way programs can make sure they will see certain tags, even - * if the user tries to supress them with -x/-X - */ - struct vbitmap *vbm_select; /* index: tag */ - - /* Bit map of tags selected/supressed with -[iIxX] options */ - struct vbitmap *vbm_supress; /* index: tag */ - - int regflags; - vre_t *regincl; - vre_t *regexcl; - - unsigned long skip; - unsigned long keep; - - int vsl_fd; - struct shmloghead *vsl_lh; -}; - -#ifndef MAP_HASSEMAPHORE -#define MAP_HASSEMAPHORE 0 /* XXX Linux */ -#endif - -static int vsl_nextlog(struct VSL_data *vd, unsigned char **pp); - -/*--------------------------------------------------------------------*/ - -const char *VSL_tags[256] = { -#define SLTM(foo) [SLT_##foo] = #foo, -#include "shmlog_tags.h" -#undef SLTM -}; - -/*--------------------------------------------------------------------*/ - -int -VSL_Open(struct VSL_data *vd) -{ - int i; - struct shmloghead slh; - char *logname; - - if (vd->vsl_lh != NULL) - return (0); - - if (vin_n_arg(vd->n_opt, NULL, NULL, &logname)) { - fprintf(stderr, "Invalid instance name: %s\n", - strerror(errno)); - return (1); - } - - vd->vsl_fd = open(logname, O_RDONLY); - if (vd->vsl_fd < 0) { - fprintf(stderr, "Cannot open %s: %s\n", - logname, strerror(errno)); - return (1); - } - i = read(vd->vsl_fd, &slh, sizeof slh); - if (i != sizeof slh) { - fprintf(stderr, "Cannot read %s: %s\n", - logname, strerror(errno)); - return (1); - } - if (slh.magic != SHMLOGHEAD_MAGIC) { - fprintf(stderr, "Wrong magic number in file %s\n", - logname); - return (1); - } - - vd->vsl_lh = (void *)mmap(NULL, slh.size + sizeof slh, - PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsl_fd, 0); - if (vd->vsl_lh == MAP_FAILED) { - fprintf(stderr, "Cannot mmap %s: %s\n", - logname, strerror(errno)); - return (1); - } - return (0); -} - -/*--------------------------------------------------------------------*/ - -struct VSL_data * -VSL_New(void) -{ - struct VSL_data *vd; - - vd = calloc(sizeof *vd, 1); - assert(vd != NULL); - vd->regflags = 0; - vd->magic = VSL_MAGIC; - vd->vsl_fd = -1; - - /* XXX: Allocate only if log access */ - vd->vbm_client = vbit_init(4096); - vd->vbm_backend = vbit_init(4096); - vd->vbm_supress = vbit_init(256); - vd->vbm_select = vbit_init(256); - - vd->r_fd = -1; - /* XXX: Allocate only if -r option given ? */ - vd->rbuflen = SHMLOG_NEXTTAG + 256; - vd->rbuf = malloc(vd->rbuflen); - assert(vd->rbuf != NULL); - - return (vd); -} - -/*--------------------------------------------------------------------*/ - -void -VSL_Delete(struct VSL_data *vd) -{ - - VSL_Close(vd); - vbit_destroy(vd->vbm_client); - vbit_destroy(vd->vbm_backend); - vbit_destroy(vd->vbm_supress); - vbit_destroy(vd->vbm_select); - free(vd->n_opt); - free(vd->rbuf); - free(vd); -} - -/*--------------------------------------------------------------------*/ - -void -VSL_Select(struct VSL_data *vd, unsigned tag) -{ - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - vbit_set(vd->vbm_select, tag); -} - -/*--------------------------------------------------------------------*/ - -int -VSL_OpenLog(struct VSL_data *vd) -{ - unsigned char *p; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (vd->r_fd != -1) - return (0); - - if (VSL_Open(vd)) - return (-1); - - vd->head = vd->vsl_lh; - vd->logstart = (unsigned char *)vd->vsl_lh + vd->vsl_lh->start; - vd->logend = vd->logstart + vd->vsl_lh->size; - vd->ptr = vd->logstart; - - if (!vd->d_opt && vd->r_fd == -1) { - for (p = vd->ptr; *p != SLT_ENDMARKER; ) - p += SHMLOG_LEN(p) + SHMLOG_NEXTTAG; - vd->ptr = p; - } - return (0); -} - -/*--------------------------------------------------------------------*/ - -void -VSL_NonBlocking(struct VSL_data *vd, int nb) -{ - if (nb) - vd->flags |= F_NON_BLOCKING; - else - vd->flags &= ~F_NON_BLOCKING; -} - -/*--------------------------------------------------------------------*/ - -static int -vsl_nextlog(struct VSL_data *vd, unsigned char **pp) -{ - unsigned char *p; - unsigned w, l; - int i; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (vd->r_fd != -1) { - assert(vd->rbuflen >= SHMLOG_DATA); - i = read(vd->r_fd, vd->rbuf, SHMLOG_DATA); - if (i != SHMLOG_DATA) - return (-1); - l = SHMLOG_LEN(vd->rbuf) + SHMLOG_NEXTTAG; - if (vd->rbuflen < l) { - l += 200; - vd->rbuf = realloc(vd->rbuf, l); - assert(vd->rbuf != NULL); - vd->rbuflen = l; - } - l = SHMLOG_LEN(vd->rbuf) + 1; - i = read(vd->r_fd, vd->rbuf + SHMLOG_DATA, l); - if (i != l) - return (-1); - *pp = vd->rbuf; - return (1); - } - - p = vd->ptr; - for (w = 0; w < TIMEOUT_USEC;) { - if (*p == SLT_WRAPMARKER) { - p = vd->logstart; - continue; - } - if (*p == SLT_ENDMARKER) { - if (vd->flags & F_NON_BLOCKING) - return (-1); - w += SLEEP_USEC; - usleep(SLEEP_USEC); - continue; - } - l = SHMLOG_LEN(p); - vd->ptr = p + l + SHMLOG_NEXTTAG; - *pp = p; - return (1); - } - vd->ptr = p; - return (0); -} - -int -VSL_NextLog(struct VSL_data *vd, unsigned char **pp) -{ - unsigned char *p, t; - unsigned u, l; - int i; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - while (1) { - i = vsl_nextlog(vd, &p); - if (i != 1) - return (i); - u = SHMLOG_ID(p); - l = SHMLOG_LEN(p); - switch(p[SHMLOG_TAG]) { - case SLT_SessionOpen: - case SLT_ReqStart: - vbit_set(vd->vbm_client, u); - vbit_clr(vd->vbm_backend, u); - break; - case SLT_BackendOpen: - case SLT_BackendXID: - vbit_clr(vd->vbm_client, u); - vbit_set(vd->vbm_backend, u); - break; - default: - break; - } - if (vd->skip) { - --vd->skip; - continue; - } else if (vd->keep) { - if (--vd->keep == 0) - return (-1); - } - t = p[SHMLOG_TAG]; - if (vbit_test(vd->vbm_select, t)) { - *pp = p; - return (1); - } - if (vbit_test(vd->vbm_supress, t)) - continue; - if (vd->b_opt && !vbit_test(vd->vbm_backend, u)) - continue; - if (vd->c_opt && !vbit_test(vd->vbm_client, u)) - continue; - if (vd->regincl != NULL) { - i = VRE_exec(vd->regincl, - (char *)p + SHMLOG_DATA, - SHMLOG_LEN(p) - SHMLOG_DATA, /* Length */ - 0, 0, NULL, 0); - if (i == VRE_ERROR_NOMATCH) - continue; - } - if (vd->regexcl != NULL) { - i = VRE_exec(vd->regincl, - (char *)p + SHMLOG_DATA, - SHMLOG_LEN(p) - SHMLOG_DATA, /* Length */ - 0, 0, NULL, 0); - if (i != VRE_ERROR_NOMATCH) - continue; - } - *pp = p; - return (1); - } -} - -/*--------------------------------------------------------------------*/ - -int -VSL_Dispatch(struct VSL_data *vd, vsl_handler *func, void *priv) -{ - int i; - unsigned u, l, s; - unsigned char *p; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - while (1) { - i = VSL_NextLog(vd, &p); - if (i <= 0) - return (i); - u = SHMLOG_ID(p); - l = SHMLOG_LEN(p); - s = 0; - if (vbit_test(vd->vbm_backend, u)) - s |= VSL_S_BACKEND; - if (vbit_test(vd->vbm_client, u)) - s |= VSL_S_CLIENT; - if (func(priv, - p[SHMLOG_TAG], u, l, s, (char *)p + SHMLOG_DATA)) - return (1); - } -} - -/*--------------------------------------------------------------------*/ - -int -VSL_H_Print(void *priv, enum shmlogtag tag, unsigned fd, unsigned len, - unsigned spec, const char *ptr) -{ - FILE *fo = priv; - int type; - - assert(fo != NULL); - - type = (spec & VSL_S_CLIENT) ? 'c' : - (spec & VSL_S_BACKEND) ? 'b' : '-'; - - if (tag == SLT_Debug) { - fprintf(fo, "%5d %-12s %c \"", fd, VSL_tags[tag], type); - while (len-- > 0) { - if (*ptr >= ' ' && *ptr <= '~') - fprintf(fo, "%c", *ptr); - else - fprintf(fo, "%%%02x", (unsigned char)*ptr); - ptr++; - } - fprintf(fo, "\"\n"); - return (0); - } - fprintf(fo, "%5d %-12s %c %.*s\n", fd, VSL_tags[tag], type, len, ptr); - return (0); -} - -/*--------------------------------------------------------------------*/ - -static int -vsl_r_arg(struct VSL_data *vd, const char *opt) -{ - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (!strcmp(opt, "-")) - vd->r_fd = STDIN_FILENO; - else - vd->r_fd = open(opt, O_RDONLY); - if (vd->r_fd < 0) { - perror(opt); - return (-1); - } - return (1); -} - -/*--------------------------------------------------------------------*/ - -static int -vsl_IX_arg(struct VSL_data *vd, const char *opt, int arg) -{ - vre_t **rp; - const char *error; - int erroroffset; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (arg == 'I') - rp = &vd->regincl; - else - rp = &vd->regexcl; - if (*rp != NULL) { - fprintf(stderr, "Option %c can only be given once", arg); - return (-1); - } - *rp = VRE_compile(opt, vd->regflags, &error, &erroroffset); - if (*rp == NULL) { - fprintf(stderr, "Illegal regex: %s\n", error); - return (-1); - } - return (1); -} - -/*--------------------------------------------------------------------*/ - -static int -vsl_ix_arg(struct VSL_data *vd, const char *opt, int arg) -{ - int i, j, l; - const char *b, *e, *p, *q; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - /* If first option is 'i', set all bits for supression */ - if (arg == 'i' && !(vd->flags & F_SEEN_IX)) - for (i = 0; i < 256; i++) - vbit_set(vd->vbm_supress, i); - vd->flags |= F_SEEN_IX; - - for (b = opt; *b; b = e) { - while (isspace(*b)) - b++; - e = strchr(b, ','); - if (e == NULL) - e = strchr(b, '\0'); - l = e - b; - if (*e == ',') - e++; - while (isspace(b[l - 1])) - l--; - for (i = 0; i < 256; i++) { - if (VSL_tags[i] == NULL) - continue; - p = VSL_tags[i]; - q = b; - for (j = 0; j < l; j++) - if (tolower(*q++) != tolower(*p++)) - break; - if (j != l || *p != '\0') - continue; - - if (arg == 'x') - vbit_set(vd->vbm_supress, i); - else - vbit_clr(vd->vbm_supress, i); - break; - } - if (i == 256) { - fprintf(stderr, - "Could not match \"%*.*s\" to any tag\n", l, l, b); - return (-1); - } - } - return (1); -} - -/*--------------------------------------------------------------------*/ - -static int -vsl_s_arg(struct VSL_data *vd, const char *opt) -{ - char *end; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (*opt == '\0') { - fprintf(stderr, "number required for -s\n"); - return (-1); - } - vd->skip = strtoul(opt, &end, 10); - if (*end != '\0') { - fprintf(stderr, "invalid number for -s\n"); - return (-1); - } - return (1); -} -/*--------------------------------------------------------------------*/ - -static int -vsl_k_arg(struct VSL_data *vd, const char *opt) -{ - char *end; - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (*opt == '\0') { - fprintf(stderr, "number required for -k\n"); - return (-1); - } - vd->keep = strtoul(opt, &end, 10); - if (*end != '\0') { - fprintf(stderr, "invalid number for -k\n"); - return (-1); - } - return (1); -} - -/*--------------------------------------------------------------------*/ - -int -VSL_Arg(struct VSL_data *vd, int arg, const char *opt) -{ - - CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - switch (arg) { - case 'b': vd->b_opt = !vd->b_opt; return (1); - case 'c': vd->c_opt = !vd->c_opt; return (1); - case 'd': - vd->d_opt = !vd->d_opt; - vd->flags |= F_NON_BLOCKING; - return (1); - case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); - case 'k': return (vsl_k_arg(vd, opt)); - case 'n': - free(vd->n_opt); - vd->n_opt = strdup(opt); - assert(vd->n_opt != NULL); - return (1); - case 'r': return (vsl_r_arg(vd, opt)); - case 's': return (vsl_s_arg(vd, opt)); - case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); - case 'C': vd->regflags = VRE_CASELESS; return (1); - case 'L': - vd->L_opt = strtoul(opt, NULL, 0); - if (vd->L_opt < 1024 || vd->L_opt > 65000) { - fprintf(stderr, "%s\n", VIN_L_MSG); - exit (1); - } - free(vd->n_opt); - vd->n_opt = vin_L_arg(vd->L_opt); - assert(vd->n_opt != NULL); - return (1); - default: - return (0); - } -} - -/*--------------------------------------------------------------------*/ - -static -struct shmalloc * -vsl_find_alloc(struct VSL_data *vd, const char *type, const char *ident) -{ - struct shmalloc *sha; - - assert (vd->vsl_lh != NULL); - for(sha = &vd->vsl_lh->head; ; sha = SHA_NEXT(sha)) { - CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); - if (strcmp(sha->type, type)) - continue; - if (ident != NULL && strcmp(sha->ident, ident)) - continue; - return (sha); - } - return (NULL); -} - -struct varnish_stats * -VSL_OpenStats(struct VSL_data *vd) -{ - struct shmalloc *sha; - - if (VSL_Open(vd)) - return (NULL); - sha = vsl_find_alloc(vd, VSL_STAT_TYPE, ""); - assert(sha != NULL); - return (SHA_PTR(sha)); -} - -void -VSL_Close(struct VSL_data *vd) -{ - if (vd->vsl_lh == NULL) - return; - assert(0 == munmap((void*)vd->vsl_lh, - vd->vsl_lh->size + sizeof *vd->vsl_lh)); - vd->vsl_lh = NULL; - assert(vd->vsl_fd >= 0); - assert(0 == close(vd->vsl_fd)); - vd->vsl_fd = -1; -} - -const char * -VSL_Name(struct VSL_data *vd) -{ - - return (vd->n_opt); -} Copied: trunk/varnish-cache/lib/libvarnishapi/vsl.c (from rev 4803, trunk/varnish-cache/lib/libvarnishapi/shmlog.c) =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c (rev 0) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-17 15:27:51 UTC (rev 4804) @@ -0,0 +1,232 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id$") + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "vre.h" +#include "vbm.h" +#include "miniobj.h" +#include "varnishapi.h" + +#include "vsl.h" + +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + +/*--------------------------------------------------------------------*/ + +struct VSL_data * +VSL_New(void) +{ + struct VSL_data *vd; + + vd = calloc(sizeof *vd, 1); + assert(vd != NULL); + vd->regflags = 0; + vd->magic = VSL_MAGIC; + vd->vsl_fd = -1; + + /* XXX: Allocate only if log access */ + vd->vbm_client = vbit_init(4096); + vd->vbm_backend = vbit_init(4096); + vd->vbm_supress = vbit_init(256); + vd->vbm_select = vbit_init(256); + + vd->r_fd = -1; + /* XXX: Allocate only if -r option given ? */ + vd->rbuflen = SHMLOG_NEXTTAG + 256; + vd->rbuf = malloc(vd->rbuflen); + assert(vd->rbuf != NULL); + + return (vd); +} + +/*--------------------------------------------------------------------*/ + +void +VSL_Delete(struct VSL_data *vd) +{ + + VSL_Close(vd); + vbit_destroy(vd->vbm_client); + vbit_destroy(vd->vbm_backend); + vbit_destroy(vd->vbm_supress); + vbit_destroy(vd->vbm_select); + free(vd->n_opt); + free(vd->rbuf); + free(vd); +} + +/*--------------------------------------------------------------------*/ + +int +VSL_Open(struct VSL_data *vd) +{ + int i; + struct shmloghead slh; + char *logname; + + if (vd->vsl_lh != NULL) + return (0); + + if (vin_n_arg(vd->n_opt, NULL, NULL, &logname)) { + fprintf(stderr, "Invalid instance name: %s\n", + strerror(errno)); + return (1); + } + + vd->vsl_fd = open(logname, O_RDONLY); + if (vd->vsl_fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + logname, strerror(errno)); + return (1); + } + i = read(vd->vsl_fd, &slh, sizeof slh); + if (i != sizeof slh) { + fprintf(stderr, "Cannot read %s: %s\n", + logname, strerror(errno)); + return (1); + } + if (slh.magic != SHMLOGHEAD_MAGIC) { + fprintf(stderr, "Wrong magic number in file %s\n", + logname); + return (1); + } + + vd->vsl_lh = (void *)mmap(NULL, slh.size + sizeof slh, + PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsl_fd, 0); + if (vd->vsl_lh == MAP_FAILED) { + fprintf(stderr, "Cannot mmap %s: %s\n", + logname, strerror(errno)); + return (1); + } + return (0); +} + +/*--------------------------------------------------------------------*/ + +void +VSL_Close(struct VSL_data *vd) +{ + if (vd->vsl_lh == NULL) + return; + assert(0 == munmap((void*)vd->vsl_lh, + vd->vsl_lh->size + sizeof *vd->vsl_lh)); + vd->vsl_lh = NULL; + assert(vd->vsl_fd >= 0); + assert(0 == close(vd->vsl_fd)); + vd->vsl_fd = -1; +} + +/*--------------------------------------------------------------------*/ + +static struct shmalloc * +vsl_find_alloc(struct VSL_data *vd, const char *type, const char *ident) +{ + struct shmalloc *sha; + + assert (vd->vsl_lh != NULL); + for(sha = &vd->vsl_lh->head; ; sha = SHA_NEXT(sha)) { + CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); + if (strcmp(sha->type, type)) + continue; + if (ident != NULL && strcmp(sha->ident, ident)) + continue; + return (sha); + } + return (NULL); +} + +/*--------------------------------------------------------------------*/ + +struct varnish_stats * +VSL_OpenStats(struct VSL_data *vd) +{ + struct shmalloc *sha; + + if (VSL_Open(vd)) + return (NULL); + sha = vsl_find_alloc(vd, VSL_STAT_TYPE, ""); + assert(sha != NULL); + return (SHA_PTR(sha)); +} + +/*--------------------------------------------------------------------*/ + +int +VSL_OpenLog(struct VSL_data *vd) +{ + unsigned char *p; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (vd->r_fd != -1) + return (0); + + if (VSL_Open(vd)) + return (-1); + + vd->head = vd->vsl_lh; + vd->logstart = (unsigned char *)vd->vsl_lh + vd->vsl_lh->start; + vd->logend = vd->logstart + vd->vsl_lh->size; + vd->ptr = vd->logstart; + + if (!vd->d_opt && vd->r_fd == -1) { + for (p = vd->ptr; *p != SLT_ENDMARKER; ) + p += SHMLOG_LEN(p) + SHMLOG_NEXTTAG; + vd->ptr = p; + } + return (0); +} + +/*--------------------------------------------------------------------*/ + +const char * +VSL_Name(struct VSL_data *vd) +{ + + return (vd->n_opt); +} Copied: trunk/varnish-cache/lib/libvarnishapi/vsl.h (from rev 4803, trunk/varnish-cache/lib/libvarnishapi/shmlog.c) =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.h (rev 0) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-17 15:27:51 UTC (rev 4804) @@ -0,0 +1,86 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id$ + */ + +/* Parameters */ +#define SLEEP_USEC (50*1000) +#define TIMEOUT_USEC (5*1000*1000) + +struct VSL_data { + unsigned magic; +#define VSL_MAGIC 0x6e3bd69b + + struct shmloghead *head; + unsigned char *logstart; + unsigned char *logend; + unsigned char *ptr; + + /* for -r option */ + int r_fd; + unsigned rbuflen; + unsigned char *rbuf; + + unsigned L_opt; + char *n_opt; + int b_opt; + int c_opt; + int d_opt; + + unsigned flags; +#define F_SEEN_IX (1 << 0) +#define F_NON_BLOCKING (1 << 1) + + /* + * These two bitmaps mark fd's as belonging to client or backend + * transactions respectively. + */ + struct vbitmap *vbm_client; + struct vbitmap *vbm_backend; + + /* + * Bit map of programatically selected tags, that cannot be suppressed. + * This way programs can make sure they will see certain tags, even + * if the user tries to supress them with -x/-X + */ + struct vbitmap *vbm_select; /* index: tag */ + + /* Bit map of tags selected/supressed with -[iIxX] options */ + struct vbitmap *vbm_supress; /* index: tag */ + + int regflags; + vre_t *regincl; + vre_t *regexcl; + + unsigned long skip; + unsigned long keep; + + int vsl_fd; + struct shmloghead *vsl_lh; +}; Copied: trunk/varnish-cache/lib/libvarnishapi/vsl_arg.c (from rev 4803, trunk/varnish-cache/lib/libvarnishapi/shmlog.c) =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl_arg.c (rev 0) +++ trunk/varnish-cache/lib/libvarnishapi/vsl_arg.c 2010-05-17 15:27:51 UTC (rev 4804) @@ -0,0 +1,233 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id$") + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "vre.h" +#include "vbm.h" +#include "miniobj.h" +#include "varnishapi.h" + +#include "vsl.h" + +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + +/*--------------------------------------------------------------------*/ + +static int +vsl_r_arg(struct VSL_data *vd, const char *opt) +{ + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (!strcmp(opt, "-")) + vd->r_fd = STDIN_FILENO; + else + vd->r_fd = open(opt, O_RDONLY); + if (vd->r_fd < 0) { + perror(opt); + return (-1); + } + return (1); +} + +/*--------------------------------------------------------------------*/ + +static int +vsl_IX_arg(struct VSL_data *vd, const char *opt, int arg) +{ + vre_t **rp; + const char *error; + int erroroffset; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (arg == 'I') + rp = &vd->regincl; + else + rp = &vd->regexcl; + if (*rp != NULL) { + fprintf(stderr, "Option %c can only be given once", arg); + return (-1); + } + *rp = VRE_compile(opt, vd->regflags, &error, &erroroffset); + if (*rp == NULL) { + fprintf(stderr, "Illegal regex: %s\n", error); + return (-1); + } + return (1); +} + +/*--------------------------------------------------------------------*/ + +static int +vsl_ix_arg(struct VSL_data *vd, const char *opt, int arg) +{ + int i, j, l; + const char *b, *e, *p, *q; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + /* If first option is 'i', set all bits for supression */ + if (arg == 'i' && !(vd->flags & F_SEEN_IX)) + for (i = 0; i < 256; i++) + vbit_set(vd->vbm_supress, i); + vd->flags |= F_SEEN_IX; + + for (b = opt; *b; b = e) { + while (isspace(*b)) + b++; + e = strchr(b, ','); + if (e == NULL) + e = strchr(b, '\0'); + l = e - b; + if (*e == ',') + e++; + while (isspace(b[l - 1])) + l--; + for (i = 0; i < 256; i++) { + if (VSL_tags[i] == NULL) + continue; + p = VSL_tags[i]; + q = b; + for (j = 0; j < l; j++) + if (tolower(*q++) != tolower(*p++)) + break; + if (j != l || *p != '\0') + continue; + + if (arg == 'x') + vbit_set(vd->vbm_supress, i); + else + vbit_clr(vd->vbm_supress, i); + break; + } + if (i == 256) { + fprintf(stderr, + "Could not match \"%*.*s\" to any tag\n", l, l, b); + return (-1); + } + } + return (1); +} + +/*--------------------------------------------------------------------*/ + +static int +vsl_s_arg(struct VSL_data *vd, const char *opt) +{ + char *end; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (*opt == '\0') { + fprintf(stderr, "number required for -s\n"); + return (-1); + } + vd->skip = strtoul(opt, &end, 10); + if (*end != '\0') { + fprintf(stderr, "invalid number for -s\n"); + return (-1); + } + return (1); +} +/*--------------------------------------------------------------------*/ + +static int +vsl_k_arg(struct VSL_data *vd, const char *opt) +{ + char *end; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (*opt == '\0') { + fprintf(stderr, "number required for -k\n"); + return (-1); + } + vd->keep = strtoul(opt, &end, 10); + if (*end != '\0') { + fprintf(stderr, "invalid number for -k\n"); + return (-1); + } + return (1); +} + +/*--------------------------------------------------------------------*/ + +int +VSL_Arg(struct VSL_data *vd, int arg, const char *opt) +{ + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + switch (arg) { + case 'b': vd->b_opt = !vd->b_opt; return (1); + case 'c': vd->c_opt = !vd->c_opt; return (1); + case 'd': + vd->d_opt = !vd->d_opt; + vd->flags |= F_NON_BLOCKING; + return (1); + case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); + case 'k': return (vsl_k_arg(vd, opt)); + case 'n': + free(vd->n_opt); + vd->n_opt = strdup(opt); + assert(vd->n_opt != NULL); + return (1); + case 'r': return (vsl_r_arg(vd, opt)); + case 's': return (vsl_s_arg(vd, opt)); + case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); + case 'C': vd->regflags = VRE_CASELESS; return (1); + case 'L': + vd->L_opt = strtoul(opt, NULL, 0); + if (vd->L_opt < 1024 || vd->L_opt > 65000) { + fprintf(stderr, "%s\n", VIN_L_MSG); + exit (1); + } + free(vd->n_opt); + vd->n_opt = vin_L_arg(vd->L_opt); + assert(vd->n_opt != NULL); + return (1); + default: + return (0); + } +} Copied: trunk/varnish-cache/lib/libvarnishapi/vsl_log.c (from rev 4803, trunk/varnish-cache/lib/libvarnishapi/shmlog.c) =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl_log.c (rev 0) +++ trunk/varnish-cache/lib/libvarnishapi/vsl_log.c 2010-05-17 15:27:51 UTC (rev 4804) @@ -0,0 +1,266 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id$") + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "vre.h" +#include "vbm.h" +#include "miniobj.h" +#include "varnishapi.h" + +#include "vsl.h" + +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + +static int vsl_nextlog(struct VSL_data *vd, unsigned char **pp); + +/*--------------------------------------------------------------------*/ + +const char *VSL_tags[256] = { +#define SLTM(foo) [SLT_##foo] = #foo, +#include "shmlog_tags.h" +#undef SLTM +}; + +/*--------------------------------------------------------------------*/ + +void +VSL_Select(struct VSL_data *vd, unsigned tag) +{ + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + vbit_set(vd->vbm_select, tag); +} + + +/*--------------------------------------------------------------------*/ + +void +VSL_NonBlocking(struct VSL_data *vd, int nb) +{ + if (nb) + vd->flags |= F_NON_BLOCKING; + else + vd->flags &= ~F_NON_BLOCKING; +} + +/*--------------------------------------------------------------------*/ + +static int +vsl_nextlog(struct VSL_data *vd, unsigned char **pp) +{ + unsigned char *p; + unsigned w, l; + int i; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (vd->r_fd != -1) { + assert(vd->rbuflen >= SHMLOG_DATA); + i = read(vd->r_fd, vd->rbuf, SHMLOG_DATA); + if (i != SHMLOG_DATA) + return (-1); + l = SHMLOG_LEN(vd->rbuf) + SHMLOG_NEXTTAG; + if (vd->rbuflen < l) { + l += 200; + vd->rbuf = realloc(vd->rbuf, l); + assert(vd->rbuf != NULL); + vd->rbuflen = l; + } + l = SHMLOG_LEN(vd->rbuf) + 1; + i = read(vd->r_fd, vd->rbuf + SHMLOG_DATA, l); + if (i != l) + return (-1); + *pp = vd->rbuf; + return (1); + } + + p = vd->ptr; + for (w = 0; w < TIMEOUT_USEC;) { + if (*p == SLT_WRAPMARKER) { + p = vd->logstart; + continue; + } + if (*p == SLT_ENDMARKER) { + if (vd->flags & F_NON_BLOCKING) + return (-1); + w += SLEEP_USEC; + usleep(SLEEP_USEC); + continue; + } + l = SHMLOG_LEN(p); + vd->ptr = p + l + SHMLOG_NEXTTAG; + *pp = p; + return (1); + } + vd->ptr = p; + return (0); +} + +int +VSL_NextLog(struct VSL_data *vd, unsigned char **pp) +{ + unsigned char *p, t; + unsigned u, l; + int i; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + while (1) { + i = vsl_nextlog(vd, &p); + if (i != 1) + return (i); + u = SHMLOG_ID(p); + l = SHMLOG_LEN(p); + switch(p[SHMLOG_TAG]) { + case SLT_SessionOpen: + case SLT_ReqStart: + vbit_set(vd->vbm_client, u); + vbit_clr(vd->vbm_backend, u); + break; + case SLT_BackendOpen: + case SLT_BackendXID: + vbit_clr(vd->vbm_client, u); + vbit_set(vd->vbm_backend, u); + break; + default: + break; + } + if (vd->skip) { + --vd->skip; + continue; + } else if (vd->keep) { + if (--vd->keep == 0) + return (-1); + } + t = p[SHMLOG_TAG]; + if (vbit_test(vd->vbm_select, t)) { + *pp = p; + return (1); + } + if (vbit_test(vd->vbm_supress, t)) + continue; + if (vd->b_opt && !vbit_test(vd->vbm_backend, u)) + continue; + if (vd->c_opt && !vbit_test(vd->vbm_client, u)) + continue; + if (vd->regincl != NULL) { + i = VRE_exec(vd->regincl, + (char *)p + SHMLOG_DATA, + SHMLOG_LEN(p) - SHMLOG_DATA, /* Length */ + 0, 0, NULL, 0); + if (i == VRE_ERROR_NOMATCH) + continue; + } + if (vd->regexcl != NULL) { + i = VRE_exec(vd->regincl, + (char *)p + SHMLOG_DATA, + SHMLOG_LEN(p) - SHMLOG_DATA, /* Length */ + 0, 0, NULL, 0); + if (i != VRE_ERROR_NOMATCH) + continue; + } + *pp = p; + return (1); + } +} + +/*--------------------------------------------------------------------*/ + +int +VSL_Dispatch(struct VSL_data *vd, vsl_handler *func, void *priv) +{ + int i; + unsigned u, l, s; + unsigned char *p; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + while (1) { + i = VSL_NextLog(vd, &p); + if (i <= 0) + return (i); + u = SHMLOG_ID(p); + l = SHMLOG_LEN(p); + s = 0; + if (vbit_test(vd->vbm_backend, u)) + s |= VSL_S_BACKEND; + if (vbit_test(vd->vbm_client, u)) + s |= VSL_S_CLIENT; + if (func(priv, + p[SHMLOG_TAG], u, l, s, (char *)p + SHMLOG_DATA)) + return (1); + } +} + +/*--------------------------------------------------------------------*/ + +int +VSL_H_Print(void *priv, enum shmlogtag tag, unsigned fd, unsigned len, + unsigned spec, const char *ptr) +{ + FILE *fo = priv; + int type; + + assert(fo != NULL); + + type = (spec & VSL_S_CLIENT) ? 'c' : + (spec & VSL_S_BACKEND) ? 'b' : '-'; + + if (tag == SLT_Debug) { + fprintf(fo, "%5d %-12s %c \"", fd, VSL_tags[tag], type); + while (len-- > 0) { + if (*ptr >= ' ' && *ptr <= '~') + fprintf(fo, "%c", *ptr); + else + fprintf(fo, "%%%02x", (unsigned char)*ptr); + ptr++; + } + fprintf(fo, "\"\n"); + return (0); + } + fprintf(fo, "%5d %-12s %c %.*s\n", fd, VSL_tags[tag], type, len, ptr); + return (0); +} From phk at varnish-cache.org Mon May 17 15:48:05 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 17:48:05 +0200 Subject: r4805 - in trunk/varnish-cache: bin/varnishd include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-17 17:48:05 +0200 (Mon, 17 May 2010) New Revision: 4805 Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/include/shmlog.h trunk/varnish-cache/include/stats.h trunk/varnish-cache/include/vbm.h trunk/varnish-cache/lib/libvarnishapi/vsl.c trunk/varnish-cache/lib/libvarnishapi/vsl.h trunk/varnish-cache/lib/libvarnishapi/vsl_log.c Log: Move the log records to an allocated chunk of shmem. Warning: may contain nut^H^H^Hnuts. Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-17 15:48:05 UTC (rev 4805) @@ -55,19 +55,21 @@ vsl_wrap(void) { - assert(loghead->magic == SHMLOGHEAD_MAGIC); - *logstart = SLT_ENDMARKER; - logstart[loghead->ptr] = SLT_WRAPMARKER; - loghead->ptr = 0; + assert(vsl_log_nxt < vsl_log_end); + vsl_log_start[1] = SLT_ENDMARKER; + MEMORY_BARRIER(); + *vsl_log_nxt = SLT_WRAPMARKER; + MEMORY_BARRIER(); + vsl_log_start[0]++; + vsl_log_nxt = vsl_log_start + 1; VSL_stats->shm_cycles++; - assert(loghead->magic == SHMLOGHEAD_MAGIC); } static void vsl_hdr(enum shmlogtag tag, unsigned char *p, unsigned len, unsigned id) { - assert(loghead->magic == SHMLOGHEAD_MAGIC); + assert(vsl_log_nxt + SHMLOG_NEXTTAG + len < vsl_log_end); assert(len < 0x10000); p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff; p[__SHMLOG_LEN_LOW] = len & 0xff; @@ -76,11 +78,28 @@ p[__SHMLOG_ID_MEDLOW] = (id >> 8) & 0xff; p[__SHMLOG_ID_LOW] = id & 0xff; p[SHMLOG_DATA + len] = '\0'; - p[SHMLOG_NEXTTAG + len] = SLT_ENDMARKER; - /* XXX: Write barrier here */ + MEMORY_BARRIER(); p[SHMLOG_TAG] = tag; } +static uint8_t * +vsl_get(unsigned len) +{ + uint8_t *p; + + assert(vsl_log_nxt < vsl_log_end); + + /* Wrap if necessary */ + if (vsl_log_nxt + SHMLOG_NEXTTAG + len + 1 >= vsl_log_end) /* XXX: + 1 ?? */ + vsl_wrap(); + p = vsl_log_nxt; + + vsl_log_nxt += SHMLOG_NEXTTAG + len; + assert(vsl_log_nxt < vsl_log_end); + *vsl_log_nxt = SLT_ENDMARKER; + return (p); +} + /*-------------------------------------------------------------------- * This variant copies a byte-range directly to the log, without * taking the detour over sprintf() @@ -106,14 +125,7 @@ LOCKSHM(&vsl_mtx); VSL_stats->shm_writes++; VSL_stats->shm_records++; - assert(loghead->ptr < loghead->size); - - /* Wrap if necessary */ - if (loghead->ptr + SHMLOG_NEXTTAG + l + 1 >= loghead->size) - vsl_wrap(); - p = logstart + loghead->ptr; - loghead->ptr += SHMLOG_NEXTTAG + l; - assert(loghead->ptr < loghead->size); + p = vsl_get(l); UNLOCKSHM(&vsl_mtx); memcpy(p + SHMLOG_DATA, t.b, l); @@ -142,21 +154,25 @@ LOCKSHM(&vsl_mtx); VSL_stats->shm_writes++; VSL_stats->shm_records++; - assert(loghead->ptr < loghead->size); + assert(vsl_log_nxt < vsl_log_end); /* Wrap if we cannot fit a full size record */ - if (loghead->ptr + SHMLOG_NEXTTAG + mlen + 1 >= loghead->size) + if (vsl_log_nxt + SHMLOG_NEXTTAG + mlen + 1 >= vsl_log_end) vsl_wrap(); - p = logstart + loghead->ptr; + p = vsl_log_nxt; /* +1 for the NUL */ n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); if (n > mlen) n = mlen; /* we truncate long fields */ + + vsl_log_nxt += SHMLOG_NEXTTAG + n; + assert(vsl_log_nxt < vsl_log_end); + *vsl_log_nxt = SLT_ENDMARKER; + + UNLOCKSHM(&vsl_mtx); + vsl_hdr(tag, p, n, id); - loghead->ptr += SHMLOG_NEXTTAG + n; - assert(loghead->ptr < loghead->size); - UNLOCKSHM(&vsl_mtx); } va_end(ap); } @@ -166,7 +182,7 @@ void WSL_Flush(struct worker *w, int overflow) { - unsigned char *p; + uint8_t *p; unsigned l; l = pdiff(w->wlb, w->wlp); @@ -176,15 +192,11 @@ VSL_stats->shm_flushes += overflow; VSL_stats->shm_writes++; VSL_stats->shm_records += w->wlr; - if (loghead->ptr + l + 1 >= loghead->size) - vsl_wrap(); - p = logstart + loghead->ptr; - p[l] = SLT_ENDMARKER; - loghead->ptr += l; - assert(loghead->ptr < loghead->size); + p = vsl_get(l); UNLOCKSHM(&vsl_mtx); + memcpy(p + 1, w->wlb + 1, l - 1); - /* XXX: memory barrier here */ + MEMORY_BARRIER(); p[0] = w->wlb[0]; w->wlp = w->wlb; w->wlr = 0; @@ -269,10 +281,6 @@ VSL_Init(void) { - assert(loghead->magic == SHMLOGHEAD_MAGIC); - assert(loghead->hdrsize == sizeof *loghead); - /* XXX more check sanity of loghead ? */ - logstart = (unsigned char *)loghead + loghead->start; AZ(pthread_mutex_init(&vsl_mtx, NULL)); loghead->starttime = TIM_real(); loghead->panicstr[0] = '\0'; Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-17 15:48:05 UTC (rev 4805) @@ -41,7 +41,9 @@ /* mgt_shmem.c */ extern struct varnish_stats *VSL_stats; extern struct shmloghead *loghead; -extern unsigned char *logstart; +extern uint8_t *vsl_log_start; +extern uint8_t *vsl_log_end; +extern uint8_t *vsl_log_nxt; /* varnishd.c */ struct vsb; Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 15:48:05 UTC (rev 4805) @@ -56,6 +56,9 @@ struct varnish_stats *VSL_stats; struct shmloghead *loghead; unsigned char *logstart; +uint8_t *vsl_log_start; +uint8_t *vsl_log_end; +uint8_t *vsl_log_nxt; static int vsl_fd = -1; @@ -102,7 +105,7 @@ loghead->alloc_seq = seq++; MEMORY_BARRIER(); - return ((void*)(sha + 1)); + return (SHA_PTR(sha)); } return (NULL); @@ -114,7 +117,7 @@ */ static int -vsl_goodold(int fd, unsigned size, unsigned s2) +vsl_goodold(int fd, unsigned size) { struct shmloghead slh; int i; @@ -153,7 +156,7 @@ /* Sanity checks */ - if (slh.start != s2) + if (slh.shm_size != size) return (0); if (st.st_size != size) @@ -163,7 +166,7 @@ } static void -vsl_buildnew(const char *fn, unsigned size, int fill, unsigned s2) +vsl_buildnew(const char *fn, unsigned size, int fill) { struct shmloghead slh; int i; @@ -181,9 +184,7 @@ memset(&slh, 0, sizeof slh); slh.magic = SHMLOGHEAD_MAGIC; slh.hdrsize = sizeof slh; - slh.size = size; - slh.ptr = 0; - slh.start = s2; + slh.shm_size = size; i = write(vsl_fd, &slh, sizeof slh); xxxassert(i == sizeof slh); @@ -272,13 +273,13 @@ size &= ~(ps - 1); i = open(fn, O_RDWR, 0644); - if (i >= 0 && vsl_goodold(i, size, s2)) { + if (i >= 0 && vsl_goodold(i, size)) { fprintf(stderr, "Using old SHMFILE\n"); vsl_fd = i; } else { fprintf(stderr, "Creating new SHMFILE\n"); (void)close(i); - vsl_buildnew(fn, size, fill, s2); + vsl_buildnew(fn, size, fill); } loghead = (void *)mmap(NULL, size, @@ -295,11 +296,11 @@ memset(&loghead->head, 0, sizeof loghead->head); loghead->head.magic = SHMALLOC_MAGIC; - loghead->head.len = s2 - sizeof *loghead; + loghead->head.len = size - sizeof *loghead; bprintf(loghead->head.type, "%s", "Free"); MEMORY_BARRIER(); - VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_STAT_TYPE, ""); + VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_TYPE_STAT, ""); AN(VSL_stats); pp = mgt_SHM_Alloc(sizeof *pp, "Params", ""); @@ -307,6 +308,15 @@ *pp = *params; params = pp; + vsl_log_start = mgt_SHM_Alloc(s1, VSL_TYPE_LOG, ""); + AN(vsl_log_start); + vsl_log_end = vsl_log_start + s1; + vsl_log_nxt = vsl_log_start + 1; + *vsl_log_nxt = SLT_ENDMARKER; + MEMORY_BARRIER(); + *vsl_log_start = random(); + MEMORY_BARRIER(); + loghead->alloc_seq = random(); MEMORY_BARRIER(); } Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/include/shmlog.h 2010-05-17 15:48:05 UTC (rev 4805) @@ -68,18 +68,8 @@ pid_t master_pid; pid_t child_pid; - /* - * Byte offset into the file where the fifolog starts - * This allows the header to expand later. - */ - unsigned start; + unsigned shm_size; - /* Length of the fifolog area in bytes */ - unsigned size; - - /* Current write position relative to the beginning of start */ - unsigned ptr; - /* Panic message buffer */ char panicstr[64 * 1024]; @@ -88,6 +78,8 @@ struct shmalloc head; }; +#define VSL_TYPE_LOG "Log" + /* * Record format is as follows: * Modified: trunk/varnish-cache/include/stats.h =================================================================== --- trunk/varnish-cache/include/stats.h 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/include/stats.h 2010-05-17 15:48:05 UTC (rev 4805) @@ -31,7 +31,7 @@ #include -#define VSL_STAT_TYPE "Stats" +#define VSL_TYPE_STAT "Stats" struct varnish_stats { #define MAC_STAT(n, t, l, f, e) t n; Modified: trunk/varnish-cache/include/vbm.h =================================================================== --- trunk/varnish-cache/include/vbm.h 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/include/vbm.h 2010-05-17 15:48:05 UTC (rev 4805) @@ -89,6 +89,9 @@ vbit_set(struct vbitmap *vb, unsigned bit) { + if (0) /* XXX: HACK: ref it, to silence compiler */ + vbit_destroy(vb); + if (bit >= vb->nbits) vbit_expand(vb, bit); vb->bits[VBITMAP_IDX(bit)] |= VBITMAP_BIT(bit); Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-17 15:48:05 UTC (rev 4805) @@ -137,7 +137,7 @@ return (1); } - vd->vsl_lh = (void *)mmap(NULL, slh.size + sizeof slh, + vd->vsl_lh = (void *)mmap(NULL, slh.shm_size, PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsl_fd, 0); if (vd->vsl_lh == MAP_FAILED) { fprintf(stderr, "Cannot mmap %s: %s\n", @@ -154,8 +154,7 @@ { if (vd->vsl_lh == NULL) return; - assert(0 == munmap((void*)vd->vsl_lh, - vd->vsl_lh->size + sizeof *vd->vsl_lh)); + assert(0 == munmap((void*)vd->vsl_lh, vd->vsl_lh->shm_size)); vd->vsl_lh = NULL; assert(vd->vsl_fd >= 0); assert(0 == close(vd->vsl_fd)); @@ -188,9 +187,10 @@ { struct shmalloc *sha; + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (VSL_Open(vd)) return (NULL); - sha = vsl_find_alloc(vd, VSL_STAT_TYPE, ""); + sha = vsl_find_alloc(vd, VSL_TYPE_STAT, ""); assert(sha != NULL); return (SHA_PTR(sha)); } @@ -201,23 +201,22 @@ VSL_OpenLog(struct VSL_data *vd) { unsigned char *p; + struct shmalloc *sha; CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (vd->r_fd != -1) - return (0); - if (VSL_Open(vd)) return (-1); + sha = vsl_find_alloc(vd, VSL_TYPE_LOG, ""); + assert(sha != NULL); - vd->head = vd->vsl_lh; - vd->logstart = (unsigned char *)vd->vsl_lh + vd->vsl_lh->start; - vd->logend = vd->logstart + vd->vsl_lh->size; - vd->ptr = vd->logstart; + vd->log_start = SHA_PTR(sha); + vd->log_end = vd->log_start + sha->len - sizeof *sha; + vd->log_ptr = vd->log_start + 1; if (!vd->d_opt && vd->r_fd == -1) { - for (p = vd->ptr; *p != SLT_ENDMARKER; ) + for (p = vd->log_ptr; *p != SLT_ENDMARKER; ) p += SHMLOG_LEN(p) + SHMLOG_NEXTTAG; - vd->ptr = p; + vd->log_ptr = p; } return (0); } Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.h =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-17 15:48:05 UTC (rev 4805) @@ -37,11 +37,14 @@ unsigned magic; #define VSL_MAGIC 0x6e3bd69b - struct shmloghead *head; - unsigned char *logstart; - unsigned char *logend; - unsigned char *ptr; + int vsl_fd; + struct shmloghead *vsl_lh; + + unsigned char *log_start; + unsigned char *log_end; + unsigned char *log_ptr; + /* for -r option */ int r_fd; unsigned rbuflen; @@ -80,7 +83,4 @@ unsigned long skip; unsigned long keep; - - int vsl_fd; - struct shmloghead *vsl_lh; }; Modified: trunk/varnish-cache/lib/libvarnishapi/vsl_log.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl_log.c 2010-05-17 15:27:51 UTC (rev 4804) +++ trunk/varnish-cache/lib/libvarnishapi/vsl_log.c 2010-05-17 15:48:05 UTC (rev 4805) @@ -119,10 +119,10 @@ return (1); } - p = vd->ptr; + p = vd->log_ptr; for (w = 0; w < TIMEOUT_USEC;) { if (*p == SLT_WRAPMARKER) { - p = vd->logstart; + p = vd->log_start + 1; continue; } if (*p == SLT_ENDMARKER) { @@ -133,11 +133,11 @@ continue; } l = SHMLOG_LEN(p); - vd->ptr = p + l + SHMLOG_NEXTTAG; + vd->log_ptr = p + l + SHMLOG_NEXTTAG; *pp = p; return (1); } - vd->ptr = p; + vd->log_ptr = p; return (0); } From phk at varnish-cache.org Mon May 17 20:09:45 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 22:09:45 +0200 Subject: r4806 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 22:09:45 +0200 (Mon, 17 May 2010) New Revision: 4806 Modified: trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/mgt_shmem.c Log: Expose mgt_SHM_Alloc() Fix an safely-off-by-some-bytes error which made it harder to find the end of the alloc-arena in the shmlog file. Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-17 15:48:05 UTC (rev 4805) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-17 20:09:45 UTC (rev 4806) @@ -71,6 +71,7 @@ /* mgt_shmem.c */ void mgt_SHM_Init(const char *fn, const char *arg); void mgt_SHM_Pid(void); +void *mgt_SHM_Alloc(unsigned size, const char *type, const char *ident); /* mgt_vcc.c */ void mgt_vcc_init(void); Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 15:48:05 UTC (rev 4805) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-17 20:09:45 UTC (rev 4806) @@ -64,7 +64,7 @@ /*--------------------------------------------------------------------*/ -static void * +void * mgt_SHM_Alloc(unsigned size, const char *type, const char *ident) { struct shmalloc *sha, *sha2; @@ -221,8 +221,6 @@ if (av[0] != NULL) ARGV_ERR("\t-l ...: %s", av[0]); - printf("<%s> <%s> <%s>\n", av[1], av[2], av[3]); - ap = av + 1; /* Size of SHMLOG */ @@ -296,7 +294,8 @@ memset(&loghead->head, 0, sizeof loghead->head); loghead->head.magic = SHMALLOC_MAGIC; - loghead->head.len = size - sizeof *loghead; + loghead->head.len = + (uint8_t*)(loghead) + size - (uint8_t*)&loghead->head; bprintf(loghead->head.type, "%s", "Free"); MEMORY_BARRIER(); From phk at varnish-cache.org Mon May 17 20:10:29 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 22:10:29 +0200 Subject: r4807 - in trunk/varnish-cache: include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-17 22:10:28 +0200 (Mon, 17 May 2010) New Revision: 4807 Modified: trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/vsl.c trunk/varnish-cache/lib/libvarnishapi/vsl.h Log: Expose VSL_Find_Alloc() Properly handle the case where the looked for allocation is not found. Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-17 20:09:45 UTC (rev 4806) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-17 20:10:28 UTC (rev 4807) @@ -62,5 +62,8 @@ struct varnish_stats *VSL_OpenStats(struct VSL_data *vd); const char *VSL_Name(struct VSL_data *vd); extern const char *VSL_tags[256]; +void *VSL_Find_Alloc(struct VSL_data *vd, const char *type, const char *ident, + unsigned *lenp); + #endif Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-17 20:09:45 UTC (rev 4806) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-17 20:10:28 UTC (rev 4807) @@ -144,6 +144,7 @@ logname, strerror(errno)); return (1); } + vd->vsl_end = (uint8_t *)vd->vsl_lh + slh.shm_size; return (0); } @@ -169,7 +170,7 @@ struct shmalloc *sha; assert (vd->vsl_lh != NULL); - for(sha = &vd->vsl_lh->head; ; sha = SHA_NEXT(sha)) { + for(sha = &vd->vsl_lh->head; (void*)sha < vd->vsl_end; sha = SHA_NEXT(sha)) { CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); if (strcmp(sha->type, type)) continue; @@ -182,6 +183,25 @@ /*--------------------------------------------------------------------*/ +void * +VSL_Find_Alloc(struct VSL_data *vd, const char *type, const char *ident, + unsigned *lenp) +{ + struct shmalloc *sha; + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (VSL_Open(vd)) + return (NULL); + sha = vsl_find_alloc(vd, type, ident); + if (sha == NULL) + return (NULL); + if (lenp != NULL) + *lenp = sha->len - sizeof *sha; + return (SHA_PTR(sha)); +} + +/*--------------------------------------------------------------------*/ + struct varnish_stats * VSL_OpenStats(struct VSL_data *vd) { Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.h =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-17 20:09:45 UTC (rev 4806) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-17 20:10:28 UTC (rev 4807) @@ -40,6 +40,7 @@ int vsl_fd; struct shmloghead *vsl_lh; + void *vsl_end; unsigned char *log_start; unsigned char *log_end; From phk at varnish-cache.org Mon May 17 20:11:01 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 22:11:01 +0200 Subject: r4808 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-17 22:11:01 +0200 (Mon, 17 May 2010) New Revision: 4808 Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c Log: Store the -S and -T arguments in the shmlog. Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2010-05-17 20:10:28 UTC (rev 4807) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2010-05-17 20:11:01 UTC (rev 4808) @@ -506,7 +506,13 @@ { int i, fd; char buf[BUFSIZ]; + char *p; + /* Save in shmem */ + i = strlen(S_arg); + p = mgt_SHM_Alloc(i + 1, "Arg", "-S"); + AN(p); + strcpy(p, S_arg); srandomdev(); fd = open(S_arg, O_RDONLY); @@ -535,8 +541,16 @@ char *addr, *port; int i, n, sock, good; struct telnet *tn; + char *p; + /* Save in shmem */ + i = strlen(T_arg); + p = mgt_SHM_Alloc(i + 1, "Arg", "-T"); + AN(p); + strcpy(p, T_arg); + XXXAZ(VSS_parse(T_arg, &addr, &port)); + n = VSS_resolve(addr, port, &ta); if (n == 0) { fprintf(stderr, "Could not open management port\n"); From phk at varnish-cache.org Mon May 17 20:11:59 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 17 May 2010 22:11:59 +0200 Subject: r4809 - trunk/varnish-cache/bin/varnishadm Message-ID: Author: phk Date: 2010-05-17 22:11:59 +0200 (Mon, 17 May 2010) New Revision: 4809 Modified: trunk/varnish-cache/bin/varnishadm/Makefile.am trunk/varnish-cache/bin/varnishadm/varnishadm.c Log: Give varnishadm the ability to find -T and possibly -S arguments if given a -n argument. Modified: trunk/varnish-cache/bin/varnishadm/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishadm/Makefile.am 2010-05-17 20:11:01 UTC (rev 4808) +++ trunk/varnish-cache/bin/varnishadm/Makefile.am 2010-05-17 20:11:59 UTC (rev 4809) @@ -11,5 +11,6 @@ varnishadm_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ + $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ $(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \ ${PTHREAD_LIBS} ${NET_LIBS} Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c =================================================================== --- trunk/varnish-cache/bin/varnishadm/varnishadm.c 2010-05-17 20:11:01 UTC (rev 4808) +++ trunk/varnish-cache/bin/varnishadm/varnishadm.c 2010-05-17 20:11:59 UTC (rev 4809) @@ -44,6 +44,7 @@ #include "cli.h" #include "cli_common.h" #include "libvarnish.h" +#include "varnishapi.h" #include "vss.h" static double timeout = 5; @@ -214,10 +215,16 @@ { const char *T_arg = NULL; const char *S_arg = NULL; + const char *n_arg = NULL; + struct VSL_data *vsd; + char *p; int opt, sock; - while ((opt = getopt(argc, argv, "S:T:t:")) != -1) { + while ((opt = getopt(argc, argv, "n:S:T:t:")) != -1) { switch (opt) { + case 'n': + n_arg = optarg; + break; case 'S': S_arg = optarg; break; @@ -235,6 +242,25 @@ argc -= optind; argv += optind; + if (n_arg != NULL) { + vsd = VSL_New(); + assert(VSL_Arg(vsd, 'n', n_arg)); + if (!VSL_Open(vsd)) { + if (T_arg == NULL) { + p = VSL_Find_Alloc(vsd, "Arg", "-T", NULL); + if (p != NULL) { + T_arg = strdup(p); + } + } + if (S_arg == NULL) { + p = VSL_Find_Alloc(vsd, "Arg", "-S", NULL); + if (p != NULL) { + S_arg = strdup(p); + } + } + } + } + if (T_arg == NULL) usage(); From perbu at varnish-cache.org Tue May 18 11:06:50 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Tue, 18 May 2010 13:06:50 +0200 Subject: r4810 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-18 13:06:50 +0200 (Tue, 18 May 2010) New Revision: 4810 Removed: trunk/varnish-cache/doc/sphinx/tutorial/intro.rst Modified: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/index.rst trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst Log: Heading cleanups + merged introduction and index Modified: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -1,5 +1,5 @@ Advanced Backend configuration -============================== +------------------------------ At some point you might need Varnish to cache content from several servers. You might want Varnish to map all the URL into one single Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -1,7 +1,7 @@ .. _tutorial-increasing_your_hitrate: Achieving a high hitrate -======================== +------------------------ Now that Varnish is up and running, and you can access your web application through Varnish. Unless your application is specifically @@ -60,6 +60,7 @@ Headers* and *Firebug* can show you what headers are beeing sent and recieved. + HTTP Headers ------------ Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -4,9 +4,24 @@ Varnish Tutorial %%%%%%%%%%%%%%%% -.. toctree:: +Varnish is a web accelerator. It is installed in frond of your web +application and it caches the reponses, making your web site run Varnish +is fast, flexible and easy to use. - intro.rst +This tutorial does not go through every bit of functionality Varnish +has. It will give you a good overview of what Varnish does and how it +is done. + +We assume you have a web server and a web application up and running +and that you want to accelerate this application with Varnish. + +Furthermore we assume you have read the :ref:`Installation` and that +it is installed with the default configuration. + + + +.. toctree:: + backend_servers.rst starting_varnish.rst logging.rst Deleted: trunk/varnish-cache/doc/sphinx/tutorial/intro.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/intro.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/intro.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -1,21 +0,0 @@ -%%%%%%%%%%%% -Introduction -%%%%%%%%%%%% - -Varnish is a web accelerator. It is installed in frond of your web -application and it caches the reponses, making your web site run Varnish -is fast, flexible and easy to use. - -This tutorial does not go through every bit of functionality Varnish -has. It will give you a good overview of what Varnish does and how it -is done. - -We assume you have a web server and a web application up and running -and that you want to accelerate this application with Varnish. - -Furthermore we assume you have read the :ref:`Installation` and that -it is installed with the default configuration. - - - - Modified: trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/statistics.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -8,7 +8,7 @@ doing. There are several tools that can help. varnishtop -========== +~~~~~~~~~~ The varnishtop utility reads the shared memory logs and presents a continuously updated list of the most commonly occurring log entries. @@ -24,7 +24,7 @@ the client are sendning you. varnishhist -=========== +~~~~~~~~~~~ The varnishhist utility reads varnishd(1) shared memory logs and presents a continuously updated histogram showing the distribution of @@ -35,7 +35,7 @@ varnishsizes -============ +~~~~~~~~~~~~ Varnishsizes does the same as varnishhist, except it shows the size of the objects and not the time take to complete the request. This gives @@ -43,7 +43,7 @@ varnishstat -=========== +~~~~~~~~~~~ Varnish has lots of counters. We count misses, hits, information about the storage, threads created, deleted objects. Just about Modified: trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -1,9 +1,9 @@ Troubleshooting Varnish -======================= +----------------------- When Varnish won't start ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ Sometimes Varnish wont start. There is a pletphora of reasons why Varnish wont start on your machine. We've seen everything from wrong @@ -45,6 +45,6 @@ Varnish is crashing -------------------- +~~~~~~~~~~~~~~~~~~~ When varnish goes bust. Modified: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-17 20:11:59 UTC (rev 4809) +++ trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-18 11:06:50 UTC (rev 4810) @@ -1,7 +1,4 @@ Varnish Configuration Language - VCL -==================================== - -How ordinary configuration files work ------------------------------------- Varnish has a really neat configuration system. Most other systems use @@ -27,9 +24,6 @@ the first? Or does Squid try to combine all the matching rules. I actually don't know. -Enter VCL ---------- - Now enter Varnish. Varnish takes your configuration file and translates it to C code, then runs it through a compiler and loads it. When requests come along varnish just executes the relevant From phk at varnish-cache.org Tue May 18 11:41:32 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 18 May 2010 13:41:32 +0200 Subject: r4811 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-18 13:41:32 +0200 (Tue, 18 May 2010) New Revision: 4811 Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Give the malloc storage a proper "sc" structure and mini-obj'ify the sma structure. Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-18 11:06:50 UTC (rev 4810) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-18 11:41:32 UTC (rev 4811) @@ -38,47 +38,65 @@ #include #include -#include #include "shmlog.h" #include "cache.h" #include "stevedore.h" -static size_t sma_max = SIZE_MAX; -static struct lock sma_mtx; +struct sma_sc { + unsigned magic; +#define SMA_SC_MAGIC 0x1ac8a345 + struct lock sma_mtx; + size_t sma_max; +}; struct sma { + unsigned magic; +#define SMA_MAGIC 0x69ae9bb9 struct storage s; size_t sz; + struct sma_sc *sc; }; static struct storage * sma_alloc(struct stevedore *st, size_t size, struct objcore *oc) { + struct sma_sc *sma_sc; struct sma *sma; + CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); (void)oc; - Lck_Lock(&sma_mtx); + Lck_Lock(&sma_sc->sma_mtx); VSL_stats->sma_nreq++; - if (VSL_stats->sma_nbytes + size > sma_max) + if (VSL_stats->sma_nbytes + size > sma_sc->sma_max) size = 0; else { VSL_stats->sma_nobj++; VSL_stats->sma_nbytes += size; VSL_stats->sma_balloc += size; } - Lck_Unlock(&sma_mtx); + Lck_Unlock(&sma_sc->sma_mtx); if (size == 0) return (NULL); - sma = calloc(sizeof *sma, 1); + /* + * Do not collaps the sma allocation with sma->s.ptr: it is not + * a good idea. Not only would it make ->trim impossible, + * performance-wise it would be a catastropy with chunksized + * allocations growing another full page, just to accomodate the sma. + */ + ALLOC_OBJ(sma, SMA_MAGIC); if (sma == NULL) - return (NULL); + return (NULL); /* XXX: stats suffer */ + sma->sc = sma_sc; sma->sz = size; sma->s.priv = sma; sma->s.ptr = malloc(size); - XXXAN(sma->s.ptr); + if (sma->s.ptr == NULL) { + free(sma); + return (NULL); /* XXX: stats suffer */ + } sma->s.len = 0; sma->s.space = size; sma->s.fd = -1; @@ -91,16 +109,18 @@ static void sma_free(struct storage *s) { + struct sma_sc *sma_sc; struct sma *sma; CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC); - sma = s->priv; + CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC); + sma_sc = sma->sc; assert(sma->sz == sma->s.space); - Lck_Lock(&sma_mtx); + Lck_Lock(&sma_sc->sma_mtx); VSL_stats->sma_nobj--; VSL_stats->sma_nbytes -= sma->sz; VSL_stats->sma_bfree += sma->sz; - Lck_Unlock(&sma_mtx); + Lck_Unlock(&sma_sc->sma_mtx); free(sma->s.ptr); free(sma); } @@ -108,18 +128,22 @@ static void sma_trim(struct storage *s, size_t size) { + struct sma_sc *sma_sc; struct sma *sma; void *p; CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC); - sma = s->priv; + CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC); + sma_sc = sma->sc; + assert(sma->sz == sma->s.space); + assert(size < sma->sz); if ((p = realloc(sma->s.ptr, size)) != NULL) { - Lck_Lock(&sma_mtx); + Lck_Lock(&sma_sc->sma_mtx); VSL_stats->sma_nbytes -= (sma->sz - size); VSL_stats->sma_bfree += sma->sz - size; sma->sz = size; - Lck_Unlock(&sma_mtx); + Lck_Unlock(&sma_sc->sma_mtx); sma->s.ptr = p; s->space = size; } @@ -130,8 +154,12 @@ { const char *e; uintmax_t u; + struct sma_sc *sc; - (void)parent; + ALLOC_OBJ(sc, SMA_SC_MAGIC); + AN(sc); + sc->sma_max = SIZE_MAX; + parent->priv = sc; AZ(av[ac]); if (ac > 1) @@ -148,14 +176,17 @@ printf("storage_malloc: max size %ju MB.\n", u / (1024 * 1024)); - sma_max = u; + sc->sma_max = u; + } static void sma_open(const struct stevedore *st) { - (void)st; - Lck_New(&sma_mtx); + struct sma_sc *sma_sc; + + CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); + Lck_New(&sma_sc->sma_mtx); } struct stevedore sma_stevedore = { From phk at varnish-cache.org Tue May 18 12:05:25 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 18 May 2010 14:05:25 +0200 Subject: r4812 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-18 14:05:25 +0200 (Tue, 18 May 2010) New Revision: 4812 Modified: trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Add a stevedore call to complete initialization ("ready") which is called as late as possible, before we start the child. Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2010-05-18 11:41:32 UTC (rev 4811) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2010-05-18 12:05:25 UTC (rev 4812) @@ -239,6 +239,18 @@ } void +STV_ready(void) +{ + struct stevedore *stv; + + ASSERT_MGT(); + VTAILQ_FOREACH(stv, &stevedores, list) { + if (stv->ready != NULL) + stv->ready(stv); + } +} + +void STV_open(void) { struct stevedore *stv; Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-18 11:41:32 UTC (rev 4811) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-18 12:05:25 UTC (rev 4812) @@ -36,6 +36,7 @@ struct objcore; typedef void storage_init_f(struct stevedore *, int ac, char * const *av); +typedef void storage_ready_f(struct stevedore *); typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size, struct objcore *); @@ -50,6 +51,7 @@ #define STEVEDORE_MAGIC 0x4baf43db const char *name; storage_init_f *init; /* called by mgt process */ + storage_ready_f *ready; /* called by mgt process */ storage_open_f *open; /* called by cache process */ storage_alloc_f *alloc; storage_trim_f *trim; @@ -71,6 +73,7 @@ void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); void STV_add(const struct stevedore *stv, int ac, char * const *av); +void STV_ready(void); void STV_open(void); void STV_close(void); struct lru *STV_lru(const struct storage *st); Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-18 11:41:32 UTC (rev 4811) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-18 12:05:25 UTC (rev 4812) @@ -719,6 +719,8 @@ mgt_cli_telnet(tmpbuf); } + STV_ready(); /* Complete initialization */ + MGT_Run(); if (pfh != NULL) From phk at varnish-cache.org Tue May 18 12:09:16 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 18 May 2010 14:09:16 +0200 Subject: r4813 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-18 14:09:16 +0200 (Tue, 18 May 2010) New Revision: 4813 Modified: trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/mgt.h Log: Move the mgt_SHM_Alloc() prototype Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-18 12:05:25 UTC (rev 4812) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-18 12:09:16 UTC (rev 4813) @@ -39,6 +39,7 @@ void VCA_tweak_waiter(struct cli *cli, const char *arg); /* mgt_shmem.c */ +void *mgt_SHM_Alloc(unsigned size, const char *type, const char *ident); extern struct varnish_stats *VSL_stats; extern struct shmloghead *loghead; extern uint8_t *vsl_log_start; Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-18 12:05:25 UTC (rev 4812) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-18 12:09:16 UTC (rev 4813) @@ -71,7 +71,6 @@ /* mgt_shmem.c */ void mgt_SHM_Init(const char *fn, const char *arg); void mgt_SHM_Pid(void); -void *mgt_SHM_Alloc(unsigned size, const char *type, const char *ident); /* mgt_vcc.c */ void mgt_vcc_init(void); From phk at varnish-cache.org Tue May 18 12:09:49 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 18 May 2010 14:09:49 +0200 Subject: r4814 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-18 14:09:48 +0200 (Tue, 18 May 2010) New Revision: 4814 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c Log: Protect mgt_SHM_Alloc() with a couple of asserts. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-18 12:09:16 UTC (rev 4813) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-18 12:09:48 UTC (rev 4814) @@ -70,6 +70,8 @@ struct shmalloc *sha, *sha2; unsigned seq; + ASSERT_MGT(); + AN(loghead); /* Round up to pointersize */ size += sizeof(sha) - 1; size &= ~(sizeof(sha) - 1); From phk at varnish-cache.org Wed May 19 06:45:39 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 08:45:39 +0200 Subject: r4815 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 08:45:39 +0200 (Wed, 19 May 2010) New Revision: 4815 Modified: trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Expose pick() Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-18 12:09:48 UTC (rev 4814) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-19 06:45:39 UTC (rev 4815) @@ -66,8 +66,9 @@ /* A tiny helper for choosing hash/storage modules */ struct choice { const char *name; - void *ptr; + void *ptr; /* XXX: constify */ }; +void *pick(const struct choice *cp, const char *which, const char *kind); #define NEEDLESS_RETURN(foo) return (foo) Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-18 12:09:48 UTC (rev 4814) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-19 06:45:39 UTC (rev 4815) @@ -102,7 +102,7 @@ /*--------------------------------------------------------------------*/ -static void * +void * pick(const struct choice *cp, const char *which, const char *kind) { From phk at varnish-cache.org Wed May 19 07:05:12 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 09:05:12 +0200 Subject: r4816 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 09:05:11 +0200 (Wed, 19 May 2010) New Revision: 4816 Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c Log: Note a possible idea. Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-19 06:45:39 UTC (rev 4815) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2010-05-19 07:05:11 UTC (rev 4816) @@ -142,6 +142,10 @@ unsigned n, mlen; txt t; + /* + * XXX: consider formatting into a stack buffer then move into + * XXX: shmlog with VSLR(). + */ AN(fmt); va_start(ap, fmt); mlen = params->shm_reclen; From phk at varnish-cache.org Wed May 19 07:07:15 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 09:07:15 +0200 Subject: r4817 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 09:07:15 +0200 (Wed, 19 May 2010) New Revision: 4817 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/hash_slinger.h trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Constify what pick() finds. Move hash & storage config out to cache_hash.c and stevedore.c. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2010-05-19 07:07:15 UTC (rev 4817) @@ -749,10 +749,43 @@ hash->start(); } -const struct choice hsh_choice[] = { +static const struct choice hsh_choice[] = { { "classic", &hcl_slinger }, { "simple", &hsl_slinger }, { "simple_list", &hsl_slinger }, /* backwards compat */ { "critbit", &hcb_slinger }, { NULL, NULL } }; + +/*--------------------------------------------------------------------*/ + +void +HSH_config(const char *h_arg) +{ + char **av; + int ac; + const struct hash_slinger *hp; + + av = ParseArgv(h_arg, ARGV_COMMA); + AN(av); + + if (av[0] != NULL) + ARGV_ERR("%s\n", av[0]); + + if (av[1] == NULL) + ARGV_ERR("-h argument is empty\n"); + + for (ac = 0; av[ac + 2] != NULL; ac++) + continue; + + hp = pick(hsh_choice, av[1], "hash"); + CHECK_OBJ_NOTNULL(hp, SLINGER_MAGIC); + vsb_printf(vident, ",-h%s", av[1]); + heritage.hash = hp; + if (hp->init != NULL) + hp->init(ac, av + 2); + else if (ac > 0) + ARGV_ERR("Hash method \"%s\" takes no arguments\n", + hp->name); +} + Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-19 07:07:15 UTC (rev 4817) @@ -66,9 +66,9 @@ /* A tiny helper for choosing hash/storage modules */ struct choice { const char *name; - void *ptr; /* XXX: constify */ + const void *ptr; }; -void *pick(const struct choice *cp, const char *which, const char *kind); +const void *pick(const struct choice *cp, const char *which, const char *kind); #define NEEDLESS_RETURN(foo) return (foo) Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2010-05-19 07:07:15 UTC (rev 4817) @@ -65,6 +65,7 @@ void HSH_FindBan(struct sess *sp, struct objcore **oc); struct objcore *HSH_Insert(const struct sess *sp); void HSH_Purge(struct sess *, struct objhead *, double ttl, double grace); +void HSH_config(const char *h_arg); #ifdef VARNISH_CACHE_CHILD @@ -105,4 +106,3 @@ extern struct hash_slinger hsl_slinger; extern struct hash_slinger hcl_slinger; extern struct hash_slinger hcb_slinger; -extern const struct choice hsh_choice[]; Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2010-05-19 07:07:15 UTC (rev 4817) @@ -56,7 +56,7 @@ unsigned nsocks; /* Hash method */ - struct hash_slinger *hash; + const struct hash_slinger *hash; char *name; char identity[1024]; Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2010-05-19 07:07:15 UTC (rev 4817) @@ -214,31 +214,6 @@ } void -STV_add(const struct stevedore *stv2, int ac, char * const *av) -{ - struct stevedore *stv; - - CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC); - ALLOC_OBJ(stv, STEVEDORE_MAGIC); - AN(stv); - - *stv = *stv2; - AN(stv->name); - AN(stv->alloc); - stv->lru = LRU_Alloc(); - - if (stv->init != NULL) - stv->init(stv, ac, av); - else if (ac != 0) - ARGV_ERR("(-s%s) too many arguments\n", stv->name); - - VTAILQ_INSERT_TAIL(&stevedores, stv, list); - - if (!stv_next) - stv_next = VTAILQ_FIRST(&stevedores); -} - -void STV_ready(void) { struct stevedore *stv; @@ -280,7 +255,7 @@ return (st->stevedore->lru); } -const struct choice STV_choice[] = { +static const struct choice STV_choice[] = { { "file", &smf_stevedore }, { "malloc", &sma_stevedore }, { "persistent", &smp_stevedore }, @@ -289,3 +264,51 @@ #endif { NULL, NULL } }; + +/*--------------------------------------------------------------------*/ + +void +STV_config(const char *spec) +{ + char **av; + struct stevedore *stv; + const struct stevedore *stv2; + int ac; + + av = ParseArgv(spec, ARGV_COMMA); + AN(av); + + if (av[0] != NULL) + ARGV_ERR("%s\n", av[0]); + + if (av[1] == NULL) + ARGV_ERR("-s argument is empty\n"); + + for (ac = 0; av[ac + 2] != NULL; ac++) + continue; + + stv2 = pick(STV_choice, av[1], "storage"); + AN(stv2); + vsb_printf(vident, ",-s%s", av[1]); + + av += 2; + + CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC); + ALLOC_OBJ(stv, STEVEDORE_MAGIC); + AN(stv); + + *stv = *stv2; + AN(stv->name); + AN(stv->alloc); + stv->lru = LRU_Alloc(); + + if (stv->init != NULL) + stv->init(stv, ac, av); + else if (ac != 0) + ARGV_ERR("(-s%s) too many arguments\n", stv->name); + + VTAILQ_INSERT_TAIL(&stevedores, stv, list); + + if (!stv_next) + stv_next = VTAILQ_FIRST(&stevedores); +} Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-19 07:07:15 UTC (rev 4817) @@ -50,14 +50,14 @@ unsigned magic; #define STEVEDORE_MAGIC 0x4baf43db const char *name; - storage_init_f *init; /* called by mgt process */ - storage_ready_f *ready; /* called by mgt process */ - storage_open_f *open; /* called by cache process */ - storage_alloc_f *alloc; - storage_trim_f *trim; - storage_free_f *free; - storage_object_f *object; - storage_close_f *close; + storage_init_f *init; /* called by mgt process */ + storage_ready_f *ready; /* called by mgt process */ + storage_open_f *open; /* called by cache process */ + storage_alloc_f *alloc; /* --//-- */ + storage_trim_f *trim; /* --//-- */ + storage_free_f *free; /* --//-- */ + storage_object_f *object; /* --//-- */ + storage_close_f *close; /* --//-- */ struct lru *lru; @@ -72,10 +72,10 @@ struct storage *STV_alloc(struct sess *sp, size_t size, struct objcore *oc); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); -void STV_add(const struct stevedore *stv, int ac, char * const *av); void STV_ready(void); void STV_open(void); void STV_close(void); +void STV_config(const char *spec); struct lru *STV_lru(const struct storage *st); struct lru *LRU_Alloc(void); @@ -94,4 +94,3 @@ extern struct stevedore smu_stevedore; #endif -extern const struct choice STV_choice[]; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-19 07:05:11 UTC (rev 4816) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-19 07:07:15 UTC (rev 4817) @@ -102,7 +102,7 @@ /*--------------------------------------------------------------------*/ -void * +const void * pick(const struct choice *cp, const char *which, const char *kind) { @@ -130,68 +130,6 @@ /*--------------------------------------------------------------------*/ static void -setup_storage(const char *spec) -{ - char **av; - void *priv; - int ac; - - av = ParseArgv(spec, ARGV_COMMA); - AN(av); - - if (av[0] != NULL) - ARGV_ERR("%s\n", av[0]); - - if (av[1] == NULL) - ARGV_ERR("-s argument is empty\n"); - - for (ac = 0; av[ac + 2] != NULL; ac++) - continue; - - priv = pick(STV_choice, av[1], "storage"); - AN(priv); - vsb_printf(vident, ",-s%s", av[1]); - - STV_add(priv, ac, av + 2); - - /* We do not free av, to make life simpler for stevedores */ -} - -/*--------------------------------------------------------------------*/ - -static void -setup_hash(const char *h_arg) -{ - char **av; - int ac; - struct hash_slinger *hp; - - av = ParseArgv(h_arg, ARGV_COMMA); - AN(av); - - if (av[0] != NULL) - ARGV_ERR("%s\n", av[0]); - - if (av[1] == NULL) - ARGV_ERR("-h argument is empty\n"); - - for (ac = 0; av[ac + 2] != NULL; ac++) - continue; - - hp = pick(hsh_choice, av[1], "hash"); - CHECK_OBJ_NOTNULL(hp, SLINGER_MAGIC); - vsb_printf(vident, ",-h%s", av[1]); - heritage.hash = hp; - if (hp->init != NULL) - hp->init(ac, av + 2); - else if (ac > 0) - ARGV_ERR("Hash method \"%s\" takes no arguments\n", - hp->name); -} - -/*--------------------------------------------------------------------*/ - -static void usage(void) { #define FMT " %-28s # %s\n" @@ -554,7 +492,7 @@ break; case 's': s_arg_given = 1; - setup_storage(optarg); + STV_config(optarg); break; case 't': MCF_ParamSet(cli, "default_ttl", optarg); @@ -595,7 +533,7 @@ if (L_arg) { /* Learner mode */ if (!s_arg_given) { - setup_storage("malloc,1m"); + STV_config("malloc,1m"); s_arg_given = 1; } } @@ -679,9 +617,9 @@ exit (0); if (!s_arg_given) - setup_storage(s_arg); + STV_config(s_arg); - setup_hash(h_arg); + HSH_config(h_arg); mgt_SHM_Init(SHMLOG_FILENAME, l_arg); From phk at varnish-cache.org Wed May 19 07:32:28 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 09:32:28 +0200 Subject: r4818 - in trunk/varnish-cache: include lib/libvarnish Message-ID: Author: phk Date: 2010-05-19 09:32:27 +0200 (Wed, 19 May 2010) New Revision: 4818 Added: trunk/varnish-cache/include/vmb.h trunk/varnish-cache/lib/libvarnish/vmb.c Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am Log: Face the music and do memory barriers sensibly. This really belongs in POSIX pthreads :-( Added: trunk/varnish-cache/include/vmb.h =================================================================== --- trunk/varnish-cache/include/vmb.h (rev 0) +++ trunk/varnish-cache/include/vmb.h 2010-05-19 07:32:27 UTC (rev 4818) @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id$ + * + * Memory barriers + * + */ + +#ifndef VMB_H_INCLUDED +#define VMB_H_INCLUDED + +void vmb_pthread(void); + +#if defined(__FreeBSD__) +#include +#include +#define VMB() mb() +#define VWMB() wmb() +#define VRMB() rmb() +#else +#define VMB() vmb_pthread() +#define VWMB() vmb_pthread() +#define VRMB() vmb_pthread() +#endif + +#endif /* VMB_H_INCLUDED */ Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2010-05-19 07:07:15 UTC (rev 4817) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2010-05-19 07:32:27 UTC (rev 4818) @@ -24,6 +24,7 @@ vev.c \ vin.c \ vlu.c \ + vmb.c \ vpf.c \ vre.c \ vsb.c \ Added: trunk/varnish-cache/lib/libvarnish/vmb.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vmb.c (rev 0) +++ trunk/varnish-cache/lib/libvarnish/vmb.c 2010-05-19 07:32:27 UTC (rev 4818) @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2010 Redpill Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libvarnish.h" +#include "vmb.h" + +static pthread_mutex_t mb_mtx; +static pthread_once_t mb_mtx_once = PTHREAD_ONCE_INIT; + +static void +vmb_init(void) +{ + + AZ(pthread_mutex_init(&mb_mtx, NULL)); +} + + +void +vmb_pthread(void) +{ + + AZ(pthread_once(&mb_mtx_once, vmb_init)); + + AZ(pthread_mutex_lock(&mb_mtx)); + AZ(pthread_mutex_unlock(&mb_mtx)); +} From phk at varnish-cache.org Wed May 19 08:15:23 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:15:23 +0200 Subject: r4819 - trunk/varnish-cache/include Message-ID: Author: phk Date: 2010-05-19 10:15:23 +0200 (Wed, 19 May 2010) New Revision: 4819 Added: trunk/varnish-cache/include/vend.h Log: Import my v{be|le}{16|32|64}{enc|dec}() from Freebsd. #if 0 out those not used at present. Added: trunk/varnish-cache/include/vend.h =================================================================== --- trunk/varnish-cache/include/vend.h (rev 0) +++ trunk/varnish-cache/include/vend.h 2010-05-19 08:15:23 UTC (rev 4819) @@ -0,0 +1,154 @@ +/*- + * Copyright (c) 2003,2010 Poul-Henning Kamp + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * From: + * $FreeBSD: head/sys/sys/endian.h 121122 2003-10-15 20:05:57Z obrien $ + * + * $Id$ + * + * Endian conversion functions + */ + +#ifndef VEND_H_INCLUDED + +/* Alignment-agnostic encode/decode bytestream to/from little/big endian. */ + +#if 0 +static __inline uint16_t +vbe16dec(const void *pp) +{ + uint8_t const *p = (uint8_t const *)pp; + + return ((p[0] << 8) | p[1]); +} +#endif + +static __inline uint32_t +vbe32dec(const void *pp) +{ + uint8_t const *p = (uint8_t const *)pp; + + return (((unsigned)p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]); +} + +#if 0 +static __inline uint64_t +vbe64dec(const void *pp) +{ + uint8_t const *p = (uint8_t const *)pp; + + return (((uint64_t)vbe32dec(p) << 32) | vbe32dec(p + 4)); +} + +static __inline uint16_t +vle16dec(const void *pp) +{ + uint8_t const *p = (uint8_t const *)pp; + + return ((p[1] << 8) | p[0]); +} +#endif + +static __inline uint32_t +vle32dec(const void *pp) +{ + uint8_t const *p = (uint8_t const *)pp; + + return (((unsigned)p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]); +} + +#if 0 +static __inline uint64_t +vle64dec(const void *pp) +{ + uint8_t const *p = (uint8_t const *)pp; + + return (((uint64_t)vle32dec(p + 4) << 32) | vle32dec(p)); +} + +static __inline void +vbe16enc(void *pp, uint16_t u) +{ + uint8_t *p = (uint8_t *)pp; + + p[0] = (u >> 8) & 0xff; + p[1] = u & 0xff; +} + +#endif + +static __inline void +vbe32enc(void *pp, uint32_t u) +{ + uint8_t *p = (uint8_t *)pp; + + p[0] = (u >> 24) & 0xff; + p[1] = (u >> 16) & 0xff; + p[2] = (u >> 8) & 0xff; + p[3] = u & 0xff; +} + +static __inline void +vbe64enc(void *pp, uint64_t u) +{ + uint8_t *p = (uint8_t *)pp; + + vbe32enc(p, (uint32_t)(u >> 32)); + vbe32enc(p + 4, (uint32_t)(u & 0xffffffffU)); +} + +#if 0 + +static __inline void +vle16enc(void *pp, uint16_t u) +{ + uint8_t *p = (uint8_t *)pp; + + p[0] = u & 0xff; + p[1] = (u >> 8) & 0xff; +} + +static __inline void +vle32enc(void *pp, uint32_t u) +{ + uint8_t *p = (uint8_t *)pp; + + p[0] = u & 0xff; + p[1] = (u >> 8) & 0xff; + p[2] = (u >> 16) & 0xff; + p[3] = (u >> 24) & 0xff; +} + +static __inline void +vle64enc(void *pp, uint64_t u) +{ + uint8_t *p = (uint8_t *)pp; + + vle32enc(p, (uint32_t)(u & 0xffffffffU)); + vle32enc(p + 4, (uint32_t)(u >> 32)); +} +#endif + +#endif From phk at varnish-cache.org Wed May 19 08:15:54 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:15:54 +0200 Subject: r4820 - trunk/varnish-cache/include Message-ID: Author: phk Date: 2010-05-19 10:15:54 +0200 (Wed, 19 May 2010) New Revision: 4820 Modified: trunk/varnish-cache/include/vbm.h Log: Unhack the vbit_destroy hack, I had forgotten "inline" Modified: trunk/varnish-cache/include/vbm.h =================================================================== --- trunk/varnish-cache/include/vbm.h 2010-05-19 08:15:23 UTC (rev 4819) +++ trunk/varnish-cache/include/vbm.h 2010-05-19 08:15:54 UTC (rev 4820) @@ -75,7 +75,7 @@ return (vb); } -static void +static inline void vbit_destroy(struct vbitmap *vb) { @@ -89,9 +89,6 @@ vbit_set(struct vbitmap *vb, unsigned bit) { - if (0) /* XXX: HACK: ref it, to silence compiler */ - vbit_destroy(vb); - if (bit >= vb->nbits) vbit_expand(vb, bit); vb->bits[VBITMAP_IDX(bit)] |= VBITMAP_BIT(bit); From phk at varnish-cache.org Wed May 19 08:16:49 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:16:49 +0200 Subject: r4821 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 10:16:49 +0200 (Wed, 19 May 2010) New Revision: 4821 Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c Log: Unhack vbit_destroy() hiding. Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2010-05-19 08:15:54 UTC (rev 4820) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2010-05-19 08:16:49 UTC (rev 4821) @@ -615,9 +615,6 @@ REPORT(LOG_ERR, "vev_schedule() = %d", i); REPORT0(LOG_ERR, "manager dies"); - - /* XXX: quench compiler warning about unused func */ - vbit_destroy(NULL); } /*--------------------------------------------------------------------*/ From phk at varnish-cache.org Wed May 19 08:17:21 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:17:21 +0200 Subject: r4822 - trunk/varnish-cache/lib/libvarnish Message-ID: Author: phk Date: 2010-05-19 10:17:21 +0200 (Wed, 19 May 2010) New Revision: 4822 Modified: trunk/varnish-cache/lib/libvarnish/vsha256.c Log: Use vend.h version of endian macros. Modified: trunk/varnish-cache/lib/libvarnish/vsha256.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vsha256.c 2010-05-19 08:16:49 UTC (rev 4821) +++ trunk/varnish-cache/lib/libvarnish/vsha256.c 2010-05-19 08:17:21 UTC (rev 4822) @@ -47,27 +47,8 @@ #include "libvarnish.h" #include "vsha256.h" +#include "vend.h" -static inline void -mybe32enc(void *pp, uint32_t u) -{ - unsigned char *p = (unsigned char *)pp; - - p[0] = (u >> 24) & 0xff; - p[1] = (u >> 16) & 0xff; - p[2] = (u >> 8) & 0xff; - p[3] = u & 0xff; -} - -static inline void -mybe64enc(void *pp, uint64_t u) -{ - unsigned char *p = (unsigned char *)pp; - - mybe32enc(p, u >> 32); - mybe32enc(p + 4, u & 0xffffffff); -} - #if defined(VBYTE_ORDER) && VBYTE_ORDER == VBIG_ENDIAN /* Copy a vector of big-endian uint32_t into a vector of bytes */ @@ -80,14 +61,6 @@ #else /* BYTE_ORDER != BIG_ENDIAN or in doubt... */ -static inline uint32_t -mybe32dec(const void *pp) -{ - unsigned char const *p = (unsigned char const *)pp; - - return (((unsigned)p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]); -} - /* * Encode a length len/4 vector of (uint32_t) into a length len vector of * (unsigned char) in big-endian form. Assumes len is a multiple of 4. @@ -98,7 +71,7 @@ size_t i; for (i = 0; i < len / 4; i++) - mybe32enc(dst + i * 4, src[i]); + vbe32enc(dst + i * 4, src[i]); } /* @@ -111,7 +84,7 @@ size_t i; for (i = 0; i < len / 4; i++) - dst[i] = mybe32dec(src + i * 4); + dst[i] = vbe32dec(src + i * 4); } #endif @@ -251,7 +224,7 @@ * -- we do this now rather than later because the length * will change after we pad. */ - mybe64enc(len, ctx->count << 3); + vbe64enc(len, ctx->count << 3); /* Add 1--64 bytes so that the resulting length is 56 mod 64 */ r = ctx->count & 0x3f; From phk at varnish-cache.org Wed May 19 08:17:59 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:17:59 +0200 Subject: r4823 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 10:17:57 +0200 (Wed, 19 May 2010) New Revision: 4823 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c Log: Fix a minor confusion. Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt_re.c 2010-05-19 08:17:21 UTC (rev 4822) +++ trunk/varnish-cache/bin/varnishd/cache_vrt_re.c 2010-05-19 08:17:57 UTC (rev 4823) @@ -100,7 +100,7 @@ if (str == NULL) return (""); t = re; - memset(&ovector, 0, sizeof(ovector)); + memset(ovector, 0, sizeof(ovector)); i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30); /* If it didn't match, we can return the original string */ From phk at varnish-cache.org Wed May 19 08:18:57 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:18:57 +0200 Subject: r4824 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 10:18:57 +0200 (Wed, 19 May 2010) New Revision: 4824 Modified: trunk/varnish-cache/bin/varnishd/cache_dir_random.c Log: Use vend.h version of endian conversion. Modified: trunk/varnish-cache/bin/varnishd/cache_dir_random.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_dir_random.c 2010-05-19 08:17:57 UTC (rev 4823) +++ trunk/varnish-cache/bin/varnishd/cache_dir_random.c 2010-05-19 08:18:57 UTC (rev 4824) @@ -58,6 +58,7 @@ #include "cache_backend.h" #include "vrt.h" #include "vsha256.h" +#include "vend.h" /*--------------------------------------------------------------------*/ @@ -90,7 +91,7 @@ struct vbe_conn *vbe; struct director *d2; struct SHA256Context ctx; - unsigned char sign[SHA256_LEN], *hp; + uint8_t sign[SHA256_LEN], *hp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -124,10 +125,7 @@ * amongst the healthy set. */ if (vs->criteria != c_random) { - u = hp[3] << 24; - u |= hp[2] << 16; - u |= hp[1] << 8; - u |= hp[0] << 0; + u = vle32dec(hp); r = u / 4294967296.0; assert(r >= 0.0 && r < 1.0); r *= vs->tot_weight; From phk at varnish-cache.org Wed May 19 08:19:22 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 10:19:22 +0200 Subject: r4825 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 10:19:21 +0200 (Wed, 19 May 2010) New Revision: 4825 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_slinger.h Log: Constify arg to HSH_Ref() Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2010-05-19 08:18:57 UTC (rev 4824) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2010-05-19 08:19:21 UTC (rev 4825) @@ -595,7 +595,7 @@ } void -HSH_Ref(struct object *o) +HSH_Ref(const struct object *o) { struct objhead *oh; Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2010-05-19 08:18:57 UTC (rev 4824) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2010-05-19 08:19:21 UTC (rev 4825) @@ -56,7 +56,7 @@ void HSH_Freestore(struct object *o); struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh); void HSH_Unbusy(const struct sess *sp); -void HSH_Ref(struct object *o); +void HSH_Ref(const struct object *o); void HSH_Drop(struct sess *sp); double HSH_Grace(double g); void HSH_Init(void); From perbu at varnish-cache.org Wed May 19 10:15:35 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Wed, 19 May 2010 12:15:35 +0200 Subject: r4826 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-19 12:15:35 +0200 (Wed, 19 May 2010) New Revision: 4826 Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst Log: more on increasing hit rates Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-19 08:19:21 UTC (rev 4825) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-19 10:15:35 UTC (rev 4826) @@ -47,29 +47,32 @@ prints repsonse headers and -d discards the actual content. We dont really care about the content, only the headers. -As you can see VG ads quite a bit of information in their headers. A -lot of this information is added in their VCL. +As you can see VG ads quite a bit of information in their +headers. Some of the headers, like the X-Rick-Would-Never are specific +to vg.no and their somewhat odd sense of humour. Others, like the +X-VG-Webcache are for debugging purposes. So, to check whether a site sets cookies for a specific URL just do -``GET -Used http://example.com/ |grep Set-Cookie`` +``GET -Used http://example.com/ |grep ^Set-Cookie`` Firefox plugins ~~~~~~~~~~~~~~~ There are also a couple of great plugins for Firefox. Both *Live HTTP Headers* and *Firebug* can show you what headers are beeing sent and -recieved. +recieved. Try googling for them if you want to check them out. They +are both recommended. -HTTP Headers ------------- +The role of HTTP Headers +~~~~~~~~~~~~~~~~~~~~~~~~ Varnish considers itself part of the actual webserver, since its under your control. The role of *surrogate origin cache* is not really well defined by the IETF so RFC 2616 doesn't always tell us what we should -do +do. -Cache-control +Cache-Control ~~~~~~~~~~~~~ The Cache-Control instructs caches how to handle the content. Varnish @@ -80,6 +83,21 @@ :ref:`tutorial-increasing_your_hitrate-pragma:` for an example on how to implement support. +So make sure use issue a Cache-Control header with a max-age +header. You can have a look at what Varnish Softwares drupal server +issues::: + + $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control + Cache-Control: public, max-age=600 + +Age +~~~ + +Varnish adds a Age header to indicate how long the object has been +kept inside Varnish. You can grep out Age from varnishlog like this:: + + varnishlog -i TxHeader -I ^Age + Cookies ~~~~~~~ @@ -105,10 +123,26 @@ like removing one out of several cookies, things get difficult. Unfornunatly Varnish doesn't have good tools for manipulating the Cookies. We have to use regular expressions to do the -work. Let me show you an example where we remove everything the the cookies named COOKIE1 and COOKIE2 and you can marvel at it.:: +work. +Let me show you what Varnish Software uses. We use some cookies for +Google Analytics tracking and similar tools. The cookies are all set +and used by Javascript. Varnish and Drupal doesn't need to see those +cookies and since Varnish will cease caching of pages when the client +sends cookies we will discard these unnecessary cookies in VCL. + +In the following VCL we discard all cookies that start with a underscore.:: + + // Remove has_js and Google Analytics __* cookies. + set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", ""); + // Remove a ";" prefix, if present. + set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", ""); + +Let me show you an example where we remove everything the the cookies +named COOKIE1 and COOKIE2 and you can marvel at it.:: + sub vcl_recv { - if (req.http.Cookie) { + if (req.http.Cookie) { set req.http.Cookie = ";" req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1="); @@ -127,8 +161,44 @@ ~~~~ The Vary header is sent by the web server to indicate what makes a -HTTP object Vary. +HTTP object Vary. This makes a lot of sense with headers like +Accept-Encoding. When a server issues a "Vary: Accept-Encoding" it +tells Varnish that its needs to cache a separate version for every +different Accept-Encoding that is comming from the clients. So, if a +clients only accepts gzip encoding Varnish wont't serve the version of +the page encoded with the deflate encoding. +The problem is that the Accept-Encoding field contains a lot of +different encodings. If one browser sends:: + + Accept-Encodign: gzip,deflate + +And another one sends:: + + Accept-Encoding:: deflate, gzip + +Varnish will keep two variants of the page requested due to the +different Accept-Encoding headers. Normalizing the accept-encoding +header will sure that you have as few variants as possible. The +following VCL code will normalize the Accept-Encoding headers.:: + + if (req.http.Accept-Encoding) { + if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") { + # No point in compressing these + remove req.http.Accept-Encoding; + } elsif (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate") { + set req.http.Accept-Encoding = "deflate"; + } else { + # unkown algorithm + remove req.http.Accept-Encoding; + } + } + +The code sets the Accept-Encoding header from the client to either +gzip, deflate with a preference for gzip. + .. _tutorial-increasing_your_hitrate-pragma: Pragma @@ -148,12 +218,14 @@ ~~~~~~~~~~~~~~ Normalizing your namespace --------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. _tutorial-increasing_your_hitrate-purging: Purging -------- +~~~~~~~ HTTP Purges Modified: trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-19 08:19:21 UTC (rev 4825) +++ trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-19 10:15:35 UTC (rev 4826) @@ -41,10 +41,18 @@ Could not open sockets And here we have our problem. Something else is bound to the HTTP port -of Varnish. +of Varnish. If this doesn't help try strace or truss or come find us +on IRC. Varnish is crashing ~~~~~~~~~~~~~~~~~~~ When varnish goes bust. + + +Varnish doesn't cache +~~~~~~~~~~~~~~~~~~~~~ + +See :ref:`_tutorial-increasing_your_hitrate:`. + From phk at varnish-cache.org Wed May 19 10:42:37 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 12:42:37 +0200 Subject: r4827 - trunk/varnish-cache/bin/varnishtest/tests Message-ID: Author: phk Date: 2010-05-19 12:42:37 +0200 (Wed, 19 May 2010) New Revision: 4827 Modified: trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc Log: Add a delay to make this case more predictable. Modified: trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc 2010-05-19 10:15:35 UTC (rev 4826) +++ trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc 2010-05-19 10:42:37 UTC (rev 4827) @@ -67,6 +67,9 @@ varnish v1 -expect n_vcl == 2 varnish v1 -cli "vcl.discard vcl4" +varnish v1 -cli "vcl.list" +delay 1 +varnish v1 -cli "vcl.list" varnish v1 -expect n_backend == 1 varnish v1 -expect n_vcl == 1 From perbu at varnish-cache.org Wed May 19 12:35:36 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Wed, 19 May 2010 14:35:36 +0200 Subject: r4828 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-19 14:35:36 +0200 (Wed, 19 May 2010) New Revision: 4828 Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst Log: authorization and normalizing namespace Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-19 10:42:37 UTC (rev 4827) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-19 12:35:36 UTC (rev 4828) @@ -52,9 +52,10 @@ to vg.no and their somewhat odd sense of humour. Others, like the X-VG-Webcache are for debugging purposes. -So, to check whether a site sets cookies for a specific URL just do -``GET -Used http://example.com/ |grep ^Set-Cookie`` +So, to check whether a site sets cookies for a specific URL just do:: + GET -Used http://example.com/ |grep ^Set-Cookie + Firefox plugins ~~~~~~~~~~~~~~~ @@ -214,13 +215,27 @@ pass; } -Authentication -~~~~~~~~~~~~~~ +Authorization +~~~~~~~~~~~~~ +If Varnish sees a Authorization header it will pass the request. If +this is not what you want you can unset the header. + + Normalizing your namespace ~~~~~~~~~~~~~~~~~~~~~~~~~~ +Some sites are accessed via lots of +hostnames. http://www.varnish-software.com , +http://varnish-software.com and http://varnishsoftware.com/ all point +at the same site. Since Varnish doesn't know they are different +Varnish will cache different versions of every page for every +hostname. You can mitigate this in your web server config by setting +up redirects or by useing the following VCL::: + if (req.http.host ~ "^(www.)?varnish-?software.com") { + set req.http.host = "varnish-software.com"; + } .. _tutorial-increasing_your_hitrate-purging: From phk at varnish-cache.org Wed May 19 19:53:52 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 21:53:52 +0200 Subject: r4829 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: Author: phk Date: 2010-05-19 21:53:52 +0200 (Wed, 19 May 2010) New Revision: 4829 Added: trunk/varnish-cache/bin/varnishtest/tests/r00700.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Allow TAB in the 3rd field of the first line of HTTP requests and responses. Fixes: #700 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2010-05-19 12:35:36 UTC (rev 4828) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2010-05-19 19:53:52 UTC (rev 4829) @@ -474,7 +474,7 @@ q = p; if (!vct_iscrlf(*p)) { for (; !vct_iscrlf(*p); p++) - if (vct_isctl(*p)) + if (!vct_issep(*p) && vct_isctl(*p)) return (-1); } hp->hd[h3].b = q; Added: trunk/varnish-cache/bin/varnishtest/tests/r00700.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00700.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00700.vtc 2010-05-19 19:53:52 UTC (rev 4829) @@ -0,0 +1,16 @@ +# $Id$ + +test "check TAB in 3 header field" + +server s1 { + rxreq + send "HTTP/1.1 666 foo\tbar\n\nFOO" +} -start + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq + rxresp + expect resp.status == 666 +} -run From phk at varnish-cache.org Wed May 19 20:55:22 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 22:55:22 +0200 Subject: r4830 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: Author: phk Date: 2010-05-19 22:55:22 +0200 (Wed, 19 May 2010) New Revision: 4830 Added: trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c Log: Fix an off-by one error in vrt_assemble_string() when we use up exactly the number of bytes in the workspace. This is the most obscure test-case I have written so far, and I wonder how portable it is. I may have to drop it if it does not work on systems with different width/alignment requirements. Fixes: #693 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-19 19:53:52 UTC (rev 4829) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-19 20:55:22 UTC (rev 4830) @@ -179,7 +179,7 @@ } else { e = b; b = hp->ws->f; - WS_Release(hp->ws, 1 + e - b); + WS_Release(hp->ws, e - b); return (b); } } Added: trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc 2010-05-19 20:55:22 UTC (rev 4830) @@ -0,0 +1,83 @@ +# $Id$ +# + +test "check boundary condition on vrt_assemble_string()" + +server s1 { + rxreq + expect req.http.baz == "req.http.baz" + txresp -status 201 + + rxreq + expect req.http.baz == "req.http.baz" + txresp -status 202 + + rxreq + expect req.http.baz == "BAZ" + txresp -status 203 + +} -start + +varnish v1 -arg "-p sess_workspace=1024" -vcl+backend { + + sub vcl_recv { + set req.http.foo = + req.http.bar + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" "0123456789abcdef" + "0123456789abcdef" + "01234567"; + set req.http.baz = "BAZ"; + return (pass); + } + sub vcl_hash { + set req.hash += req.url; + return (hash); + } +} -start + +client c1 { + # This should soak up all bytes but the last in the workspace + txreq -hdr "foo: x" -hdr "bar: A" + rxresp + expect resp.status == 201 + + # This should soak up all bytes in the workspace + txreq -hdr "foo: x" -hdr "bar: AB" + rxresp + expect resp.status == 202 + + # This overcommits the workspace, failing the "bar" set, + # Thus allowing the "baz" set to work. + txreq -hdr "foo: x" -hdr "bar: ABC" + rxresp + expect resp.status == 203 + +} -run From phk at varnish-cache.org Wed May 19 21:05:38 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 19 May 2010 23:05:38 +0200 Subject: r4831 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-19 23:05:37 +0200 (Wed, 19 May 2010) New Revision: 4831 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: add -hcritbit and -spersist to usage. Fixes: #683 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-19 20:55:22 UTC (rev 4830) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2010-05-19 21:05:37 UTC (rev 4831) @@ -144,8 +144,9 @@ fprintf(stderr, FMT, "-f file", "VCL script"); fprintf(stderr, FMT, "-F", "Run in foreground"); fprintf(stderr, FMT, "-h kind[,hashoptions]", "Hash specification"); + fprintf(stderr, FMT, "", " -h critbit [default]"); fprintf(stderr, FMT, "", " -h simple_list"); - fprintf(stderr, FMT, "", " -h classic [default]"); + fprintf(stderr, FMT, "", " -h classic"); fprintf(stderr, FMT, "", " -h classic,"); fprintf(stderr, FMT, "-i identity", "Identity of varnish instance"); fprintf(stderr, FMT, "-l shl,free,fill", "Size of shared memory file"); @@ -165,6 +166,7 @@ fprintf(stderr, FMT, "", " -s file [default: use /tmp]"); fprintf(stderr, FMT, "", " -s file,"); fprintf(stderr, FMT, "", " -s file,,"); + fprintf(stderr, FMT, "", " -s persist{experimenta}"); fprintf(stderr, FMT, "", " -s file,,,"); fprintf(stderr, FMT, "-t", "Default TTL"); From phk at varnish-cache.org Thu May 20 07:24:19 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 09:24:19 +0200 Subject: r4832 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 09:24:18 +0200 (Thu, 20 May 2010) New Revision: 4832 Modified: trunk/varnish-cache/bin/varnishd/flint.sh Log: Add a diff facility for the output Modified: trunk/varnish-cache/bin/varnishd/flint.sh =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.sh 2010-05-19 21:05:37 UTC (rev 4831) +++ trunk/varnish-cache/bin/varnishd/flint.sh 2010-05-20 07:24:18 UTC (rev 4832) @@ -10,4 +10,10 @@ *.c \ ../../lib/libvarnish/*.c \ ../../lib/libvarnishcompat/execinfo.c \ - ../../lib/libvcl/*.c + ../../lib/libvcl/*.c 2>&1 | tee _.fl + +if [ -f _.fl.old ] ; then + diff -u _.fl.old _.fl +fi + +mv _.fl _.fl.old From phk at varnish-cache.org Thu May 20 07:24:36 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 09:24:36 +0200 Subject: r4833 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 09:24:36 +0200 (Thu, 20 May 2010) New Revision: 4833 Modified: trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/bin/varnishd/storage_persistent.c trunk/varnish-cache/bin/varnishd/storage_umem.c Log: Constify. Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-20 07:24:18 UTC (rev 4832) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-20 07:24:36 UTC (rev 4833) @@ -87,10 +87,10 @@ /* Synthetic Storage */ void SMS_Init(void); -extern struct stevedore sma_stevedore; -extern struct stevedore smf_stevedore; -extern struct stevedore smp_stevedore; +extern const struct stevedore sma_stevedore; +extern const struct stevedore smf_stevedore; +extern const struct stevedore smp_stevedore; #ifdef HAVE_LIBUMEM -extern struct stevedore smu_stevedore; +extern const struct stevedore smu_stevedore; #endif Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2010-05-20 07:24:18 UTC (rev 4832) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2010-05-20 07:24:36 UTC (rev 4833) @@ -541,7 +541,7 @@ /*--------------------------------------------------------------------*/ -struct stevedore smf_stevedore = { +const struct stevedore smf_stevedore = { .magic = STEVEDORE_MAGIC, .name = "file", .init = smf_init, Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-20 07:24:18 UTC (rev 4832) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-20 07:24:36 UTC (rev 4833) @@ -189,7 +189,7 @@ Lck_New(&sma_sc->sma_mtx); } -struct stevedore sma_stevedore = { +const struct stevedore sma_stevedore = { .magic = STEVEDORE_MAGIC, .name = "malloc", .init = sma_init, Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2010-05-20 07:24:18 UTC (rev 4832) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2010-05-20 07:24:36 UTC (rev 4833) @@ -1497,7 +1497,7 @@ /*--------------------------------------------------------------------*/ -struct stevedore smp_stevedore = { +const struct stevedore smp_stevedore = { .magic = STEVEDORE_MAGIC, .name = "persistent", .init = smp_init, Modified: trunk/varnish-cache/bin/varnishd/storage_umem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_umem.c 2010-05-20 07:24:18 UTC (rev 4832) +++ trunk/varnish-cache/bin/varnishd/storage_umem.c 2010-05-20 07:24:36 UTC (rev 4833) @@ -161,7 +161,7 @@ AZ(pthread_mutex_init(&smu_mtx, NULL)); } -struct stevedore smu_stevedore = { +const struct stevedore smu_stevedore = { .magic = STEVEDORE_MAGIC, .name = "umem", .init = smu_init, From phk at varnish-cache.org Thu May 20 07:28:40 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 09:28:40 +0200 Subject: r4834 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 09:28:40 +0200 (Thu, 20 May 2010) New Revision: 4834 Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c trunk/varnish-cache/bin/varnishd/hash_critbit.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c trunk/varnish-cache/bin/varnishd/hash_slinger.h Log: constify Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2010-05-20 07:24:36 UTC (rev 4833) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2010-05-20 07:28:40 UTC (rev 4834) @@ -179,7 +179,7 @@ /*--------------------------------------------------------------------*/ -struct hash_slinger hcl_slinger = { +const struct hash_slinger hcl_slinger = { .magic = SLINGER_MAGIC, .name = "classic", .init = hcl_init, Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-20 07:24:36 UTC (rev 4833) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-20 07:28:40 UTC (rev 4834) @@ -477,7 +477,7 @@ } -struct hash_slinger hcb_slinger = { +const struct hash_slinger hcb_slinger = { .magic = SLINGER_MAGIC, .name = "critbit", .start = hcb_start, Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2010-05-20 07:24:36 UTC (rev 4833) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2010-05-20 07:28:40 UTC (rev 4834) @@ -116,7 +116,7 @@ /*--------------------------------------------------------------------*/ -struct hash_slinger hsl_slinger = { +const struct hash_slinger hsl_slinger = { .magic = SLINGER_MAGIC, .name = "simple", .start = hsl_start, Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2010-05-20 07:24:36 UTC (rev 4833) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2010-05-20 07:28:40 UTC (rev 4834) @@ -103,6 +103,6 @@ void HSH_Deref(struct worker *w, struct object **o); #endif /* VARNISH_CACHE_CHILD */ -extern struct hash_slinger hsl_slinger; -extern struct hash_slinger hcl_slinger; -extern struct hash_slinger hcb_slinger; +extern const struct hash_slinger hsl_slinger; +extern const struct hash_slinger hcl_slinger; +extern const struct hash_slinger hcb_slinger; From phk at varnish-cache.org Thu May 20 07:32:24 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 09:32:24 +0200 Subject: r4835 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 09:32:24 +0200 (Thu, 20 May 2010) New Revision: 4835 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c Log: Remove unused variable. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-20 07:28:40 UTC (rev 4834) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-20 07:32:24 UTC (rev 4835) @@ -53,9 +53,8 @@ #define MAP_NOSYNC 0 /* XXX Linux */ #endif -struct varnish_stats *VSL_stats; -struct shmloghead *loghead; -unsigned char *logstart; +struct varnish_stats *VSL_stats; +struct shmloghead *loghead; uint8_t *vsl_log_start; uint8_t *vsl_log_end; uint8_t *vsl_log_nxt; From phk at varnish-cache.org Thu May 20 08:05:28 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 10:05:28 +0200 Subject: r4836 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: Author: phk Date: 2010-05-20 10:05:27 +0200 (Thu, 20 May 2010) New Revision: 4836 Added: trunk/varnish-cache/bin/varnishtest/tests/b00032.vtc Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c trunk/varnish-cache/bin/varnishd/mgt_cli.h trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Make it possible to name storage devices. The syntax is: -s [name=]type[, args] Add storage.list command to show storage devices, more details will be added later. Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2010-05-20 07:32:24 UTC (rev 4835) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2010-05-20 08:05:27 UTC (rev 4836) @@ -355,6 +355,7 @@ AZ(CLS_AddFunc(cls, MCF_NOAUTH, cli_auth)); AZ(CLS_AddFunc(cls, MCF_AUTH, cli_proto)); AZ(CLS_AddFunc(cls, MCF_AUTH, cli_debug)); + AZ(CLS_AddFunc(cls, MCF_AUTH, cli_stv)); AZ(CLS_AddFunc(cls, MCF_AUTH, cli_askchild)); } Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.h 2010-05-20 07:32:24 UTC (rev 4835) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.h 2010-05-20 08:05:27 UTC (rev 4836) @@ -44,3 +44,6 @@ cli_func_t mcf_config_discard; cli_func_t mcf_config_list; cli_func_t mcf_config_show; + +/* stevedore.c */ +extern struct cli_proto cli_stv[]; Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2010-05-20 07:32:24 UTC (rev 4835) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2010-05-20 08:05:27 UTC (rev 4836) @@ -38,6 +38,7 @@ #include "cache.h" #include "stevedore.h" +#include "cli_priv.h" static VTAILQ_HEAD(, stevedore) stevedores = VTAILQ_HEAD_INITIALIZER(stevedores); @@ -271,11 +272,20 @@ STV_config(const char *spec) { char **av; + const char *p, *q; struct stevedore *stv; const struct stevedore *stv2; - int ac; + int ac, l; + static unsigned seq = 0; - av = ParseArgv(spec, ARGV_COMMA); + p = strchr(spec, '='); + q = strchr(spec, ','); + if (p != NULL && (q == NULL || q > p)) { + av = ParseArgv(p + 1, ARGV_COMMA); + } else { + av = ParseArgv(spec, ARGV_COMMA); + p = NULL; + } AN(av); if (av[0] != NULL) @@ -289,6 +299,8 @@ stv2 = pick(STV_choice, av[1], "storage"); AN(stv2); + + /* Append to ident string */ vsb_printf(vident, ",-s%s", av[1]); av += 2; @@ -300,6 +312,16 @@ *stv = *stv2; AN(stv->name); AN(stv->alloc); + + if (p == NULL) + bprintf(stv->ident, "storage_%u", seq++); + else { + l = p - spec; + if (l > sizeof stv->ident - 1) + l = sizeof stv->ident - 1; + bprintf(stv->ident, "%*.*s", l, l, spec); + } + stv->lru = LRU_Alloc(); if (stv->init != NULL) @@ -312,3 +334,28 @@ if (!stv_next) stv_next = VTAILQ_FIRST(&stevedores); } + +/*--------------------------------------------------------------------*/ + +static void +stv_cli_list(struct cli *cli, const char * const *av, void *priv) +{ + struct stevedore *stv; + + ASSERT_MGT(); + (void)av; + (void)priv; + cli_out(cli, "Storage devices:\n"); + VTAILQ_FOREACH(stv, &stevedores, list) { + cli_out(cli, "\tstorage.%s.%s\n", stv->name, stv->ident); + } +} + +/*--------------------------------------------------------------------*/ + +struct cli_proto cli_stv[] = { + { "storage.list", "storage.list", "List storage devices\n", + 0, 0, "", stv_cli_list }, + { NULL} +}; + Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-20 07:32:24 UTC (rev 4835) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2010-05-20 08:05:27 UTC (rev 4836) @@ -65,6 +65,7 @@ void *priv; VTAILQ_ENTRY(stevedore) list; + char ident[16]; }; struct object *STV_NewObject(struct sess *sp, unsigned len, double ttl, Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-20 07:32:24 UTC (rev 4835) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-20 08:05:27 UTC (rev 4836) @@ -174,7 +174,7 @@ if ((u != (uintmax_t)(size_t)u)) ARGV_ERR("(-smalloc) size \"%s\": too big\n", av[0]); - printf("storage_malloc: max size %ju MB.\n", + printf("storage.malloc.%s: max size %ju MB.\n", parent->ident, u / (1024 * 1024)); sc->sma_max = u; Added: trunk/varnish-cache/bin/varnishtest/tests/b00032.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/b00032.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/b00032.vtc 2010-05-20 08:05:27 UTC (rev 4836) @@ -0,0 +1,5 @@ +# $Id$ + +test "CLI coverage test" + +varnish v1 -cliok storage.list From phk at varnish-cache.org Thu May 20 08:42:08 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 10:42:08 +0200 Subject: r4837 - in trunk/varnish-cache: bin/varnishadm bin/varnishd include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-20 10:42:07 +0200 (Thu, 20 May 2010) New Revision: 4837 Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/mgt_cli.c trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/include/shmlog.h trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/vsl.c Log: Make the shmlog-alloc's id threepart: class+type+ident to make things simpler down the road. Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c =================================================================== --- trunk/varnish-cache/bin/varnishadm/varnishadm.c 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/bin/varnishadm/varnishadm.c 2010-05-20 08:42:07 UTC (rev 4837) @@ -247,13 +247,13 @@ assert(VSL_Arg(vsd, 'n', n_arg)); if (!VSL_Open(vsd)) { if (T_arg == NULL) { - p = VSL_Find_Alloc(vsd, "Arg", "-T", NULL); + p = VSL_Find_Alloc(vsd, "Arg", "-T", "", NULL); if (p != NULL) { T_arg = strdup(p); } } if (S_arg == NULL) { - p = VSL_Find_Alloc(vsd, "Arg", "-S", NULL); + p = VSL_Find_Alloc(vsd, "Arg", "-S", "", NULL); if (p != NULL) { S_arg = strdup(p); } Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-20 08:42:07 UTC (rev 4837) @@ -39,7 +39,7 @@ void VCA_tweak_waiter(struct cli *cli, const char *arg); /* mgt_shmem.c */ -void *mgt_SHM_Alloc(unsigned size, const char *type, const char *ident); +void *mgt_SHM_Alloc(unsigned size, const char *class, const char *type, const char *ident); extern struct varnish_stats *VSL_stats; extern struct shmloghead *loghead; extern uint8_t *vsl_log_start; Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2010-05-20 08:42:07 UTC (rev 4837) @@ -511,7 +511,7 @@ /* Save in shmem */ i = strlen(S_arg); - p = mgt_SHM_Alloc(i + 1, "Arg", "-S"); + p = mgt_SHM_Alloc(i + 1, "Arg", "-S", ""); AN(p); strcpy(p, S_arg); @@ -546,7 +546,7 @@ /* Save in shmem */ i = strlen(T_arg); - p = mgt_SHM_Alloc(i + 1, "Arg", "-T"); + p = mgt_SHM_Alloc(i + 1, "Arg", "-T", ""); AN(p); strcpy(p, T_arg); Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-20 08:42:07 UTC (rev 4837) @@ -64,7 +64,7 @@ /*--------------------------------------------------------------------*/ void * -mgt_SHM_Alloc(unsigned size, const char *type, const char *ident) +mgt_SHM_Alloc(unsigned size, const char *class, const char *type, const char *ident) { struct shmalloc *sha, *sha2; unsigned seq; @@ -99,6 +99,7 @@ MEMORY_BARRIER(); sha->len = size; + bprintf(sha->class, "%s", class); bprintf(sha->type, "%s", type); bprintf(sha->ident, "%s", ident); MEMORY_BARRIER(); @@ -300,15 +301,15 @@ bprintf(loghead->head.type, "%s", "Free"); MEMORY_BARRIER(); - VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_TYPE_STAT, ""); + VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_CLASS_STAT, "", ""); AN(VSL_stats); - pp = mgt_SHM_Alloc(sizeof *pp, "Params", ""); + pp = mgt_SHM_Alloc(sizeof *pp, "Params", "", ""); AN(pp); *pp = *params; params = pp; - vsl_log_start = mgt_SHM_Alloc(s1, VSL_TYPE_LOG, ""); + vsl_log_start = mgt_SHM_Alloc(s1, VSL_CLASS_LOG, "", ""); AN(vsl_log_start); vsl_log_end = vsl_log_start + s1; vsl_log_nxt = vsl_log_start + 1; Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/include/shmlog.h 2010-05-20 08:42:07 UTC (rev 4837) @@ -51,15 +51,16 @@ #define SHMALLOC_MAGIC 0x43907b6e /* From /dev/random */ unsigned magic; unsigned len; + char class[8]; char type[8]; - char ident[8]; + char ident[16]; }; #define SHA_NEXT(sha) ((void*)((uintptr_t)(sha) + (sha)->len)) #define SHA_PTR(sha) ((void*)((uintptr_t)((sha) + 1))) struct shmloghead { -#define SHMLOGHEAD_MAGIC 4185512500U /* From /dev/random */ +#define SHMLOGHEAD_MAGIC 4185512501U /* From /dev/random */ unsigned magic; unsigned hdrsize; @@ -78,7 +79,8 @@ struct shmalloc head; }; -#define VSL_TYPE_LOG "Log" +#define VSL_CLASS_LOG "Log" +#define VSL_CLASS_STAT "Stat" /* * Record format is as follows: Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-20 08:42:07 UTC (rev 4837) @@ -62,8 +62,7 @@ struct varnish_stats *VSL_OpenStats(struct VSL_data *vd); const char *VSL_Name(struct VSL_data *vd); extern const char *VSL_tags[256]; -void *VSL_Find_Alloc(struct VSL_data *vd, const char *type, const char *ident, - unsigned *lenp); +void *VSL_Find_Alloc(struct VSL_data *vd, const char *class, const char *type, + const char *ident, unsigned *lenp); - #endif Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-20 08:05:27 UTC (rev 4836) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-20 08:42:07 UTC (rev 4837) @@ -165,15 +165,17 @@ /*--------------------------------------------------------------------*/ static struct shmalloc * -vsl_find_alloc(struct VSL_data *vd, const char *type, const char *ident) +vsl_find_alloc(struct VSL_data *vd, const char *class, const char *type, const char *ident) { struct shmalloc *sha; assert (vd->vsl_lh != NULL); for(sha = &vd->vsl_lh->head; (void*)sha < vd->vsl_end; sha = SHA_NEXT(sha)) { CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); - if (strcmp(sha->type, type)) + if (strcmp(sha->class, class)) continue; + if (type != NULL && strcmp(sha->type, type)) + continue; if (ident != NULL && strcmp(sha->ident, ident)) continue; return (sha); @@ -184,7 +186,7 @@ /*--------------------------------------------------------------------*/ void * -VSL_Find_Alloc(struct VSL_data *vd, const char *type, const char *ident, +VSL_Find_Alloc(struct VSL_data *vd, const char *class, const char *type, const char *ident, unsigned *lenp) { struct shmalloc *sha; @@ -192,7 +194,7 @@ CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (VSL_Open(vd)) return (NULL); - sha = vsl_find_alloc(vd, type, ident); + sha = vsl_find_alloc(vd, class, type, ident); if (sha == NULL) return (NULL); if (lenp != NULL) @@ -210,7 +212,7 @@ CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (VSL_Open(vd)) return (NULL); - sha = vsl_find_alloc(vd, VSL_TYPE_STAT, ""); + sha = vsl_find_alloc(vd, VSL_CLASS_STAT, "", ""); assert(sha != NULL); return (SHA_PTR(sha)); } @@ -226,7 +228,7 @@ CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (VSL_Open(vd)) return (-1); - sha = vsl_find_alloc(vd, VSL_TYPE_LOG, ""); + sha = vsl_find_alloc(vd, VSL_CLASS_LOG, "", ""); assert(sha != NULL); vd->log_start = SHA_PTR(sha); From phk at varnish-cache.org Thu May 20 09:08:50 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 11:08:50 +0200 Subject: r4838 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 11:08:50 +0200 (Thu, 20 May 2010) New Revision: 4838 Modified: trunk/varnish-cache/bin/varnishd/common.h Log: Oops, #warning, not #warn. I have clearly been doing too much assembly programming recently. Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-20 08:42:07 UTC (rev 4837) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-20 09:08:50 UTC (rev 4838) @@ -83,6 +83,6 @@ #include #define MEMORY_BARRIER() mb() #else -#warn "MEMORY_BARRIER() is expensive" +#warning "MEMORY_BARRIER() is expensive" #define MEMORY_BARRIER() close(-1) #endif From phk at varnish-cache.org Thu May 20 09:23:43 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 11:23:43 +0200 Subject: r4839 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 11:23:42 +0200 (Thu, 20 May 2010) New Revision: 4839 Modified: trunk/varnish-cache/bin/varnishd/common.h Log: Just remove the dang #warning, it's going to be removed as soon as I convert to vmb.h anyway. Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2010-05-20 09:08:50 UTC (rev 4838) +++ trunk/varnish-cache/bin/varnishd/common.h 2010-05-20 09:23:42 UTC (rev 4839) @@ -83,6 +83,5 @@ #include #define MEMORY_BARRIER() mb() #else -#warning "MEMORY_BARRIER() is expensive" #define MEMORY_BARRIER() close(-1) #endif From phk at varnish-cache.org Thu May 20 09:45:22 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 11:45:22 +0200 Subject: r4840 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 11:45:20 +0200 (Thu, 20 May 2010) New Revision: 4840 Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c Log: Correctly scope a local variable Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_panic.c 2010-05-20 09:23:42 UTC (rev 4839) +++ trunk/varnish-cache/bin/varnishd/cache_panic.c 2010-05-20 09:45:20 UTC (rev 4840) @@ -294,7 +294,6 @@ pan_ic(const char *func, const char *file, int line, const char *cond, int err, int xxx) { - char *p; const char *q; const struct sess *sp; @@ -347,6 +346,7 @@ #ifdef HAVE_ABORT2 if (params->diag_bitmap & 0x8000) { void *arg[1]; + char *p; for (p = loghead->panicstr; *p; p++) if (*p == '\n') From phk at varnish-cache.org Thu May 20 09:46:31 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 11:46:31 +0200 Subject: r4841 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-20 11:46:30 +0200 (Thu, 20 May 2010) New Revision: 4841 Modified: trunk/varnish-cache/bin/varnishd/mgt.h Log: Always include Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-20 09:45:20 UTC (rev 4840) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2010-05-20 09:46:30 UTC (rev 4841) @@ -29,8 +29,10 @@ * $Id$ */ -#include +#include +#include "vqueue.h" + #include "common.h" #include "miniobj.h" From phk at varnish-cache.org Thu May 20 10:10:48 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 12:10:48 +0200 Subject: r4842 - trunk/varnish-cache/include Message-ID: Author: phk Date: 2010-05-20 12:10:48 +0200 (Thu, 20 May 2010) New Revision: 4842 Modified: trunk/varnish-cache/include/Makefile.am Log: List vend.h and vmb.h Modified: trunk/varnish-cache/include/Makefile.am =================================================================== --- trunk/varnish-cache/include/Makefile.am 2010-05-20 09:46:30 UTC (rev 4841) +++ trunk/varnish-cache/include/Makefile.am 2010-05-20 10:10:48 UTC (rev 4842) @@ -36,9 +36,11 @@ vcl.h \ vcl_returns.h \ vct.h \ + vend.h \ vev.h \ vlu.h \ vbm.h \ + vmb.h \ vre.h \ vrt.h \ vrt_obj.h \ From phk at varnish-cache.org Thu May 20 10:40:21 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 12:40:21 +0200 Subject: r4843 - trunk/varnish-cache/lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-20 12:40:21 +0200 (Thu, 20 May 2010) New Revision: 4843 Modified: trunk/varnish-cache/lib/libvarnishapi/Makefile.am Log: List vsl.h Modified: trunk/varnish-cache/lib/libvarnishapi/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2010-05-20 10:10:48 UTC (rev 4842) +++ trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2010-05-20 10:40:21 UTC (rev 4843) @@ -7,6 +7,8 @@ libvarnishapi_la_LDFLAGS = -version-info 1:0:0 libvarnishapi_la_SOURCES = \ + vsl.h \ + \ ../libvarnish/vin.c \ base64.c \ vsl.c \ From phk at varnish-cache.org Thu May 20 10:53:16 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 20 May 2010 12:53:16 +0200 Subject: r4844 - in trunk/varnish-cache/bin/varnishtest: . tests Message-ID: Author: phk Date: 2010-05-20 12:53:16 +0200 (Thu, 20 May 2010) New Revision: 4844 Modified: trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc trunk/varnish-cache/bin/varnishtest/vtc.c Log: Try to limit r00693 to 64 bit systems. Modified: trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc 2010-05-20 10:40:21 UTC (rev 4843) +++ trunk/varnish-cache/bin/varnishtest/tests/r00693.vtc 2010-05-20 10:53:16 UTC (rev 4844) @@ -3,6 +3,8 @@ test "check boundary condition on vrt_assemble_string()" +feature 64bit + server s1 { rxreq expect req.http.baz == "req.http.baz" Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-20 10:40:21 UTC (rev 4843) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-20 10:53:16 UTC (rev 4844) @@ -482,6 +482,9 @@ if (!strcmp(av[i], "SO_RCVTIMEO_WORKS")) continue; #endif + if (sizeof(void*) == 8 && !strcmp(av[i], "64bit")) + continue; + vtc_log(vl, 1, "SKIPPING test, missing feature %s", av[i]); vtc_stop = 1; return; From phk at varnish-cache.org Fri May 21 07:27:41 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 09:27:41 +0200 Subject: r4845 - trunk/varnish-cache/bin/varnishtest Message-ID: Author: phk Date: 2010-05-21 09:27:41 +0200 (Fri, 21 May 2010) New Revision: 4845 Modified: trunk/varnish-cache/bin/varnishtest/vtc.c Log: Explicitly seed the random() function, we might run on a debian system. Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-20 10:53:16 UTC (rev 4844) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2010-05-21 07:27:41 UTC (rev 4845) @@ -673,6 +673,7 @@ init_macro(); init_sema(); + srandomdev(); bprintf(vtc_tmpdir, "/tmp/vtc.%d.%08x", getpid(), (unsigned)random()); AZ(mkdir(vtc_tmpdir, 0700)); macro_def(vltop, NULL, "tmpdir", vtc_tmpdir); From phk at varnish-cache.org Fri May 21 07:59:59 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 09:59:59 +0200 Subject: r4846 - trunk/varnish-cache/bin/varnishstat Message-ID: Author: phk Date: 2010-05-21 09:59:59 +0200 (Fri, 21 May 2010) New Revision: 4846 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Assert curses returns, and generally polish for FlexeLint Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-21 07:27:41 UTC (rev 4845) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-21 07:59:59 UTC (rev 4846) @@ -39,7 +39,6 @@ #include #include -#include #include #include #include @@ -53,7 +52,9 @@ #define FIELD_EXCLUSION_CHARACTER '^' +#define AC(x) assert((x) != ERR) + static void myexp(double *acc, double val, unsigned *n, unsigned nmax) { @@ -66,8 +67,8 @@ static int show_field(const char* field, const char *fields) { - char* field_start; - char* field_end; + const char* field_start; + const char* field_end; int field_length; int match_value = 1; @@ -94,7 +95,7 @@ } static void -do_curses(struct VSL_data *vd, struct varnish_stats *VSL_stats, int delay, const char *fields) +do_curses(struct VSL_data *vd, const struct varnish_stats *VSL_stats, int delay, const char *fields) { struct varnish_stats copy; struct varnish_stats seen; @@ -112,26 +113,26 @@ a1 = a2 = a3 = 0.0; n1 = n2 = n3 = 0; - initscr(); - raw(); - noecho(); - nonl(); - intrflush(stdscr, FALSE); - curs_set(0); - erase(); + (void)initscr(); + AC(raw()); + AC(noecho()); + AC(nonl()); + AC(intrflush(stdscr, FALSE)); + AC(curs_set(0)); + AC(erase()); lt = 0; while (1) { - gettimeofday(&tv, NULL); + AZ(gettimeofday(&tv, NULL)); tt = tv.tv_usec * 1e-6 + tv.tv_sec; lt = tt - lt; rt = VSL_stats->uptime; up = rt; - mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd)); - mvprintw(0, 0, "%d+%02d:%02d:%02d", rt / 86400, - (rt % 86400) / 3600, (rt % 3600) / 60, rt % 60); + AC(mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd))); + AC(mvprintw(0, 0, "%d+%02d:%02d:%02d", rt / 86400, + (rt % 86400) / 3600, (rt % 3600) / 60, rt % 60)); hit = VSL_stats->cache_hit - copy.cache_hit; miss = VSL_stats->cache_miss - copy.cache_miss; @@ -143,55 +144,55 @@ myexp(&a2, ratio, &n2, 100); myexp(&a3, ratio, &n3, 1000); } - mvprintw(1, 0, "Hitrate ratio: %8u %8u %8u", n1, n2, n3); - mvprintw(2, 0, "Hitrate avg: %8.4f %8.4f %8.4f", a1, a2, a3); + AC(mvprintw(1, 0, "Hitrate ratio: %8u %8u %8u", n1, n2, n3)); + AC(mvprintw(2, 0, "Hitrate avg: %8.4f %8.4f %8.4f", a1, a2, a3)); line = 3; -#define MAC_STAT(n, t, l, f, d) \ +#define MAC_STAT(n, t, l, ff, d) \ if ((fields == NULL || show_field( #n, fields )) && line < LINES) { \ ju = VSL_stats->n; \ if (ju == 0 && !seen.n) { \ - } else if (f == 'a') { \ + } else if (ff == 'a') { \ seen.n = 1; \ line++; \ - mvprintw(line, 0, "%12ju %12.2f %12.2f %s\n", \ - ju, (ju - (intmax_t)copy.n)/lt, ju / up, d); \ + AC(mvprintw(line, 0, "%12ju %12.2f %12.2f %s\n", \ + ju, (ju - (intmax_t)copy.n)/lt, ju / up, d)); \ copy.n = ju; \ } else { \ seen.n = 1; \ line++; \ - mvprintw(line, 0, "%12ju %12s %12s %s\n", \ - ju, ". ", ". ", d); \ + AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", \ + ju, ". ", ". ", d)); \ } \ } #include "stat_field.h" #undef MAC_STAT lt = tt; - refresh(); + AC(refresh()); timeout(delay * 1000); switch ((ch = getch())) { case ERR: break; #ifdef KEY_RESIZE case KEY_RESIZE: - erase(); + AC(erase()); break; #endif case '\014': /* Ctrl-L */ case '\024': /* Ctrl-T */ - redrawwin(stdscr); - refresh(); + AC(redrawwin(stdscr)); + AC(refresh()); break; case '\003': /* Ctrl-C */ - raise(SIGINT); + AZ(raise(SIGINT)); break; case '\032': /* Ctrl-Z */ - raise(SIGTSTP); + AZ(raise(SIGTSTP)); break; case '\021': /* Ctrl-Q */ case 'Q': case 'q': - endwin(); + AC(endwin()); exit(0); case '0': case '1': @@ -203,24 +204,24 @@ case '7': case '8': case '9': - delay = 1 << (ch - '0'); + delay = 1U << (ch - '0'); break; default: - beep(); + AC(beep()); break; } } } static void -do_xml(struct varnish_stats *VSL_stats, const char* fields) +do_xml(const struct varnish_stats *VSL_stats, const char* fields) { char time_stamp[20]; time_t now; printf("\n"); now = time(NULL); - strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now)); + (void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now)); printf("\n", time_stamp); #define MAC_STAT(n, t, l, f, d) \ do { \ @@ -238,19 +239,19 @@ } static void -do_once(struct varnish_stats *VSL_stats, const char* fields) +do_once(const struct varnish_stats *VSL_stats, const char* fields) { struct timeval tv; double up; - gettimeofday(&tv, NULL); + AZ(gettimeofday(&tv, NULL)); up = VSL_stats->uptime; -#define MAC_STAT(n, t, l, f, d) \ +#define MAC_STAT(n, t, l, ff, d) \ do { \ if (fields != NULL && ! show_field( #n, fields )) break; \ intmax_t ju = VSL_stats->n; \ - if (f == 'a') \ + if (ff == 'a') \ printf("%-16s %12ju %12.2f %s\n", #n, ju, ju / up, d); \ else \ printf("%-16s %12ju %12s %s\n", #n, ju, ". ", d); \ @@ -330,9 +331,9 @@ } if (!valid_field) { - fputs("The field '", stderr); - fwrite(field_start, 1, field_length, stderr); - fputs("' is not a valid field\n", stderr); + (void)fputs("The field '", stderr); + (void)fwrite(field_start, 1, field_length, stderr); + (void)fputs("' is not a valid field\n", stderr); return (0); } @@ -344,7 +345,7 @@ } int -main(int argc, char **argv) +main(int argc, char * const *argv) { int c; struct VSL_data *vd; From phk at varnish-cache.org Fri May 21 08:13:51 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 10:13:51 +0200 Subject: r4847 - trunk/varnish-cache/bin/varnishstat Message-ID: Author: phk Date: 2010-05-21 10:13:51 +0200 (Fri, 21 May 2010) New Revision: 4847 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: polishing Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-21 07:59:59 UTC (rev 4846) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-21 08:13:51 UTC (rev 4847) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2009 Linpro AS + * Copyright (c) 2006-2010 Linpro AS * All rights reserved. * * Author: Poul-Henning Kamp @@ -148,22 +148,24 @@ AC(mvprintw(2, 0, "Hitrate avg: %8.4f %8.4f %8.4f", a1, a2, a3)); line = 3; -#define MAC_STAT(n, t, l, ff, d) \ +#define MAC_STAT(n, t, l, ff, d) \ if ((fields == NULL || show_field( #n, fields )) && line < LINES) { \ - ju = VSL_stats->n; \ - if (ju == 0 && !seen.n) { \ - } else if (ff == 'a') { \ - seen.n = 1; \ - line++; \ - AC(mvprintw(line, 0, "%12ju %12.2f %12.2f %s\n", \ - ju, (ju - (intmax_t)copy.n)/lt, ju / up, d)); \ - copy.n = ju; \ - } else { \ - seen.n = 1; \ - line++; \ - AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", \ - ju, ". ", ". ", d)); \ - } \ + ju = VSL_stats->n; \ + if (ju == 0 && !seen.n) { \ + } else if (ff == 'a') { \ + seen.n = 1; \ + line++; \ + AC(mvprintw(line, 0, \ + "%12ju %12.2f %12.2f %s\n", \ + ju, (ju - (intmax_t)copy.n)/lt, \ + ju / up, d)); \ + copy.n = ju; \ + } else { \ + seen.n = 1; \ + line++; \ + AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", \ + ju, ". ", ". ", d)); \ + } \ } #include "stat_field.h" #undef MAC_STAT @@ -223,15 +225,16 @@ now = time(NULL); (void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now)); printf("\n", time_stamp); -#define MAC_STAT(n, t, l, f, d) \ - do { \ - if (fields != NULL && ! show_field( #n, fields )) break; \ - intmax_t ju = VSL_stats->n; \ - printf("\t\n"); \ - printf("\t\t%s\n", #n); \ - printf("\t\t%ju\n", ju); \ - printf("\t\t%s\n", d); \ - printf("\t\n"); \ +#define MAC_STAT(n, t, l, f, d) \ + do { \ + if (fields != NULL && ! show_field( #n, fields )) \ + break; \ + intmax_t ju = VSL_stats->n; \ + printf("\t\n"); \ + printf("\t\t%s\n", #n); \ + printf("\t\t%ju\n", ju); \ + printf("\t\t%s\n", d); \ + printf("\t\n"); \ } while (0); #include "stat_field.h" #undef MAC_STAT @@ -247,14 +250,17 @@ AZ(gettimeofday(&tv, NULL)); up = VSL_stats->uptime; -#define MAC_STAT(n, t, l, ff, d) \ - do { \ - if (fields != NULL && ! show_field( #n, fields )) break; \ - intmax_t ju = VSL_stats->n; \ - if (ff == 'a') \ - printf("%-16s %12ju %12.2f %s\n", #n, ju, ju / up, d); \ - else \ - printf("%-16s %12ju %12s %s\n", #n, ju, ". ", d); \ +#define MAC_STAT(n, t, l, ff, d) \ + do { \ + if (fields != NULL && ! show_field( #n, fields )) \ + break; \ + intmax_t ju = VSL_stats->n; \ + if (ff == 'a') \ + printf("%-16s %12ju %12.2f %s\n", \ + #n, ju, ju / up, d); \ + else \ + printf("%-16s %12ju %12s %s\n", \ + #n, ju, ". ", d); \ } while (0); #include "stat_field.h" #undef MAC_STAT @@ -291,9 +297,9 @@ fprintf(stderr, "Field name Description\n"); fprintf(stderr, "---------- -----------\n"); -#define MAC_STAT(n, t, l, f, d) \ - do { \ - fprintf(stderr, "%-20s %s\n", #n, d);\ +#define MAC_STAT(n, t, l, f, d) \ + do { \ + fprintf(stderr, "%-20s %s\n", #n, d); \ } while (0); #include "stat_field.h" #undef MAC_STAT @@ -349,7 +355,7 @@ { int c; struct VSL_data *vd; - struct varnish_stats *VSL_stats; + const struct varnish_stats *VSL_stats; int delay = 1, once = 0, xml = 0; const char *fields = NULL; From phk at varnish-cache.org Fri May 21 10:34:52 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 12:34:52 +0200 Subject: r4848 - in trunk/varnish-cache: include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-21 12:34:52 +0200 (Fri, 21 May 2010) New Revision: 4848 Modified: trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/vsl.c trunk/varnish-cache/lib/libvarnishapi/vsl.h Log: Add a supported iterator over shmlog allocations. Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-21 08:13:51 UTC (rev 4847) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-21 10:34:52 UTC (rev 4848) @@ -65,4 +65,10 @@ void *VSL_Find_Alloc(struct VSL_data *vd, const char *class, const char *type, const char *ident, unsigned *lenp); +struct shmalloc *vsl_iter0(struct VSL_data *vd); +struct shmalloc *vsl_itern(struct VSL_data *vd, struct shmalloc **pp); + +#define VSL_FOREACH(var, vd) \ + for((var) = vsl_iter0((vd)); (var) != NULL; vsl_itern((vd), &(var))) + #endif Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-21 08:13:51 UTC (rev 4847) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-21 10:34:52 UTC (rev 4848) @@ -145,6 +145,8 @@ return (1); } vd->vsl_end = (uint8_t *)vd->vsl_lh + slh.shm_size; + + vd->alloc_seq = slh.alloc_seq; return (0); } @@ -164,13 +166,41 @@ /*--------------------------------------------------------------------*/ +struct shmalloc * +vsl_iter0(struct VSL_data *vd) +{ + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (vd->alloc_seq != vd->vsl_lh->alloc_seq) + return(NULL); + CHECK_OBJ_NOTNULL(&vd->vsl_lh->head, SHMALLOC_MAGIC); + return (&vd->vsl_lh->head); +} + +struct shmalloc * +vsl_itern(struct VSL_data *vd, struct shmalloc **pp) +{ + + CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); + if (vd->alloc_seq != vd->vsl_lh->alloc_seq) + return(NULL); + CHECK_OBJ_NOTNULL(*pp, SHMALLOC_MAGIC); + *pp = SHA_NEXT(*pp); + if ((void*)*pp >= vd->vsl_end) + return (NULL); + CHECK_OBJ_NOTNULL(*pp, SHMALLOC_MAGIC); + return (*pp); +} + +/*--------------------------------------------------------------------*/ + static struct shmalloc * vsl_find_alloc(struct VSL_data *vd, const char *class, const char *type, const char *ident) { struct shmalloc *sha; assert (vd->vsl_lh != NULL); - for(sha = &vd->vsl_lh->head; (void*)sha < vd->vsl_end; sha = SHA_NEXT(sha)) { + VSL_FOREACH(sha, vd) { CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); if (strcmp(sha->class, class)) continue; Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.h =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-21 08:13:51 UTC (rev 4847) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.h 2010-05-21 10:34:52 UTC (rev 4848) @@ -41,6 +41,7 @@ int vsl_fd; struct shmloghead *vsl_lh; void *vsl_end; + unsigned alloc_seq; unsigned char *log_start; unsigned char *log_end; From phk at varnish-cache.org Fri May 21 10:53:34 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 12:53:34 +0200 Subject: r4849 - trunk/varnish-cache/bin/varnishstat Message-ID: Author: phk Date: 2010-05-21 12:53:34 +0200 (Fri, 21 May 2010) New Revision: 4849 Added: trunk/varnish-cache/bin/varnishstat/varnishstat.h trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Move the curses stuff into its own file Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishstat/Makefile.am 2010-05-21 10:34:52 UTC (rev 4848) +++ trunk/varnish-cache/bin/varnishstat/Makefile.am 2010-05-21 10:53:34 UTC (rev 4849) @@ -6,7 +6,11 @@ dist_man_MANS = varnishstat.1 -varnishstat_SOURCES = varnishstat.c +varnishstat_SOURCES = \ + varnishstat.h \ + \ + varnishstat.c \ + varnishstat_curses.c varnishstat_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-21 10:34:52 UTC (rev 4848) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2010-05-21 10:53:34 UTC (rev 4849) @@ -27,7 +27,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Log tailer for Varnish + * Statistics output program */ #include "config.h" @@ -37,9 +37,7 @@ #include -#include #include -#include #include #include #include @@ -49,22 +47,11 @@ #include "libvarnish.h" #include "shmlog.h" #include "varnishapi.h" +#include "varnishstat.h" #define FIELD_EXCLUSION_CHARACTER '^' -#define AC(x) assert((x) != ERR) - - -static void -myexp(double *acc, double val, unsigned *n, unsigned nmax) -{ - - if (*n < nmax) - (*n)++; - (*acc) += (val - *acc) / (double)*n; -} - -static int +int show_field(const char* field, const char *fields) { const char* field_start; @@ -95,127 +82,6 @@ } static void -do_curses(struct VSL_data *vd, const struct varnish_stats *VSL_stats, int delay, const char *fields) -{ - struct varnish_stats copy; - struct varnish_stats seen; - intmax_t ju; - struct timeval tv; - double tt, lt, hit, miss, ratio, up; - double a1, a2, a3; - unsigned n1, n2, n3; - time_t rt; - int ch, line; - - memset(©, 0, sizeof copy); - memset(&seen, 0, sizeof seen); - - a1 = a2 = a3 = 0.0; - n1 = n2 = n3 = 0; - - (void)initscr(); - AC(raw()); - AC(noecho()); - AC(nonl()); - AC(intrflush(stdscr, FALSE)); - AC(curs_set(0)); - AC(erase()); - - lt = 0; - while (1) { - AZ(gettimeofday(&tv, NULL)); - tt = tv.tv_usec * 1e-6 + tv.tv_sec; - lt = tt - lt; - - rt = VSL_stats->uptime; - up = rt; - - AC(mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd))); - AC(mvprintw(0, 0, "%d+%02d:%02d:%02d", rt / 86400, - (rt % 86400) / 3600, (rt % 3600) / 60, rt % 60)); - - hit = VSL_stats->cache_hit - copy.cache_hit; - miss = VSL_stats->cache_miss - copy.cache_miss; - hit /= lt; - miss /= lt; - if (hit + miss != 0) { - ratio = hit / (hit + miss); - myexp(&a1, ratio, &n1, 10); - myexp(&a2, ratio, &n2, 100); - myexp(&a3, ratio, &n3, 1000); - } - AC(mvprintw(1, 0, "Hitrate ratio: %8u %8u %8u", n1, n2, n3)); - AC(mvprintw(2, 0, "Hitrate avg: %8.4f %8.4f %8.4f", a1, a2, a3)); - - line = 3; -#define MAC_STAT(n, t, l, ff, d) \ - if ((fields == NULL || show_field( #n, fields )) && line < LINES) { \ - ju = VSL_stats->n; \ - if (ju == 0 && !seen.n) { \ - } else if (ff == 'a') { \ - seen.n = 1; \ - line++; \ - AC(mvprintw(line, 0, \ - "%12ju %12.2f %12.2f %s\n", \ - ju, (ju - (intmax_t)copy.n)/lt, \ - ju / up, d)); \ - copy.n = ju; \ - } else { \ - seen.n = 1; \ - line++; \ - AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", \ - ju, ". ", ". ", d)); \ - } \ - } -#include "stat_field.h" -#undef MAC_STAT - lt = tt; - AC(refresh()); - timeout(delay * 1000); - switch ((ch = getch())) { - case ERR: - break; -#ifdef KEY_RESIZE - case KEY_RESIZE: - AC(erase()); - break; -#endif - case '\014': /* Ctrl-L */ - case '\024': /* Ctrl-T */ - AC(redrawwin(stdscr)); - AC(refresh()); - break; - case '\003': /* Ctrl-C */ - AZ(raise(SIGINT)); - break; - case '\032': /* Ctrl-Z */ - AZ(raise(SIGTSTP)); - break; - case '\021': /* Ctrl-Q */ - case 'Q': - case 'q': - AC(endwin()); - exit(0); - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - delay = 1U << (ch - '0'); - break; - default: - AC(beep()); - break; - } - } -} - -static void do_xml(const struct varnish_stats *VSL_stats, const char* fields) { char time_stamp[20]; @@ -388,6 +254,9 @@ } } + if (VSL_Open(vd)) + exit(1); + if ((VSL_stats = VSL_OpenStats(vd)) == NULL) exit(1); Added: trunk/varnish-cache/bin/varnishstat/varnishstat.h =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.h (rev 0) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.h 2010-05-21 10:53:34 UTC (rev 4849) @@ -0,0 +1,31 @@ +/*- + * Copyright (c) 2010 Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +int show_field(const char* field, const char *fields); +void do_curses(struct VSL_data *vd, const struct varnish_stats *VSL_stats, int delay, const char *fields); Copied: trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c (from rev 4847, trunk/varnish-cache/bin/varnishstat/varnishstat.c) =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c (rev 0) +++ trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c 2010-05-21 10:53:34 UTC (rev 4849) @@ -0,0 +1,184 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2010 Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Dag-Erling Sm?rgrav + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Statistics output program + */ + +#include "config.h" + +#include "svnid.h" +SVNID("$Id$") + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libvarnish.h" +#include "shmlog.h" +#include "varnishapi.h" +#include "varnishstat.h" + +#define AC(x) assert((x) != ERR) + +static void +myexp(double *acc, double val, unsigned *n, unsigned nmax) +{ + + if (*n < nmax) + (*n)++; + (*acc) += (val - *acc) / (double)*n; +} + +void +do_curses(struct VSL_data *vd, const struct varnish_stats *VSL_stats, int delay, const char *fields) +{ + struct varnish_stats copy; + struct varnish_stats seen; + intmax_t ju; + struct timeval tv; + double tt, lt, hit, miss, ratio, up; + double a1, a2, a3; + unsigned n1, n2, n3; + time_t rt; + int ch, line; + + memset(©, 0, sizeof copy); + memset(&seen, 0, sizeof seen); + + a1 = a2 = a3 = 0.0; + n1 = n2 = n3 = 0; + + (void)initscr(); + AC(raw()); + AC(noecho()); + AC(nonl()); + AC(intrflush(stdscr, FALSE)); + AC(curs_set(0)); + AC(erase()); + + lt = 0; + while (1) { + AZ(gettimeofday(&tv, NULL)); + tt = tv.tv_usec * 1e-6 + tv.tv_sec; + lt = tt - lt; + + rt = VSL_stats->uptime; + up = rt; + + AC(mvprintw(0, 0, "%*s", COLS - 1, VSL_Name(vd))); + AC(mvprintw(0, 0, "%d+%02d:%02d:%02d", rt / 86400, + (rt % 86400) / 3600, (rt % 3600) / 60, rt % 60)); + + hit = VSL_stats->cache_hit - copy.cache_hit; + miss = VSL_stats->cache_miss - copy.cache_miss; + hit /= lt; + miss /= lt; + if (hit + miss != 0) { + ratio = hit / (hit + miss); + myexp(&a1, ratio, &n1, 10); + myexp(&a2, ratio, &n2, 100); + myexp(&a3, ratio, &n3, 1000); + } + AC(mvprintw(1, 0, "Hitrate ratio: %8u %8u %8u", n1, n2, n3)); + AC(mvprintw(2, 0, "Hitrate avg: %8.4f %8.4f %8.4f", a1, a2, a3)); + + line = 3; +#define MAC_STAT(n, t, l, ff, d) \ + if ((fields == NULL || show_field( #n, fields )) && line < LINES) { \ + ju = VSL_stats->n; \ + if (ju == 0 && !seen.n) { \ + } else if (ff == 'a') { \ + seen.n = 1; \ + line++; \ + AC(mvprintw(line, 0, \ + "%12ju %12.2f %12.2f %s\n", \ + ju, (ju - (intmax_t)copy.n)/lt, \ + ju / up, d)); \ + copy.n = ju; \ + } else { \ + seen.n = 1; \ + line++; \ + AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", \ + ju, ". ", ". ", d)); \ + } \ + } +#include "stat_field.h" +#undef MAC_STAT + lt = tt; + AC(refresh()); + timeout(delay * 1000); + switch ((ch = getch())) { + case ERR: + break; +#ifdef KEY_RESIZE + case KEY_RESIZE: + AC(erase()); + break; +#endif + case '\014': /* Ctrl-L */ + case '\024': /* Ctrl-T */ + AC(redrawwin(stdscr)); + AC(refresh()); + break; + case '\003': /* Ctrl-C */ + AZ(raise(SIGINT)); + break; + case '\032': /* Ctrl-Z */ + AZ(raise(SIGTSTP)); + break; + case '\021': /* Ctrl-Q */ + case 'Q': + case 'q': + AC(endwin()); + exit(0); + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + delay = 1U << (ch - '0'); + break; + default: + AC(beep()); + break; + } + } +} From phk at varnish-cache.org Fri May 21 11:11:13 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 13:11:13 +0200 Subject: r4850 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-21 13:11:13 +0200 (Fri, 21 May 2010) New Revision: 4850 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c Log: Move free indication to class field. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-21 10:53:34 UTC (rev 4849) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-21 11:11:13 UTC (rev 4850) @@ -80,7 +80,7 @@ while (1) { CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); - if (strcmp(sha->type, "Free")) { + if (strcmp(sha->class, "Free")) { sha = SHA_NEXT(sha); continue; } @@ -95,7 +95,7 @@ memset(sha2, 0, sizeof *sha2); sha2->magic = SHMALLOC_MAGIC; sha2->len = sha->len - size; - bprintf(sha2->type, "%s", "Free"); + bprintf(sha2->class, "%s", "Free"); MEMORY_BARRIER(); sha->len = size; @@ -298,7 +298,7 @@ loghead->head.magic = SHMALLOC_MAGIC; loghead->head.len = (uint8_t*)(loghead) + size - (uint8_t*)&loghead->head; - bprintf(loghead->head.type, "%s", "Free"); + bprintf(loghead->head.class, "%s", "Free"); MEMORY_BARRIER(); VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_CLASS_STAT, "", ""); From phk at varnish-cache.org Fri May 21 11:11:50 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 13:11:50 +0200 Subject: r4851 - in trunk/varnish-cache: include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-21 13:11:50 +0200 (Fri, 21 May 2010) New Revision: 4851 Modified: trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/vsl.c Log: Make iterator deal correctly with end of list. Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2010-05-21 11:11:13 UTC (rev 4850) +++ trunk/varnish-cache/include/varnishapi.h 2010-05-21 11:11:50 UTC (rev 4851) @@ -66,7 +66,7 @@ const char *ident, unsigned *lenp); struct shmalloc *vsl_iter0(struct VSL_data *vd); -struct shmalloc *vsl_itern(struct VSL_data *vd, struct shmalloc **pp); +void vsl_itern(struct VSL_data *vd, struct shmalloc **pp); #define VSL_FOREACH(var, vd) \ for((var) = vsl_iter0((vd)); (var) != NULL; vsl_itern((vd), &(var))) Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-21 11:11:13 UTC (rev 4850) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-21 11:11:50 UTC (rev 4851) @@ -177,19 +177,22 @@ return (&vd->vsl_lh->head); } -struct shmalloc * +void vsl_itern(struct VSL_data *vd, struct shmalloc **pp) { CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); - if (vd->alloc_seq != vd->vsl_lh->alloc_seq) - return(NULL); + if (vd->alloc_seq != vd->vsl_lh->alloc_seq) { + *pp = NULL; + return; + } CHECK_OBJ_NOTNULL(*pp, SHMALLOC_MAGIC); *pp = SHA_NEXT(*pp); - if ((void*)*pp >= vd->vsl_end) - return (NULL); + if ((void*)(*pp) >= vd->vsl_end) { + *pp = NULL; + return; + } CHECK_OBJ_NOTNULL(*pp, SHMALLOC_MAGIC); - return (*pp); } /*--------------------------------------------------------------------*/ From perbu at varnish-cache.org Fri May 21 11:14:18 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Fri, 21 May 2010 13:14:18 +0200 Subject: r4852 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-21 13:14:18 +0200 (Fri, 21 May 2010) New Revision: 4852 Added: trunk/varnish-cache/doc/sphinx/tutorial/handling_misbehaving_servers.rst Modified: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/index.rst trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst Log: probes, grace, saint Modified: trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-21 11:11:50 UTC (rev 4851) +++ trunk/varnish-cache/doc/sphinx/tutorial/advanced_backend_servers.rst 2010-05-21 11:14:18 UTC (rev 4852) @@ -78,6 +78,8 @@ requests to the healthy server? Sure it can. This is where the Health Checks come into play. +.. _tutorial-advanced_backend_servers-health: + Health checks ------------- @@ -124,8 +126,6 @@ threshold How many of the .window last polls must be good for the backend to be declared healthy. -XXX: Ref to reference guide. - Now we define the director.:: director example_director round-robin { @@ -141,4 +141,6 @@ You use this director just as you would use any other director or backend. Varnish will not send traffic to hosts that are marked as -unhealty. +unhealty. Varnish can also serve stale content if all the backends are +down. See :ref:`tutorial-handling_misbehaving_servers` for more +information on how to enable this. Added: trunk/varnish-cache/doc/sphinx/tutorial/handling_misbehaving_servers.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/handling_misbehaving_servers.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/tutorial/handling_misbehaving_servers.rst 2010-05-21 11:14:18 UTC (rev 4852) @@ -0,0 +1,76 @@ +.. _tutorial-handling_misbehaving_servers: + +Misbehaving servers +------------------- + +A key feature of Varnish is its ability to shield you from misbehaving +web- and application servers. + + + +Grace mode +~~~~~~~~~~ + +When several clients are requesting the same page Varnish will send +one request to the backend and place the others on hold while fetching +one copy from the back end. + +If you are serving thousands of hits per second this queue can get +huge. Nobody likes to wait so there is a possibility to serve stale +content to waiting users. In order to do this we must instruct Varnish +to keep the objects in cache beyond their TTL. So, to keep all objects +for 30 minutes beyond their TTL use the following VCL::: + + sub vcl_fetch { + set beresp.grace = 30m; + } + +Varnish still won't serve the stale objects. In order to enable +Varnish to actually serve the stale object we must enable this on the +request. Lets us say that we accept serving 15s old object.:: + + sub vcl_recv { + set req.grace = 15s; + } + +You might wonder why we should keep the objects in the cache for 30 +minutes if we are unable to serve them? Well, if you have enabled +:ref:`tutorial-advanced_backend_servers-health` you can check if the +backend is healthy and serve the content for as longer.:: + + if (! req.backend.healthy) { + set req.grace = 5m; + } else { + set req.grace = 15s; + } + +Saint mode +~~~~~~~~~~ + +Sometimes servers get flaky. They start throwing out random +errors. You can instruct Varnish to try to handle this in a +more-than-graceful way - enter *Saint mode*. Saint mode enables you to +discard a certain page from one backend server and either try another +server or serve stale content from cache. Lets have a look at how this +can be enabled in VCL::: + + sub vcl_fetch { + if (beresp.status == 500) { + set beresp.saintmode = 10s; + restart; + } + set beresp.grace = 5m; + } + +When we set beresp.saintmode to 10 seconds Varnish will not ask *that* +server for URL for 10 seconds. A blacklist, more or less. Also a +restart is performed so if you have other backends capable of serving +that content Varnish will try those. When you are out of backends +Varnish will serve the content from its stale cache. + +This can really be a life saver. + +God mode +~~~~~~~~ +Not implemented yet. :-) + Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-21 11:11:50 UTC (rev 4851) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-21 11:14:18 UTC (rev 4852) @@ -25,9 +25,10 @@ there are quite clueful. So its interesting to look at their HTTP Headers. Lets send a GET requst for their home page.:: - $ GET -H 'Host: vg.no' -Used http://vg.no/ - GET http://www.vg.no/ - User-Agent: Rickzilla 1.0 + $ GET -H 'Host: www.vg.no' -Used http://vg.no/ + GET http://vg.no/ + Host: www.vg.no + User-Agent: lwp-request/5.834 libwww-perl/5.834 200 OK Cache-Control: must-revalidate @@ -59,10 +60,10 @@ Firefox plugins ~~~~~~~~~~~~~~~ -There are also a couple of great plugins for Firefox. Both *Live HTTP -Headers* and *Firebug* can show you what headers are beeing sent and -recieved. Try googling for them if you want to check them out. They -are both recommended. +There is also a plugin for Firefox. *Live HTTP Headers* can show you +what headers are beeing sent and recieved. Live HTTP Headers can be +found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by +googling "Live HTTP Headers". The role of HTTP Headers @@ -124,15 +125,19 @@ like removing one out of several cookies, things get difficult. Unfornunatly Varnish doesn't have good tools for manipulating the Cookies. We have to use regular expressions to do the -work. +work. If you are familiar with regular expressions you'll understand +whats going on. If you don't I suggest you either pick up a book on +the subject, read through the *pcrepattern* man page or read through +one of many online guides. Let me show you what Varnish Software uses. We use some cookies for Google Analytics tracking and similar tools. The cookies are all set and used by Javascript. Varnish and Drupal doesn't need to see those cookies and since Varnish will cease caching of pages when the client -sends cookies we will discard these unnecessary cookies in VCL. +sends cookies we will discard these unnecessary cookies in VCL. -In the following VCL we discard all cookies that start with a underscore.:: +In the following VCL we discard all cookies that start with a +underscore.:: // Remove has_js and Google Analytics __* cookies. set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", ""); Modified: trunk/varnish-cache/doc/sphinx/tutorial/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-21 11:11:50 UTC (rev 4851) +++ trunk/varnish-cache/doc/sphinx/tutorial/index.rst 2010-05-21 11:14:18 UTC (rev 4852) @@ -30,6 +30,7 @@ statistics.rst increasing_your_hitrate.rst advanced_backend_servers.rst + handling_misbehaving_servers.rst troubleshooting.rst .. todo:: Modified: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-21 11:11:50 UTC (rev 4851) +++ trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-21 11:14:18 UTC (rev 4852) @@ -56,3 +56,6 @@ trigger ESI processing, try alternate backend servers in case the request failed. + +Example one +~~~~~~~~~~~ From perbu at varnish-cache.org Fri May 21 11:43:45 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Fri, 21 May 2010 13:43:45 +0200 Subject: r4853 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-21 13:43:45 +0200 (Fri, 21 May 2010) New Revision: 4853 Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst Log: tools. troubleshooting Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-21 11:14:18 UTC (rev 4852) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-21 11:43:45 UTC (rev 4853) @@ -14,8 +14,8 @@ varnishlog, but sometimes a separate tool makes sense. Here are the ones I use. -lwp-request -~~~~~~~~~~~ +Tool: lwp-request +~~~~~~~~~~~~~~~~~ lwp-request is part of The World-Wide Web library for Perl. It's couple of really basic programs that can execute a HTTP request and @@ -57,8 +57,8 @@ GET -Used http://example.com/ |grep ^Set-Cookie -Firefox plugins -~~~~~~~~~~~~~~~ +Tool: Live HTTP Headers +~~~~~~~~~~~~~~~~~~~~~~~ There is also a plugin for Firefox. *Live HTTP Headers* can show you what headers are beeing sent and recieved. Live HTTP Headers can be Modified: trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-21 11:14:18 UTC (rev 4852) +++ trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-21 11:43:45 UTC (rev 4853) @@ -1,7 +1,12 @@ Troubleshooting Varnish ----------------------- +Sometimes Varnish misbehaves. In order for you to understand whats +going on there are a couple of places you can check. varnishlog, +/var/log/syslog, /var/log/messages are all places where varnish might +leave clues of whats going on. + When Varnish won't start ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -51,6 +56,12 @@ When varnish goes bust. +Varnish gives me Guru meditation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +First find the relevant log entries in varnishlog. That will probably +give you a clue. + Varnish doesn't cache ~~~~~~~~~~~~~~~~~~~~~ From phk at varnish-cache.org Fri May 21 11:48:16 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 13:48:16 +0200 Subject: r4854 - trunk/varnish-cache/bin/varnishstat Message-ID: Author: phk Date: 2010-05-21 13:48:16 +0200 (Fri, 21 May 2010) New Revision: 4854 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c Log: Build a list of things we might display up front, and then iterate over that list, rather than enumerate all stats for each iteration. Modified: trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c 2010-05-21 11:43:45 UTC (rev 4853) +++ trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c 2010-05-21 11:48:16 UTC (rev 4854) @@ -48,12 +48,103 @@ #include "libvarnish.h" #include "shmlog.h" +#include "vqueue.h" #include "varnishapi.h" #include "varnishstat.h" +#include "miniobj.h" #define AC(x) assert((x) != ERR) +struct pt { + VTAILQ_ENTRY(pt) next; + const volatile uint64_t *ptr; + uint64_t ref; + char type; + char seen; + const char *name; +}; + +static VTAILQ_HEAD(, pt) pthead = VTAILQ_HEAD_INITIALIZER(pthead); + +static struct pt * +add_pt(const uint64_t *ptr, int type, const char *c, const char *t, const char *i) +{ + struct pt *pt; + char buf[128]; + + pt = calloc(sizeof *pt, 1); + AN(pt); + VTAILQ_INSERT_TAIL(&pthead, pt, next); + + pt->ptr = ptr; + pt->ref = *ptr; + pt->type = type; + + *buf = '\0'; + if (c != NULL) { + strcat(buf, c); + strcat(buf, "."); + } + if (t != NULL) { + strcat(buf, t); + strcat(buf, "."); + } + if (i != NULL) { + strcat(buf, i); + strcat(buf, "."); + } + pt->name = strdup(buf); + AN(pt->name); + return (pt); +} + static void +main_stat(void *ptr, const char *fields) +{ + struct varnish_stats *st = ptr; + +#define MAC_STAT(nn, tt, ll, ff, dd) \ + if (fields == NULL || show_field( #nn, fields )) \ + (void)add_pt(&st->nn, ff, NULL, NULL, dd); +#include "stat_field.h" +#undef MAC_STAT +} + +#if 0 +static void +sma_stat(struct shmalloc *sha, const char *fields) +{ + struct varnish_stats_sma *st = SHA_PTR(sha); + +#define MAC_STAT_SMA(nn, tt, ll, ff, dd) \ + if (fields == NULL || show_field( #nn, fields )) \ + (void)add_pt(&st->nn, ff, "SMA", sha->ident, dd); +#include "stat_field.h" +#undef MAC_STAT_SMA +} +#endif + +static void +prep_pts(struct VSL_data *vd, const char *fields) +{ + struct shmalloc *sha; + + VSL_FOREACH(sha, vd) { + CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); + if (strcmp(sha->class, VSL_CLASS_STAT)) + continue; + if (!strcmp(sha->type, "")) + main_stat(SHA_PTR(sha), fields); +#if 0 + else if (!strcmp(sha->type, VSL_TYPE_STAT_SMA)) + sma_stat(sha, fields); +#endif + else + fprintf(stderr, "Unknwon Statistics"); + } +} + +static void myexp(double *acc, double val, unsigned *n, unsigned nmax) { @@ -66,7 +157,6 @@ do_curses(struct VSL_data *vd, const struct varnish_stats *VSL_stats, int delay, const char *fields) { struct varnish_stats copy; - struct varnish_stats seen; intmax_t ju; struct timeval tv; double tt, lt, hit, miss, ratio, up; @@ -74,9 +164,10 @@ unsigned n1, n2, n3; time_t rt; int ch, line; + struct pt *pt; + prep_pts(vd, fields); memset(©, 0, sizeof copy); - memset(&seen, 0, sizeof seen); a1 = a2 = a3 = 0.0; n1 = n2 = n3 = 0; @@ -116,27 +207,25 @@ AC(mvprintw(2, 0, "Hitrate avg: %8.4f %8.4f %8.4f", a1, a2, a3)); line = 3; -#define MAC_STAT(n, t, l, ff, d) \ - if ((fields == NULL || show_field( #n, fields )) && line < LINES) { \ - ju = VSL_stats->n; \ - if (ju == 0 && !seen.n) { \ - } else if (ff == 'a') { \ - seen.n = 1; \ - line++; \ - AC(mvprintw(line, 0, \ - "%12ju %12.2f %12.2f %s\n", \ - ju, (ju - (intmax_t)copy.n)/lt, \ - ju / up, d)); \ - copy.n = ju; \ - } else { \ - seen.n = 1; \ - line++; \ - AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", \ - ju, ". ", ". ", d)); \ - } \ - } -#include "stat_field.h" -#undef MAC_STAT + VTAILQ_FOREACH(pt, &pthead, next) { + if (line >= LINES) + break; + ju = *pt->ptr; + if (ju == 0 && !pt->seen) + continue; + pt->seen = 1; + line++; + if (pt->type == 'a') { + AC(mvprintw(line, 0, + "%12ju %12.2f %12.2f %s\n", + ju, (ju - (intmax_t)pt->ref)/lt, + ju / up, pt->name)); + pt->ref = ju; + } else { + AC(mvprintw(line, 0, "%12ju %12s %12s %s\n", + ju, ". ", ". ", pt->name)); + } + } lt = tt; AC(refresh()); timeout(delay * 1000); From phk at varnish-cache.org Fri May 21 12:01:16 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 21 May 2010 14:01:16 +0200 Subject: r4855 - in trunk/varnish-cache: bin/varnishd bin/varnishstat include lib/libvarnishapi Message-ID: Author: phk Date: 2010-05-21 14:01:16 +0200 (Fri, 21 May 2010) New Revision: 4855 Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c trunk/varnish-cache/include/stat_field.h trunk/varnish-cache/include/stats.h trunk/varnish-cache/lib/libvarnishapi/vsl.c Log: Start segmenting stats, so that we can have per-whatever stats, starting with -smalloc stats: Split SMA relevant fields into their own MAC_STAT_SMA macros, and make it possible to include stat_field.h without all MAC_STAT* macrods defined. Move SMA relevant fields from struct varnish_stats to their own varnish_stats_sma. Teach varnishstat to enumerate fields from a varnish_stats_sma. (This code looks like it should/could be much more general, and probably libvarnisapi candidate). For now only curses mode updated. In storage_malloc.c, allocate a shm_alloc for a struct varnish_stats_sma for each instance (in ->ready(), because we have no shm during arg processing in ->init(), update the counters therein. Modified: trunk/varnish-cache/bin/varnishd/mgt_shmem.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-21 11:48:16 UTC (rev 4854) +++ trunk/varnish-cache/bin/varnishd/mgt_shmem.c 2010-05-21 12:01:16 UTC (rev 4855) @@ -301,7 +301,8 @@ bprintf(loghead->head.class, "%s", "Free"); MEMORY_BARRIER(); - VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, VSL_CLASS_STAT, "", ""); + VSL_stats = mgt_SHM_Alloc(sizeof *VSL_stats, + VSL_CLASS_STAT, VSL_TYPE_STAT, ""); AN(VSL_stats); pp = mgt_SHM_Alloc(sizeof *pp, "Params", "", ""); Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-21 11:48:16 UTC (rev 4854) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2010-05-21 12:01:16 UTC (rev 4855) @@ -48,6 +48,7 @@ #define SMA_SC_MAGIC 0x1ac8a345 struct lock sma_mtx; size_t sma_max; + struct varnish_stats_sma *stats; }; struct sma { @@ -67,13 +68,13 @@ CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); (void)oc; Lck_Lock(&sma_sc->sma_mtx); - VSL_stats->sma_nreq++; - if (VSL_stats->sma_nbytes + size > sma_sc->sma_max) + sma_sc->stats->sma_nreq++; + if (sma_sc->stats->sma_nbytes + size > sma_sc->sma_max) size = 0; else { - VSL_stats->sma_nobj++; - VSL_stats->sma_nbytes += size; - VSL_stats->sma_balloc += size; + sma_sc->stats->sma_nobj++; + sma_sc->stats->sma_nbytes += size; + sma_sc->stats->sma_balloc += size; } Lck_Unlock(&sma_sc->sma_mtx); @@ -117,9 +118,9 @@ sma_sc = sma->sc; assert(sma->sz == sma->s.space); Lck_Lock(&sma_sc->sma_mtx); - VSL_stats->sma_nobj--; - VSL_stats->sma_nbytes -= sma->sz; - VSL_stats->sma_bfree += sma->sz; + sma_sc->stats->sma_nobj--; + sma_sc->stats->sma_nbytes -= sma->sz; + sma_sc->stats->sma_bfree += sma->sz; Lck_Unlock(&sma_sc->sma_mtx); free(sma->s.ptr); free(sma); @@ -140,8 +141,8 @@ assert(size < sma->sz); if ((p = realloc(sma->s.ptr, size)) != NULL) { Lck_Lock(&sma_sc->sma_mtx); - VSL_stats->sma_nbytes -= (sma->sz - size); - VSL_stats->sma_bfree += sma->sz - size; + sma_sc->stats->sma_nbytes -= (sma->sz - size); + sma_sc->stats->sma_bfree += sma->sz - size; sma->sz = size; Lck_Unlock(&sma_sc->sma_mtx); sma->s.ptr = p; @@ -156,6 +157,7 @@ uintmax_t u; struct sma_sc *sc; + ASSERT_MGT(); ALLOC_OBJ(sc, SMA_SC_MAGIC); AN(sc); sc->sma_max = SIZE_MAX; @@ -181,6 +183,17 @@ } static void +sma_ready(struct stevedore *st) +{ + struct sma_sc *sma_sc; + + CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC); + sma_sc->stats = mgt_SHM_Alloc(sizeof *sma_sc->stats, + VSL_CLASS_STAT, VSL_TYPE_STAT_SMA, st->ident); + memset(sma_sc->stats, 0, sizeof *sma_sc->stats); +} + +static void sma_open(const struct stevedore *st) { struct sma_sc *sma_sc; @@ -193,6 +206,7 @@ .magic = STEVEDORE_MAGIC, .name = "malloc", .init = sma_init, + .ready = sma_ready, .open = sma_open, .alloc = sma_alloc, .free = sma_free, Modified: trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c 2010-05-21 11:48:16 UTC (rev 4854) +++ trunk/varnish-cache/bin/varnishstat/varnishstat_curses.c 2010-05-21 12:01:16 UTC (rev 4855) @@ -110,7 +110,6 @@ #undef MAC_STAT } -#if 0 static void sma_stat(struct shmalloc *sha, const char *fields) { @@ -122,7 +121,6 @@ #include "stat_field.h" #undef MAC_STAT_SMA } -#endif static void prep_pts(struct VSL_data *vd, const char *fields) @@ -133,12 +131,10 @@ CHECK_OBJ_NOTNULL(sha, SHMALLOC_MAGIC); if (strcmp(sha->class, VSL_CLASS_STAT)) continue; - if (!strcmp(sha->type, "")) + if (!strcmp(sha->type, VSL_TYPE_STAT)) main_stat(SHA_PTR(sha), fields); -#if 0 else if (!strcmp(sha->type, VSL_TYPE_STAT_SMA)) sma_stat(sha, fields); -#endif else fprintf(stderr, "Unknwon Statistics"); } Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2010-05-21 11:48:16 UTC (rev 4854) +++ trunk/varnish-cache/include/stat_field.h 2010-05-21 12:01:16 UTC (rev 4855) @@ -32,6 +32,12 @@ * stats structure. */ +#ifndef MAC_STAT +#define MAC_STAT(a, b, c, d, e) +#define __MAC_STAT +#endif + + MAC_STAT(client_conn, uint64_t, 0, 'a', "Client connections accepted") MAC_STAT(client_drop, uint64_t, 0, 'a', "Connection dropped, no sess/wrk") MAC_STAT(client_req, uint64_t, 1, 'a', "Client requests received") @@ -118,12 +124,6 @@ MAC_STAT(sm_balloc, uint64_t, 0, 'i', "bytes allocated") MAC_STAT(sm_bfree, uint64_t, 0, 'i', "bytes free") -MAC_STAT(sma_nreq, uint64_t, 0, 'a', "SMA allocator requests") -MAC_STAT(sma_nobj, uint64_t, 0, 'i', "SMA outstanding allocations") -MAC_STAT(sma_nbytes, uint64_t, 0, 'i', "SMA outstanding bytes") -MAC_STAT(sma_balloc, uint64_t, 0, 'i', "SMA bytes allocated") -MAC_STAT(sma_bfree, uint64_t, 0, 'i', "SMA bytes free") - MAC_STAT(sms_nreq, uint64_t, 0, 'a', "SMS allocator requests") MAC_STAT(sms_nobj, uint64_t, 0, 'i', "SMS outstanding allocations") MAC_STAT(sms_nbytes, uint64_t, 0, 'i', "SMS outstanding bytes") @@ -152,3 +152,24 @@ MAC_STAT(accept_fail, uint64_t, 0, 'a', "Accept failures") MAC_STAT(client_drop_late, uint64_t, 0, 'a', "Connection dropped late") MAC_STAT(uptime, uint64_t, 0, 'a', "Client uptime") + +#ifdef __MAC_STAT +#undef MAC_STAT +#undef __MAC_STAT +#endif + +#ifndef MAC_STAT_SMA +#define MAC_STAT_SMA(a, b, c, d, e) +#define __MAC_STAT_SMA +#endif + +MAC_STAT_SMA(sma_nreq, uint64_t, 0, 'a', "Allocator requests") +MAC_STAT_SMA(sma_nobj, uint64_t, 0, 'i', "Outstanding allocations") +MAC_STAT_SMA(sma_nbytes, uint64_t, 0, 'i', "Outstanding bytes") +MAC_STAT_SMA(sma_balloc, uint64_t, 0, 'i', "Bytes allocated") +MAC_STAT_SMA(sma_bfree, uint64_t, 0, 'i', "Bytes free") + +#ifdef __MAC_STAT_SMA +#undef MAC_STAT_SMA +#undef __MAC_STAT_SMA +#endif Modified: trunk/varnish-cache/include/stats.h =================================================================== --- trunk/varnish-cache/include/stats.h 2010-05-21 11:48:16 UTC (rev 4854) +++ trunk/varnish-cache/include/stats.h 2010-05-21 12:01:16 UTC (rev 4855) @@ -31,10 +31,18 @@ #include -#define VSL_TYPE_STAT "Stats" +#define VSL_TYPE_STAT "" struct varnish_stats { #define MAC_STAT(n, t, l, f, e) t n; #include "stat_field.h" #undef MAC_STAT }; + +#define VSL_TYPE_STAT_SMA "SMA" + +struct varnish_stats_sma { +#define MAC_STAT_SMA(n, t, l, f, e) t n; +#include "stat_field.h" +#undef MAC_STAT_SMA +}; Modified: trunk/varnish-cache/lib/libvarnishapi/vsl.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-21 11:48:16 UTC (rev 4854) +++ trunk/varnish-cache/lib/libvarnishapi/vsl.c 2010-05-21 12:01:16 UTC (rev 4855) @@ -261,7 +261,7 @@ CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (VSL_Open(vd)) return (-1); - sha = vsl_find_alloc(vd, VSL_CLASS_LOG, "", ""); + sha = vsl_find_alloc(vd, VSL_CLASS_LOG, VSL_TYPE_STAT_SMA, ""); assert(sha != NULL); vd->log_start = SHA_PTR(sha); From perbu at varnish-cache.org Tue May 25 06:59:11 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Tue, 25 May 2010 08:59:11 +0200 Subject: r4856 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-25 08:59:10 +0200 (Tue, 25 May 2010) New Revision: 4856 Modified: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst Log: VCL Basics Modified: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-21 12:01:16 UTC (rev 4855) +++ trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-25 06:59:10 UTC (rev 4856) @@ -31,8 +31,13 @@ Varnish will execute these subroutines of code at different stages of its work. Since its code it's execute line by line and precedence -isn't a problem. +isn't a problem. At some point you call an action in this subroutine +and then the execution of the subroutine stops. +If you don't call an action in your subroutine and it reaches the end +Varnish will execute some built in code as well. We discuss this in +XXX: Appendix A - the builtin VCL. + 99% of all the changes you'll need to do will be done in two of these subroutines. @@ -47,7 +52,6 @@ In vcl_recv you can also alter the request, dropping cookies, rewrite headers. - vcl_fetch ~~~~~~~~~ @@ -56,6 +60,69 @@ trigger ESI processing, try alternate backend servers in case the request failed. +actions +~~~~~~~ -Example one -~~~~~~~~~~~ +The most common actions to call are these: + +*pass* + When you call pass the request and subsequent response will be passed + to and from the backend server. It won't be cached. pass can be called + in both vcl_recv and vcl_fetch. + +*lookup* + When you call lookup from vcl_recv you tell Varnish to deliver content + from cache even if the request othervise indicates that the request + should be passed. You can't call lookup from vcl_fetch. + +*pipe* + + +*deliver* + +*esi* + ESI-process the fetched document. + +Example 1 - manipulating headers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Lets say we want to remove the cookie for all objects in the /static +directory of our web server::: + + sub vcl_recv { + if (req.url ~ "^/images") { + unset req.http.cookie; + } + } + +Now, when the request is handled to the backend server there will be +no cookie header. The interesting line is the one with the +if-statement. It probably needs a bit of explaining. Varnish has a few +objects that are available throughout the VCL. The important ones are: + +*req* + The request object. Each HTTP transaction contains a request and a + response. When Varnish has recieved the request the req object is + created and populated. Most of the work you do in vcl_fetch you + do on or with the req object. + +*beresp* + The backend respons object. It contains the headers of the object + comming from the backend. Most of the work you do in vcl_fetch you + do on the beresp object. + +Example 2 - manipulating beresp +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here we override the TTL of a object comming from the backend if it +matches certain criteria::: + + sub vcl_fetch { + if (beresp.url ~ "\.(png|gif|jpg)$") { + unset beresp.http.set-cookie; + beresp.ttl = 3600; + } + } + +Example 3 - ACLs +~~~~~~~~~~~~~~~~ From phk at varnish-cache.org Tue May 25 10:47:21 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Tue, 25 May 2010 12:47:21 +0200 Subject: r4857 - in trunk/varnish-cache: bin/varnishd include Message-ID: Author: phk Date: 2010-05-25 12:47:20 +0200 (Tue, 25 May 2010) New Revision: 4857 Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c trunk/varnish-cache/include/stat_field.h Log: Add a counter so we know how many objhdr's are on the cooling list. Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-25 06:59:10 UTC (rev 4856) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-25 10:47:20 UTC (rev 4857) @@ -367,6 +367,7 @@ continue; if (++oh->digest[0] > params->critbit_cooloff) { VTAILQ_REMOVE(&laylow, oh, coollist); + VSL_stats->critbit_cooler--; break; } } @@ -415,6 +416,7 @@ assert(VTAILQ_EMPTY(&oh->waitinglist)); oh->digest[0] = 0; VTAILQ_INSERT_TAIL(&laylow, oh, coollist); + VSL_stats->critbit_cooler++; Lck_Unlock(&hcb_mtx); } Lck_Unlock(&oh->mtx); @@ -428,7 +430,7 @@ hcb_lookup(const struct sess *sp, struct objhead *noh) { struct objhead *oh; - volatile unsigned u; + unsigned u; unsigned with_lock; (void)sp; @@ -439,11 +441,11 @@ Lck_Lock(&hcb_mtx); VSL_stats->hcb_lock++; assert(noh->refcnt == 1); - oh = hcb_insert(&hcb_root, noh, 1); + oh = hcb_insert(&hcb_root, noh, 1); Lck_Unlock(&hcb_mtx); } else { VSL_stats->hcb_nolock++; - oh = hcb_insert(&hcb_root, noh, 0); + oh = hcb_insert(&hcb_root, noh, 0); } if (oh != NULL && oh == noh) { @@ -468,8 +470,12 @@ * under us, so fall through and try with the lock held. */ u = oh->refcnt; - if (u > 0) + if (u > 0) { oh->refcnt++; + } else { + assert(!with_lock); + with_lock = 1; + } Lck_Unlock(&oh->mtx); if (u > 0) return (oh); Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2010-05-25 06:59:10 UTC (rev 4856) +++ trunk/varnish-cache/include/stat_field.h 2010-05-25 10:47:20 UTC (rev 4857) @@ -153,6 +153,8 @@ MAC_STAT(client_drop_late, uint64_t, 0, 'a', "Connection dropped late") MAC_STAT(uptime, uint64_t, 0, 'a', "Client uptime") +MAC_STAT(critbit_cooler, uint64_t, 0, 'i', "Objhdr's on cool list") + #ifdef __MAC_STAT #undef MAC_STAT #undef __MAC_STAT From perbu at varnish-cache.org Wed May 26 08:21:58 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Wed, 26 May 2010 10:21:58 +0200 Subject: r4858 - in trunk/varnish-cache/doc/sphinx: installation tutorial Message-ID: Author: perbu Date: 2010-05-26 10:21:57 +0200 (Wed, 26 May 2010) New Revision: 4858 Modified: trunk/varnish-cache/doc/sphinx/installation/install.rst trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst Log: rewrote VCL + removed all warnings Modified: trunk/varnish-cache/doc/sphinx/installation/install.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-25 10:47:20 UTC (rev 4857) +++ trunk/varnish-cache/doc/sphinx/installation/install.rst 2010-05-26 08:21:57 UTC (rev 4858) @@ -43,7 +43,7 @@ You are probably best of compiling your own code. See `Compiling Varnish from source`_. If that worked for you, you can skip the rest of this document -for now, and and start reading the much more interesting :ref:`Tutorial` +for now, and and start reading the much more interesting :ref:`tutorial-index` instead. Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-25 10:47:20 UTC (rev 4857) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-26 08:21:57 UTC (rev 4858) @@ -22,8 +22,8 @@ give you the result. I use two programs, GET and HEAD. vg.no was the first site to use Varnish and the people running Varnish -there are quite clueful. So its interesting to look at their HTTP -Headers. Lets send a GET requst for their home page.:: +there are quite cluefull. So its interesting to look at their HTTP +Headers. Lets send a GET request for their home page.:: $ GET -H 'Host: www.vg.no' -Used http://vg.no/ GET http://vg.no/ @@ -81,9 +81,8 @@ cares about the *max-age* parameter and uses it to calculate the TTL for an object. -"Cache-Control: nocache" is ignored. See -:ref:`tutorial-increasing_your_hitrate-pragma:` for an example on how -to implement support. +"Cache-Control: nocache" is ignored but if you need this you can +easyli add support for it. So make sure use issue a Cache-Control header with a max-age header. You can have a look at what Varnish Softwares drupal server Modified: trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-25 10:47:20 UTC (rev 4857) +++ trunk/varnish-cache/doc/sphinx/tutorial/troubleshooting.rst 2010-05-26 08:21:57 UTC (rev 4858) @@ -65,5 +65,5 @@ Varnish doesn't cache ~~~~~~~~~~~~~~~~~~~~~ -See :ref:`_tutorial-increasing_your_hitrate:`. +See :ref:`tutorial-increasing_your_hitrate`. Modified: trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-25 10:47:20 UTC (rev 4857) +++ trunk/varnish-cache/doc/sphinx/tutorial/vcl.rst 2010-05-26 08:21:57 UTC (rev 4858) @@ -1,45 +1,28 @@ Varnish Configuration Language - VCL ------------------------------------- -Varnish has a really neat configuration system. Most other systems use -configuration directives, where you basically turn on and off a bunch -of switches. +Varnish has a great configuration system. Most other systems use +configuration directives, where you basically turn on and off lots of +switches. Varnish uses a domain specific language called Varnish +Configuration Language, or VCL for short. Varnish translates this +configuration into binary code which is then executed when requests +arrive. -A very common thing to do in Varnish is to override the cache headers -from our backend. Lets see how this looks in Squid, which has a -standard configuration.:: +The VCL files are divided into subroutines. The different subroutines +are executed at different times. One is executed when we get the +request, another when files are fetched from the backend server. - refresh_pattern ^http://images. 3600 20% 7200 - refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 - refresh_pattern -i (/\.jpg) 1800 10% 3600 override-expire - refresh_pattern . 0 20% 4320 - -If you are familiar with squid that probably made sense to you. But -lets point out a few weaknesses with this model. - -1) It's not intuitive. You can guess what the options mean, and you - can (and should) document it in your configuration file. - -2) Which rules have precedence? Does the last rule to match stick? Or - the first? Or does Squid try to combine all the matching rules. I - actually don't know. - -Now enter Varnish. Varnish takes your configuration file and -translates it to C code, then runs it through a compiler and loads -it. When requests come along varnish just executes the relevant -subroutines of the configuration at the relevant times. - Varnish will execute these subroutines of code at different stages of -its work. Since its code it's execute line by line and precedence +its work. Because it is code it is execute line by line precedence isn't a problem. At some point you call an action in this subroutine -and then the execution of the subroutine stops. +and then the execution of the subroutine stops. If you don't call an action in your subroutine and it reaches the end -Varnish will execute some built in code as well. We discuss this in -XXX: Appendix A - the builtin VCL. +Varnish will execute some built in VCL code. You will see this VCL +code commented out in default.vcl. 99% of all the changes you'll need to do will be done in two of these -subroutines. +subroutines. vcl_recv and vcl_fetch. vcl_recv ~~~~~~~~ From phk at varnish-cache.org Wed May 26 09:27:11 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Wed, 26 May 2010 11:27:11 +0200 Subject: r4859 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-26 11:27:11 +0200 (Wed, 26 May 2010) New Revision: 4859 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_critbit.c Log: Make the critbit "Y" a independently allocated struct, rather than using the objhead as carrier for it. The time difference between objhead freeing and Y freeing is very significant on systems with rolling URLS (ie: "article=%d") resulting in far too many objheads being stuck on the cooling list. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2010-05-26 08:21:57 UTC (rev 4858) +++ trunk/varnish-cache/bin/varnishd/cache.h 2010-05-26 09:27:11 UTC (rev 4859) @@ -206,6 +206,7 @@ #define WORKER_MAGIC 0x6391adcf struct objhead *nobjhead; struct objcore *nobjcore; + void *nhashpriv; struct dstat stats; double lastused; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2010-05-26 08:21:57 UTC (rev 4858) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2010-05-26 09:27:11 UTC (rev 4859) @@ -146,6 +146,10 @@ w->nobjhead = NULL; w->stats.n_objecthead--; } + if (w->nhashpriv != NULL) { + free(w->nhashpriv); + w->nhashpriv = NULL; + } } void Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-26 08:21:57 UTC (rev 4858) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-26 09:27:11 UTC (rev 4859) @@ -46,8 +46,6 @@ static struct lock hcb_mtx; -static VTAILQ_HEAD(,objhead) laylow = VTAILQ_HEAD_INITIALIZER(laylow); - /********************************************************************** * Table for finding out how many bits two bytes have in common, * counting from the MSB towards the LSB. @@ -93,10 +91,13 @@ */ struct hcb_y { + unsigned magic; +#define HCB_Y_MAGIC 0x125c4bd2 unsigned short critbit; unsigned char ptr; unsigned char bitmask; volatile uintptr_t leaf[2]; + VSTAILQ_ENTRY(hcb_y) list; }; #define HCB_BIT_NODE (1<<0) @@ -108,6 +109,9 @@ static struct hcb_root hcb_root; +static VSTAILQ_HEAD(, hcb_y) cool = VSTAILQ_HEAD_INITIALIZER(cool); +static VSTAILQ_HEAD(, hcb_y) dead = VSTAILQ_HEAD_INITIALIZER(dead); + /********************************************************************** * Pointer accessor functions */ @@ -146,6 +150,7 @@ hcb_r_y(struct hcb_y *y) { + CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC); assert(!((uintptr_t)y & (HCB_BIT_NODE|HCB_BIT_Y))); return (HCB_BIT_Y | (uintptr_t)y); } @@ -169,6 +174,7 @@ { unsigned char u, r; + CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC); for (u = 0; u < DIGEST_LEN && oh1->digest[u] == oh2->digest[u]; u++) ; assert(u < DIGEST_LEN); @@ -186,7 +192,7 @@ */ static struct objhead * -hcb_insert(struct hcb_root *root, struct objhead *oh, int has_lock) +hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, int has_lock) { volatile uintptr_t *p; uintptr_t pp; @@ -205,6 +211,7 @@ while(hcb_is_y(pp)) { y = hcb_l_y(pp); + CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC); assert(y->ptr < DIGEST_LEN); s = (oh->digest[y->ptr] & y->bitmask) != 0; assert(s < 2); @@ -231,8 +238,8 @@ /* Insert */ - y2 = (void*)&oh->u; - memset(y2, 0, sizeof *y2); + CAST_OBJ_NOTNULL(y2, wrk->nhashpriv, HCB_Y_MAGIC); + wrk->nhashpriv = NULL; (void)hcb_crit_bit(oh, hcb_l_node(*p), y2); s2 = (oh->digest[y2->ptr] & y2->bitmask) != 0; assert(s2 < 2); @@ -244,6 +251,7 @@ while(hcb_is_y(*p)) { y = hcb_l_y(*p); + CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC); assert(y->critbit != y2->critbit); if (y->critbit > y2->critbit) break; @@ -282,8 +290,7 @@ assert(s < 2); if (y->leaf[s] == hcb_r_node(oh)) { *p = y->leaf[1 - s]; - y->leaf[0] = 0; - y->leaf[1] = 0; + VSTAILQ_INSERT_TAIL(&cool, y, list); return; } p = &y->leaf[s]; @@ -321,21 +328,12 @@ static void hcb_dump(struct cli *cli, const char * const *av, void *priv) { - struct objhead *oh, *oh2; - struct hcb_y *y; (void)priv; (void)av; cli_out(cli, "HCB dump:\n"); dumptree(cli, hcb_root.origo, 0); cli_out(cli, "Coollist:\n"); - Lck_Lock(&hcb_mtx); - VTAILQ_FOREACH_SAFE(oh, &laylow, coollist, oh2) { - y = (void *)&oh->u; - cli_out(cli, "%p ref %d, y{%u, %u}\n", oh, - oh->refcnt, y->leaf[0], y->leaf[1]); - } - Lck_Unlock(&hcb_mtx); } static struct cli_proto hcb_cmds[] = { @@ -348,8 +346,7 @@ static void * hcb_cleaner(void *priv) { - struct objhead *oh, *oh2; - struct hcb_y *y; + struct hcb_y *y, *y2; struct worker ww; memset(&ww, 0, sizeof ww); @@ -358,26 +355,15 @@ THR_SetName("hcb_cleaner"); (void)priv; while (1) { - Lck_Lock(&hcb_mtx); - VTAILQ_FOREACH_SAFE(oh, &laylow, coollist, oh2) { - CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - AZ(oh->refcnt); - y = (void *)&oh->u; - if (y->leaf[0] || y->leaf[1]) - continue; - if (++oh->digest[0] > params->critbit_cooloff) { - VTAILQ_REMOVE(&laylow, oh, coollist); - VSL_stats->critbit_cooler--; - break; - } + VSTAILQ_FOREACH_SAFE(y, &dead, list, y2) { + VSTAILQ_REMOVE_HEAD(&dead, list); + FREE_OBJ(y); } + Lck_Lock(&hcb_mtx); + VSTAILQ_CONCAT(&dead, &cool); Lck_Unlock(&hcb_mtx); - if (oh == NULL) { - WRK_SumStat(&ww); - (void)sleep(1); - } else { - HSH_DeleteObjHead(&ww, oh); - } + WRK_SumStat(&ww); + TIM_sleep(params->critbit_cooloff); } NEEDLESS_RETURN(NULL); } @@ -394,7 +380,6 @@ CLI_AddFuncs(hcb_cmds); Lck_New(&hcb_mtx); AZ(pthread_create(&tp, NULL, hcb_cleaner, NULL)); - assert(sizeof(struct hcb_y) <= sizeof(oh->u)); memset(&hcb_root, 0, sizeof hcb_root); hcb_build_bittbl(); } @@ -412,12 +397,10 @@ if (oh->refcnt == 0) { Lck_Lock(&hcb_mtx); hcb_delete(&hcb_root, oh); + Lck_Unlock(&hcb_mtx); assert(VTAILQ_EMPTY(&oh->objcs)); assert(VTAILQ_EMPTY(&oh->waitinglist)); - oh->digest[0] = 0; - VTAILQ_INSERT_TAIL(&laylow, oh, coollist); - VSL_stats->critbit_cooler++; - Lck_Unlock(&hcb_mtx); + r = 0; } Lck_Unlock(&oh->mtx); #ifdef PHK @@ -430,6 +413,7 @@ hcb_lookup(const struct sess *sp, struct objhead *noh) { struct objhead *oh; + struct hcb_y *y; unsigned u; unsigned with_lock; @@ -438,14 +422,19 @@ with_lock = 0; while (1) { if (with_lock) { + if (sp->wrk->nhashpriv == NULL) { + ALLOC_OBJ(y, HCB_Y_MAGIC); + sp->wrk->nhashpriv = y; + } + AN(sp->wrk->nhashpriv); Lck_Lock(&hcb_mtx); VSL_stats->hcb_lock++; assert(noh->refcnt == 1); - oh = hcb_insert(&hcb_root, noh, 1); + oh = hcb_insert(sp->wrk, &hcb_root, noh, 1); Lck_Unlock(&hcb_mtx); } else { VSL_stats->hcb_nolock++; - oh = hcb_insert(&hcb_root, noh, 0); + oh = hcb_insert(sp->wrk, &hcb_root, noh, 0); } if (oh != NULL && oh == noh) { From phk at varnish-cache.org Thu May 27 10:48:01 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Thu, 27 May 2010 12:48:01 +0200 Subject: r4860 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-27 12:48:01 +0200 (Thu, 27 May 2010) New Revision: 4860 Modified: trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/hash_critbit.c Log: Also cool off objheaders Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2010-05-26 09:27:11 UTC (rev 4859) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2010-05-27 10:48:01 UTC (rev 4860) @@ -157,6 +157,7 @@ -emacro((826), VTAILQ_PREV) // Suspicious pointer-to-pointer conversion (area too small) -emacro((826), VTAILQ_LAST) // Suspicious pointer-to-pointer conversion (area too small) -emacro(506, VTAILQ_FOREACH_SAFE) // constant value boolean +-emacro(506, VSTAILQ_FOREACH_SAFE) // constant value boolean -esym(765, vcc_ProcAction) // could be made static -esym(759, vcc_ProcAction) // could be moved to module Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-26 09:27:11 UTC (rev 4859) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-27 10:48:01 UTC (rev 4860) @@ -109,8 +109,10 @@ static struct hcb_root hcb_root; -static VSTAILQ_HEAD(, hcb_y) cool = VSTAILQ_HEAD_INITIALIZER(cool); -static VSTAILQ_HEAD(, hcb_y) dead = VSTAILQ_HEAD_INITIALIZER(dead); +static VSTAILQ_HEAD(, hcb_y) cool_y = VSTAILQ_HEAD_INITIALIZER(cool_y); +static VSTAILQ_HEAD(, hcb_y) dead_y = VSTAILQ_HEAD_INITIALIZER(dead_y); +static VTAILQ_HEAD(, objhead) cool_h = VTAILQ_HEAD_INITIALIZER(cool_h); +static VTAILQ_HEAD(, objhead) dead_h = VTAILQ_HEAD_INITIALIZER(dead_h); /********************************************************************** * Pointer accessor functions @@ -290,7 +292,7 @@ assert(s < 2); if (y->leaf[s] == hcb_r_node(oh)) { *p = y->leaf[1 - s]; - VSTAILQ_INSERT_TAIL(&cool, y, list); + VSTAILQ_INSERT_TAIL(&cool_y, y, list); return; } p = &y->leaf[s]; @@ -348,6 +350,7 @@ { struct hcb_y *y, *y2; struct worker ww; + struct objhead *oh, *oh2; memset(&ww, 0, sizeof ww); ww.magic = WORKER_MAGIC; @@ -355,12 +358,17 @@ THR_SetName("hcb_cleaner"); (void)priv; while (1) { - VSTAILQ_FOREACH_SAFE(y, &dead, list, y2) { - VSTAILQ_REMOVE_HEAD(&dead, list); + VSTAILQ_FOREACH_SAFE(y, &dead_y, list, y2) { + VSTAILQ_REMOVE_HEAD(&dead_y, list); FREE_OBJ(y); } + VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) { + VTAILQ_REMOVE(&dead_h, oh, hoh_list); + HSH_DeleteObjHead(&ww, oh); + } Lck_Lock(&hcb_mtx); - VSTAILQ_CONCAT(&dead, &cool); + VSTAILQ_CONCAT(&dead_y, &cool_y); + VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list); Lck_Unlock(&hcb_mtx); WRK_SumStat(&ww); TIM_sleep(params->critbit_cooloff); @@ -397,10 +405,10 @@ if (oh->refcnt == 0) { Lck_Lock(&hcb_mtx); hcb_delete(&hcb_root, oh); + VTAILQ_INSERT_TAIL(&cool_h, oh, hoh_list); Lck_Unlock(&hcb_mtx); assert(VTAILQ_EMPTY(&oh->objcs)); assert(VTAILQ_EMPTY(&oh->waitinglist)); - r = 0; } Lck_Unlock(&oh->mtx); #ifdef PHK From phk at varnish-cache.org Fri May 28 09:28:15 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 28 May 2010 11:28:15 +0200 Subject: r4861 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-28 11:28:15 +0200 (Fri, 28 May 2010) New Revision: 4861 Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c Log: Add a single write memory barrier, to be absolutely 100% sure that the tree is sane at all points. Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-27 10:48:01 UTC (rev 4860) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2010-05-28 09:28:15 UTC (rev 4861) @@ -43,6 +43,7 @@ #include "cache.h" #include "hash_slinger.h" #include "cli_priv.h" +#include "vmb.h" static struct lock hcb_mtx; @@ -242,7 +243,7 @@ CAST_OBJ_NOTNULL(y2, wrk->nhashpriv, HCB_Y_MAGIC); wrk->nhashpriv = NULL; - (void)hcb_crit_bit(oh, hcb_l_node(*p), y2); + (void)hcb_crit_bit(oh, oh2, y2); s2 = (oh->digest[y2->ptr] & y2->bitmask) != 0; assert(s2 < 2); y2->leaf[s2] = hcb_r_node(oh); @@ -263,6 +264,7 @@ p = &y->leaf[s]; } y2->leaf[s2] = *p; + VWMB(); *p = hcb_r_y(y2); return(oh); } From phk at varnish-cache.org Fri May 28 10:55:09 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Fri, 28 May 2010 12:55:09 +0200 Subject: r4862 - trunk/varnish-cache/doc/sphinx/phk Message-ID: Author: phk Date: 2010-05-28 12:55:08 +0200 (Fri, 28 May 2010) New Revision: 4862 Added: trunk/varnish-cache/doc/sphinx/phk/thoughts.rst Modified: trunk/varnish-cache/doc/sphinx/phk/index.rst Log: upd Modified: trunk/varnish-cache/doc/sphinx/phk/index.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/phk/index.rst 2010-05-28 09:28:15 UTC (rev 4861) +++ trunk/varnish-cache/doc/sphinx/phk/index.rst 2010-05-28 10:55:08 UTC (rev 4862) @@ -8,6 +8,7 @@ .. toctree:: + thoughts.rst + autocrap.rst sphinx.rst - autocrap.rst Added: trunk/varnish-cache/doc/sphinx/phk/thoughts.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/phk/thoughts.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/phk/thoughts.rst 2010-05-28 10:55:08 UTC (rev 4862) @@ -0,0 +1,35 @@ +.. _phk_thoughts: + +========================= +What were they thinking ? +========================= + +The reason I try to write these notes is the chinese wall. + +Ever since I first saw it on a school-book map, I have been wondering +what the decision making process were like. + +We would like to think that the emperor asked for ideas, and that +advisors came up with analyses, budgets, cost/benefit calculations +and project plans for various proposals, and that the emperor applied +his wisdom to choose the better idea. + +But it could also be, that The Assistant to The Deputy Viceminister of +Northern Affairs, edged in sideways, at a carefully chosen time where +the emperor looked relaxed and friendly, and sort of happend to mention +that 50 villages had been sort of raided by the barbarians, hoping +for the reply, which would not be a career opportunity +for The Assistant to The Assistant to The Deputy Viceminister of +Northern Affairs. + +And likely as not, the emperor absentmindedly grunted "Why don't +you just build a wall to keep them out or something ?" probably +wondering about the competence of an administration, which could +not figure out to build palisades around border villages without +bothering him and causing a monument to the Peter Principle and +Parkinssons Law to be built, which can be seen from orbit, and +possibly from the moon, if you bring your binoculars. + +If somebody had written some notes, we might have known. + +Poul-Henning, 2010-05-28 From perbu at varnish-cache.org Fri May 28 11:37:39 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Fri, 28 May 2010 13:37:39 +0200 Subject: r4863 - trunk/varnish-cache/doc/sphinx/tutorial Message-ID: Author: perbu Date: 2010-05-28 13:37:39 +0200 (Fri, 28 May 2010) New Revision: 4863 Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst Log: added varnishlog and varnishtop to increasing hitrate chapter Modified: trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-28 10:55:08 UTC (rev 4862) +++ trunk/varnish-cache/doc/sphinx/tutorial/increasing_your_hitrate.rst 2010-05-28 11:37:39 UTC (rev 4863) @@ -10,16 +10,33 @@ to get a high hitrate in Varnish. Note that you need a tool to see what HTTP headers fly between you and -the web server. If you have Varnish the easiest might be to use -varnishlog, but sometimes a separate tool makes sense. Here are the -ones I use. +the web server. If you have Varnish the easiest is to use varnishlog +and varnishtop but sometimes a client-side tool makes sense. Here are +the ones I use. +Tool: varnistop +~~~~~~~~~~~~~~~ + +You can use varnishtop to identify what URLs are hitting the backend +the most. ``varnishtop -i txurl`` is a essential command. You can see +some other examples of varnishtop usage in :ref:`tutorial-statistics`. + + +Tool: varnishlog +~~~~~~~~~~~~~~~~ + +When you have identified the an URL which is frequently sent to the +backend you can use varnishlog to have a look at the whole request. +``varnishlog -c -o /foo/bar`` will give the whole (-o) requests coming +from the client (-c) matching /foo/bar. + + Tool: lwp-request ~~~~~~~~~~~~~~~~~ lwp-request is part of The World-Wide Web library for Perl. It's couple of really basic programs that can execute a HTTP request and -give you the result. I use two programs, GET and HEAD. +give you the result. I mostly use two programs, GET and HEAD. vg.no was the first site to use Varnish and the people running Varnish there are quite cluefull. So its interesting to look at their HTTP From phk at varnish-cache.org Mon May 31 07:51:43 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 31 May 2010 09:51:43 +0200 Subject: r4864 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-31 09:51:43 +0200 (Mon, 31 May 2010) New Revision: 4864 Modified: trunk/varnish-cache/bin/varnishd/cache_wrw.c Log: Clarify a debug message Modified: trunk/varnish-cache/bin/varnishd/cache_wrw.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_wrw.c 2010-05-28 11:37:39 UTC (rev 4863) +++ trunk/varnish-cache/bin/varnishd/cache_wrw.c 2010-05-31 07:51:43 UTC (rev 4864) @@ -105,7 +105,7 @@ if (i != w->liov) { w->werr++; WSL(w, SLT_Debug, *w->wfd, - "Write error, len = %d/%d, errno = %s", + "Write error, retval = %d, len = %d, errno = %s", i, w->liov, strerror(errno)); } } From phk at varnish-cache.org Mon May 31 08:00:23 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 31 May 2010 10:00:23 +0200 Subject: r4865 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: Author: phk Date: 2010-05-31 10:00:23 +0200 (Mon, 31 May 2010) New Revision: 4865 Added: trunk/varnish-cache/bin/varnishtest/tests/r00702.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_response.c Log: Fix an off-by-one bug, when requested range exceeds available data. Submitted by: Luc Saillard Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2010-05-31 07:51:43 UTC (rev 4864) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2010-05-31 08:00:23 UTC (rev 4865) @@ -169,7 +169,7 @@ return; if (high >= sp->obj->len) - high = sp->obj->len; + high = sp->obj->len - 1; if (low > high) return; Added: trunk/varnish-cache/bin/varnishtest/tests/r00702.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00702.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00702.vtc 2010-05-31 08:00:23 UTC (rev 4865) @@ -0,0 +1,22 @@ +# $Id$ + +test "Range bug" + +server s1 { + rxreq + txresp -bodylen 100 +} -start + +varnish v1 -vcl+backend { +} -start + +varnish v1 -cliok "param.set http_range_support on" + + +client c1 { + txreq -hdr "Range: bytes=50-200" + rxresp + expect resp.status == 206 + expect resp.bodylen == 50 +} -run + From phk at varnish-cache.org Mon May 31 08:50:36 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 31 May 2010 10:50:36 +0200 Subject: r4866 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: Author: phk Date: 2010-05-31 10:50:35 +0200 (Mon, 31 May 2010) New Revision: 4866 Added: trunk/varnish-cache/bin/varnishtest/tests/r00704.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishtest/tests/c00034.vtc Log: I have no idea how I overlooked that a "bytes=-100" range was from the end of the object, but I did. Fixes #704. Reported by: Luc Saillard Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2010-05-31 08:00:23 UTC (rev 4865) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2010-05-31 08:50:35 UTC (rev 4866) @@ -131,7 +131,7 @@ static void res_dorange(struct sess *sp, const char *r, unsigned *plow, unsigned *phigh) { - unsigned low, high; + unsigned low, high, has_low; (void)sp; if (strncmp(r, "bytes=", 6)) @@ -139,10 +139,11 @@ r += 6; /* The low end of range */ - low = 0; + has_low = low = 0; if (!vct_isdigit(*r) && *r != '-') return; while (vct_isdigit(*r)) { + has_low = 1; low *= 10; low += *r - '0'; r++; @@ -163,6 +164,10 @@ high += *r - '0'; r++; } + if (!has_low) { + low = sp->obj->len - high; + high = sp->obj->len - 1; + } } else high = sp->obj->len - 1; if (*r != '\0') Modified: trunk/varnish-cache/bin/varnishtest/tests/c00034.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/c00034.vtc 2010-05-31 08:00:23 UTC (rev 4865) +++ trunk/varnish-cache/bin/varnishtest/tests/c00034.vtc 2010-05-31 08:50:35 UTC (rev 4866) @@ -59,7 +59,7 @@ txreq -hdr "Range: bytes=-9" rxresp expect resp.status == 206 - expect resp.bodylen == 10 + expect resp.bodylen == 9 txreq -hdr "Range: bytes=-" rxresp Added: trunk/varnish-cache/bin/varnishtest/tests/r00704.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00704.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00704.vtc 2010-05-31 08:50:35 UTC (rev 4866) @@ -0,0 +1,22 @@ +# $Id$ + +test "Range bug" + +server s1 { + rxreq + txresp -bodylen 100 +} -start + +varnish v1 -vcl+backend { +} -start + +varnish v1 -cliok "param.set http_range_support on" + + +client c1 { + txreq -hdr "Range: bytes=-20" + rxresp + expect resp.status == 206 + expect resp.bodylen == 20 +} -run + From phk at varnish-cache.org Mon May 31 10:08:39 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 31 May 2010 12:08:39 +0200 Subject: r4867 - trunk/varnish-cache/bin/varnishd Message-ID: Author: phk Date: 2010-05-31 12:08:38 +0200 (Mon, 31 May 2010) New Revision: 4867 Modified: trunk/varnish-cache/bin/varnishd/default.vcl Log: Don't bog our webserver down when somebodys backend croaks. Modified: trunk/varnish-cache/bin/varnishd/default.vcl =================================================================== --- trunk/varnish-cache/bin/varnishd/default.vcl 2010-05-31 08:50:35 UTC (rev 4866) +++ trunk/varnish-cache/bin/varnishd/default.vcl 2010-05-31 10:08:38 UTC (rev 4867) @@ -132,9 +132,7 @@

Guru Meditation:

XID: "} req.xid {"

-
- Varnish cache server -
+

Varnish cache server

"}; From phk at varnish-cache.org Mon May 31 11:31:11 2010 From: phk at varnish-cache.org (phk at varnish-cache.org) Date: Mon, 31 May 2010 13:31:11 +0200 Subject: r4868 - in trunk/varnish-cache: bin/varnishd bin/varnishtest include lib/libvarnish Message-ID: Author: phk Date: 2010-05-31 13:31:11 +0200 (Mon, 31 May 2010) New Revision: 4868 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishtest/vtc_server.c trunk/varnish-cache/include/libvarnish.h trunk/varnish-cache/lib/libvarnish/tcp.c Log: Attempt an even more comprehensive fix for a bug in Solaris: If the remote end RST's a TCP connection, all socket related syscalls seems to return EINVAL. Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-31 10:08:38 UTC (rev 4867) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2010-05-31 11:31:11 UTC (rev 4868) @@ -696,10 +696,12 @@ struct sockaddr * VRT_r_server_ip(struct sess *sp) { + int i; - if (sp->mysockaddr->sa_family == AF_UNSPEC) - assert(!getsockname(sp->fd, sp->mysockaddr, &sp->mysockaddrlen) - || errno == ECONNRESET); + if (sp->mysockaddr->sa_family == AF_UNSPEC) { + i = getsockname(sp->fd, sp->mysockaddr, &sp->mysockaddrlen); + assert(TCP_Check(i)); + } return (sp->mysockaddr); } Modified: trunk/varnish-cache/bin/varnishtest/vtc_server.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_server.c 2010-05-31 10:08:38 UTC (rev 4867) +++ trunk/varnish-cache/bin/varnishtest/vtc_server.c 2010-05-31 11:31:11 UTC (rev 4868) @@ -85,7 +85,7 @@ { struct server *s; struct vtclog *vl; - int i, fd; + int i, j, fd; struct sockaddr_storage addr_s; struct sockaddr *addr; socklen_t l; @@ -107,8 +107,9 @@ vtc_log(vl, 3, "accepted fd %d", fd); http_process(vl, s->spec, fd, s->sock); vtc_log(vl, 3, "shutting fd %d", fd); - assert((shutdown(fd, SHUT_WR) == 0) - || errno == ENOTCONN || errno == ECONNRESET); + j = shutdown(fd, SHUT_WR); + if (!TCP_Check(j)) + vtc_log(vl, 0, "Shutdown failed: %s", strerror(errno)); TCP_close(&fd); } macro_def(s->vl, s->name, "addr", NULL); Modified: trunk/varnish-cache/include/libvarnish.h =================================================================== --- trunk/varnish-cache/include/libvarnish.h 2010-05-31 10:08:38 UTC (rev 4867) +++ trunk/varnish-cache/include/libvarnish.h 2010-05-31 11:31:11 UTC (rev 4868) @@ -60,7 +60,16 @@ #define TCP_ADDRBUFSIZE 64 #define TCP_PORTBUFSIZE 16 +#if defined (__SVR4) && defined (__sun) +/* + * Solaris returns EINVAL if the other end unexepectedly reset the + * connection. This is a bug in Solaris. + */ +#define TCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN \ + || errno == EINVAL) +#else #define TCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN) +#endif #define TCP_Assert(a) assert(TCP_Check(a)) Modified: trunk/varnish-cache/lib/libvarnish/tcp.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/tcp.c 2010-05-31 10:08:38 UTC (rev 4867) +++ trunk/varnish-cache/lib/libvarnish/tcp.c 2010-05-31 11:31:11 UTC (rev 4868) @@ -236,9 +236,11 @@ void TCP_close(int *s) { - assert (close(*s) == 0 || - errno == ECONNRESET || - errno == ENOTCONN); + int i; + + i = close(*s); + + assert (TCP_Check(i)); *s = -1; } From perbu at varnish-cache.org Mon May 31 17:22:29 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Mon, 31 May 2010 19:22:29 +0200 Subject: r4869 - trunk/varnish-cache/doc/sphinx/reference Message-ID: Author: perbu Date: 2010-05-31 19:22:29 +0200 (Mon, 31 May 2010) New Revision: 4869 Added: trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst Log: Converted varnisadm(1) to rst. Isn't it beutiful? Added: trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst (rev 0) +++ trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst 2010-05-31 17:22:29 UTC (rev 4869) @@ -0,0 +1,74 @@ +=========== + varnishadm +=========== + +---------------------------------- +Control a running varnish instance +---------------------------------- + +:Author: Cecilie Fritzvold +:Author: Per Buer +:Date: 2008-01-31 +:Copyright: public domain +:Version: 0.2 +:Manual section: 1 + +SYNOPSIS +======== + + varnishadm [-t timeout] [-S secret_file] -T address:port [command [...]] + +DESCRIPTION +=========== + +The varnishadm utility establishes a CLI connection using the -T and -S arguments. + +If a command is given, the command and arguments are sent over the CLI +connection and the result returned on stdout. + +If no command argument is given varnishadm will pass commands and +replies between the CLI socket and stdin/stdout. + +OPTIONS +======= + +-t timeout Wait no longer than this many seconds for an operation to finish. + +-S secret_file Specify the authentication secret file. This should be the same -S + argument as was given to varnishd. Only processes which can read + the contents of this file, will be able to authenticate the CLI connection. + +-T address:port Connect to the management interface at the specified address and port. + + +Available commands and parameters are documented in the varnishd(1) +manual page. Additionally, a summary of commands can be obtained by +issuing the *help* command, and a summary of parameters can be +obtained by issuing the *param.show* command. + +EXIT STATUS +=========== + +If a command is given, the exit status of the varnishadm utility is +zero if the command succeeded, and non-zero otherwise. + +EXAMPLES +======== + +Some ways you can use varnishadm::: + + varnishadm -T localhost:999 -S /var/db/secret vcl.use foo + echo vcl.use foo | varnishadm -T localhost:999 -S /var/db/secret + echo vcl.use foo | ssh vhost varnishadm -T localhost:999 -S /var/db/secret + +SEE ALSO +======== + +* varnishd(1) + +HISTORY +======= + +The varnishadm utility and this manual page were written by Cecilie +Fritzvold. Converted to reStructured and updated in 2010 by Per +Buer. From perbu at varnish-cache.org Mon May 31 17:43:25 2010 From: perbu at varnish-cache.org (perbu at varnish-cache.org) Date: Mon, 31 May 2010 19:43:25 +0200 Subject: r4870 - trunk/varnish-cache/doc/sphinx/reference Message-ID: Author: perbu Date: 2010-05-31 19:43:25 +0200 (Mon, 31 May 2010) New Revision: 4870 Modified: trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst Log: Made sensible copyright information Modified: trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst =================================================================== --- trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst 2010-05-31 17:22:29 UTC (rev 4869) +++ trunk/varnish-cache/doc/sphinx/reference/varnishadm.rst 2010-05-31 17:43:25 UTC (rev 4870) @@ -8,8 +8,7 @@ :Author: Cecilie Fritzvold :Author: Per Buer -:Date: 2008-01-31 -:Copyright: public domain +:Date: 2010-05-31 :Version: 0.2 :Manual section: 1 @@ -72,3 +71,14 @@ The varnishadm utility and this manual page were written by Cecilie Fritzvold. Converted to reStructured and updated in 2010 by Per Buer. + +COPYRIGHT +========= + +This document is licensed under the same licence as Varnish +itself. See LICENCE for details. + +* Copyright (c) 2006 Verdens Gang AS +* Copyright (c) 2006-2008 Linpro AS +* Copyright (c) 2008-2010 Redpill Linpro AS +* Copyright (c) 2010 Varnish Software AS