From tfheen at projects.linpro.no Mon Mar 2 09:49:15 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 2 Mar 2009 10:49:15 +0100 (CET) Subject: r3847 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302094915.ECD631F7497@projects.linpro.no> Author: tfheen Date: 2009-03-02 10:49:15 +0100 (Mon, 02 Mar 2009) New Revision: 3847 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c Log: Solaris fix for VRT_error If VRT_error is called with reason==NULL, WSL() will eventually call strlen(0), which will cause a SIGSEGV on (Open)Solaris. Fix this by checking if the reason is empty and just use (null) in that case. Fixes: 458 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-02-28 22:06:39 UTC (rev 3846) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-02 09:49:15 UTC (rev 3847) @@ -62,7 +62,8 @@ { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason); + WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ? + reason : "(null)"); sp->err_code = code ? code : 503; sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code); } From phk at projects.linpro.no Mon Mar 2 10:43:56 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 11:43:56 +0100 (CET) Subject: r3848 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302104356.A6CF21F74E7@projects.linpro.no> Author: phk Date: 2009-03-02 11:43:56 +0100 (Mon, 02 Mar 2009) New Revision: 3848 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: Add an assert to make sure 32 stays 32 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 09:49:15 UTC (rev 3847) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 10:43:56 UTC (rev 3848) @@ -572,6 +572,7 @@ HSH_Init(void) { + assert(DIGEST_LEN == SHA256_LEN); /* avoid #include pollution */ save_hash = params->save_hash; hash = heritage.hash; if (hash->start != NULL) From phk at projects.linpro.no Mon Mar 2 11:22:25 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 12:22:25 +0100 (CET) Subject: r3849 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090302112225.1D19928464@projects.linpro.no> Author: phk Date: 2009-03-02 12:22:24 +0100 (Mon, 02 Mar 2009) New Revision: 3849 Added: trunk/varnish-cache/bin/varnishtest/tests/c00023.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_critbit.c trunk/varnish-cache/bin/varnishd/mgt_param.c Log: Add a debug facility for substituting digests with hand-created digests with edge-bit conditions and run the critbit through them. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 10:43:56 UTC (rev 3848) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 11:22:24 UTC (rev 3849) @@ -272,6 +272,83 @@ sp->lhashptr += l + 1; } +/********************************************************************** + * This is a debugging hack to enable testing of boundary conditions + * in the hash algorithm. + * We trap the first 9 different digests and translate them to different + * digests with edge bit conditions + */ + +static struct hsh_magiclist { + unsigned char was[SHA256_LEN]; + unsigned char now[SHA256_LEN]; +} hsh_magiclist[] = { + { .now = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { .now = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } }, + { .now = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 } }, + { .now = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40 } }, + { .now = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 } }, + { .now = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { .now = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { .now = { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { .now = { 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, +}; + +#define HSH_NMAGIC (sizeof hsh_magiclist / sizeof hsh_magiclist[0]) + +static void +hsh_testmagic(void *result) +{ + int i, j; + static int nused; + + for (i = 0; i < nused; i++) + if (!memcmp(hsh_magiclist[i].was, result, SHA256_LEN)) + break; + if (i == nused && i < HSH_NMAGIC) + memcpy(hsh_magiclist[nused++].was, result, SHA256_LEN); + if (i == nused) + return; + fprintf(stderr, "HASHMAGIC: <"); + for (j = 0; j < SHA256_LEN; j++) + fprintf(stderr, "%02x", ((unsigned char*)result)[j]); + fprintf(stderr, "> -> <"); + memcpy(result, hsh_magiclist[i].now, SHA256_LEN); + for (j = 0; j < SHA256_LEN; j++) + fprintf(stderr, "%02x", ((unsigned char*)result)[j]); + fprintf(stderr, ">\n"); +} + +/**********************************************************************/ + + struct objcore * HSH_Lookup(struct sess *sp, struct objhead **poh) { @@ -289,6 +366,8 @@ HSH_Prealloc(sp); SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx); + if (params->diag_bitmap & 0x80000000) + hsh_testmagic(sp->wrk->nobjhead->digest); if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-03-02 10:43:56 UTC (rev 3848) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-03-02 11:22:24 UTC (rev 3849) @@ -197,10 +197,8 @@ while(hcb_is_y(*p)) { y = hcb_l_y(*p); - if (y->ptr > DIGEST_LEN) - s = 0; - else - s = (oh->digest[y->ptr] & y->bitmask) != 0; + assert(y->ptr < DIGEST_LEN); + s = (oh->digest[y->ptr] & y->bitmask) != 0; assert(s < 2); root->cmps++; p = &y->leaf[s]; @@ -233,10 +231,8 @@ y = hcb_l_y(*p); if (y->critbit > y2->critbit) break; - if (y->ptr > DIGEST_LEN) - s = 0; - else - s = (oh->digest[y->ptr] & y->bitmask) != 0; + assert(y->ptr < DIGEST_LEN); + s = (oh->digest[y->ptr] & y->bitmask) != 0; assert(s < 2); root->cmps++; p = &y->leaf[s]; @@ -266,10 +262,8 @@ while(1) { assert(hcb_is_y(*p)); y = hcb_l_y(*p); - if (y->ptr > DIGEST_LEN) - s = 0; - else - s = (oh->digest[y->ptr] & y->bitmask) != 0; + assert(y->ptr < DIGEST_LEN); + s = (oh->digest[y->ptr] & y->bitmask) != 0; assert(s < 2); if (y->leaf[s] == hcb_r_node(oh)) { *p = y->leaf[1 - s]; Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-03-02 10:43:56 UTC (rev 3848) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-03-02 11:22:24 UTC (rev 3849) @@ -746,6 +746,7 @@ " 0x00008000 - panic to abort2().\n" #endif " 0x00010000 - synchronize shmlog.\n" + " 0x80000000 - do edge-detection on digest.\n" "Use 0x notation and do the bitor in your head :-)\n", 0, "0", "bitmap" }, Added: trunk/varnish-cache/bin/varnishtest/tests/c00023.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/c00023.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/c00023.vtc 2009-03-02 11:22:24 UTC (rev 3849) @@ -0,0 +1,156 @@ +# $Id$ + +test "Test -h critbit for digest edges" + +server s1 { + rxreq + expect req.url == "/1" + txresp -body "\n" + rxreq + expect req.url == "/2" + txresp -body "x\n" + rxreq + expect req.url == "/3" + txresp -body "xx\n" + rxreq + expect req.url == "/4" + txresp -body "xxx\n" + rxreq + expect req.url == "/5" + txresp -body "xxxx\n" + rxreq + expect req.url == "/6" + txresp -body "xxxxx\n" + rxreq + expect req.url == "/7" + txresp -body "xxxxxx\n" + rxreq + expect req.url == "/8" + txresp -body "xxxxxxx\n" + rxreq + expect req.url == "/9" + txresp -body "xxxxxxxx\n" +} -start + +varnish v1 -arg "-hcritbit" -vcl+backend { } -start +varnish v1 -cliok "param.set diag_bitmap 0x8000000" + +client c1 { + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + expect resp.http.X-Varnish == "1001" + + txreq -url "/2" + rxresp + expect resp.bodylen == 2 + expect resp.status == 200 + expect resp.http.X-Varnish == "1002" + + txreq -url "/3" + rxresp + expect resp.bodylen == 3 + expect resp.status == 200 + expect resp.http.X-Varnish == "1003" + + txreq -url "/4" + rxresp + expect resp.bodylen == 4 + expect resp.status == 200 + expect resp.http.X-Varnish == "1004" + + txreq -url "/5" + rxresp + expect resp.bodylen == 5 + expect resp.status == 200 + expect resp.http.X-Varnish == "1005" + + txreq -url "/6" + rxresp + expect resp.bodylen == 6 + expect resp.status == 200 + expect resp.http.X-Varnish == "1006" + + txreq -url "/7" + rxresp + expect resp.bodylen == 7 + expect resp.status == 200 + expect resp.http.X-Varnish == "1007" + + txreq -url "/8" + rxresp + expect resp.bodylen == 8 + expect resp.status == 200 + expect resp.http.X-Varnish == "1008" + + txreq -url "/9" + rxresp + expect resp.bodylen == 9 + expect resp.status == 200 + expect resp.http.X-Varnish == "1009" +} -run + + +client c1 { + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + expect resp.http.X-Varnish == "1010 1001" + + txreq -url "/2" + rxresp + expect resp.bodylen == 2 + expect resp.status == 200 + expect resp.http.X-Varnish == "1011 1002" + + txreq -url "/3" + rxresp + expect resp.bodylen == 3 + expect resp.status == 200 + expect resp.http.X-Varnish == "1012 1003" + + txreq -url "/4" + rxresp + expect resp.bodylen == 4 + expect resp.status == 200 + expect resp.http.X-Varnish == "1013 1004" + + txreq -url "/5" + rxresp + expect resp.bodylen == 5 + expect resp.status == 200 + expect resp.http.X-Varnish == "1014 1005" + + txreq -url "/6" + rxresp + expect resp.bodylen == 6 + expect resp.status == 200 + expect resp.http.X-Varnish == "1015 1006" + + txreq -url "/7" + rxresp + expect resp.bodylen == 7 + expect resp.status == 200 + expect resp.http.X-Varnish == "1016 1007" + + txreq -url "/8" + rxresp + expect resp.bodylen == 8 + expect resp.status == 200 + expect resp.http.X-Varnish == "1017 1008" + + txreq -url "/9" + rxresp + expect resp.bodylen == 9 + expect resp.status == 200 + expect resp.http.X-Varnish == "1018 1009" +} -run + +varnish v1 -cliok "hcb.dump" + +varnish v1 -expect client_conn == 2 +varnish v1 -expect cache_hit == 9 +varnish v1 -expect cache_miss == 9 +varnish v1 -expect client_req == 18 From tfheen at projects.linpro.no Mon Mar 2 11:54:28 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 2 Mar 2009 12:54:28 +0100 (CET) Subject: r3850 - trunk/varnish-cache/bin/varnishadm Message-ID: <20090302115428.20CDA28449@projects.linpro.no> Author: tfheen Date: 2009-03-02 12:54:27 +0100 (Mon, 02 Mar 2009) New Revision: 3850 Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c Log: Make varnishadm able to handle cli_banner=on Send a ping and ignore the first reply if it doesn't include a PONG. Fixes: #457 Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c =================================================================== --- trunk/varnish-cache/bin/varnishadm/varnishadm.c 2009-03-02 11:22:24 UTC (rev 3849) +++ trunk/varnish-cache/bin/varnishadm/varnishadm.c 2009-03-02 11:54:27 UTC (rev 3850) @@ -41,6 +41,43 @@ #define STATUS_OK 200 +static void +parse_reply(int sock, long *status, long *answerlen, char **answer) +{ + int n; + char buf[13]; + char *p, *pp; + + n = read(sock, buf, 13); + if (n != 13) { + fprintf(stderr, "An error occured in receiving status.\n"); + exit(1); + } + if (!(p = strchr(buf, ' '))) { + fprintf(stderr, + "An error occured in parsing of status code.\n"); + exit(1); + } + *p = '\0'; + *status = strtol(buf, &p, 10); + pp = p+1; + if (!(p = strchr(pp, '\n'))) { + fprintf(stderr, "An error occured " + "in parsing of number of bytes returned.\n"); + exit(1); + } + *p = '\0'; + *answerlen = strtol(pp, &p, 10); + + *answer = malloc(*answerlen+1); + n = read(sock, *answer, *answerlen); + read(sock, buf, 1); /* Read the trailing \n */ + if (n != *answerlen) { + fprintf(stderr, "An error occured in receiving answer.\n"); + exit(1); + } +} + /* * This function establishes a connection to the specified ip and port and * sends a command to varnishd. If varnishd returns an OK status, the result @@ -76,6 +113,25 @@ } free(ta); + write(sock, "ping\n", 5); + parse_reply(sock, &status, &bytes, &answer); + if (status != 200) { + fprintf(stderr, "No pong received from server\n", status); + exit(1); + } + + if (strstr(answer, "PONG") == NULL) { + /* The first one was probably just the banner, + see if there are more replies.*/ + free(answer); + parse_reply(sock, &status, &answerlen, &answer); + if (status != 200 || strstr(answer, "PONG") == NULL) { + fprintf(stderr, "No pong received from server\n", status); + exit(1); + } + } + free(answer); + for (i=0; i 0) write(sock, " ", 1); @@ -83,33 +139,8 @@ } write(sock, "\n", 1); - n = read(sock, buf, 13); - if (n != 13) { - fprintf(stderr, "An error occured in receiving status.\n"); - exit(1); - } - if (!(p = strchr(buf, ' '))) { - fprintf(stderr, - "An error occured in parsing of status code.\n"); - exit(1); - } - *p = '\0'; - status = strtol(buf, &p, 10); - pp = p+1; - if (!(p = strchr(pp, '\n'))) { - fprintf(stderr, "An error occured " - "in parsing of number of bytes returned.\n"); - exit(1); - } - *p = '\0'; - bytes = strtol(pp, &p, 10); + parse_reply(sock, &status, &bytes, &answer); - answer = malloc(bytes+1); - n = read(sock, answer, bytes); - if (n != bytes) { - fprintf(stderr, "An error occured in receiving answer.\n"); - exit(1); - } answer[bytes] = '\0'; close(sock); From tfheen at projects.linpro.no Mon Mar 2 12:01:31 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 2 Mar 2009 13:01:31 +0100 (CET) Subject: r3851 - trunk/varnish-cache/bin/varnishadm Message-ID: <20090302120131.1850D38086@projects.linpro.no> Author: tfheen Date: 2009-03-02 13:01:30 +0100 (Mon, 02 Mar 2009) New Revision: 3851 Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c Log: Fix typo in varnishadm.c (variable name) Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c =================================================================== --- trunk/varnish-cache/bin/varnishadm/varnishadm.c 2009-03-02 11:54:27 UTC (rev 3850) +++ trunk/varnish-cache/bin/varnishadm/varnishadm.c 2009-03-02 12:01:30 UTC (rev 3851) @@ -124,7 +124,7 @@ /* The first one was probably just the banner, see if there are more replies.*/ free(answer); - parse_reply(sock, &status, &answerlen, &answer); + parse_reply(sock, &status, &bytes, &answer); if (status != 200 || strstr(answer, "PONG") == NULL) { fprintf(stderr, "No pong received from server\n", status); exit(1); From tfheen at projects.linpro.no Mon Mar 2 12:01:33 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 2 Mar 2009 13:01:33 +0100 (CET) Subject: r3852 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302120133.8884638654@projects.linpro.no> Author: tfheen Date: 2009-03-02 13:01:33 +0100 (Mon, 02 Mar 2009) New Revision: 3852 Modified: trunk/varnish-cache/bin/varnishd/default.vcl Log: Documentation for vcl_pipe and connection: close Note in the default VCL that connection: close might be wanted. Modified: trunk/varnish-cache/bin/varnishd/default.vcl =================================================================== --- trunk/varnish-cache/bin/varnishd/default.vcl 2009-03-02 12:01:30 UTC (rev 3851) +++ trunk/varnish-cache/bin/varnishd/default.vcl 2009-03-02 12:01:33 UTC (rev 3852) @@ -62,6 +62,12 @@ } sub vcl_pipe { + # Note that only the first request to the backend will have + # X-Forwarded-For set. If you use X-Forwarded-For and want to + # have it set for all requests, make sure to have: + # set req.http.connection = "close"; + # here. It is not set by default as it might break some broken web + # applications, like IIS with NTLM authentication. return (pipe); } From tfheen at projects.linpro.no Mon Mar 2 12:04:04 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 2 Mar 2009 13:04:04 +0100 (CET) Subject: r3853 - trunk/varnish-cache/bin/varnishtest/tests Message-ID: <20090302120404.C835C28449@projects.linpro.no> Author: tfheen Date: 2009-03-02 13:04:04 +0100 (Mon, 02 Mar 2009) New Revision: 3853 Added: trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc Removed: trunk/varnish-cache/bin/varnishtest/tests/r00433.vcl Log: Rename r00433.vcl to r00433.vtc Deleted: trunk/varnish-cache/bin/varnishtest/tests/r00433.vcl =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00433.vcl 2009-03-02 12:01:33 UTC (rev 3852) +++ trunk/varnish-cache/bin/varnishtest/tests/r00433.vcl 2009-03-02 12:04:04 UTC (rev 3853) @@ -1,48 +0,0 @@ -# $Id$ - -test "noidx" - -server s1 { - rxreq - expect req.url == "/foo" - txresp -hdr "Connection: close" - send { - -FOO{ - -}FOO - -The end. -} - -} -start - -server s2 -listen 127.0.0.1:9082 { - rxreq - expect req.url == "/bar" - txresp -body "bar" -} -start - -varnish v1 -vcl+backend { - sub vcl_recv { - if (req.url == "/foo") { - set req.backend = s1; - } else { - set req.backend = s2; - } - } - - sub vcl_fetch { - esi; - } -} -start - -varnish v1 -cliok "param.set esi_syntax 4" -varnish v1 -cliok "param.set diag_bitmap 0x10000" -varnish v1 -cliok "debug.fragfetch 32" - -client c1 { - txreq -url /foo - rxresp -} -run Copied: trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc (from rev 3852, trunk/varnish-cache/bin/varnishtest/tests/r00433.vcl) =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc 2009-03-02 12:04:04 UTC (rev 3853) @@ -0,0 +1,48 @@ +# $Id$ + +test "noidx" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -hdr "Connection: close" + send { + +FOO{ + +}FOO + +The end. +} + +} -start + +server s2 -listen 127.0.0.1:9082 { + rxreq + expect req.url == "/bar" + txresp -body "bar" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/foo") { + set req.backend = s1; + } else { + set req.backend = s2; + } + } + + sub vcl_fetch { + esi; + } +} -start + +varnish v1 -cliok "param.set esi_syntax 4" +varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "debug.fragfetch 32" + +client c1 { + txreq -url /foo + rxresp +} -run From tfheen at projects.linpro.no Mon Mar 2 12:54:11 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 2 Mar 2009 13:54:11 +0100 (CET) Subject: r3854 - trunk/varnish-cache/bin/varnishadm Message-ID: <20090302125411.0B03B38086@projects.linpro.no> Author: tfheen Date: 2009-03-02 13:54:10 +0100 (Mon, 02 Mar 2009) New Revision: 3854 Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c Log: Fix compilation warnings Modified: trunk/varnish-cache/bin/varnishadm/varnishadm.c =================================================================== --- trunk/varnish-cache/bin/varnishadm/varnishadm.c 2009-03-02 12:04:04 UTC (rev 3853) +++ trunk/varnish-cache/bin/varnishadm/varnishadm.c 2009-03-02 12:54:10 UTC (rev 3854) @@ -93,8 +93,6 @@ int sock; long status, bytes; char *answer = NULL; - char buf[13]; - char *p, *pp; XXXAZ(VSS_parse(T_arg, &addr, &port)); XXXAN(n = VSS_resolve(addr, port, &ta)); @@ -116,7 +114,7 @@ write(sock, "ping\n", 5); parse_reply(sock, &status, &bytes, &answer); if (status != 200) { - fprintf(stderr, "No pong received from server\n", status); + fprintf(stderr, "No pong received from server\n"); exit(1); } @@ -126,7 +124,7 @@ free(answer); parse_reply(sock, &status, &bytes, &answer); if (status != 200 || strstr(answer, "PONG") == NULL) { - fprintf(stderr, "No pong received from server\n", status); + fprintf(stderr, "No pong received from server\n"); exit(1); } } From phk at projects.linpro.no Mon Mar 2 12:56:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 13:56:03 +0100 (CET) Subject: r3855 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302125603.633D538086@projects.linpro.no> Author: phk Date: 2009-03-02 13:56:03 +0100 (Mon, 02 Mar 2009) New Revision: 3855 Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c Log: Give coords for WRONG Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_panic.c 2009-03-02 12:54:10 UTC (rev 3854) +++ trunk/varnish-cache/bin/varnishd/cache_panic.c 2009-03-02 12:56:03 UTC (rev 3855) @@ -260,7 +260,7 @@ switch(xxx) { case 3: vsb_printf(vsp, - "Wrong turn:\n%s\n", cond); + "Wrong turn at %s:%d:\n%s\n", file, line, cond); break; case 2: vsb_printf(vsp, From phk at projects.linpro.no Mon Mar 2 13:39:09 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 14:39:09 +0100 (CET) Subject: r3856 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302133909.C72B828449@projects.linpro.no> Author: phk Date: 2009-03-02 14:39:09 +0100 (Mon, 02 Mar 2009) New Revision: 3856 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Cache the vcl_fetch{} result, in case vcl_discard{} (via LRU) overwrites it during body fetch. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-02 12:56:03 UTC (rev 3855) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-02 13:39:09 UTC (rev 3856) @@ -380,6 +380,7 @@ struct http *hp, *hp2; struct object *o; char *b; + unsigned handling; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); @@ -442,8 +443,15 @@ VCL_fetch_method(sp); - o = HSH_NewObject(sp, sp->handling != VCL_RET_DELIVER); + /* + * When we fetch the body, we may hit the LRU cleanup and that + * will overwrite sp->handling, so we have to save our plans + * here. + */ + handling = sp->handling; + o = HSH_NewObject(sp, handling != VCL_RET_DELIVER); + if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); @@ -498,7 +506,7 @@ VBE_free_bereq(&sp->bereq); - switch (sp->handling) { + switch (handling) { case VCL_RET_RESTART: HSH_Drop(sp); sp->director = NULL; From phk at projects.linpro.no Mon Mar 2 14:38:55 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 15:38:55 +0100 (CET) Subject: r3857 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302143855.CEC7028449@projects.linpro.no> Author: phk Date: 2009-03-02 15:38:55 +0100 (Mon, 02 Mar 2009) New Revision: 3857 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: Clarify the waitinglist handling a little bit Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 13:39:09 UTC (rev 3856) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 14:38:55 UTC (rev 3857) @@ -521,25 +521,23 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); o = sp->obj; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + oh = o->objhead; + CHECK_OBJ(oh, OBJHEAD_MAGIC); + AN(ObjIsBusy(o)); assert(o->objcore->obj == o); assert(o->refcnt > 0); + assert(oh->refcnt > 0); if (o->ws_o->overflow) VSL_stats->n_objoverflow++; if (params->diag_bitmap & 0x40) WSP(sp, SLT_Debug, "Object %u workspace free %u", o->xid, WS_Free(o->ws_o)); - oh = o->objhead; - if (oh != NULL) { - CHECK_OBJ(oh, OBJHEAD_MAGIC); - Lck_Lock(&oh->mtx); - } + Lck_Lock(&oh->mtx); o->objcore->flags &= ~OC_F_BUSY; - if (oh != NULL) { - hsh_rush(oh); - Lck_Unlock(&oh->mtx); - } + hsh_rush(oh); + Lck_Unlock(&oh->mtx); } void @@ -605,9 +603,10 @@ Lck_Lock(&oh->mtx); assert(o->refcnt > 0); r = --o->refcnt; - if (!r) + if (!r) { + assert(VTAILQ_EMPTY(&oh->waitinglist)); VTAILQ_REMOVE(&oh->objcs, oc, list); - else + } else hsh_rush(oh); Lck_Unlock(&oh->mtx); } From phk at projects.linpro.no Mon Mar 2 16:15:58 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 17:15:58 +0100 (CET) Subject: r3858 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302161558.DF15E1F7536@projects.linpro.no> Author: phk Date: 2009-03-02 17:15:58 +0100 (Mon, 02 Mar 2009) New Revision: 3858 Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c Log: Add a missing refcnt in the case where we lost a collision while doing the lockless lookup. Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-03-02 14:38:55 UTC (rev 3857) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-03-02 16:15:58 UTC (rev 3858) @@ -443,12 +443,14 @@ fprintf(stderr, "hcb_lookup %d\n", __LINE__); #endif } else { + CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); free(noh->hash); noh->hash = NULL; VSL_stats->hcb_lock++; #ifdef PHK fprintf(stderr, "hcb_lookup %d\n", __LINE__); #endif + oh->refcnt++; } Lck_Unlock(&hcb_mtx); return (oh); From phk at projects.linpro.no Mon Mar 2 17:29:11 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 18:29:11 +0100 (CET) Subject: r3859 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302172911.6251238086@projects.linpro.no> Author: phk Date: 2009-03-02 18:29:11 +0100 (Mon, 02 Mar 2009) New Revision: 3859 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: I was confused, always rush when we deref an *object*, it is the *objhead* we need to be careful about. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 16:15:58 UTC (rev 3858) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-02 17:29:11 UTC (rev 3859) @@ -601,13 +601,12 @@ CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); Lck_Lock(&oh->mtx); + assert(oh->refcnt > 0); assert(o->refcnt > 0); r = --o->refcnt; - if (!r) { - assert(VTAILQ_EMPTY(&oh->waitinglist)); + if (!r) VTAILQ_REMOVE(&oh->objcs, oc, list); - } else - hsh_rush(oh); + hsh_rush(oh); Lck_Unlock(&oh->mtx); } From phk at projects.linpro.no Mon Mar 2 17:29:45 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 2 Mar 2009 18:29:45 +0100 (CET) Subject: r3860 - trunk/varnish-cache/bin/varnishd Message-ID: <20090302172945.763AE1F7535@projects.linpro.no> Author: phk Date: 2009-03-02 18:29:45 +0100 (Mon, 02 Mar 2009) New Revision: 3860 Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c Log: Be much more careful about volatile and locking Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-03-02 17:29:11 UTC (rev 3859) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-03-02 17:29:45 UTC (rev 3860) @@ -176,38 +176,44 @@ return (y->critbit); } -/**********************************************************************/ +/********************************************************************* + * Unless we have the lock, we need to be very careful about pointer + * references into the tree, we cannot trust things to be the same + * in two consequtive memory accesses. + */ static struct objhead * hcb_insert(struct hcb_root *root, struct objhead *oh, int has_lock) { - uintptr_t *p; + volatile uintptr_t *p; + uintptr_t pp; struct hcb_y *y, *y2; struct objhead *oh2; unsigned s, s2; - p = &root->origo; - if (*p == 0) { + pp = *p; + if (pp == 0) { if (!has_lock) return (NULL); *p = hcb_r_node(oh); return (oh); } - while(hcb_is_y(*p)) { - y = hcb_l_y(*p); + while(hcb_is_y(pp)) { + y = hcb_l_y(pp); assert(y->ptr < DIGEST_LEN); s = (oh->digest[y->ptr] & y->bitmask) != 0; assert(s < 2); root->cmps++; p = &y->leaf[s]; + pp = *p; } - assert(hcb_is_node(*p)); + assert(hcb_is_node(pp)); /* We found a node, does it match ? */ - oh2 = hcb_l_node(*p); + oh2 = hcb_l_node(pp); if (!memcmp(oh2->digest, oh->digest, DIGEST_LEN)) return (oh2); @@ -356,12 +362,12 @@ y = (void *)&oh->u; if (y->leaf[0] || y->leaf[1]) continue; - if (++oh->refcnt > COOL_DURATION) { + if (++oh->digest[0] > COOL_DURATION) { VTAILQ_REMOVE(&laylow, oh, coollist); #ifdef PHK fprintf(stderr, "OH %p is cold enough\n", oh); #endif - oh->refcnt = 0; + AZ(oh->refcnt); HSH_DeleteObjHead(&ww, oh); } } @@ -399,6 +405,9 @@ if (--oh->refcnt == 0) { Lck_Lock(&hcb_mtx); hcb_delete(&hcb_root, oh); + assert(VTAILQ_EMPTY(&oh->objcs)); + assert(VTAILQ_EMPTY(&oh->waitinglist)); + oh->digest[0] = 0; VTAILQ_INSERT_TAIL(&laylow, oh, coollist); Lck_Unlock(&hcb_mtx); } @@ -420,6 +429,10 @@ /* Assert that we didn't muck with the tree without lock */ assert(oh != noh); Lck_Lock(&oh->mtx); + /* + * A refcount of zero indicates that the tree changed + * under us, so fall through and try with the lock held. + */ u = oh->refcnt; if (u) oh->refcnt++; @@ -436,6 +449,7 @@ */ HSH_Copy(sp, noh); Lck_Lock(&hcb_mtx); + assert(noh->refcnt == 1); oh = hcb_insert(&hcb_root, noh, 1); if (oh == noh) { VSL_stats->hcb_insert++; @@ -450,7 +464,9 @@ #ifdef PHK fprintf(stderr, "hcb_lookup %d\n", __LINE__); #endif + Lck_Lock(&oh->mtx); oh->refcnt++; + Lck_Unlock(&oh->mtx); } Lck_Unlock(&hcb_mtx); return (oh); From phk at projects.linpro.no Tue Mar 3 09:33:27 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 3 Mar 2009 10:33:27 +0100 (CET) Subject: r3861 - trunk/varnish-cache/bin/varnishd Message-ID: <20090303093327.F25E71F74DA@projects.linpro.no> Author: phk Date: 2009-03-03 10:33:27 +0100 (Tue, 03 Mar 2009) New Revision: 3861 Modified: trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Minor bit of cleanup: initialize CLI from main() instead of mgt_run(). Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2009-03-02 17:29:45 UTC (rev 3860) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2009-03-03 09:33:27 UTC (rev 3861) @@ -37,11 +37,12 @@ struct cli; extern struct vev_base *mgt_evb; -extern unsigned d_flag; +extern unsigned d_flag; +extern pid_t mgt_pid; /* mgt_child.c */ -void mgt_run(const char *T_arg); -extern pid_t mgt_pid, child_pid; +extern pid_t child_pid; +void MGT_Run(void); void mgt_stop_child(void); /* mgt_cli.c */ Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2009-03-02 17:29:45 UTC (rev 3860) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2009-03-03 09:33:27 UTC (rev 3861) @@ -64,7 +64,6 @@ #include "vss.h" #include "vbm.h" -pid_t mgt_pid; pid_t child_pid = -1; static struct vbitmap *fd_map; @@ -89,7 +88,6 @@ [CH_DIED] = "died, (restarting)", }; -struct vev_base *mgt_evb; static struct vev *ev_poker; static struct vev *ev_listen; static struct vlu *vlu; @@ -499,23 +497,12 @@ */ void -mgt_run(const char *T_arg) +MGT_Run(void) { struct sigaction sac; struct vev *e; int i; - mgt_pid = getpid(); - - mgt_evb = vev_new_base(); - XXXAN(mgt_evb); - - if (d_flag) - mgt_cli_setup(0, 1, 1, "debug"); - - if (T_arg) - mgt_cli_telnet(T_arg); - e = vev_new(); XXXAN(e); e->sig = SIGTERM; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-03-02 17:29:45 UTC (rev 3860) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-03-03 09:33:27 UTC (rev 3861) @@ -58,6 +58,7 @@ #endif #include "vsb.h" +#include "vev.h" #include "vpf.h" #include "vsha256.h" @@ -76,10 +77,14 @@ #define INFTIM -1 #endif -struct heritage heritage; -volatile struct params *params; -unsigned d_flag = 0; +struct heritage heritage; +volatile struct params *params; +unsigned d_flag = 0; +pid_t mgt_pid; +struct vev_base *mgt_evb; + + /*--------------------------------------------------------------------*/ static void * @@ -605,8 +610,18 @@ if (pfh != NULL && vpf_write(pfh)) fprintf(stderr, "NOTE: Could not write PID file\n"); - mgt_run(T_arg); + mgt_pid = getpid(); + mgt_evb = vev_new_base(); + XXXAN(mgt_evb); + + if (d_flag) + mgt_cli_setup(0, 1, 1, "debug"); + if (T_arg) + mgt_cli_telnet(T_arg); + + MGT_Run(); + if (pfh != NULL) (void)vpf_remove(pfh); exit(0); From petter at projects.linpro.no Tue Mar 3 09:47:49 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Tue, 3 Mar 2009 10:47:49 +0100 (CET) Subject: r3862 - in trunk/varnish-tools/webgui: . Varnish Message-ID: <20090303094749.690B11F74FB@projects.linpro.no> Author: petter Date: 2009-03-03 10:47:49 +0100 (Tue, 03 Mar 2009) New Revision: 3862 Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm trunk/varnish-tools/webgui/Varnish/Util.pm trunk/varnish-tools/webgui/start.pl Log: Added the use of config file (at last),which is given as argument to web ui. If not given, it will check for /etc/varnish/webui.conf and then fall back to the default config. Added a new config value, 'document_root', which is the root of the web server and contains the templates, images and CSS. Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-03 09:33:27 UTC (rev 3861) +++ trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-03 09:47:49 UTC (rev 3862) @@ -135,10 +135,11 @@ defined($use_master_template) ? $use_master_template : 1; if ($content_template) { + my $document_root = get_config_value('document_root'); my %template_options = - (die_on_bad_params => 0, global_vars => 1, loop_context_vars => 1); + (die_on_bad_params => 0, global_vars => 1, loop_context_vars => 1, path => [$document_root]); if ($use_master_template) { - my $template_text = read_file("templates/master.tmpl"); + my $template_text = read_file("$document_root/templates/master.tmpl"); $template_text =~ s/CONTENT_TEMPLATE/$content_template/; my $template = HTML::Template->new_scalar_ref( \$template_text, Modified: trunk/varnish-tools/webgui/Varnish/Util.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/Util.pm 2009-03-03 09:33:27 UTC (rev 3861) +++ trunk/varnish-tools/webgui/Varnish/Util.pm 2009-03-03 09:47:49 UTC (rev 3862) @@ -8,7 +8,9 @@ use Algorithm::Diff; our @EXPORT = qw( + read_config set_config + print_config get_config_value read_file get_formatted_percentage @@ -29,6 +31,25 @@ my $error; my $log_handle; + sub read_config { + my ($filename) = @_; + + my $handle; + if (!open($handle, "<$filename")) { + die "Could not open config file $filename\n"; + } + + while (<$handle>) { + if (/^(\w+)\s*=\s*(.*?)$/) { + my $key = lc $1; + my $value = $2; + $config{$key} = $value; + } + } + + close($handle); + } + sub set_config { my ($config_ref) = @_; @@ -44,6 +65,13 @@ Varnish::DB->init($config{'db_filename'}); } + sub print_config { + print "Config:\n"; + while (my ($k, $v) = each(%config)) { + print "$k: $v\n"; + } + } + sub get_config_value { my ($key) = @_; Modified: trunk/varnish-tools/webgui/start.pl =================================================================== --- trunk/varnish-tools/webgui/start.pl 2009-03-03 09:33:27 UTC (rev 3861) +++ trunk/varnish-tools/webgui/start.pl 2009-03-03 09:47:49 UTC (rev 3862) @@ -14,9 +14,8 @@ use Varnish::Statistics; use Varnish::DB; - -# Configuration starts here -my %config = ( +my $global_config_filename = '/etc/varnish/webui.conf'; +my %default_config = ( # 'address' is the IP to bind to. If not set, it listens on all. # address => localhost, @@ -39,14 +38,27 @@ large_graph_height => 500, # 'log_filename' is the filename to log errors and information about actions done in the GUI - log_filename => "varnish.log", + log_filename => 'varnish.log', # 'db_filename' is the sqlite3 database created with the SQL outputed from create_db_data.pl db_filename => 'varnish.db', + +# 'document_root' is the root of the templates and css file + document_root => '.', ); -# End of configuration -set_config(\%config); +set_config(\%default_config); +my $config_filename; +if (@ARGV == 1 && -f $ARGV[0]) { + $config_filename = $ARGV[0]; +} +elsif (-f $global_config_filename) { + $config_filename = $global_config_filename; +} +if ($config_filename) { + print "Using config file $config_filename\n"; + read_config($config_filename); +} # catch interupt to stop the daemon $SIG{'INT'} = sub { @@ -59,8 +71,8 @@ }; log_info("Starting HTTP daemon"); -my $daemon = HTTP::Daemon->new( LocalPort => $config{'port'}, - LocalAddr => $config{'address'}, +my $daemon = HTTP::Daemon->new( LocalPort => get_config_value('port'), + LocalAddr => get_config_value('address'), ReuseAddr => 1 ); if (!$daemon) { @@ -72,6 +84,7 @@ my $running :shared; $running = 1; my $data_collector_handle = threads->create('data_collector_thread'); +my $document_root = get_config_value('document_root'); while (my $connection = $daemon->accept) { REQUEST: while (my $request = $connection->get_request) { @@ -81,7 +94,7 @@ $request->uri =~ m{/(.*?\.ico)}) { my $filename = $1; - $connection->send_file_response($filename); + $connection->send_file_response("$document_root/$filename"); next REQUEST; } elsif ($request->uri =~ m{/(.*?\.css)}) { @@ -115,7 +128,7 @@ sub data_collector_thread { my $url = $daemon->url . "collect_data"; - my $interval = $config{'poll_interval'}; + my $interval = get_config_value('poll_interval'); log_info("Data collector thread started. Polling URL $url at $interval seconds interval"); sleep 1; # wait for the server to come up From petter at projects.linpro.no Tue Mar 3 09:48:43 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Tue, 3 Mar 2009 10:48:43 +0100 (CET) Subject: r3863 - trunk/varnish-tools/webgui Message-ID: <20090303094843.5F0C31F74DA@projects.linpro.no> Author: petter Date: 2009-03-03 10:48:43 +0100 (Tue, 03 Mar 2009) New Revision: 3863 Added: trunk/varnish-tools/webgui/varnish-webui.pl Removed: trunk/varnish-tools/webgui/start.pl Log: Renamed the program to something more logical. Deleted: trunk/varnish-tools/webgui/start.pl =================================================================== --- trunk/varnish-tools/webgui/start.pl 2009-03-03 09:47:49 UTC (rev 3862) +++ trunk/varnish-tools/webgui/start.pl 2009-03-03 09:48:43 UTC (rev 3863) @@ -1,145 +0,0 @@ -#!/usr/bin/perl -use threads; -use threads::shared; -use strict; -use warnings; -use HTTP::Daemon; -use HTTP::Status; -use HTTP::Request; -use LWP::UserAgent; -use Varnish::Util; -use Varnish::RequestHandler; -use Varnish::NodeManager; -use Varnish::Node; -use Varnish::Statistics; -use Varnish::DB; - -my $global_config_filename = '/etc/varnish/webui.conf'; -my %default_config = ( -# 'address' is the IP to bind to. If not set, it listens on all. -# address => localhost, - -# 'port' is the port of the web server - port => 8000, - -# 'poll_intervall' is the polling interval for the statistics - poll_interval => 5, - -# 'restricted' gives a restricted version of the web GUI, disabling the user -# from changing any values - restricted => 0, - -# 'graph_width' and 'graph_height' are width and height for the graphs in 'View stats' - graph_width => 250, - graph_height => 125, -# 'large_graph_width' and 'large_graph_height' are width and height for the full size graph -# when clicking a stat graph in 'View stats' - large_graph_width => 1000, - large_graph_height => 500, - -# 'log_filename' is the filename to log errors and information about actions done in the GUI - log_filename => 'varnish.log', - -# 'db_filename' is the sqlite3 database created with the SQL outputed from create_db_data.pl - db_filename => 'varnish.db', - -# 'document_root' is the root of the templates and css file - document_root => '.', -); - -set_config(\%default_config); -my $config_filename; -if (@ARGV == 1 && -f $ARGV[0]) { - $config_filename = $ARGV[0]; -} -elsif (-f $global_config_filename) { - $config_filename = $global_config_filename; -} -if ($config_filename) { - print "Using config file $config_filename\n"; - read_config($config_filename); -} - -# catch interupt to stop the daemon -$SIG{'INT'} = sub { - print "Interrupt detected.\n"; -}; - -# ignore the occational sigpipe -$SIG{'PIPE'} = sub { -# print "Pipe ignored\n"; -}; - -log_info("Starting HTTP daemon"); -my $daemon = HTTP::Daemon->new( LocalPort => get_config_value('port'), - LocalAddr => get_config_value('address'), - ReuseAddr => 1 ); - -if (!$daemon) { - log_error("Could not start HTTP daemon"); - die "Could not start web server"; -} -log_info("HTTP daemon started with URL " . $daemon->url); -print "Web server started with URL: " . $daemon->url, "\n"; -my $running :shared; -$running = 1; -my $data_collector_handle = threads->create('data_collector_thread'); -my $document_root = get_config_value('document_root'); -while (my $connection = $daemon->accept) { - REQUEST: - while (my $request = $connection->get_request) { - $connection->force_last_request; -# print "Request for: " . $request->uri . "\n"; - if ($request->uri =~ m{/(.*?\.png)} || - $request->uri =~ m{/(.*?\.ico)}) { - my $filename = $1; - - $connection->send_file_response("$document_root/$filename"); - next REQUEST; - } - elsif ($request->uri =~ m{/(.*?\.css)}) { - my $filename = $1; - - $connection->send_basic_header(); - print $connection "Content-Type: text/css"; - $connection->send_crlf(); - $connection->send_crlf(); - $connection->send_file($filename); - next REQUEST; - } - - my $request_handler = Varnish::RequestHandler->new(\$request, $connection); - $request_handler->process(); - - my $response = HTTP::Response->new(200); - $response->header( $request_handler->get_response_header() ); - $response->content( $request_handler->get_response_content() ); - $connection->send_response($response); - } - $connection->close(); - undef($connection); -} -log_info("Shutting down web server"); -$daemon->close(); -$running = 0; -Varnish::DB->finish(); -log_info("Stopping data collector thread"); -$data_collector_handle->join(); - -sub data_collector_thread { - my $url = $daemon->url . "collect_data"; - my $interval = get_config_value('poll_interval'); - - log_info("Data collector thread started. Polling URL $url at $interval seconds interval"); - sleep 1; # wait for the server to come up - while (1) { - my $user_agent = LWP::UserAgent->new; - - $user_agent->timeout(10); - my $response = $user_agent->get($url); - sleep($interval); - - last if (!$running); - } - print "Data collector thread stopped.\n"; -} Copied: trunk/varnish-tools/webgui/varnish-webui.pl (from rev 3862, trunk/varnish-tools/webgui/start.pl) =================================================================== --- trunk/varnish-tools/webgui/varnish-webui.pl (rev 0) +++ trunk/varnish-tools/webgui/varnish-webui.pl 2009-03-03 09:48:43 UTC (rev 3863) @@ -0,0 +1,145 @@ +#!/usr/bin/perl +use threads; +use threads::shared; +use strict; +use warnings; +use HTTP::Daemon; +use HTTP::Status; +use HTTP::Request; +use LWP::UserAgent; +use Varnish::Util; +use Varnish::RequestHandler; +use Varnish::NodeManager; +use Varnish::Node; +use Varnish::Statistics; +use Varnish::DB; + +my $global_config_filename = '/etc/varnish/webui.conf'; +my %default_config = ( +# 'address' is the IP to bind to. If not set, it listens on all. +# address => localhost, + +# 'port' is the port of the web server + port => 8000, + +# 'poll_intervall' is the polling interval for the statistics + poll_interval => 5, + +# 'restricted' gives a restricted version of the web GUI, disabling the user +# from changing any values + restricted => 0, + +# 'graph_width' and 'graph_height' are width and height for the graphs in 'View stats' + graph_width => 250, + graph_height => 125, +# 'large_graph_width' and 'large_graph_height' are width and height for the full size graph +# when clicking a stat graph in 'View stats' + large_graph_width => 1000, + large_graph_height => 500, + +# 'log_filename' is the filename to log errors and information about actions done in the GUI + log_filename => 'varnish.log', + +# 'db_filename' is the sqlite3 database created with the SQL outputed from create_db_data.pl + db_filename => 'varnish.db', + +# 'document_root' is the root of the templates and css file + document_root => '.', +); + +set_config(\%default_config); +my $config_filename; +if (@ARGV == 1 && -f $ARGV[0]) { + $config_filename = $ARGV[0]; +} +elsif (-f $global_config_filename) { + $config_filename = $global_config_filename; +} +if ($config_filename) { + print "Using config file $config_filename\n"; + read_config($config_filename); +} + +# catch interupt to stop the daemon +$SIG{'INT'} = sub { + print "Interrupt detected.\n"; +}; + +# ignore the occational sigpipe +$SIG{'PIPE'} = sub { +# print "Pipe ignored\n"; +}; + +log_info("Starting HTTP daemon"); +my $daemon = HTTP::Daemon->new( LocalPort => get_config_value('port'), + LocalAddr => get_config_value('address'), + ReuseAddr => 1 ); + +if (!$daemon) { + log_error("Could not start HTTP daemon"); + die "Could not start web server"; +} +log_info("HTTP daemon started with URL " . $daemon->url); +print "Web server started with URL: " . $daemon->url, "\n"; +my $running :shared; +$running = 1; +my $data_collector_handle = threads->create('data_collector_thread'); +my $document_root = get_config_value('document_root'); +while (my $connection = $daemon->accept) { + REQUEST: + while (my $request = $connection->get_request) { + $connection->force_last_request; +# print "Request for: " . $request->uri . "\n"; + if ($request->uri =~ m{/(.*?\.png)} || + $request->uri =~ m{/(.*?\.ico)}) { + my $filename = $1; + + $connection->send_file_response("$document_root/$filename"); + next REQUEST; + } + elsif ($request->uri =~ m{/(.*?\.css)}) { + my $filename = $1; + + $connection->send_basic_header(); + print $connection "Content-Type: text/css"; + $connection->send_crlf(); + $connection->send_crlf(); + $connection->send_file($filename); + next REQUEST; + } + + my $request_handler = Varnish::RequestHandler->new(\$request, $connection); + $request_handler->process(); + + my $response = HTTP::Response->new(200); + $response->header( $request_handler->get_response_header() ); + $response->content( $request_handler->get_response_content() ); + $connection->send_response($response); + } + $connection->close(); + undef($connection); +} +log_info("Shutting down web server"); +$daemon->close(); +$running = 0; +Varnish::DB->finish(); +log_info("Stopping data collector thread"); +$data_collector_handle->join(); + +sub data_collector_thread { + my $url = $daemon->url . "collect_data"; + my $interval = get_config_value('poll_interval'); + + log_info("Data collector thread started. Polling URL $url at $interval seconds interval"); + sleep 1; # wait for the server to come up + while (1) { + my $user_agent = LWP::UserAgent->new; + + $user_agent->timeout(10); + my $response = $user_agent->get($url); + sleep($interval); + + last if (!$running); + } + print "Data collector thread stopped.\n"; +} From petter at projects.linpro.no Tue Mar 3 11:18:42 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Tue, 3 Mar 2009 12:18:42 +0100 (CET) Subject: r3864 - trunk/varnish-tools/webgui/Varnish Message-ID: <20090303111842.8CD2B1F74E1@projects.linpro.no> Author: petter Date: 2009-03-03 12:18:41 +0100 (Tue, 03 Mar 2009) New Revision: 3864 Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm Log: Add some polish to the graphs by using sensible font sizes and x ticks. Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-03 09:48:43 UTC (rev 3863) +++ trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-03 11:18:41 UTC (rev 3864) @@ -10,7 +10,7 @@ use Varnish::Node; use Varnish::Statistics; use GD::Graph::lines; -use GD qw(gdTinyFont); +use GD qw(gdTinyFont gdSmallFont gdLargeFont gdGiantFont); use POSIX qw(strftime); use List::Util qw(first); use Socket; @@ -1110,26 +1110,28 @@ use_delta => $param{'custom_delta'}, }, ); + + my $x_tick_factor = $param{'width'} / 300; my %time_span_graph_parameters = ( minute => { x_label => 'Time', - x_tick_number => 4, # need to be set to make x_number_format work + x_tick_number => 4 * $x_tick_factor, # need to be set to make x_number_format work x_number_format => sub { return strftime("%H:%M:%S", localtime($_[0])); }, }, hour => { x_label => 'Time', - x_tick_number => 6, # need to be set to make x_number_format work + x_tick_number => 6 * $x_tick_factor, # need to be set to make x_number_format work x_number_format => sub { return strftime("%H:%M", localtime($_[0])); }, }, day => { x_label => 'Time', - x_tick_number => 4, # need to be set to make x_number_format work - x_number_format => sub { return strftime("%H", localtime($_[0])); }, + x_tick_number => 6 * $x_tick_factor, # need to be set to make x_number_format work + x_number_format => sub { return strftime("%H:%M", localtime($_[0])); }, }, week => { x_label => 'Time', - x_tick_number => 7, # need to be set to make x_number_format work - x_number_format => sub { return strftime("%d", localtime($_[0])); }, + x_tick_number => 7 * $x_tick_factor, # need to be set to make x_number_format work + x_number_format => sub { return strftime("%d.%m", localtime($_[0])); }, }, month => { x_label => 'Time', @@ -1165,10 +1167,20 @@ return if (!$data_ref); my $graph = GD::Graph::lines->new($param{'width'}, $param{'height'}); - if ($param{'width'} < 300) { - $graph->set_title_font(gdTinyFont); - $graph->set_legend_font(gdTinyFont); + my $title_font = gdSmallFont; + my $axis_font = gdTinyFont; + my $label_font = gdSmallFont; + if ($param{'width'} > 300) { + $title_font = gdGiantFont; + $axis_font = gdSmallFont; + $label_font = gdLargeFont; } + $graph->set_title_font($title_font); + $graph->set_legend_font($title_font); + $graph->set_x_label_font($label_font); + $graph->set_y_label_font($label_font); + $graph->set_x_axis_font($axis_font); + $graph->set_y_axis_font($axis_font); $graph->set((%{$graph_info{$param{'type'}}->{'graph_parameter'}}, %{$time_span_graph_parameters{$param{'time_span'}}}), dclrs => ["#990200"], From phk at projects.linpro.no Tue Mar 3 11:25:46 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 3 Mar 2009 12:25:46 +0100 (CET) Subject: r3865 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090303112546.C6E531F74E4@projects.linpro.no> Author: phk Date: 2009-03-03 12:25:46 +0100 (Tue, 03 Mar 2009) New Revision: 3865 Modified: trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/mgt_cli.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/cli.h trunk/varnish-cache/include/cli_common.h Log: Add support for authenticating CLI telnet connections Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2009-03-03 11:18:41 UTC (rev 3864) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2009-03-03 11:25:46 UTC (rev 3865) @@ -52,6 +52,7 @@ void mgt_cli_start_child(int fdi, int fdo); void mgt_cli_stop_child(void); void mgt_cli_telnet(const char *T_arg); +void mgt_cli_secret(const char *S_arg); /* mgt_param.c */ void MCF_ParamSync(void); Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-03-03 11:18:41 UTC (rev 3864) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-03-03 11:25:46 UTC (rev 3865) @@ -57,12 +57,14 @@ #include "mgt.h" #include "mgt_cli.h" #include "vev.h" +#include "vsha256.h" #include "shmlog.h" #include "vlu.h" #include "vss.h" static int cli_i = -1, cli_o = -1; +static const char *secret_file; struct telnet { int fd; @@ -70,6 +72,19 @@ VTAILQ_ENTRY(telnet) list; }; +struct cli_port { + unsigned magic; +#define CLI_PORT_MAGIC 0x5791079f + struct vev *ev; + int fdi; + int fdo; + int verbose; + struct vlu *vlu; + struct cli cli[1]; + char *name; + char challenge[34]; +}; + static VTAILQ_HEAD(,telnet) telnets = VTAILQ_HEAD_INITIALIZER(telnets); static void telnet_close_all(void); static void telnet_close_one(int fd); @@ -166,7 +181,6 @@ /*--------------------------------------------------------------------*/ - static void mcf_panic(struct cli *cli, const char * const *av, void *priv) { @@ -248,20 +262,94 @@ /* XXX: kick any users */ } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Validate the authentication + */ -struct cli_port { - unsigned magic; -#define CLI_PORT_MAGIC 0x5791079f - struct vev *ev; - int fdi; - int fdo; - int verbose; - struct vlu *vlu; - struct cli cli[1]; - char *name; +static void +mcf_auth(struct cli *cli, const char *const *av, void *priv) +{ + char buf[1025]; + int i, fd; + struct SHA256Context sha256ctx; + unsigned char digest[SHA256_LEN]; + struct cli_port *cp; + + AN(av[2]); + CAST_OBJ_NOTNULL(cp, cli->priv, CLI_PORT_MAGIC); + (void)priv; + AN(secret_file); + fd = open(secret_file, O_RDONLY); + if (fd < 0) { + cli_out(cli, "Cannot open secret file (%s)\n", + strerror(errno)); + cli_result(cli, CLIS_CANT); + return; + } + i = read(fd, buf, sizeof buf); + if (i == 0) { + cli_out(cli, "Empty secret file"); + cli_result(cli, CLIS_CANT); + return; + } + if (i < 0) { + cli_out(cli, "Read error on secret file (%s)\n", + strerror(errno)); + cli_result(cli, CLIS_CANT); + return; + } + if (i == sizeof buf) { + cli_out(cli, "Secret file too long (> %d)\n", + sizeof buf - 1); + cli_result(cli, CLIS_CANT); + return; + } + buf[i] = '\0'; + AZ(close(fd)); + SHA256_Init(&sha256ctx); + SHA256_Update(&sha256ctx, cp->challenge, strlen(cp->challenge)); + SHA256_Update(&sha256ctx, buf, i); + SHA256_Update(&sha256ctx, cp->challenge, strlen(cp->challenge)); + SHA256_Final(digest, &sha256ctx); + for (i = 0; i < SHA256_LEN; i++) + sprintf(buf + i + i, "%02x", digest[i]); + if (strcasecmp(buf, av[2])) { + cli_result(cli, CLIS_UNKNOWN); + return; + } + cp->challenge[0] = '\0'; + cli_result(cli, CLIS_OK); + if (params->cli_banner) + mcf_banner(cli, av, priv); +} + +static struct cli_proto cli_auth[] = { + { CLI_HELP, mcf_help, cli_auth }, + { CLI_AUTH, mcf_auth, NULL }, + { CLI_QUIT, mcf_close, NULL}, + { NULL } }; +/*-------------------------------------------------------------------- + * Generate a random challenge + */ + +static void +mgt_cli_challenge(struct cli_port *cp) +{ + int i; + + for (i = 0; i + 2 < sizeof cp->challenge; i++) + cp->challenge[i] = (random() % 26) + 'a'; + cp->challenge[i++] = '\n'; + cp->challenge[i] = '\0'; + cli_out(cp->cli, "%s", cp->challenge); + cli_out(cp->cli, "\nAuthentication required.\n"); + cli_result(cp->cli, CLIS_AUTH); +} + +/*--------------------------------------------------------------------*/ + static int mgt_cli_vlu(void *priv, const char *p) { @@ -281,31 +369,39 @@ if (*p == '\0') return (0); - cli_dispatch(cp->cli, cli_proto, p); - if (cp->cli->result == CLIS_UNKNOWN) - cli_dispatch(cp->cli, cli_debug, p); - if (cp->cli->result == CLIS_UNKNOWN) { - /* - * Command not recognized in master, try cacher if it is - * running. - */ - vsb_clear(cp->cli->sb); - cp->cli->result = CLIS_OK; - if (cli_o <= 0) { - cli_result(cp->cli, CLIS_UNKNOWN); - cli_out(cp->cli, - "Unknown request in manager process " - "(child not running).\n" - "Type 'help' for more info."); - } else { - i = write(cli_o, p, strlen(p)); - xxxassert(i == strlen(p)); - i = write(cli_o, "\n", 1); - xxxassert(i == 1); - (void)cli_readres(cli_i, &u, &q, params->cli_timeout); - cli_result(cp->cli, u); - cli_out(cp->cli, "%s", q); - free(q); + if (secret_file != NULL && cp->challenge[0] != '\0') { + /* Authentication not yet passed */ + cli_dispatch(cp->cli, cli_auth, p); + if (cp->cli->result == CLIS_UNKNOWN) + mgt_cli_challenge(cp); + } else { + cli_dispatch(cp->cli, cli_proto, p); + if (cp->cli->result == CLIS_UNKNOWN) + cli_dispatch(cp->cli, cli_debug, p); + if (cp->cli->result == CLIS_UNKNOWN) { + /* + * Command not recognized in master, try cacher if it is + * running. + */ + vsb_clear(cp->cli->sb); + cp->cli->result = CLIS_OK; + if (cli_o <= 0) { + cli_result(cp->cli, CLIS_UNKNOWN); + cli_out(cp->cli, + "Unknown request in manager process " + "(child not running).\n" + "Type 'help' for more info."); + } else { + i = write(cli_o, p, strlen(p)); + xxxassert(i == strlen(p)); + i = write(cli_o, "\n", 1); + xxxassert(i == 1); + (void)cli_readres(cli_i, + &u, &q, params->cli_timeout); + cli_result(cp->cli, u); + cli_out(cp->cli, "%s", q); + free(q); + } } } vsb_finish(cp->cli->sb); @@ -403,8 +499,16 @@ cp->cli->sb = vsb_newauto(); XXXAN(cp->cli->sb); + cp->cli->priv = cp; - if (params->cli_banner) + /* + * If we have a secret file authenticate all CLI connections + * except the stdin/stdout debug port. + */ + if (cp->fdi != 0 && secret_file != NULL) { + mgt_cli_challenge(cp); + (void)VLU_Data("auth -\n", -1, cp->vlu); + } else if (params->cli_banner) (void)VLU_Data("banner\n", -1, cp->vlu); cp->ev = calloc(sizeof *cp->ev, 1); @@ -433,7 +537,6 @@ } } - static void telnet_close_all() { @@ -490,6 +593,31 @@ } void +mgt_cli_secret(const char *S_arg) +{ + int i, fd; + char buf[BUFSIZ]; + + + srandomdev(); + fd = open(S_arg, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Can not open secret-file \"%s\"\n", S_arg); + exit (2); + } + i = read(fd, buf, sizeof buf); + if (i == 0) { + fprintf(stderr, "Empty secret-file \"%s\"\n", S_arg); + exit (2); + } + if (i < 0) { + fprintf(stderr, "Can not read secret-file \"%s\"\n", S_arg); + exit (2); + } + secret_file = S_arg; +} + +void mgt_cli_telnet(const char *T_arg) { struct vss_addr **ta; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-03-03 11:18:41 UTC (rev 3864) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-03-03 11:25:46 UTC (rev 3865) @@ -207,6 +207,8 @@ fprintf(stderr, FMT, "", " -s file,,,"); fprintf(stderr, FMT, "-t", "Default TTL"); + fprintf(stderr, FMT, "-S secret-file", + "Secret file for CLI authentication"); fprintf(stderr, FMT, "-T address:port", "Telnet listen address and port"); fprintf(stderr, FMT, "-V", "version"); @@ -404,6 +406,7 @@ const char *h_arg = "classic"; const char *n_arg = NULL; const char *P_arg = NULL; + const char *S_arg = NULL; const char *s_arg = "file"; int s_arg_given = 0; const char *T_arg = NULL; @@ -445,7 +448,7 @@ cli_check(cli); while ((o = getopt(argc, argv, - "a:b:Cdf:Fg:h:l:n:P:p:s:T:t:u:Vw:")) != -1) + "a:b:Cdf:Fg:h:l:n:P:p:S:s:T:t:u:Vw:")) != -1) switch (o) { case 'a': MCF_ParamSet(cli, "listen_address", optarg); @@ -497,6 +500,9 @@ case 't': MCF_ParamSet(cli, "default_ttl", optarg); break; + case 'S': + S_arg = optarg; + break; case 'T': T_arg = optarg; break; @@ -617,7 +623,9 @@ if (d_flag) mgt_cli_setup(0, 1, 1, "debug"); - if (T_arg) + if (S_arg != NULL) + mgt_cli_secret(S_arg); + if (T_arg != NULL) mgt_cli_telnet(T_arg); MGT_Run(); Modified: trunk/varnish-cache/include/cli.h =================================================================== --- trunk/varnish-cache/include/cli.h 2009-03-03 11:18:41 UTC (rev 3864) +++ trunk/varnish-cache/include/cli.h 2009-03-03 11:25:46 UTC (rev 3865) @@ -231,6 +231,12 @@ "\tPrint welcome banner.", \ 0, 0 +#define CLI_AUTH \ + "auth", \ + "auth response", \ + "\tAuthenticate.", \ + 1, 1 + #define CLI_HIDDEN(foo, min_arg, max_arg) \ foo, NULL, NULL, min_arg, max_arg, @@ -245,6 +251,7 @@ CLIS_TOOFEW = 104, CLIS_TOOMANY = 105, CLIS_PARAM = 106, + CLIS_AUTH = 107, CLIS_OK = 200, CLIS_CANT = 300, CLIS_COMMS = 400, Modified: trunk/varnish-cache/include/cli_common.h =================================================================== --- trunk/varnish-cache/include/cli_common.h 2009-03-03 11:18:41 UTC (rev 3864) +++ trunk/varnish-cache/include/cli_common.h 2009-03-03 11:25:46 UTC (rev 3865) @@ -33,6 +33,7 @@ /* XXX: should be MINI_OBJ */ struct vsb *sb; enum cli_status_e result; + void *priv; }; int cli_writeres(int fd, const struct cli *cli); From phk at projects.linpro.no Tue Mar 3 12:23:11 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 3 Mar 2009 13:23:11 +0100 (CET) Subject: r3866 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish Message-ID: <20090303122311.B9FEB1F74E1@projects.linpro.no> Author: phk Date: 2009-03-03 13:23:11 +0100 (Tue, 03 Mar 2009) New Revision: 3866 Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c trunk/varnish-cache/include/vlu.h trunk/varnish-cache/lib/libvarnish/vlu.c Log: Add minimal facility for dealing with TELNET option negotiation in a way that telnet clients don't get confused about: Return WONT on to all DO or DONT and ignore everything else. Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-03-03 11:25:46 UTC (rev 3865) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-03-03 12:23:11 UTC (rev 3866) @@ -501,6 +501,10 @@ XXXAN(cp->cli->sb); cp->cli->priv = cp; + /* Deal with TELNET options */ + if (cp->fdi != 0) + VLU_SetTelnet(cp->vlu, cp->fdo); + /* * If we have a secret file authenticate all CLI connections * except the stdin/stdout debug port. Modified: trunk/varnish-cache/include/vlu.h =================================================================== --- trunk/varnish-cache/include/vlu.h 2009-03-03 11:25:46 UTC (rev 3865) +++ trunk/varnish-cache/include/vlu.h 2009-03-03 12:23:11 UTC (rev 3866) @@ -38,5 +38,6 @@ int VLU_File(FILE *f, struct vlu *l); int VLU_Data(const void *ptr, int len, struct vlu *l); void VLU_Destroy(struct vlu *l); +void VLU_SetTelnet(struct vlu *l, int fd); #endif Modified: trunk/varnish-cache/lib/libvarnish/vlu.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vlu.c 2009-03-03 11:25:46 UTC (rev 3865) +++ trunk/varnish-cache/lib/libvarnish/vlu.c 2009-03-03 12:23:11 UTC (rev 3866) @@ -46,6 +46,7 @@ unsigned bufl; unsigned bufp; void *priv; + int telnet; vlu_f *func; }; @@ -66,11 +67,20 @@ FREE_OBJ(l); l = NULL; } + l->telnet = -1; } return (l); } void +VLU_SetTelnet(struct vlu *l, int fd) +{ + CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC); + assert(fd >= 0); + l->telnet = fd; +} + +void VLU_Destroy(struct vlu *l) { @@ -80,6 +90,51 @@ } static int +vlu_dotelnet(struct vlu *l, char *p) +{ + char *e; + char tno[3]; + int i; + + e = l->buf + l->bufp; + assert(p >= l->buf && p < e); + assert(*p == (char)255); + + /* We need at least two characters */ + if (p == e - 1) + return (1); + + /* And three for will/wont/do/dont */ + if (p[1] >= (char)251 && p[1] <= (char)254 && p == e - 2) + return (1); + + switch (p[1]) { + case (char)251: /* WILL */ + case (char)252: /* WONT */ + /* Ignore these */ + i = 3; + break; + case (char)253: /* DO */ + case (char)254: /* DONT */ + /* Return WONT for these */ + memcpy(tno, p, 3); + tno[1] = (char)252; + write(l->telnet, tno, 3); + i = 3; + break; + default: + /* Ignore the rest */ + /* XXX: only p[1] >= 240 ? */ + i = 2; + } + + /* Remove telnet sequence from buffer */ + memmove(p, p + i, 1 + e - (p + i)); + l->bufp -= i; + return (0); +} + +static int LineUpProcess(struct vlu *l) { char *p, *q; @@ -88,9 +143,13 @@ l->buf[l->bufp] = '\0'; for (p = l->buf; *p != '\0'; p = q) { /* Find first CR or NL */ - for (q = p; *q != '\0'; q++) + for (q = p; *q != '\0'; q++) { + while (l->telnet >= 0 && *q == (char)255) + if (vlu_dotelnet(l, q)) + return (0); if (*q == '\n' || *q == '\r') break; + } if (*q == '\0') break; *q++ = '\0'; From phk at phk.freebsd.dk Tue Mar 3 12:24:13 2009 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 03 Mar 2009 12:24:13 +0000 Subject: r3866 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish In-Reply-To: Your message of "Tue, 03 Mar 2009 13:23:11 +0100." <20090303122311.B9FEB1F74E1@projects.linpro.no> Message-ID: <60444.1236083053@critter.freebsd.dk> In message <20090303122311.B9FEB1F74E1 at projects.linpro.no>, phk at projects.linpro .no writes: >Date: 2009-03-03 13:23:11 +0100 (Tue, 03 Mar 2009) >New Revision: 3866 > >Log: >Add minimal facility for dealing with TELNET option negotiation >in a way that telnet clients don't get confused about: > >Return WONT on to all DO or DONT and ignore everything else. This should prevent a telnet(1) client from hanging if you press CTRL-C by accident. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Tue Mar 3 12:29:55 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 3 Mar 2009 13:29:55 +0100 (CET) Subject: r3867 - trunk/varnish-cache/lib/libvarnish Message-ID: <20090303122955.9A69828370@projects.linpro.no> Author: phk Date: 2009-03-03 13:29:55 +0100 (Tue, 03 Mar 2009) New Revision: 3867 Modified: trunk/varnish-cache/lib/libvarnish/vlu.c Log: Flexelint nitpickery. Modified: trunk/varnish-cache/lib/libvarnish/vlu.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vlu.c 2009-03-03 12:23:11 UTC (rev 3866) +++ trunk/varnish-cache/lib/libvarnish/vlu.c 2009-03-03 12:29:55 UTC (rev 3867) @@ -62,12 +62,12 @@ l->func = func; l->priv = priv; l->bufl = bufsize - 1; + l->telnet = -1; l->buf = malloc(l->bufl + 1); if (l->buf == NULL) { FREE_OBJ(l); l = NULL; } - l->telnet = -1; } return (l); } @@ -119,7 +119,7 @@ /* Return WONT for these */ memcpy(tno, p, 3); tno[1] = (char)252; - write(l->telnet, tno, 3); + (void)write(l->telnet, tno, 3); i = 3; break; default: @@ -198,7 +198,7 @@ VLU_Data(const void *ptr, int len, struct vlu *l) { const char *p; - int i; + int i = 0; p = ptr; CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC); From tfheen at projects.linpro.no Tue Mar 3 14:14:23 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 3 Mar 2009 15:14:23 +0100 (CET) Subject: r3868 - in trunk/varnish-tools: . python-varnishadm Message-ID: <20090303141423.E74691F7497@projects.linpro.no> Author: tfheen Date: 2009-03-03 15:14:23 +0100 (Tue, 03 Mar 2009) New Revision: 3868 Added: trunk/varnish-tools/python-varnishadm/ trunk/varnish-tools/python-varnishadm/setup.py trunk/varnish-tools/python-varnishadm/test.py trunk/varnish-tools/python-varnishadm/varnishadmin.py Log: Add python class to emulate varnishadm Added: trunk/varnish-tools/python-varnishadm/setup.py =================================================================== --- trunk/varnish-tools/python-varnishadm/setup.py (rev 0) +++ trunk/varnish-tools/python-varnishadm/setup.py 2009-03-03 14:14:23 UTC (rev 3868) @@ -0,0 +1,6 @@ +from distutils.core import setup +setup(name='varnishadmin', + version='0.1', + py_modules=['varnishadmin'], + ) + Added: trunk/varnish-tools/python-varnishadm/test.py =================================================================== --- trunk/varnish-tools/python-varnishadm/test.py (rev 0) +++ trunk/varnish-tools/python-varnishadm/test.py 2009-03-03 14:14:23 UTC (rev 3868) @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Yes, this is very much a hack, but it's something... +from varnishadmin import VarnishAdmin + +v = VarnishAdmin() +print v.status() +print v.status() +print v.status() +print v.status() +print v.purge('^/test1$') +print v.status() +print v.purge('^/test2$') +print v.status() +print v.stats() +print v.purge('^/$') +print v.showparam("backend_http11") +#print v.allparams() +print v.setparam("backend_http11","on") +print v.showparam("backend_http11") + Added: trunk/varnish-tools/python-varnishadm/varnishadmin.py =================================================================== --- trunk/varnish-tools/python-varnishadm/varnishadmin.py (rev 0) +++ trunk/varnish-tools/python-varnishadm/varnishadmin.py 2009-03-03 14:14:23 UTC (rev 3868) @@ -0,0 +1,136 @@ +""" +Copyright (c) 2009 Redpill Linpro AB +All rights reserved. + +Author: Magnus Hagander + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer + in this position and unchanged. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +$Id$ + +""" + +import socket + +class VarnishAdmin(object): + """ + VarnishAdmin is a thin wrapper class around the admin interface to + varnish (see www.varnish-cache.org). At this point, it contains + the most common operations only - use the "help" command in the + admin interface to find out about other availble commands. + """ + def __init__(self, port=6082, host="localhost"): + self.port = port + self.host = host + self.file = None + + def __connect(self): + """ If a connection has not yet been made, make one. """ + if self.file: + return + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.host, self.port)) + self.file = sock.makefile() + sock.close() # it's been dup()ed into self.file + + def disconnect(self): + """ Disconnect from the varnish server. Will automatically + reconnect if needed later. """ + self.file.close() + self.file = None + + def __command(self, command): + """ Execute a command and return a string with the full + response from the varnish server. """ + self.__connect() + self.file.write("%s\n" % command) + self.file.flush() + + status = self.file.readline() + (code, blen) = status.split() + + msg = self.file.read(int(blen)+1) # 1 extra for newline + + if code != "200": + raise Exception("Error code %s returned from Varnish, message is '%s'" % (code, msg)) + return msg.rstrip() + + def status(self): + """ Get the status of the server. """ + return self.__command("status") + + def start(self): + """ Start the server (if stopped). """ + return self.__command("start") + + def stop(self): + """ Stop the server (if started). """ + return self.__command("stop") + + def purge(self, purgere): + """ Purge URLs matching a regexp from the cache. """ + return self.__command("url.purge %s" % purgere) + + def stats(self): + """ Get all statistics from the server as a + dictionary. """ + d = {} + for (val,key) in [l.strip().split(None,1) for l in + self.__command("stats").splitlines()]: + d[key] = int(val) + return d + + def __parseparam(self, param): + """ Parse parameter output into dictionary of dictionaries. """ + params = {} + current = "" + for l in self.__command("param.show %s" % param).splitlines(): + if not l.strip(): + continue + if l.startswith(" "): + # Starts with space, append to the current one + params[current]['description'] += l.lstrip() + " " + else: + # Starts with something else, this is a new parameter + (current, txt) = l.split(None, 1) + params[current] = { + 'value': txt, + 'description': '', + } + return params + + def showparam(self, param): + """ Return the value of a single parameter as a dictionary with + keys 'description' and 'value'. """ + return self.__parseparam(param)[param] + + def allparams(self): + """ Return a dictionary containing all parameters. The + dictionary is indexed by parameter name, and each entry + is a dictionary with keys 'description' and 'value'. """ + return self.__parseparam("-l") + + def setparam(self, param, value): + """ Set the value of a parameter. """ + return self.__command("param.set %s %s" % (param, value)) + From petter at projects.linpro.no Tue Mar 3 14:33:57 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Tue, 3 Mar 2009 15:33:57 +0100 (CET) Subject: r3869 - trunk/varnish-tools/webgui Message-ID: <20090303143357.B007128396@projects.linpro.no> Author: petter Date: 2009-03-03 15:33:57 +0100 (Tue, 03 Mar 2009) New Revision: 3869 Added: trunk/varnish-tools/webgui/example.conf Log: Example config file with default values for running it from a working directory. Added: trunk/varnish-tools/webgui/example.conf =================================================================== --- trunk/varnish-tools/webgui/example.conf (rev 0) +++ trunk/varnish-tools/webgui/example.conf 2009-03-03 14:33:57 UTC (rev 3869) @@ -0,0 +1,22 @@ +# 'address' is the IP to bind to. If not set, it listens on all. +# address=localhost +# 'port' is the port of the web server +port=8000 +# 'poll_intervall' is the polling interval for the statistics +poll_interval=5, +# 'restricted' gives a restricted version of the web GUI, disabling the user +# from changing any values +restricted=0 +# 'graph_width' and 'graph_height' are width and height for the graphs in 'View stats' +graph_width=250, +graph_height=125, +# 'large_graph_width' and 'large_graph_height' are width and height for the full size graph +# when clicking a stat graph in 'View stats' +large_graph_width=1000 +large_graph_height=500 +# 'log_filename' is the filename to log errors and information about actions done in the GUI +log_filename=/opt/varnish/webui/varnish.log +# 'db_filename' is the sqlite3 database created with the SQL outputed from create_db_data.pl +db_filename=/opt/varnish/webui/varnish.db +# 'document_root' is the root of the templates and css file +document_root=/opt/varnish/webui From petter at projects.linpro.no Tue Mar 3 15:25:45 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Tue, 3 Mar 2009 16:25:45 +0100 (CET) Subject: r3870 - in trunk/varnish-tools/webgui: Varnish templates Message-ID: <20090303152545.36CEB28370@projects.linpro.no> Author: petter Date: 2009-03-03 16:25:44 +0100 (Tue, 03 Mar 2009) New Revision: 3870 Added: trunk/varnish-tools/webgui/templates/access_denied.tmpl Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm Log: Added real restrictions in the backend as well. Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-03 14:33:57 UTC (rev 3869) +++ trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-03 15:25:44 UTC (rev 3870) @@ -71,6 +71,11 @@ return %parameter; } + sub _access_denied { + + return ("access_denied.tmpl", undef); + } + sub process { my ($self) = @_; @@ -113,7 +118,12 @@ ($content_template, $param) = management_console(\%request_parameter); } elsif ($operation eq 'send_management_command') { - $response_content = send_management_command(\%request_parameter); + if (get_config_value('restricted')) { + ($content_template, $param) = _access_denied(); + } + else { + $response_content = send_management_command(\%request_parameter); + } } elsif ($operation eq 'generate_graph') { $response_header_ref_of{$self}->{'Content-Type'} = "image/png"; @@ -127,7 +137,7 @@ $response_content = "Ok"; } else { - return; + ($content_template, $param) = _access_denied(); } $response_header_ref_of{$self}->{'Connection'} = "Close"; @@ -196,6 +206,12 @@ my $successfull_save = 0; my $editing_new_vcl = 0; + if (get_config_value('restricted') + && $param{'operation'} + && $param{'operation'} ne 'load') { + return _access_denied(); + } + if ($param{'operation'} eq "make_active") { my $unit; if ($param{'is_node'}) { @@ -619,6 +635,10 @@ my $nodes_ref = Varnish::NodeManager->get_nodes(); my $groups_ref = Varnish::NodeManager->get_groups(); if (%changed_parameter) { + if (get_config_value('restricted')) { + return _access_denied(); + } + my $unit_name; my $node = Varnish::NodeManager->get_node($param{'node_id'}); if ($node) { @@ -767,6 +787,11 @@ my $error = ""; my $status = ""; + if (get_config_value('restricted') + && $param{'operation'} ne '') { + return _access_denied(); + } + if ($param{'operation'} eq "add_group") { if ($param{'group_name'}) { my $new_group = Varnish::Group->new({name => $param{'group_name'}}); Added: trunk/varnish-tools/webgui/templates/access_denied.tmpl =================================================================== --- trunk/varnish-tools/webgui/templates/access_denied.tmpl (rev 0) +++ trunk/varnish-tools/webgui/templates/access_denied.tmpl 2009-03-03 15:25:44 UTC (rev 3870) @@ -0,0 +1 @@ +Acess denied From petter at projects.linpro.no Wed Mar 4 07:38:30 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Wed, 4 Mar 2009 08:38:30 +0100 (CET) Subject: r3871 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304073830.B95621F74C3@projects.linpro.no> Author: petter Date: 2009-03-04 08:38:30 +0100 (Wed, 04 Mar 2009) New Revision: 3871 Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c Log: Added missing header Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-03-03 15:25:44 UTC (rev 3870) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-03-04 07:38:30 UTC (rev 3871) @@ -49,6 +49,10 @@ #include "compat/vasprintf.h" #endif +#ifndef HAVE_SRANDOMDEV +#include "compat/srandomdev.h" +#endif + #include "cli_priv.h" #include "cli.h" #include "vsb.h" From phk at projects.linpro.no Wed Mar 4 09:22:47 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 10:22:47 +0100 (CET) Subject: r3872 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304092247.A6E261F753B@projects.linpro.no> Author: phk Date: 2009-03-04 10:22:47 +0100 (Wed, 04 Mar 2009) New Revision: 3872 Modified: trunk/varnish-cache/bin/varnishd/stevedore.c Log: Try quite hard to allocate all storage segments for an object from the same stevedore as the object was allocated from. I am not sure why I didn't spot this back when we added multiple storage ability, but it is surely dumb to have to find the object structure on one disk and the body of the object on another, or even several other, disks. For persistence this changes from a performance issue to problem: Any object spread over silos will be lost on restart. Further changes to this stuff is therefore in the pipeline, but I wanted this fix in a version that can be merged back to 2.0.x. Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-03-04 07:38:30 UTC (rev 3871) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-03-04 09:22:47 UTC (rev 3872) @@ -46,20 +46,33 @@ STV_alloc(struct sess *sp, size_t size) { struct storage *st; - struct stevedore *stv; + struct stevedore *stv = NULL; + unsigned fail = 0; + /* + * Always try the stevedore which allocated the object in order to + * not needlessly split an object across multiple stevedores. + */ + if (sp->obj != NULL) { + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + if (sp->obj->objstore != NULL) { + stv = sp->obj->objstore->stevedore; + CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); + } + } + for (;;) { + if (stv == NULL) { + /* pick a stevedore and bump the head along */ + stv = VTAILQ_NEXT(stv_next, list); + if (stv == NULL) + stv = VTAILQ_FIRST(&stevedores); + AN(stv); + AN(stv->name); + stv_next = stv; + fail = 0; + } - /* pick a stevedore and bump the head along */ - stv = VTAILQ_NEXT(stv_next, list); - if (stv == NULL) - stv = VTAILQ_FIRST(&stevedores); - AN(stv); - AN(stv->name); - - /* XXX: only safe as long as pointer writes are atomic */ - stv_next = stv; - /* try to allocate from it */ AN(stv->alloc); st = stv->alloc(stv, size); @@ -69,6 +82,10 @@ /* no luck; try to free some space and keep trying */ if (EXP_NukeOne(sp) == -1) break; + + /* Enough is enough: try another if we have one */ + if (++fail == 50) + stv = NULL; } CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); return (st); From phk at projects.linpro.no Wed Mar 4 09:40:06 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 10:40:06 +0100 (CET) Subject: r3873 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090304094006.482731F753B@projects.linpro.no> Author: phk Date: 2009-03-04 10:40:06 +0100 (Wed, 04 Mar 2009) New Revision: 3873 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/include/stat_field.h Log: Make n_bereq a worker-thead buffered counter and thus precise. Any continous growth is now indications of leaks. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 09:22:47 UTC (rev 3872) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 09:40:06 UTC (rev 3873) @@ -439,7 +439,7 @@ void VBE_GetFd(struct sess *sp); void VBE_ClosedFd(struct sess *sp); void VBE_RecycleFd(struct sess *sp); -struct bereq * VBE_new_bereq(void); +struct bereq * VBE_new_bereq(struct sess *sp); void VBE_free_bereq(struct bereq **bereq); void VBE_AddHostHeader(const struct sess *sp); void VBE_Poll(void); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 09:22:47 UTC (rev 3872) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 09:40:06 UTC (rev 3873) @@ -142,7 +142,7 @@ */ struct bereq * -VBE_new_bereq(void) +VBE_new_bereq(struct sess *sp) { struct bereq *bereq; volatile unsigned len; @@ -161,7 +161,7 @@ return (NULL); bereq->magic = BEREQ_MAGIC; WS_Init(bereq->ws, "bereq", bereq + 1, len); - VSL_stats->n_bereq++; + sp->wrk->stats->n_bereq++; } http_Setup(bereq->bereq, bereq->ws); http_Setup(bereq->beresp, bereq->ws); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-03-04 09:22:47 UTC (rev 3872) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-03-04 09:40:06 UTC (rev 3873) @@ -635,7 +635,7 @@ struct http *hp; AZ(sp->bereq); - bereq = VBE_new_bereq(); + bereq = VBE_new_bereq(sp); AN(bereq); hp = bereq->bereq; hp->logtag = HTTP_Tx; Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-03-04 09:22:47 UTC (rev 3872) +++ trunk/varnish-cache/include/stat_field.h 2009-03-04 09:40:06 UTC (rev 3873) @@ -55,7 +55,7 @@ MAC_STAT(n_smf_frag, uint64_t, 0, 'i', "N small free smf") MAC_STAT(n_smf_large, uint64_t, 0, 'i', "N large free smf") MAC_STAT(n_vbe_conn, uint64_t, 0, 'i', "N struct vbe_conn") -MAC_STAT(n_bereq, uint64_t, 0, 'i', "N struct bereq") +MAC_STAT(n_bereq, uint64_t, 1, 'i', "N struct bereq") MAC_STAT(n_wrk, uint64_t, 0, 'i', "N worker threads") MAC_STAT(n_wrk_create, uint64_t, 0, 'a', "N worker threads created") MAC_STAT(n_wrk_failed, uint64_t, 0, 'a', "N worker threads not created") From phk at projects.linpro.no Wed Mar 4 10:55:28 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 11:55:28 +0100 (CET) Subject: r3874 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304105528.BF8081F753B@projects.linpro.no> Author: phk Date: 2009-03-04 11:55:28 +0100 (Wed, 04 Mar 2009) New Revision: 3874 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Begin moving bereq into the worker thread. The original expectation was a ratio of 1:10 bereq:worker but once again reality lets down data-less speculation. It might have been true in a scenario where the majority of objects bind the worker thread for significant time during delivery (ie: ISOs over 28.8 modems) but that is not what happens in reality. With objects rapidly delivered, backend contacts will size the worker thread pool, since that is where they spend their time "out of the loop", and consequenty, ratios of bereq/worker is much close to 1:1 than 1:10. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 09:40:06 UTC (rev 3873) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 10:55:28 UTC (rev 3874) @@ -217,6 +217,9 @@ unsigned wlr; struct SHA256Context *sha256ctx; + + struct ws ws[1]; + struct http http[3]; }; /* Work Request for worker thread ------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 09:40:06 UTC (rev 3873) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 10:55:28 UTC (rev 3874) @@ -291,18 +291,17 @@ /*--------------------------------------------------------------------*/ static void * -wrk_thread(void *priv) +wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace) { struct worker *w, ww; - struct wq *qp; - unsigned char wlog[params->shm_workspace]; + unsigned char wlog[shm_workspace]; + unsigned char ws[sess_workspace]; struct SHA256Context sha256; struct dstat stats; unsigned stats_clean = 0; THR_SetName("cache-worker"); w = &ww; - CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC); memset(w, 0, sizeof *w); memset(&stats, 0, sizeof stats); w->magic = WORKER_MAGIC; @@ -313,6 +312,8 @@ w->sha256ctx = &sha256; AZ(pthread_cond_init(&w->cond, NULL)); + WS_Init(w->ws, "wrk", ws, sess_workspace); + VSL(SLT_WorkThread, 0, "%p start", w); Lck_Lock(&qp->mtx); @@ -342,6 +343,10 @@ AN(w->wrq->func); w->lastused = NAN; stats_clean = 0; + WS_Reset(w->ws, NULL); + http_Setup(&w->http[0], w->ws); + http_Setup(&w->http[1], w->ws); + http_Setup(&w->http[2], w->ws); w->wrq->func(w, w->wrq->priv); AZ(w->wfd); assert(w->wlp == w->wlb); @@ -368,6 +373,18 @@ return (NULL); } +static void * +wrk_thread(void *priv) +{ + struct wq *qp; + + CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC); + /* We need to snapshot these two for consistency */ + return (wrk_thread_real(qp, + params->shm_workspace, + params->sess_workspace)); +} + /*-------------------------------------------------------------------- * Queue a workrequest if possible. * From phk at projects.linpro.no Wed Mar 4 11:28:07 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 12:28:07 +0100 (CET) Subject: r3875 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304112807.C30551F753E@projects.linpro.no> Author: phk Date: 2009-03-04 12:28:07 +0100 (Wed, 04 Mar 2009) New Revision: 3875 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_esi.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vrt.c Log: Move bereq.* to worker. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 11:28:07 UTC (rev 3875) @@ -220,6 +220,10 @@ struct ws ws[1]; struct http http[3]; + struct http *bereq; + struct http *beresp1; + struct http *beresp; + struct http *resp; }; /* Work Request for worker thread ------------------------------------*/ @@ -244,7 +248,6 @@ #define BEREQ_MAGIC 0x3b6d250c VTAILQ_ENTRY(bereq) list; struct ws ws[1]; - struct http bereq[1]; struct http beresp[2]; struct http_conn htc[1]; unsigned cacheable; Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -65,10 +65,10 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq->bereq, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC); CHECK_OBJ_NOTNULL(sp->vbe, VBE_CONN_MAGIC); CHECK_OBJ_NOTNULL(sp->vbe->backend, BACKEND_MAGIC); - http_PrintfHeader(sp->wrk, sp->fd, sp->bereq->bereq, + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq, "Host: %s", sp->vbe->backend->hosthdr); } @@ -163,7 +163,6 @@ WS_Init(bereq->ws, "bereq", bereq + 1, len); sp->wrk->stats->n_bereq++; } - http_Setup(bereq->bereq, bereq->ws); http_Setup(bereq->beresp, bereq->ws); return (bereq); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -339,6 +339,7 @@ assert(sp->handling == VCL_RET_DELIVER); sp->err_code = 0; sp->err_reason = NULL; + sp->wrk->bereq = NULL; sp->step = STP_DELIVER; return (0); } @@ -536,6 +537,7 @@ HSH_Unbusy(sp); } sp->acct_req.fetch++; + sp->wrk->bereq = NULL; sp->step = STP_DELIVER; return (0); } @@ -622,6 +624,7 @@ if (sp->handling == VCL_RET_DELIVER) { /* Dispose of any body part of the request */ FetchReqBody(sp); + sp->wrk->bereq = NULL; sp->step = STP_DELIVER; return (0); } @@ -771,6 +774,7 @@ AZ(sp->obj); AN(sp->objcore); AN(sp->objhead); + sp->wrk->bereq = &sp->wrk->http[0]; http_FilterHeader(sp, HTTPH_R_FETCH); VCL_miss_method(sp); switch(sp->handling) { @@ -836,6 +840,7 @@ CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); AZ(sp->obj); + sp->wrk->bereq = &sp->wrk->http[0]; http_FilterHeader(sp, HTTPH_R_PASS); VCL_pass_method(sp); @@ -884,6 +889,7 @@ CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); sp->acct_req.pipe++; + sp->wrk->bereq = &sp->wrk->http[0]; http_FilterHeader(sp, HTTPH_R_PIPE); VCL_pipe_method(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -398,8 +398,8 @@ */ CHECK_OBJ_NOTNULL(ew->sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(ew->sp->bereq, BEREQ_MAGIC); - CHECK_OBJ_NOTNULL(ew->sp->bereq->bereq, HTTP_MAGIC); - tag = ew->sp->bereq->bereq->hd[HTTP_HDR_URL]; + CHECK_OBJ_NOTNULL(ew->sp->wrk->bereq, HTTP_MAGIC); + tag = ew->sp->wrk->bereq->hd[HTTP_HDR_URL]; /* Use the objects WS to store the result */ CHECK_OBJ_NOTNULL(ew->sp->obj, OBJECT_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -333,7 +333,7 @@ w = sp->wrk; bereq = sp->bereq; - hp = bereq->bereq; + hp = sp->wrk->bereq; VBE_GetFd(sp); if (sp->vbe == NULL) @@ -421,7 +421,7 @@ vc = sp->vbe; - is_head = (strcasecmp(http_GetReq(sp->bereq->bereq), "head") == 0); + is_head = (strcasecmp(http_GetReq(sp->wrk->bereq), "head") == 0); /* Determine if we have a body or not */ cls = 0; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -637,7 +637,7 @@ AZ(sp->bereq); bereq = VBE_new_bereq(sp); AN(bereq); - hp = bereq->bereq; + hp = sp->wrk->bereq; hp->logtag = HTTP_Tx; http_copyreq(hp, sp->http, how); Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -84,7 +84,7 @@ TCP_blocking(vc->fd); WRW_Reserve(w, &vc->fd); - sp->acct_req.hdrbytes += http_Write(w, bereq->bereq, 0); + sp->acct_req.hdrbytes += http_Write(w, sp->wrk->bereq, 0); if (sp->htc->pipeline.b != NULL) sp->acct_req.bodybytes += Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -347,6 +347,10 @@ http_Setup(&w->http[0], w->ws); http_Setup(&w->http[1], w->ws); http_Setup(&w->http[2], w->ws); + w->bereq = NULL; + w->beresp1 = NULL; + w->beresp = NULL; + w->resp = NULL; w->wrq->func(w, w->wrq->priv); AZ(w->wfd); assert(w->wlp == w->wlb); Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 10:55:28 UTC (rev 3874) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 11:28:07 UTC (rev 3875) @@ -101,7 +101,7 @@ hp = sp->http; break; case HDR_BEREQ: - hp = sp->bereq->bereq; + hp = sp->wrk->bereq; break; case HDR_BERESP: hp = sp->bereq->beresp; @@ -244,9 +244,9 @@ VRT_DO_HDR(req, request, sp->http, HTTP_HDR_REQ) VRT_DO_HDR(req, url, sp->http, HTTP_HDR_URL) VRT_DO_HDR(req, proto, sp->http, HTTP_HDR_PROTO) -VRT_DO_HDR(bereq, request, sp->bereq->bereq, HTTP_HDR_REQ) -VRT_DO_HDR(bereq, url, sp->bereq->bereq, HTTP_HDR_URL) -VRT_DO_HDR(bereq, proto, sp->bereq->bereq, HTTP_HDR_PROTO) +VRT_DO_HDR(bereq, request, sp->wrk->bereq, HTTP_HDR_REQ) +VRT_DO_HDR(bereq, url, sp->wrk->bereq, HTTP_HDR_URL) +VRT_DO_HDR(bereq, proto, sp->wrk->bereq, HTTP_HDR_PROTO) VRT_DO_HDR(obj, proto, sp->obj->http, HTTP_HDR_PROTO) VRT_DO_HDR(obj, response, sp->obj->http, HTTP_HDR_RESPONSE) VRT_DO_HDR(resp, proto, sp->http, HTTP_HDR_PROTO) From phk at projects.linpro.no Wed Mar 4 11:37:39 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 12:37:39 +0100 (CET) Subject: r3876 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304113739.D556197C83@projects.linpro.no> Author: phk Date: 2009-03-04 12:37:39 +0100 (Wed, 04 Mar 2009) New Revision: 3876 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Move beresp from bereq to worker Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 11:28:07 UTC (rev 3875) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 11:37:39 UTC (rev 3876) @@ -248,7 +248,6 @@ #define BEREQ_MAGIC 0x3b6d250c VTAILQ_ENTRY(bereq) list; struct ws ws[1]; - struct http beresp[2]; struct http_conn htc[1]; unsigned cacheable; double age; Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 11:28:07 UTC (rev 3875) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 11:37:39 UTC (rev 3876) @@ -163,7 +163,6 @@ WS_Init(bereq->ws, "bereq", bereq + 1, len); sp->wrk->stats->n_bereq++; } - http_Setup(bereq->beresp, bereq->ws); return (bereq); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 11:28:07 UTC (rev 3875) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 11:37:39 UTC (rev 3876) @@ -390,13 +390,16 @@ AN(sp->director); AZ(sp->vbe); + sp->wrk->beresp = &sp->wrk->http[1]; + sp->wrk->beresp1 = &sp->wrk->http[2]; + i = FetchHdr(sp); /* * Save a copy before it might get mangled in VCL. When it comes to * dealing with the body, we want to see the unadultered headers. */ - sp->bereq->beresp[1] = sp->bereq->beresp[0]; + *sp->wrk->beresp1 = *sp->wrk->beresp; if (i) { sp->err_code = 503; @@ -411,7 +414,7 @@ return (0); } - sp->err_code = http_GetStatus(sp->bereq->beresp); + sp->err_code = http_GetStatus(sp->wrk->beresp); /* * Initial cacheability determination per [RFC2616, 13.4] @@ -477,7 +480,7 @@ WS_Assert(sp->obj->ws_o); /* Filter into object */ - hp = sp->bereq->beresp; + hp = sp->wrk->beresp; hp2 = sp->obj->http; hp2->logtag = HTTP_Obj; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 11:28:07 UTC (rev 3875) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 11:37:39 UTC (rev 3876) @@ -384,7 +384,7 @@ return (__LINE__); } - hp = bereq->beresp; + hp = sp->wrk->beresp; if (http_DissectResponse(sp->wrk, bereq->htc, hp)) { VBE_ClosedFd(sp); @@ -413,7 +413,7 @@ CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* We use the unmodified headers */ - hp = &sp->bereq->beresp[1]; + hp = sp->wrk->beresp1; AN(sp->director); if (sp->obj->objcore != NULL) /* pass has no objcore */ AN(ObjIsBusy(sp->obj)); Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 11:28:07 UTC (rev 3875) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 11:37:39 UTC (rev 3876) @@ -104,7 +104,7 @@ hp = sp->wrk->bereq; break; case HDR_BERESP: - hp = sp->bereq->beresp; + hp = sp->wrk->beresp; break; case HDR_RESP: hp = sp->http; @@ -251,8 +251,8 @@ VRT_DO_HDR(obj, response, sp->obj->http, HTTP_HDR_RESPONSE) VRT_DO_HDR(resp, proto, sp->http, HTTP_HDR_PROTO) VRT_DO_HDR(resp, response, sp->http, HTTP_HDR_RESPONSE) -VRT_DO_HDR(beresp, proto, sp->bereq->beresp, HTTP_HDR_PROTO) -VRT_DO_HDR(beresp, response, sp->bereq->beresp, HTTP_HDR_RESPONSE) +VRT_DO_HDR(beresp, proto, sp->wrk->beresp, HTTP_HDR_PROTO) +VRT_DO_HDR(beresp, response, sp->wrk->beresp, HTTP_HDR_RESPONSE) /*--------------------------------------------------------------------*/ @@ -372,12 +372,12 @@ char *p; assert(num >= 100 && num <= 999); - p = WS_Alloc(sp->bereq->beresp->ws, 4); + p = WS_Alloc(sp->wrk->beresp->ws, 4); if (p == NULL) WSP(sp, SLT_LostHeader, "%s", "obj.status"); else sprintf(p, "%d", num); - http_SetH(sp->bereq->beresp, HTTP_HDR_STATUS, p); + http_SetH(sp->wrk->beresp, HTTP_HDR_STATUS, p); } int @@ -386,10 +386,10 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* XXX: use http_GetStatus() */ - if (sp->bereq->beresp->status) - return (sp->bereq->beresp->status); - AN(sp->bereq->beresp->hd[HTTP_HDR_STATUS].b); - return (atoi(sp->bereq->beresp->hd[HTTP_HDR_STATUS].b)); + if (sp->wrk->beresp->status) + return (sp->wrk->beresp->status); + AN(sp->wrk->beresp->hd[HTTP_HDR_STATUS].b); + return (atoi(sp->wrk->beresp->hd[HTTP_HDR_STATUS].b)); } Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2009-03-04 11:28:07 UTC (rev 3875) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2009-03-04 11:37:39 UTC (rev 3876) @@ -78,7 +78,7 @@ const struct http *hp; CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); - hp = sp->bereq->beresp; + hp = sp->wrk->beresp; assert(sp->bereq->entered != 0.0 && !isnan(sp->bereq->entered)); /* If all else fails, cache using default ttl */ From phk at projects.linpro.no Wed Mar 4 11:45:11 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 12:45:11 +0100 (CET) Subject: r3877 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304114511.3BDDF1F7542@projects.linpro.no> Author: phk Date: 2009-03-04 12:45:10 +0100 (Wed, 04 Mar 2009) New Revision: 3877 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Move the htc from bereq to worker Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 11:37:39 UTC (rev 3876) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 11:45:10 UTC (rev 3877) @@ -218,6 +218,7 @@ struct SHA256Context *sha256ctx; + struct http_conn htc[1]; struct ws ws[1]; struct http http[3]; struct http *bereq; @@ -247,8 +248,6 @@ unsigned magic; #define BEREQ_MAGIC 0x3b6d250c VTAILQ_ENTRY(bereq) list; - struct ws ws[1]; - struct http_conn htc[1]; unsigned cacheable; double age; double entered; Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 11:37:39 UTC (rev 3876) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 11:45:10 UTC (rev 3877) @@ -145,7 +145,6 @@ VBE_new_bereq(struct sess *sp) { struct bereq *bereq; - volatile unsigned len; Lck_Lock(&VBE_mtx); bereq = VTAILQ_FIRST(&bereq_head); @@ -155,12 +154,10 @@ if (bereq != NULL) { CHECK_OBJ(bereq, BEREQ_MAGIC); } else { - len = params->sess_workspace; - bereq = calloc(sizeof *bereq + len, 1); + bereq = calloc(sizeof *bereq, 1); if (bereq == NULL) return (NULL); bereq->magic = BEREQ_MAGIC; - WS_Init(bereq->ws, "bereq", bereq + 1, len); sp->wrk->stats->n_bereq++; } return (bereq); @@ -180,7 +177,6 @@ *bereqp = NULL; CHECK_OBJ_NOTNULL(bereq, BEREQ_MAGIC); - WS_Reset(bereq->ws, NULL); Lck_Lock(&VBE_mtx); VTAILQ_INSERT_HEAD(&bereq_head, bereq, list); Lck_Unlock(&VBE_mtx); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 11:37:39 UTC (rev 3876) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 11:45:10 UTC (rev 3877) @@ -370,10 +370,10 @@ /* Receive response */ - HTC_Init(bereq->htc, bereq->ws, vc->fd); + HTC_Init(sp->wrk->htc, sp->wrk->ws, vc->fd); TCP_set_read_timeout(vc->fd, sp->first_byte_timeout); do { - i = HTC_Rx(bereq->htc); + i = HTC_Rx(sp->wrk->htc); TCP_set_read_timeout(vc->fd, sp->between_bytes_timeout); } while (i == 0); @@ -386,7 +386,7 @@ hp = sp->wrk->beresp; - if (http_DissectResponse(sp->wrk, bereq->htc, hp)) { + if (http_DissectResponse(sp->wrk, sp->wrk->htc, hp)) { VBE_ClosedFd(sp); /* XXX: other cleanup ? */ return (__LINE__); @@ -429,10 +429,10 @@ if (is_head) { /* nothing */ } else if (http_GetHdr(hp, H_Content_Length, &b)) { - cls = fetch_straight(sp, sp->bereq->htc, b); + cls = fetch_straight(sp, sp->wrk->htc, b); mklen = 1; } else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { - cls = fetch_chunked(sp, sp->bereq->htc); + cls = fetch_chunked(sp, sp->wrk->htc); mklen = 1; } else if (http_GetHdr(hp, H_Transfer_Encoding, &b)) { /* XXX: AUGH! */ @@ -451,13 +451,13 @@ * If we have connection closed, it is safe to read what * comes in any case. */ - cls = fetch_eof(sp, sp->bereq->htc); + cls = fetch_eof(sp, sp->wrk->htc); mklen = 1; } else if (hp->protover < 1.1) { /* * With no Connection header, assume EOF */ - cls = fetch_eof(sp, sp->bereq->htc); + cls = fetch_eof(sp, sp->wrk->htc); mklen = 1; } else { /* From phk at projects.linpro.no Wed Mar 4 12:13:54 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 13:13:54 +0100 (CET) Subject: r3878 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090304121354.8B2661F753B@projects.linpro.no> Author: phk Date: 2009-03-04 13:13:54 +0100 (Wed, 04 Mar 2009) New Revision: 3878 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_cli.c trunk/varnish-cache/bin/varnishd/cache_esi.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/include/stat_field.h Log: Integrate the rest of bereq into the worker structure. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-04 12:13:54 UTC (rev 3878) @@ -225,6 +225,13 @@ struct http *beresp1; struct http *beresp; struct http *resp; + + unsigned cacheable; + double age; + double entered; + double ttl; + double grace; + unsigned do_esi; }; /* Work Request for worker thread ------------------------------------*/ @@ -242,20 +249,6 @@ void *priv; }; -/* Backend Request ---------------------------------------------------*/ - -struct bereq { - unsigned magic; -#define BEREQ_MAGIC 0x3b6d250c - VTAILQ_ENTRY(bereq) list; - unsigned cacheable; - double age; - double entered; - double ttl; - double grace; - unsigned do_esi; -}; - /* Storage -----------------------------------------------------------*/ struct storage { @@ -398,7 +391,6 @@ struct director *director; struct vbe_conn *vbe; - struct bereq *bereq; struct object *obj; struct objcore *objcore; struct objhead *objhead; @@ -443,8 +435,6 @@ void VBE_GetFd(struct sess *sp); void VBE_ClosedFd(struct sess *sp); void VBE_RecycleFd(struct sess *sp); -struct bereq * VBE_new_bereq(struct sess *sp); -void VBE_free_bereq(struct bereq **bereq); void VBE_AddHostHeader(const struct sess *sp); void VBE_Poll(void); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -51,11 +51,6 @@ */ static VTAILQ_HEAD(,vbe_conn) vbe_conns = VTAILQ_HEAD_INITIALIZER(vbe_conns); -/* - * List of cached bereq's - */ -static VTAILQ_HEAD(,bereq) bereq_head = VTAILQ_HEAD_INITIALIZER(bereq_head); - /*-------------------------------------------------------------------- * Create default Host: header for backend request */ @@ -64,7 +59,6 @@ { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC); CHECK_OBJ_NOTNULL(sp->vbe, VBE_CONN_MAGIC); CHECK_OBJ_NOTNULL(sp->vbe->backend, BACKEND_MAGIC); @@ -135,54 +129,6 @@ } /*-------------------------------------------------------------------- - * Get a bereq structure for talking HTTP with the backend. - * First attempt to pick one from our stash, else make a new. - * - * Can fail with NULL. - */ - -struct bereq * -VBE_new_bereq(struct sess *sp) -{ - struct bereq *bereq; - - Lck_Lock(&VBE_mtx); - bereq = VTAILQ_FIRST(&bereq_head); - if (bereq != NULL) - VTAILQ_REMOVE(&bereq_head, bereq, list); - Lck_Unlock(&VBE_mtx); - if (bereq != NULL) { - CHECK_OBJ(bereq, BEREQ_MAGIC); - } else { - bereq = calloc(sizeof *bereq, 1); - if (bereq == NULL) - return (NULL); - bereq->magic = BEREQ_MAGIC; - sp->wrk->stats->n_bereq++; - } - return (bereq); -} - -/*-------------------------------------------------------------------- - * Return a bereq to the stash. - */ - -void -VBE_free_bereq(struct bereq **bereqp) -{ - struct bereq *bereq; - - AN(bereqp); - bereq = *bereqp; - *bereqp = NULL; - - CHECK_OBJ_NOTNULL(bereq, BEREQ_MAGIC); - Lck_Lock(&VBE_mtx); - VTAILQ_INSERT_HEAD(&bereq_head, bereq, list); - Lck_Unlock(&VBE_mtx); -} - -/*-------------------------------------------------------------------- * Manage a pool of vbe_conn structures. * XXX: as an experiment, make this caching controled by a parameter * XXX: so we can see if it has any effect. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -155,8 +155,6 @@ CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); - AZ(sp->bereq); - sp->t_resp = TIM_real(); if (sp->obj->objhead != NULL) { if ((sp->t_resp - sp->obj->last_lru) > params->lru_timeout && @@ -208,7 +206,6 @@ AZ(sp->obj); AZ(sp->vbe); - AZ(sp->bereq); sp->director = NULL; sp->restarts = 0; @@ -305,7 +302,6 @@ char date[40]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - AZ(sp->bereq); /* We always close when we take this path */ sp->doclose = "error"; @@ -386,7 +382,6 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); - AN(sp->bereq); AN(sp->director); AZ(sp->vbe); @@ -404,7 +399,6 @@ if (i) { sp->err_code = 503; sp->step = STP_ERROR; - VBE_free_bereq(&sp->bereq); if (sp->objhead) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); @@ -428,22 +422,22 @@ case 302: /* Moved Temporarily */ case 410: /* Gone */ case 404: /* Not Found */ - sp->bereq->cacheable = 1; + sp->wrk->cacheable = 1; break; default: - sp->bereq->cacheable = 0; + sp->wrk->cacheable = 0; break; } - sp->bereq->entered = TIM_real(); - sp->bereq->age = 0; - sp->bereq->ttl = RFC2616_Ttl(sp); + sp->wrk->entered = TIM_real(); + sp->wrk->age = 0; + sp->wrk->ttl = RFC2616_Ttl(sp); - if (sp->bereq->ttl == 0.) - sp->bereq->cacheable = 0; + if (sp->wrk->ttl == 0.) + sp->wrk->cacheable = 0; - sp->bereq->do_esi = 0; - sp->bereq->grace = NAN; + sp->wrk->do_esi = 0; + sp->wrk->grace = NAN; VCL_fetch_method(sp); @@ -470,13 +464,13 @@ sp->obj->xid = sp->xid; sp->obj->response = sp->err_code; - sp->obj->cacheable = sp->bereq->cacheable; - sp->obj->ttl = sp->bereq->ttl; - sp->obj->grace = sp->bereq->grace; + sp->obj->cacheable = sp->wrk->cacheable; + sp->obj->ttl = sp->wrk->ttl; + sp->obj->grace = sp->wrk->grace; if (sp->obj->ttl == 0.) sp->obj->cacheable = 0; - sp->obj->age = sp->bereq->age; - sp->obj->entered = sp->bereq->entered; + sp->obj->age = sp->wrk->age; + sp->obj->entered = sp->wrk->entered; WS_Assert(sp->obj->ws_o); /* Filter into object */ @@ -499,17 +493,14 @@ if (i) { sp->err_code = 503; sp->step = STP_ERROR; - VBE_free_bereq(&sp->bereq); HSH_Drop(sp); AZ(sp->obj); return (0); } - if (sp->bereq->do_esi) + if (sp->wrk->do_esi) ESI_Parse(sp); - VBE_free_bereq(&sp->bereq); - switch (handling) { case VCL_RET_RESTART: HSH_Drop(sp); @@ -783,11 +774,9 @@ switch(sp->handling) { case VCL_RET_ERROR: HSH_DerefObjCore(sp); - VBE_free_bereq(&sp->bereq); sp->step = STP_ERROR; return (0); case VCL_RET_PASS: - VBE_free_bereq(&sp->bereq); HSH_DerefObjCore(sp); sp->step = STP_PASS; return (0); @@ -796,7 +785,6 @@ return (0); case VCL_RET_RESTART: HSH_DerefObjCore(sp); - VBE_free_bereq(&sp->bereq); INCOMPL(); default: WRONG("Illegal action in vcl_miss{}"); @@ -848,7 +836,6 @@ VCL_pass_method(sp); if (sp->handling == VCL_RET_ERROR) { - VBE_free_bereq(&sp->bereq); sp->step = STP_ERROR; return (0); } @@ -902,7 +889,6 @@ assert(sp->handling == VCL_RET_PIPE); PipeSession(sp); - AZ(sp->bereq); AZ(sp->wrk->wfd); sp->step = STP_DONE; return (0); Modified: trunk/varnish-cache/bin/varnishd/cache_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_cli.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_cli.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -189,7 +189,6 @@ SZOF(struct acct); SZOF(struct worker); SZOF(struct workreq); - SZOF(struct bereq); SZOF(struct storage); SZOF(struct object); SZOF(struct objcore); Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -397,7 +397,6 @@ * different and have been rewritten underway. */ CHECK_OBJ_NOTNULL(ew->sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(ew->sp->bereq, BEREQ_MAGIC); CHECK_OBJ_NOTNULL(ew->sp->wrk->bereq, HTTP_MAGIC); tag = ew->sp->wrk->bereq->hd[HTTP_HDR_URL]; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -314,12 +314,10 @@ struct worker *w; char *b; struct http *hp; - struct bereq *bereq; int i; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); AN(sp->director); AZ(sp->obj); if (sp->objcore != NULL) { /* pass has no objcore */ @@ -327,12 +325,10 @@ AN(sp->objhead); /* details in hash_slinger.h */ AN(sp->objcore->flags & OC_F_BUSY); } - AN(sp->bereq); /* Transmit request */ w = sp->wrk; - bereq = sp->bereq; hp = sp->wrk->bereq; VBE_GetFd(sp); @@ -410,14 +406,12 @@ CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* We use the unmodified headers */ hp = sp->wrk->beresp1; AN(sp->director); if (sp->obj->objcore != NULL) /* pass has no objcore */ AN(ObjIsBusy(sp->obj)); - AN(sp->bereq); vc = sp->vbe; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -631,13 +631,10 @@ void http_FilterHeader(struct sess *sp, unsigned how) { - struct bereq *bereq; struct http *hp; - AZ(sp->bereq); - bereq = VBE_new_bereq(sp); - AN(bereq); hp = sp->wrk->bereq; + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); hp->logtag = HTTP_Tx; http_copyreq(hp, sp->http, how); @@ -645,8 +642,6 @@ http_PrintfHeader(sp->wrk, sp->fd, hp, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->wrk, sp->fd, hp, "X-Forwarded-For: %s", sp->addr); - - sp->bereq = bereq; } /*-------------------------------------------------------------------- Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -67,15 +67,12 @@ { struct vbe_conn *vc; struct worker *w; - struct bereq *bereq; struct pollfd fds[2]; int i; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); w = sp->wrk; - bereq = sp->bereq; - sp->bereq = NULL; VBE_GetFd(sp); if (sp->vbe == NULL) @@ -91,7 +88,6 @@ WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline)); i = WRW_FlushRelease(w); - VBE_free_bereq(&bereq); if (i) { vca_close_session(sp, "pipe"); Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -313,16 +313,14 @@ VRT_l_##dir##_##onm(const struct sess *sp, type a) \ { \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* XXX */ \ - sp->bereq->field = a; \ + sp->wrk->field = a; \ } \ \ type \ VRT_r_##dir##_##onm(const struct sess *sp) \ { \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* XXX */ \ - return (sp->bereq->field); \ + return (sp->wrk->field); \ } VBEREQ(beresp, unsigned, cacheable, cacheable) @@ -344,7 +342,6 @@ { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* XXX */ WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req); /* * If people set obj.ttl = 0s, they don't expect it to be cacheable @@ -353,17 +350,16 @@ * We special case and make sure that rounding does not surprise. */ if (a <= 0) - sp->bereq->ttl = sp->t_req - 1; + sp->wrk->ttl = sp->t_req - 1; else - sp->bereq->ttl = sp->t_req + a; + sp->wrk->ttl = sp->t_req + a; } double VRT_r_beresp_ttl(const struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* XXX */ - return (sp->bereq->ttl - sp->t_req); + return (sp->wrk->ttl - sp->t_req); } void @@ -384,7 +380,6 @@ VRT_r_beresp_status(const struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); /* XXX: use http_GetStatus() */ if (sp->wrk->beresp->status) return (sp->wrk->beresp->status); @@ -780,7 +775,6 @@ void VRT_ESI(struct sess *sp) { - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); if (sp->cur_method != VCL_MET_FETCH) { /* XXX: we should catch this at compile time */ @@ -789,7 +783,7 @@ return; } - sp->bereq->do_esi = 1; + sp->wrk->do_esi = 1; } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2009-03-04 12:13:54 UTC (rev 3878) @@ -77,10 +77,9 @@ char *p; const struct http *hp; - CHECK_OBJ_NOTNULL(sp->bereq, BEREQ_MAGIC); hp = sp->wrk->beresp; - assert(sp->bereq->entered != 0.0 && !isnan(sp->bereq->entered)); + assert(sp->wrk->entered != 0.0 && !isnan(sp->wrk->entered)); /* If all else fails, cache using default ttl */ ttl = params->default_ttl; @@ -103,7 +102,7 @@ max_age = strtoul(p, NULL, 0); if (http_GetHdr(hp, H_Age, &p)) { age = strtoul(p, NULL, 0); - sp->bereq->age = age; + sp->wrk->age = age; } if (age > max_age) @@ -130,14 +129,14 @@ } if (h_date == 0 || - (h_date < sp->bereq->entered + params->clock_skew && - h_date + params->clock_skew > sp->bereq->entered)) { + (h_date < sp->wrk->entered + params->clock_skew && + h_date + params->clock_skew > sp->wrk->entered)) { /* * If we have no Date: header or if it is * sufficiently close to our clock we will * trust Expires: relative to our own clock. */ - if (h_expires < sp->bereq->entered) + if (h_expires < sp->wrk->entered) ttl = 0; else ttd = h_expires; @@ -154,12 +153,12 @@ } while (0); if (ttl > 0 && ttd == 0) - ttd = sp->bereq->entered + ttl; + ttd = sp->wrk->entered + ttl; /* calculated TTL, Our time, Date, Expires, max-age, age */ WSP(sp, SLT_TTL, "%u RFC %d %d %d %d %u %u", sp->xid, - ttd ? (int)(ttd - sp->bereq->entered) : 0, - (int)sp->bereq->entered, (int)h_date, + ttd ? (int)(ttd - sp->wrk->entered) : 0, + (int)sp->wrk->entered, (int)h_date, (int)h_expires, max_age, age); return (ttd); Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-03-04 11:45:10 UTC (rev 3877) +++ trunk/varnish-cache/include/stat_field.h 2009-03-04 12:13:54 UTC (rev 3878) @@ -55,7 +55,6 @@ MAC_STAT(n_smf_frag, uint64_t, 0, 'i', "N small free smf") MAC_STAT(n_smf_large, uint64_t, 0, 'i', "N large free smf") MAC_STAT(n_vbe_conn, uint64_t, 0, 'i', "N struct vbe_conn") -MAC_STAT(n_bereq, uint64_t, 1, 'i', "N struct bereq") MAC_STAT(n_wrk, uint64_t, 0, 'i', "N worker threads") MAC_STAT(n_wrk_create, uint64_t, 0, 'a', "N worker threads created") MAC_STAT(n_wrk_failed, uint64_t, 0, 'a', "N worker threads not created") From phk at projects.linpro.no Wed Mar 4 12:28:17 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 13:28:17 +0100 (CET) Subject: r3879 - trunk/varnish-cache/bin/varnishd Message-ID: <20090304122817.0C09A38087@projects.linpro.no> Author: phk Date: 2009-03-04 13:28:16 +0100 (Wed, 04 Mar 2009) New Revision: 3879 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Setup the http structs in worker when we annex them for use. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 12:13:54 UTC (rev 3878) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 12:28:16 UTC (rev 3879) @@ -385,8 +385,9 @@ AN(sp->director); AZ(sp->vbe); + /* sp->wrk->http[0] is (still) bereq */ sp->wrk->beresp = &sp->wrk->http[1]; - sp->wrk->beresp1 = &sp->wrk->http[2]; + http_Setup(sp->wrk->beresp, sp->wrk->ws); i = FetchHdr(sp); @@ -394,17 +395,21 @@ * Save a copy before it might get mangled in VCL. When it comes to * dealing with the body, we want to see the unadultered headers. */ + sp->wrk->beresp1 = &sp->wrk->http[2]; *sp->wrk->beresp1 = *sp->wrk->beresp; if (i) { - sp->err_code = 503; - sp->step = STP_ERROR; if (sp->objhead) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); HSH_DerefObjCore(sp); } AZ(sp->obj); + sp->wrk->bereq = NULL; + sp->wrk->beresp = NULL; + sp->wrk->beresp1 = NULL; + sp->err_code = 503; + sp->step = STP_ERROR; return (0); } @@ -491,10 +496,13 @@ AN(sp->director); if (i) { + HSH_Drop(sp); + AZ(sp->obj); + sp->wrk->bereq = NULL; + sp->wrk->beresp = NULL; + sp->wrk->beresp1 = NULL; sp->err_code = 503; sp->step = STP_ERROR; - HSH_Drop(sp); - AZ(sp->obj); return (0); } @@ -506,6 +514,9 @@ HSH_Drop(sp); sp->director = NULL; sp->restarts++; + sp->wrk->bereq = NULL; + sp->wrk->beresp = NULL; + sp->wrk->beresp1 = NULL; sp->step = STP_RECV; return (0); case VCL_RET_PASS: @@ -517,8 +528,11 @@ case VCL_RET_DELIVER: break; case VCL_RET_ERROR: + HSH_Drop(sp); + sp->wrk->bereq = NULL; + sp->wrk->beresp = NULL; + sp->wrk->beresp1 = NULL; sp->step = STP_ERROR; - HSH_Drop(sp); return (0); default: WRONG("Illegal action in vcl_fetch{}"); @@ -532,6 +546,8 @@ } sp->acct_req.fetch++; sp->wrk->bereq = NULL; + sp->wrk->beresp = NULL; + sp->wrk->beresp1 = NULL; sp->step = STP_DELIVER; return (0); } @@ -769,6 +785,7 @@ AN(sp->objcore); AN(sp->objhead); sp->wrk->bereq = &sp->wrk->http[0]; + http_Setup(sp->wrk->bereq, sp->wrk->ws); http_FilterHeader(sp, HTTPH_R_FETCH); VCL_miss_method(sp); switch(sp->handling) { @@ -832,6 +849,7 @@ AZ(sp->obj); sp->wrk->bereq = &sp->wrk->http[0]; + http_Setup(sp->wrk->bereq, sp->wrk->ws); http_FilterHeader(sp, HTTPH_R_PASS); VCL_pass_method(sp); @@ -880,6 +898,7 @@ sp->acct_req.pipe++; sp->wrk->bereq = &sp->wrk->http[0]; + http_Setup(sp->wrk->bereq, sp->wrk->ws); http_FilterHeader(sp, HTTPH_R_PIPE); VCL_pipe_method(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 12:13:54 UTC (rev 3878) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 12:28:16 UTC (rev 3879) @@ -344,9 +344,6 @@ w->lastused = NAN; stats_clean = 0; WS_Reset(w->ws, NULL); - http_Setup(&w->http[0], w->ws); - http_Setup(&w->http[1], w->ws); - http_Setup(&w->http[2], w->ws); w->bereq = NULL; w->beresp1 = NULL; w->beresp = NULL; From phk at projects.linpro.no Wed Mar 4 13:00:18 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 4 Mar 2009 14:00:18 +0100 (CET) Subject: r3880 - in trunk/varnish-cache: bin/varnishd lib/libvcl Message-ID: <20090304130018.6C7FE1F7474@projects.linpro.no> Author: phk Date: 2009-03-04 14:00:18 +0100 (Wed, 04 Mar 2009) New Revision: 3880 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Untested to any meaningfull extent: Make req.* available in vcl_deliver{} Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-04 13:00:18 UTC (rev 3880) @@ -162,6 +162,8 @@ sp->obj->last_lru = sp->t_resp; /* XXX: locking ? */ sp->obj->last_use = sp->t_resp; /* XXX: locking ? */ } + sp->wrk->resp = &sp->wrk->http[2]; + http_Setup(sp->wrk->resp, sp->wrk->ws); RES_BuildHttp(sp); VCL_deliver_method(sp); switch (sp->handling) { @@ -180,6 +182,7 @@ RES_WriteObj(sp); AZ(sp->wrk->wfd); HSH_Deref(sp->wrk, &sp->obj); + sp->wrk->resp = NULL; sp->step = STP_DONE; return (0); } @@ -1116,6 +1119,7 @@ CHECK_OBJ_ORNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC); + WS_Assert(w->ws); switch (sp->step) { #define STEP(l,u) \ @@ -1129,6 +1133,7 @@ default: WRONG("State engine misfire"); } + WS_Assert(w->ws); CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC); } WSL_Flush(w, 0); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-03-04 13:00:18 UTC (rev 3880) @@ -349,6 +349,11 @@ w->beresp = NULL; w->resp = NULL; w->wrq->func(w, w->wrq->priv); + AZ(w->bereq); + AZ(w->beresp1); + AZ(w->beresp); + AZ(w->resp); + WS_Assert(w->ws); AZ(w->wfd); assert(w->wlp == w->wlb); w->wrq = NULL; Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2009-03-04 13:00:18 UTC (rev 3880) @@ -49,16 +49,16 @@ WSP(sp, SLT_Length, "%u", 0); - http_ClrHeader(sp->http); - sp->http->logtag = HTTP_Tx; - http_SetResp(sp->http, "HTTP/1.1", "304", "Not Modified"); + http_ClrHeader(sp->wrk->resp); + sp->wrk->resp->logtag = HTTP_Tx; + http_SetResp(sp->wrk->resp, "HTTP/1.1", "304", "Not Modified"); TIM_format(sp->t_req, lm); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Date: %s", lm); - http_SetHeader(sp->wrk, sp->fd, sp->http, "Via: 1.1 varnish"); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "X-Varnish: %u", sp->xid); + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", lm); + http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish"); + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "X-Varnish: %u", sp->xid); TIM_format(sp->obj->last_modified, lm); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Last-Modified: %s", lm); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Connection: %s", + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Last-Modified: %s", lm); + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s", sp->doclose ? "close" : "keep-alive"); sp->wantbody = 0; } @@ -99,29 +99,29 @@ WSP(sp, SLT_Length, "%u", sp->obj->len); - http_ClrHeader(sp->http); - sp->http->logtag = HTTP_Tx; - http_CopyResp(sp->http, sp->obj->http); - http_FilterFields(sp->wrk, sp->fd, sp->http, sp->obj->http, + http_ClrHeader(sp->wrk->resp); + sp->wrk->resp->logtag = HTTP_Tx; + http_CopyResp(sp->wrk->resp, sp->obj->http); + http_FilterFields(sp->wrk, sp->fd, sp->wrk->resp, sp->obj->http, HTTPH_A_DELIVER); /* Only HTTP 1.1 can do Chunked encoding */ if (sp->http->protover < 1.1 && !VTAILQ_EMPTY(&sp->obj->esibits)) - http_Unset(sp->http, H_Transfer_Encoding); + http_Unset(sp->wrk->resp, H_Transfer_Encoding); TIM_format(TIM_real(), time_str); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Date: %s", time_str); + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", time_str); if (sp->xid != sp->obj->xid) - http_PrintfHeader(sp->wrk, sp->fd, sp->http, + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "X-Varnish: %u %u", sp->xid, sp->obj->xid); else - http_PrintfHeader(sp->wrk, sp->fd, sp->http, + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "X-Varnish: %u", sp->xid); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Age: %.0f", + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Age: %.0f", sp->obj->age + sp->t_resp - sp->obj->entered); - http_SetHeader(sp->wrk, sp->fd, sp->http, "Via: 1.1 varnish"); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Connection: %s", + http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish"); + http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s", sp->doclose ? "close" : "keep-alive"); } @@ -140,7 +140,7 @@ WRW_Reserve(sp->wrk, &sp->fd); if (sp->esis == 0) - sp->acct_req.hdrbytes += http_Write(sp->wrk, sp->http, 1); + sp->acct_req.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1); if (sp->wantbody && !VTAILQ_EMPTY(&sp->obj->esibits)) { if (WRW_FlushRelease(sp->wrk)) { Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-03-04 13:00:18 UTC (rev 3880) @@ -107,7 +107,7 @@ hp = sp->wrk->beresp; break; case HDR_RESP: - hp = sp->http; + hp = sp->wrk->resp; break; case HDR_OBJ: hp = sp->obj->http; @@ -249,8 +249,8 @@ VRT_DO_HDR(bereq, proto, sp->wrk->bereq, HTTP_HDR_PROTO) VRT_DO_HDR(obj, proto, sp->obj->http, HTTP_HDR_PROTO) VRT_DO_HDR(obj, response, sp->obj->http, HTTP_HDR_RESPONSE) -VRT_DO_HDR(resp, proto, sp->http, HTTP_HDR_PROTO) -VRT_DO_HDR(resp, response, sp->http, HTTP_HDR_RESPONSE) +VRT_DO_HDR(resp, proto, sp->wrk->resp, HTTP_HDR_PROTO) +VRT_DO_HDR(resp, response, sp->wrk->resp, HTTP_HDR_RESPONSE) VRT_DO_HDR(beresp, proto, sp->wrk->beresp, HTTP_HDR_PROTO) VRT_DO_HDR(beresp, response, sp->wrk->beresp, HTTP_HDR_RESPONSE) @@ -290,20 +290,20 @@ char *p; assert(num >= 100 && num <= 999); - p = WS_Alloc(sp->http->ws, 4); + p = WS_Alloc(sp->wrk->ws, 4); if (p == NULL) WSP(sp, SLT_LostHeader, "%s", "resp.status"); else sprintf(p, "%d", num); - http_SetH(sp->http, HTTP_HDR_STATUS, p); + http_SetH(sp->wrk->resp, HTTP_HDR_STATUS, p); } int VRT_r_resp_status(const struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); - return (atoi(sp->http->hd[HTTP_HDR_STATUS].b)); + CHECK_OBJ_NOTNULL(sp->wrk->resp, HTTP_MAGIC); + return (atoi(sp->wrk->resp->hd[HTTP_HDR_STATUS].b)); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-03-04 13:00:18 UTC (rev 3880) @@ -322,7 +322,7 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 3834 2009-02-27 12:02:50Z "); + vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 3838 2009-02-27 15:20:57Z "); vsb_cat(sb, "phk $\n *\n * NB: This file is machine generated, DO "); vsb_cat(sb, "NOT EDIT!\n *\n * Edit vcc_gen_obj.tcl instead\n"); vsb_cat(sb, " */\n\nstruct sockaddr * VRT_r_client_ip(const struct "); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2009-03-04 13:00:18 UTC (rev 3880) @@ -59,22 +59,22 @@ # Request paramters { req.request RW STRING - {recv pipe pass hash miss hit fetch error } + {recv pipe pass hash miss hit fetch deliver error } "const struct sess *" } { req.url RW STRING - {recv pipe pass hash miss hit fetch error } + {recv pipe pass hash miss hit fetch deliver error } "const struct sess *" } { req.proto RW STRING - {recv pipe pass hash miss hit fetch error } + {recv pipe pass hash miss hit fetch deliver error } "const struct sess *" } { req.http. RW HDR_REQ - {recv pipe pass hash miss hit fetch error } + {recv pipe pass hash miss hit fetch deliver error } "const struct sess *" } @@ -86,7 +86,7 @@ } { req.backend RW BACKEND - {recv pipe pass hash miss hit fetch error } + {recv pipe pass hash miss hit fetch deliver error } "struct sess *" } { req.restarts Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-03-04 12:28:16 UTC (rev 3879) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-03-04 13:00:18 UTC (rev 3880) @@ -36,25 +36,29 @@ "VRT_r_req_request(sp)", "VRT_l_req_request(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH - | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_ERROR + | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR }, { "req.url", STRING, 7, "VRT_r_req_url(sp)", "VRT_l_req_url(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH - | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_ERROR + | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR }, { "req.proto", STRING, 9, "VRT_r_req_proto(sp)", "VRT_l_req_proto(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH - | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_ERROR + | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR }, { "req.http.", HEADER, 9, "VRT_r_req_http_(sp)", "VRT_l_req_http_(sp, ", V_RW, "HDR_REQ", VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH - | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_ERROR + | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR }, { "req.hash", HASH, 8, NULL, "VRT_l_req_hash(sp, ", @@ -65,7 +69,8 @@ "VRT_r_req_backend(sp)", "VRT_l_req_backend(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH - | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_ERROR + | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR }, { "req.restarts", INT, 12, "VRT_r_req_restarts(sp)", NULL, From petter at projects.linpro.no Wed Mar 4 15:06:17 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Wed, 4 Mar 2009 16:06:17 +0100 (CET) Subject: r3881 - in trunk/varnish-tools/webgui: . Varnish templates Message-ID: <20090304150617.B353B1F753B@projects.linpro.no> Author: petter Date: 2009-03-04 16:06:17 +0100 (Wed, 04 Mar 2009) New Revision: 3881 Modified: trunk/varnish-tools/webgui/Varnish/DB.pm trunk/varnish-tools/webgui/Varnish/Management.pm trunk/varnish-tools/webgui/Varnish/Node.pm trunk/varnish-tools/webgui/Varnish/NodeManager.pm trunk/varnish-tools/webgui/Varnish/RequestHandler.pm trunk/varnish-tools/webgui/Varnish/Util.pm trunk/varnish-tools/webgui/create_db_data.pl trunk/varnish-tools/webgui/templates/node_management.tmpl trunk/varnish-tools/webgui/varnish-webui.pl trunk/varnish-tools/webgui/varnish_webgui.sql Log: Added support for the authentication of the management console. This requires changes to the DB schema, so the DB must be rebuilt. Added possibility to set the inheritance when moving nodes around. Fixed sending of correct Content-Type header. Modified: trunk/varnish-tools/webgui/Varnish/DB.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/DB.pm 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/Varnish/DB.pm 2009-03-04 15:06:17 UTC (rev 3881) @@ -101,14 +101,15 @@ sub add_node { my ($self, $node) = @_; - my $fields = "name, address, port, group_id, management_port"; - my $sql = "INSERT INTO node($fields) VALUES(?, ?, ?, ?, ?)"; + my $fields = "name, address, port, group_id, management_port, management_secret"; + my $sql = "INSERT INTO node($fields) VALUES(?, ?, ?, ?, ?, ?)"; $dbh->do($sql, undef, $node->get_name(), $node->get_address(), $node->get_port(), $node->get_group_id(), - $node->get_management_port()); + $node->get_management_port(), + $node->get_management_secret()); $dbh->commit(); $node->set_id($dbh->func('last_insert_rowid')); @@ -119,9 +120,9 @@ my $sql = "UPDATE node SET name = ?, address = ?, port = ?, group_id = ?, " - . "management_port = ? where id = ?"; + . "management_port = ?, management_secret = ? WHERE id = ?"; $dbh->do($sql, undef, $node->get_name, $node->get_address(), $node->get_port(), - $node->get_group_id(), $node->get_management_port(), + $node->get_group_id(), $node->get_management_port(), $node->get_management_secret(), $node->get_id()); $dbh->commit(); } Modified: trunk/varnish-tools/webgui/Varnish/Management.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/Management.pm 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/Varnish/Management.pm 2009-03-04 15:06:17 UTC (rev 3881) @@ -7,19 +7,22 @@ use Exporter; use List::Util qw(first); use Varnish::Util qw(set_error get_error no_error); +use Digest::SHA qw(sha256_hex); { my %hostname_of; my %port_of; my %socket_of; + my %secret_of; sub new { - my ($class, $hostname, $port) = @_; + my ($class, $hostname, $port, $secret) = @_; my $new_object = bless \do{ my $anon_scalar; }, $class; $hostname_of{$new_object} = $hostname; $port_of{$new_object} = $port; + $secret_of{$new_object} = $secret; return $new_object; } @@ -59,13 +62,28 @@ my $select = IO::Select->new(); $select->add($socket); + my $status_code; + my $response; # wait 100ms, tops, before assuming we don't get a banner if ($select->can_read(0.1)) { - _read_cli_response($socket); + ($status_code, $response) = _read_cli_response($socket); } my $flags = fcntl($socket, F_GETFL, 0); $flags = fcntl($socket, F_SETFL, $flags & ~O_NONBLOCK); + + if ($status_code && $status_code eq "107") { + my ($challenge) = ($response =~ /^(.*)$/m); + my $challenge_response_text = + "$challenge\n" . $secret_of{$self} . "\n$challenge\n"; + print $socket "auth " . sha256_hex($challenge_response_text) . "\n"; + my ($status_code, $response) = _read_cli_response($socket); + if ($status_code ne "200") { + close($socket); + return ("666", "Management port authentication failed."); + } + } + $socket_of{$self} = $socket; } my $socket = $socket_of{$self}; @@ -242,8 +260,8 @@ sub ping { my ($self) = @_; - my ($status_code, $response) = _send_command($self, "stats"); - + my ($status_code, $response) = _send_command($self, "ping"); + return no_error($self) if ($status_code eq "200"); return set_error($response); } Modified: trunk/varnish-tools/webgui/Varnish/Node.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/Node.pm 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/Varnish/Node.pm 2009-03-04 15:06:17 UTC (rev 3881) @@ -13,6 +13,7 @@ my %group_id_of; my %management_of; my %management_port_of; + my %management_secret_of; my %id_of; sub new { @@ -35,8 +36,10 @@ $group_id_of{$new_object} = 0; } $management_port_of{$new_object} = $arg_ref->{'management_port'}; + $management_secret_of{$new_object} = $arg_ref->{'management_secret'}; $management_of{$new_object} = Varnish::Management->new($arg_ref->{'address'}, - $arg_ref->{'management_port'}); + $arg_ref->{'management_port'}, + $arg_ref->{'management_secret'}); return $new_object; } @@ -118,6 +121,20 @@ $management_port_of{$self} = $management_port; } + sub get_management_secret { + my ($self) = @_; + + return $management_secret_of{$self}; + } + + sub set_management_secret { + my ($self, $management_secret) = @_; + + $management_secret_of{$self} = $management_secret; + } + + + sub set_id { my ($self, $id) = @_; Modified: trunk/varnish-tools/webgui/Varnish/NodeManager.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/NodeManager.pm 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/Varnish/NodeManager.pm 2009-03-04 15:06:17 UTC (rev 3881) @@ -57,7 +57,7 @@ $inheritance ||= 0; my $management = $node->get_management(); if (!$management->ping()) { - return set_error($self, "Could not connect to management port: " + return set_error("Could not connect to management port: " . get_error()); } Varnish::DB->add_node($node); @@ -140,14 +140,23 @@ } sub update_node { - my ($self, $node) = @_; + my ($self, $node, $inheritance) = @_; + $inheritance ||= 0; my $current = get_node($self, $node->get_id()); if ($current->get_group_id() != $node->get_group_id() - && $node->get_group_id() > 0) { + && $node->get_group_id() > 0 + && $inheritance) { my $group = get_group($self, $node->get_group_id()); - _clone_unit($group, $node); + + if ($inheritance == 1) { + _clone_unit($node, $group); + } + elsif ($inheritance == 2) { + _clone_unit($group, $node); + } } + Varnish::DB->update_node($node); } Modified: trunk/varnish-tools/webgui/Varnish/RequestHandler.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/Varnish/RequestHandler.pm 2009-03-04 15:06:17 UTC (rev 3881) @@ -50,8 +50,9 @@ sub get_response_header { my ($self) = @_; - return $response_header_ref_of{$self}; + return %{$response_header_ref_of{$self}}; } + sub get_response_content { my ($self) = @_; @@ -102,6 +103,7 @@ my $param; my $use_master_template; + $response_header_ref_of{$self}->{'Content-Type'} = "text/html"; if ($operation eq 'view_stats' || $operation eq '') { ($content_template, $param, $use_master_template) = view_stats(\%request_parameter); } @@ -764,6 +766,7 @@ $param{'address'} = $$parameter_ref{'address'} || ""; $param{'port'} ||= ""; $param{'management_port'} ||= ""; + $param{'management_secret'} ||= ""; $param{'inheritance'} ||= 0; $param{'edit_node'} ||= -1; @@ -782,6 +785,7 @@ $tmpl_var{'show_add_node'} = 1; $tmpl_var{'show_node_in_backend_health'} = 1; $tmpl_var{'show_inheritance_settings'} = 1; + $tmpl_var{'show_management_secret'} = 1; $tmpl_var{'inheritance_settings'} = []; my $error = ""; @@ -857,10 +861,17 @@ address => $param{'address'}, port => $param{'port'}, group_id => $param{'group_id'}, - management_port => $param{'management_port'} + management_port => $param{'management_port'}, + management_secret => $param{'management_secret'} }); - Varnish::NodeManager->add_node($node, $param{'inheritance'}); - $status .= "Node " . $node->get_name() . " added successfully."; + if (Varnish::NodeManager->add_node($node, $param{'inheritance'})) { + $status .= "Node " . $node->get_name() . " added successfully."; + } + else { + $error .= "Could not add node " . $node->get_name() . ": " . get_error(); + @tmpl_var{'new_name', 'new_address', 'new_port', 'new_management_port', 'new_management_secret'} = + @param{'name', 'address', 'port', 'management_port', 'management_secret'}; + } my $group = Varnish::NodeManager->get_group($param{'group_id'}); my $group_name = ($group ? $group->get_name() : ""); @@ -873,6 +884,7 @@ . " [port=" . $node->get_port() . "]" . " [group=" . $group_name . "]" . " [management_port=" . $node->get_management_port() . "]" + . " [management_port=******]" . " [settings_inheritance=$inheritance]"); } else { @@ -880,6 +892,7 @@ $error .= "Name: " . $param{'name'} . ":\n"; $error .= "Address: " . $param{'address'} . ":\n"; $error .= "Port: " . $param{'port'} . ":\n"; + $error .= "Management secret: ******\n"; $error .= "Management port: " . $param{'management_port'} . ":\n"; } } @@ -892,8 +905,9 @@ $node->set_port($param{'port'}); $node->set_group_id($param{'node_group_id'}); $node->set_management_port($param{'management_port'}); + $node->set_management_secret($param{'management_secret'}); - Varnish::NodeManager->update_node($node); + Varnish::NodeManager->update_node($node, $param{'inheritance'}); $status .= "Node " . $node->get_name() . " updated successfully."; @@ -904,6 +918,7 @@ . " [address=" . $node->get_address() . "]" . " [port=" . $node->get_port() . "]" . " [group=" . $group_name . "]" + . " [management_secret=******]" . " [management_port=" . $node->get_management_port() . "]"); } } @@ -1000,6 +1015,7 @@ address => $node->get_address(), port => $node->get_port(), management_port => $node->get_management_port(), + management_secret => $node->get_management_secret() ? "******" : "", group => $group_name, edit => $node->get_id() == $param{'edit_node'}, }; @@ -1020,7 +1036,8 @@ $tmpl_var{'add_group'} = 1; } - if ($tmpl_var{'group_id'} > 0) { + if ($tmpl_var{'group_id'} > 0 + || $param{'edit_node'} > -1) { my @inheritance_settings; push @inheritance_settings, { value => 2, @@ -1049,6 +1066,7 @@ $tmpl_var{'group_name'} = $selected_group->get_name(); } $tmpl_var{'show_group_controls'} = $tmpl_var{'group_id'} > 0; + $tmpl_var{'show_management_secret'} = $tmpl_var{'group_id'} >= 0; $tmpl_var{'show_group'} = $tmpl_var{'group_id'} == -1 || $param{'edit_node'} > -1; $tmpl_var{'show_add_node'} = $tmpl_var{'group_id'} >= 0; $tmpl_var{'error'} = $error; Modified: trunk/varnish-tools/webgui/Varnish/Util.pm =================================================================== --- trunk/varnish-tools/webgui/Varnish/Util.pm 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/Varnish/Util.pm 2009-03-04 15:06:17 UTC (rev 3881) @@ -52,17 +52,18 @@ sub set_config { my ($config_ref) = @_; + + my @keys = keys %$config_ref; + for my $key (@keys) { + $config{$key} = $config_ref->{$key}; + } - %config = %{$config_ref}; - if ($config{'log_filename'}) { if (!open($log_handle, ">>" . $config{'log_filename'})) { die "Could not open log file " . $config{'log_filename'} . " for writing"; } $log_handle->autoflush(1); # FIXME: Remove it, or is it usefull? } - - Varnish::DB->init($config{'db_filename'}); } sub print_config { @@ -128,7 +129,7 @@ sub set_error { my ($new_error) = @_; - + $error = $new_error; return; @@ -136,7 +137,7 @@ sub get_error { - return $error; + return $error ? $error : ''; } sub no_error { Modified: trunk/varnish-tools/webgui/create_db_data.pl =================================================================== --- trunk/varnish-tools/webgui/create_db_data.pl 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/create_db_data.pl 2009-03-04 15:06:17 UTC (rev 3881) @@ -82,7 +82,7 @@ port TEXT, group_id INTEGER, management_port TEXT, - is_master BOOLEAN + management_secret TEXT ); CREATE TABLE stat ( Modified: trunk/varnish-tools/webgui/templates/node_management.tmpl =================================================================== --- trunk/varnish-tools/webgui/templates/node_management.tmpl 2009-03-04 13:00:18 UTC (rev 3880) +++ trunk/varnish-tools/webgui/templates/node_management.tmpl 2009-03-04 15:06:17 UTC (rev 3881) @@ -43,6 +43,9 @@ Address Port Management
port + +Management
secret + Inheritance
settings @@ -91,8 +94,17 @@ + + + - + + + @@ -111,6 +123,9 @@ + + + @@ -156,13 +171,16 @@ - + - - - + + + + + +