From sky at projects.linpro.no Mon Aug 3 10:25:42 2009 From: sky at projects.linpro.no (sky at projects.linpro.no) Date: Mon, 3 Aug 2009 12:25:42 +0200 (CEST) Subject: r4171 - trunk/varnish-cache/lib/libvarnish Message-ID: <20090803102542.EFEFA1F7C72@projects.linpro.no> Author: sky Date: 2009-08-03 12:25:42 +0200 (Mon, 03 Aug 2009) New Revision: 4171 Modified: trunk/varnish-cache/lib/libvarnish/tcp.c Log: Implement the linux equivalent of BSD accept filter using TCP_DEFER_ACCEPT Modified: trunk/varnish-cache/lib/libvarnish/tcp.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/tcp.c 2009-07-31 15:27:34 UTC (rev 4170) +++ trunk/varnish-cache/lib/libvarnish/tcp.c 2009-08-03 10:25:42 UTC (rev 4171) @@ -37,6 +37,10 @@ #include +#ifdef __linux +#include +#endif + #include #include #ifdef HAVE_SYS_FILIO_H @@ -114,6 +118,10 @@ printf("Acceptfilter(%d, httpready): %d %s\n", sock, i, strerror(errno)); return (i); +#elif defined(__linux) + int defer = 1; + setsockopt(sock, SOL_TCP,TCP_DEFER_ACCEPT,(char *) &defer, sizeof(int)); + return (0); #else (void)sock; return (0); From sky at projects.linpro.no Mon Aug 3 10:46:32 2009 From: sky at projects.linpro.no (sky at projects.linpro.no) Date: Mon, 3 Aug 2009 12:46:32 +0200 (CEST) Subject: r4172 - in branches/sky/response/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20090803104632.BABBC1F7C71@projects.linpro.no> Author: sky Date: 2009-08-03 12:46:32 +0200 (Mon, 03 Aug 2009) New Revision: 4172 Modified: branches/sky/response/varnish-cache/bin/varnishd/cache_center.c branches/sky/response/varnish-cache/bin/varnishd/steps.h branches/sky/response/varnish-cache/include/vcl.h branches/sky/response/varnish-cache/include/vcl_returns.h branches/sky/response/varnish-cache/lib/libvcl/vcc_fixed_token.c branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_obj.tcl branches/sky/response/varnish-cache/lib/libvcl/vcc_obj.c Log: rename resp to response since otherwise it uhm, conflicts with the resp object Modified: branches/sky/response/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/sky/response/varnish-cache/bin/varnishd/cache_center.c 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/bin/varnishd/cache_center.c 2009-08-03 10:46:32 UTC (rev 4172) @@ -341,7 +341,7 @@ TIM_format(TIM_real(), date); http_PrintfHeader(w, sp->fd, h, "Date: %s", date); http_PrintfHeader(w, sp->fd, h, "Server: Varnish"); - if (sp->step != STP_RESP) + if (sp->step != STP_RESPONSE) http_PrintfHeader(w, sp->fd, h, "Retry-After: %d", params->err_ttl); if (sp->err_reason != NULL) @@ -350,8 +350,8 @@ http_PutResponse(w, sp->fd, h, http_StatusMessage(sp->err_code)); - if (sp->step == STP_RESP) { - VCL_resp_method(sp); + if (sp->step == STP_RESPONSE) { + VCL_response_method(sp); } else { VCL_error_method(sp); } @@ -365,7 +365,7 @@ } /* We always close when we take this path */ - if (sp->step != STP_RESP) + if (sp->step != STP_RESPONSE) sp->doclose = "error"; sp->wantbody = 1; @@ -392,7 +392,7 @@ */ static int -cnt_resp(struct sess *sp) +cnt_response(struct sess *sp) { sp->err_code = 200; return cnt_error(sp); @@ -1041,8 +1041,8 @@ /* XXX: discard req body, if any */ sp->step = STP_ERROR; return (0); - case VCL_RET_RESP: - sp->step = STP_RESP; + case VCL_RET_RESPONSE: + sp->step = STP_RESPONSE; return (0); default: WRONG("Illegal action in vcl_recv{}"); Modified: branches/sky/response/varnish-cache/bin/varnishd/steps.h =================================================================== --- branches/sky/response/varnish-cache/bin/varnishd/steps.h 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/bin/varnishd/steps.h 2009-08-03 10:46:32 UTC (rev 4172) @@ -42,4 +42,4 @@ STEP(deliver, DELIVER) STEP(error, ERROR) STEP(done, DONE) -STEP(resp, RESP) +STEP(response, RESPONSE) Modified: branches/sky/response/varnish-cache/include/vcl.h =================================================================== --- branches/sky/response/varnish-cache/include/vcl.h 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/include/vcl.h 2009-08-03 10:46:32 UTC (rev 4172) @@ -23,7 +23,7 @@ #define VCL_MET_FETCH (1 << 6) #define VCL_MET_DELIVER (1 << 7) #define VCL_MET_ERROR (1 << 8) -#define VCL_MET_RESP (1 << 9) +#define VCL_MET_RESPONSE (1 << 9) #define VCL_MET_MAX 10 @@ -38,7 +38,7 @@ #define VCL_RET_DISCARD 7 #define VCL_RET_KEEP 8 #define VCL_RET_RESTART 9 -#define VCL_RET_RESP 10 +#define VCL_RET_RESPONSE 10 #define VCL_RET_MAX 11 @@ -71,5 +71,5 @@ vcl_func_f *fetch_func; vcl_func_f *deliver_func; vcl_func_f *error_func; - vcl_func_f *resp_func; + vcl_func_f *response_func; }; Modified: branches/sky/response/varnish-cache/include/vcl_returns.h =================================================================== --- branches/sky/response/varnish-cache/include/vcl_returns.h 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/include/vcl_returns.h 2009-08-03 10:46:32 UTC (rev 4172) @@ -17,13 +17,13 @@ VCL_RET_MAC(discard, DISCARD) VCL_RET_MAC(keep, KEEP) VCL_RET_MAC(restart, RESTART) -VCL_RET_MAC(resp, RESP) +VCL_RET_MAC(response, RESPONSE) #endif #ifdef VCL_MET_MAC VCL_MET_MAC(recv,RECV, ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_RESP) + | (1 << VCL_RET_RESPONSE) | (1 << VCL_RET_PASS) | (1 << VCL_RET_PIPE) | (1 << VCL_RET_LOOKUP) @@ -42,12 +42,14 @@ )) VCL_MET_MAC(miss,MISS, ((1 << VCL_RET_ERROR) + | (1 << VCL_RET_RESPONSE) | (1 << VCL_RET_RESTART) | (1 << VCL_RET_PASS) | (1 << VCL_RET_FETCH) )) VCL_MET_MAC(hit,HIT, ((1 << VCL_RET_ERROR) + | (1 << VCL_RET_RESPONSE) | (1 << VCL_RET_RESTART) | (1 << VCL_RET_PASS) | (1 << VCL_RET_DELIVER) @@ -66,7 +68,7 @@ ((1 << VCL_RET_RESTART) | (1 << VCL_RET_DELIVER) )) -VCL_MET_MAC(resp,RESP, +VCL_MET_MAC(response,RESPONSE, ((1 << VCL_RET_RESTART) | (1 << VCL_RET_DELIVER) )) Modified: branches/sky/response/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- branches/sky/response/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-03 10:46:32 UTC (rev 4172) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10:41:38Z phk $ + * $Id: vcc_gen_fixed_token.tcl 4149 2009-07-28 13:30:49Z sky $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,8 +159,8 @@ /* ../../include/vcl.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10"); - vsb_cat(sb, ":41:38Z phk $\n *\n * NB: This file is machine genera"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4149 2009-07-28 13"); + vsb_cat(sb, ":30:49Z sky $\n *\n * NB: This file is machine genera"); vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sess;\n"); vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); @@ -175,7 +175,7 @@ vsb_cat(sb, "#define VCL_MET_FETCH\t\t(1 << 6)\n"); vsb_cat(sb, "#define VCL_MET_DELIVER\t\t(1 << 7)\n"); vsb_cat(sb, "#define VCL_MET_ERROR\t\t(1 << 8)\n"); - vsb_cat(sb, "#define VCL_MET_RESP\t\t(1 << 9)\n"); + vsb_cat(sb, "#define VCL_MET_RESPONSE\t(1 << 9)\n"); vsb_cat(sb, "\n#define VCL_MET_MAX\t\t10\n\n"); vsb_cat(sb, "/* VCL Returns */\n#define VCL_RET_ERROR\t\t0\n"); vsb_cat(sb, "#define VCL_RET_LOOKUP\t\t1\n#define VCL_RET_HASH\t\t2"); @@ -183,7 +183,7 @@ vsb_cat(sb, "\n#define VCL_RET_FETCH\t\t5\n#define VCL_RET_DELIVER\t"); vsb_cat(sb, "\t6\n#define VCL_RET_DISCARD\t\t7\n"); vsb_cat(sb, "#define VCL_RET_KEEP\t\t8\n#define VCL_RET_RESTART\t\t"); - vsb_cat(sb, "9\n#define VCL_RET_RESP\t\t10\n"); + vsb_cat(sb, "9\n#define VCL_RET_RESPONSE\t\t10\n"); vsb_cat(sb, "\n#define VCL_RET_MAX\t\t11\n\n"); vsb_cat(sb, "struct VCL_conf {\n\tunsigned\tmagic;\n"); vsb_cat(sb, "#define VCL_CONF_MAGIC\t0x7406c509\t/* from /dev/rando"); @@ -199,7 +199,8 @@ vsb_cat(sb, "\tvcl_func_f\t*miss_func;\n\tvcl_func_f\t*hit_func;\n"); vsb_cat(sb, "\tvcl_func_f\t*fetch_func;\n\tvcl_func_f\t*deliver_fun"); vsb_cat(sb, "c;\n\tvcl_func_f\t*error_func;\n"); - vsb_cat(sb, "\tvcl_func_f\t*resp_func;\n};\n"); + vsb_cat(sb, "\tvcl_func_f\t*response_func;\n"); + vsb_cat(sb, "};\n"); /* ../../include/vrt.h */ @@ -321,8 +322,8 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 4100 2009-06-09 10:41:38Z "); - vsb_cat(sb, "phk $\n *\n * NB: This file is machine generated, DO "); + vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 4149 2009-07-28 13:30:49Z "); + vsb_cat(sb, "sky $\n *\n * NB: This file is machine generated, DO "); vsb_cat(sb, "NOT EDIT!\n *\n * Edit vcc_gen_obj.tcl instead\n"); vsb_cat(sb, " */\n\nstruct sockaddr * VRT_r_client_ip(const struct "); vsb_cat(sb, "sess *);\nstruct sockaddr * VRT_r_server_ip(struct ses"); Modified: branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-03 10:46:32 UTC (rev 4172) @@ -34,16 +34,16 @@ # Second element is list of valid return actions. # set methods { - {recv {error resp pass pipe lookup}} + {recv {error response pass pipe lookup}} {pipe {error pipe}} {pass {error restart pass}} {hash {hash}} - {miss {error restart pass fetch}} - {hit {error restart pass deliver}} + {miss {error response restart pass fetch}} + {hit {error response restart pass deliver}} {fetch {error restart pass deliver}} {deliver {restart deliver}} {error {restart deliver}} - {resp {restart deliver}} + {response {restart deliver}} } # These are the return actions @@ -59,7 +59,7 @@ discard keep restart - resp + response } # Language keywords Modified: branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2009-08-03 10:46:32 UTC (rev 4172) @@ -38,7 +38,7 @@ # Connection related parameters { client.ip RO IP - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "const struct sess *" } { client.bandwidth # Not implemented yet @@ -48,47 +48,47 @@ } { server.ip RO IP - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "struct sess *" } { server.hostname RO STRING - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "struct sess *" } { server.identity RO STRING - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "struct sess *" } { server.port RO INT - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "struct sess *" } # Request paramters { req.request RW STRING - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "const struct sess *" } { req.url RW STRING - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "const struct sess *" } { req.proto RW STRING - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "const struct sess *" } { req.http. RW HDR_REQ - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "const struct sess *" } - # Possibly misnamed, not really part of the request resp + # Possibly misnamed, not really part of the request response { req.hash WO HASH { hash error } @@ -96,12 +96,12 @@ } { req.backend RW BACKEND - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "struct sess *" } { req.restarts RO INT - {recv pipe pass hash miss hit fetch deliver resp error } + {recv pipe pass hash miss hit fetch deliver response error } "const struct sess *" } { req.grace @@ -112,7 +112,7 @@ { req.xid RO STRING - {recv pipe pass hash miss hit fetch deliver resp error} + {recv pipe pass hash miss hit fetch deliver response error} "struct sess *" } @@ -202,27 +202,27 @@ # The (possibly) cached object { obj.proto RW STRING - { hit resp error} + { hit response error} "const struct sess *" } { obj.status RW INT - { resp error} + { response error} "const struct sess *" } { obj.response RW STRING - { resp error} + { response error} "const struct sess *" } { obj.hits RO INT - { hit deliver resp } + { hit deliver response } "const struct sess *" } { obj.http. RW HDR_OBJ - { hit resp error} + { hit response error} "const struct sess *" } Modified: branches/sky/response/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- branches/sky/response/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-03 10:25:42 UTC (rev 4171) +++ branches/sky/response/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-03 10:46:32 UTC (rev 4172) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_obj.tcl 4100 2009-06-09 10:41:38Z phk $ + * $Id: vcc_gen_obj.tcl 4149 2009-07-28 13:30:49Z sky $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -16,63 +16,63 @@ V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "server.ip", IP, 9, "VRT_r_server_ip(sp)", NULL, V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "server.hostname", STRING, 15, "VRT_r_server_hostname(sp)", NULL, V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "server.identity", STRING, 15, "VRT_r_server_identity(sp)", NULL, V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "server.port", INT, 11, "VRT_r_server_port(sp)", NULL, V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.request", STRING, 11, "VRT_r_req_request(sp)", "VRT_l_req_request(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.url", STRING, 7, "VRT_r_req_url(sp)", "VRT_l_req_url(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.proto", STRING, 9, "VRT_r_req_proto(sp)", "VRT_l_req_proto(sp, ", V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.http.", HEADER, 9, "VRT_r_req_http_(sp)", "VRT_l_req_http_(sp, ", V_RW, "HDR_REQ", VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.hash", HASH, 8, NULL, "VRT_l_req_hash(sp, ", @@ -84,14 +84,14 @@ V_RW, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.restarts", INT, 12, "VRT_r_req_restarts(sp)", NULL, V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.grace", TIME, 9, "VRT_r_req_grace(sp)", "VRT_l_req_grace(sp, ", @@ -105,7 +105,7 @@ V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - | VCL_MET_RESP | VCL_MET_ERROR + | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "req.esi", BOOL, 7, "VRT_r_req_esi(sp)", "VRT_l_req_esi(sp, ", @@ -185,27 +185,27 @@ { "obj.proto", STRING, 9, "VRT_r_obj_proto(sp)", "VRT_l_obj_proto(sp, ", V_RW, 0, - VCL_MET_HIT | VCL_MET_RESP | VCL_MET_ERROR + VCL_MET_HIT | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "obj.status", INT, 10, "VRT_r_obj_status(sp)", "VRT_l_obj_status(sp, ", V_RW, 0, - VCL_MET_RESP | VCL_MET_ERROR + VCL_MET_RESPONSE | VCL_MET_ERROR }, { "obj.response", STRING, 12, "VRT_r_obj_response(sp)", "VRT_l_obj_response(sp, ", V_RW, 0, - VCL_MET_RESP | VCL_MET_ERROR + VCL_MET_RESPONSE | VCL_MET_ERROR }, { "obj.hits", INT, 8, "VRT_r_obj_hits(sp)", NULL, V_RO, 0, - VCL_MET_HIT | VCL_MET_DELIVER | VCL_MET_RESP + VCL_MET_HIT | VCL_MET_DELIVER | VCL_MET_RESPONSE }, { "obj.http.", HEADER, 9, "VRT_r_obj_http_(sp)", "VRT_l_obj_http_(sp, ", V_RW, "HDR_OBJ", - VCL_MET_HIT | VCL_MET_RESP | VCL_MET_ERROR + VCL_MET_HIT | VCL_MET_RESPONSE | VCL_MET_ERROR }, { "obj.cacheable", BOOL, 13, "VRT_r_obj_cacheable(sp)", "VRT_l_obj_cacheable(sp, ", From tfheen at projects.linpro.no Tue Aug 4 08:04:13 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 4 Aug 2009 10:04:13 +0200 (CEST) Subject: r4173 - trunk/varnish-cache/bin/varnishtop Message-ID: <20090804080413.AD8FE1F7B45@projects.linpro.no> Author: tfheen Date: 2009-08-04 10:04:13 +0200 (Tue, 04 Aug 2009) New Revision: 4173 Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c Log: varnishtop: handle shm_reclen > 255 varnishtop used to have a hard-coded max record length of 255 bytes. This broke with the change that allows larger record length. Use a dynamic buffer instead. Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2009-08-03 10:46:32 UTC (rev 4172) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2009-08-04 08:04:13 UTC (rev 4173) @@ -55,7 +55,8 @@ #include "varnishapi.h" struct top { - unsigned char rec[4 + 255]; + unsigned char rec[4]; + unsigned char *rec_data; unsigned clen; unsigned hash; VTAILQ_ENTRY(top) list; @@ -100,7 +101,7 @@ continue; if (tp->clen != q - p) continue; - if (memcmp(p + SHMLOG_DATA, tp->rec + SHMLOG_DATA, + if (memcmp(p + SHMLOG_DATA, tp->rec_data, q - (p + SHMLOG_DATA))) continue; tp->count += 1.0; @@ -110,12 +111,15 @@ ntop++; tp = calloc(sizeof *tp, 1); assert(tp != NULL); + tp->rec_data = calloc(l, 1); + assert(tp->rec_data != NULL); tp->hash = u; tp->count = 1.0; tp->clen = q - p; VTAILQ_INSERT_TAIL(&top_head, tp, list); } - memcpy(tp->rec, p, SHMLOG_DATA + l); + memcpy(tp->rec, p, SHMLOG_DATA - 1); + memcpy(tp->rec_data, p + SHMLOG_DATA, l); while (1) { tp2 = VTAILQ_PREV(tp, tophead, list); if (tp2 == NULL || tp2->count >= tp->count) @@ -158,12 +162,13 @@ mvprintw(l, 0, "%9.2f %-*.*s %*.*s\n", tp->count, maxfieldlen, maxfieldlen, VSL_tags[tp->rec[SHMLOG_TAG]], - len, len, tp->rec + SHMLOG_DATA); + len, len, tp->rec_data); t = tp->count; } tp->count *= .999; if (tp->count * 10 < t || l > LINES * 10) { VTAILQ_REMOVE(&top_head, tp, list); + free(tp->rec_data); free(tp); ntop--; } From sky at projects.linpro.no Tue Aug 4 10:54:17 2009 From: sky at projects.linpro.no (sky at projects.linpro.no) Date: Tue, 4 Aug 2009 12:54:17 +0200 (CEST) Subject: r4174 - trunk/varnish-cache/bin/varnishd Message-ID: <20090804105417.6F2DF1F7B45@projects.linpro.no> Author: sky Date: 2009-08-04 12:54:17 +0200 (Tue, 04 Aug 2009) New Revision: 4174 Modified: trunk/varnish-cache/bin/varnishd/cache_lck.c Log: In Lck__Trylock we probably do want to try locks -- and check return value not errno Modified: trunk/varnish-cache/bin/varnishd/cache_lck.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_lck.c 2009-08-04 08:04:13 UTC (rev 4173) +++ trunk/varnish-cache/bin/varnishd/cache_lck.c 2009-08-04 10:54:17 UTC (rev 4174) @@ -115,8 +115,8 @@ int r; CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); - r = pthread_mutex_lock(&ilck->mtx); - assert(r == 0 || errno == EBUSY); + r = pthread_mutex_trylock(&ilck->mtx); + assert(r == 0 || r == EBUSY); if (params->diag_bitmap & 0x8) VSL(SLT_Debug, 0, "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w); From des at des.no Tue Aug 4 17:07:25 2009 From: des at des.no (=?utf-8?Q?Dag-Erling_Sm=C3=B8rgrav?=) Date: Tue, 04 Aug 2009 19:07:25 +0200 Subject: r4161 - trunk/varnish-cache/lib/libvarnish In-Reply-To: <20090729124333.4EF0A1F7BD6@projects.linpro.no> (sky@projects.linpro.no's message of "Wed, 29 Jul 2009 14:43:33 +0200 (CEST)") References: <20090729124333.4EF0A1F7BD6@projects.linpro.no> Message-ID: <86ws5j1p9u.fsf@ds4.des.no> sky at projects.linpro.no writes: > void > TCP_blocking(int sock) > { > int i; > > +#ifdef __sun > + i = fcntl (sock, F_GETFL,0); > + fcntl(sock, F_SETFL, i & ~O_NONBLOCK); > +#else > i = 0; > AZ(ioctl(sock, FIONBIO, &i)); > +#endif > } Missing error check for the fcntl case. DES -- Dag-Erling Sm?rgrav - des at des.no From des at des.no Tue Aug 4 17:08:48 2009 From: des at des.no (=?utf-8?Q?Dag-Erling_Sm=C3=B8rgrav?=) Date: Tue, 04 Aug 2009 19:08:48 +0200 Subject: r4164 - branches/2.0/varnish-cache/lib/libvarnish In-Reply-To: <20090730071055.C73F31F7C1D@projects.linpro.no> (sky@projects.linpro.no's message of "Thu, 30 Jul 2009 09:10:55 +0200 (CEST)") References: <20090730071055.C73F31F7C1D@projects.linpro.no> Message-ID: <86skg71p7j.fsf@ds4.des.no> sky at projects.linpro.no writes: > Modified: > branches/2.0/varnish-cache/lib/libvarnish/tcp.c > Log: > revert merge r4162 You wouldn't have needed to do this if you had waited until r4162 was properly tested before merging it. DES -- Dag-Erling Sm?rgrav - des at des.no From des at des.no Tue Aug 4 17:16:24 2009 From: des at des.no (=?utf-8?Q?Dag-Erling_Sm=C3=B8rgrav?=) Date: Tue, 04 Aug 2009 19:16:24 +0200 Subject: r4173 - trunk/varnish-cache/bin/varnishtop In-Reply-To: <20090804080413.AD8FE1F7B45@projects.linpro.no> (tfheen@projects.linpro.no's message of "Tue, 4 Aug 2009 10:04:13 +0200 (CEST)") References: <20090804080413.AD8FE1F7B45@projects.linpro.no> Message-ID: <86ljlz1ouv.fsf@ds4.des.no> tfheen at projects.linpro.no writes: > varnishtop used to have a hard-coded max record length of 255 bytes. > This broke with the change that allows larger record length. Use a > dynamic buffer instead. A VLA would be simpler and more efficient. (performance *does* matter - on a busy server, an inefficient varnish{log,stat,top} can drag the entire server down) DES -- Dag-Erling Sm?rgrav - des at des.no From tfheen at projects.linpro.no Thu Aug 6 10:53:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 6 Aug 2009 12:53:18 +0200 (CEST) Subject: r4175 - trunk/varnish-cache/lib/libvarnish Message-ID: <20090806105318.3AC1F28461@projects.linpro.no> Author: tfheen Date: 2009-08-06 12:53:17 +0200 (Thu, 06 Aug 2009) New Revision: 4175 Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am trunk/varnish-cache/lib/libvarnish/version.c Log: Reinstate svn_version code Reinstate code to embed the svn version (or git commit id) in the output given by varnishd -V. Also, make sure we don't overwrite version.c when the tree is exported, but generate it if it's completely missing. (This should only happen if somebody does svn export rather than make dist, but some people might do that.) Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2009-08-04 10:54:17 UTC (rev 4174) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2009-08-06 10:53:17 UTC (rev 4175) @@ -15,6 +15,7 @@ cli_common.c \ flopen.c \ num.c \ + svn_version.c \ time.c \ tcp.c \ vct.c \ @@ -29,6 +30,24 @@ libvarnish_la_LIBADD = ${RT_LIBS} ${NET_LIBS} ${LIBM} +DISTCLEANFILES = svn_version.c +svn_version.c: FORCE + V="$$(git log -n 1 --pretty=format:%h 2>/dev/null || LANG=C svnversion -n $(top_srcdir))" \ + H="$$(head -n 1 svn_version.c 2>/dev/null || true)"; \ + [ "$$V" = "exported" ] && [ -e svn_version.c ] && exit 0 ; \ + if [ "/* $$V */" != "$$H" ]; then \ + ( \ + echo "/* $$V */" ;\ + echo "#include " ;\ + echo "const char* svn_version(void)" ;\ + echo "{" ;\ + echo " const char* SVN_Version = \"$$V\";" ;\ + echo " return SVN_Version;" ;\ + echo "}" ;\ + ) > svn_version.c ; \ + fi +FORCE: + if ENABLE_TESTS TESTS = num_c_test Modified: trunk/varnish-cache/lib/libvarnish/version.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/version.c 2009-08-04 10:54:17 UTC (rev 4174) +++ trunk/varnish-cache/lib/libvarnish/version.c 2009-08-06 10:53:17 UTC (rev 4175) @@ -41,8 +41,8 @@ void varnish_version(const char *progname) { - fprintf(stderr, "%s (%s-%s)\n", progname, - PACKAGE_TARNAME, PACKAGE_VERSION); + fprintf(stderr, "%s (%s-%s SVN %s)\n", progname, + PACKAGE_TARNAME, PACKAGE_VERSION, svn_version()); fprintf(stderr, "Copyright (c) 2006-2009 Linpro AS / Verdens Gang AS\n"); } From des at des.no Thu Aug 6 13:57:44 2009 From: des at des.no (=?utf-8?Q?Dag-Erling_Sm=C3=B8rgrav?=) Date: Thu, 06 Aug 2009 15:57:44 +0200 Subject: r4175 - trunk/varnish-cache/lib/libvarnish In-Reply-To: <20090806105318.3AC1F28461@projects.linpro.no> (tfheen@projects.linpro.no's message of "Thu, 6 Aug 2009 12:53:18 +0200 (CEST)") References: <20090806105318.3AC1F28461@projects.linpro.no> Message-ID: <86my6d6o4n.fsf@ds4.des.no> tfheen at projects.linpro.no writes: > Modified: > trunk/varnish-cache/lib/libvarnish/Makefile.am > trunk/varnish-cache/lib/libvarnish/version.c > Log: > Reinstate svn_version code You could have had the decency to wait for the discussion to settle. It hasn't, yet. DES -- Dag-Erling Sm?rgrav - des at des.no From phk at phk.freebsd.dk Thu Aug 6 14:12:15 2009 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Thu, 06 Aug 2009 14:12:15 +0000 Subject: r4175 - trunk/varnish-cache/lib/libvarnish In-Reply-To: Your message of "Thu, 06 Aug 2009 15:57:44 +0200." <86my6d6o4n.fsf@ds4.des.no> Message-ID: <3048.1249567935@critter.freebsd.dk> In message <86my6d6o4n.fsf at ds4.des.no>, =?utf-8?Q?Dag-Erling_Sm=C3=B8rgrav?= wr ites: >You could have had the decency to wait for the discussion to settle. It >hasn't, yet. Yes it has. You have made your point, none of us have been able to see the problem as being significant, given the level of details you have provided. You have also not offered a better solution. Next step in this, is when somebody has an improvement to offer. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From des at des.no Thu Aug 6 14:16:47 2009 From: des at des.no (=?utf-8?Q?Dag-Erling_Sm=C3=B8rgrav?=) Date: Thu, 06 Aug 2009 16:16:47 +0200 Subject: r4175 - trunk/varnish-cache/lib/libvarnish In-Reply-To: <3048.1249567935@critter.freebsd.dk> (Poul-Henning Kamp's message of "Thu, 06 Aug 2009 14:12:15 +0000") References: <3048.1249567935@critter.freebsd.dk> Message-ID: <86zlad58og.fsf@ds4.des.no> "Poul-Henning Kamp" writes: > You have also not offered a better solution. I have. Pay attention! DES -- Dag-Erling Sm?rgrav - des at des.no From phk at projects.linpro.no Fri Aug 7 17:32:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Aug 2009 19:32:03 +0200 (CEST) Subject: r4176 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish Message-ID: <20090807173203.E4EDA1F7CA9@projects.linpro.no> Author: phk Date: 2009-08-07 19:32:03 +0200 (Fri, 07 Aug 2009) New Revision: 4176 Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/include/vsb.h trunk/varnish-cache/lib/libvarnish/cli_common.c trunk/varnish-cache/lib/libvarnish/vsb.c Log: Give vsb_quote() an optional length paramter (Pass -1 for strlen). Add a complementary vsb_unquote() function. Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-06 10:53:17 UTC (rev 4175) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-07 17:32:03 UTC (rev 4176) @@ -291,7 +291,7 @@ } vsb_printf(b->vsb, "%s %s ", a1, a2); - vsb_quote(b->vsb, a3, 0); + vsb_quote(b->vsb, a3, -1, 0); return (0); } Modified: trunk/varnish-cache/include/vsb.h =================================================================== --- trunk/varnish-cache/include/vsb.h 2009-08-06 10:53:17 UTC (rev 4175) +++ trunk/varnish-cache/include/vsb.h 2009-08-07 17:32:03 UTC (rev 4176) @@ -77,7 +77,8 @@ int vsb_len(struct vsb *); int vsb_done(const struct vsb *); void vsb_delete(struct vsb *); -void vsb_quote(struct vsb *s, const char *p, int how); +void vsb_quote(struct vsb *s, const char *p, int len, int how); +const char *vsb_unquote(struct vsb *s, const char *p, int len, int how); #ifdef __cplusplus }; #endif Modified: trunk/varnish-cache/lib/libvarnish/cli_common.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-08-06 10:53:17 UTC (rev 4175) +++ trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-08-07 17:32:03 UTC (rev 4176) @@ -74,7 +74,7 @@ cli_quote(struct cli *cli, const char *s) { - vsb_quote(cli->sb, s, 0); + vsb_quote(cli->sb, s, -1, 0); } void Modified: trunk/varnish-cache/lib/libvarnish/vsb.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vsb.c 2009-08-06 10:53:17 UTC (rev 4175) +++ trunk/varnish-cache/lib/libvarnish/vsb.c 2009-08-07 17:32:03 UTC (rev 4176) @@ -482,25 +482,27 @@ * Quote a string */ void -vsb_quote(struct vsb *s, const char *p, int how) +vsb_quote(struct vsb *s, const char *p, int len, int how) { const char *q; int quote = 0; (void)how; /* For future enhancements */ + if (len == -1) + len = strlen(p); - for (q = p; *q != '\0'; q++) { + for (q = p; q < p + len; q++) { if (!isgraph(*q) || *q == '"') { quote++; break; } } if (!quote) { - (void)vsb_cat(s, p); + (void)vsb_bcat(s, p, len); return; } (void)vsb_putc(s, '"'); - for (q = p; *q != '\0'; q++) { + for (q = p; q < p + len; q++) { switch (*q) { case ' ': (void)vsb_putc(s, *q); @@ -523,9 +525,66 @@ if (isgraph(*q)) (void)vsb_putc(s, *q); else - (void)vsb_printf(s, "\\%o", *q); + (void)vsb_printf(s, "\\%o", *q & 0xff); break; } } (void)vsb_putc(s, '"'); } + +/* + * Unquote a string + */ +const char * +vsb_unquote(struct vsb *s, const char *p, int len, int how) +{ + const char *q; + char *r; + unsigned long u; + char c; + + (void)how; /* For future enhancements */ + + if (len == -1) + len = strlen(p); + + for (q = p; q < p + len; q++) { + if (*q != '\\') { + (void)vsb_bcat(s, q, 1); + continue; + } + if (++q >= p + len) + return ("Incomplete '\\'-sequence at end of string"); + + switch(*q) { + case 'n': + (void)vsb_bcat(s, "\n", 1); + continue; + case 'r': + (void)vsb_bcat(s, "\r", 1); + continue; + case 't': + (void)vsb_bcat(s, "\t", 1); + continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + errno = 0; + u = strtoul(q, &r, 8); + if (errno != 0 || (u & ~0xff)) + return ("\\ooo sequence out of range"); + c = (char)u; + (void)vsb_bcat(s, &c, 1); + q = r - 1; + continue; + default: + (void)vsb_bcat(s, q, 1); + } + } + return (NULL); +} From phk at projects.linpro.no Mon Aug 10 10:07:08 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Aug 2009 12:07:08 +0200 (CEST) Subject: r4177 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090810100708.0D88C1F7CAD@projects.linpro.no> Author: phk Date: 2009-08-10 12:07:07 +0200 (Mon, 10 Aug 2009) New Revision: 4177 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/include/stat_field.h Log: Add a stats counter for backend connections which we came too late to recycle Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-08-07 17:32:03 UTC (rev 4176) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-08-10 10:07:07 UTC (rev 4177) @@ -256,11 +256,11 @@ if (VBE_CheckFd(vc->fd)) { /* XXX locking of stats */ VSL_stats->backend_reuse += 1; - VSL_stats->backend_conn++; WSP(sp, SLT_Backend, "%d %s %s", vc->fd, sp->director->vcl_name, bp->vcl_name); return (vc); } + VSL_stats->backend_toolate++; sp->vbe = vc; VBE_ClosedFd(sp); } Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-08-07 17:32:03 UTC (rev 4176) +++ trunk/varnish-cache/include/stat_field.h 2009-08-10 10:07:07 UTC (rev 4177) @@ -37,14 +37,14 @@ MAC_STAT(cache_hitpass, uint64_t, 0, 'a', "Cache hits for pass") MAC_STAT(cache_miss, uint64_t, 0, 'a', "Cache misses") -MAC_STAT(backend_conn, uint64_t, 0, 'a', "Backend connections success") -MAC_STAT(backend_unhealthy, uint64_t, 0, 'a', - "Backend connections not attempted") -MAC_STAT(backend_busy, uint64_t, 0, 'a', "Backend connections too many") -MAC_STAT(backend_fail, uint64_t, 0, 'a', "Backend connections failures") -MAC_STAT(backend_reuse, uint64_t, 0, 'a', "Backend connections reuses") -MAC_STAT(backend_recycle, uint64_t, 0, 'a', "Backend connections recycles") -MAC_STAT(backend_unused, uint64_t, 0, 'a', "Backend connections unused") +MAC_STAT(backend_conn, uint64_t, 0, 'a', "Backend conn. success") +MAC_STAT(backend_unhealthy, uint64_t, 0, 'a', "Backend conn. not attempted") +MAC_STAT(backend_busy, uint64_t, 0, 'a', "Backend conn. too many") +MAC_STAT(backend_fail, uint64_t, 0, 'a', "Backend conn. failures") +MAC_STAT(backend_reuse, uint64_t, 0, 'a', "Backend conn. reuses") +MAC_STAT(backend_toolate, uint64_t, 0, 'a', "Backend conn. was closed") +MAC_STAT(backend_recycle, uint64_t, 0, 'a', "Backend conn. recycles") +MAC_STAT(backend_unused, uint64_t, 0, 'a', "Backend conn. unused") MAC_STAT(n_sess_mem, uint64_t, 0, 'i', "N struct sess_mem") MAC_STAT(n_sess, uint64_t, 0, 'i', "N struct sess") From phk at projects.linpro.no Mon Aug 10 11:23:44 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Aug 2009 13:23:44 +0200 (CEST) Subject: r4178 - trunk/varnish-cache/bin/varnishd Message-ID: <20090810112344.D1FEC1F7C7E@projects.linpro.no> Author: phk Date: 2009-08-10 13:23:44 +0200 (Mon, 10 Aug 2009) New Revision: 4178 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: "Hit-for-pass" objects are not transient. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-10 10:07:07 UTC (rev 4177) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-10 11:23:44 UTC (rev 4178) @@ -483,8 +483,13 @@ handling = sp->handling; if (sp->objhead == NULL) + /* This is a pass from vcl_recv */ transient = 1; + else if (sp->handling == VCL_RET_PASS) + /* A pass from vcl_fetch is not transient */ + transient = 0; else if (sp->handling == VCL_RET_DELIVER) + /* Regular object */ transient = 0; else transient = 1; @@ -582,7 +587,7 @@ } sp->obj->cacheable = 1; - if (sp->obj->objhead != NULL) { + if (!transient) { VRY_Create(sp); EXP_Insert(sp->obj); AN(sp->obj->ban); From sky at crucially.net Tue Aug 4 17:41:26 2009 From: sky at crucially.net (Artur Bergman) Date: Tue, 4 Aug 2009 19:41:26 +0200 Subject: r4161 - trunk/varnish-cache/lib/libvarnish In-Reply-To: <86ws5j1p9u.fsf@ds4.des.no> References: <20090729124333.4EF0A1F7BD6@projects.linpro.no> <86ws5j1p9u.fsf@ds4.des.no> Message-ID: <5CCCA6D8-BCE5-45AD-BD43-D5FE1161FA24@crucially.net> On Aug 4, 2009, at 7:07 PM, Dag-Erling Sm?rgrav wrote: > sky at projects.linpro.no writes: >> void >> TCP_blocking(int sock) >> { >> int i; >> >> +#ifdef __sun >> + i = fcntl (sock, F_GETFL,0); >> + fcntl(sock, F_SETFL, i & ~O_NONBLOCK); >> +#else >> i = 0; >> AZ(ioctl(sock, FIONBIO, &i)); >> +#endif >> } > > Missing error check for the fcntl case. Yes, also reverted since it didn't help (initial user feedback was wrong) Artur From tfheen at projects.linpro.no Thu Aug 13 11:50:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 13 Aug 2009 13:50:18 +0200 (CEST) Subject: r4179 - trunk/varnish-tools Message-ID: <20090813115018.EA0EA1F7CCE@projects.linpro.no> Author: tfheen Date: 2009-08-13 13:50:18 +0200 (Thu, 13 Aug 2009) New Revision: 4179 Removed: trunk/varnish-tools/regress/ Log: Get rid of empty directories From kristian at projects.linpro.no Fri Aug 14 10:54:54 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Fri, 14 Aug 2009 12:54:54 +0200 (CEST) Subject: r4180 - in trunk/varnish-tools: . security.vcl security.vcl/tools security.vcl/vcl security.vcl/vcl/breach security.vcl/vcl/modules Message-ID: <20090814105454.548DC1F7C9F@projects.linpro.no> Author: kristian Date: 2009-08-14 12:54:53 +0200 (Fri, 14 Aug 2009) New Revision: 4180 Added: trunk/varnish-tools/security.vcl/ trunk/varnish-tools/security.vcl/LICENSE trunk/varnish-tools/security.vcl/README trunk/varnish-tools/security.vcl/tools/ trunk/varnish-tools/security.vcl/tools/2vcl.pl trunk/varnish-tools/security.vcl/tools/all2vcl.sh trunk/varnish-tools/security.vcl/tools/check_variables trunk/varnish-tools/security.vcl/tools/generate_variables trunk/varnish-tools/security.vcl/tools/rule_test trunk/varnish-tools/security.vcl/tools/varnish_reload trunk/varnish-tools/security.vcl/vcl/ trunk/varnish-tools/security.vcl/vcl/Makefile trunk/varnish-tools/security.vcl/vcl/breach/ trunk/varnish-tools/security.vcl/vcl/breach/20_protocol_violations.vcl trunk/varnish-tools/security.vcl/vcl/breach/21_protocol_anomalies.vcl trunk/varnish-tools/security.vcl/vcl/breach/23_request_limits.vcl trunk/varnish-tools/security.vcl/vcl/breach/30_http_policy.vcl trunk/varnish-tools/security.vcl/vcl/breach/35_bad_robots.vcl trunk/varnish-tools/security.vcl/vcl/breach/40_generic_attacks.vcl trunk/varnish-tools/security.vcl/vcl/breach/45_trojans.vcl trunk/varnish-tools/security.vcl/vcl/breach/50_outbound.vcl trunk/varnish-tools/security.vcl/vcl/main.vcl trunk/varnish-tools/security.vcl/vcl/modules/ trunk/varnish-tools/security.vcl/vcl/modules/cmd.vcl trunk/varnish-tools/security.vcl/vcl/modules/content-encoding.vcl trunk/varnish-tools/security.vcl/vcl/modules/content-type.vcl trunk/varnish-tools/security.vcl/vcl/modules/demo.vcl trunk/varnish-tools/security.vcl/vcl/modules/localfiles.vcl trunk/varnish-tools/security.vcl/vcl/modules/php.vcl trunk/varnish-tools/security.vcl/vcl/modules/request.vcl trunk/varnish-tools/security.vcl/vcl/modules/restricted-file-extensions.vcl trunk/varnish-tools/security.vcl/vcl/modules/sql.vcl trunk/varnish-tools/security.vcl/vcl/modules/user-agent.vcl trunk/varnish-tools/security.vcl/vcl/modules/xss.vcl Log: Initial commit of Security.VCL Added: trunk/varnish-tools/security.vcl/LICENSE =================================================================== --- trunk/varnish-tools/security.vcl/LICENSE (rev 0) +++ trunk/varnish-tools/security.vcl/LICENSE 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. Added: trunk/varnish-tools/security.vcl/README =================================================================== --- trunk/varnish-tools/security.vcl/README (rev 0) +++ trunk/varnish-tools/security.vcl/README 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,70 @@ +Security.VCL + +This is work in progress. This was (as is too common) developed internally +and thrown into a public repository. Use at your own discretion. + +================================= +1. About Security.VCL +2. Basic usage +3. The Architecture +4. Handlers +================================= + + +1. About Security.VCL +===================== + +Security.VCL aims to provide: + - A standardized framework for security-related filters + - Several core rule-sets + - A tool to generate Security.VCL modules from mod_security rules. + - A limited set of default 'handlers', for instance CGI scripts to call + upon when Bad Stuff happens. + +This is done mainly by using clever VCL, and with as little impact on +normal operation as possible. The incident handlers are mainly CGI-like +scripts on a backend. + +2. Basic usage +============== + +To use Security.VCL, you currently have to: + +$ cd vcl/ +$ make +$ cd .. +$ cp -a vcl/ /etc/varnish/security.vcl/ + (alternatively you could symlink it, of course). + +Then you edit your normal VCL and add this line near the top: + + include "/etc/varnish/security.vcl/main.vcl"; + +At this point, you should only need to reload your varnish configuration. +You may have to or want to modify main.vcl to fit your needs. At the +moment, paths are hardcoded. + +3. The Architecture +=================== + +Security.VCL currently have two categories of rules: rules generated from +mod_security, located in vcl/breach/ and our own rules, vcl/modules/. + +Security.VCL works by including all modules, then defining a number of +standard functions. Each module will call sec_sevN, where N is the +severity, which in turn typically calls error or some other handler. + +4. Handlers +=========== + +Handlers are still not well developed, but the general concept is that +Security.VCL either throws an error (vcl_error) of some kind, which can +potentially redirect the client or do any other synthetic response, or +Security.VCL can rewrite the original request and send it to a backend +designed to do more clever things, like: + +- Block the client in a firewall +- Log the event +- Test-run the code. +- Paint you a pretty picture.... +- .... Added: trunk/varnish-tools/security.vcl/tools/2vcl.pl =================================================================== --- trunk/varnish-tools/security.vcl/tools/2vcl.pl (rev 0) +++ trunk/varnish-tools/security.vcl/tools/2vcl.pl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,380 @@ +#!/usr/bin/perl +# +# 2009-08-04 Kacper Wysocki +# +# Usage: 2vcl.pl foo.conf > bar.vcl +# +# Unsupported variables: +# REQUEST_LINE +# X-Sec-VCL-mod +use strict; + +# kill newlines and leading spaces +our $TABLEVEL = 0; +our $TABSIZE = 3; +our $TABCHAR = ' '; +our $DEFAULT_ACTION = q(call sec_sev1;); +our $DENY_ACTION = q(call sec_sev1;); + +our $DEBUG = 0; + +# don't touch these, eh? + +# ".*?\\".*?[^\\]" +# + +my $re_vararg = '[\w_-\d\:\/\*&!\|\'()^$]+'; +#my $re_op = q("[^\\"]*(\\.[^\\"]*)*"|[^\s"']+|'[^\\']*(\\.[^\\']*)*');#|"[^"]*"|'[^']*'); +my $re_op = qq(".*?[^\\\\]"|[^\s"']|'.*?[^\\\\]'); +my $re_var = '[\w_\d]+'; +#my $re_arg = ':[\w\d\*\/\|\'\-()\^\$]+'; +my $re_arg = q(:'.+?[^\\\\]'|:[\w\d\*\/\|\-()\^\$]+); +my $re_num = q('?(\d*)'?); + +sub skip_rule { + my ($var, $func, $arg) = @_; + # this is an expression that returns true or false + $var =~ /^(XML| # XML body parse + WEBSERVER_ERROR_LOG| # errorlog something + REQUEST_BODY| # raw body + REQUEST_LINE| # whole HTTP request + REQBODY_PROCESSOR| # body parse error + REMOTE_HOST| # we don't resolve + REMOTE_ADDR| # we don't resolve + RESPONSE_ # any response logic + )/x or $func =~ + /validate(UrlEncoding|ByteRange)|lt|le|gt|ge|eq|ne|pm/x or + ($var eq 'REQUEST_HEADERS' and not $arg) +} +sub emit { + my $out = join " ", @_; + print $TABCHAR x ($TABLEVEL * $TABSIZE) . qq($out\n); +} +# indent more +sub memit { + $TABLEVEL++; + emit @_; +} +# indent less +sub lemit { + $TABLEVEL--; + emit @_; +} +# indent after +sub aemit { + emit @_; + $TABLEVEL++; +} + +sub killnil { + s/^\s*(.*)\n?$/\1/; +} + +# skip empty lines and comments +sub skip_line { + while($_ and /^\s*#?\s*$/){ + $_ = <>; + chomp; + } +} + +sub normalize_line { + killnil; + # normalize line: join escaped multilines + while(/\\$/){ + chop; + $_ .= <>; + killnil; + } +} + +sub parse_input { + aemit q(sub vcl_recv {); + emit q(set req.http.X-Sec-Module = "2vcl";); + + while(<>){ + parse_line(); + # I monk around with <>, so we gots to recheck eof + last if eof; + } + lemit qq(}\n); +} + +sub parse_line { + skip_line; + normalize_line; + # Syntax: SecRule VARIABLES OPERATOR [ACTIONS] + if(/^SecRule\s*("?$re_vararg"?)\s*($re_op)\s*(.*)?$/){ + my ($vars, $ops, $actions) = ($1, $2, $3); + print STDERR "# line: VAR: $vars\n#OP: $ops\n#ACT: $actions\n" if $DEBUG == 2; + #emit "# line: VAR: $vars\n#OP: $ops\n#ACT: $actions\n"; + parse_secrule($vars,$ops,$actions); + }elsif(/^SecRule\s*($re_vararg)\s*/){ + print STDERR "#2 Rule fell thru cracks: $_\n"; + }elsif(/^SecRule/){ + print STDERR "#3 Rule fell thru cracks: $_\n"; + } + # Other rule modifiers / matcher +} + +sub emit_rule { + my ($target, $ops, $neg_op, $neg_var, $func, $action) = @_; + + # pm pmFromFile rbl + # validateUrlEncoding/Utf8Encoding verifyCC + # within => acl + $func = parse_func($func); + if($func eq 'beginsWith'){ + $func = '~'; + $ops = '^'.$ops; + }elsif($func eq 'endsWith'){ + $func = '~'; + $ops .= '$'; + }elsif($func eq 'pm'){ + # uses funky Aho-Corasick fast collection matching + # we could use ACLs here to match on collection + $func = '~'; + #$ops = emit_acl(split / /,$ops); + + $ops = '('. join "|",split / /,$ops .')'; + }elsif($func eq 'within'){ + $func = '~'; + my $tmp = '('. join "|",split / /,$ops .')'; + $ops = $target; + $target = $tmp; + } + + aemit qq(if($neg_op$target $func "$ops"){); + emit_action($action,$target,$ops,$neg_op,$neg_var,$func); + lemit qq(}); +} + +# translate funcs +sub parse_func { + my ($func) = @_; + if(not $func or $func eq 'rx') { + $func = '~'; + }elsif($func eq 'eq'){ + $func = '=='; + }elsif($func eq 'ge'){ + $func = '>='; + }elsif($func eq 'le'){ + $func = '<='; + }elsif($func eq 'gt'){ + $func = '>'; + }elsif($func eq 'lt'){ + $func = '<'; + }elsif($func eq 'contains'){ + $func = '~'; + } + return $func; +} +sub parse_secrule { + my ($vars, $ops, $actions) = @_; + + # parse OPERATOR (regex default) + my $neg_op; + my $func; + ($func, $ops) = parse_ops($ops); + # parse VARIABLES + print STDERR ";;$vars;;\n" if $DEBUG == 2; + + my @var = split_vars($vars); + #my @var = split /\|/, $vars; + for (@var){ + my ($var, $arg, $neg_var, $amp) = split_args($_); + emit "#1 $vars: Var slipped thu: '$_'\n" if not $var; + + # skip this rule if it is not interesting + # ie if we can't match for it in VCL yet + if(skip_rule($var,$func,$arg) ){ + print STDERR "skipped $var $func :$arg\n" if $DEBUG == 2; + emit qq(# skipped $neg_var $amp $var $func $arg $ops); + next; + } + emit qq(## Rule: $var $func :$arg); + + emit_code($var,$ops,$actions,$func,$arg,$neg_var,$neg_op,$amp); + } +} +sub split_vars { + my ($vars, @var) = @_; + $vars =~ s/^"(.*)"$/$1/; + while($vars){ + $vars =~ s/([!&]?(?:$re_var)(?:$re_arg)?)\|?//; # welcome to ehll + #print "PUSH $1\n"; + push @var, $1; + #print "#REDUCE :$vars\n"; + } + return @var; +} + +sub split_args { + ($_) = @_; + my ($neg_var, $amp, $var, $arg) = /(!?)(&?)($re_var)($re_arg)?/; + # get rid of :'/( + emit qq(## $neg_var$amp$var, $arg); + if($arg =~ /:'\/^?\(([^']*)\)\$?\/'/){ + $arg = $1; + emit q(# AA ); + }elsif($arg =~ /:"\/^?\(([^"]*)\)\$?\/"/){ + $arg = $1; + emit q(# AB ); + }elsif($arg =~ /:([^'"]+)/){ + $arg = $1; + emit qq(# AC $arg ); + } + return ($var, $arg, $neg_var, $amp); +} + +sub parse_ops { + my ($ops) = @_; + my $neg; + my $func = 'rx'; # regex is the default operator + if($ops =~ /^"?(!?)(@?)(.*?[^\\])"?$/){ + $neg = $1; + if($2 eq '@'){ + ($func, $ops) = split / /,$3,2; + }else{ + $ops = $3; + } + # translate pcre to posix regex + $ops =~ s/\\?%/%25/g; + $ops =~ s/\\"/%22/g; + $ops =~ s/\(\?:/\(/g; + #$ops =~ s/([^\\])([\{\}])/$1\\$2/g; + #print "OP: $neg_op\@$func $ops\n"; + }else{ + print STDERR "error '$ops'\n$_\n"; + } + return ($func, $ops, $neg); +} + +# Do the dirty deed: map ModSec to vcl +sub emit_code { + my ($var,$ops,$action,$func,$arg,$neg_var,$neg_op,$amp) = @_; + my $target; + my $status = '800'; + my $msg = 'Hack attack, try again.'; + print STDERR "# code VAR: $var\n#OP: $ops\n#ACT: $action\n" if $DEBUG; + + #emit qq(## $neg_var $amp $var \@$func $arg $ops); # : $actions + if($var eq 'REQUEST_HEADERS' and $arg){ + # Support REQUEST_HEADERS:foo + $arg =~ s#^:'/\^\((.*?)\)\$/'#$1#; + emit qq(# AAA $arg); + my @args = split /\|/,$arg; + for(@args){ + $target = "req.http.$_"; + emit_rule($target,$ops,$neg_op,$neg_var,$func,$action); + } + }elsif($var =~/^ARGS/){ # ARGS, ARGS_NAMES, ARGS_GET etc.. + # XXX ARGS should only apply to param _values_ ?p=v&q=w + $target = 'req.url'; + emit_rule($target,$ops,$neg_op,$neg_var,$func,$action); + }else{ + if($var =~ /^(REQUEST_URI_RAW|REQUEST_URI)$/){ + $target = 'req.url'; + }elsif($var eq 'REQUEST_METHOD'){ + $target = "req.request"; + }elsif($var eq 'REQUEST_PROTOCOL'){ + $target = "req.proto"; + }elsif($var eq 'REMOTE_ADDR'){ + $target = 'client.ip'; + # More hacks! + $ops =~ s/\\|\^|\$//g; + }elsif($var =~ /^REQUEST_COOKIES/){ + $target = 'req.http.Cookie'; + }elsif($var eq 'REQUEST_FILENAME'){ + # XXX should be URL minus QUERY + $target = 'req.url'; + } + + if($target){ + emit_rule($target,$ops,$neg_op,$neg_var,$func,$action); + } + } + +} +# default action: phase:2,log,auditlog,pass +# deal with chains! +sub emit_action { + my ($action,$target,$ops,$neg_op,$neg_var,$func) = @_; + $action =~ s/^"([^"]*)"$/$1/; + #emit "## ACTION $action\n"; + my @act = split /,/, $action; + my ($chain, $phase); + my $transforms = ''; #no default xforms + my $end = $DEFAULT_ACTION; + my $id; + for(@act){ + # APEShiT: The Action Parse Engine Short Circuit + # Warning! An expression must return TRUE + # to stop matching the next 'or' clause. + # In particular, assigments return the assigned value + # which is FALSE if it is an empty string + /phase:(\d*)/ and $phase = $1 or + /chain/ and $chain = 1 or + /status:$re_num/ and + emit qq(set req.http.X-Sec-Return = "$1";) or + /severity:'?(\d*)'/ and + emit qq(set req.http.X-Sec-Severity = "$1";) or + /id:$re_num/ and + $id = $1 or + /rev:$re_num/ and + $id .= "-$1" or + /tag:'([^']*)'/ and + emit qq(set req.http.X-Sec-RuleName = "$1";) or + /msg:'([^']*)'/ and + emit qq(set req.http.X-Sec-RuleInfo = "$1";) or + /t:none/ and + $transforms = '' or 1 or + /t:(.*)/ and + $transforms .= "$1;" or + /allow:phase/ and + emit qq(# should last in this phase..) and + emit qq(deliver;) or + /allow:request/ and + emit qq(# skip to RESPONSE_HEADERS..) and + emit qq(deliver;) or + /allow/ and emit qq(deliver;) or + + /(audit)?log(.*)/ and emit qq(# $1log$2 this plz) or + /block/ and + $end = $DEFAULT_ACTION or + /deny/ and + $end = $DENY_ACTION or + /drop/ and + emit qq(# send a FIN and drop) and + $end = $DENY_ACTION or + /pass/ and + emit qq(# pass to next rule) and $end = '' or + /skip:$re_num/ and + emit qq(# rule action skips next $1 rules!) or + /skipAfter:$re_num/ and + emit qq(# rule action skips after id/marker $1) or + /pause:(.*)/ and + emit qq(# sleep $1) or + /proxy:(.*)/ and + emit qq(# proxy to host: $1) or + /redirect:(.*)/ and + emit qq(# redirect to $1) or + /exec:(.*)/ and emit qq(# exec $1) or + /capture/ and emit qq(# capture action) or + /ctl:(.*)/ and emit qq(# ctl:$1) or + /(pre|ap)pend:(.*)/ and emit qq(# body $1pend $2) or + emit qq(# action : $_) + } + if($id){ + emit qq(set req.http.X-Sec-RuleId = "$id";); + } + emit qq(# transforms: $transforms) if $transforms; + if($chain){ + emit qq(# chained rule); + $end = ''; + $_ = ''; parse_line; + } + emit $end if $end; +} + +parse_input; Property changes on: trunk/varnish-tools/security.vcl/tools/2vcl.pl ___________________________________________________________________ Name: svn:executable + * Added: trunk/varnish-tools/security.vcl/tools/all2vcl.sh =================================================================== --- trunk/varnish-tools/security.vcl/tools/all2vcl.sh (rev 0) +++ trunk/varnish-tools/security.vcl/tools/all2vcl.sh 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,8 @@ +#!/bin/sh +for i in modsecurity-apache_2.5.9/rules/modsecurity_crs_[2345]* +do + v=`basename $i` + v=${v#modsecurity_crs_} + echo $v + ./tools/2vcl.pl $i > vcl/breach/${v%.conf}.vcl +done Property changes on: trunk/varnish-tools/security.vcl/tools/all2vcl.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/varnish-tools/security.vcl/tools/check_variables =================================================================== --- trunk/varnish-tools/security.vcl/tools/check_variables (rev 0) +++ trunk/varnish-tools/security.vcl/tools/check_variables 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Braindead script to verify that all X-SEC-VCL-variables are part of +# VARIABLES. + +if [ ! -f VARIABLES ]; then + echo 1>&2 "No VARIABLES-file found. Bailing." + exit 1; +fi + +RET=0 + +for a in `grep --exclude=variables.vcl 'X-SEC-' *vcl modules/*vcl | sed 's/.*X-SEC/X-SEC/;s/\s.*//;s/\;\s*$//' | sort | uniq`; do + if ! grep -q $a VARIABLES; then + echo >&2 "Variable $a found in a module, but not in VARIABLES! Unsafe." + grep $a *vcl modules/*vcl | cat 1>&2 + if [ "x$a" = "xX-SEC-Info" ]; then + echo >&2 "Are you sure you didn't mean X-SEC-RuleInfo?" + fi + RET=1; + fi +done + +exit $RET Property changes on: trunk/varnish-tools/security.vcl/tools/check_variables ___________________________________________________________________ Name: svn:executable + * Added: trunk/varnish-tools/security.vcl/tools/generate_variables =================================================================== --- trunk/varnish-tools/security.vcl/tools/generate_variables (rev 0) +++ trunk/varnish-tools/security.vcl/tools/generate_variables 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Braindead script to generate variables.vcl + + +TARGET=build/variables.vcl + +if [ ! -f VARIABLES ]; then + echo 1>&2 "No VARIABLES-file found. Bailing." + exit 1; +fi + +cat > $TARGET << __EOF__ +/* Autogenerated with $0 at $(date) + * + * Do not modify, modify VARIABLES and rerun $0 instead. + */ + +sub vcl_recv +{ +__EOF__ + +grep -Ev '^(#.*|$)' VARIABLES | while read a; do + echo " unset req.http.$a;" >> $TARGET +done + +echo "}" >> $TARGET Property changes on: trunk/varnish-tools/security.vcl/tools/generate_variables ___________________________________________________________________ Name: svn:executable + * Added: trunk/varnish-tools/security.vcl/tools/rule_test =================================================================== --- trunk/varnish-tools/security.vcl/tools/rule_test (rev 0) +++ trunk/varnish-tools/security.vcl/tools/rule_test 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,26 @@ +#!/usr/bin/awk -f +# This thing is quite ugly, and should probably not be used as-is. +# Author: Kristian Lyngstol +/\/\/TEST:/ { + FS=":" + ORS="" + print $3 " -Used 'http://localhost:8081" + for(n=4; n<=NF; n++) { + if (n > 4) + print ":" + print $n + } + print "' | grep -q " $2" && echo Rule positive test " $2 " Ok || echo Rule positive test" $2 " FAILED" + print "\n" +} END { print "\n"} BEGIN { FS = ":" } +/\/\/TESTN:/ { + FS=":" + ORS="" + print $3 " -Used 'http://localhost:8081" + for(n=4; n<=NF; n++) { + if (n > 4) + print ":" + print $n + } + print "' | grep -q " $2" && echo Rule negative test " $2 " FAILED || echo Rule negative test" $2 " Ok" +} END { print "\n"} BEGIN { FS = ":" } Property changes on: trunk/varnish-tools/security.vcl/tools/rule_test ___________________________________________________________________ Name: svn:executable + * Added: trunk/varnish-tools/security.vcl/tools/varnish_reload =================================================================== --- trunk/varnish-tools/security.vcl/tools/varnish_reload (rev 0) +++ trunk/varnish-tools/security.vcl/tools/varnish_reload 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,23 @@ +#!/bin/bash +# Reload a varnish config +# Author: Kristian Lyngstol + +FILE="/etc/varnish/default.vcl" + +# Hostname and management port +# (defined in /etc/default/varnish or on startup) +HOSTPORT="localhost:6082" +NOW=`date +%s` + +error() +{ + echo 1>&2 "Failed to reload $FILE." + exit 1 +} + +varnishadm -T $HOSTPORT vcl.load reload$NOW $FILE || error +sleep 0.1 +varnishadm -T $HOSTPORT vcl.use reload$NOW || error +sleep 0.1 +echo Current configs: +varnishadm -T $HOSTPORT vcl.list Property changes on: trunk/varnish-tools/security.vcl/tools/varnish_reload ___________________________________________________________________ Name: svn:executable + * Added: trunk/varnish-tools/security.vcl/vcl/Makefile =================================================================== --- trunk/varnish-tools/security.vcl/vcl/Makefile (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/Makefile 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,40 @@ +# Braindead Makefile. Feel free to improve. +# Copyright (c) 2009 Redpill Linpro AS +# Author: Kristian Lyngstol ]+|utonomous_transaction\b)|o(r\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|pen(rowset|query)\b)|having\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|print\b\W*?\@\@|cast\b\W*?\()|(;\W*?\b(shutdown|drop)|\@\@version)\b|'(s(qloledb|a)|msdasql|dbo)')"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950001"; + call sec_sev1; + } + ## ARGS, + ## Rule: ARGS rx : + if(req.url ~ "(\b((s(elect\b(.{1,100}?\b((length|count|top)\b.{1,100}?\bfrom|from\b.{1,100}?\bwhere)|.*?\b(d(ump\b.*\bfrom|ata_type)|(to_(numbe|cha)|inst)r))|p_((addextendedpro|sqlexe)c|(oacreat|prepar)e|execute(sql)?|makewebtask)|ql_(longvarchar|variant))|xp_(reg(re(movemultistring|ad)|delete(value|key)|enum(value|key)s|addmultistring|write)|e(xecresultset|numdsn)|(terminat|dirtre)e|availablemedia|loginconfig|cmdshell|filelist|makecab|ntsec)|u(nion\b.{1,100}?\bselect|tl_(file|http))|group\b.*\bby\b.{1,100}?\bhaving|d(elete\b\W*?\bfrom|bms_java)|load\b\W*?\bdata\b.*\binfile|(n?varcha|tbcreato)r)\b|i(n(to\b\W*?\b(dump|out)file|sert\b\W*?\binto|ner\b\W*?\bjoin)\b|(f(\b\W*?\(\W*?\bbenchmark|null\b)|snull\b)\W*?\()|a(nd\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|utonomous_transaction\b)|o(r\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|pen(rowset|query)\b)|having\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|print\b\W*?\@\@|cast\b\W*?\()|(;\W*?\b(shutdown|drop)|\@\@version)\b|'(s(qloledb|a)|msdasql|dbo)')"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950001"; + call sec_sev1; + } + ## ARGS_NAMES, + ## Rule: ARGS_NAMES rx : + if(req.url ~ "(\b((s(elect\b(.{1,100}?\b((length|count|top)\b.{1,100}?\bfrom|from\b.{1,100}?\bwhere)|.*?\b(d(ump\b.*\bfrom|ata_type)|(to_(numbe|cha)|inst)r))|p_((addextendedpro|sqlexe)c|(oacreat|prepar)e|execute(sql)?|makewebtask)|ql_(longvarchar|variant))|xp_(reg(re(movemultistring|ad)|delete(value|key)|enum(value|key)s|addmultistring|write)|e(xecresultset|numdsn)|(terminat|dirtre)e|availablemedia|loginconfig|cmdshell|filelist|makecab|ntsec)|u(nion\b.{1,100}?\bselect|tl_(file|http))|group\b.*\bby\b.{1,100}?\bhaving|d(elete\b\W*?\bfrom|bms_java)|load\b\W*?\bdata\b.*\binfile|(n?varcha|tbcreato)r)\b|i(n(to\b\W*?\b(dump|out)file|sert\b\W*?\binto|ner\b\W*?\bjoin)\b|(f(\b\W*?\(\W*?\bbenchmark|null\b)|snull\b)\W*?\()|a(nd\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|utonomous_transaction\b)|o(r\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|pen(rowset|query)\b)|having\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|print\b\W*?\@\@|cast\b\W*?\()|(;\W*?\b(shutdown|drop)|\@\@version)\b|'(s(qloledb|a)|msdasql|dbo)')"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950001"; + call sec_sev1; + } + ## REQUEST_HEADERS, + # skipped REQUEST_HEADERS rx (\b((s(elect\b(.{1,100}?\b((length|count|top)\b.{1,100}?\bfrom|from\b.{1,100}?\bwhere)|.*?\b(d(ump\b.*\bfrom|ata_type)|(to_(numbe|cha)|inst)r))|p_((addextendedpro|sqlexe)c|(oacreat|prepar)e|execute(sql)?|makewebtask)|ql_(longvarchar|variant))|xp_(reg(re(movemultistring|ad)|delete(value|key)|enum(value|key)s|addmultistring|write)|e(xecresultset|numdsn)|(terminat|dirtre)e|availablemedia|loginconfig|cmdshell|filelist|makecab|ntsec)|u(nion\b.{1,100}?\bselect|tl_(file|http))|group\b.*\bby\b.{1,100}?\bhaving|d(elete\b\W*?\bfrom|bms_java)|load\b\W*?\bdata\b.*\binfile|(n?varcha|tbcreato)r)\b|i(n(to\b\W*?\b(dump|out)file|sert\b\W*?\binto|ner\b\W*?\bjoin)\b|(f(\b\W*?\(\W*?\bbenchmark|null\b)|snull\b)\W*?\()|a(nd\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|utonomous_transaction\b)|o(r\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|pen(rowset|query)\b)|having\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|print\b\W*?\@\@|cast\b\W*?\()|(;\W*?\b(shutdown|drop)|\@\@version)\b|'(s(qloledb|a)|msdasql|dbo)') + ## XML, :/*| + # AC /*| + # skipped XML rx /*| (\b((s(elect\b(.{1,100}?\b((length|count|top)\b.{1,100}?\bfrom|from\b.{1,100}?\bwhere)|.*?\b(d(ump\b.*\bfrom|ata_type)|(to_(numbe|cha)|inst)r))|p_((addextendedpro|sqlexe)c|(oacreat|prepar)e|execute(sql)?|makewebtask)|ql_(longvarchar|variant))|xp_(reg(re(movemultistring|ad)|delete(value|key)|enum(value|key)s|addmultistring|write)|e(xecresultset|numdsn)|(terminat|dirtre)e|availablemedia|loginconfig|cmdshell|filelist|makecab|ntsec)|u(nion\b.{1,100}?\bselect|tl_(file|http))|group\b.*\bby\b.{1,100}?\bhaving|d(elete\b\W*?\bfrom|bms_java)|load\b\W*?\bdata\b.*\binfile|(n?varcha|tbcreato)r)\b|i(n(to\b\W*?\b(dump|out)file|sert\b\W*?\binto|ner\b\W*?\bjoin)\b|(f(\b\W*?\(\W*?\bbenchmark|null\b)|snull\b)\W*?\()|a(nd\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|utonomous_transaction\b)|o(r\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|pen(rowset|query)\b)|having\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|print\b\W*?\@\@|cast\b\W*?\()|(;\W*?\b(shutdown|drop)|\@\@version)\b|'(s(qloledb|a)|msdasql|dbo)') + ## !REQUEST_HEADERS, :Referer + # AC Referer + ## Rule: REQUEST_HEADERS rx :Referer + # AAA Referer + if(req.http.Referer ~ "(\b((s(elect\b(.{1,100}?\b((length|count|top)\b.{1,100}?\bfrom|from\b.{1,100}?\bwhere)|.*?\b(d(ump\b.*\bfrom|ata_type)|(to_(numbe|cha)|inst)r))|p_((addextendedpro|sqlexe)c|(oacreat|prepar)e|execute(sql)?|makewebtask)|ql_(longvarchar|variant))|xp_(reg(re(movemultistring|ad)|delete(value|key)|enum(value|key)s|addmultistring|write)|e(xecresultset|numdsn)|(terminat|dirtre)e|availablemedia|loginconfig|cmdshell|filelist|makecab|ntsec)|u(nion\b.{1,100}?\bselect|tl_(file|http))|group\b.*\bby\b.{1,100}?\bhaving|d(elete\b\W*?\bfrom|bms_java)|load\b\W*?\bdata\b.*\binfile|(n?varcha|tbcreato)r)\b|i(n(to\b\W*?\b(dump|out)file|sert\b\W*?\binto|ner\b\W*?\bjoin)\b|(f(\b\W*?\(\W*?\bbenchmark|null\b)|snull\b)\W*?\()|a(nd\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|utonomous_transaction\b)|o(r\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|pen(rowset|query)\b)|having\b ?(\d{1,10}|[\'%22][^=]{1,10}[\'%22]) ?[=<>]+|print\b\W*?\@\@|cast\b\W*?\()|(;\W*?\b(shutdown|drop)|\@\@version)\b|'(s(qloledb|a)|msdasql|dbo)')"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "959001"; + call sec_sev1; + } + ## REQUEST_FILENAME, + ## Rule: REQUEST_FILENAME rx : + if(req.url ~ "\b(\d+) ?= ?\1\b|[\'%22](\w+)[\'%22] ?= ?[\'%22]\2\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950901"; + call sec_sev1; + } + ## ARGS, + ## Rule: ARGS rx : + if(req.url ~ "\b(\d+) ?= ?\1\b|[\'%22](\w+)[\'%22] ?= ?[\'%22]\2\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950901"; + call sec_sev1; + } + ## ARGS_NAMES, + ## Rule: ARGS_NAMES rx : + if(req.url ~ "\b(\d+) ?= ?\1\b|[\'%22](\w+)[\'%22] ?= ?[\'%22]\2\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950901"; + call sec_sev1; + } + ## REQUEST_HEADERS, + # skipped REQUEST_HEADERS rx \b(\d+) ?= ?\1\b|[\'%22](\w+)[\'%22] ?= ?[\'%22]\2\b + ## XML, :/*| + # AC /*| + # skipped XML rx /*| \b(\d+) ?= ?\1\b|[\'%22](\w+)[\'%22] ?= ?[\'%22]\2\b + ## !REQUEST_HEADERS, :Referer + # AC Referer + ## Rule: REQUEST_HEADERS rx :Referer + # AAA Referer + if(req.http.Referer ~ "\b(\d+) ?= ?\1\b|[\'%22](\w+)[\'%22] ?= ?[\'%22]\2\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "959901"; + call sec_sev1; + } + ## REQUEST_FILENAME, + # skipped REQUEST_FILENAME pm user_objects object_type substr all_objects mb_users column_name rownum atttypid substring object_id user_group user_tables pg_attribute user_users column_id user_password attrelid object_name table_name pg_class + ## ARGS, + # skipped ARGS pm user_objects object_type substr all_objects mb_users column_name rownum atttypid substring object_id user_group user_tables pg_attribute user_users column_id user_password attrelid object_name table_name pg_class + ## REQUEST_HEADERS, + # skipped REQUEST_HEADERS pm user_objects object_type substr all_objects mb_users column_name rownum atttypid substring object_id user_group user_tables pg_attribute user_users column_id user_password attrelid object_name table_name pg_class + ## XML, :/*| + # AC /*| + # skipped XML pm /*| user_objects object_type substr all_objects mb_users column_name rownum atttypid substring object_id user_group user_tables pg_attribute user_users column_id user_password attrelid object_name table_name pg_class + ## !REQUEST_HEADERS, :Referer + # AC Referer + # skipped ! REQUEST_HEADERS pm Referer user_objects object_type substr all_objects mb_users column_name rownum atttypid substring object_id user_group user_tables pg_attribute user_users column_id user_password attrelid object_name table_name pg_class + ## REQUEST_FILENAME, + ## Rule: REQUEST_FILENAME rx : + if(req.url ~ "\b(user_((object|table|user)s|password|group)|a(tt(rel|typ)id|ll_objects)|object_((nam|typ)e|id)|pg_(attribute|class)|column_(name|id)|substr(ing)?|table_name|mb_users|rownum)\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950906"; + call sec_sev1; + } + ## ARGS, + ## Rule: ARGS rx : + if(req.url ~ "\b(user_((object|table|user)s|password|group)|a(tt(rel|typ)id|ll_objects)|object_((nam|typ)e|id)|pg_(attribute|class)|column_(name|id)|substr(ing)?|table_name|mb_users|rownum)\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950906"; + call sec_sev1; + } + ## REQUEST_HEADERS, + # skipped REQUEST_HEADERS rx \b(user_((object|table|user)s|password|group)|a(tt(rel|typ)id|ll_objects)|object_((nam|typ)e|id)|pg_(attribute|class)|column_(name|id)|substr(ing)?|table_name|mb_users|rownum)\b + ## XML, :/*| + # AC /*| + # skipped XML rx /*| \b(user_((object|table|user)s|password|group)|a(tt(rel|typ)id|ll_objects)|object_((nam|typ)e|id)|pg_(attribute|class)|column_(name|id)|substr(ing)?|table_name|mb_users|rownum)\b + ## !REQUEST_HEADERS, :Referer + # AC Referer + ## Rule: REQUEST_HEADERS rx :Referer + # AAA Referer + if(req.http.Referer ~ "\b(user_((object|table|user)s|password|group)|a(tt(rel|typ)id|ll_objects)|object_((nam|typ)e|id)|pg_(attribute|class)|column_(name|id)|substr(ing)?|table_name|mb_users|rownum)\b"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "959906"; + call sec_sev1; + } + ## REQUEST_FILENAME, + ## Rule: REQUEST_FILENAME rx : + if(req.url ~ "\b(coalesce\b|root\@)"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950908"; + call sec_sev1; + } + ## ARGS, + ## Rule: ARGS rx : + if(req.url ~ "\b(coalesce\b|root\@)"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950908"; + call sec_sev1; + } + ## ARGS_NAMES, + ## Rule: ARGS_NAMES rx : + if(req.url ~ "\b(coalesce\b|root\@)"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950908"; + call sec_sev1; + } + ## !REQUEST_HEADERS, :via + # AC via + ## Rule: REQUEST_HEADERS rx :via + # AAA via + if(req.http.via ~ "\b(coalesce\b|root\@)"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "950908"; + call sec_sev1; + } + ## REQUEST_HEADERS, + # skipped REQUEST_HEADERS rx \b(coalesce\b|root\@) + ## XML, :/*| + # AC /*| + # skipped XML rx /*| \b(coalesce\b|root\@) + ## !REQUEST_HEADERS, :Referer| + # AC Referer| + ## Rule: REQUEST_HEADERS rx :Referer| + # AAA Referer| + if(req.http.Referer ~ "\b(coalesce\b|root\@)"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "959908"; + call sec_sev1; + } + ## !REQUEST_HEADERS, :via + # AC via + ## Rule: REQUEST_HEADERS rx :via + # AAA via + if(req.http.via ~ "\b(coalesce\b|root\@)"){ + set req.http.X-Sec-RuleInfo = "SQL Injection Attack"; + set req.http.X-Sec-RuleName = "WEB_ATTACK/SQL_INJECTION"; + set req.http.X-Sec-Severity = "2"; + set req.http.X-Sec-RuleId = "959908"; + call sec_sev1; + } + ## REQUEST_FILENAME, + # skipped REQUEST_FILENAME pm jscript onsubmit copyparentfolder javascript meta onmove onkeydown onchange onkeyup activexobject expression onmouseup ecmascript onmouseover vbscript: [^<]*?(\b((c(ehennemden|gi-telnet)|gamma web shell)\b|imhabirligi phpftp)|(r(emote explorer|57shell)|aventis klasvayv|zehir)\b|\.::(news remote php shell injection::\.| rhtools\b)|ph(p(( commander|-terminal)\b|remoteview)|vayv)|myshell)|\b(((microsoft windows\b.{,10}?\bversion\b.{,20}?\(c\) copyright 1985-.{,10}?\bmicrosoft corp|ntdaddy v1\.9 - obzerve \| fux0r inc)\.|(www\.sanalteror\.org - indexer and read|haxplor)er|php(konsole| shell)|c99shell)\b|aventgrup\.
|drwxr)) +} + Added: trunk/varnish-tools/security.vcl/vcl/breach/50_outbound.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/breach/50_outbound.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/breach/50_outbound.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,58 @@ +sub vcl_recv { + set req.http.X-Sec-Module = "2vcl"; + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx \b(Th(is (summary was generated by.{0,100}?(w(ebcruncher|wwstat)|analog|Jware)|analysis was produced by.{0,100}?(calamaris|EasyStat|analog)|report was generated by WebLog)|ese statistics were produced by (getstats|PeLAB))|[gG]enerated by.{0,100}?[Ww]ebalizer)\b + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx (\b((s(elect list because it is not contained in (an aggregate function and there is no|either an aggregate function or the) GROUP BY clause|upplied argument is not a valid ((M(S |y)|Postgre)SQL|O(racle|DBC)))|S(yntax error converting the \w+ value .*? to a column of data type|QL Server does not exist or access denied)|Either BOF or EOF is True, or the current record has been deleted(; the operation|\. Requested)|The column prefix .{0,50}? does not match with a table name or alias name used in the query|Could not find server '\w+' in sysservers\. execute sp_addlinkedserver)\b|Un(closed quotation mark before the character string\b|able to connect to PostgreSQL server:)|(Microsoft OLE DB Provider for .{0,30} [eE]rror |error '800a01b8)'|(Warning: mysql_connect\(\)|PostgreSQL query failed):|You have an error in your SQL syntax( near '|;)|cannot take a \w+ data type as an argument\.|incorrect syntax near (\'|the\b|@@error\b)|microsoft jet database engine error '8|ORA-\d{5}: )|\[Microsoft\]\[ODBC ) + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx (\b(A(DODB\.Command\b.{0,100}?\b(Application uses a value of the wrong type for the current operation\b|error')| trappable error occurred in an external object\. The script cannot continue running\b)|Microsoft VBScript (compilation (\(0x8|error)|runtime (Error|\(0x8))\b|Object required: '|error '800)|Version Information:<\/b>( |\s)(Microsoft \.NET Framework|ASP\.NET) Version:|(\/[Ee]rror[Mm]essage\.aspx?\?[Ee]rror|>error 'ASP)\b) + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx \bServer Error in.{0,50}?\bApplication\b + ## RESPONSE_STATUS, + # skipped RESPONSE_STATUS rx ^404$ + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx

Site Error<\/h2>.{0,20}

An error was encountered while publishing this resource\. + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx \bThe error occurred in\b.{0,100}: line\b.{0,1000}\bColdFusion\b.*?\bStack Trace \(click to expand\)\b + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx Warning<\/b>.{0,100}?:.{0,1000}?\bon line\b + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx \b403 Forbidden\b.*?\bInternet Security and Acceleration Server\b + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx (<(TITLE>Index of.*?Index of.*?Index of|>\[To Parent Directory\]<\/[Aa]>
) + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx (\b((s(erver\.(((htm|ur)lencod|execut)e|createobject|mappath)|cripting\.filesystemobject)|(response\.(binary)?writ|vbscript\.encod)e|wscript\.(network|shell))\b|javax\.servlet)|\.(((createtex|ge)t|loadfrom)file|addheader)\b|.{1,20}?error '800(04005|40e31)'.{1,40}?Timeout expired| \(0x80040e31\)
Timeout expired
)|

internal server error<\/h1>.*?

part of the server has crashed or it has a configuration error\.<\/h2>|cannot connect to the server: timed out) + ## RESPONSE_STATUS, + # skipped RESPONSE_STATUS rx ^500$ + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx t:none,JSP compile error<\/title> + ## RESPONSE_BODY, + # skipped RESPONSE_BODY rx href\s?=[\s%22\']*[A-Za-z]\:\x5c([^%22\']+) + ## TX, :1 + # AC 1 + ## Rule: TX rx :1 +} + Added: trunk/varnish-tools/security.vcl/vcl/main.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/main.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/main.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,81 @@ + +/* Security.vcl main VCL file + * Copyright (C) 2009 Redpill Linpro AS + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Author: Kristian Lyngst?l <kristian at redpill-linpro.com> + * + * FIXME: We might need a Makefile for the paths here, but for now, they are + * hardcoded. Blah. + */ + +include "/etc/varnish/security/build/variables.vcl"; +include "/etc/varnish/security/modules/demo.vcl"; +include "/etc/varnish/security/modules/php.vcl"; +include "/etc/varnish/security/modules/sql.vcl"; +include "/etc/varnish/security/modules/xss.vcl"; +include "/etc/varnish/security/modules/cmd.vcl"; +include "/etc/varnish/security/modules/restricted-file-extensions.vcl"; +include "/etc/varnish/security/modules/content-encoding.vcl"; +include "/etc/varnish/security/modules/content-type.vcl"; +include "/etc/varnish/security/modules/localfiles.vcl"; +include "/etc/varnish/security/modules/request.vcl"; + +#include "/etc/varnish/security/modules/user-agent.vcl"; + +include "/etc/varnish/security/breach/20_protocol_violations.vcl"; +include "/etc/varnish/security/breach/21_protocol_anomalies.vcl"; +include "/etc/varnish/security/breach/23_request_limits.vcl"; +include "/etc/varnish/security/breach/30_http_policy.vcl"; +include "/etc/varnish/security/breach/35_bad_robots.vcl"; +include "/etc/varnish/security/breach/40_generic_attacks.vcl"; +include "/etc/varnish/security/breach/45_trojans.vcl"; +include "/etc/varnish/security/breach/50_outbound.vcl"; + +/* The value of '800' and up is used because it is not actual HTTP error + * codes. They should not be exposed. + * + * The list thus far: + * 800 - Debug + * 801 - Plain error (401-unauthorized might be a bad rewrite here) + * 802 - Redirect + */ +sub vcl_error { + if (obj.status == 800) { + set obj.http.X-SEC-Rule = req.http.X-SEC-Module "-" req.http.X-SEC-RuleId; + + set obj.status = 200; + } elsif (obj.status == 801) { + set obj.status = 401; + set obj.response = "Here be dragons! YARR! Wait, that's pirates."; + } elsif (obj.status == 802) { + set obj.status = 302; + set obj.response = "Redirected for fun and profit"; + set obj.http.Location = "http://images.google.com/images?q=llama"; + deliver; + } +} + +/* Catch-all handler */ +sub sec_general { + error 800 "BOOOYA!"; +} + +sub sec_sev1 { + call sec_general; +} + +/* vim: set syntax=c tw=76: */ Added: trunk/varnish-tools/security.vcl/vcl/modules/cmd.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/cmd.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/cmd.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,79 @@ + +# For now +sub sec_cmd_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + + +sub vcl_recv { + set req.http.X-SEC-Module = "cmd"; + +# Should it be "wget%20", "wget " or "wget\s+" ? +# "=cmd\W+" or "=cmd.+" is the best I can think of at the moment +# What about "=cmd(\%20| )" or... ? + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)wget.+") { + set req.http.X-SEC-RuleName = "Common command in URL: wget"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: wget"; + call sec_cmd_sev1; + } + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)curl.+") { + set req.http.X-SEC-RuleName = "Common command in URL: curl"; + set req.http.X-SEC-RuleId = "2"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: curl"; + call sec_cmd_sev1; + } + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)echo.+") { + set req.http.X-SEC-RuleName = "Common command in URL: curl"; + set req.http.X-SEC-RuleId = "3"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: curl"; + call sec_cmd_sev1; + } + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)cat.+") { + set req.http.X-SEC-RuleName = "Common command in URL: curl"; + set req.http.X-SEC-RuleId = "4"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: curl"; + call sec_cmd_sev1; + } + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)cmd.exe.+") { + set req.http.X-SEC-RuleName = "Common command in URL: cmd.exe"; + set req.http.X-SEC-RuleId = "5"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: cmd.exe"; + call sec_cmd_sev1; + } + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)nc(.exe)?.+(\-(l|p)?)?") { + set req.http.X-SEC-RuleName = "Common command in URL: netcat"; + set req.http.X-SEC-RuleId = "6"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: netcat"; + call sec_cmd_sev1; + } + + # Checks if someone tries to inject a common command name in URL + if (req.url ~ "(=|;|&&|%7C%7C)(whoami|who|uptime|last|df).*") { + set req.http.X-SEC-RuleName = "Common command in URL: cmd.exe"; + set req.http.X-SEC-RuleId = "7"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject a common command name in URL: cmd.exe"; + call sec_cmd_sev1; + } + + # Checks if someone tries to redirect output to /dev/null + if (req.url ~ "(>|%3E|-o)+" && req.url ~ "/dev/null") { + set req.http.X-SEC-RuleName = "Common redirect of command ouput in URL: /dev/null"; + set req.http.X-SEC-RuleId = "100"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to redirect command output in URL: /dev/null"; + call sec_cmd_sev1; + } +} Added: trunk/varnish-tools/security.vcl/vcl/modules/content-encoding.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/content-encoding.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/content-encoding.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,18 @@ + +# For now +sub sec_contentencoding_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + +sub vcl_recv { + set req.http.X-SEC-Module = "contentencoding"; + + # Security.vcl does not support content encodings + if(req.http.Content-Encoding ~ "!^Identity$"){ + set req.http.X-SEC-RuleName = "Inbound compressed content"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Blocks inbound compressed content"; + call sec_contentencoding_sev1; + } +} Added: trunk/varnish-tools/security.vcl/vcl/modules/content-type.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/content-type.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/content-type.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,21 @@ + +# For now +sub sec_contenttype_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + +sub vcl_recv { + set req.http.X-SEC-Module = "contenttype"; + + # Checks for which content-types we accept in GET and HEAD request: application/x-www-form-urlencoded, multipart/form-data request and text/xml + if(( req.request == "GET" || req.request == "HEAD" ) + # Content-type: application/x-www-form-urlencoded; charset=utf-8 +# && req.http.Content-Type ~ "(?:^(?:application\/x-www-form-urlencoded(?:;(?:\s?charset\s?=\s?[\w\d\-]{1,18})?)??$|multipart/form-data;)|text/xml)" ) { + && req.http.Content-Type ~ "application\/x-www-form-urlencoded;(\s?charset\s?=\s?[\w\d\-]{1,18})?)??$|multipart/form-data;)|text/xml)" ) { + set req.http.X-SEC-RuleName = "Request content type restricted"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks for accepted content-types"; + call sec_contenttype_sev1; + } +} Added: trunk/varnish-tools/security.vcl/vcl/modules/demo.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/demo.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/demo.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,43 @@ + +/* Security.VCL demonstration rules + * Copyright (C) 2009 Redpill Linpro AS + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Author: Kristian Lyngst?l <kristian at redpill-linpro.com> + * + * This file demonstrates the intended use of Security VCL for + * rule-matching and how to handle the fallout. + */ + +sub sec_demo_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + +sub vcl_recv { + set req.http.X-SEC-Module = "demo"; + + if (req.url ~ "/exploit/") { + //TEST:demo-1:GET:/exploit/foo/bar:bla + //TESTN:demo-1:GET:/notexploit/foo/bar + set req.http.X-SEC-RuleName = "Awsome demo for Security.VCL"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "This rule triggers when an 31337 h4x0r accesses a dir with name /exploit/"; + call sec_demo_sev1; + } +} + +/* vim: set syntax=c tw=76: */ Added: trunk/varnish-tools/security.vcl/vcl/modules/localfiles.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/localfiles.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/localfiles.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,60 @@ + +# For now +sub sec_localfiles_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + + +sub vcl_recv { + set req.http.X-SEC-Module = "localfiles"; + + # Checks if someone tries to access common files from /etc/ dir + if (req.url ~ "/etc/(passwd(\-)?|(g)?shadow(\-)?|motd|group(\-)?)") { + set req.http.X-SEC-RuleName = "Local file access attempt in: /etc/"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to access known local files in: /etc/"; + call sec_localfiles_sev1; + } + + # Checks if someone tries to access common dirs in /etc/ dir + if (req.url ~ "/etc/(apache(2)?|httpd|phpmyadmin|mysql|php(4|5)?)/") { + set req.http.X-SEC-RuleName = "Local dir access attempt in: /etc/"; + set req.http.X-SEC-RuleId = "2"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to access known local directories in: /etc/"; + call sec_localfiles_sev1; + } + + # Checks if someone tries to access /tmp/ dir + if (req.url ~ "/tmp/") { + set req.http.X-SEC-RuleName = "Local dir access attempt: /tmp/"; + set req.http.X-SEC-RuleId = "3"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries access a local directory: /tmp/"; + call sec_localfiles_sev1; + } + + # Checks if someone tries to access common dirs in /var/ dir + if (req.url ~ "/var/(log|backups|mail|www)/") { + set req.http.X-SEC-RuleName = "Local dir access attempt in: /var/"; + set req.http.X-SEC-RuleId = "4"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries access a local directory in: /var/"; + call sec_localfiles_sev1; + } + + # Checks if someone tries to access common files from /proc/ dir + if (req.url ~ "/proc/(self/environ|cmdline|cpuinfo|mounts|mdstat|partitions|version(_signature)?|uptime)") { + set req.http.X-SEC-RuleName = "Local file access attempt in: /proc/"; + set req.http.X-SEC-RuleId = "5"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to access known local files in: /proc/"; + call sec_localfiles_sev1; + } + + # Checks if someone tries a directory traversal + if (req.url ~ "\.(\.)?/\.(\.)?/\.(\.)?") { + set req.http.X-SEC-RuleName = "Directory traversal attempt: ../../.. or ././../ etc"; + set req.http.X-SEC-RuleId = "6"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries a directory traversal of more than 3 dirs"; + call sec_localfiles_sev1; + } + +} Added: trunk/varnish-tools/security.vcl/vcl/modules/php.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/php.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/php.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,134 @@ + +# For now +sub sec_php_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + + +sub vcl_recv { + set req.http.X-SEC-Module = "php"; + + # Checks if someone tries to alter predefined $GLOBALS variable via url + if (req.url ~ "GLOBALS\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable GLOBALS"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: GLOBALS"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _SERVER variable via url + if (req.url ~ "_SERVER\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _SERVER"; + set req.http.X-SEC-RuleId = "2"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _SERVER"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _GET variable via url + if (req.url ~ "_GET\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _GET"; + set req.http.X-SEC-RuleId = "3"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _GET"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _POST variable via url + if (req.url ~ "_POST\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _POST"; + set req.http.X-SEC-RuleId = "4"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _POST"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _FILE variable via url + if (req.url ~ "_FILES\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _FILES"; + set req.http.X-SEC-RuleId = "5"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _FILES"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _REQUEST variable via url + if (req.url ~ "_REQUEST\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _REQUEST"; + set req.http.X-SEC-RuleId = "6"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _REQUEST"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _SESSION variable via url + if (req.url ~ "_SESSION\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _SESSION"; + set req.http.X-SEC-RuleId = "7"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _SESSION"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _ENV variable via url + if (req.url ~ "_ENV\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _ENV"; + set req.http.X-SEC-RuleId = "8"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _ENV"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _COOKIE variable via url + if (req.url ~ "_COOKIE\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _COOKIE"; + set req.http.X-SEC-RuleId = "9"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _COOKIE"; + call sec_php_sev1; + } + + # Checks if someone tries to alter predefined _REQUEST variable via url + if (req.url ~ "_REQUEST\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _REQUEST"; + set req.http.X-SEC-RuleId = "8"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _REQUEST"; + call sec_php_sev1; + } + +# One could make one long regexp with common php statements. For now: + + # Generic check for code execution + if (req.url ~ "system\(") { + set req.http.X-SEC-RuleName = "PHP command: system()"; + set req.http.X-SEC-RuleId = "9"; + set req.http.X-SEC-RuleInfo = "Generic check for PHP commands in URL: system()"; + call sec_php_sev1; + } + + # Generic check for code execution + if (req.url ~ "passthru\(") { + set req.http.X-SEC-RuleName = "PHP command: passthru()"; + set req.http.X-SEC-RuleId = "10"; + set req.http.X-SEC-RuleInfo = "Generic check for PHP commands in URL: passthru()"; + call sec_php_sev1; + } + + # Generic check for code execution + if (req.url ~ "eval\(") { + set req.http.X-SEC-RuleName = "PHP command: eval()"; + set req.http.X-SEC-RuleId = "11"; + set req.http.X-SEC-RuleInfo = "Generic check for PHP commands in URL: eval()"; + call sec_php_sev1; + } + + # Generic check for PHP code inclusion in URL + if (req.url ~ "(<|\%3C)?\?(php)?.*(php)?\?(>|\%3E)?") { + set req.http.X-SEC-RuleName = "PHP code inclusion in URL: <?php ..code.. ?>"; + set req.http.X-SEC-RuleId = "12"; + set req.http.X-SEC-RuleInfo = "Generic check for PHP code in URL: <?php ..code.. ?>"; + call sec_php_sev1; + } + + # Generic check for remote code inclusion from external sites + if (req.url ~ "=?(https?|ftps?|php)://") { + set req.http.X-SEC-RuleName = "Remote site in URL parameter"; + set req.http.X-SEC-RuleId = "100"; + set req.http.X-SEC-RuleInfo = "Generic check for remote code inclusion from external sites"; + call sec_php_sev1; + } + +} Added: trunk/varnish-tools/security.vcl/vcl/modules/request.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/request.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/request.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,24 @@ + +# For now +sub sec_request_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + +sub vcl_recv { + set req.http.X-SEC-Module = "request"; + +#sub vcl_recv { + # Checks if someone tries use a blacklisted request method + if ( req.request == "PUT" +# || req.request == "POST" + || req.request == "TRACE" + || req.request == "OPTIONS" + || req.request == "CONNECT" + || req.request == "DELETE") { + set req.http.X-SEC-RuleName = "Blocked Requestmethods"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries use a blacklisted request method"; + call sec_request_sev1; + } +} Added: trunk/varnish-tools/security.vcl/vcl/modules/restricted-file-extensions.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/restricted-file-extensions.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/restricted-file-extensions.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,19 @@ + +# For now +sub sec_restrictedfileextentions_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + +sub vcl_recv { + set req.http.X-SEC-Module = "restrictedfileextentions"; + + # List of file extensions to not allow (blacklist) +# if ( req.url ~ "\.(?:c(?:o(?:nf(?:ig)?|m)|s(?:proj|r)?|dx|er|fg|md)|p(?:rinter|ass|db|ol|wd)|v(?:b(?:proj|s)?|sdisco)|a(?:s(?:ax?|cx)|xd)|d(?:bf?|at|ll|os)|i(?:d[acq]|n[ci])|ba(?:[kt]|ckup)|res(?:ources|x)|s(?:h?tm|ql|ys)|l(?:icx|nk|og)|\w{0,5}~|webinfo|ht[rw]|xs[dx]|key|mdb|old)$" ) { + if ( req.url ~ "\.(c(o(nf(ig)?|m)|s(proj|r)?|dx|er|fg|md)|p(rinter|ass|db|ol|wd)|v(b(proj|s)?|sdisco)|a(s(ax?|cx)|xd)|d(bf?|at|ll|os)|i(d[acq]|n[ci])|ba([kt]|ckup)|res(ources|x)|s(h?tm|ql|ys)|l(icx|nk|og)|\w{0,5}~|webinfo|ht[rw]|xs[dx]|key|mdb|old)$" ) { + set req.http.X-SEC-RuleName = "Restricted file extensions"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks for file extensions that are not allowed"; + call sec_restrictedfileextentions_sev1; + } +} Added: trunk/varnish-tools/security.vcl/vcl/modules/sql.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/sql.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/sql.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,76 @@ + +# For now +sub sec_sql_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + + +sub vcl_recv { + set req.http.X-SEC-Module = "sql"; + + # Checks if someone tries to use SQL statement in URL: SELECT FROM + if (req.url ~ ".+SELECT.+FROM") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: SELECT FROM"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: SELECT FROM"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: UNION SELECT + if (req.url ~ ".+UNION\s+SELECT") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: UNION SELECT"; + set req.http.X-SEC-RuleId = "2"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: UNION SELECT"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: UPDATE SET + if (req.url ~ ".+UPDATE.+SET") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: UPDATE SET"; + set req.http.X-SEC-RuleId = "3"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: UPDATE SET"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: INSERT INTO + if (req.url ~ ".+INSERT.+INTO") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: INSERT INTO"; + set req.http.X-SEC-RuleId = "4"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: INSERT INTO"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: DELETE FROM + if (req.url ~ ".+DELETE.+FROM") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: DELETE FROM"; + set req.http.X-SEC-RuleId = "5"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: DELETE FROM"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: ASCII SELECT + if (req.url ~ ".+ASCII\(.+SELECT") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: ASCII SELECT"; + set req.http.X-SEC-RuleId = "6"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: ASCII SELECT"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: DROP TABLE + if (req.url ~ ".+DROP.+TABLE") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: DROP TABLE"; + set req.http.X-SEC-RuleId = "7"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: DROP TABLE"; + call sec_sql_sev1; + } + + # Checks if someone tries to use SQL statement in URL: DROP DATABASE + if (req.url ~ ".+DROP.+DATABASE") { + set req.http.X-SEC-RuleName = "SQL Injection Attempt: DROP DATABASE"; + set req.http.X-SEC-RuleId = "8"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to use SQL statement in URL: DROP DATABASE"; + call sec_sql_sev1; + } + +} Added: trunk/varnish-tools/security.vcl/vcl/modules/user-agent.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/user-agent.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/user-agent.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,200 @@ + +# For now +sub sec_useragent_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + + +sub vcl_recv { + set req.http.X-SEC-Module = "useragent"; + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "php_uname\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: php_uname"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: php_uname"; + call sec_useragent_sev1; + } + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "curl_init\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: curl_init"; + set req.http.X-SEC-RuleId = "2"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: curl_init"; + call sec_useragent_sev1; + } + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "curl_setopt\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: curl_setopt"; + set req.http.X-SEC-RuleId = "3"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: curl_setopt"; + call sec_useragent_sev1; + } + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "curl_exec\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: curl_exec"; + set req.http.X-SEC-RuleId = "4"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: curl_exec"; + call sec_useragent_sev1; + } + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "curl_close\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: curl_close"; + set req.http.X-SEC-RuleId = "5"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: curl_close"; + call sec_useragent_sev1; + } + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "fopen\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: fopen"; + set req.http.X-SEC-RuleId = "6"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: fopen"; + call sec_useragent_sev1; + } + + # Checks for php code in User-Agent + if (req.http.user-agent ~ "fwrite\(") { + set req.http.X-SEC-RuleName = "PHP Code in User-Agent: fwrite"; + set req.http.X-SEC-RuleId = "7"; + set req.http.X-SEC-RuleInfo = "Checks for php code in User-Agent: fwrite"; + call sec_useragent_sev1; + } + + # Checks for bad User-Agent + if ( + req.http.user-agent ~ "^$" + || req.http.user-agent ~ "^Java" + || req.http.user-agent ~ "^Jakarta" + || req.http.user-agent ~ "IDBot" + || req.http.user-agent ~ "id-search" + || req.http.user-agent ~ "User-Agent" + || req.http.user-agent ~ "compatible ;" + || req.http.user-agent ~ "ConveraCrawler" + || req.http.user-agent ~ "^Mozilla$" + || req.http.user-agent ~ "libwww" + || req.http.user-agent ~ "lwp-trivial" + || req.http.user-agent ~ "curl" + || req.http.user-agent ~ "PHP/" + || req.http.user-agent ~ "urllib" + || req.http.user-agent ~ "GT:WWW" + || req.http.user-agent ~ "Snoopy" + || req.http.user-agent ~ "MFC_Tear_Sample" + || req.http.user-agent ~ "HTTP::Lite" + || req.http.user-agent ~ "PHPCrawl" + || req.http.user-agent ~ "URI::Fetch" + || req.http.user-agent ~ "Zend_Http_Client" + || req.http.user-agent ~ "http client" + || req.http.user-agent ~ "PECL::HTTP" + || req.http.user-agent ~ "panscient.com" + || req.http.user-agent ~ "IBM EVV" + || req.http.user-agent ~ "Bork-edition" + || req.http.user-agent ~ "Fetch API Request" + || req.http.user-agent ~ "PleaseCrawl" + || req.http.user-agent ~ "[A-Z][a-z]{3,} [a-z]{4,} [a-z]{4,}" + || req.http.user-agent ~ "layeredtech.com" + || req.http.user-agent ~ "WEP Search" + || req.http.user-agent ~ "Wells Search II" + || req.http.user-agent ~ "Missigua Locator" + || req.http.user-agent ~ "ISC Systems iRc Search 2.1" + || req.http.user-agent ~ "Microsoft URL Control" + || req.http.user-agent ~ "Indy Library" + || req.http.user-agent == "8484 Boston Project v 1.0" + || req.http.user-agent == "Atomic_Email_Hunter/4.0" + || req.http.user-agent == "atSpider/1.0" + || req.http.user-agent == "autoemailspider" + || req.http.user-agent == "China Local Browse 2.6" + || req.http.user-agent == "ContactBot/0.2" + || req.http.user-agent == "ContentSmartz" + || req.http.user-agent == "DataCha0s/2.0" + || req.http.user-agent == "DataCha0s/2.0" + || req.http.user-agent == "DBrowse 1.4b" + || req.http.user-agent == "DBrowse 1.4d" + || req.http.user-agent == "Demo Bot DOT 16b" + || req.http.user-agent == "Demo Bot Z 16b" + || req.http.user-agent == "DSurf15a 01" + || req.http.user-agent == "DSurf15a 71" + || req.http.user-agent == "DSurf15a 81" + || req.http.user-agent == "DSurf15a VA" + || req.http.user-agent == "EBrowse 1.4b" + || req.http.user-agent == "Educate Search VxB" + || req.http.user-agent == "EmailSiphon" + || req.http.user-agent == "EmailWolf 1.00" + || req.http.user-agent == "ESurf15a 15" + || req.http.user-agent == "ExtractorPro" + || req.http.user-agent == "Franklin Locator 1.8" + || req.http.user-agent == "FSurf15a 01" + || req.http.user-agent == "Full Web Bot 0416B" + || req.http.user-agent == "Full Web Bot 0516B" + || req.http.user-agent == "Full Web Bot 2816B" + || req.http.user-agent == "Guestbook Auto Submitter" + || req.http.user-agent == "Industry Program 1.0.x" + || req.http.user-agent == "ISC Systems iRc Search 2.1" + || req.http.user-agent == "IUPUI Research Bot v 1.9a" + || req.http.user-agent == "LARBIN-EXPERIMENTAL (efp at gmx.net)" + || req.http.user-agent == "LetsCrawl.com/1.0 +http://letscrawl.com/" + || req.http.user-agent == "Lincoln State Web Browser" + || req.http.user-agent == "LMQueueBot/0.2" + || req.http.user-agent == "LWP::Simple/5.803" + || req.http.user-agent == "Mac Finder 1.0.xx" + || req.http.user-agent == "MFC Foundation Class Library 4.0" + || req.http.user-agent == "Microsoft URL Control - 6.00.8xxx" + || req.http.user-agent == "Missauga Locate 1.0.0" + || req.http.user-agent == "Missigua Locator 1.9" + || req.http.user-agent == "Missouri College Browse" + || req.http.user-agent == "Mizzu Labs 2.2" + || req.http.user-agent == "Mo College 1.9" + || req.http.user-agent == "Mozilla/2.0 (compatible; NEWT ActiveX; Win32)" + || req.http.user-agent == "Mozilla/3.0 (compatible; Indy Library)" + || req.http.user-agent == "Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)" + || req.http.user-agent == "Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)" + || req.http.user-agent == "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent" + || req.http.user-agent == "Mozilla/4.0 efp at gmx.net" + || req.http.user-agent == "Mozilla/5.0 (Version: xxxx Type:xx)" + || req.http.user-agent == "MVAClient" + || req.http.user-agent == "NameOfAgent (CMS Spider)" + || req.http.user-agent == "NASA Search 1.0" + || req.http.user-agent == "Nsauditor/1.x" + || req.http.user-agent == "PBrowse 1.4b" + || req.http.user-agent == "PEval 1.4b" + || req.http.user-agent == "Poirot" + || req.http.user-agent == "Port Huron Labs" + || req.http.user-agent == "Production Bot 0116B" + || req.http.user-agent == "Production Bot 2016B" + || req.http.user-agent == "Production Bot DOT 3016B" + || req.http.user-agent == "Program Shareware 1.0.2" + || req.http.user-agent == "PSurf15a 11" + || req.http.user-agent == "PSurf15a 51" + || req.http.user-agent == "PSurf15a VA" + || req.http.user-agent == "psycheclone" + || req.http.user-agent == "RSurf15a 41" + || req.http.user-agent == "RSurf15a 51" + || req.http.user-agent == "RSurf15a 81" + || req.http.user-agent == "searchbot admin at google.com" + || req.http.user-agent == "ShablastBot 1.0" + || req.http.user-agent == "snap.com beta crawler v0" + || req.http.user-agent == "Snapbot/1.0" + || req.http.user-agent == "sogou develop spider" + || req.http.user-agent == "Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" + || req.http.user-agent == "sogou spider" + || req.http.user-agent == "Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" + || req.http.user-agent == "sohu agent" + || req.http.user-agent == "SSurf15a 11" + || req.http.user-agent == "TSurf15a 11" + || req.http.user-agent == "Under the Rainbow 2.2" + || req.http.user-agent == "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" + || req.http.user-agent == "VadixBot" + || req.http.user-agent == "WebVulnCrawl.blogspot.com/1.0 libwww-perl/5.803" + || req.http.user-agent == "Wells Search II" + || req.http.user-agent == "WEP Search 00" + ) { + set req.http.X-SEC-RuleName = "Unwanted User-Agent"; + set req.http.X-SEC-RuleId = "100"; + set req.http.X-SEC-RuleInfo = "Checks if User-Agent is in banned list"; + call sec_useragent_sev1; + } +} Added: trunk/varnish-tools/security.vcl/vcl/modules/xss.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/xss.vcl (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/modules/xss.vcl 2009-08-14 10:54:53 UTC (rev 4180) @@ -0,0 +1,58 @@ + +# For now +sub sec_xss_sev1 { + set req.http.X-SEC-Severity = "1"; + call sec_sev1; +} + +sub vcl_recv { + set req.http.X-SEC-Module = "xss"; + +# # Checks if someone tries to inject java/vb script for XSS in URL +# if (req.url ~ "<?(java|vb)?script>?.*<.+\/script>?") { +# set req.http.X-SEC-RuleName = "Cross Site Scripting Attempt"; +# set req.http.X-SEC-RuleId = "1"; +# set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject java/vb script for XSS in URL"; +# call sec_xss_sev1; +# } + + # Checks if someone tries to inject java/vb script for XSS in URL + if (req.url ~ "(<|\%3C)?(java|vb)?script(>|\%3E).*(<|\%3C).*\/script(>|\%3E)?") { + set req.http.X-SEC-RuleName = "Cross Site Scripting Attempt"; + set req.http.X-SEC-RuleId = "1"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject java/vb script for XSS in URL"; + call sec_xss_sev1; + } + + # Checks if someone tries to inject java/vb script for XSS in URL + if (req.url ~ "(java|vb)?script:") { + set req.http.X-SEC-RuleName = "Cross Site Scripting Attempt: (java|vb)script:"; + set req.http.X-SEC-RuleId = "2"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject java/vb script for XSS in URL"; + call sec_xss_sev1; + } + + # Checks if someone tries to inject java/vb script for XSS in URL + if (req.url ~ "\(.*javascript.*\)") { + set req.http.X-SEC-RuleName = "Cross Site Scripting Attempt: (javascript)"; + set req.http.X-SEC-RuleId = "3"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject java/vb script for XSS in URL"; + call sec_xss_sev1; + } + + # Checks if someone tries to inject java/vb script for XSS in URL + if (req.url ~ "\(.*vbscript.*\)") { + set req.http.X-SEC-RuleName = "Cross Site Scripting Attempt: (vbscript)"; + set req.http.X-SEC-RuleId = "4"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject java/vb script for XSS in URL"; + call sec_xss_sev1; + } + + # Checks if someone tries to inject java/vb script for XSS in URL + if (req.url ~ ":?.*url\(") { + set req.http.X-SEC-RuleName = "Cross Site Scripting Attempt: :url("; + set req.http.X-SEC-RuleId = "5"; + set req.http.X-SEC-RuleInfo = "Checks if someone tries to inject java/vb script for XSS in URL"; + call sec_xss_sev1; + } +} From kristian at projects.linpro.no Fri Aug 14 11:55:53 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Fri, 14 Aug 2009 13:55:53 +0200 (CEST) Subject: r4181 - trunk/varnish-tools/security.vcl/vcl Message-ID: <20090814115553.97AD61F7C74@projects.linpro.no> Author: kristian Date: 2009-08-14 13:55:53 +0200 (Fri, 14 Aug 2009) New Revision: 4181 Added: trunk/varnish-tools/security.vcl/vcl/VARIABLES Log: Security.VCL: Add missing VARIABLES-file Added: trunk/varnish-tools/security.vcl/vcl/VARIABLES =================================================================== --- trunk/varnish-tools/security.vcl/vcl/VARIABLES (rev 0) +++ trunk/varnish-tools/security.vcl/vcl/VARIABLES 2009-08-14 11:55:53 UTC (rev 4181) @@ -0,0 +1,27 @@ +# List of variables, one per line. +# This is used to auto-generate variables.vcl AND to verify that no +# X-SEC-variables occur in modules that aren't listed here. +# Comments start with #, blank lines are skipped. + +X-SEC-Severity +# return value for function +X-SEC-Return +# arguments to function +X-SEC-Arg +X-SEC-Module + +# Rule IDs +# 1-99,999; reserved for local (internal) use. Use as you see fit but do not use this range for rules that are distributed to others. +# 100,000-199,999; reserved for internal use of the engine, to assign to rules that do not have explicit IDs. +# 200,000-299,999; reserved for rules published at modsecurity.org. +# 300,000-399,999; reserved for rules published at gotroot.com. +# 400,000-419,999; unused (available for reservation). +# 420,000-429,999; reserved for ScallyWhack. +# 430,000-899,999; unused (available for reservation). +# 900,000-999,999; reserved for the Core Rules project. +# 1,000,000 and above; unused (available for reservation). +X-SEC-RuleId +# Rule = Module-RuleId; +X-SEC-Rule +X-SEC-RuleName +X-SEC-RuleInfo From phk at projects.linpro.no Mon Aug 17 08:45:36 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 17 Aug 2009 10:45:36 +0200 (CEST) Subject: r4182 - trunk/varnish-cache/bin/varnishd Message-ID: <20090817084537.007EB1F7B9A@projects.linpro.no> Author: phk Date: 2009-08-17 10:45:36 +0200 (Mon, 17 Aug 2009) New Revision: 4182 Modified: trunk/varnish-cache/bin/varnishd/flint.lnt Log: Silence 679 and 776, which are noisy on 64p32i systems Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2009-08-14 11:55:53 UTC (rev 4181) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2009-08-17 08:45:36 UTC (rev 4182) @@ -8,6 +8,7 @@ -e454 // mutex not released (...ReleaseLocked) -e457 // unprotected access -e777 // float equality comparison +-e679 // Suspicious Truncation in arithmetic expression combining with pointer -esym(458, lbv_assert) // unlocked access -esym(458, params) // unlocked access @@ -144,6 +145,7 @@ -e574 // Signed-unsigned mix with relational -e712 // Loss of precision (assignment) (long long to -e747 // Significant prototype coercion (arg. no. 2) long +-e776 // Possible truncation of addition /* From phk at projects.linpro.no Mon Aug 17 10:53:18 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 17 Aug 2009 12:53:18 +0200 (CEST) Subject: r4183 - trunk/varnish-cache/bin/varnishd Message-ID: <20090817105318.0C2111F7B9A@projects.linpro.no> Author: phk Date: 2009-08-17 12:53:17 +0200 (Mon, 17 Aug 2009) New Revision: 4183 Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c trunk/varnish-cache/bin/varnishd/cache_waiter_kqueue.c trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c trunk/varnish-cache/bin/varnishd/cache_waiter_ports.c Log: Disable SO_LINGER when we time out a connection due to sess_timeout, so that we do not RST connections that have still not transmitted their data. Since we were able to get the writev(2) to detach the socket, we should not end up sleeping in the close(2) either. We still RST the socket for all error conditions. Ideally I would still like to RST connections that have no outstanding data after their sess_timeout, in order to avoid the 2*RTT+misc timeouts delays associated with loosing a TCP socket for a client that have gone to meet some other IP#. In particular with load-balancers, this allows the load balancer to declare the session dead right away, and reuse it for something more productive. Unfortunately, this lacks OS support in all presently released OS'es: you cannot ask if a socket is done transmitting what you asked it to. FreeBSD-8.0 will have experimental support for this (FIONWRITE) and I will revisit it in that context. Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c 2009-08-17 08:45:36 UTC (rev 4182) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c 2009-08-17 10:53:17 UTC (rev 4183) @@ -190,6 +190,7 @@ if (sp->t_open > deadline) break; VTAILQ_REMOVE(&sesshead, sp, list); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_kqueue.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_kqueue.c 2009-08-17 08:45:36 UTC (rev 4182) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_kqueue.c 2009-08-17 10:53:17 UTC (rev 4183) @@ -198,6 +198,7 @@ if (sp->t_open > deadline) break; VTAILQ_REMOVE(&sesshead, sp, list); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c 2009-08-17 08:45:36 UTC (rev 4182) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c 2009-08-17 10:53:17 UTC (rev 4183) @@ -147,6 +147,7 @@ continue; VTAILQ_REMOVE(&sesshead, sp, list); vca_unpoll(fd); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_ports.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_ports.c 2009-08-17 08:45:36 UTC (rev 4182) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_ports.c 2009-08-17 10:53:17 UTC (rev 4183) @@ -139,6 +139,7 @@ VTAILQ_REMOVE(&sesshead, sp, list); if(sp->fd != -1) vca_del(sp->fd); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } From phk at projects.linpro.no Mon Aug 17 11:26:20 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 17 Aug 2009 13:26:20 +0200 (CEST) Subject: r4184 - trunk/varnish-cache/bin/varnishd Message-ID: <20090817112620.383161F7B9A@projects.linpro.no> Author: phk Date: 2009-08-17 13:26:19 +0200 (Mon, 17 Aug 2009) New Revision: 4184 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend_poll.c Log: An explanatory comment. Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2009-08-17 10:53:17 UTC (rev 4183) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2009-08-17 11:26:19 UTC (rev 4184) @@ -69,6 +69,13 @@ pthread_t VCA_thread; static struct timeval tv_sndtimeo; static struct timeval tv_rcvtimeo; + +/* + * We want to get out of any kind of touble-hit TCP connections as fast + * as absolutely possible, so we set them LINGER enabled with zero timeout, + * so that even if there are outstanding write data on the socket, a close(2) + * will return immediately. + */ static const struct linger linger = { .l_onoff = 1, }; Modified: trunk/varnish-cache/bin/varnishd/cache_backend_poll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend_poll.c 2009-08-17 10:53:17 UTC (rev 4183) +++ trunk/varnish-cache/bin/varnishd/cache_backend_poll.c 2009-08-17 11:26:19 UTC (rev 4184) @@ -242,18 +242,85 @@ } /*-------------------------------------------------------------------- - * One thread per backend to be poked. + * Record pokings... */ -static void * -vbp_wrk_poll_backend(void *priv) +static void +vbp_start_poke(struct vbp_target *vt) { - struct vbp_target *vt; + CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); + +#define BITMAP(n, c, t, b) vt->n <<= 1; +#include "cache_backend_poll.h" +#undef BITMAP + + vt->last = 0; + vt->resp_buf[0] = '\0'; +} + +static void +vbp_has_poked(struct vbp_target *vt) +{ unsigned i, j; uint64_t u; const char *logmsg; char bits[10]; + CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); + + /* Calculate exponential average */ + if (vt->happy & 1) { + if (vt->rate < AVG_RATE) + vt->rate += 1.0; + vt->avg += (vt->last - vt->avg) / vt->rate; + } + + i = 0; +#define BITMAP(n, c, t, b) bits[i++] = (vt->n & 1) ? c : '-'; +#include "cache_backend_poll.h" +#undef BITMAP + bits[i] = '\0'; + + u = vt->happy; + for (i = j = 0; i < vt->probe.window; i++) { + if (u & 1) + j++; + u >>= 1; + } + vt->good = j; + + if (vt->good >= vt->probe.threshold) { + if (vt->backend->healthy) + logmsg = "Still healthy"; + else + logmsg = "Back healthy"; + vt->backend->healthy = 1; + } else { + if (vt->backend->healthy) + logmsg = "Went sick"; + else + logmsg = "Still sick"; + vt->backend->healthy = 0; + } + VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", + vt->backend->vcl_name, logmsg, bits, + vt->good, vt->probe.threshold, vt->probe.window, + vt->last, vt->avg, vt->resp_buf); + + if (!vt->stop) + TIM_sleep(vt->probe.interval); +} + +/*-------------------------------------------------------------------- + * One thread per backend to be poked. + */ + +static void * +vbp_wrk_poll_backend(void *priv) +{ + struct vbp_target *vt; + unsigned u; + THR_SetName("backend poll"); CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC); @@ -273,59 +340,29 @@ if (vt->probe.threshold == 0) vt->probe.threshold = 3; - printf("Probe(\"%s\", %g, %g)\n", - vt->req, - vt->probe.timeout, - vt->probe.interval); + if (vt->probe.threshold == ~0) + vt->probe.initial = vt->probe.threshold - 1; - /*lint -e{525} indent */ - while (!vt->stop) { -#define BITMAP(n, c, t, b) vt->n <<= 1; -#include "cache_backend_poll.h" -#undef BITMAP - vt->last = 0; - vt->resp_buf[0] = '\0'; - vbp_poke(vt); + if (vt->probe.initial > vt->probe.threshold) + vt->probe.initial = vt->probe.threshold; - /* Calculate exponential average */ - if (vt->happy & 1) { - if (vt->rate < AVG_RATE) - vt->rate += 1.0; - vt->avg += (vt->last - vt->avg) / vt->rate; - } +printf("Initial %u\n", vt->probe.initial); - i = 0; -#define BITMAP(n, c, t, b) bits[i++] = (vt->n & 1) ? c : '-'; -#include "cache_backend_poll.h" -#undef BITMAP - bits[i] = '\0'; + printf("Probe(\"%s\", %g, %g)\n", + vt->req, vt->probe.timeout, vt->probe.interval); - u = vt->happy; - for (i = j = 0; i < vt->probe.window; i++) { - if (u & 1) - j++; - u >>= 1; - } - vt->good = j; +if (0) { + for (u = 0; u < vt->probe.initial; u++) { + vbp_start_poke(vt); + vt->happy |= 1; + vbp_has_poked(vt); + } +} - if (vt->good >= vt->probe.threshold) { - if (vt->backend->healthy) - logmsg = "Still healthy"; - else - logmsg = "Back healthy"; - vt->backend->healthy = 1; - } else { - if (vt->backend->healthy) - logmsg = "Went sick"; - else - logmsg = "Still sick"; - vt->backend->healthy = 0; - } - VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", - vt->backend->vcl_name, logmsg, bits, - vt->good, vt->probe.threshold, vt->probe.window, - vt->last, vt->avg, vt->resp_buf); - + while (!vt->stop) { + vbp_start_poke(vt); + vbp_poke(vt); + vbp_has_poked(vt); if (!vt->stop) TIM_sleep(vt->probe.interval); } From phk at projects.linpro.no Mon Aug 17 11:53:02 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 17 Aug 2009 13:53:02 +0200 (CEST) Subject: r4185 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include lib/libvcl Message-ID: <20090817115302.3F9F51F7B9A@projects.linpro.no> Author: phk Date: 2009-08-17 13:53:01 +0200 (Mon, 17 Aug 2009) New Revision: 4185 Modified: trunk/varnish-cache/bin/varnishd/cache_backend_poll.c trunk/varnish-cache/bin/varnishtest/tests/r00306.vtc trunk/varnish-cache/bin/varnishtest/tests/s00002.vtc trunk/varnish-cache/bin/varnishtest/tests/v00014.vtc trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcc_backend.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c Log: Add a ".initial" property to backend probe specifications. This is the number of good probes we pretend to have already seen when we start up, in order to speed up getting healthy backends. The default value is one less than the .threshold, so the backend will be set healthy if it manages to respond correctly to the very first probe we send to it. (A bit of this commit leaked in during r4184) Modified: trunk/varnish-cache/bin/varnishd/cache_backend_poll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend_poll.c 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/bin/varnishd/cache_backend_poll.c 2009-08-17 11:53:01 UTC (rev 4185) @@ -306,9 +306,6 @@ vt->backend->vcl_name, logmsg, bits, vt->good, vt->probe.threshold, vt->probe.window, vt->last, vt->avg, vt->resp_buf); - - if (!vt->stop) - TIM_sleep(vt->probe.interval); } /*-------------------------------------------------------------------- @@ -340,24 +337,20 @@ if (vt->probe.threshold == 0) vt->probe.threshold = 3; - if (vt->probe.threshold == ~0) + if (vt->probe.threshold == ~0U) vt->probe.initial = vt->probe.threshold - 1; if (vt->probe.initial > vt->probe.threshold) vt->probe.initial = vt->probe.threshold; -printf("Initial %u\n", vt->probe.initial); - printf("Probe(\"%s\", %g, %g)\n", vt->req, vt->probe.timeout, vt->probe.interval); -if (0) { for (u = 0; u < vt->probe.initial; u++) { vbp_start_poke(vt); vt->happy |= 1; vbp_has_poked(vt); } -} while (!vt->stop) { vbp_start_poke(vt); Modified: trunk/varnish-cache/bin/varnishtest/tests/r00306.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00306.vtc 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/bin/varnishtest/tests/r00306.vtc 2009-08-17 11:53:01 UTC (rev 4185) @@ -27,6 +27,7 @@ .host = "127.0.0.1"; .port = "9180"; .probe = { .url = "/"; + .initial = 0; } } director foo random { Modified: trunk/varnish-cache/bin/varnishtest/tests/s00002.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/s00002.vtc 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/bin/varnishtest/tests/s00002.vtc 2009-08-17 11:53:01 UTC (rev 4185) @@ -18,6 +18,7 @@ .interval = 1s; .window = 2; .threshold = 1; + .initial = 0; } } sub vcl_fetch { Modified: trunk/varnish-cache/bin/varnishtest/tests/v00014.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/v00014.vtc 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/bin/varnishtest/tests/v00014.vtc 2009-08-17 11:53:01 UTC (rev 4185) @@ -20,6 +20,7 @@ .interval = 1s; .window = 3; .threshold = 2; + .initial = 0; } } Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/include/vrt.h 2009-08-17 11:53:01 UTC (rev 4185) @@ -54,6 +54,7 @@ double interval; unsigned window; unsigned threshold; + unsigned initial; }; /* Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_backend.c 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/lib/libvcl/vcc_backend.c 2009-08-17 11:53:01 UTC (rev 4185) @@ -341,7 +341,8 @@ struct fld_spec *fs; struct token *t_field; struct token *t_did = NULL, *t_window = NULL, *t_threshold = NULL; - unsigned window, threshold; + struct token *t_initial = NULL; + unsigned window, threshold, initial; fs = vcc_FldSpec(tl, "?url", @@ -350,6 +351,7 @@ "?interval", "?window", "?threshold", + "?initial", NULL); ExpectErr(tl, '{'); @@ -357,6 +359,7 @@ window = 0; threshold = 0; + initial = 0; Fb(tl, 0, "\t.probe = {\n"); while (tl->t->tok != '}') { @@ -397,6 +400,11 @@ window = vcc_UintVal(tl); vcc_NextToken(tl); ERRCHK(tl); + } else if (vcc_IdIs(t_field, "initial")) { + t_initial = tl->t; + initial = vcc_UintVal(tl); + vcc_NextToken(tl); + ERRCHK(tl); } else if (vcc_IdIs(t_field, "threshold")) { t_threshold = tl->t; threshold = vcc_UintVal(tl); @@ -442,8 +450,12 @@ vcc_ErrWhere(tl, t_window); } Fb(tl, 0, "\t\t.window = %u,\n", window); - Fb(tl, 0, "\t\t.threshold = %u\n", threshold); + Fb(tl, 0, "\t\t.threshold = %u,\n", threshold); } + if (t_initial != NULL) + Fb(tl, 0, "\t\t.initial = %u,\n", initial); + else + Fb(tl, 0, "\t\t.initial = ~0U,\n", initial); Fb(tl, 0, "\t},\n"); ExpectErr(tl, '}'); vcc_NextToken(tl); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-17 11:26:19 UTC (rev 4184) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-17 11:53:01 UTC (rev 4185) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4099 2009-06-08 21:40:48Z phk $ + * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10:41:38Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,8 +159,8 @@ /* ../../include/vcl.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4099 2009-06-08 21"); - vsb_cat(sb, ":40:48Z phk $\n *\n * NB: This file is machine genera"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10"); + vsb_cat(sb, ":41:38Z phk $\n *\n * NB: This file is machine genera"); vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sess;\n"); vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); @@ -242,12 +242,12 @@ vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n"); vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n"); vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n"); - vsb_cat(sb, "\tunsigned\tthreshold;\n};\n\n/*\n"); - vsb_cat(sb, " * A backend is a host+port somewhere on the network\n"); - vsb_cat(sb, " */\nstruct vrt_backend {\n\tconst char\t\t\t*vcl_name"); - vsb_cat(sb, ";\n\tconst char\t\t\t*ident;\n\n"); - vsb_cat(sb, "\tconst char\t\t\t*hosthdr;\n\n"); - vsb_cat(sb, "\tconst unsigned char\t\t*ipv4_sockaddr;\n"); + vsb_cat(sb, "\tunsigned\tthreshold;\n\tunsigned\tinitial;\n"); + vsb_cat(sb, "};\n\n/*\n * A backend is a host+port somewhere on the"); + vsb_cat(sb, " network\n */\nstruct vrt_backend {\n"); + vsb_cat(sb, "\tconst char\t\t\t*vcl_name;\n\tconst char\t\t\t*ident"); + vsb_cat(sb, ";\n\n\tconst char\t\t\t*hosthdr;\n"); + vsb_cat(sb, "\n\tconst unsigned char\t\t*ipv4_sockaddr;\n"); vsb_cat(sb, "\tconst unsigned char\t\t*ipv6_sockaddr;\n"); vsb_cat(sb, "\n\tdouble\t\t\t\tconnect_timeout;\n"); vsb_cat(sb, "\tdouble\t\t\t\tfirst_byte_timeout;\n"); @@ -319,8 +319,8 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 4066 2009-05-10 21:21:36Z "); - vsb_cat(sb, "sky $\n *\n * NB: This file is machine generated, DO "); + vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 4099 2009-06-08 21:40:48Z "); + vsb_cat(sb, "phk $\n *\n * NB: This file is machine generated, DO "); vsb_cat(sb, "NOT EDIT!\n *\n * Edit vcc_gen_obj.tcl instead\n"); vsb_cat(sb, " */\n\nstruct sockaddr * VRT_r_client_ip(const struct "); vsb_cat(sb, "sess *);\nstruct sockaddr * VRT_r_server_ip(struct ses"); From phk at projects.linpro.no Mon Aug 17 12:27:11 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 17 Aug 2009 14:27:11 +0200 (CEST) Subject: r4186 - trunk/varnish-cache/lib/libvcl Message-ID: <20090817122711.B8D2D1F7B9A@projects.linpro.no> Author: phk Date: 2009-08-17 14:27:11 +0200 (Mon, 17 Aug 2009) New Revision: 4186 Modified: trunk/varnish-cache/lib/libvcl/vcc_parse.c Log: Make string comparisons against other than string literals possible. Modified: trunk/varnish-cache/lib/libvcl/vcc_parse.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_parse.c 2009-08-17 11:53:01 UTC (rev 4185) +++ trunk/varnish-cache/lib/libvcl/vcc_parse.c 2009-08-17 12:27:11 UTC (rev 4186) @@ -235,10 +235,11 @@ Fb(tl, 1, "%sVRT_strcmp(%s, ", tl->t->tok == T_EQ ? "!" : "", vp->rname); vcc_NextToken(tl); - ExpectErr(tl, CSTR); - EncToken(tl->fb, tl->t); + if (!vcc_StringVal(tl)) { + vcc_ExpectedStringval(tl); + break; + } Fb(tl, 0, ")\n"); - vcc_NextToken(tl); break; default: Fb(tl, 1, "%s != (void*)0\n", vp->rname); From phk at projects.linpro.no Mon Aug 17 12:44:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 17 Aug 2009 14:44:03 +0200 (CEST) Subject: r4187 - trunk/varnish-cache/bin/varnishtest/tests Message-ID: <20090817124403.9FEAD1F7B9A@projects.linpro.no> Author: phk Date: 2009-08-17 14:44:03 +0200 (Mon, 17 Aug 2009) New Revision: 4187 Added: trunk/varnish-cache/bin/varnishtest/tests/v00024.vtc Log: New test for string comparisons. Added: trunk/varnish-cache/bin/varnishtest/tests/v00024.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/v00024.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/v00024.vtc 2009-08-17 12:44:03 UTC (rev 4187) @@ -0,0 +1,27 @@ +# $Id$ + +test "Test that headers can be compared" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 -body "1" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.etag == req.http.if-none-match) { + error 400 "FOO"; + } + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + txreq -url "/foo" -hdr "etag: foo" -hdr "if-none-match: foo" + rxresp + expect resp.status == 400 +} -run From phk at projects.linpro.no Tue Aug 18 08:29:28 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 10:29:28 +0200 (CEST) Subject: r4188 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20090818082928.39BF01F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 10:29:27 +0200 (Tue, 18 Aug 2009) New Revision: 4188 Removed: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl Modified: trunk/varnish-cache/include/vcl.h trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/Makefile.am trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: I can never remember which of the two VCC generation scripts to run first, so combine them into one and avoid the issue entirely. Modified: trunk/varnish-cache/include/vcl.h =================================================================== --- trunk/varnish-cache/include/vcl.h 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/include/vcl.h 2009-08-18 08:29:27 UTC (rev 4188) @@ -27,18 +27,16 @@ #define VCL_MET_MAX 9 /* VCL Returns */ -#define VCL_RET_ERROR 0 -#define VCL_RET_LOOKUP 1 -#define VCL_RET_HASH 2 -#define VCL_RET_PIPE 3 -#define VCL_RET_PASS 4 -#define VCL_RET_FETCH 5 -#define VCL_RET_DELIVER 6 -#define VCL_RET_DISCARD 7 -#define VCL_RET_KEEP 8 -#define VCL_RET_RESTART 9 +#define VCL_RET_DELIVER 0 +#define VCL_RET_ERROR 1 +#define VCL_RET_FETCH 2 +#define VCL_RET_HASH 3 +#define VCL_RET_LOOKUP 4 +#define VCL_RET_PASS 5 +#define VCL_RET_PIPE 6 +#define VCL_RET_RESTART 7 -#define VCL_RET_MAX 10 +#define VCL_RET_MAX 8 struct VCL_conf { unsigned magic; Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/include/vcl_returns.h 2009-08-18 08:29:27 UTC (rev 4188) @@ -7,15 +7,13 @@ */ #ifdef VCL_RET_MAC +VCL_RET_MAC(deliver, DELIVER) VCL_RET_MAC(error, ERROR) -VCL_RET_MAC(lookup, LOOKUP) +VCL_RET_MAC(fetch, FETCH) VCL_RET_MAC(hash, HASH) -VCL_RET_MAC(pipe, PIPE) +VCL_RET_MAC(lookup, LOOKUP) VCL_RET_MAC(pass, PASS) -VCL_RET_MAC(fetch, FETCH) -VCL_RET_MAC(deliver, DELIVER) -VCL_RET_MAC(discard, DISCARD) -VCL_RET_MAC(keep, KEEP) +VCL_RET_MAC(pipe, PIPE) VCL_RET_MAC(restart, RESTART) #endif Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/include/vrt_obj.h 2009-08-18 08:29:27 UTC (rev 4188) @@ -3,7 +3,7 @@ * * NB: This file is machine generated, DO NOT EDIT! * - * Edit vcc_gen_obj.tcl instead + * Edit and run vcc_gen_fixed_token.tcl instead */ struct sockaddr * VRT_r_client_ip(const struct sess *); Modified: trunk/varnish-cache/lib/libvcl/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvcl/Makefile.am 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/lib/libvcl/Makefile.am 2009-08-18 08:29:27 UTC (rev 4188) @@ -25,11 +25,11 @@ vcc_var.c \ vcc_xref.c -EXTRA_DIST = vcc_gen_obj.tcl \ +EXTRA_DIST = \ vcc_gen_fixed_token.tcl -$(srcdir)/vcc_obj.c: $(srcdir)/vcc_gen_obj.tcl - cd $(srcdir) && @TCLSH@ vcc_gen_obj.tcl || true +$(srcdir)/vcc_obj.c: $(srcdir)/vcc_gen_fixed_token.tcl + cd $(srcdir) && @TCLSH@ vcc_gen_fixed_token.tcl || true $(srcdir)/vcc_fixed_token.c: $(srcdir)/vcc_gen_fixed_token.tcl $(top_srcdir)/include/vcl.h $(top_srcdir)/include/vrt.h $(top_srcdir)/include/vrt_obj.h cd $(srcdir) && @TCLSH@ vcc_gen_fixed_token.tcl || true Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-18 08:29:27 UTC (rev 4188) @@ -176,17 +176,15 @@ vsb_cat(sb, "#define VCL_MET_DELIVER\t\t(1 << 7)\n"); vsb_cat(sb, "#define VCL_MET_ERROR\t\t(1 << 8)\n"); vsb_cat(sb, "\n#define VCL_MET_MAX\t\t9\n\n/* VCL Returns */\n"); - vsb_cat(sb, "#define VCL_RET_ERROR\t\t0\n#define VCL_RET_LOOKUP\t\t"); - vsb_cat(sb, "1\n#define VCL_RET_HASH\t\t2\n#define VCL_RET_PIPE\t\t"); - vsb_cat(sb, "3\n#define VCL_RET_PASS\t\t4\n#define VCL_RET_FETCH\t\t"); - vsb_cat(sb, "5\n#define VCL_RET_DELIVER\t\t6\n"); - vsb_cat(sb, "#define VCL_RET_DISCARD\t\t7\n#define VCL_RET_KEEP\t\t"); - vsb_cat(sb, "8\n#define VCL_RET_RESTART\t\t9\n"); - vsb_cat(sb, "\n#define VCL_RET_MAX\t\t10\n\n"); - vsb_cat(sb, "struct VCL_conf {\n\tunsigned\tmagic;\n"); - vsb_cat(sb, "#define VCL_CONF_MAGIC\t0x7406c509\t/* from /dev/rando"); - vsb_cat(sb, "m */\n\n\tstruct director\t**director;\n"); - vsb_cat(sb, "\tunsigned\tndirector;\n\tstruct vrt_ref\t*ref;\n"); + vsb_cat(sb, "#define VCL_RET_DELIVER\t\t0\n#define VCL_RET_ERROR\t\t"); + vsb_cat(sb, "1\n#define VCL_RET_FETCH\t\t2\n"); + vsb_cat(sb, "#define VCL_RET_HASH\t\t3\n#define VCL_RET_LOOKUP\t\t4"); + vsb_cat(sb, "\n#define VCL_RET_PASS\t\t5\n#define VCL_RET_PIPE\t\t6"); + vsb_cat(sb, "\n#define VCL_RET_RESTART\t\t7\n"); + vsb_cat(sb, "\n#define VCL_RET_MAX\t\t8\n\nstruct VCL_conf {\n"); + vsb_cat(sb, "\tunsigned\tmagic;\n#define VCL_CONF_MAGIC\t0x7406c509"); + vsb_cat(sb, "\t/* from /dev/random */\n\n\tstruct director\t**direc"); + vsb_cat(sb, "tor;\n\tunsigned\tndirector;\n\tstruct vrt_ref\t*ref;\n"); vsb_cat(sb, "\tunsigned\tnref;\n\tunsigned\tbusy;\n"); vsb_cat(sb, "\tunsigned\tdiscard;\n\n\tunsigned\tnsrc;\n"); vsb_cat(sb, "\tconst char\t**srcname;\n\tconst char\t**srcbody;\n"); @@ -230,8 +228,8 @@ vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI"); vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT"); vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"); - vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4025 2009-04-03 21:"); - vsb_cat(sb, "52:44Z des $\n *\n * Runtime support for compiled VCL "); + vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4185 2009-08-17 11:"); + vsb_cat(sb, "53:01Z phk $\n *\n * Runtime support for compiled VCL "); vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/"); vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n"); vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n"); @@ -319,26 +317,26 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 4099 2009-06-08 21:40:48Z "); - vsb_cat(sb, "phk $\n *\n * NB: This file is machine generated, DO "); - vsb_cat(sb, "NOT EDIT!\n *\n * Edit vcc_gen_obj.tcl instead\n"); - vsb_cat(sb, " */\n\nstruct sockaddr * VRT_r_client_ip(const struct "); - vsb_cat(sb, "sess *);\nstruct sockaddr * VRT_r_server_ip(struct ses"); - vsb_cat(sb, "s *);\nconst char * VRT_r_server_hostname(struct sess "); - vsb_cat(sb, "*);\nconst char * VRT_r_server_identity(struct sess *)"); - vsb_cat(sb, ";\nint VRT_r_server_port(struct sess *);\n"); - vsb_cat(sb, "const char * VRT_r_req_request(const struct sess *);\n"); - vsb_cat(sb, "void VRT_l_req_request(const struct sess *, const char"); - vsb_cat(sb, " *, ...);\nconst char * VRT_r_req_url(const struct ses"); - vsb_cat(sb, "s *);\nvoid VRT_l_req_url(const struct sess *, const c"); - vsb_cat(sb, "har *, ...);\nconst char * VRT_r_req_proto(const struc"); - vsb_cat(sb, "t sess *);\nvoid VRT_l_req_proto(const struct sess *, "); - vsb_cat(sb, "const char *, ...);\nvoid VRT_l_req_hash(struct sess *"); - vsb_cat(sb, ", const char *);\nstruct director * VRT_r_req_backend("); - vsb_cat(sb, "struct sess *);\nvoid VRT_l_req_backend(struct sess *,"); - vsb_cat(sb, " struct director *);\nint VRT_r_req_restarts(const str"); - vsb_cat(sb, "uct sess *);\ndouble VRT_r_req_grace(struct sess *);\n"); - vsb_cat(sb, "void VRT_l_req_grace(struct sess *, double);\n"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10"); + vsb_cat(sb, ":41:38Z phk $\n *\n * NB: This file is machine genera"); + vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); + vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sockaddr * VRT_r_clien"); + vsb_cat(sb, "t_ip(const struct sess *);\nstruct sockaddr * VRT_r_se"); + vsb_cat(sb, "rver_ip(struct sess *);\nconst char * VRT_r_server_hos"); + vsb_cat(sb, "tname(struct sess *);\nconst char * VRT_r_server_ident"); + vsb_cat(sb, "ity(struct sess *);\nint VRT_r_server_port(struct sess"); + vsb_cat(sb, " *);\nconst char * VRT_r_req_request(const struct sess"); + vsb_cat(sb, " *);\nvoid VRT_l_req_request(const struct sess *, cons"); + vsb_cat(sb, "t char *, ...);\nconst char * VRT_r_req_url(const stru"); + vsb_cat(sb, "ct sess *);\nvoid VRT_l_req_url(const struct sess *, c"); + vsb_cat(sb, "onst char *, ...);\nconst char * VRT_r_req_proto(const"); + vsb_cat(sb, " struct sess *);\nvoid VRT_l_req_proto(const struct se"); + vsb_cat(sb, "ss *, const char *, ...);\nvoid VRT_l_req_hash(struct "); + vsb_cat(sb, "sess *, const char *);\nstruct director * VRT_r_req_ba"); + vsb_cat(sb, "ckend(struct sess *);\nvoid VRT_l_req_backend(struct s"); + vsb_cat(sb, "ess *, struct director *);\nint VRT_r_req_restarts(con"); + vsb_cat(sb, "st struct sess *);\ndouble VRT_r_req_grace(struct sess"); + vsb_cat(sb, " *);\nvoid VRT_l_req_grace(struct sess *, double);\n"); vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n"); vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n"); vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-18 08:29:27 UTC (rev 4188) @@ -45,21 +45,6 @@ {error {restart deliver}} } -# These are the return actions -# -set returns { - error - lookup - hash - pipe - pass - fetch - deliver - discard - keep - restart -} - # Language keywords # set keywords { @@ -97,6 +82,292 @@ set extras {ID VAR CNUM CSTR EOI CSRC} #---------------------------------------------------------------------- +# Variables available in sessions +# Comments are stripped from #...\n +set spobj { + + # Connection related parameters + { client.ip + RO IP + {recv pipe pass hash miss hit fetch deliver error } + "const struct sess *" + } + { client.bandwidth # Not implemented yet + NO + { } + "const struct sess *" + } + { server.ip + RO IP + {recv pipe pass hash miss hit fetch deliver error } + "struct sess *" + } + { server.hostname + RO STRING + {recv pipe pass hash miss hit fetch deliver error } + "struct sess *" + } + { server.identity + RO STRING + {recv pipe pass hash miss hit fetch deliver error } + "struct sess *" + } + { server.port + RO INT + {recv pipe pass hash miss hit fetch deliver error } + "struct sess *" + } + # Request paramters + { req.request + RW STRING + {recv pipe pass hash miss hit fetch deliver error } + "const struct sess *" + } + { req.url + RW STRING + {recv pipe pass hash miss hit fetch deliver error } + "const struct sess *" + } + { req.proto + RW STRING + {recv pipe pass hash miss hit fetch deliver error } + "const struct sess *" + } + { req.http. + RW HDR_REQ + {recv pipe pass hash miss hit fetch deliver error } + "const struct sess *" + } + + # Possibly misnamed, not really part of the request + { req.hash + WO HASH + { hash error } + "struct sess *" + } + { req.backend + RW BACKEND + {recv pipe pass hash miss hit fetch deliver error } + "struct sess *" + } + { req.restarts + RO INT + {recv pipe pass hash miss hit fetch deliver error } + "const struct sess *" + } + { req.grace + RW TIME + {recv pipe pass hash miss hit fetch deliver error } + "struct sess *" + } + + { req.xid + RO STRING + {recv pipe pass hash miss hit fetch deliver error} + "struct sess *" + } + + { req.esi + RW BOOL + {recv fetch deliver error} + "struct sess *" + } + + ####################################################################### + # Request sent to backend + { bereq.request + RW STRING + { pipe pass miss fetch } + "const struct sess *" + } + { bereq.url + RW STRING + { pipe pass miss fetch } + "const struct sess *" + } + { bereq.proto + RW STRING + { pipe pass miss fetch } + "const struct sess *" + } + { bereq.http. + RW HDR_BEREQ + { pipe pass miss fetch } + "const struct sess *" + } + { bereq.connect_timeout + RW TIME + { pass miss } + "struct sess *" + } + { bereq.first_byte_timeout + RW TIME + { pass miss } + "struct sess *" + } + { bereq.between_bytes_timeout + RW TIME + { pass miss } + "struct sess *" + } + + ####################################################################### + # Response from the backend + { beresp.proto + RW STRING + { fetch } + "const struct sess *" + } + { beresp.status + RW INT + { fetch } + "const struct sess *" + } + { beresp.response + RW STRING + { fetch } + "const struct sess *" + } + { beresp.http. + RW HDR_BERESP + { fetch } + "const struct sess *" + } + { beresp.cacheable + RW BOOL + { fetch } + "const struct sess *" + } + { beresp.ttl + RW TIME + { fetch } + "const struct sess *" + } + { beresp.grace + RW TIME + { fetch } + "const struct sess *" + } + + ####################################################################### + # The (possibly) cached object + { obj.proto + RW STRING + { hit error} + "const struct sess *" + } + { obj.status + RW INT + { error} + "const struct sess *" + } + { obj.response + RW STRING + { error} + "const struct sess *" + } + { obj.hits + RO INT + { hit deliver } + "const struct sess *" + } + { obj.http. + RW HDR_OBJ + { hit error} + "const struct sess *" + } + + { obj.cacheable + RW BOOL + { hit } + "const struct sess *" + } + { obj.ttl + RW TIME + { hit error} + "const struct sess *" + } + { obj.grace + RW TIME + { hit error} + "const struct sess *" + } + { obj.lastuse + RO TIME + { hit deliver error} + "const struct sess *" + } + { obj.hash + RO STRING + { miss hit deliver error} + "const struct sess *" + } + + ####################################################################### + # The response we send back + { resp.proto + RW STRING + { deliver } + "const struct sess *" + } + { resp.status + RW INT + { deliver } + "const struct sess *" + } + { resp.response + RW STRING + { deliver } + "const struct sess *" + } + { resp.http. + RW HDR_RESP + { deliver } + "const struct sess *" + } + + # Miscellaneous + # XXX: I'm not happy about this one. All times should be relative + # XXX: or delta times in VCL programs, so this shouldn't be needed /phk + { now + RO TIME + {recv pipe pass hash miss hit fetch deliver } + "const struct sess *" + } + { req.backend.healthy RO BOOL + {recv pipe pass hash miss hit fetch deliver } + "const struct sess *" + } + +} + +set tt(IP) "struct sockaddr *" +set tt(STRING) "const char *" +set tt(BOOL) "unsigned" +set tt(BACKEND) "struct director *" +set tt(TIME) "double" +set tt(RTIME) "double" +set tt(INT) "int" +set tt(HDR_RESP) "const char *" +set tt(HDR_OBJ) "const char *" +set tt(HDR_REQ) "const char *" +set tt(HDR_BEREQ) "const char *" +set tt(HOSTNAME) "const char *" +set tt(PORTNAME) "const char *" +set tt(HASH) "const char *" +set tt(SET) "struct vrt_backend_entry *" + +#---------------------------------------------------------------------- +# Figure out the union list of return actions +foreach i $methods { + foreach j [lindex $i 1] { + set tmp($j) 1 + } +} +set returns [lsort [array names tmp]] +unset tmp + +#---------------------------------------------------------------------- # Boilerplate warning for all generated files. proc warns {fd} { @@ -113,6 +384,148 @@ } #---------------------------------------------------------------------- +# Include a .h file as a string. + +proc copy_include {n} { + global fo + + puts $fo "\n\t/* $n */\n" + set fi [open $n] + set n 0 + while {[gets $fi a] >= 0} { + for {set b 0} {$b < [string length $a]} {incr b} { + if {$n == 0} { + puts -nonewline $fo "\tvsb_cat(sb, \"" + } + set c [string index $a $b] + if {"$c" == "\\"} { + puts -nonewline $fo "\\\\" + incr n + } elseif {"$c" == "\t"} { + puts -nonewline $fo "\\t" + incr n + } else { + puts -nonewline $fo "$c" + } + incr n + if {$n > 53} { + puts $fo "\");" + set n 0 + } + } + if {$n == 0} { + puts -nonewline $fo "\tvsb_cat(sb, \"" + } + puts -nonewline $fo "\\n" + incr n 2 + if {$n > 30} { + puts $fo "\");" + set n 0 + } + } + if {$n > 0} { + puts $fo "\");" + } + close $fi +} + +#---------------------------------------------------------------------- +# Build the variable related .c and .h files + +set fo [open vcc_obj.c w] +warns $fo +set fp [open ../../include/vrt_obj.h w] +warns $fp + +proc method_map {m} { + + set l1 "" + set l2 "" + foreach i $m { + if {[string length $l2] > 55} { + if {$l1 != ""} { + append l1 "\n\t " + } + append l1 "$l2" + set l2 "" + } + if {$l2 != "" || $l1 != ""} { + append l2 " | " + } + append l2 VCL_MET_[string toupper $i] + } + if {$l2 != ""} { + if {$l1 != ""} { + append l1 "\n\t " + } + append l1 "$l2" + } + if {$l1 == ""} { + return "0" + } + return $l1 +} + +proc vars {v pa} { + global tt fo fp + + regsub -all "#\[^\n\]*\n" $v "" v + foreach v $v { + set n [lindex $v 0] + regsub -all {[.]} $n "_" m + set a [lindex $v 1] + if {$a == "NO"} continue + set t [lindex $v 2] + set ty [lindex $v 4] + if {[regexp HDR_ $t]} { + puts $fo "\t\{ \"$n\", HEADER, [string length $n]," + } else { + puts $fo "\t\{ \"$n\", $t, [string length $n]," + } + if {$a == "RO" || $a == "RW"} { + puts -nonewline $fo "\t \"VRT_r_${m}($pa)\"," + if {![regexp HDR_ $t]} { + puts $fp "$tt($t) VRT_r_${m}($ty);" + } + } else { + puts -nonewline $fo "\t NULL," + } + if {$a == "WO" || $a == "RW"} { + puts $fo "\t \"VRT_l_${m}($pa, \"," + if {[regexp HDR_ $t]} { + } elseif {$t == "STRING"} { + puts $fp "void VRT_l_${m}($ty, $tt($t), ...);" + } else { + puts $fp "void VRT_l_${m}($ty, $tt($t));" + } + } else { + puts $fo "\t NULL," + } + puts -nonewline $fo "\t V_$a," + if {![regexp HDR_ $t]} { + puts $fo "\t 0," + } else { + puts $fo "\t \"$t\"," + } + puts $fo "\t [method_map [lindex $v 3]]" + puts $fo "\t\}," + + } + puts $fo "\t{ NULL }" +} + +puts $fo "#include \"config.h\"" +puts $fo "#include <stdio.h>" +puts $fo "#include \"vcc_compile.h\"" +puts $fo "" + +puts $fo "struct var vcc_vars\[\] = {" +vars $spobj "sp" +puts $fo "};" +close $fp + + +#---------------------------------------------------------------------- # Build the vcl.h #include file set fo [open ../../include/vcl.h w] @@ -290,8 +703,8 @@ puts $fo " case '$ch':" set retval "0" set m1 0 - foreach tt $l { - set k [lindex $tt 0] + foreach ty $l { + set k [lindex $ty 0] if {[string length $k] == 1} { puts $fo "\t\tM1();" set m1 1 @@ -300,7 +713,7 @@ if {[string length $k] == 2} { puts -nonewline $fo " M2(" puts -nonewline $fo "'[string index $k 1]'" - puts $fo ", [lindex $tt 1]);" + puts $fo ", [lindex $ty 1]);" continue; } puts -nonewline $fo " if (" @@ -314,7 +727,7 @@ } puts -nonewline $fo "p\[$i\] == '[string index $k $i]'" } - if {[lindex $tt 2]} { + if {[lindex $ty 2]} { if {[expr $i % 3] == 1} { puts -nonewline $fo "\n\t\t " } @@ -322,7 +735,7 @@ } puts $fo ") {" puts $fo "\t\t\t*q = p + [string length $k];" - puts $fo "\t\t\treturn ([lindex $tt 1]);" + puts $fo "\t\t\treturn ([lindex $ty 1]);" puts $fo "\t\t}" } if {$m1 == 0} { @@ -345,53 +758,6 @@ } puts $fo "};" -#---------------------------------------------------------------------- -# Create the C-code which emits the boilerplate definitions for the -# generated C code output - -proc copy_include {n} { - global fo - - puts $fo "\n\t/* $n */\n" - set fi [open $n] - set n 0 - while {[gets $fi a] >= 0} { - for {set b 0} {$b < [string length $a]} {incr b} { - if {$n == 0} { - puts -nonewline $fo "\tvsb_cat(sb, \"" - } - set c [string index $a $b] - if {"$c" == "\\"} { - puts -nonewline $fo "\\\\" - incr n - } elseif {"$c" == "\t"} { - puts -nonewline $fo "\\t" - incr n - } else { - puts -nonewline $fo "$c" - } - incr n - if {$n > 53} { - puts $fo "\");" - set n 0 - } - } - if {$n == 0} { - puts -nonewline $fo "\tvsb_cat(sb, \"" - } - puts -nonewline $fo "\\n" - incr n 2 - if {$n > 30} { - puts $fo "\");" - set n 0 - } - } - if {$n > 0} { - puts $fo "\");" - } - close $fi -} - puts $fo "" puts $fo "void" puts $fo "vcl_output_lang_h(struct vsb *sb)" @@ -405,3 +771,4 @@ close $foh close $fo + Deleted: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2009-08-18 08:29:27 UTC (rev 4188) @@ -1,415 +0,0 @@ -#!/usr/local/bin/tclsh8.4 -#- -# Copyright (c) 2006 Verdens Gang AS -# Copyright (c) 2006-2009 Linpro AS -# All rights reserved. -# -# Author: Poul-Henning Kamp <phk at phk.freebsd.dk> -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. -# -# Generate various .c and .h files for the VCL compiler and the interfaces -# for it. - - -# Variables available in sessions -# Comments are stripped from #...\n -set spobj { - - # Connection related parameters - { client.ip - RO IP - {recv pipe pass hash miss hit fetch deliver error } - "const struct sess *" - } - { client.bandwidth # Not implemented yet - NO - { } - "const struct sess *" - } - { server.ip - RO IP - {recv pipe pass hash miss hit fetch deliver error } - "struct sess *" - } - { server.hostname - RO STRING - {recv pipe pass hash miss hit fetch deliver error } - "struct sess *" - } - { server.identity - RO STRING - {recv pipe pass hash miss hit fetch deliver error } - "struct sess *" - } - { server.port - RO INT - {recv pipe pass hash miss hit fetch deliver error } - "struct sess *" - } - # Request paramters - { req.request - RW STRING - {recv pipe pass hash miss hit fetch deliver error } - "const struct sess *" - } - { req.url - RW STRING - {recv pipe pass hash miss hit fetch deliver error } - "const struct sess *" - } - { req.proto - RW STRING - {recv pipe pass hash miss hit fetch deliver error } - "const struct sess *" - } - { req.http. - RW HDR_REQ - {recv pipe pass hash miss hit fetch deliver error } - "const struct sess *" - } - - # Possibly misnamed, not really part of the request - { req.hash - WO HASH - { hash error } - "struct sess *" - } - { req.backend - RW BACKEND - {recv pipe pass hash miss hit fetch deliver error } - "struct sess *" - } - { req.restarts - RO INT - {recv pipe pass hash miss hit fetch deliver error } - "const struct sess *" - } - { req.grace - RW TIME - {recv pipe pass hash miss hit fetch deliver error } - "struct sess *" - } - - { req.xid - RO STRING - {recv pipe pass hash miss hit fetch deliver error} - "struct sess *" - } - - { req.esi - RW BOOL - {recv fetch deliver error} - "struct sess *" - } - - ####################################################################### - # Request sent to backend - { bereq.request - RW STRING - { pipe pass miss fetch } - "const struct sess *" - } - { bereq.url - RW STRING - { pipe pass miss fetch } - "const struct sess *" - } - { bereq.proto - RW STRING - { pipe pass miss fetch } - "const struct sess *" - } - { bereq.http. - RW HDR_BEREQ - { pipe pass miss fetch } - "const struct sess *" - } - { bereq.connect_timeout - RW TIME - { pass miss } - "struct sess *" - } - { bereq.first_byte_timeout - RW TIME - { pass miss } - "struct sess *" - } - { bereq.between_bytes_timeout - RW TIME - { pass miss } - "struct sess *" - } - - ####################################################################### - # Response from the backend - { beresp.proto - RW STRING - { fetch } - "const struct sess *" - } - { beresp.status - RW INT - { fetch } - "const struct sess *" - } - { beresp.response - RW STRING - { fetch } - "const struct sess *" - } - { beresp.http. - RW HDR_BERESP - { fetch } - "const struct sess *" - } - { beresp.cacheable - RW BOOL - { fetch } - "const struct sess *" - } - { beresp.ttl - RW TIME - { fetch } - "const struct sess *" - } - { beresp.grace - RW TIME - { fetch } - "const struct sess *" - } - - ####################################################################### - # The (possibly) cached object - { obj.proto - RW STRING - { hit error} - "const struct sess *" - } - { obj.status - RW INT - { error} - "const struct sess *" - } - { obj.response - RW STRING - { error} - "const struct sess *" - } - { obj.hits - RO INT - { hit deliver } - "const struct sess *" - } - { obj.http. - RW HDR_OBJ - { hit error} - "const struct sess *" - } - - { obj.cacheable - RW BOOL - { hit } - "const struct sess *" - } - { obj.ttl - RW TIME - { hit error} - "const struct sess *" - } - { obj.grace - RW TIME - { hit error} - "const struct sess *" - } - { obj.lastuse - RO TIME - { hit deliver error} - "const struct sess *" - } - { obj.hash - RO STRING - { miss hit deliver error} - "const struct sess *" - } - - ####################################################################### - # The response we send back - { resp.proto - RW STRING - { deliver } - "const struct sess *" - } - { resp.status - RW INT - { deliver } - "const struct sess *" - } - { resp.response - RW STRING - { deliver } - "const struct sess *" - } - { resp.http. - RW HDR_RESP - { deliver } - "const struct sess *" - } - - # Miscellaneous - # XXX: I'm not happy about this one. All times should be relative - # XXX: or delta times in VCL programs, so this shouldn't be needed /phk - { now - RO TIME - {recv pipe pass hash miss hit fetch deliver } - "const struct sess *" - } - { req.backend.healthy RO BOOL - {recv pipe pass hash miss hit fetch deliver } - "const struct sess *" - } - -} - -set tt(IP) "struct sockaddr *" -set tt(STRING) "const char *" -set tt(BOOL) "unsigned" -set tt(BACKEND) "struct director *" -set tt(TIME) "double" -set tt(RTIME) "double" -set tt(INT) "int" -set tt(HDR_RESP) "const char *" -set tt(HDR_OBJ) "const char *" -set tt(HDR_REQ) "const char *" -set tt(HDR_BEREQ) "const char *" -set tt(HOSTNAME) "const char *" -set tt(PORTNAME) "const char *" -set tt(HASH) "const char *" -set tt(SET) "struct vrt_backend_entry *" - -#---------------------------------------------------------------------- -# Boilerplate warning for all generated files. - -proc warns {fd} { - - puts $fd "/*" - puts $fd { * $Id$} - puts $fd " *" - puts $fd " * NB: This file is machine generated, DO NOT EDIT!" - puts $fd " *" - puts $fd " * Edit vcc_gen_obj.tcl instead" - puts $fd " */" - puts $fd "" -} - -set fo [open vcc_obj.c w] -warns $fo -set fp [open ../../include/vrt_obj.h w] -warns $fp - -proc method_map {m} { - - set l1 "" - set l2 "" - foreach i $m { - if {[string length $l2] > 55} { - if {$l1 != ""} { - append l1 "\n\t " - } - append l1 "$l2" - set l2 "" - } - if {$l2 != "" || $l1 != ""} { - append l2 " | " - } - append l2 VCL_MET_[string toupper $i] - } - if {$l2 != ""} { - if {$l1 != ""} { - append l1 "\n\t " - } - append l1 "$l2" - } - if {$l1 == ""} { - return "0" - } - return $l1 -} - -proc vars {v pa} { - global tt fo fp - - regsub -all "#\[^\n\]*\n" $v "" v - foreach v $v { - set n [lindex $v 0] - regsub -all {[.]} $n "_" m - set a [lindex $v 1] - if {$a == "NO"} continue - set t [lindex $v 2] - set ty [lindex $v 4] - if {[regexp HDR_ $t]} { - puts $fo "\t\{ \"$n\", HEADER, [string length $n]," - } else { - puts $fo "\t\{ \"$n\", $t, [string length $n]," - } - if {$a == "RO" || $a == "RW"} { - puts -nonewline $fo "\t \"VRT_r_${m}($pa)\"," - if {![regexp HDR_ $t]} { - puts $fp "$tt($t) VRT_r_${m}($ty);" - } - } else { - puts -nonewline $fo "\t NULL," - } - if {$a == "WO" || $a == "RW"} { - puts $fo "\t \"VRT_l_${m}($pa, \"," - if {[regexp HDR_ $t]} { - } elseif {$t == "STRING"} { - puts $fp "void VRT_l_${m}($ty, $tt($t), ...);" - } else { - puts $fp "void VRT_l_${m}($ty, $tt($t));" - } - } else { - puts $fo "\t NULL," - } - puts -nonewline $fo "\t V_$a," - if {![regexp HDR_ $t]} { - puts $fo "\t 0," - } else { - puts $fo "\t \"$t\"," - } - puts $fo "\t [method_map [lindex $v 3]]" - puts $fo "\t\}," - - } - puts $fo "\t{ NULL }" -} - -puts $fo "#include \"config.h\"" -puts $fo "#include <stdio.h>" -puts $fo "#include \"vcc_compile.h\"" -puts $fo "" - -puts $fo "struct var vcc_vars\[\] = {" -vars $spobj "sp" -puts $fo "};" - -close $fp Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-17 12:44:03 UTC (rev 4187) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-18 08:29:27 UTC (rev 4188) @@ -1,9 +1,9 @@ /* - * $Id: vcc_gen_obj.tcl 4099 2009-06-08 21:40:48Z phk $ + * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10:41:38Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * - * Edit vcc_gen_obj.tcl instead + * Edit and run vcc_gen_fixed_token.tcl instead */ #include "config.h" From phk at projects.linpro.no Tue Aug 18 08:45:41 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 10:45:41 +0200 (CEST) Subject: r4189 - trunk/varnish-cache/lib/libvcl Message-ID: <20090818084541.B8D2B1F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 10:45:41 +0200 (Tue, 18 Aug 2009) New Revision: 4189 Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl Log: Remove last traces of client.bandwidth object Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-18 08:29:27 UTC (rev 4188) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-18 08:45:41 UTC (rev 4189) @@ -92,11 +92,6 @@ {recv pipe pass hash miss hit fetch deliver error } "const struct sess *" } - { client.bandwidth # Not implemented yet - NO - { } - "const struct sess *" - } { server.ip RO IP {recv pipe pass hash miss hit fetch deliver error } From phk at projects.linpro.no Tue Aug 18 09:17:22 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 11:17:22 +0200 (CEST) Subject: r4190 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20090818091722.4771E1F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 11:17:21 +0200 (Tue, 18 Aug 2009) New Revision: 4190 Modified: trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Clean up the vcc generator script a little bit Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2009-08-18 08:45:41 UTC (rev 4189) +++ trunk/varnish-cache/include/vrt_obj.h 2009-08-18 09:17:21 UTC (rev 4190) @@ -26,6 +26,7 @@ const char * VRT_r_req_xid(struct sess *); unsigned VRT_r_req_esi(struct sess *); void VRT_l_req_esi(struct sess *, unsigned); +unsigned VRT_r_req_backend_healthy(const struct sess *); const char * VRT_r_bereq_request(const struct sess *); void VRT_l_bereq_request(const struct sess *, const char *, ...); const char * VRT_r_bereq_url(const struct sess *); @@ -72,4 +73,3 @@ const char * VRT_r_resp_response(const struct sess *); void VRT_l_resp_response(const struct sess *, const char *, ...); double VRT_r_now(const struct sess *); -unsigned VRT_r_req_backend_healthy(const struct sess *); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-18 08:45:41 UTC (rev 4189) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-18 09:17:21 UTC (rev 4190) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10:41:38Z phk $ + * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08:29:27Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,8 +159,8 @@ /* ../../include/vcl.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10"); - vsb_cat(sb, ":41:38Z phk $\n *\n * NB: This file is machine genera"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08"); + vsb_cat(sb, ":29:27Z phk $\n *\n * NB: This file is machine genera"); vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sess;\n"); vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); @@ -317,8 +317,8 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10"); - vsb_cat(sb, ":41:38Z phk $\n *\n * NB: This file is machine genera"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08"); + vsb_cat(sb, ":29:27Z phk $\n *\n * NB: This file is machine genera"); vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sockaddr * VRT_r_clien"); vsb_cat(sb, "t_ip(const struct sess *);\nstruct sockaddr * VRT_r_se"); @@ -340,13 +340,14 @@ vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n"); vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n"); vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n"); - vsb_cat(sb, "const char * VRT_r_bereq_request(const struct sess *);"); - vsb_cat(sb, "\nvoid VRT_l_bereq_request(const struct sess *, const "); - vsb_cat(sb, "char *, ...);\nconst char * VRT_r_bereq_url(const stru"); - vsb_cat(sb, "ct sess *);\nvoid VRT_l_bereq_url(const struct sess *,"); - vsb_cat(sb, " const char *, ...);\nconst char * VRT_r_bereq_proto(c"); - vsb_cat(sb, "onst struct sess *);\nvoid VRT_l_bereq_proto(const str"); - vsb_cat(sb, "uct sess *, const char *, ...);\n"); + vsb_cat(sb, "unsigned VRT_r_req_backend_healthy(const struct sess *"); + vsb_cat(sb, ");\nconst char * VRT_r_bereq_request(const struct sess"); + vsb_cat(sb, " *);\nvoid VRT_l_bereq_request(const struct sess *, co"); + vsb_cat(sb, "nst char *, ...);\nconst char * VRT_r_bereq_url(const "); + vsb_cat(sb, "struct sess *);\nvoid VRT_l_bereq_url(const struct ses"); + vsb_cat(sb, "s *, const char *, ...);\nconst char * VRT_r_bereq_pro"); + vsb_cat(sb, "to(const struct sess *);\nvoid VRT_l_bereq_proto(const"); + vsb_cat(sb, " struct sess *, const char *, ...);\n"); vsb_cat(sb, "double VRT_r_bereq_connect_timeout(struct sess *);\n"); vsb_cat(sb, "void VRT_l_bereq_connect_timeout(struct sess *, double"); vsb_cat(sb, ");\ndouble VRT_r_bereq_first_byte_timeout(struct sess "); @@ -388,6 +389,4 @@ vsb_cat(sb, "const char * VRT_r_resp_response(const struct sess *);"); vsb_cat(sb, "\nvoid VRT_l_resp_response(const struct sess *, const "); vsb_cat(sb, "char *, ...);\ndouble VRT_r_now(const struct sess *);\n"); - vsb_cat(sb, "unsigned VRT_r_req_backend_healthy(const struct sess *"); - vsb_cat(sb, ");\n"); } Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-18 08:45:41 UTC (rev 4189) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-18 09:17:21 UTC (rev 4190) @@ -87,253 +87,254 @@ set spobj { # Connection related parameters - { client.ip - RO IP - {recv pipe pass hash miss hit fetch deliver error } + { client.ip IP + RO + all "const struct sess *" } - { server.ip - RO IP - {recv pipe pass hash miss hit fetch deliver error } + { server.ip IP + RO + all "struct sess *" } - { server.hostname - RO STRING - {recv pipe pass hash miss hit fetch deliver error } + { server.hostname STRING + RO + all "struct sess *" } - { server.identity - RO STRING - {recv pipe pass hash miss hit fetch deliver error } + { server.identity STRING + RO + all "struct sess *" } - { server.port - RO INT - {recv pipe pass hash miss hit fetch deliver error } + { server.port INT + RO + all "struct sess *" } # Request paramters - { req.request - RW STRING - {recv pipe pass hash miss hit fetch deliver error } + { req.request STRING + RW + all "const struct sess *" } - { req.url - RW STRING - {recv pipe pass hash miss hit fetch deliver error } + { req.url STRING + RW + all "const struct sess *" } - { req.proto - RW STRING - {recv pipe pass hash miss hit fetch deliver error } + { req.proto STRING + RW + all "const struct sess *" } - { req.http. - RW HDR_REQ - {recv pipe pass hash miss hit fetch deliver error } + { req.http. HDR_REQ + RW + all "const struct sess *" } # Possibly misnamed, not really part of the request - { req.hash - WO HASH - { hash error } + { req.hash HASH + WO + { hash error } "struct sess *" } - { req.backend - RW BACKEND - {recv pipe pass hash miss hit fetch deliver error } + { req.backend BACKEND + RW + all "struct sess *" } - { req.restarts - RO INT - {recv pipe pass hash miss hit fetch deliver error } + { req.restarts INT + RO + all "const struct sess *" } - { req.grace - RW TIME - {recv pipe pass hash miss hit fetch deliver error } + { req.grace TIME + RW + all "struct sess *" } - { req.xid - RO STRING - {recv pipe pass hash miss hit fetch deliver error} + { req.xid STRING + RO + all "struct sess *" } - { req.esi - RW BOOL - {recv fetch deliver error} + { req.esi BOOL + RW + {recv fetch deliver error } "struct sess *" } + { req.backend.healthy BOOL + RO + all + "const struct sess *" + } + ####################################################################### # Request sent to backend - { bereq.request - RW STRING - { pipe pass miss fetch } + { bereq.request STRING + RW + { pipe pass miss fetch } "const struct sess *" } - { bereq.url - RW STRING - { pipe pass miss fetch } + { bereq.url STRING + RW + { pipe pass miss fetch } "const struct sess *" } - { bereq.proto - RW STRING - { pipe pass miss fetch } + { bereq.proto STRING + RW + { pipe pass miss fetch } "const struct sess *" } - { bereq.http. - RW HDR_BEREQ - { pipe pass miss fetch } + { bereq.http. HDR_BEREQ + RW + { pipe pass miss fetch } "const struct sess *" } - { bereq.connect_timeout - RW TIME - { pass miss } + { bereq.connect_timeout TIME + RW + { pass miss } "struct sess *" } - { bereq.first_byte_timeout - RW TIME - { pass miss } + { bereq.first_byte_timeout TIME + RW + { pass miss } "struct sess *" } - { bereq.between_bytes_timeout - RW TIME - { pass miss } + { bereq.between_bytes_timeout TIME + RW + { pass miss } "struct sess *" } ####################################################################### # Response from the backend - { beresp.proto - RW STRING - { fetch } + { beresp.proto STRING + RW + { fetch } "const struct sess *" } - { beresp.status - RW INT - { fetch } + { beresp.status INT + RW + { fetch } "const struct sess *" } - { beresp.response - RW STRING - { fetch } + { beresp.response STRING + RW + { fetch } "const struct sess *" } - { beresp.http. - RW HDR_BERESP - { fetch } + { beresp.http. HDR_BERESP + RW + { fetch } "const struct sess *" } - { beresp.cacheable - RW BOOL - { fetch } + { beresp.cacheable BOOL + RW + { fetch } "const struct sess *" } - { beresp.ttl - RW TIME - { fetch } + { beresp.ttl TIME + RW + { fetch } "const struct sess *" } - { beresp.grace - RW TIME - { fetch } + { beresp.grace TIME + RW + { fetch } "const struct sess *" } ####################################################################### # The (possibly) cached object - { obj.proto - RW STRING - { hit error} + { obj.proto STRING + RW + { hit error } "const struct sess *" } - { obj.status - RW INT - { error} + { obj.status INT + RW + { error } "const struct sess *" } - { obj.response - RW STRING - { error} + { obj.response STRING + RW + { error } "const struct sess *" } - { obj.hits - RO INT - { hit deliver } + { obj.hits INT + RO + { hit deliver } "const struct sess *" } - { obj.http. - RW HDR_OBJ - { hit error} + { obj.http. HDR_OBJ + RW + { hit error } "const struct sess *" } - { obj.cacheable - RW BOOL - { hit } + { obj.cacheable BOOL + RW + { hit } "const struct sess *" } - { obj.ttl - RW TIME - { hit error} + { obj.ttl TIME + RW + { hit error } "const struct sess *" } - { obj.grace - RW TIME - { hit error} + { obj.grace TIME + RW + { hit error } "const struct sess *" } - { obj.lastuse - RO TIME - { hit deliver error} + { obj.lastuse TIME + RO + { hit deliver error } "const struct sess *" } - { obj.hash - RO STRING - { miss hit deliver error} + { obj.hash STRING + RO + { miss hit deliver error } "const struct sess *" } ####################################################################### # The response we send back - { resp.proto - RW STRING - { deliver } + { resp.proto STRING + RW + { deliver } "const struct sess *" } - { resp.status - RW INT - { deliver } + { resp.status INT + RW + { deliver } "const struct sess *" } - { resp.response - RW STRING - { deliver } + { resp.response STRING + RW + { deliver } "const struct sess *" } - { resp.http. - RW HDR_RESP - { deliver } + { resp.http. HDR_RESP + RW + { deliver } "const struct sess *" } # Miscellaneous # XXX: I'm not happy about this one. All times should be relative # XXX: or delta times in VCL programs, so this shouldn't be needed /phk - { now - RO TIME - {recv pipe pass hash miss hit fetch deliver } - "const struct sess *" + { now TIME + RO + all + "const struct sess *" } - { req.backend.healthy RO BOOL - {recv pipe pass hash miss hit fetch deliver } - "const struct sess *" - } - } set tt(IP) "struct sockaddr *" @@ -433,7 +434,14 @@ warns $fp proc method_map {m} { + global methods + if {$m == "all"} { + set m "" + foreach i $methods { + lappend m [lindex $i 0] + } + } set l1 "" set l2 "" foreach i $m { @@ -468,9 +476,9 @@ foreach v $v { set n [lindex $v 0] regsub -all {[.]} $n "_" m - set a [lindex $v 1] + set t [lindex $v 1] + set a [lindex $v 2] if {$a == "NO"} continue - set t [lindex $v 2] set ty [lindex $v 4] if {[regexp HDR_ $t]} { puts $fo "\t\{ \"$n\", HEADER, [string length $n]," Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-18 08:45:41 UTC (rev 4189) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-18 09:17:21 UTC (rev 4190) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4100 2009-06-09 10:41:38Z phk $ + * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08:29:27Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -112,6 +112,13 @@ V_RW, 0, VCL_MET_RECV | VCL_MET_FETCH | VCL_MET_DELIVER | VCL_MET_ERROR }, + { "req.backend.healthy", BOOL, 19, + "VRT_r_req_backend_healthy(sp)", NULL, + V_RO, 0, + VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH + | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR + }, { "bereq.request", STRING, 13, "VRT_r_bereq_request(sp)", "VRT_l_bereq_request(sp, ", V_RW, 0, @@ -257,12 +264,7 @@ V_RO, 0, VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER + | VCL_MET_ERROR }, - { "req.backend.healthy", BOOL, 19, - "VRT_r_req_backend_healthy(sp)", NULL, - V_RO, 0, - VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH - | VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER - }, { NULL } }; From phk at projects.linpro.no Tue Aug 18 10:08:02 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 12:08:02 +0200 (CEST) Subject: r4191 - in trunk/varnish-cache: bin/varnishd lib/libvarnishapi Message-ID: <20090818100802.88C491F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 12:08:02 +0200 (Tue, 18 Aug 2009) New Revision: 4191 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Make the -d option to varnishlog et all exit when the log is emptied. Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 09:17:21 UTC (rev 4190) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 10:08:02 UTC (rev 4191) @@ -264,31 +264,36 @@ continue; } - o = oc->obj; - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(o->objhead, OBJHEAD_MAGIC); - assert(oc->flags & OC_F_ONLRU); + /* It's time... */ + + /* Remove from binheap */ assert(oc->timer_idx != BINHEAP_NOIDX); binheap_delete(exp_heap, oc->timer_idx); assert(oc->timer_idx == BINHEAP_NOIDX); - lru = STV_lru(o->objstore); - AN(lru); - VTAILQ_REMOVE(lru, o->objcore, lru_list); - oc->flags &= ~OC_F_ONLRU; - { /* Sanity checking */ - struct objcore *oc2 = binheap_root(exp_heap); - if (oc2 != NULL) { - assert(oc2->timer_idx != BINHEAP_NOIDX); - assert(oc2->timer_when >= oc->timer_when); - } + /* And from LRU */ + if (oc->flags & OC_F_ONLRU) { + assert(!(oc->flags & OC_F_PERSISTENT)); + lru = STV_lru(o->objstore); + AN(lru); + VTAILQ_REMOVE(lru, o->objcore, lru_list); + oc->flags &= ~OC_F_ONLRU; + } else { + assert(oc->flags & OC_F_PERSISTENT); } VSL_stats->n_expired++; + Lck_Unlock(&exp_mtx); + + if (!(oc->flags & OC_F_PERSISTENT)) { + o = oc->obj; + CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + HSH_Deref(sp->wrk, &o); + CHECK_OBJ_NOTNULL(o->objhead, OBJHEAD_MAGIC); + } WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", o->xid, (int)(o->ttl - t)); - HSH_Deref(sp->wrk, &o); } } Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2009-08-18 09:17:21 UTC (rev 4190) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2009-08-18 10:08:02 UTC (rev 4191) @@ -540,7 +540,10 @@ switch (arg) { case 'b': vd->b_opt = !vd->b_opt; return (1); case 'c': vd->c_opt = !vd->c_opt; return (1); - case 'd': vd->d_opt = !vd->d_opt; return (1); + case 'd': + vd->d_opt = !vd->d_opt; + vd->flags |= F_NON_BLOCKING; + return (1); case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); case 'r': return (vsl_r_arg(vd, opt)); case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); From phk at projects.linpro.no Tue Aug 18 10:12:32 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 12:12:32 +0200 (CEST) Subject: r4192 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818101232.476E41F7C98@projects.linpro.no> Author: phk Date: 2009-08-18 12:12:32 +0200 (Tue, 18 Aug 2009) New Revision: 4192 Modified: trunk/varnish-cache/bin/varnishd/cache_cli.c Log: Log that the worker stops in the shmlog Modified: trunk/varnish-cache/bin/varnishd/cache_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_cli.c 2009-08-18 10:08:02 UTC (rev 4191) +++ trunk/varnish-cache/bin/varnishd/cache_cli.c 2009-08-18 10:12:32 UTC (rev 4192) @@ -159,8 +159,7 @@ continue; assert(i == 1); if (pfd[0].revents & POLLHUP) { - fprintf(stderr, - "EOF on CLI connection, exiting\n"); + VSL(SLT_CLI, 0, "EOF on CLI connection, worker stops"); break; } i = VLU_Fd(heritage.cli_in, vlu); From phk at projects.linpro.no Tue Aug 18 10:35:19 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 12:35:19 +0200 (CEST) Subject: r4193 - trunk/varnish-cache/bin/varnishtest Message-ID: <20090818103519.1416E1F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 12:35:18 +0200 (Tue, 18 Aug 2009) New Revision: 4193 Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c Log: Fail gracefully on Rx error Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc_http.c 2009-08-18 10:12:32 UTC (rev 4192) +++ trunk/varnish-cache/bin/varnishtest/vtc_http.c 2009-08-18 10:35:18 UTC (rev 4193) @@ -326,7 +326,9 @@ i = read(hp->fd, hp->rxbuf + hp->prxbuf, n); if (i == 0) return (i); - assert(i > 0); + if (i <= 0) + vtc_log(hp->vl, 0, "HTTP rx failed (%s)", + strerror(errno)); hp->prxbuf += i; hp->rxbuf[hp->prxbuf] = '\0'; n -= i; From phk at projects.linpro.no Tue Aug 18 11:14:35 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 13:14:35 +0200 (CEST) Subject: r4194 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818111435.884E41F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 13:14:35 +0200 (Tue, 18 Aug 2009) New Revision: 4194 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Move the objhead reference from the object to the objcore, we need it for expiry of non-instantiated objects from -spersistent Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-18 11:14:35 UTC (rev 4194) @@ -281,6 +281,7 @@ unsigned magic; #define OBJCORE_MAGIC 0x4d301302 struct object *obj; + struct objhead *objhead; double timer_when; unsigned char flags; #define OC_F_ONLRU (1<<0) @@ -301,7 +302,6 @@ #define OBJECT_MAGIC 0x32851d42 unsigned refcnt; unsigned xid; - struct objhead *objhead; struct storage *objstore; struct objcore *objcore; Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-18 11:14:35 UTC (rev 4194) @@ -157,9 +157,10 @@ { (void)sp; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(o->objhead, OBJHEAD_MAGIC); - AN(o->objhead->hash); - return(ban_cond_str(bt, o->objhead->hash)); + CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(o->objcore->objhead, OBJHEAD_MAGIC); + AN(o->objcore->objhead->hash); + return(ban_cond_str(bt, o->objcore->objhead->hash)); } static int Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-18 11:14:35 UTC (rev 4194) @@ -175,7 +175,7 @@ CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); sp->t_resp = TIM_real(); - if (sp->obj->objhead != NULL) { + if (sp->obj->objcore != NULL) { if ((sp->t_resp - sp->obj->last_lru) > params->lru_timeout && EXP_Touch(sp->obj)) sp->obj->last_lru = sp->t_resp; /* XXX: locking ? */ @@ -505,7 +505,7 @@ CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); sp->objcore->obj = sp->obj; sp->obj->objcore = sp->objcore; - sp->obj->objhead = sp->objhead; + sp->objcore->objhead = sp->objhead; sp->objhead = NULL; /* refcnt follows pointer. */ sp->objcore = NULL; /* refcnt follows pointer. */ } Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 11:14:35 UTC (rev 4194) @@ -128,11 +128,10 @@ struct objcore_head *lru; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - AN(o->objhead); + CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); AN(ObjIsBusy(o)); assert(o->cacheable); HSH_Ref(o); - CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); oc = o->objcore; assert(o->entered != 0 && !isnan(o->entered)); @@ -175,8 +174,8 @@ lru = STV_lru(o->objstore); if (lru == NULL) return (retval); - AN(o->objhead); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); if (Lck_Trylock(&exp_mtx)) return (retval); if (oc->flags & OC_F_ONLRU) { @@ -274,11 +273,13 @@ /* And from LRU */ if (oc->flags & OC_F_ONLRU) { assert(!(oc->flags & OC_F_PERSISTENT)); + o = oc->obj; lru = STV_lru(o->objstore); AN(lru); VTAILQ_REMOVE(lru, o->objcore, lru_list); oc->flags &= ~OC_F_ONLRU; } else { + o = NULL; assert(oc->flags & OC_F_PERSISTENT); } @@ -289,11 +290,14 @@ if (!(oc->flags & OC_F_PERSISTENT)) { o = oc->obj; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); + WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", + o->xid, (int)(o->ttl - t)); HSH_Deref(sp->wrk, &o); - CHECK_OBJ_NOTNULL(o->objhead, OBJHEAD_MAGIC); + } else { + WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", + o, (int)(oc->timer_when - t)); } - WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", - o->xid, (int)(o->ttl - t)); } } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-18 11:14:35 UTC (rev 4194) @@ -589,7 +589,8 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); o = sp->obj; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - oh = o->objhead; + CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); + oh = o->objcore->objhead; CHECK_OBJ(oh, OBJHEAD_MAGIC); AN(ObjIsBusy(o)); @@ -616,7 +617,8 @@ struct objhead *oh; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - oh = o->objhead; + CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); + oh = o->objcore->objhead; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_Lock(&oh->mtx); assert(o->refcnt > 0); @@ -660,16 +662,15 @@ o = *oo; *oo = NULL; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - oh = o->objhead; - if (oh == NULL) { - oc = NULL; + oc = o->objcore; + if (oc == NULL) { assert(o->refcnt > 0); r = --o->refcnt; + oh = NULL; } else { + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + oh = oc->objhead; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - - oc = o->objcore; - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); @@ -706,11 +707,11 @@ o = NULL; w->stats->n_object--; - if (oh == NULL) { - AZ(oc); + if (oc == NULL) { + AZ(oh); return; } - AN(oc); + AN(oh); FREE_OBJ(oc); /* Drop our ref on the objhead */ assert(oh->refcnt > 0); Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-08-18 11:14:35 UTC (rev 4194) @@ -731,10 +731,11 @@ if (sp->obj == NULL) return (NULL); CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); - if (sp->obj->objhead == NULL) + if (sp->obj->objcore == NULL) return (NULL); - CHECK_OBJ_NOTNULL(sp->obj->objhead, OBJHEAD_MAGIC); - return (sp->obj->objhead->hash); + CHECK_OBJ_NOTNULL(sp->obj->objcore, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj->objcore->objhead, OBJHEAD_MAGIC); + return (sp->obj->objcore->objhead->hash); } unsigned Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 10:35:18 UTC (rev 4193) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 11:14:35 UTC (rev 4194) @@ -623,7 +623,7 @@ /* refcnt is one because the object is in the hash */ oc->obj->refcnt = 1; oc->obj->objcore = oc; - oc->obj->objhead = oh; + oc->objhead = oh; oc->obj->ban = oc->ban; sg->nfixed++; @@ -1067,7 +1067,7 @@ } assert(sg->nalloc < sg->maxobj); so = &sg->objs[sg->nalloc++]; - memcpy(so->hash, sp->obj->objhead->digest, DIGEST_LEN); + memcpy(so->hash, sp->obj->objcore->objhead->digest, DIGEST_LEN); so->ttl = sp->obj->ttl; so->ptr = sp->obj; so->ban = sp->obj->ban_t; From phk at projects.linpro.no Tue Aug 18 11:16:06 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 13:16:06 +0200 (CEST) Subject: r4195 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818111606.220221F7C98@projects.linpro.no> Author: phk Date: 2009-08-18 13:16:05 +0200 (Tue, 18 Aug 2009) New Revision: 4195 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Avoid the 10 second sleep while waiting for the VCL to appear. Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 11:14:35 UTC (rev 4194) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 11:16:05 UTC (rev 4195) @@ -246,7 +246,6 @@ struct objcore_head *lru; (void)priv; - AZ(sleep(10)); /* XXX: Takes time for VCL to arrive */ VCL_Get(&sp->vcl); t = TIM_real(); while (1) { Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2009-08-18 11:14:35 UTC (rev 4194) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2009-08-18 11:16:05 UTC (rev 4195) @@ -84,7 +84,13 @@ void VCL_Get(struct VCL_conf **vcc) { + static int once; + while (!once && vcl_active == NULL) { + sleep(1); + } + once = 1; + Lck_Lock(&vcl_mtx); AN(vcl_active); *vcc = vcl_active->conf; From phk at projects.linpro.no Tue Aug 18 11:23:41 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 13:23:41 +0200 (CEST) Subject: r4196 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818112341.B12581F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 13:23:41 +0200 (Tue, 18 Aug 2009) New Revision: 4196 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Muffle Flexelint a little bit. Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2009-08-18 11:16:05 UTC (rev 4195) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2009-08-18 11:23:41 UTC (rev 4196) @@ -84,10 +84,10 @@ void VCL_Get(struct VCL_conf **vcc) { - static int once; + static int once = 0; while (!once && vcl_active == NULL) { - sleep(1); + (void)sleep(1); } once = 1; From phk at projects.linpro.no Tue Aug 18 11:46:06 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 13:46:06 +0200 (CEST) Subject: r4197 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090818114606.C496E1F7C98@projects.linpro.no> Author: phk Date: 2009-08-18 13:46:06 +0200 (Tue, 18 Aug 2009) New Revision: 4197 Added: trunk/varnish-cache/bin/varnishtest/tests/p00005.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc Log: Fix expiry of non-instantiated objects. Add test-case for same. Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 11:23:41 UTC (rev 4196) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-18 11:46:06 UTC (rev 4197) @@ -262,7 +262,9 @@ continue; } + /* It's time... */ + CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); /* Remove from binheap */ assert(oc->timer_idx != BINHEAP_NOIDX); @@ -286,16 +288,19 @@ Lck_Unlock(&exp_mtx); + CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); if (!(oc->flags & OC_F_PERSISTENT)) { o = oc->obj; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", o->xid, (int)(o->ttl - t)); HSH_Deref(sp->wrk, &o); } else { - WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", + WSL(sp->wrk, SLT_ExpKill, 1, "%u %d", o, (int)(oc->timer_when - t)); + sp->objhead = oc->objhead; + sp->objcore = oc; + HSH_DerefObjCore(sp); } } } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-18 11:23:41 UTC (rev 4196) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-18 11:46:06 UTC (rev 4197) @@ -406,6 +406,7 @@ /* XXX: Should this not be ..._HEAD now ? */ VTAILQ_INSERT_TAIL(&oh->objcs, oc, list); /* NB: do not deref objhead the new object inherits our reference */ + oc->objhead = oh; Lck_Unlock(&oh->mtx); sp->wrk->stats->n_object++; return (oc); @@ -642,6 +643,7 @@ Lck_Lock(&oh->mtx); VTAILQ_REMOVE(&oh->objcs, oc, list); + sp->wrk->stats->n_object--; Lck_Unlock(&oh->mtx); assert(oh->refcnt > 0); FREE_OBJ(oc); Added: trunk/varnish-cache/bin/varnishtest/tests/p00005.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/p00005.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/p00005.vtc 2009-08-18 11:46:06 UTC (rev 4197) @@ -0,0 +1,50 @@ +# $Id$ + +test "Check expiry of non-instantiated object" + +shell "rm -f /tmp/__v1/_.per" + +server s1 { + rxreq + txresp -hdr "Foo: foo1" +} -start + +varnish v1 \ + -arg "-pdiag_bitmap=0x30000" \ + -arg "-spersistent,/tmp/__v1/_.per,10m" \ + -vcl+backend { + sub vcl_fetch { + set beresp.ttl = 3s; + } + } -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" + expect resp.http.foo == "foo1" +} -run + +varnish v1 -expect n_object == 1 + +varnish v1 -stop + +server s1 -wait { + rxreq + txresp -hdr "Foo: foo2" +} -start + +varnish v1 -start + +delay 5 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" + expect resp.http.foo == "foo2" +} -run + +varnish v1 -expect n_object == 1 Modified: trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc 2009-08-18 11:23:41 UTC (rev 4196) +++ trunk/varnish-cache/bin/varnishtest/tests/v00004.vtc 2009-08-18 11:46:06 UTC (rev 4197) @@ -29,6 +29,8 @@ } varnish v1 -expect n_backend == 2 +# give the expiry thread a chance to let go of its VCL +delay 2 varnish v1 -expect n_vcl == 3 From phk at projects.linpro.no Tue Aug 18 12:03:07 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 14:03:07 +0200 (CEST) Subject: r4198 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818120307.40CEC1F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 14:03:06 +0200 (Tue, 18 Aug 2009) New Revision: 4198 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Make sure new seg is inside silo. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 11:46:06 UTC (rev 4197) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 12:03:06 UTC (rev 4198) @@ -892,6 +892,7 @@ } else { sg->offset = sg2->offset + sg2->length; assert(sg->offset < sc->mediasize); + assert(sg->offset + sg2->length < sc->mediasize); } sg->length = sc->aim_segl; sg->length &= ~7; From phk at projects.linpro.no Tue Aug 18 12:19:10 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 14:19:10 +0200 (CEST) Subject: r4199 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818121910.6007B1F7C97@projects.linpro.no> Author: phk Date: 2009-08-18 14:19:10 +0200 (Tue, 18 Aug 2009) New Revision: 4199 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Don't allocate if we do not fit in the segment. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 12:03:06 UTC (rev 4198) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 12:19:10 UTC (rev 4199) @@ -892,7 +892,6 @@ } else { sg->offset = sg2->offset + sg2->length; assert(sg->offset < sc->mediasize); - assert(sg->offset + sg2->length < sc->mediasize); } sg->length = sc->aim_segl; sg->length &= ~7; @@ -1093,6 +1092,10 @@ sg = sc->cur_seg; /* XXX: size fit check */ + if (sg->next_addr + sizeof *ss + size > sg->length) { + Lck_Unlock(&sc->mtx); + return (NULL); + } AN(sg->next_addr); ss = (void *)(sc->ptr + sg->next_addr); sg->next_addr += size + sizeof *ss; From phk at projects.linpro.no Tue Aug 18 15:48:08 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 17:48:08 +0200 (CEST) Subject: r4200 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818154808.A56E21F7C98@projects.linpro.no> Author: phk Date: 2009-08-18 17:48:08 +0200 (Tue, 18 Aug 2009) New Revision: 4200 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c Log: Be more paranoid about shmlog magics, Don't explode manager if it gets overwritten. Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2009-08-18 12:19:10 UTC (rev 4199) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2009-08-18 15:48:08 UTC (rev 4200) @@ -72,16 +72,19 @@ vsl_wrap(void) { + assert(loghead->magic == SHMLOGHEAD_MAGIC); *logstart = SLT_ENDMARKER; logstart[loghead->ptr] = SLT_WRAPMARKER; loghead->ptr = 0; VSL_stats->shm_cycles++; + assert(loghead->magic == SHMLOGHEAD_MAGIC); } static void vsl_hdr(enum shmlogtag tag, unsigned char *p, unsigned len, unsigned id) { + assert(loghead->magic == SHMLOGHEAD_MAGIC); assert(len < 0x10000); assert(id < 0x10000); p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff; @@ -281,13 +284,18 @@ void VSL_Panic(int *len, char **ptr) { + static char a[1] = { '\0' }; AN(len); AN(ptr); - assert(loghead->magic == SHMLOGHEAD_MAGIC); - assert(loghead->hdrsize == sizeof *loghead); - *len = sizeof(loghead->panicstr); - *ptr = loghead->panicstr; + if (loghead->magic == SHMLOGHEAD_MAGIC) { + assert(loghead->hdrsize == sizeof *loghead); + *len = sizeof(loghead->panicstr); + *ptr = loghead->panicstr; + } else { + *len = 0; + *ptr = a; + } } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Tue Aug 18 15:57:07 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 17:57:07 +0200 (CEST) Subject: r4201 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818155707.CC1AE37F3A@projects.linpro.no> Author: phk Date: 2009-08-18 17:57:07 +0200 (Tue, 18 Aug 2009) New Revision: 4201 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Hack up -spersistent some more. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 15:48:08 UTC (rev 4200) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 15:57:07 UTC (rev 4201) @@ -129,6 +129,7 @@ struct smp_seghead segments; struct smp_seg *cur_seg; + uint64_t objreserv; pthread_t thread; VTAILQ_ENTRY(smp_sc) list; @@ -328,7 +329,6 @@ smp_reset_sign(&sc->idn); si = sc->ident; -printf("NEW: %p\n", si); memset(si, 0, sizeof *si); strcpy(si->ident, SMP_IDENT_STRING); @@ -444,7 +444,7 @@ * XXX: the lines of "one object per silo". */ - sc->min_nseg = 100; + sc->min_nseg = 40; sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg; fprintf(stderr, "min_nseg = %u, max_segl = %ju\n", @@ -538,6 +538,20 @@ ARGV_ERR("(-spersistent) failed to mmap (%s)\n", strerror(errno)); + if (1) { + /* + * XXX: This (magically ?) prevents a memory corruption + * XXX: which I have not been able to find any rhyme and + * XXX: reason in. + */ + void *foo; + + + foo = mmap(sc->ptr + sc->mediasize, sc->granularity, + PROT_NONE, MAP_ANON, -1, 0); + assert(foo == sc->ptr + sc->mediasize); + } + smp_def_sign(sc, &sc->idn, 0, "SILO"); sc->ident = SIGN_DATA(&sc->idn); @@ -919,6 +933,7 @@ sizeof (struct smp_segment) + SHA256_LEN; memcpy(sc->ptr + sg->next_addr, "HERE", 4); + sc->objreserv = sizeof *sg->objs + SHA256_LEN; } /*-------------------------------------------------------------------- @@ -928,13 +943,19 @@ static void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg) { + size_t sz; (void)sc; + /* XXX: if segment is empty, delete instead */ + + /* Copy the objects into the segment */ - memcpy(sc->ptr + sg->next_addr, - sg->objs, sizeof *sg->objs * sg->nalloc); + sz = sizeof *sg->objs * sg->nalloc; + assert(sg->next_addr + sz <= sg->offset + sg->length); + memcpy(sc->ptr + sg->next_addr, sg->objs, sz); + /* Update the segment header */ sg->segment.objlist = sg->next_addr; sg->segment.nalloc = sg->nalloc; @@ -1059,6 +1080,7 @@ Lck_Lock(&sc->mtx); sg = sc->cur_seg; + sc->objreserv += sizeof *so; if (sg->nalloc >= sg->maxobj) { smp_close_seg(sc, sc->cur_seg); smp_new_seg(sc); @@ -1085,20 +1107,47 @@ struct smp_sc *sc; struct storage *ss; struct smp_seg *sg; + size_t sz2; CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_Lock(&sc->mtx); sg = sc->cur_seg; - /* XXX: size fit check */ - if (sg->next_addr + sizeof *ss + size > sg->length) { - Lck_Unlock(&sc->mtx); - return (NULL); + AN(sg->next_addr); + + /* + * XXX: We do not have a mechanism for deciding which allocations + * XXX: have mandatory size and which can be chopped into smaller + * XXX: allocations. + * XXX: The primary unchoppable allocation is from HSH_NewObject() + * XXX: which needs an object + workspace. + */ + sz2 = sizeof (struct object) + 1024; + if (size < sz2) + sz2 = size; + + /* If the segment is full, get a new one right away */ + if (sg->next_addr + sizeof *ss + sz2 + sc->objreserv > + sg->offset + sg->length) { + smp_close_seg(sc, sc->cur_seg); + smp_new_seg(sc); + sg = sc->cur_seg; } - AN(sg->next_addr); + + assert (sg->next_addr + sizeof *ss + sz2 + sc->objreserv <= + sg->offset + sg->length); + + if (sg->next_addr + sizeof *ss + size + sc->objreserv > + sg->offset + sg->length) + size = (sg->offset + sg->length) - + (sg->next_addr + sizeof *ss + sc->objreserv); + ss = (void *)(sc->ptr + sg->next_addr); + sg->next_addr += size + sizeof *ss; + assert(sg->next_addr <= sg->offset + sg->length); + memcpy(sc->ptr + sg->next_addr, "HERE", 4); Lck_Unlock(&sc->mtx); /* Grab and fill a storage structure */ @@ -1110,7 +1159,6 @@ ss->stevedore = st; ss->fd = sc->fd; ss->where = sg->next_addr + sizeof *ss; - memcpy(sc->ptr + sg->next_addr, "HERE", 4); return (ss); } @@ -1121,6 +1169,8 @@ struct smp_seg *sg; const char z[4] = { 0, 0, 0, 0}; + return; + CAST_OBJ_NOTNULL(sc, ss->priv, SMP_SC_MAGIC); /* We want 16 bytes alignment */ From phk at projects.linpro.no Tue Aug 18 19:08:06 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 21:08:06 +0200 (CEST) Subject: r4202 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818190806.5AD6937F3A@projects.linpro.no> Author: phk Date: 2009-08-18 21:08:06 +0200 (Tue, 18 Aug 2009) New Revision: 4202 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: A long time ago, I wrote fetch_straight() in fact it was one of the very first functions ever written in Varnish. Since I knew that the file stevedore would always return the size you asked for, or panic, I didn't bother to actually check how big the storage segment he allocated was, I just knew it would be the right size. Guess what, segments are finite size in the -spersistent stevedore so you may in fact _not_ get what you ask for. (Cue music: Theme Music from The Keystone Kops) DuH! Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-08-18 15:57:07 UTC (rev 4201) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-08-18 19:08:06 UTC (rev 4202) @@ -53,7 +53,7 @@ int i; unsigned char *p; uintmax_t cll; - unsigned cl; + unsigned cl, sl; struct storage *st; cll = strtoumax(b, NULL, 0); @@ -63,18 +63,24 @@ cl = (unsigned)cll; assert((uintmax_t)cl == cll); /* Protect against bogusly large values */ - st = STV_alloc(sp, cl); - VTAILQ_INSERT_TAIL(&sp->obj->store, st, list); - st->len = cl; - sp->obj->len = cl; - p = st->ptr; + while (cl > 0) { + st = STV_alloc(sp, cl); + VTAILQ_INSERT_TAIL(&sp->obj->store, st, list); + sl = st->space; + if (sl > cl) + sl = cl; + p = st->ptr; - while (cl > 0) { - i = HTC_Read(htc, p, cl); - if (i <= 0) - return (-1); - p += i; - cl -= i; + while (sl > 0) { + i = HTC_Read(htc, p, sl); + if (i <= 0) + return (-1); + p += i; + st->len += i; + sp->obj->len += i; + sl -= i; + cl -= i; + } } return (0); } From phk at projects.linpro.no Tue Aug 18 20:43:25 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Aug 2009 22:43:25 +0200 (CEST) Subject: r4203 - trunk/varnish-cache/bin/varnishd Message-ID: <20090818204325.BB47237F3A@projects.linpro.no> Author: phk Date: 2009-08-18 22:43:25 +0200 (Tue, 18 Aug 2009) New Revision: 4203 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Various random improvements to get closer to something that works. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 19:08:06 UTC (rev 4202) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-18 20:43:25 UTC (rev 4203) @@ -444,7 +444,7 @@ * XXX: the lines of "one object per silo". */ - sc->min_nseg = 40; + sc->min_nseg = 10; sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg; fprintf(stderr, "min_nseg = %u, max_segl = %ju\n", @@ -458,6 +458,11 @@ sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg; sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg; + while (sc->min_segl < sizeof(struct object)) { + sc->max_nseg /= 2; + sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg; + } + fprintf(stderr, "max_nseg = %u, min_segl = %ju\n", sc->max_nseg, (uintmax_t)sc->min_segl); @@ -538,20 +543,6 @@ ARGV_ERR("(-spersistent) failed to mmap (%s)\n", strerror(errno)); - if (1) { - /* - * XXX: This (magically ?) prevents a memory corruption - * XXX: which I have not been able to find any rhyme and - * XXX: reason in. - */ - void *foo; - - - foo = mmap(sc->ptr + sc->mediasize, sc->granularity, - PROT_NONE, MAP_ANON, -1, 0); - assert(foo == sc->ptr + sc->mediasize); - } - smp_def_sign(sc, &sc->idn, 0, "SILO"); sc->ident = SIGN_DATA(&sc->idn); @@ -890,6 +881,7 @@ { struct smp_seg *sg, *sg2; + Lck_AssertHeld(&sc->mtx); ALLOC_OBJ(sg, SMP_SEG_MAGIC); AN(sg); @@ -947,6 +939,8 @@ (void)sc; + Lck_AssertHeld(&sc->mtx); + /* XXX: if segment is empty, delete instead */ @@ -968,7 +962,7 @@ sg->next_addr += sizeof *sg->objs * sg->nalloc; sg->length = sg->next_addr - sg->offset; - sg->length |= 7; + sg->length |= 15; sg->length++; /* Save segment list */ @@ -1015,6 +1009,7 @@ CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_New(&sc->mtx); + Lck_Lock(&sc->mtx); /* We trust the parent to give us a valid silo, for good measure: */ AZ(smp_valid_silo(sc)); @@ -1043,6 +1038,7 @@ WRK_BgThread(&sc->thread, "persistence", smp_thread, sc); VTAILQ_INSERT_TAIL(&silos, sc, list); + Lck_Unlock(&sc->mtx); } /*-------------------------------------------------------------------- @@ -1057,7 +1053,9 @@ ASSERT_CLI(); CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); + Lck_Lock(&sc->mtx); smp_close_seg(sc, sc->cur_seg); + Lck_Unlock(&sc->mtx); /* XXX: reap thread */ } @@ -1159,6 +1157,9 @@ ss->stevedore = st; ss->fd = sc->fd; ss->where = sg->next_addr + sizeof *ss; + assert((uintmax_t)ss->space == (uintmax_t)size); + assert((char*)ss->ptr > (char*)ss); + assert((char*)ss->ptr + ss->space <= (char*)sc->ptr + sc->mediasize); return (ss); } From sky at projects.linpro.no Wed Aug 19 02:15:07 2009 From: sky at projects.linpro.no (sky at projects.linpro.no) Date: Wed, 19 Aug 2009 04:15:07 +0200 (CEST) Subject: r4204 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090819021507.2652537F3A@projects.linpro.no> Author: sky Date: 2009-08-19 04:15:06 +0200 (Wed, 19 Aug 2009) New Revision: 4204 Added: trunk/varnish-cache/bin/varnishtest/tests/c00028.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: it was possible to have infinite recursion between vcl_error returning restart and cnt_recv calling vcl_error when you had too many restarts Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-18 20:43:25 UTC (rev 4203) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-19 02:15:06 UTC (rev 4204) @@ -350,13 +350,15 @@ http_StatusMessage(sp->err_code)); VCL_error_method(sp); - if (sp->handling == VCL_RET_RESTART) { + if (sp->handling == VCL_RET_RESTART && sp->restarts < params->max_restarts) { HSH_Drop(sp); sp->director = NULL; sp->restarts++; sp->step = STP_RECV; return (0); - } + } else if (sp->handling == VCL_RET_RESTART) + sp->handling = VCL_RET_DELIVER; + /* We always close when we take this path */ sp->doclose = "error"; Added: trunk/varnish-cache/bin/varnishtest/tests/c00028.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/c00028.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/c00028.vtc 2009-08-19 02:15:06 UTC (rev 4204) @@ -0,0 +1,24 @@ +# $Id$ + +test "Test that we can't recurse restarts forever" + +varnish v1 -vcl { + backend bad { + .host = "127.0.0.1"; + .port = "9090"; + } + + sub vcl_recv { + set req.backend = bad; + } + sub vcl_error { + restart; + } + } -start + +client c1 { + txreq -url "/" + rxresp + expect resp.status == 503 +} -run + From phk at projects.linpro.no Wed Aug 19 18:53:47 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Aug 2009 20:53:47 +0200 (CEST) Subject: r4205 - trunk/varnish-cache/bin/varnishd Message-ID: <20090819185347.3DA721F750D@projects.linpro.no> Author: phk Date: 2009-08-19 20:53:46 +0200 (Wed, 19 Aug 2009) New Revision: 4205 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_ban.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Do the simple part of ban list lurker: link the objcores off the bans the reference. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-19 02:15:06 UTC (rev 4204) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-19 18:53:46 UTC (rev 4205) @@ -291,6 +291,7 @@ unsigned timer_idx; VTAILQ_ENTRY(objcore) list; VTAILQ_ENTRY(objcore) lru_list; + VTAILQ_ENTRY(objcore) ban_list; struct smp_seg *smp_seg; struct ban *ban; }; @@ -470,7 +471,7 @@ void BAN_Reload(double t0, unsigned flags, const char *ban); struct ban *BAN_TailRef(void); void BAN_Compile(void); -struct ban *BAN_RefBan(double t0, const struct ban *tail); +struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail); void BAN_Deref(struct ban **ban); /* cache_center.c [CNT] */ Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-19 02:15:06 UTC (rev 4204) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-19 18:53:46 UTC (rev 4205) @@ -81,6 +81,7 @@ return (NULL); } VTAILQ_INIT(&b->tests); + VTAILQ_INIT(&b->objcore); return (b); } @@ -103,6 +104,9 @@ struct ban_test *bt; CHECK_OBJ_NOTNULL(b, BAN_MAGIC); + AZ(b->refcount); + assert(VTAILQ_EMPTY(&b->objcore)); + if (b->vsb != NULL) vsb_delete(b->vsb); if (b->test != NULL) @@ -365,10 +369,12 @@ { CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); AZ(o->ban); Lck_Lock(&ban_mtx); o->ban = ban_start; ban_start->refcount++; + VTAILQ_INSERT_TAIL(&ban_start->objcore, o->objcore, ban_list); Lck_Unlock(&ban_mtx); o->ban_t = o->ban->t0; } @@ -402,6 +408,7 @@ Lck_Lock(&ban_mtx); assert(o->ban->refcount > 0); o->ban->refcount--; + VTAILQ_REMOVE(&o->ban->objcore, o->objcore, ban_list); o->ban = NULL; /* Attempt to purge last ban entry */ @@ -449,8 +456,11 @@ Lck_Lock(&ban_mtx); o->ban->refcount--; - if (b == o->ban) /* not banned */ + VTAILQ_REMOVE(&o->ban->objcore, o->objcore, ban_list); + if (b == o->ban) { /* not banned */ + VTAILQ_INSERT_TAIL(&b0->objcore, o->objcore, ban_list); b0->refcount++; + } VSL_stats->n_purge_obj_test++; VSL_stats->n_purge_re_test += tests; Lck_Unlock(&ban_mtx); @@ -511,7 +521,7 @@ */ struct ban * -BAN_RefBan(double t0, const struct ban *tail) +BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail) { struct ban *b; @@ -525,6 +535,7 @@ assert(b->t0 == t0); Lck_Lock(&ban_mtx); b->refcount++; + VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list); Lck_Unlock(&ban_mtx); return (b); } Modified: trunk/varnish-cache/bin/varnishd/cache_ban.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-08-19 02:15:06 UTC (rev 4204) +++ trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-08-19 18:53:46 UTC (rev 4205) @@ -58,6 +58,7 @@ #define BAN_F_GONE (1 << 0) #define BAN_F_PENDING (1 << 1) VTAILQ_HEAD(,ban_test) tests; + VTAILQ_HEAD(,objcore) objcore; double t0; struct vsb *vsb; char *test; Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-19 02:15:06 UTC (rev 4204) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-19 18:53:46 UTC (rev 4205) @@ -510,10 +510,9 @@ sp->objcore->objhead = sp->objhead; sp->objhead = NULL; /* refcnt follows pointer. */ sp->objcore = NULL; /* refcnt follows pointer. */ + BAN_NewObj(sp->obj); } - BAN_NewObj(sp->obj); - sp->obj->xid = sp->xid; sp->obj->response = sp->err_code; sp->obj->cacheable = sp->wrk->cacheable; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-19 02:15:06 UTC (rev 4204) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-19 18:53:46 UTC (rev 4205) @@ -688,7 +688,8 @@ if (r != 0) return; - BAN_DestroyObj(o); + if (oh != NULL) + BAN_DestroyObj(o); AZ(o->ban); DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u", o->xid, WS_Free(o->ws_o)); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-19 02:15:06 UTC (rev 4204) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-19 18:53:46 UTC (rev 4205) @@ -830,7 +830,7 @@ oc->flags &= ~OC_F_BUSY; oc->obj = (void*)so; oc->smp_seg = sg; - oc->ban = BAN_RefBan(so->ban, sc->tailban); + oc->ban = BAN_RefBan(oc, so->ban, sc->tailban); memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN); (void)HSH_Insert(sp); EXP_Inject(oc, NULL, so->ttl); From phk at projects.linpro.no Wed Aug 19 21:04:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Aug 2009 23:04:03 +0200 (CEST) Subject: r4206 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include lib/libvcl Message-ID: <20090819210403.A2C501F7502@projects.linpro.no> Author: phk Date: 2009-08-19 23:04:03 +0200 (Wed, 19 Aug 2009) New Revision: 4206 Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_ban.h trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_slinger.h trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/mgt_param.c trunk/varnish-cache/bin/varnishd/storage_persistent.c trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc trunk/varnish-cache/include/purge_vars.h trunk/varnish-cache/lib/libvcl/vcc_action.c Log: Add an experimental thread to lurk at the bottom of the ban-list and try to push objects away from the tail end so that the list can be shortened. It won't work if your bans contains any req.* tests, because the lurker has no request to compare against. By default the lurker is disabled, you enable it by setting the 'ban_lurker_sleep' parameter to the amount of time the lurker should sleep between doing things. If it cannot do anything, it always sleeps for a second. NB: This feature is on probation, there is currently no guarantee it will end up in a future varnish release, it depends on the impact and usability. Feedback much anticipated. One of the quasi-nasty details of this feature, is the need to work against the lock-order in cache_hash.c, see details in comments for HSH_FindBan(). Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-19 21:04:03 UTC (rev 4206) @@ -63,6 +63,7 @@ static VTAILQ_HEAD(banhead,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head); static struct lock ban_mtx; static struct ban *ban_magic; +static pthread_t ban_thread; /*-------------------------------------------------------------------- * Manipulation of bans @@ -285,6 +286,8 @@ if (strncmp(a1, pv->name, strlen(pv->name))) continue; bt->func = pv->func; + if (pv->flag & 2) + b->flags |= BAN_F_REQ; if (pv->flag & 1) ban_parse_http(bt, a1 + strlen(pv->name)); break; @@ -419,8 +422,8 @@ } -int -BAN_CheckObject(struct object *o, const struct sess *sp) +static int +ban_check_object(struct object *o, const struct sess *sp, int has_req) { struct ban *b; struct ban_test *bt; @@ -445,6 +448,8 @@ for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) { if (b->flags & BAN_F_GONE) continue; + if (!has_req && (b->flags & BAN_F_REQ)) + return (0); VTAILQ_FOREACH(bt, &b->tests, list) { tests++; if (bt->func(bt, o, sp)) @@ -484,7 +489,69 @@ } } +int +BAN_CheckObject(struct object *o, const struct sess *sp) +{ + + return ban_check_object(o, sp, 1); +} + /*-------------------------------------------------------------------- + * Ban tail lurker thread + */ + +static void * +ban_lurker(struct sess *sp, void *priv) +{ + struct ban *b, *bf; + struct objcore *oc; + struct object *o; + + (void)priv; + while (1) { + WSL_Flush(sp->wrk, 0); + WRK_SumStat(sp->wrk); + if (params->ban_lurker_sleep == 0.0) { + AZ(sleep(1)); + continue; + } + Lck_Lock(&ban_mtx); + + /* First try to route the last ban */ + bf = BAN_CheckLast(); + if (bf != NULL) { + Lck_Unlock(&ban_mtx); + BAN_Free(bf); + TIM_sleep(params->ban_lurker_sleep); + continue; + } + /* Then try to poke the first object on the last ban */ + oc = NULL; + while (1) { + b = VTAILQ_LAST(&ban_head, banhead); + if (b == ban_start) + break; + oc = VTAILQ_FIRST(&b->objcore); + if (oc == NULL) + break; + HSH_FindBan(sp, &oc); + break; + } + Lck_Unlock(&ban_mtx); + if (oc == NULL) { + TIM_sleep(1.0); + continue; + } + o = oc->obj; + (void)ban_check_object(o, sp, 0); + WSP(sp, SLT_Debug, "lurker: %p %g", oc, o->ttl); + HSH_Deref(sp->wrk, &o); + TIM_sleep(params->ban_lurker_sleep); + } +} + + +/*-------------------------------------------------------------------- * Release a reference */ @@ -767,4 +834,5 @@ AN(ban_magic); ban_magic->flags |= BAN_F_GONE; BAN_Insert(ban_magic); + WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL); } Modified: trunk/varnish-cache/bin/varnishd/cache_ban.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-08-19 21:04:03 UTC (rev 4206) @@ -57,6 +57,7 @@ int flags; #define BAN_F_GONE (1 << 0) #define BAN_F_PENDING (1 << 1) +#define BAN_F_REQ (1 << 2) VTAILQ_HEAD(,ban_test) tests; VTAILQ_HEAD(,objcore) objcore; double t0; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-19 21:04:03 UTC (rev 4206) @@ -652,7 +652,40 @@ HSH_DeleteObjHead(sp->wrk, oh); } +/******************************************************************* + * This one is slightly tricky. This is called from the BAN module + * to try to wash the object which holds the oldest ban. + * We compete against HSH_Deref() which comes in the opposite + * locking order, we need to hold the BAN mutex, to stop the + * BAN_DestroyObj() call in HSH_Deref(), so that the objhead + * will not be removed under us. + * NB: Do not call this function any other way or from any other + * NB: place in the code. It will not work for you. + */ + void +HSH_FindBan(struct sess *sp, struct objcore **oc) +{ + struct objcore *oc1, *oc2; + struct objhead *oh; + + oc1 = *oc; + CHECK_OBJ_NOTNULL(oc1, OBJCORE_MAGIC); + oh = oc1->objhead; + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + Lck_Lock(&oh->mtx); + VTAILQ_FOREACH(oc2, &oh->objcs, list) + if (oc1 == oc2) { + oc1->obj->refcnt++; + break; + } + if (oc2 != NULL && oc2->flags & OC_F_PERSISTENT) + SMP_Fixup(sp, oh, oc2); + Lck_Unlock(&oc1->objhead->mtx); + *oc = oc2; +} + +void HSH_Deref(const struct worker *w, struct object **oo) { struct object *o; Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-19 21:04:03 UTC (rev 4206) @@ -66,6 +66,7 @@ void HSH_BeforeVclHash(struct sess *sp, unsigned hashcount); void HSH_AfterVclHash(const struct sess *sp); void HSH_DerefObjCore(struct sess *sp); +void HSH_FindBan(struct sess *sp, struct objcore **oc); struct objcore *HSH_Insert(const struct sess *sp); #ifdef VARNISH_CACHE_CHILD Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-19 21:04:03 UTC (rev 4206) @@ -195,6 +195,9 @@ /* CLI banner */ unsigned cli_banner; + + /* How long time does the ban lurker sleep */ + double ban_lurker_sleep; }; extern volatile struct params *params; Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-08-19 21:04:03 UTC (rev 4206) @@ -758,6 +758,14 @@ "Set to off for compatibility with pre 2.1 versions.\n", 0, "on", "bool" }, + { "ban_lurker_sleep", tweak_timeout_double, + &master.ban_lurker_sleep, 0.0, UINT_MAX, + "How long time does the ban lurker thread sleeps between " + "successfull attempts to push the last item up the purge " + " list. It always sleeps a second when nothing can be done.\n" + "A value of zero disables the ban lurker.", + 0, + "0.0", "s" }, { NULL, NULL, NULL } }; Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-19 21:04:03 UTC (rev 4206) @@ -610,6 +610,7 @@ struct smp_seg *sg; struct smp_object *so; + Lck_AssertHeld(&oh->mtx); (void)sp; sg = oc->smp_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); Modified: trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc 2009-08-19 21:04:03 UTC (rev 4206) @@ -53,6 +53,9 @@ varnish v1 -cli "vcl.discard vcl1" +# Give expiry thread a chance to let go +delay 2 + # It won't go away as long as the workthread holds a VCL reference varnish v1 -expect n_backend == 2 varnish v1 -expect n_vcl_avail == 1 Modified: trunk/varnish-cache/include/purge_vars.h =================================================================== --- trunk/varnish-cache/include/purge_vars.h 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/include/purge_vars.h 2009-08-19 21:04:03 UTC (rev 4206) @@ -28,9 +28,13 @@ * $Id$ * * Define which variables we can purge on, and which function does it. + * + * Middle field is a bitmap: + * 1 = HTTP header search + * 2 = Needs a client supplied request for test evaluation */ -PVAR("req.url", 0, ban_cond_url) -PVAR("obj.hash", 0, ban_cond_hash) -PVAR("req.http.", 1, ban_cond_req_http) -PVAR("obj.http.", 1, ban_cond_obj_http) +PVAR("req.url", 2|0, ban_cond_url) +PVAR("obj.hash", 0|0, ban_cond_hash) +PVAR("req.http.", 2|1, ban_cond_req_http) +PVAR("obj.http.", 0|1, ban_cond_obj_http) Modified: trunk/varnish-cache/lib/libvcl/vcc_action.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_action.c 2009-08-19 18:53:46 UTC (rev 4205) +++ trunk/varnish-cache/lib/libvcl/vcc_action.c 2009-08-19 21:04:03 UTC (rev 4206) @@ -383,7 +383,7 @@ vcc_ErrWhere(tl, tl->t); return; } - if (pv->flag && + if ((pv->flag & 1) && tl->t->b + strlen(pv->name) >= tl->t->e) { vsb_printf(tl->sb, "Missing header name."); vcc_ErrWhere(tl, tl->t); From kristian at projects.linpro.no Thu Aug 20 12:11:47 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Thu, 20 Aug 2009 14:11:47 +0200 (CEST) Subject: r4207 - trunk/varnish-cache/man Message-ID: <20090820121147.D3DA71F7C37@projects.linpro.no> Author: kristian Date: 2009-08-20 14:11:47 +0200 (Thu, 20 Aug 2009) New Revision: 4207 Modified: trunk/varnish-cache/man/vcl.7so Log: Add obj.hits to the vcl(7) manual Modified: trunk/varnish-cache/man/vcl.7so =================================================================== --- trunk/varnish-cache/man/vcl.7so 2009-08-19 21:04:03 UTC (rev 4206) +++ trunk/varnish-cache/man/vcl.7so 2009-08-20 12:11:47 UTC (rev 4207) @@ -541,6 +541,9 @@ .It Va obj.lastuse The approximate time elapsed since the object was last requests, in seconds. +.It Va obj.hits +The approximate number of times the object has been delivered. A value of 0 +indicates a cache miss. .El .Pp The following variables are available while determining the hash key From kristian at projects.linpro.no Mon Aug 24 14:51:26 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Mon, 24 Aug 2009 16:51:26 +0200 (CEST) Subject: r4208 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include lib/libvcl Message-ID: <20090824145126.64FD41F72A9@projects.linpro.no> Author: kristian Date: 2009-08-24 16:51:26 +0200 (Mon, 24 Aug 2009) New Revision: 4208 Added: trunk/varnish-cache/bin/varnishtest/tests/s00003.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_backend.h trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c trunk/varnish-cache/bin/varnishd/cache_dir_random.c trunk/varnish-cache/bin/varnishd/cache_dir_round_robin.c trunk/varnish-cache/bin/varnishd/cache_dir_simple.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Saint mode: add beresp.saintmode to vcl_fetch. This allows us to put items on a blacklist for a specific backend based on the response it gave us. The basic syntax is (vcl_fetch): if (beresp.status == 500) { set beresp.saintmode = 20s; restart; } Health checks are modified accordingly, which means graced objects are used if no healthy backends are left to try. Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -228,6 +228,52 @@ sp->vbe = sp->director->getfd(sp); } +/* + * It evaluates if a backend is healthy _for_a_specific_object_. + * That means that it relies on sp->objhead. This is mainly for saint-mode, + * but also takes backend->healthy into account. + */ +unsigned int +backend_is_healthy(const struct sess *sp, struct backend *backend) +{ + struct trouble *tr; + struct trouble *tr2; + struct trouble *old = NULL; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC); + + if (!backend->healthy) + return 0; + + /* No need to test if we don't have an object head to test against. + * FIXME: Should check the magic too, but probably not assert? + */ + if (!sp->objhead) + return 1; + + Lck_Lock(&backend->mtx); + VTAILQ_FOREACH_SAFE(tr, &backend->troublelist, list, tr2) { + CHECK_OBJ_NOTNULL(tr, TROUBLE_MAGIC); + if (tr->timeout < sp->t_req) { + VTAILQ_REMOVE(&backend->troublelist, tr, list); + old = tr; + break; + } + + if (tr->objhead == sp->objhead) { + Lck_Unlock(&backend->mtx); + return 0; + } + } + Lck_Unlock(&backend->mtx); + + if (old) + FREE_OBJ(old); + + return 1; +} + /*-------------------------------------------------------------------- * Get a connection to a particular backend. */ @@ -265,7 +311,7 @@ VBE_ClosedFd(sp); } - if (!bp->healthy) { + if (!backend_is_healthy(sp, bp)) { VSL_stats->backend_unhealthy++; return (NULL); } Modified: trunk/varnish-cache/bin/varnishd/cache_backend.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.h 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_backend.h 2009-08-24 14:51:26 UTC (rev 4208) @@ -93,6 +93,18 @@ }; /*-------------------------------------------------------------------- + * List of objectheads that have recently been rejected by VCL. + */ + +struct trouble { + unsigned magic; +#define TROUBLE_MAGIC 0x4211ab21 + void *objhead; /* NB: only comparison */ + double timeout; + VTAILQ_ENTRY(trouble) list; +}; + +/*-------------------------------------------------------------------- * An instance of a backend from a VCL program. */ @@ -124,11 +136,13 @@ struct vbp_target *probe; unsigned healthy; + VTAILQ_HEAD(, trouble) troublelist; }; /* cache_backend.c */ void VBE_ReleaseConn(struct vbe_conn *vc); struct vbe_conn *VBE_GetVbe(struct sess *sp, struct backend *bp); +unsigned int backend_is_healthy(const struct sess *sp, struct backend *backend); /* cache_backend_cfg.c */ extern struct lock VBE_mtx; Modified: trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -226,6 +226,8 @@ VTAILQ_INIT(&b->connlist); b->hash = u; + VTAILQ_INIT(&b->troublelist); + /* * This backend may live longer than the VCL that instantiated it * so we cannot simply reference the VCL's copy of things. Modified: trunk/varnish-cache/bin/varnishd/cache_dir_random.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_dir_random.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_dir_random.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -81,7 +81,7 @@ /* Sum up the weights of healty backends */ s1 = 0.0; for (i = 0; i < vs->nhosts; i++) - if (vs->hosts[i].backend->healthy) + if (backend_is_healthy(sp,vs->hosts[i].backend)) s1 += vs->hosts[i].weight; if (s1 == 0.0) @@ -94,7 +94,7 @@ s1 = 0.0; for (i = 0; i < vs->nhosts; i++) { - if (!vs->hosts[i].backend->healthy) + if (!backend_is_healthy(sp, vs->hosts[i].backend)) continue; s1 += vs->hosts[i].weight; if (r >= s1) @@ -120,7 +120,7 @@ CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_RANDOM_MAGIC); for (i = 0; i < vs->nhosts; i++) { - if (vs->hosts[i].backend->healthy) + if (backend_is_healthy(sp,vs->hosts[i].backend)) return 1; } return 0; Modified: trunk/varnish-cache/bin/varnishd/cache_dir_round_robin.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_dir_round_robin.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_dir_round_robin.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -75,7 +75,7 @@ for (i = 0; i < vs->nhosts; i++) { backend = vs->hosts[vs->next_host].backend; vs->next_host = (vs->next_host + 1) % vs->nhosts; - if (!backend->healthy) + if (!backend_is_healthy(sp, backend)) continue; vbe = VBE_GetVbe(sp, backend); if (vbe != NULL) @@ -96,7 +96,7 @@ CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_ROUND_ROBIN_MAGIC); for (i = 0; i < vs->nhosts; i++) { - if (vs->hosts[i].backend->healthy) + if (backend_is_healthy(sp, vs->hosts[i].backend)) return 1; } return 0; Modified: trunk/varnish-cache/bin/varnishd/cache_dir_simple.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_dir_simple.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_dir_simple.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -75,7 +75,7 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_SIMPLE_MAGIC); - return vs->backend->healthy; + return (backend_is_healthy(sp, vs->backend)); } /*lint -e{818} not const-able */ Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -493,6 +493,11 @@ * XXX: this until the object is unbusy'ed, so in practice we * XXX: serialize fetch of all Vary's if grace is possible. */ + /* Grace-stuff: sp->objhead is evaluated in healthy() for 'saint + * mode'. Is this entirely wrong, or just ugly? Why isn't objhead + * set here? FIXME:Grace. + */ + sp->objhead = oh; if (oc == NULL && grace_oc != NULL && (busy_oc != NULL || !sp->director->healthy(sp))) { o = grace_oc->obj; @@ -500,6 +505,7 @@ if (o->ttl + HSH_Grace(sp->grace) >= sp->t_req) oc = grace_oc; } + sp->objhead = NULL; if (oc != NULL) { o = oc->obj; Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -279,6 +279,59 @@ http_SetH(sp->obj->http, HTTP_HDR_STATUS, p); } +/* Add an objecthead to the saintmode list for the (hopefully) relevant + * backend. Some double-up asserting here to avoid assert-errors when there + * is no object. + */ +void +VRT_l_beresp_saintmode(const struct sess *sp, double a) +{ + struct trouble *new; + struct trouble *tr; + struct trouble *tr2; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (!sp->vbe) + return; + CHECK_OBJ_NOTNULL(sp->vbe, VBE_CONN_MAGIC); + if (!sp->vbe->backend) + return; + CHECK_OBJ_NOTNULL(sp->vbe->backend, BACKEND_MAGIC); + if (!sp->objhead) + return; + CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); + + /* Setting a negative holdoff period is a mistake. Detecting this + * when compiling the VCL would be better. + */ + assert(a > 0); + + ALLOC_OBJ(new, TROUBLE_MAGIC); + new->objhead = sp->objhead; + new->timeout = sp->t_req + a; + + /* Insert the new item on the list before the first item with a + * timeout at a later date (ie: sort by which entry will time out + * from the list + */ + Lck_Lock(&sp->vbe->backend->mtx); + VTAILQ_FOREACH_SAFE(tr, &sp->vbe->backend->troublelist, list, tr2) { + if (tr->timeout < new->timeout) { + VTAILQ_INSERT_BEFORE(tr, new, list); + new = NULL; + break; + } + } + + /* Insert the item at the end if the list is empty or all other + * items have a longer timeout. + */ + if (new) + VTAILQ_INSERT_TAIL(&sp->vbe->backend->troublelist, new, list); + + Lck_Unlock(&sp->vbe->backend->mtx); +} + int VRT_r_obj_status(const struct sess *sp) { Added: trunk/varnish-cache/bin/varnishtest/tests/s00003.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/s00003.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/s00003.vtc 2009-08-24 14:51:26 UTC (rev 4208) @@ -0,0 +1,57 @@ +# $Id$ + +test "Check saint mode with sick pages" + +server s1 -listen 127.0.0.1:9080 { + timeout 10 + + rxreq + expect req.url == "/" + txresp -status 200 -hdr "foo: 1" + rxreq + expect req.url == "/" + txresp -status 200 -hdr "foo: 2" + rxreq + expect req.url == "/" + txresp -status 200 -hdr "foo: 3" +} -start + +varnish v1 -vcl { + backend b { .host = "127.0.0.1"; .port = "9080"; } + sub vcl_fetch { + set beresp.ttl = 1s; + set beresp.grace = 10m; + set beresp.cacheable = true; + if (beresp.http.foo == "2") + { + set beresp.saintmode = 2s; + restart; + } + deliver; + } +} -start + +client c1 { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.foo == 1 +} -run + +delay 2 + +client c2 { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.foo == 1 +} -run + +delay 2 + +client c3 { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.foo == 3 +} -run Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/include/vrt_obj.h 2009-08-24 14:51:26 UTC (rev 4208) @@ -41,6 +41,7 @@ void VRT_l_bereq_between_bytes_timeout(struct sess *, double); const char * VRT_r_beresp_proto(const struct sess *); void VRT_l_beresp_proto(const struct sess *, const char *, ...); +void VRT_l_beresp_saintmode(const struct sess *, double); int VRT_r_beresp_status(const struct sess *); void VRT_l_beresp_status(const struct sess *, int); const char * VRT_r_beresp_response(const struct sess *); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08:29:27Z phk $ + * $Id$ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,10 +159,9 @@ /* ../../include/vcl.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08"); - vsb_cat(sb, ":29:27Z phk $\n *\n * NB: This file is machine genera"); - vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); - vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sess;\n"); + vsb_cat(sb, "/*\n * $Id$\n *\n * NB: This file is machine generate"); + vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok"); + vsb_cat(sb, "en.tcl instead\n */\n\nstruct sess;\n"); vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n"); vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n"); @@ -228,15 +227,15 @@ vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI"); vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT"); vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"); - vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4185 2009-08-17 11:"); - vsb_cat(sb, "53:01Z phk $\n *\n * Runtime support for compiled VCL "); - vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/"); - vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n"); - vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n"); - vsb_cat(sb, "\nstruct sess;\nstruct vsb;\nstruct cli;\n"); - vsb_cat(sb, "struct director;\nstruct VCL_conf;\n"); - vsb_cat(sb, "struct sockaddr;\n\n/*\n * A backend probe specificati"); - vsb_cat(sb, "on\n */\n\nextern void *vrt_magic_string_end;\n"); + vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id$\n *\n"); + vsb_cat(sb, " * Runtime support for compiled VCL programs.\n"); + vsb_cat(sb, " *\n * XXX: When this file is changed, lib/libvcl/vcc_"); + vsb_cat(sb, "gen_fixed_token.tcl\n * XXX: *MUST* be rerun.\n"); + vsb_cat(sb, " */\n\nstruct sess;\nstruct vsb;\n"); + vsb_cat(sb, "struct cli;\nstruct director;\n"); + vsb_cat(sb, "struct VCL_conf;\nstruct sockaddr;\n"); + vsb_cat(sb, "\n/*\n * A backend probe specification\n"); + vsb_cat(sb, " */\n\nextern void *vrt_magic_string_end;\n"); vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n"); vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n"); vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n"); @@ -317,26 +316,25 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08"); - vsb_cat(sb, ":29:27Z phk $\n *\n * NB: This file is machine genera"); - vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t"); - vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sockaddr * VRT_r_clien"); - vsb_cat(sb, "t_ip(const struct sess *);\nstruct sockaddr * VRT_r_se"); - vsb_cat(sb, "rver_ip(struct sess *);\nconst char * VRT_r_server_hos"); - vsb_cat(sb, "tname(struct sess *);\nconst char * VRT_r_server_ident"); - vsb_cat(sb, "ity(struct sess *);\nint VRT_r_server_port(struct sess"); - vsb_cat(sb, " *);\nconst char * VRT_r_req_request(const struct sess"); - vsb_cat(sb, " *);\nvoid VRT_l_req_request(const struct sess *, cons"); - vsb_cat(sb, "t char *, ...);\nconst char * VRT_r_req_url(const stru"); - vsb_cat(sb, "ct sess *);\nvoid VRT_l_req_url(const struct sess *, c"); - vsb_cat(sb, "onst char *, ...);\nconst char * VRT_r_req_proto(const"); - vsb_cat(sb, " struct sess *);\nvoid VRT_l_req_proto(const struct se"); - vsb_cat(sb, "ss *, const char *, ...);\nvoid VRT_l_req_hash(struct "); - vsb_cat(sb, "sess *, const char *);\nstruct director * VRT_r_req_ba"); - vsb_cat(sb, "ckend(struct sess *);\nvoid VRT_l_req_backend(struct s"); - vsb_cat(sb, "ess *, struct director *);\nint VRT_r_req_restarts(con"); - vsb_cat(sb, "st struct sess *);\ndouble VRT_r_req_grace(struct sess"); - vsb_cat(sb, " *);\nvoid VRT_l_req_grace(struct sess *, double);\n"); + vsb_cat(sb, "/*\n * $Id$\n *\n * NB: This file is machine generate"); + vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok"); + vsb_cat(sb, "en.tcl instead\n */\n\nstruct sockaddr * VRT_r_client_"); + vsb_cat(sb, "ip(const struct sess *);\nstruct sockaddr * VRT_r_serv"); + vsb_cat(sb, "er_ip(struct sess *);\nconst char * VRT_r_server_hostn"); + vsb_cat(sb, "ame(struct sess *);\nconst char * VRT_r_server_identit"); + vsb_cat(sb, "y(struct sess *);\nint VRT_r_server_port(struct sess *"); + vsb_cat(sb, ");\nconst char * VRT_r_req_request(const struct sess *"); + vsb_cat(sb, ");\nvoid VRT_l_req_request(const struct sess *, const "); + vsb_cat(sb, "char *, ...);\nconst char * VRT_r_req_url(const struct"); + vsb_cat(sb, " sess *);\nvoid VRT_l_req_url(const struct sess *, con"); + vsb_cat(sb, "st char *, ...);\nconst char * VRT_r_req_proto(const s"); + vsb_cat(sb, "truct sess *);\nvoid VRT_l_req_proto(const struct sess"); + vsb_cat(sb, " *, const char *, ...);\nvoid VRT_l_req_hash(struct se"); + vsb_cat(sb, "ss *, const char *);\nstruct director * VRT_r_req_back"); + vsb_cat(sb, "end(struct sess *);\nvoid VRT_l_req_backend(struct ses"); + vsb_cat(sb, "s *, struct director *);\nint VRT_r_req_restarts(const"); + vsb_cat(sb, " struct sess *);\ndouble VRT_r_req_grace(struct sess *"); + vsb_cat(sb, ");\nvoid VRT_l_req_grace(struct sess *, double);\n"); vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n"); vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n"); vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n"); @@ -357,7 +355,8 @@ vsb_cat(sb, "(struct sess *, double);\nconst char * VRT_r_beresp_pr"); vsb_cat(sb, "oto(const struct sess *);\nvoid VRT_l_beresp_proto(con"); vsb_cat(sb, "st struct sess *, const char *, ...);\n"); - vsb_cat(sb, "int VRT_r_beresp_status(const struct sess *);\n"); + vsb_cat(sb, "void VRT_l_beresp_saintmode(const struct sess *, doubl"); + vsb_cat(sb, "e);\nint VRT_r_beresp_status(const struct sess *);\n"); vsb_cat(sb, "void VRT_l_beresp_status(const struct sess *, int);\n"); vsb_cat(sb, "const char * VRT_r_beresp_response(const struct sess *"); vsb_cat(sb, ");\nvoid VRT_l_beresp_response(const struct sess *, co"); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-08-24 14:51:26 UTC (rev 4208) @@ -219,6 +219,11 @@ { fetch } "const struct sess *" } + { beresp.saintmode TIME + WO + { fetch } + "const struct sess *" + } { beresp.status INT RW { fetch } Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-20 12:11:47 UTC (rev 4207) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-08-24 14:51:26 UTC (rev 4208) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08:29:27Z phk $ + * $Id$ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,6 +159,11 @@ V_RW, 0, VCL_MET_FETCH }, + { "beresp.saintmode", TIME, 16, + NULL, "VRT_l_beresp_saintmode(sp, ", + V_WO, 0, + VCL_MET_FETCH + }, { "beresp.status", INT, 13, "VRT_r_beresp_status(sp)", "VRT_l_beresp_status(sp, ", V_RW, 0, From phk at projects.linpro.no Wed Aug 26 07:25:55 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 26 Aug 2009 09:25:55 +0200 (CEST) Subject: r4209 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20090826072555.33D3F1F7230@projects.linpro.no> Author: phk Date: 2009-08-26 09:25:54 +0200 (Wed, 26 Aug 2009) New Revision: 4209 Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/purge_vars.h trunk/varnish-cache/lib/libvcl/vcc_action.c Log: Silence some FlexeLint warnings Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-24 14:51:26 UTC (rev 4208) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-26 07:25:54 UTC (rev 4209) @@ -235,7 +235,7 @@ unsigned flag; ban_cond_f *func; } pvars[] = { -#define PVAR(a, b, c) { a, b, c }, +#define PVAR(a, b, c) { (a), (b), (c) }, #include "purge_vars.h" #undef PVAR { 0, 0, 0} @@ -286,9 +286,9 @@ if (strncmp(a1, pv->name, strlen(pv->name))) continue; bt->func = pv->func; - if (pv->flag & 2) + if (pv->flag & PVAR_REQ) b->flags |= BAN_F_REQ; - if (pv->flag & 1) + if (pv->flag & PVAR_HTTP) ban_parse_http(bt, a1 + strlen(pv->name)); break; } Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-08-24 14:51:26 UTC (rev 4208) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-08-26 07:25:54 UTC (rev 4209) @@ -307,6 +307,7 @@ assert(a > 0); ALLOC_OBJ(new, TROUBLE_MAGIC); + AN(new); new->objhead = sp->objhead; new->timeout = sp->t_req + a; Modified: trunk/varnish-cache/include/purge_vars.h =================================================================== --- trunk/varnish-cache/include/purge_vars.h 2009-08-24 14:51:26 UTC (rev 4208) +++ trunk/varnish-cache/include/purge_vars.h 2009-08-26 07:25:54 UTC (rev 4209) @@ -29,12 +29,12 @@ * * Define which variables we can purge on, and which function does it. * - * Middle field is a bitmap: - * 1 = HTTP header search - * 2 = Needs a client supplied request for test evaluation */ -PVAR("req.url", 2|0, ban_cond_url) -PVAR("obj.hash", 0|0, ban_cond_hash) -PVAR("req.http.", 2|1, ban_cond_req_http) -PVAR("obj.http.", 0|1, ban_cond_obj_http) +#define PVAR_HTTP 1 +#define PVAR_REQ 2 + +PVAR("req.url", PVAR_REQ, ban_cond_url) +PVAR("obj.hash", 0, ban_cond_hash) +PVAR("req.http.", PVAR_REQ|PVAR_HTTP, ban_cond_req_http) +PVAR("obj.http.", PVAR_HTTP, ban_cond_obj_http) Modified: trunk/varnish-cache/lib/libvcl/vcc_action.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_action.c 2009-08-24 14:51:26 UTC (rev 4208) +++ trunk/varnish-cache/lib/libvcl/vcc_action.c 2009-08-26 07:25:54 UTC (rev 4209) @@ -350,7 +350,7 @@ const char *name; unsigned flag; } purge_var[] = { -#define PVAR(a, b, c) { a, b }, +#define PVAR(a, b, c) { (a), (b) }, #include "purge_vars.h" #undef PVAR { 0, 0 } @@ -383,7 +383,7 @@ vcc_ErrWhere(tl, tl->t); return; } - if ((pv->flag & 1) && + if ((pv->flag & PVAR_HTTP) && tl->t->b + strlen(pv->name) >= tl->t->e) { vsb_printf(tl->sb, "Missing header name."); vcc_ErrWhere(tl, tl->t); From phk at projects.linpro.no Wed Aug 26 11:12:48 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 26 Aug 2009 13:12:48 +0200 (CEST) Subject: r4210 - trunk/varnish-cache/bin/varnishd Message-ID: <20090826111248.5978A38C51@projects.linpro.no> Author: phk Date: 2009-08-26 13:12:48 +0200 (Wed, 26 Aug 2009) New Revision: 4210 Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Factor getdtablesize() out of close loop. Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2009-08-26 07:25:54 UTC (rev 4209) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2009-08-26 11:12:48 UTC (rev 4210) @@ -235,7 +235,7 @@ unsigned u; char *p; struct vev *e; - int i, cp[2]; + int i, j, cp[2]; if (child_state != CH_STOPPED && child_state != CH_DIED) return; @@ -303,7 +303,8 @@ /* Close anything we shouldn't know about */ closelog(); printf("Closed fds:"); - for (i = STDERR_FILENO + 1; i < getdtablesize(); i++) { + j = getdtablesize(); + for (i = STDERR_FILENO + 1; i < j; i++) { if (vbit_test(fd_map, i)) continue; if (close(i) == 0) Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-08-26 07:25:54 UTC (rev 4209) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-08-26 11:12:48 UTC (rev 4210) @@ -314,7 +314,8 @@ pipes[1][1] = 1; /* close the rest */ - for (i = 5; i < getdtablesize(); i++) + j = getdtablesize(); + for (i = 5; i < j; i++) (void)close(i); pfd[0].fd = pipes[0][0]; From phk at projects.linpro.no Wed Aug 26 21:13:17 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 26 Aug 2009 23:13:17 +0200 (CEST) Subject: r4211 - trunk/varnish-cache/bin/varnishd Message-ID: <20090826211317.41DF71F72C0@projects.linpro.no> Author: phk Date: 2009-08-26 23:13:16 +0200 (Wed, 26 Aug 2009) New Revision: 4211 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Counter-intuitively change the LRU list from a VTAILQ to a VLIST. The cruical difference is that we can remove from a VLIST without having the head of the VLIST, which comes handy when expiring a non-instantiated object from a -spersistent stevedore. We flip the LRU list around, so that the frontmost element has not been accessed for the longest time, and simulate tail-inserts by putting a senteniel on the tail of the list. Presently the senteniel is a full objcore, which is a non-issue with few machines having more than a handful of stevedores, but once the persistent stevedore moves to per-segment LRU lists, this needs to be addressed, probably by wrapping the LRU linkage in a separate structure. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-26 11:12:48 UTC (rev 4210) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-26 21:13:16 UTC (rev 4211) @@ -290,7 +290,7 @@ #define OC_F_PERSISTENT (1<<3) unsigned timer_idx; VTAILQ_ENTRY(objcore) list; - VTAILQ_ENTRY(objcore) lru_list; + VLIST_ENTRY(objcore) lru_list; VTAILQ_ENTRY(objcore) ban_list; struct smp_seg *smp_seg; struct ban *ban; @@ -488,11 +488,11 @@ /* cache_expiry.c */ void EXP_Insert(struct object *o); -void EXP_Inject(struct objcore *oc, struct objcore_head *lru, double ttl); +void EXP_Inject(struct objcore *oc, struct objcore *lrut, double ttl); void EXP_Init(void); void EXP_Rearm(const struct object *o); int EXP_Touch(const struct object *o); -int EXP_NukeOne(struct sess *sp, struct objcore_head *lru); +int EXP_NukeOne(struct sess *sp, const struct objcore_head *lru); /* cache_fetch.c */ int FetchHdr(struct sess *sp); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-26 11:12:48 UTC (rev 4210) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-26 21:13:16 UTC (rev 4211) @@ -97,7 +97,7 @@ */ void -EXP_Inject(struct objcore *oc, struct objcore_head *lru, double ttl) +EXP_Inject(struct objcore *oc, struct objcore *lrut, double ttl) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); @@ -107,10 +107,8 @@ oc->timer_when = ttl; binheap_insert(exp_heap, oc); assert(oc->timer_idx != BINHEAP_NOIDX); - if (lru != NULL) { - VTAILQ_INSERT_TAIL(lru, oc, lru_list); - oc->flags |= OC_F_ONLRU; - } + VLIST_INSERT_BEFORE(lrut, oc, lru_list); + oc->flags |= OC_F_ONLRU; Lck_Unlock(&exp_mtx); } @@ -124,8 +122,7 @@ void EXP_Insert(struct object *o) { - struct objcore *oc; - struct objcore_head *lru; + struct objcore *oc, *lrut; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); @@ -136,14 +133,15 @@ assert(o->entered != 0 && !isnan(o->entered)); o->last_lru = o->entered; + lrut = STV_lru(o->objstore); Lck_Lock(&exp_mtx); assert(oc->timer_idx == BINHEAP_NOIDX); (void)update_object_when(o); binheap_insert(exp_heap, oc); assert(oc->timer_idx != BINHEAP_NOIDX); - lru = STV_lru(o->objstore); - if (lru != NULL) { - VTAILQ_INSERT_TAIL(lru, oc, lru_list); + if (o->objstore != NULL) { + CHECK_OBJ_NOTNULL(lrut, OBJCORE_MAGIC); + VLIST_INSERT_BEFORE(lrut, oc, lru_list); oc->flags |= OC_F_ONLRU; } Lck_Unlock(&exp_mtx); @@ -163,24 +161,25 @@ int EXP_Touch(const struct object *o) { - struct objcore *oc; int retval = 0; - struct objcore_head *lru; + struct objcore *oc, *lrut; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = o->objcore; if (oc == NULL) return (retval); - lru = STV_lru(o->objstore); - if (lru == NULL) - return (retval); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + /* We must have an objhead, otherwise we have no business on a LRU */ CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); + if (o->objstore == NULL) /* XXX ?? */ + return (retval); + lrut = STV_lru(o->objstore); + CHECK_OBJ_NOTNULL(lrut, OBJCORE_MAGIC); if (Lck_Trylock(&exp_mtx)) return (retval); - if (oc->flags & OC_F_ONLRU) { - VTAILQ_REMOVE(lru, oc, lru_list); - VTAILQ_INSERT_TAIL(lru, oc, lru_list); + if (oc->flags & OC_F_ONLRU) { /* XXX ?? */ + VLIST_REMOVE(oc, lru_list); + VLIST_INSERT_BEFORE(lrut, oc, lru_list); VSL_stats->n_lru_moved++; retval = 1; } @@ -243,7 +242,6 @@ struct objcore *oc; struct object *o; double t; - struct objcore_head *lru; (void)priv; VCL_Get(&sp->vcl); @@ -273,16 +271,9 @@ /* And from LRU */ if (oc->flags & OC_F_ONLRU) { - assert(!(oc->flags & OC_F_PERSISTENT)); - o = oc->obj; - lru = STV_lru(o->objstore); - AN(lru); - VTAILQ_REMOVE(lru, o->objcore, lru_list); + VLIST_REMOVE(oc, lru_list); oc->flags &= ~OC_F_ONLRU; - } else { - o = NULL; - assert(oc->flags & OC_F_PERSISTENT); - } + } VSL_stats->n_expired++; @@ -296,8 +287,8 @@ o->xid, (int)(o->ttl - t)); HSH_Deref(sp->wrk, &o); } else { - WSL(sp->wrk, SLT_ExpKill, 1, "%u %d", - o, (int)(oc->timer_when - t)); + WSL(sp->wrk, SLT_ExpKill, 1, "-1 %d", + (int)(oc->timer_when - t)); sp->objhead = oc->objhead; sp->objcore = oc; HSH_DerefObjCore(sp); @@ -312,7 +303,7 @@ */ int -EXP_NukeOne(struct sess *sp, struct objcore_head *lru) +EXP_NukeOne(struct sess *sp, const struct objcore_head *lru) { struct objcore *oc; @@ -327,7 +318,7 @@ * */ Lck_Lock(&exp_mtx); - VTAILQ_FOREACH(oc, lru, lru_list) { + VLIST_FOREACH(oc, lru, lru_list) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); if (oc->timer_idx == BINHEAP_NOIDX) /* exp_timer has it */ continue; @@ -335,7 +326,7 @@ break; } if (oc != NULL) { - VTAILQ_REMOVE(lru, oc, lru_list); + VLIST_REMOVE(oc, lru_list); oc->flags &= ~OC_F_ONLRU; binheap_delete(exp_heap, oc->timer_idx); assert(oc->timer_idx == BINHEAP_NOIDX); Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2009-08-26 11:12:48 UTC (rev 4210) +++ trunk/varnish-cache/bin/varnishd/common.h 2009-08-26 21:13:16 UTC (rev 4211) @@ -67,5 +67,5 @@ /* Sort of hack-ish... */ struct objcore; -VTAILQ_HEAD(objcore_head, objcore); +VLIST_HEAD(objcore_head, objcore); Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-08-26 11:12:48 UTC (rev 4210) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-08-26 21:13:16 UTC (rev 4211) @@ -124,7 +124,9 @@ *stv = *stv2; AN(stv->name); AN(stv->alloc); - VTAILQ_INIT(&stv->lru); + ALLOC_OBJ(stv->lru_tail, OBJCORE_MAGIC); + VLIST_INIT(&stv->lru); + VLIST_INSERT_HEAD(&stv->lru, stv->lru_tail, lru_list); if (stv->init != NULL) stv->init(stv, ac, av); @@ -159,14 +161,12 @@ } } -struct objcore_head * +struct objcore * STV_lru(struct storage *st) { - if (st == NULL) - return (NULL); - CHECK_OBJ(st, STORAGE_MAGIC); + CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); - return (&st->stevedore->lru); + return (st->stevedore->lru_tail); } const struct choice STV_choice[] = { Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-08-26 11:12:48 UTC (rev 4210) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-08-26 21:13:16 UTC (rev 4211) @@ -55,6 +55,7 @@ storage_close_f *close; struct objcore_head lru; + struct objcore *lru_tail; /* private fields */ void *priv; @@ -68,8 +69,9 @@ void STV_add(const struct stevedore *stv, int ac, char * const *av); void STV_open(void); void STV_close(void); -struct objcore_head *STV_lru(struct storage *st); +struct objcore *STV_lru(struct storage *st); + int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-26 11:12:48 UTC (rev 4210) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-26 21:13:16 UTC (rev 4211) @@ -113,6 +113,7 @@ struct smp_sc { unsigned magic; #define SMP_SC_MAGIC 0x7b73af0a + struct stevedore *parent; unsigned flags; #define SMP_F_LOADED (1 << 0) @@ -519,6 +520,7 @@ /* Allocate softc */ ALLOC_OBJ(sc, SMP_SC_MAGIC); XXXAN(sc); + sc->parent = parent; sc->fd = -1; VTAILQ_INIT(&sc->segments); @@ -834,7 +836,7 @@ oc->ban = BAN_RefBan(oc, so->ban, sc->tailban); memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN); (void)HSH_Insert(sp); - EXP_Inject(oc, NULL, so->ttl); + EXP_Inject(oc, sc->parent->lru_tail, so->ttl); sg->nalloc++; } WRK_SumStat(sp->wrk); From phk at projects.linpro.no Wed Aug 26 23:01:06 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 27 Aug 2009 01:01:06 +0200 (CEST) Subject: r4212 - trunk/varnish-cache/bin/varnishd Message-ID: <20090826230106.BC4A628503@projects.linpro.no> Author: phk Date: 2009-08-27 01:01:06 +0200 (Thu, 27 Aug 2009) New Revision: 4212 Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Fix an reference count issue relating to non-instantiated objects. Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-26 21:13:16 UTC (rev 4211) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-26 23:01:06 UTC (rev 4212) @@ -506,6 +506,7 @@ struct ban *b, *bf; struct objcore *oc; struct object *o; + int i; (void)priv; while (1) { @@ -542,9 +543,10 @@ TIM_sleep(1.0); continue; } + AZ(oc->flags & OC_F_PERSISTENT); o = oc->obj; - (void)ban_check_object(o, sp, 0); - WSP(sp, SLT_Debug, "lurker: %p %g", oc, o->ttl); + i = ban_check_object(o, sp, 0); + WSP(sp, SLT_Debug, "lurker: %p %g %d", oc, o->ttl, i); HSH_Deref(sp->wrk, &o); TIM_sleep(params->ban_lurker_sleep); } Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-26 21:13:16 UTC (rev 4211) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-26 23:01:06 UTC (rev 4212) @@ -101,6 +101,7 @@ { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(lrut, OBJCORE_MAGIC); Lck_Lock(&exp_mtx); assert(oc->timer_idx == BINHEAP_NOIDX); @@ -260,7 +261,6 @@ continue; } - /* It's time... */ CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-26 21:13:16 UTC (rev 4211) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-26 23:01:06 UTC (rev 4212) @@ -401,6 +401,7 @@ /* Insert (precreated) objcore in objecthead */ oc = w->nobjcore; w->nobjcore = NULL; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->flags & OC_F_BUSY); /* XXX: Should this not be ..._HEAD now ? */ @@ -681,13 +682,13 @@ CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_Lock(&oh->mtx); VTAILQ_FOREACH(oc2, &oh->objcs, list) - if (oc1 == oc2) { - oc1->obj->refcnt++; + if (oc1 == oc2) break; - } if (oc2 != NULL && oc2->flags & OC_F_PERSISTENT) SMP_Fixup(sp, oh, oc2); - Lck_Unlock(&oc1->objhead->mtx); + if (oc2 != NULL) + oc2->obj->refcnt++; + Lck_Unlock(&oh->mtx); *oc = oc2; } Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-26 21:13:16 UTC (rev 4211) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-26 23:01:06 UTC (rev 4212) @@ -836,6 +836,7 @@ oc->ban = BAN_RefBan(oc, so->ban, sc->tailban); memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN); (void)HSH_Insert(sp); + AZ(sp->wrk->nobjcore); EXP_Inject(oc, sc->parent->lru_tail, so->ttl); sg->nalloc++; } @@ -993,6 +994,7 @@ sc->flags |= SMP_F_LOADED; BAN_Deref(&sc->tailban); sc->tailban = NULL; + printf("Silo completely loaded\n"); while (1) sleep (1); return (NULL); From phk at projects.linpro.no Thu Aug 27 16:34:23 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 27 Aug 2009 18:34:23 +0200 (CEST) Subject: r4213 - trunk/varnish-cache/bin/varnishd Message-ID: <20090827163423.D869F1F72B8@projects.linpro.no> Author: phk Date: 2009-08-27 18:34:23 +0200 (Thu, 27 Aug 2009) New Revision: 4213 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Clean up the LRU stuff a bit, wrap the senteniel with the list head etc. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-26 23:01:06 UTC (rev 4212) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-27 16:34:23 UTC (rev 4213) @@ -296,6 +296,15 @@ struct ban *ban; }; +/*--------------------------------------------------------------------*/ + +struct lru { + unsigned magic; +#define LRU_MAGIC 0x3fec7bb0 + VLIST_HEAD(,objcore) lru_head; + struct objcore senteniel; +}; + /* Object structure --------------------------------------------------*/ struct object { @@ -488,11 +497,11 @@ /* cache_expiry.c */ void EXP_Insert(struct object *o); -void EXP_Inject(struct objcore *oc, struct objcore *lrut, double ttl); +void EXP_Inject(struct objcore *oc, struct lru *lru, double ttl); void EXP_Init(void); void EXP_Rearm(const struct object *o); int EXP_Touch(const struct object *o); -int EXP_NukeOne(struct sess *sp, const struct objcore_head *lru); +int EXP_NukeOne(struct sess *sp, const struct lru *lru); /* cache_fetch.c */ int FetchHdr(struct sess *sp); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-26 23:01:06 UTC (rev 4212) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-27 16:34:23 UTC (rev 4213) @@ -97,18 +97,18 @@ */ void -EXP_Inject(struct objcore *oc, struct objcore *lrut, double ttl) +EXP_Inject(struct objcore *oc, struct lru *lru, double ttl) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - CHECK_OBJ_NOTNULL(lrut, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); Lck_Lock(&exp_mtx); assert(oc->timer_idx == BINHEAP_NOIDX); oc->timer_when = ttl; binheap_insert(exp_heap, oc); assert(oc->timer_idx != BINHEAP_NOIDX); - VLIST_INSERT_BEFORE(lrut, oc, lru_list); + VLIST_INSERT_BEFORE(&lru->senteniel, oc, lru_list); oc->flags |= OC_F_ONLRU; Lck_Unlock(&exp_mtx); } @@ -123,7 +123,8 @@ void EXP_Insert(struct object *o) { - struct objcore *oc, *lrut; + struct objcore *oc; + struct lru *lru; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); @@ -134,15 +135,15 @@ assert(o->entered != 0 && !isnan(o->entered)); o->last_lru = o->entered; - lrut = STV_lru(o->objstore); Lck_Lock(&exp_mtx); assert(oc->timer_idx == BINHEAP_NOIDX); (void)update_object_when(o); binheap_insert(exp_heap, oc); assert(oc->timer_idx != BINHEAP_NOIDX); if (o->objstore != NULL) { - CHECK_OBJ_NOTNULL(lrut, OBJCORE_MAGIC); - VLIST_INSERT_BEFORE(lrut, oc, lru_list); + lru = STV_lru(o->objstore); + CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); + VLIST_INSERT_BEFORE(&lru->senteniel, oc, lru_list); oc->flags |= OC_F_ONLRU; } Lck_Unlock(&exp_mtx); @@ -163,7 +164,8 @@ EXP_Touch(const struct object *o) { int retval = 0; - struct objcore *oc, *lrut; + struct objcore *oc; + struct lru *lru; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = o->objcore; @@ -174,13 +176,13 @@ CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); if (o->objstore == NULL) /* XXX ?? */ return (retval); - lrut = STV_lru(o->objstore); - CHECK_OBJ_NOTNULL(lrut, OBJCORE_MAGIC); + lru = STV_lru(o->objstore); + CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); if (Lck_Trylock(&exp_mtx)) return (retval); if (oc->flags & OC_F_ONLRU) { /* XXX ?? */ VLIST_REMOVE(oc, lru_list); - VLIST_INSERT_BEFORE(lrut, oc, lru_list); + VLIST_INSERT_BEFORE(&lru->senteniel, oc, lru_list); VSL_stats->n_lru_moved++; retval = 1; } @@ -303,7 +305,7 @@ */ int -EXP_NukeOne(struct sess *sp, const struct objcore_head *lru) +EXP_NukeOne(struct sess *sp, const struct lru *lru) { struct objcore *oc; @@ -318,7 +320,12 @@ * */ Lck_Lock(&exp_mtx); - VLIST_FOREACH(oc, lru, lru_list) { + VLIST_FOREACH(oc, &lru->lru_head, lru_list) { + if (oc == &lru->senteniel) { + AZ(VLIST_NEXT(oc, lru_list)); + oc = NULL; + break; + } CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); if (oc->timer_idx == BINHEAP_NOIDX) /* exp_timer has it */ continue; Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2009-08-26 23:01:06 UTC (rev 4212) +++ trunk/varnish-cache/bin/varnishd/common.h 2009-08-27 16:34:23 UTC (rev 4213) @@ -64,8 +64,3 @@ const char *name; void *ptr; }; - -/* Sort of hack-ish... */ -struct objcore; -VLIST_HEAD(objcore_head, objcore); - Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-08-26 23:01:06 UTC (rev 4212) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-08-27 16:34:23 UTC (rev 4213) @@ -43,6 +43,18 @@ static const struct stevedore * volatile stv_next; +static struct lru * +LRU_Alloc(void) +{ + struct lru *l; + + ALLOC_OBJ(l, LRU_MAGIC); + AN(l); + VLIST_INIT(&l->lru_head); + VLIST_INSERT_HEAD(&l->lru_head, &l->senteniel, lru_list); + return (l); +} + struct storage * STV_alloc(struct sess *sp, size_t size) { @@ -81,7 +93,7 @@ break; /* no luck; try to free some space and keep trying */ - if (EXP_NukeOne(sp, &stv->lru) == -1) + if (EXP_NukeOne(sp, stv->lru) == -1) break; /* Enough is enough: try another if we have one */ @@ -124,9 +136,7 @@ *stv = *stv2; AN(stv->name); AN(stv->alloc); - ALLOC_OBJ(stv->lru_tail, OBJCORE_MAGIC); - VLIST_INIT(&stv->lru); - VLIST_INSERT_HEAD(&stv->lru, stv->lru_tail, lru_list); + stv->lru = LRU_Alloc(); if (stv->init != NULL) stv->init(stv, ac, av); @@ -161,12 +171,12 @@ } } -struct objcore * -STV_lru(struct storage *st) +struct lru * +STV_lru(const struct storage *st) { CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); - return (st->stevedore->lru_tail); + return (st->stevedore->lru); } const struct choice STV_choice[] = { Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-08-26 23:01:06 UTC (rev 4212) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-08-27 16:34:23 UTC (rev 4213) @@ -42,6 +42,7 @@ typedef void storage_object_f(const struct sess *sp); typedef void storage_close_f(const struct stevedore *); + struct stevedore { unsigned magic; #define STEVEDORE_MAGIC 0x4baf43db @@ -54,8 +55,7 @@ storage_object_f *object; storage_close_f *close; - struct objcore_head lru; - struct objcore *lru_tail; + struct lru *lru; /* private fields */ void *priv; @@ -69,7 +69,7 @@ void STV_add(const struct stevedore *stv, int ac, char * const *av); void STV_open(void); void STV_close(void); -struct objcore *STV_lru(struct storage *st); +struct lru *STV_lru(const struct storage *st); int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-26 23:01:06 UTC (rev 4212) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-27 16:34:23 UTC (rev 4213) @@ -837,7 +837,7 @@ memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN); (void)HSH_Insert(sp); AZ(sp->wrk->nobjcore); - EXP_Inject(oc, sc->parent->lru_tail, so->ttl); + EXP_Inject(oc, sc->parent->lru, so->ttl); sg->nalloc++; } WRK_SumStat(sp->wrk); From petter at projects.linpro.no Fri Aug 28 14:19:41 2009 From: petter at projects.linpro.no (petter at projects.linpro.no) Date: Fri, 28 Aug 2009 16:19:41 +0200 (CEST) Subject: r4214 - trunk/varnish-cache/bin/varnishd Message-ID: <20090828141941.8A9FA1F72B8@projects.linpro.no> Author: petter Date: 2009-08-28 16:19:41 +0200 (Fri, 28 Aug 2009) New Revision: 4214 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c trunk/varnish-cache/bin/varnishd/hash_critbit.c Log: gcc 4.4.1 really expects the code to go to infinity and beyond. Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -268,6 +268,7 @@ WRK_QueueSession(sp); } } + return NULL; } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -550,6 +550,7 @@ HSH_Deref(sp->wrk, &o); TIM_sleep(params->ban_lurker_sleep); } + return NULL; } Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -296,6 +296,7 @@ HSH_DerefObjCore(sp); } } + return NULL; } /*-------------------------------------------------------------------- Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -428,6 +428,7 @@ TIM_sleep(params->wthread_purge_delay * 1e-3); } + return NULL; } /*-------------------------------------------------------------------- @@ -500,6 +501,7 @@ wrk_breed_flock(wq[u]); } } + return NULL; } /*-------------------------------------------------------------------- Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_epoll.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -195,6 +195,7 @@ SES_Delete(sp); } } + return NULL; } /*--------------------------------------------------------------------*/ @@ -214,6 +215,7 @@ TIM_sleep(100 * 1e-3); assert(read(dotimer_pipe[0], &junk, 1)); } + return NULL; } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -152,6 +152,7 @@ SES_Delete(sp); } } + return NULL; } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-27 16:34:23 UTC (rev 4213) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-28 14:19:41 UTC (rev 4214) @@ -375,6 +375,7 @@ Lck_Unlock(&hcb_mtx); WRK_SumStat(&ww); } + return NULL; } /**********************************************************************/ From phk at projects.linpro.no Fri Aug 28 14:44:02 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 28 Aug 2009 16:44:02 +0200 (CEST) Subject: r4215 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090828144402.8EFFA1F72C0@projects.linpro.no> Author: phk Date: 2009-08-28 16:44:02 +0200 (Fri, 28 Aug 2009) New Revision: 4215 Modified: trunk/varnish-cache/bin/varnishd/acct_fields.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/include/stat_field.h Log: Work over the stats while I wait for -spersistent to crash: Move most of the stat fields used in cache_center.c into the workers private stats, hopefully reducing the lock contention and stats precision at the same time. Modified: trunk/varnish-cache/bin/varnishd/acct_fields.h =================================================================== --- trunk/varnish-cache/bin/varnishd/acct_fields.h 2009-08-28 14:19:41 UTC (rev 4214) +++ trunk/varnish-cache/bin/varnishd/acct_fields.h 2009-08-28 14:44:02 UTC (rev 4215) @@ -27,6 +27,10 @@ * SUCH DAMAGE. * * $Id$ + * + * These are the stats we keep track of per session. They will be summed, + * via the sp->wrk->stats into the s_<name> fields in the SHM file. + * NB: Remember to mark those in stat_field.h to be included in struct dstat. */ ACCT(sess) Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-28 14:19:41 UTC (rev 4214) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-28 14:44:02 UTC (rev 4215) @@ -105,7 +105,8 @@ i = poll(pfd, 1, params->session_linger); if (i == 0) { WSL(sp->wrk, SLT_Debug, sp->fd, "herding"); - VSL_stats->sess_herd++; + sp->wrk->stats->sess_herd++; + SES_Charge(sp); sp->wrk = NULL; vca_return_session(sp); return (1); @@ -254,9 +255,11 @@ sp->t_resp = NAN; WSL_Flush(sp->wrk, 0); - /* If we did an ESI include, don't mess up our state */ - if (sp->esis > 0) + /* If we did an ESI include, don't mess up our state */ + if (sp->esis > 0) { + SES_Charge(sp); return (1); + } sp->t_req = NAN; @@ -269,8 +272,8 @@ vca_close_session(sp, sp->doclose); } if (sp->fd < 0) { + sp->wrk->stats->sess_closed++; SES_Charge(sp); - VSL_stats->sess_closed++; sp->wrk = NULL; SES_Delete(sp); return (1); @@ -281,21 +284,21 @@ i = HTC_Reinit(sp->htc); if (i == 1) { - VSL_stats->sess_pipeline++; + sp->wrk->stats->sess_pipeline++; sp->step = STP_START; return (0); } if (Tlen(sp->htc->rxbuf)) { - VSL_stats->sess_readahead++; + sp->wrk->stats->sess_readahead++; sp->step = STP_WAIT; return (0); } if (params->session_linger > 0) { - VSL_stats->sess_linger++; + sp->wrk->stats->sess_linger++; sp->step = STP_WAIT; return (0); } - VSL_stats->sess_herd++; + sp->wrk->stats->sess_herd++; SES_Charge(sp); sp->wrk = NULL; vca_return_session(sp); @@ -745,6 +748,7 @@ * worker thread. The hash code to restart the session, * still in STP_LOOKUP, later when the busy object isn't. */ + AZ(sp->wrk); return (1); } @@ -753,7 +757,7 @@ /* If we inserted a new object it's a miss */ if (oc->flags & OC_F_BUSY) { - VSL_stats->cache_miss++; + sp->wrk->stats->cache_miss++; AZ(oc->obj); sp->objhead = oh; @@ -767,7 +771,7 @@ sp->obj = o; if (oc->flags & OC_F_PASS) { - VSL_stats->cache_hitpass++; + sp->wrk->stats->cache_hitpass++; WSP(sp, SLT_HitPass, "%u", sp->obj->xid); HSH_Deref(sp->wrk, &sp->obj); sp->objcore = NULL; @@ -776,7 +780,7 @@ return (0); } - VSL_stats->cache_hit++; + sp->wrk->stats->cache_hit++; WSP(sp, SLT_Hit, "%u", sp->obj->xid); sp->step = STP_HIT; return (0); @@ -1045,7 +1049,7 @@ AZ(sp->vcl); /* Update stats of various sorts */ - VSL_stats->client_req++; /* XXX not locked */ + sp->wrk->stats->client_req++; sp->t_req = TIM_real(); sp->wrk->lastused = sp->t_req; sp->acct_req.req++; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-28 14:19:41 UTC (rev 4214) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-28 14:44:02 UTC (rev 4215) @@ -530,6 +530,7 @@ if (params->diag_bitmap & 0x20) WSP(sp, SLT_Debug, "on waiting list <%s>", oh->hash); + SES_Charge(sp); sp->objhead = oh; sp->wrk = NULL; Lck_Unlock(&oh->mtx); Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2009-08-28 14:19:41 UTC (rev 4214) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2009-08-28 14:44:02 UTC (rev 4215) @@ -80,27 +80,17 @@ /*--------------------------------------------------------------------*/ -static void -ses_sum_acct(struct acct *sum, const struct acct *inc) -{ - -#define ACCT(foo) sum->foo += inc->foo; -#include "acct_fields.h" -#undef ACCT -} - void SES_Charge(struct sess *sp) { struct acct *a = &sp->acct_req; - ses_sum_acct(&sp->acct, a); - Lck_Lock(&stat_mtx); -#define ACCT(foo) VSL_stats->s_##foo += a->foo; +#define ACCT(foo) \ + sp->wrk->stats->s_##foo += a->foo; \ + sp->acct.foo += a->foo; \ + a->foo = 0; #include "acct_fields.h" #undef ACCT - Lck_Unlock(&stat_mtx); - memset(a, 0, sizeof *a); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-08-28 14:19:41 UTC (rev 4214) +++ trunk/varnish-cache/include/stat_field.h 2009-08-28 14:44:02 UTC (rev 4215) @@ -27,15 +27,18 @@ * SUCH DAMAGE. * * $Id$ + * + * 3rd argument marks fields for inclusion in the per worker-thread + * stats structure. */ MAC_STAT(client_conn, uint64_t, 0, 'a', "Client connections accepted") MAC_STAT(client_drop, uint64_t, 0, 'a', "Connection dropped, no sess") -MAC_STAT(client_req, uint64_t, 0, 'a', "Client requests received") +MAC_STAT(client_req, uint64_t, 1, 'a', "Client requests received") -MAC_STAT(cache_hit, uint64_t, 0, 'a', "Cache hits") -MAC_STAT(cache_hitpass, uint64_t, 0, 'a', "Cache hits for pass") -MAC_STAT(cache_miss, uint64_t, 0, 'a', "Cache misses") +MAC_STAT(cache_hit, uint64_t, 1, 'a', "Cache hits") +MAC_STAT(cache_hitpass, uint64_t, 1, 'a', "Cache hits for pass") +MAC_STAT(cache_miss, uint64_t, 1, 'a', "Cache misses") MAC_STAT(backend_conn, uint64_t, 0, 'a', "Backend conn. success") MAC_STAT(backend_unhealthy, uint64_t, 0, 'a', "Backend conn. not attempted") @@ -75,19 +78,19 @@ MAC_STAT(n_objwrite, uint64_t, 0, 'a', "Objects sent with write") MAC_STAT(n_objoverflow, uint64_t, 0, 'a', "Objects overflowing workspace") -MAC_STAT(s_sess, uint64_t, 0, 'a', "Total Sessions") -MAC_STAT(s_req, uint64_t, 0, 'a', "Total Requests") -MAC_STAT(s_pipe, uint64_t, 0, 'a', "Total pipe") -MAC_STAT(s_pass, uint64_t, 0, 'a', "Total pass") -MAC_STAT(s_fetch, uint64_t, 0, 'a', "Total fetch") -MAC_STAT(s_hdrbytes, uint64_t, 0, 'a', "Total header bytes") -MAC_STAT(s_bodybytes, uint64_t, 0, 'a', "Total body bytes") +MAC_STAT(s_sess, uint64_t, 1, 'a', "Total Sessions") +MAC_STAT(s_req, uint64_t, 1, 'a', "Total Requests") +MAC_STAT(s_pipe, uint64_t, 1, 'a', "Total pipe") +MAC_STAT(s_pass, uint64_t, 1, 'a', "Total pass") +MAC_STAT(s_fetch, uint64_t, 1, 'a', "Total fetch") +MAC_STAT(s_hdrbytes, uint64_t, 1, 'a', "Total header bytes") +MAC_STAT(s_bodybytes, uint64_t, 1, 'a', "Total body bytes") -MAC_STAT(sess_closed, uint64_t, 0, 'a', "Session Closed") -MAC_STAT(sess_pipeline, uint64_t, 0, 'a', "Session Pipeline") -MAC_STAT(sess_readahead, uint64_t, 0, 'a', "Session Read Ahead") -MAC_STAT(sess_linger, uint64_t, 0, 'a', "Session Linger") -MAC_STAT(sess_herd, uint64_t, 0, 'a', "Session herd") +MAC_STAT(sess_closed, uint64_t, 1, 'a', "Session Closed") +MAC_STAT(sess_pipeline, uint64_t, 1, 'a', "Session Pipeline") +MAC_STAT(sess_readahead, uint64_t, 1, 'a', "Session Read Ahead") +MAC_STAT(sess_linger, uint64_t, 1, 'a', "Session Linger") +MAC_STAT(sess_herd, uint64_t, 1, 'a', "Session herd") MAC_STAT(shm_records, uint64_t, 0, 'a', "SHM records") MAC_STAT(shm_writes, uint64_t, 0, 'a', "SHM writes") From phk at projects.linpro.no Fri Aug 28 15:25:12 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 28 Aug 2009 17:25:12 +0200 (CEST) Subject: r4216 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090828152512.1A6E71F72A5@projects.linpro.no> Author: phk Date: 2009-08-28 17:25:11 +0200 (Fri, 28 Aug 2009) New Revision: 4216 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/bin/varnishd/hash_critbit.c trunk/varnish-cache/bin/varnishd/hash_slinger.h trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/mgt_pool.c trunk/varnish-cache/include/stat_field.h Log: More stats work. Add a parameter, so the worker thread gets forced to dump stats into the global pool for every N requests it serves. This should solve the problem where a wget(1) client does not show up in the stats, until the session ends. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-28 15:25:11 UTC (rev 4216) @@ -199,7 +199,7 @@ #define WORKER_MAGIC 0x6391adcf struct objhead *nobjhead; struct objcore *nobjcore; - struct dstat *stats; + struct dstat stats; double lastused; @@ -592,7 +592,7 @@ void WRK_Init(void); int WRK_Queue(struct workreq *wrq); void WRK_QueueSession(struct sess *sp); -void WRK_SumStat(const struct worker *w); +void WRK_SumStat(struct worker *w); void WRW_Reserve(struct worker *w, int *fd); unsigned WRW_Flush(struct worker *w); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-28 15:25:11 UTC (rev 4216) @@ -105,7 +105,7 @@ i = poll(pfd, 1, params->session_linger); if (i == 0) { WSL(sp->wrk, SLT_Debug, sp->fd, "herding"); - sp->wrk->stats->sess_herd++; + sp->wrk->stats.sess_herd++; SES_Charge(sp); sp->wrk = NULL; vca_return_session(sp); @@ -271,35 +271,38 @@ TCP_linger(sp->fd, 0); vca_close_session(sp, sp->doclose); } + + SES_Charge(sp); + if (sp->fd < 0) { - sp->wrk->stats->sess_closed++; - SES_Charge(sp); + sp->wrk->stats.sess_closed++; sp->wrk = NULL; SES_Delete(sp); return (1); } + if (sp->wrk->stats.client_req >= params->wthread_stats_rate) + WRK_SumStat(sp->wrk); /* Reset the workspace to the session-watermark */ WS_Reset(sp->ws, sp->ws_ses); i = HTC_Reinit(sp->htc); if (i == 1) { - sp->wrk->stats->sess_pipeline++; + sp->wrk->stats.sess_pipeline++; sp->step = STP_START; return (0); } if (Tlen(sp->htc->rxbuf)) { - sp->wrk->stats->sess_readahead++; + sp->wrk->stats.sess_readahead++; sp->step = STP_WAIT; return (0); } if (params->session_linger > 0) { - sp->wrk->stats->sess_linger++; + sp->wrk->stats.sess_linger++; sp->step = STP_WAIT; return (0); } - sp->wrk->stats->sess_herd++; - SES_Charge(sp); + sp->wrk->stats.sess_herd++; sp->wrk = NULL; vca_return_session(sp); return (1); @@ -757,7 +760,7 @@ /* If we inserted a new object it's a miss */ if (oc->flags & OC_F_BUSY) { - sp->wrk->stats->cache_miss++; + sp->wrk->stats.cache_miss++; AZ(oc->obj); sp->objhead = oh; @@ -771,7 +774,7 @@ sp->obj = o; if (oc->flags & OC_F_PASS) { - sp->wrk->stats->cache_hitpass++; + sp->wrk->stats.cache_hitpass++; WSP(sp, SLT_HitPass, "%u", sp->obj->xid); HSH_Deref(sp->wrk, &sp->obj); sp->objcore = NULL; @@ -780,7 +783,7 @@ return (0); } - sp->wrk->stats->cache_hit++; + sp->wrk->stats.cache_hit++; WSP(sp, SLT_Hit, "%u", sp->obj->xid); sp->step = STP_HIT; return (0); @@ -1049,7 +1052,7 @@ AZ(sp->vcl); /* Update stats of various sorts */ - sp->wrk->stats->client_req++; + sp->wrk->stats.client_req++; sp->t_req = TIM_real(); sp->wrk->lastused = sp->t_req; sp->acct_req.req++; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-28 15:25:11 UTC (rev 4216) @@ -117,7 +117,7 @@ o->entered = NAN; VTAILQ_INIT(&o->store); VTAILQ_INIT(&o->esibits); - sp->wrk->stats->n_object++; + sp->wrk->stats.n_object++; return (o); } @@ -164,7 +164,7 @@ VTAILQ_INIT(&oh->waitinglist); Lck_New(&oh->mtx); w->nobjhead = oh; - w->stats->n_objecthead++; + w->stats.n_objecthead++; } CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC); @@ -182,18 +182,18 @@ Lck_Delete(&w->nobjhead->mtx); FREE_OBJ(w->nobjhead); w->nobjhead = NULL; - w->stats->n_objecthead--; + w->stats.n_objecthead--; } } void -HSH_DeleteObjHead(const struct worker *w, struct objhead *oh) +HSH_DeleteObjHead(struct worker *w, struct objhead *oh) { AZ(oh->refcnt); assert(VTAILQ_EMPTY(&oh->objcs)); Lck_Delete(&oh->mtx); - w->stats->n_objecthead--; + w->stats.n_objecthead--; free(oh->hash); FREE_OBJ(oh); } @@ -409,7 +409,7 @@ /* NB: do not deref objhead the new object inherits our reference */ oc->objhead = oh; Lck_Unlock(&oh->mtx); - sp->wrk->stats->n_object++; + sp->wrk->stats.n_object++; return (oc); } @@ -608,7 +608,7 @@ assert(o->refcnt > 0); assert(oh->refcnt > 0); if (o->ws_o->overflow) - VSL_stats->n_objoverflow++; + sp->wrk->stats.n_objoverflow++; if (params->diag_bitmap & 0x40) WSP(sp, SLT_Debug, "Object %u workspace free %u", o->xid, WS_Free(o->ws_o)); @@ -651,7 +651,7 @@ Lck_Lock(&oh->mtx); VTAILQ_REMOVE(&oh->objcs, oc, list); - sp->wrk->stats->n_object--; + sp->wrk->stats.n_object--; Lck_Unlock(&oh->mtx); assert(oh->refcnt > 0); FREE_OBJ(oc); @@ -694,7 +694,7 @@ } void -HSH_Deref(const struct worker *w, struct object **oo) +HSH_Deref(struct worker *w, struct object **oo) { struct object *o; struct objhead *oh; @@ -749,7 +749,7 @@ FREE_OBJ(o); } o = NULL; - w->stats->n_object--; + w->stats.n_object--; if (oc == NULL) { AZ(oh); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-28 15:25:11 UTC (rev 4216) @@ -91,23 +91,24 @@ /*--------------------------------------------------------------------*/ static void -wrk_sumstat(const struct worker *w) +wrk_sumstat(struct worker *w) { Lck_AssertHeld(&wstat_mtx); #define L0(n) -#define L1(n) (VSL_stats->n += w->stats->n) +#define L1(n) (VSL_stats->n += w->stats.n) #define MAC_STAT(n, t, l, f, d) L##l(n); #include "stat_field.h" #undef MAC_STAT #undef L0 #undef L1 - memset(w->stats, 0, sizeof *w->stats); + memset(&w->stats, 0, sizeof w->stats); } void -WRK_SumStat(const struct worker *w) +WRK_SumStat(struct worker *w) { + Lck_Lock(&wstat_mtx); wrk_sumstat(w); Lck_Unlock(&wstat_mtx); @@ -122,15 +123,12 @@ unsigned char wlog[shm_workspace]; unsigned char ws[sess_workspace]; struct SHA256Context sha256; - struct dstat stats; - unsigned stats_clean = 0; + int stats_clean; THR_SetName("cache-worker"); w = &ww; memset(w, 0, sizeof *w); - memset(&stats, 0, sizeof stats); w->magic = WORKER_MAGIC; - w->stats = &stats; w->lastused = NAN; w->wlb = w->wlp = wlog; w->wle = wlog + sizeof wlog; @@ -143,6 +141,7 @@ Lck_Lock(&qp->mtx); qp->nthr++; + stats_clean = 1; while (1) { CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); @@ -155,19 +154,17 @@ if (isnan(w->lastused)) w->lastused = TIM_real(); VTAILQ_INSERT_HEAD(&qp->idle, w, list); - if (!stats_clean) { + if (!stats_clean) WRK_SumStat(w); - stats_clean = 1; - } Lck_CondWait(&w->cond, &qp->mtx); } if (w->wrq == NULL) break; Lck_Unlock(&qp->mtx); + stats_clean = 0; AN(w->wrq); AN(w->wrq->func); w->lastused = NAN; - stats_clean = 0; WS_Reset(w->ws, NULL); w->bereq = NULL; w->beresp1 = NULL; @@ -184,14 +181,13 @@ w->wrq = NULL; if (!Lck_Trylock(&wstat_mtx)) { wrk_sumstat(w); - stats_clean = 1; Lck_Unlock(&wstat_mtx); + stats_clean = 1; } Lck_Lock(&qp->mtx); } qp->nthr--; Lck_Unlock(&qp->mtx); - AN(stats_clean); VSL(SLT_WorkThread, 0, "%p end", w); if (w->vcl != NULL) @@ -524,19 +520,16 @@ struct worker ww; struct sess *sp; unsigned char logbuf[1024]; /* XXX: size ? */ - struct dstat stats; CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); THR_SetName(bt->name); sp = SES_Alloc(NULL, 0); XXXAN(sp); memset(&ww, 0, sizeof ww); - memset(&stats, 0, sizeof stats); sp->wrk = &ww; ww.magic = WORKER_MAGIC; ww.wlp = ww.wlb = logbuf; ww.wle = logbuf + sizeof logbuf; - ww.stats = &stats; (void)bt->func(sp, bt->priv); Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2009-08-28 15:25:11 UTC (rev 4216) @@ -86,7 +86,7 @@ struct acct *a = &sp->acct_req; #define ACCT(foo) \ - sp->wrk->stats->s_##foo += a->foo; \ + sp->wrk->stats.s_##foo += a->foo; \ sp->acct.foo += a->foo; \ a->foo = 0; #include "acct_fields.h" Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-28 15:25:11 UTC (rev 4216) @@ -347,12 +347,9 @@ struct objhead *oh, *oh2; struct hcb_y *y; struct worker ww; - struct dstat stats; memset(&ww, 0, sizeof ww); - memset(&stats, 0, sizeof stats); ww.magic = WORKER_MAGIC; - ww.stats = &stats; THR_SetName("hcb_cleaner"); (void)priv; Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-28 15:25:11 UTC (rev 4216) @@ -105,8 +105,8 @@ }; extern unsigned save_hash; -void HSH_DeleteObjHead(const struct worker *w, struct objhead *oh); -void HSH_Deref(const struct worker *w, struct object **o); +void HSH_DeleteObjHead(struct worker *w, struct objhead *oh); +void HSH_Deref(struct worker *w, struct object **o); #endif /* VARNISH_CACHE_CHILD */ extern struct hash_slinger hsl_slinger; Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-28 15:25:11 UTC (rev 4216) @@ -92,6 +92,7 @@ unsigned wthread_add_delay; unsigned wthread_fail_delay; unsigned wthread_purge_delay; + unsigned wthread_stats_rate; unsigned overflow_max; Modified: trunk/varnish-cache/bin/varnishd/mgt_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_pool.c 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/bin/varnishd/mgt_pool.c 2009-08-28 15:25:11 UTC (rev 4216) @@ -165,6 +165,16 @@ "destroyed and later recreated.\n", EXPERIMENTAL, "200", "milliseconds" }, + { "thread_stats_rate", + tweak_uint, &master.wthread_stats_rate, 0, UINT_MAX, + "Worker threads accumulate statistics, and dump these into " + "the global stats counters if the lock is free when they " + "finish a request.\n" + "This parameters defines the maximum number of requests " + "a worker thread may handle, before it is forced to dump " + "its accumulated stats into the global counters.\n", + EXPERIMENTAL, + "10", "requests" }, { "overflow_max", tweak_uint, &master.overflow_max, 0, UINT_MAX, "Percentage permitted overflow queue length.\n" "\n" Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-08-28 14:44:02 UTC (rev 4215) +++ trunk/varnish-cache/include/stat_field.h 2009-08-28 15:25:11 UTC (rev 4216) @@ -76,7 +76,7 @@ MAC_STAT(n_objsendfile, uint64_t, 0, 'a', "Objects sent with sendfile") MAC_STAT(n_objwrite, uint64_t, 0, 'a', "Objects sent with write") -MAC_STAT(n_objoverflow, uint64_t, 0, 'a', "Objects overflowing workspace") +MAC_STAT(n_objoverflow, uint64_t, 1, 'a', "Objects overflowing workspace") MAC_STAT(s_sess, uint64_t, 1, 'a', "Total Sessions") MAC_STAT(s_req, uint64_t, 1, 'a', "Total Requests") From phk at projects.linpro.no Mon Aug 31 09:10:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Aug 2009 11:10:03 +0200 (CEST) Subject: r4217 - trunk/varnish-cache/bin/varnishd Message-ID: <20090831091003.EC07A1F72D5@projects.linpro.no> Author: phk Date: 2009-08-31 11:10:02 +0200 (Mon, 31 Aug 2009) New Revision: 4217 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/hash_critbit.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Wrap needless returns in a macro that declares them as such, so that I can tell FlexeLint to ignore them. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-31 09:10:02 UTC (rev 4217) @@ -753,3 +753,4 @@ CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); return (o->objcore->flags & OC_F_BUSY); } + Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -268,7 +268,7 @@ WRK_QueueSession(sp); } } - return NULL; + NEEDLESS_RETURN(NULL); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -550,7 +550,7 @@ HSH_Deref(sp->wrk, &o); TIM_sleep(params->ban_lurker_sleep); } - return NULL; + NEEDLESS_RETURN(NULL); } Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -296,7 +296,7 @@ HSH_DerefObjCore(sp); } } - return NULL; + NEEDLESS_RETURN(NULL); } /*-------------------------------------------------------------------- Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -424,7 +424,7 @@ TIM_sleep(params->wthread_purge_delay * 1e-3); } - return NULL; + NEEDLESS_RETURN(NULL); } /*-------------------------------------------------------------------- @@ -497,7 +497,7 @@ wrk_breed_flock(wq[u]); } } - return NULL; + NEEDLESS_RETURN(NULL); } /*-------------------------------------------------------------------- @@ -535,7 +535,7 @@ WRONG("BgThread terminated"); - return (NULL); + NEEDLESS_RETURN(NULL); } void Modified: trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/cache_waiter_poll.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -152,7 +152,7 @@ SES_Delete(sp); } } - return NULL; + NEEDLESS_RETURN(NULL); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/common.h 2009-08-31 09:10:02 UTC (rev 4217) @@ -64,3 +64,5 @@ const char *name; void *ptr; }; + +#define NEEDLESS_RETURN(foo) return (foo) Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2009-08-31 09:10:02 UTC (rev 4217) @@ -55,6 +55,7 @@ -efile(766, ../../config.h) -emacro(413, offsetof) // likely null pointer -emacro(527, WRONG) // unreachable code +-emacro(527, NEEDLESS_RETURN) // unreachable code -emacro(702, WEXITSTATUS) // signed shift right -efunc(525, VCC_Return_Name) // Negative indent Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -372,7 +372,7 @@ Lck_Unlock(&hcb_mtx); WRK_SumStat(&ww); } - return NULL; + NEEDLESS_RETURN(NULL); } /**********************************************************************/ Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-28 15:25:11 UTC (rev 4216) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-31 09:10:02 UTC (rev 4217) @@ -997,7 +997,7 @@ printf("Silo completely loaded\n"); while (1) sleep (1); - return (NULL); + NEEDLESS_RETURN(NULL); } /*--------------------------------------------------------------------