Code Coverage: vinyl-cache/bin/vinyld/http1/cache_http1_fetch.c
 
82217aa4fc7094b3610340078afaf78cd95a8ec9
 
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2015 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 */
30
31
#include "config.h"
32
33
#include "cache/cache_int.h"
34
#include "cache/cache_filter.h"
35
36
#include <stdio.h>
37
#include <stdlib.h>
38
39
#include "vtcp.h"
40
#include "vtim.h"
41
42
#include "cache_http1.h"
43
44
static int
45
v1f_stackv1l(struct vdp_ctx *vdc, struct busyobj *bo, struct v1l *v1l)
46 2853
{
47 2853
        struct vrt_ctx ctx[1];
48
49 2853
        INIT_OBJ(ctx, VRT_CTX_MAGIC);
50 2853
        VCL_Bo2Ctx(ctx, bo);
51 2853
        return (VDP_Push(ctx, vdc, ctx->ws, VDP_v1l, v1l));
52 2853
}
53
54
/*--------------------------------------------------------------------
55
 * Send request to backend, including any (cached) req.body
56
 *
57
 * Return value:
58
 *       0 success
59
 *       1 failure
60
 */
61
62
int
63
V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
64
    uint64_t *ctr_bodybytes)
65 2854
{
66 2854
        struct http *hp;
67 2854
        stream_close_t sc;
68 2854
        ssize_t i;
69 2854
        uint64_t bytes, hdrbytes;
70 2854
        struct http_conn *htc;
71 2854
        struct vdp_ctx vdc[1] = {{ 0 }};
72 2854
        intmax_t cl, ocl;
73 2854
        const char *err = NULL;
74 2854
        struct v1l *v1l = NULL;
75
76 2854
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
77 2854
        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
78 2854
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
79 2854
        CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
80 2854
        AN(ctr_hdrbytes);
81 2854
        AN(ctr_bodybytes);
82
83 2854
        htc = bo->htc;
84 2854
        assert(*htc->rfd > 0);
85 2854
        hp = bo->bereq;
86
87 2854
        if (bo->bereq_body != NULL)
88 24
                cl = ObjGetLen(wrk, bo->bereq_body);
89 2830
        else if (bo->req != NULL && !bo->req->req_body_status->length_known)
90 96
                cl = -1;
91 2734
        else if (bo->req != NULL) {
92 124
                cl = http_GetContentLength(bo->req->http);
93 124
                if (cl < 0)
94 21
                        cl = 0;
95 124
        }
96 2610
        else
97 2610
                cl = 0;
98 2854
        ocl = cl;
99 2854
        VDP_Init(vdc, wrk, bo->vsl, NULL, bo, &cl);
100 2854
        if (bo->vdp_filter_list != NULL &&
101 2854
            VCL_StackVDP(vdc, bo->vcl, bo->vdp_filter_list, NULL, bo))
102 1
                err = "Failure to push processors";
103 2853
        else if ((v1l = V1L_Open(wrk->aws, htc->rfd, bo->vsl, nan(""), 0)) == NULL) {
104
                /*      ^^^^^^^^
105
                 * XXX: need a send_timeout for the backend side
106
                 * XXX: use cache_param->http1_iovs ?
107
                 */
108 1
                err = "Failure to open V1L (workspace_thread overflow)";
109 1
        }
110 2852
        else if (v1f_stackv1l(vdc, bo, v1l))
111 16
                err = "Failure to push V1L";
112
113 2854
        if (err != NULL) {
114 18
                if (v1l != NULL)
115 16
                        (void) V1L_Close(&v1l, &bytes);
116 18
                if (VALID_OBJ(vdc, VDP_CTX_MAGIC))
117 18
                        (void) VDP_Close(vdc, NULL, NULL);
118 18
                VSLbs(bo->vsl, SLT_FetchError, TOSTRAND(err));
119 18
                VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
120 18
                htc->doclose = SC_OVERLOAD;
121 18
                return (-1);
122 18
        }
123
124
125 2836
        assert(cl >= -1);
126 2836
        if (cl < 0) {
127 98
                http_Unset(hp, H_Content_Length);
128 98
                http_ForceHeader(hp, H_Transfer_Encoding, "chunked");
129 2738
        } else if (ocl != cl) {
130 4
                assert(cl >= 0);
131 4
                http_Unset(hp, H_Transfer_Encoding);
132 4
                http_Unset(hp, H_Content_Length);
133 4
                http_PrintfHeader(hp, "Content-Length: %ju", cl);
134 4
        }
135
136 2836
        VTCP_blocking(*htc->rfd);       /* XXX: we should timeout instead */
137 2836
        hdrbytes = HTTP1_Write(v1l, hp, HTTP1_Req);
138
139
        /* Deal with any message-body the request might (still) have */
140 2836
        i = 0;
141
142 2836
        if (cl != 0 && bo->bereq_body != NULL) {
143 23
                AZ(bo->req);
144 23
                if (cl < 0)
145 3
                        V1L_Chunked(v1l);
146 23
                (void)ObjIterate(bo->wrk, bo->bereq_body,
147 23
                    vdc, VDP_ObjIterate, 0);
148 23
                if (cl < 0)
149 3
                        V1L_EndChunk(v1l);
150 2813
        } else if (cl != 0 && bo->req != NULL &&
151 2813
            bo->req->req_body_status != BS_NONE) {
152
                /* Start the bereq before we wait for the req.body */
153 194
                if (bo->req->req_body_status != BS_CACHED)
154 193
                        (void)V1L_Flush(v1l);
155 194
                if (cl < 0)
156 95
                        V1L_Chunked(v1l);
157 194
                i = VRB_Iterate(wrk, bo->vsl, bo->req, VDP_ObjIterate, vdc);
158
159 194
                if (bo->req->req_body_status != BS_CACHED)
160 193
                        bo->no_retry = "req.body not cached";
161
162 194
                if (bo->req->req_body_status == BS_ERROR) {
163
                        /*
164
                         * XXX: (#2332) We should test to see if the backend
165
                         * XXX: sent us some headers explaining why.
166
                         * XXX: This is hard because of the mistaken API split
167
                         * XXX: between cache_backend.c and V1F, and therefore
168
                         * XXX: Parked in this comment, pending renovation of
169
                         * XXX: the VDI/backend-protocol API to allow non-H1
170
                         * XXX: backends.
171
                         */
172 15
                        assert(i < 0);
173 15
                        VSLb(bo->vsl, SLT_FetchError,
174 15
                            "req.body read error: %d (%s)",
175 15
                            errno, VAS_errtxt(errno));
176 15
                        bo->req->doclose = SC_RX_BODY;
177 15
                        bo->err_code = 400;
178 15
                }
179 194
                if (cl < 0)
180 95
                        V1L_EndChunk(v1l);
181 194
        }
182
183 2836
        sc = V1L_Close(&v1l, &bytes);
184 2836
        CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
185
186
        /* Bytes accounting */
187 2836
        *ctr_hdrbytes += hdrbytes;
188 2836
        *ctr_bodybytes += VDP_Close(vdc, NULL, NULL);
189
190 2837
        if (sc == SC_NULL && i < 0)
191 15
                sc = SC_TX_ERROR;
192
193 2836
        CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
194 2836
        if (sc != SC_NULL) {
195 15
                VSLb(bo->vsl, SLT_FetchError,
196 15
                    "backend write error: %d (%s) (%s)",
197 15
                    errno, VAS_errtxt(errno), sc->desc);
198 15
                VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
199 15
                htc->doclose = sc;
200 15
                return (-1);
201 15
        }
202 2821
        CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
203 2821
        VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
204 2821
        return (0);
205 2821
}
206
207
int
208
V1F_FetchRespHdr(struct busyobj *bo)
209 2797
{
210
211 2797
        struct http *hp;
212 2797
        int i;
213 2797
        double t;
214 2797
        struct http_conn *htc;
215 2797
        enum htc_status_e hs;
216 2797
        const char *name, *desc;
217
218 2797
        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
219 2797
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
220 2797
        CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
221
222 2797
        htc = bo->htc;
223 2797
        assert(*htc->rfd > 0);
224
225 2797
        VSC_C_main->backend_req++;
226
227
        /* Receive response */
228
229 2797
        HTC_RxInit(htc, bo->ws);
230 2797
        CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
231 2797
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
232
233 2797
        t = VTIM_real() + htc->first_byte_timeout;
234 2797
        hs = HTC_RxStuff(htc, HTTP1_Complete, NULL, NULL,
235 2797
            t, NAN, htc->between_bytes_timeout, cache_param->http_resp_size);
236 2797
        if (hs != HTC_S_COMPLETE) {
237 47
                bo->acct.beresp_hdrbytes +=
238 47
                    htc->rxbuf_e - htc->rxbuf_b;
239 47
                switch (hs) {
240 0
                case HTC_S_JUNK:
241 0
                        VSLb(bo->vsl, SLT_FetchError, "Received junk");
242 0
                        htc->doclose = SC_RX_JUNK;
243 0
                        break;
244 21
                case HTC_S_EOF:
245 21
                        VSLb(bo->vsl, SLT_FetchError, "backend closed");
246 21
                        htc->doclose = SC_REM_CLOSE;
247 21
                        break;
248 1
                case HTC_S_MORE:
249 1
                        VSLb(bo->vsl, SLT_FetchError, "timeout");
250 1
                        htc->doclose = SC_RX_TIMEOUT;
251 1
                        break;
252 20
                case HTC_S_OVERFLOW:
253 20
                        VSLb(bo->vsl, SLT_FetchError, "overflow");
254 20
                        htc->doclose = SC_RX_OVERFLOW;
255 20
                        break;
256 5
                case HTC_S_EMPTY:
257 5
                        VSLb(bo->vsl, SLT_FetchError, "first byte timeout");
258 5
                        htc->doclose = SC_RX_TIMEOUT;
259 5
                        break;
260 0
                default:
261 0
                        HTC_Status(hs, &name, &desc);
262 0
                        VSLb(bo->vsl, SLT_FetchError, "HTC %s (%s)",
263 0
                             name, desc);
264 0
                        htc->doclose = SC_RX_BAD;
265 0
                        break;
266 47
                }
267 47
                return (htc->rxbuf_e == htc->rxbuf_b ? 1 : -1);
268 47
        }
269 2750
        VTCP_set_read_timeout(*htc->rfd, htc->between_bytes_timeout);
270
271 2750
        hp = bo->beresp;
272
273 2750
        i = HTTP1_DissectResponse(htc, hp, bo->bereq);
274 2750
        bo->acct.beresp_hdrbytes += htc->rxbuf_e - htc->rxbuf_b;
275 2750
        htc->rxbuf_b = NULL;
276 2750
        htc->rxbuf_e = NULL;
277 2750
        if (i) {
278 23
                VSLb(bo->vsl, SLT_FetchError, "http format error");
279 23
                htc->doclose = SC_RX_JUNK;
280 23
                return (-1);
281 23
        }
282
283 2727
        htc->doclose = http_DoConnection(hp, SC_RESP_CLOSE);
284
285
        /*
286
         * Figure out how the fetch is supposed to happen, before the
287
         * headers are adulterated by VCL
288
         */
289 2727
        if (http_method_eq(bo->bereq->wkm, WKM_HEAD)) {
290
                /*
291
                 * A HEAD request can never have a body in the reply,
292
                 * no matter what the headers might say.
293
                 * [RFC7231 4.3.2 p25]
294
                 */
295 6
                bo->wrk->stats->fetch_head++;
296 6
                bo->htc->body_status = BS_NONE;
297 2721
        } else if (http_GetStatus(bo->beresp) <= 199) {
298
                /*
299
                 * 1xx responses never have a body.
300
                 * [RFC7230 3.3.2 p31]
301
                 * ... but we should never see them.
302
                 */
303 0
                bo->wrk->stats->fetch_1xx++;
304 0
                bo->htc->body_status = BS_ERROR;
305 2721
        } else if (http_IsStatus(bo->beresp, 204)) {
306
                /*
307
                 * 204 is "No Content", obviously don't expect a body.
308
                 * [RFC7230 3.3.1 p29 and 3.3.2 p31]
309
                 */
310 6
                bo->wrk->stats->fetch_204++;
311 6
                if ((http_GetHdr(bo->beresp, H_Content_Length, NULL) &&
312 6
                    bo->htc->content_length != 0) ||
313 6
                    http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
314 2
                        bo->htc->body_status = BS_ERROR;
315 4
                else
316 4
                        bo->htc->body_status = BS_NONE;
317 2715
        } else if (http_IsStatus(bo->beresp, 304)) {
318
                /*
319
                 * 304 is "Not Modified" it has no body.
320
                 * [RFC7230 3.3 p28]
321
                 */
322 45
                bo->wrk->stats->fetch_304++;
323 45
                bo->htc->body_status = BS_NONE;
324 2670
        } else if (bo->htc->body_status == BS_CHUNKED) {
325 164
                bo->wrk->stats->fetch_chunked++;
326 2506
        } else if (bo->htc->body_status == BS_LENGTH) {
327 1611
                assert(bo->htc->content_length > 0);
328 1611
                bo->wrk->stats->fetch_length++;
329 1611
        } else if (bo->htc->body_status == BS_EOF) {
330 23
                bo->wrk->stats->fetch_eof++;
331 872
        } else if (bo->htc->body_status == BS_ERROR) {
332 0
                bo->wrk->stats->fetch_bad++;
333 872
        } else if (bo->htc->body_status == BS_NONE) {
334 871
                bo->wrk->stats->fetch_none++;
335 871
        } else {
336 1
                WRONG("wrong bodystatus");
337 1
        }
338
339 2726
        assert(bo->vfc->resp == bo->beresp);
340 2726
        if (bo->htc->body_status != BS_NONE &&
341 2726
            bo->htc->body_status != BS_ERROR)
342 1798
                if (V1F_Setup_Fetch(bo->vfc, bo->htc)) {
343 16
                        VSLb(bo->vsl, SLT_FetchError, "overflow");
344 16
                        htc->doclose = SC_RX_OVERFLOW;
345 16
                        return (-1);
346 16
                }
347
348 2710
        return (0);
349 2726
}