Code Coverage: vinyl-cache/bin/vinyld/cache/cache_gzip.c
 
82217aa4fc7094b3610340078afaf78cd95a8ec9
 
0
/*-
1
 * Copyright (c) 2013-2015 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 * Interaction with the libvgz (zlib) library.
30
 *
31
 * The zlib library pollutes namespace a LOT when you include the "vgz.h"
32
 * (aka "zlib.h") file so we contain the damage by vectoring all access
33
 * to libz through this source file.
34
 *
35
 * The API defined by this file, will also insulate the rest of the code,
36
 * should we find a better gzip library at a later date.
37
 *
38
 */
39
40
/*lint -e801 goto */
41
42
#include "config.h"
43
44
#include <stdlib.h>
45
46
#include "cache_int.h"
47
#include "cache_filter.h"
48
#include "cache_objhead.h"
49
#include "cache_vgz.h"
50
51
#include "storage/storage.h"
52
53
#include "vend.h"
54
55
#include "vgz.h"
56
57
struct vgz {
58
        unsigned                magic;
59
#define VGZ_MAGIC               0x162df0cb
60
        enum {VGZ_GZ,VGZ_UN}    dir;
61
        struct vsl_log          *vsl;
62
        const char              *id;
63
        int                     last_i;
64
        int                     latch;
65
        enum vgz_flag           flag;
66
67
        struct stv_buffer       *stvbuf;
68
        char                    *m_buf;
69
        ssize_t                 m_sz;
70
        ssize_t                 m_len;
71
72
        intmax_t                bits;
73
74
        z_stream                vz;
75
};
76
77
static const char *
78
vgz_msg(const struct vgz *vg)
79 54
{
80 54
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
81 54
        return (vg->vz.msg ? vg->vz.msg : "(null)");
82 54
}
83
84
/*--------------------------------------------------------------------
85
 * Set up a gunzip instance
86
 */
87
88
static struct vgz *
89
vgz_gunzip(struct vsl_log *vsl, const char *id)
90 892
{
91 892
        struct vgz *vg;
92
93 892
        ALLOC_OBJ(vg, VGZ_MAGIC);
94 892
        AN(vg);
95 892
        vg->vsl = vsl;
96 892
        vg->id = id;
97 892
        vg->dir = VGZ_UN;
98
99
        /*
100
         * Max memory usage according to zonf.h:
101
         *      mem_needed = "a few kb" + (1 << (windowBits))
102
         * Since we don't control windowBits, we have to assume
103
         * it is 15, so 34-35KB or so.
104
         */
105 892
        assert(Z_OK == inflateInit2(&vg->vz, 31));
106 892
        return (vg);
107 892
}
108
109
static struct vgz *
110
VGZ_NewGunzip(struct vsl_log *vsl, const char *id)
111 831
{
112 831
        VSC_C_main->n_gunzip++;
113 831
        return (vgz_gunzip(vsl, id));
114 831
}
115
116
#ifndef VDPIO_TEST_DRIVER
117
static struct vgz *
118
VGZ_NewTestGunzip(struct vsl_log *vsl, const char *id)
119 61
{
120 61
        VSC_C_main->n_test_gunzip++;
121 61
        return (vgz_gunzip(vsl, id));
122 61
}
123
#endif
124
125
struct vgz *
126
VGZ_NewGzip(struct vsl_log *vsl, const char *id)
127 160
{
128 160
        struct vgz *vg;
129 160
        int i;
130
131 160
        VSC_C_main->n_gzip++;
132 160
        ALLOC_OBJ(vg, VGZ_MAGIC);
133 160
        AN(vg);
134 160
        vg->vsl = vsl;
135 160
        vg->id = id;
136 160
        vg->dir = VGZ_GZ;
137
138
        /*
139
         * From zconf.h:
140
         *
141
         *      mem_needed = "a few kb"
142
         *              + (1 << (windowBits+2))
143
         *              +  (1 << (memLevel+9))
144
         *
145
         * windowBits [8..15] (-> 1K..128K)
146
         * memLevel [1..9] (-> 1K->256K)
147
         */
148 160
        i = deflateInit2(&vg->vz,
149 160
            cache_param->gzip_level,            /* Level */
150 160
            Z_DEFLATED,                         /* Method */
151 160
            16 + 15,                            /* Window bits (16=gzip) */
152 160
            cache_param->gzip_memlevel,         /* memLevel */
153 160
            Z_DEFAULT_STRATEGY);
154 160
        assert(Z_OK == i);
155 160
        return (vg);
156 160
}
157
158
/*--------------------------------------------------------------------
159
 */
160
161
#ifndef VDPIO_TEST_DRIVER
162
static int
163
vgz_getmbuf(struct worker *wrk, struct vgz *vg)
164 686
{
165 686
        size_t sz;
166
167 686
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
168 686
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
169 686
        AZ(vg->m_sz);
170 686
        AZ(vg->m_len);
171 686
        AZ(vg->m_buf);
172 686
        AZ(vg->stvbuf);
173
174 686
        vg->stvbuf = STV_AllocBuf(wrk, stv_transient, cache_param->gzip_buffer);
175 686
        if (vg->stvbuf == NULL) {
176 0
                VSLb(vg->vsl, SLT_Gzip, "G(un)zip error:"
177 0
                    "failed to allocate gzip_buffer from transient stevedore");
178 0
                return (-1);
179 0
        }
180 686
        vg->m_buf = STV_GetBufPtr(vg->stvbuf, &sz);
181 686
        vg->m_sz = sz;
182 686
        AN(vg->m_buf);
183 686
        assert(vg->m_sz > 0);
184 686
        return (0);
185 686
}
186
#endif
187
188
/*--------------------------------------------------------------------*/
189
190
void
191
VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
192 12287
{
193
194 12287
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
195 12287
        AZ(vg->vz.avail_in);
196 12287
        assert(len >= 0);
197 12287
        if (len > 0)
198 10353
                AN(ptr);
199
200 12287
        vg->vz.next_in = TRUST_ME(ptr);
201 12287
        vg->vz.avail_in = len;
202 12287
}
203
204
int
205
VGZ_IbufEmpty(const struct vgz *vg)
206 18049
{
207
208 18049
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
209 18049
        return (vg->vz.avail_in == 0);
210 18049
}
211
212
/*--------------------------------------------------------------------*/
213
214
void
215
VGZ_Obuf(struct vgz *vg, void *ptr, ssize_t len)
216 5734
{
217
218 5734
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
219 5734
        AN(ptr);
220 5734
        assert(len > 0);
221
222 5734
        vg->vz.next_out = ptr;
223 5734
        vg->vz.avail_out = len;
224 5734
}
225
226
int
227
VGZ_ObufFull(const struct vgz *vg)
228 4185
{
229
230 4185
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
231 4185
        return (vg->vz.avail_out == 0);
232 4185
}
233
234
/*--------------------------------------------------------------------*/
235
236
#ifndef VDPIO_TEST_DRIVER
237
static enum vgzret_e
238
VGZ_Gunzip(struct vgz *vg, const void **pptr, ssize_t *plen)
239 8269
{
240 8269
        int i;
241 8269
        ssize_t l;
242 8269
        const uint8_t *before;
243
244 8269
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
245
246 8269
        *pptr = NULL;
247 8269
        *plen = 0;
248 8269
        AN(vg->vz.next_out);
249 8269
        AN(vg->vz.avail_out);
250 8269
        before = vg->vz.next_out;
251 8269
        i = inflate(&vg->vz, 0);
252 8269
        if (i == Z_OK || i == Z_STREAM_END) {
253 8263
                *pptr = before;
254 8263
                l = (const uint8_t *)vg->vz.next_out - before;
255 8263
                *plen = l;
256 8263
        }
257 8269
        vg->last_i = i;
258 8269
        if (i == Z_OK)
259 7534
                return (VGZ_OK);
260 735
        if (i == Z_STREAM_END)
261 731
                return (VGZ_END);
262 4
        if (i == Z_BUF_ERROR)
263 2
                return (VGZ_STUCK);
264 2
        VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
265 2
        return (VGZ_ERROR);
266 4
}
267
#endif
268
269
/* set vz pointers for in and out iovecs */
270
static inline void
271
vgz_iovec_update(struct vgz *vg, const struct iovec *in, const struct iovec *buf)
272 9534
{
273
274
        // can not assert on vz.next_in vs io_base because "junk" case
275 9534
        vg->vz.next_in = TRUST_ME(in->iov_base);
276 9534
        vg->vz.avail_in = in->iov_len;
277 9534
        vg->vz.next_out = TRUST_ME(buf->iov_base);
278 9534
        vg->vz.avail_out = buf->iov_len;
279 9534
}
280
281
static enum vgzret_e
282
vgz_gunzip_iovec(struct vgz *vg, struct iovec *in, struct iovec *buf, struct iovec *out, int flush)
283 9534
{
284 9534
        int i;
285
286 9534
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
287 9534
        AN(in && buf && out);
288 9534
        vgz_iovec_update(vg, in, buf);
289
290 18428
        do {
291 18428
                i = inflate(&vg->vz, flush);
292 18428
        } while (i == Z_OK);
293
294 9534
        if (i == Z_STREAM_END || i == Z_BUF_ERROR) {
295 9486
                iovec_collect(buf, out, pdiff(buf->iov_base, vg->vz.next_out));
296 9486
                in->iov_base = TRUST_ME(vg->vz.next_in);
297 9486
                in->iov_len = vg->vz.avail_in;
298 9486
        }
299 9534
        vg->last_i = i;
300
301 9534
        if (i == Z_STREAM_END)
302 288
                return (VGZ_END);
303 9246
        if (i == Z_BUF_ERROR)
304 9198
                return (VGZ_STUCK);
305
306 48
        VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
307 48
        return (VGZ_ERROR);
308 9246
}
309
310
/*--------------------------------------------------------------------*/
311
312
enum vgzret_e
313
VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
314 4264
{
315 4264
        int i;
316 4264
        int zflg;
317 4264
        ssize_t l;
318 4264
        const uint8_t *before;
319
320 4264
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
321
322 4264
        *pptr = NULL;
323 4264
        *plen = 0;
324 4264
        AN(vg->vz.next_out);
325 4264
        AN(vg->vz.avail_out);
326 4264
        before = vg->vz.next_out;
327 4264
        switch (flags) {
328 2581
        case VGZ_NORMAL:        zflg = Z_NO_FLUSH; break;
329 501
        case VGZ_ALIGN:         zflg = Z_SYNC_FLUSH; break;
330 919
        case VGZ_RESET:         zflg = Z_FULL_FLUSH; break;
331 264
        case VGZ_FINISH:        zflg = Z_FINISH; break;
332 0
        default:                WRONG("Invalid VGZ flag");
333 4264
        }
334 4264
        i = deflate(&vg->vz, zflg);
335 4264
        if (i == Z_OK || i == Z_STREAM_END) {
336 4258
                *pptr = before;
337 4258
                l = (const uint8_t *)vg->vz.next_out - before;
338 4258
                *plen = l;
339 4258
        }
340 4264
        vg->last_i = i;
341 4264
        if (i == Z_OK)
342 4044
                return (VGZ_OK);
343 220
        if (i == Z_STREAM_END)
344 213
                return (VGZ_END);
345 7
        if (i == Z_BUF_ERROR)
346 0
                return (VGZ_STUCK);
347 7
        VSLb(vg->vsl, SLT_Gzip, "Gzip error: %d (%s)", i, vgz_msg(vg));
348 7
        return (VGZ_ERROR);
349 7
}
350
351
/*--------------------------------------------------------------------
352
 * VDP for gunzip'ing
353
 */
354
355
// common for traditional interface and vdpio
356
static int vdp_gunzip_init_common(const struct vdp_ctx *vdc);
357
358
#ifndef VDPIO_TEST_DRIVER
359
static int v_matchproto_(vdp_init_f)
360
vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
361 418
{
362 418
        struct vgz *vg;
363
364 418
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
365 418
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
366 418
        CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
367 418
        CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
368 418
        AN(vdc->clen);
369 418
        AN(priv);
370
371 418
        vg = VGZ_NewGunzip(vdc->vsl, "U D -");
372 418
        AN(vg);
373 418
        if (vgz_getmbuf(vdc->wrk, vg)) {
374 0
                (void)VGZ_Destroy(vdc->wrk, &vg);
375 0
                return (-1);
376 0
        }
377
378 418
        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
379 418
        *priv = vg;
380 418
        return (vdp_gunzip_init_common(vdc));
381 418
}
382
#endif
383
384
static int
385
vdp_gunzip_init_common(const struct vdp_ctx *vdc)
386 642
{
387 642
        struct boc *boc;
388 642
        enum boc_state_e bos;
389 642
        const char *p;
390 642
        ssize_t dl;
391 642
        uint64_t u;
392 642
        http_Unset(vdc->hp, H_Content_Encoding);
393
394 642
        *vdc->clen = -1;
395
396 642
        if (vdc->oc == NULL)
397 295
                return (0);
398
399 347
        boc = HSH_RefBoc(vdc->oc);
400 347
        if (boc != NULL) {
401 15
                CHECK_OBJ(boc, BOC_MAGIC);
402 15
                bos = boc->state;
403 15
                HSH_DerefBoc(vdc->wrk, vdc->oc);
404 15
                if (bos < BOS_FINISHED)
405 15
                        return (0); /* OA_GZIPBITS is not stable yet */
406 15
        }
407
408 332
        p = ObjGetAttr(vdc->wrk, vdc->oc, OA_GZIPBITS, &dl);
409 332
        if (p != NULL && dl == 32) {
410 332
                u = vbe64dec(p + 24);
411 332
                if (u != 0)
412 331
                        *vdc->clen = u;
413 332
        }
414 332
        return (0);
415 347
}
416
417
#ifndef VDPIO_TEST_DRIVER
418
static int v_matchproto_(vdp_fini_f)
419
vdp_gunzip_fini(struct vdp_ctx *vdc, void **priv)
420 402
{
421 402
        struct vgz *vg;
422
423 402
        (void)vdc;
424 402
        TAKE_OBJ_NOTNULL(vg, priv, VGZ_MAGIC);
425 402
        AN(vg->m_buf);
426 402
        (void)VGZ_Destroy(vdc->wrk, &vg);
427 402
        return (0);
428 402
}
429
430
static int v_matchproto_(vdp_bytes_f)
431
vdp_gunzip_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv,
432
    const void *ptr, ssize_t len)
433 12596
{
434 12596
        enum vgzret_e vr;
435 12596
        ssize_t dl;
436 12596
        const void *dp;
437 12596
        struct vgz *vg;
438
439 12596
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
440 12596
        CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC);
441 12596
        (void)act;
442
443 12596
        CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
444 12596
        AN(vg->m_buf);
445
446 12596
        if (len == 0 && act == VDP_FLUSH && vg->last_i != Z_STREAM_END &&
447 12596
            VDP_bytes(vdc, VDP_FLUSH, NULL, 0))
448 0
                return (vdc->retval);
449
450 12596
        if (len == 0)
451 4993
                return (0);
452
453 7603
        VGZ_Ibuf(vg, ptr, len);
454 7603
        do {
455 7603
                vr = VGZ_Gunzip(vg, &dp, &dl);
456 7603
                if (vr == VGZ_END && !VGZ_IbufEmpty(vg)) {
457 1
                        VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
458 1
                             vr, "junk after VGZ_END");
459 1
                        return (-1);
460 1
                }
461 7602
                vg->m_len += dl;
462 7602
                if (vr < VGZ_OK)
463 0
                        return (-1);
464
                // END or STUCK
465 7602
                if (vg->m_len == vg->m_sz || vr != VGZ_OK) {
466 399
                        if (VDP_bytes(vdc, vr == VGZ_END ? VDP_END : VDP_FLUSH,
467 399
                            vg->m_buf, vg->m_len))
468 4
                                return (vdc->retval);
469 395
                        vg->m_len = 0;
470 395
                        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
471 395
                }
472 7602
        } while (!VGZ_IbufEmpty(vg));
473 7598
        assert(vr == VGZ_STUCK || vr == VGZ_OK || vr == VGZ_END);
474 7598
        return (0);
475 7598
}
476
#endif
477
478
#ifdef VDPIO_TEST_DRIVER
479
//lint -esym(754, vgz::*)
480
/*
481
 * XXX does it make sense to work on more than one buffer?
482
 */
483
static int v_matchproto_(vdpio_init_f)
484
vdpio_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, int capacity)
485 224
{
486 224
        struct vgz *vg;
487 224
        struct vscarab *in;
488
489 224
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
490 224
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
491 224
        CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
492 224
        CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
493 224
        AN(vdc->clen);
494 224
        AN(priv);
495 224
        assert(capacity >= 1);
496
497 224
        if (capacity < 4)
498 224
                capacity = 4;
499
500 224
        in = WS_Alloc(ctx->ws, VSCARAB_SIZE(capacity));
501 224
        if (in == NULL)
502 0
                return (-1);
503 224
        VSCARAB_INIT(in, capacity);
504
505 224
        vg = VGZ_NewGunzip(vdc->vsl, "U D -");
506 224
        if (vg == NULL)
507 0
                return (-1);
508
509 224
        AZ(vg->m_buf);
510 224
        vg->m_buf = (void *)in;
511
512 224
        *priv = vg;
513 224
        AZ(vdp_gunzip_init_common(vdc));
514 224
        return (1);
515 224
}
516
#endif
517
#ifndef VDPIO_TEST_DRIVER
518
static int v_matchproto_(vdpio_init_f)
519
vdpio_gunzip_upgrade(VRT_CTX, struct vdp_ctx *vdc, void **priv, int capacity)
520 16
{
521 16
        struct vgz *vg;
522 16
        struct vscarab *in;
523
524 16
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
525 16
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
526 16
        AN(priv);
527 16
        assert(capacity >= 1);
528
529 16
        if (capacity < 4)
530 16
                capacity = 4;
531
532
        /* done in vdp_gunzip_init */
533 16
        CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
534
535 16
        in = WS_Alloc(ctx->ws, VSCARAB_SIZE(capacity));
536 16
        if (in == NULL)
537 0
                return (-1);
538 16
        VSCARAB_INIT(in, capacity);
539
540
        // XXX duplicate work - remove when completing transition to VAI
541 16
        AN(vg->m_buf);
542 16
        AN(vg->stvbuf);
543 16
        STV_FreeBuf(vdc->wrk, &vg->stvbuf);
544 16
        vg->stvbuf = NULL;
545 16
        vg->m_buf = (void *)in;
546
547 16
        return (1);
548 16
}
549
#endif
550
551
static int v_matchproto_(vdpio_lease_f)
552
vdpio_gunzip_lease(struct vdp_ctx *vdc, struct vdp_entry *this, struct vscarab *out)
553 16009
{
554 16009
        struct vscarab *in;
555 16009
        enum vgzret_e vr;
556 16009
        struct vgz *vg;
557 16009
        struct viov *v, *b, *o, *end;
558 16009
        int r;
559
560 16009
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
561 16009
        CHECK_OBJ_NOTNULL(this, VDP_ENTRY_MAGIC);
562 16009
        CAST_OBJ_NOTNULL(vg, this->priv, VGZ_MAGIC);
563 16009
        CAST_OBJ_NOTNULL(in, (void*)vg->m_buf, VSCARAB_MAGIC);
564
565 16009
        this->calls++;
566
567 16009
        if (vg->latch != 0)
568 480
                return (vg->latch);
569
        // vdpio_pull*() guarantees
570 15529
        assert(out->used < out->capacity);
571
572 15529
        r = 0;
573 15529
        if (in->used < in->capacity && (in->flags & VSCARAB_F_END) == 0)
574 7215
                r = vdpio_pull(vdc, this, in);
575
576 15529
        if (in->used == 0 && (in->flags & VSCARAB_F_END) == 0)
577 1437
                return (r);
578
579 14092
        VSCARAB_LOCAL(buf, 1);
580 14092
        b = VSCARAB_GET(buf);
581 14092
        AN(b);
582
        // XXX get a smaller buffer if in->flags & VSCARAB_F_END based on a
583
        // guesstimate of the compression ratio?
584 14092
        b->iov.iov_len = cache_param->gzip_buffer;
585 14092
        r = ObjVAIbuffer(vdc->wrk, vdc->vai_hdl, buf);
586 14092
        if (r < 0)
587 4718
                return (out->used ? 0 : r);
588
589 9374
        o = VSCARAB_GET(out);
590 9374
        AN(o);
591 9374
        r = 0;
592
593 9374
    again:
594
        // no input or last empty scarab
595 9374
        if (in->used == 0 && (in->flags & VSCARAB_F_END)) {
596 500
                vr = vgz_gunzip_iovec(vg, &(struct iovec){0}, &b->iov, &o->iov, Z_FINISH);
597 500
                goto out;
598 500
        }
599
600
        // loop must be entered
601 8874
        vr = VGZ_WRONG;
602 8874
        end = in->flags & VSCARAB_F_END ? VSCARAB_LAST(in) : NULL;
603 9034
        VSCARAB_FOREACH(v, in) {
604 9034
                this->bytes_in += v->iov.iov_len;
605
606 9034
                if (vr == VGZ_END && v->iov.iov_len > 0)
607 0
                        goto junk;
608
609 9034
                vr = vgz_gunzip_iovec(vg, &v->iov, &b->iov, &o->iov,
610 9034
                    v == end ? Z_FINISH : Z_NO_FLUSH);
611
612 9034
                if (vr == VGZ_END && v->iov.iov_len > 0)
613 0
                        goto junk;
614
615 9034
                if (vr == VGZ_ERROR)
616 48
                        break;
617
618 8986
                this->bytes_in -= v->iov.iov_len;
619
620 8986
                if (vr == VGZ_STUCK && v->iov.iov_len > 0) {
621
                        // stuck because output full
622 8649
                        AZ(b->iov.iov_len);
623 8649
                        break;
624 8649
                }
625
626
                // handled after loop
627 337
                assert(vr == VGZ_STUCK || vr == VGZ_OK || vr == VGZ_END);
628 337
        }
629
630 9374
    out:
631 9374
        vdpio_consolidate_vscarab(vdc, in);
632
633 9374
        switch (vr) {
634 284
        case VGZ_END:
635 284
                out->flags |= VSCARAB_F_END;
636 284
                break;
637 0
        case VGZ_OK:
638 0
                break;
639 9042
        case VGZ_STUCK:
640
                // compare above: stuck for full buffer
641 9042
                if (b->iov.iov_len == 0)
642 8649
                        break;
643 393
                AZ(in->used);
644
                // for no input at all, we do not get a VGZ_END
645 393
                if (in->flags & VSCARAB_F_END && this->bytes_in == 0) {
646 264
                        out->flags |= VSCARAB_F_END;
647 264
                        break;
648 264
                }
649
                // stuck for input empty, try to pull more
650 129
                r = vdpio_pull(vdc, this, in);
651 129
                if (r > 0)
652 0
                        goto again;
653 129
                break;
654 129
        case VGZ_ERROR:
655 48
                r = -EBADMSG;
656 48
                goto err;
657 0
        default:
658 0
                WRONG("vr after loop");
659 9374
        }
660
661 9326
        AN(out->used);
662
663 9326
        if (o->iov.iov_len == 0) {
664 509
                AZ(o->iov.iov_base);
665 509
                out->used--;
666 509
                vdpio_return_vscarab(vdc, buf);
667 509
                return (0);
668 509
        }
669
670
        // transfer ownership of buffer
671 8817
        o->lease = b->lease;
672
        // not needed, for clarity
673 8817
        b->lease = VAI_LEASE_NORET;
674
675 8817
        return (1);
676
677 0
    junk:
678 0
        VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
679 0
                vr, "junk after VGZ_END");
680 0
        r = -EMSGSIZE;
681 48
    err:
682 48
        vg->latch = r;
683 48
        assert(r < 0);
684 48
        memset(o, 0, sizeof *o);
685 48
        AN(out->used);
686 48
        out->used--;
687 48
        vdpio_return_vscarab(vdc, buf);
688 48
        vdpio_return_vscarab(vdc, in);
689 48
        return (r);
690 48
}
691
692
static void v_matchproto_(vdpio_fini_f)
693
vdpio_gunzip_fini(struct vdp_ctx *vdc, void **priv)
694 240
{
695 240
        struct vscarab *in;
696 240
        struct vgz *vg;
697
698 240
        (void)vdc;
699 240
        TAKE_OBJ_NOTNULL(vg, priv, VGZ_MAGIC);
700 240
        CAST_OBJ_NOTNULL(in, (void *)vg->m_buf, VSCARAB_MAGIC);
701 240
        vg->m_buf = NULL;
702
703 240
        vdpio_return_vscarab(vdc, in);
704
705 240
        (void)VGZ_Destroy(vdc->wrk, &vg);
706 240
}
707
708
const struct vdp VDP_gunzip = {
709
        .name =         "gunzip",
710
#ifndef VDPIO_TEST_DRIVER
711
        .init =         vdp_gunzip_init,
712
        .bytes =        vdp_gunzip_bytes,
713
        .fini =         vdp_gunzip_fini,
714
#endif
715
716
#ifdef VDPIO_TEST_DRIVER
717
        .io_init =      vdpio_gunzip_init,
718
#endif
719
#ifndef VDPIO_TEST_DRIVER
720
        .io_upgrade =   vdpio_gunzip_upgrade,
721
#endif
722
        .io_lease =     vdpio_gunzip_lease,
723
        .io_fini =      vdpio_gunzip_fini,
724
};
725
726
#ifdef VDPIO_TEST_DRIVER
727
#include <stdio.h>
728
#include "vdpio_test.h"
729
#include "vfil.h"
730
731
int
732
main(int argc, char *const *argv)
733 1
{
734 1
        char *data, *expect;
735 1
        char fn[64];
736 1
        int fd;
737 1
        char buf[1024];
738
739 1
        VFLA_LOCAL(vdpio_test, t, VDPIO_TEST_MAGIC, tc, 2);
740 1
        t->vdp = &VDP_gunzip;
741
742
        // tests add up to 2752 bytes
743 1
        cache_param->gzip_buffer = (size_t)1;
744
745 1
        AN(argc);
746
747
        // the test is to compress our own binary
748
        // this is the simplest way I could come up with wrt loc
749 1
        bprintf(fn, "/tmp/vdoip_gzip.%s", "XXXXXX");
750 1
        fd = mkstemp(fn);
751 1
        bprintf(buf, "gzip -c %s >&%d", argv[0], fd);
752 1
        AZ(system(buf));
753 1
        close(fd);
754 1
        data = VFIL_readfile(NULL, fn, &t->datasz);
755 1
        expect = VFIL_readfile(NULL, argv[0], &t->expectsz);
756 1
        t->data = data;
757 1
        t->expect = expect;
758 1
        t->expect_empty = "";
759 1
        t->expectsz_empty = 0;
760 1
        (void)unlink(fn);
761
762 1
        struct vdpio_tc *tc;
763
764 1
        tc = VFLA_GET(t, tc);
765 1
        AN(tc);
766 1
        tc->name = "junk at end";
767 1
        tc->data = malloc(t->datasz + 1);
768 1
        AN(tc->data);
769 1
        tc->datasz = t->datasz + 1;
770 1
        tc->ret = -EBADMSG;
771
772 1
        tc = VFLA_GET(t, tc);
773 1
        AN(tc);
774 1
        tc->name = "garbage";
775 1
        data = malloc(t->datasz);
776 1
        AN(data);
777 1
        memset(data + t->datasz / 2, 0, t->datasz / 4);
778 1
        tc->data = data;
779 1
        tc->datasz = t->datasz;
780 1
        tc->ret = -EBADMSG;
781
782 1
        AZ(vdpio_mock(t));
783
784 1
        VFLA_FOREACH(tc, t, tc)
785 2
                free(TRUST_ME(tc->data));
786
787 1
        free(TRUST_ME(t->data));
788 1
        free(TRUST_ME(t->expect));
789 1
}
790
#endif
791
792
/*--------------------------------------------------------------------*/
793
794
#ifndef VDPIO_TEST_DRIVER
795
void
796
VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgzret_e e)
797 4482
{
798 4482
        char *p;
799 4482
        intmax_t ii;
800
801 4482
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
802 4482
        if (e < VGZ_OK)
803 0
                return;
804 4482
        ii = vg->vz.start_bit + vg->vz.last_bit + vg->vz.stop_bit;
805 4482
        if (e != VGZ_END && ii == vg->bits)
806 3855
                return;
807 627
        vg->bits = ii;
808 627
        p = ObjSetAttr(vc->wrk, vc->oc, OA_GZIPBITS, 32, NULL);
809 627
        AN(p);
810 627
        vbe64enc(p, vg->vz.start_bit);
811 627
        vbe64enc(p + 8, vg->vz.last_bit);
812 627
        vbe64enc(p + 16, vg->vz.stop_bit);
813 627
        if (e != VGZ_END)
814 199
                return;
815 428
        if (vg->dir == VGZ_GZ)
816 368
                vbe64enc(p + 24, vg->vz.total_in);
817 428
        if (vg->dir == VGZ_UN)
818 60
                vbe64enc(p + 24, vg->vz.total_out);
819 428
}
820
#endif
821
822
/*--------------------------------------------------------------------
823
 */
824
825
enum vgzret_e
826
VGZ_Destroy(struct worker *wrk, struct vgz **vgp)
827 1052
{
828 1052
        struct vgz *vg;
829 1052
        enum vgzret_e vr;
830 1052
        int i;
831
832 1052
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
833 1052
        TAKE_OBJ_NOTNULL(vg, vgp, VGZ_MAGIC);
834 1052
        AN(vg->id);
835 1052
        VSLb(vg->vsl, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
836 1052
            vg->id,
837 1052
            (intmax_t)vg->vz.total_in,
838 1052
            (intmax_t)vg->vz.total_out,
839 1052
            (intmax_t)vg->vz.start_bit,
840 1052
            (intmax_t)vg->vz.last_bit,
841 1052
            (intmax_t)vg->vz.stop_bit);
842 1052
        if (vg->dir == VGZ_GZ)
843 159
                i = deflateEnd(&vg->vz);
844 893
        else
845 893
                i = inflateEnd(&vg->vz);
846 1052
        if (vg->last_i == Z_STREAM_END && i == Z_OK)
847 787
                i = Z_STREAM_END;
848
#ifndef VDPIO_TEST_DRIVER
849 828
        if (vg->m_buf != NULL) {
850 670
                AN(vg->stvbuf);
851 670
                STV_FreeBuf(wrk, &vg->stvbuf);
852 670
        }
853 828
#endif
854 1052
        AZ(vg->stvbuf);
855 1052
        if (i == Z_OK)
856 263
                vr = VGZ_OK;
857 789
        else if (i == Z_STREAM_END)
858 787
                vr = VGZ_END;
859 2
        else if (i == Z_BUF_ERROR)
860 0
                vr = VGZ_STUCK;
861 2
        else {
862 2
                VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
863 2
                    i, vgz_msg(vg));
864 2
                vr = VGZ_ERROR;
865 2
        }
866 1052
        FREE_OBJ(vg);
867 1052
        return (vr);
868 828
}
869
870
/*--------------------------------------------------------------------*/
871
#ifndef VDPIO_TEST_DRIVER
872
873
static enum vfp_status v_matchproto_(vfp_init_f)
874
vfp_gzip_init(VRT_CTX, struct vfp_ctx *vc, struct vfp_entry *vfe)
875 272
{
876 272
        struct vgz *vg;
877
878 272
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
879 272
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
880 272
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
881
882
        /*
883
         * G(un)zip makes no sence on partial responses, but since
884
         * it is an pure 1:1 transform, we can just ignore it.
885
         */
886 272
        if (http_GetStatus(vc->resp) == 206)
887 4
                return (VFP_NULL);
888
889 268
        if (vfe->vfp == &VFP_gzip) {
890 18
                if (http_GetHdr(vc->resp, H_Content_Encoding, NULL))
891 0
                        return (VFP_NULL);
892 18
                vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1);
893 18
                vc->obj_flags |= OF_GZIPED | OF_CHGCE;
894 250
        } else {
895 250
                if (!http_HdrIs(vc->resp, H_Content_Encoding, "gzip"))
896 0
                        return (VFP_NULL);
897 250
                if (vfe->vfp == &VFP_gunzip) {
898 189
                        vg = VGZ_NewGunzip(vc->wrk->vsl, vfe->vfp->priv1);
899 189
                        vc->obj_flags &= ~OF_GZIPED;
900 189
                        vc->obj_flags |= OF_CHGCE;
901 189
                } else {
902 61
                        vg = VGZ_NewTestGunzip(vc->wrk->vsl, vfe->vfp->priv1);
903 61
                        vc->obj_flags |= OF_GZIPED;
904 61
                }
905 250
        }
906 268
        AN(vg);
907 268
        vfe->priv1 = vg;
908 268
        if (vgz_getmbuf(vc->wrk, vg))
909 0
                return (VFP_ERROR);
910 268
        VGZ_Ibuf(vg, vg->m_buf, 0);
911 268
        AZ(vg->m_len);
912
913 268
        if (vfe->vfp == &VFP_gunzip || vfe->vfp == &VFP_gzip) {
914 207
                http_Unset(vc->resp, H_Content_Encoding);
915 207
                http_Unset(vc->resp, H_Content_Length);
916 207
                RFC2616_Weaken_Etag(vc->resp);
917 207
        }
918
919 268
        if (vfe->vfp == &VFP_gzip)
920 18
                http_SetHeader(vc->resp, "Content-Encoding: gzip");
921
922 268
        if (vfe->vfp == &VFP_gzip || vfe->vfp == &VFP_testgunzip)
923 79
                RFC2616_Vary_AE(vc->resp);
924
925 268
        return (VFP_OK);
926 268
}
927
928
/*--------------------------------------------------------------------
929
 * VFP_GUNZIP
930
 *
931
 * A VFP for gunzip'ing an object as we receive it from the backend
932
 */
933
934
static enum vfp_status v_matchproto_(vfp_pull_f)
935
vfp_gunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
936
    ssize_t *lp)
937 584
{
938 584
        ssize_t l;
939 584
        struct vgz *vg;
940 584
        enum vgzret_e vr = VGZ_ERROR;
941 584
        const void *dp;
942 584
        ssize_t dl;
943 584
        enum vfp_status vp = VFP_OK;
944
945 584
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
946 584
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
947 584
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
948 584
        AN(p);
949 584
        AN(lp);
950 584
        l = *lp;
951 584
        *lp = 0;
952 584
        VGZ_Obuf(vg, p, l);
953 586
        do {
954 586
                if (VGZ_IbufEmpty(vg)) {
955 273
                        l = vg->m_sz;
956 273
                        vp = VFP_Suck(vc, vg->m_buf, &l);
957 273
                        if (vp == VFP_ERROR)
958 0
                                return (vp);
959 273
                        VGZ_Ibuf(vg, vg->m_buf, l);
960 273
                }
961 586
                if (!VGZ_IbufEmpty(vg) || vp == VFP_END) {
962 586
                        vr = VGZ_Gunzip(vg, &dp, &dl);
963 586
                        if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
964 1
                                return (VFP_Error(vc, "Junk after gzip data"));
965 585
                        if (vr < VGZ_OK)
966 2
                                return (VFP_Error(vc,
967 2
                                    "Invalid Gzip data: %s", vgz_msg(vg)));
968 583
                        if (dl > 0) {
969 447
                                *lp = dl;
970 447
                                assert(dp == p);
971 447
                                return (VFP_OK);
972 447
                        }
973 583
                }
974 136
                AN(VGZ_IbufEmpty(vg));
975 136
        } while (vp == VFP_OK);
976 134
        if (vr != VGZ_END)
977 1
                return (VFP_Error(vc, "Gunzip error at the very end"));
978 133
        return (vp);
979 134
}
980
981
982
/*--------------------------------------------------------------------
983
 * VFP_GZIP
984
 *
985
 * A VFP for gzip'ing an object as we receive it from the backend
986
 */
987
988
static enum vfp_status v_matchproto_(vfp_pull_f)
989
vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
990
    ssize_t *lp)
991 67
{
992 67
        ssize_t l;
993 67
        struct vgz *vg;
994 67
        enum vgzret_e vr = VGZ_ERROR;
995 67
        const void *dp;
996 67
        ssize_t dl;
997 67
        enum vfp_status vp = VFP_ERROR;
998
999 67
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
1000 67
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
1001 67
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
1002 67
        AN(p);
1003 67
        AN(lp);
1004 67
        l = *lp;
1005 67
        *lp = 0;
1006 67
        VGZ_Obuf(vg, p, l);
1007 74
        do {
1008 74
                if (VGZ_IbufEmpty(vg)) {
1009 70
                        l = vg->m_sz;
1010 70
                        vp = VFP_Suck(vc, vg->m_buf, &l);
1011 70
                        if (vp == VFP_ERROR)
1012 0
                                break;
1013 70
                        if (vp == VFP_END)
1014 24
                                vg->flag = VGZ_FINISH;
1015 70
                        VGZ_Ibuf(vg, vg->m_buf, l);
1016 70
                }
1017 74
                if (!VGZ_IbufEmpty(vg) || vg->flag == VGZ_FINISH) {
1018 74
                        vr = VGZ_Gzip(vg, &dp, &dl, vg->flag);
1019 74
                        if (vr < VGZ_OK)
1020 0
                                return (VFP_Error(vc, "Gzip failed"));
1021 74
                        if (dl > 0) {
1022 67
                                VGZ_UpdateObj(vc, vg, vr);
1023 67
                                *lp = dl;
1024 67
                                assert(dp == p);
1025 67
                                if (vr != VGZ_END || !VGZ_IbufEmpty(vg))
1026 50
                                        return (VFP_OK);
1027 67
                        }
1028 74
                }
1029 24
                AN(VGZ_IbufEmpty(vg));
1030 24
        } while (vg->flag != VGZ_FINISH);
1031
1032 17
        if (vr != VGZ_END)
1033 0
                return (VFP_Error(vc, "Gzip failed"));
1034 17
        VGZ_UpdateObj(vc, vg, VGZ_END);
1035 17
        return (VFP_END);
1036 17
}
1037
1038
/*--------------------------------------------------------------------
1039
 * VFP_TESTGZIP
1040
 *
1041
 * A VFP for testing that received gzip data is valid, and for
1042
 * collecting the magic bits while we're at it.
1043
 */
1044
1045
static enum vfp_status v_matchproto_(vfp_pull_f)
1046
vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
1047
    ssize_t *lp)
1048 83
{
1049 83
        struct vgz *vg;
1050 83
        enum vgzret_e vr = VGZ_ERROR;
1051 83
        const void *dp;
1052 83
        ssize_t dl;
1053 83
        enum vfp_status vp;
1054
1055 83
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
1056 83
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
1057 83
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
1058 83
        AN(p);
1059 83
        AN(lp);
1060 83
        vp = VFP_Suck(vc, p, lp);
1061 83
        if (vp == VFP_ERROR)
1062 2
                return (vp);
1063 81
        if (*lp > 0 || vp == VFP_END) {
1064 81
                VGZ_Ibuf(vg, p, *lp);
1065 81
                do {
1066 81
                        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
1067 81
                        vr = VGZ_Gunzip(vg, &dp, &dl);
1068 81
                        if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
1069 3
                                return (VFP_Error(vc, "Junk after gzip data"));
1070 78
                        if (vr < VGZ_OK)
1071 0
                                return (VFP_Error(vc,
1072 0
                                    "Invalid Gzip data: %s", vgz_msg(vg)));
1073 78
                } while (!VGZ_IbufEmpty(vg));
1074 81
        }
1075 78
        VGZ_UpdateObj(vc, vg, vr);
1076 78
        if (vp == VFP_END && vr != VGZ_END)
1077 1
                return (VFP_Error(vc, "tGunzip failed"));
1078 77
        return (vp);
1079 78
}
1080
1081
/*--------------------------------------------------------------------*/
1082
1083
static void v_matchproto_(vfp_fini_f)
1084
vfp_gzip_fini(struct vfp_ctx *vc, struct vfp_entry *vfe)
1085 283
{
1086 283
        struct vgz *vg;
1087
1088 283
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
1089 283
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
1090
1091 283
        if (vfe->priv1 != NULL) {
1092 268
                TAKE_OBJ_NOTNULL(vg, &vfe->priv1, VGZ_MAGIC);
1093 268
                (void)VGZ_Destroy(vc->wrk, &vg);
1094 268
        }
1095 283
}
1096
1097
/*--------------------------------------------------------------------*/
1098
1099
const struct vfp VFP_gunzip = {
1100
        .name = "gunzip",
1101
        .init = vfp_gzip_init,
1102
        .pull = vfp_gunzip_pull,
1103
        .fini = vfp_gzip_fini,
1104
        .priv1 = "U F -",
1105
};
1106
1107
const struct vfp VFP_gzip = {
1108
        .name = "gzip",
1109
        .init = vfp_gzip_init,
1110
        .pull = vfp_gzip_pull,
1111
        .fini = vfp_gzip_fini,
1112
        .priv1 = "G F -",
1113
};
1114
1115
const struct vfp VFP_testgunzip = {
1116
        .name = "testgunzip",
1117
        .init = vfp_gzip_init,
1118
        .pull = vfp_testgunzip_pull,
1119
        .fini = vfp_gzip_fini,
1120
        .priv1 = "u F -",
1121
};
1122
#endif