Code Coverage: vinyl-cache/bin/vinyld/storage/storage_simple.c
 
82217aa4fc7094b3610340078afaf78cd95a8ec9
 
0
/*-
1
 * Copyright (c) 2007-2015 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 */
30
31
#include "config.h"
32
33
#include <stdlib.h>
34
35
#include "cache/cache_int.h"
36
37
#include "cache/cache_obj.h"
38
#include "cache/cache_objhead.h"
39
40
#include "storage/storage.h"
41
#include "storage/storage_simple.h"
42
#include "storage/storage_vai.h"
43
44
#include "vtim.h"
45
46
/* Flags for allocating memory in sml_stv_alloc */
47 10765
#define LESS_MEM_ALLOCED_IS_OK  1
48
49
// marker pointer for sml_trimstore
50
static void *trim_once = &trim_once;
51
52
/*-------------------------------------------------------------------*/
53
54
static struct storage *
55
objallocwithnuke(struct worker *, const struct stevedore *, ssize_t size,
56
    int flags);
57
58
static struct storage *
59
sml_stv_alloc(const struct stevedore *stv, ssize_t size, int flags)
60 6087
{
61 6087
        struct storage *st;
62
63 6087
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
64 6087
        AN(stv->sml_alloc);
65
66 6087
        if (!(flags & LESS_MEM_ALLOCED_IS_OK)) {
67 1406
                if (size > cache_param->fetch_maxchunksize)
68 0
                        return (NULL);
69 1406
                else
70 1406
                        return (stv->sml_alloc(stv, size));
71 1406
        }
72
73 4681
        if (size > cache_param->fetch_maxchunksize)
74 0
                size = cache_param->fetch_maxchunksize;
75
76 4681
        assert(size <= UINT_MAX);       /* field limit in struct storage */
77
78 4816
        for (;;) {
79
                /* try to allocate from it */
80 4816
                assert(size > 0);
81 4816
                st = stv->sml_alloc(stv, size);
82 4816
                if (st != NULL)
83 4670
                        break;
84
85 146
                if (size <= cache_param->fetch_chunksize)
86 11
                        break;
87
88 135
                size /= 2;
89 135
        }
90 4681
        CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
91 4681
        return (st);
92 4681
}
93
94
static void
95
sml_stv_free(const struct stevedore *stv, struct storage *st)
96 8892
{
97
98 8892
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
99 8892
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
100 8892
        if (stv->sml_free != NULL)
101 8892
                stv->sml_free(st);
102 8892
}
103
104
/*--------------------------------------------------------------------
105
 * This function is called by stevedores ->allocobj() method, which
106
 * very often will be SML_allocobj() below, to convert a slab
107
 * of storage into object which the stevedore can then register in its
108
 * internal state, before returning it to STV_NewObject().
109
 * As you probably guessed: All this for persistence.
110
 */
111
112
struct object *
113
SML_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr)
114 4746
{
115 4746
        struct object *o;
116
117 4746
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
118 4746
        AN(stv->methods);
119 4746
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
120
121 4746
        assert(PAOK(ptr));
122
123 4746
        o = ptr;
124 4746
        INIT_OBJ(o, OBJECT_MAGIC);
125
126 4746
        VTAILQ_INIT(&o->list);
127
128 4746
        oc->stobj->stevedore = stv;
129 4746
        oc->stobj->priv = o;
130 4746
        oc->stobj->priv2 = 0;
131 4746
        return (o);
132 4746
}
133
134
/*--------------------------------------------------------------------
135
 * This is the default ->allocobj() which all stevedores who do not
136
 * implement persistent storage can rely on.
137
 */
138
139
int v_matchproto_(storage_allocobj_f)
140
SML_allocobj(struct worker *wrk, const struct stevedore *stv,
141
    struct objcore *oc, unsigned wsl)
142 4736
{
143 4736
        struct object *o;
144 4736
        struct storage *st = NULL;
145 4736
        unsigned ltot;
146
147 4736
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
148 4736
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
149 4736
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
150
151 4736
        AN(stv->sml_alloc);
152
153 4736
        ltot = sizeof(*o) + PRNDUP(wsl);
154
155 4739
        do {
156 4739
                st = stv->sml_alloc(stv, ltot);
157 4739
                if (st != NULL && st->space < ltot) {
158 0
                        stv->sml_free(st);
159 0
                        st = NULL;
160 0
                }
161 4739
        } while (st == NULL && LRU_NukeOne(wrk, stv->lru));
162 4736
        if (st == NULL)
163 9
                return (0);
164
165 4727
        CHECK_OBJ(st, STORAGE_MAGIC);
166 4727
        o = SML_MkObject(stv, oc, st->ptr);
167 4727
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
168 4727
        st->len = sizeof(*o);
169 4727
        o->objstore = st;
170 4727
        return (1);
171 4727
}
172
173
void * v_matchproto_(storage_allocbuf_t)
174
SML_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size,
175
    uintptr_t *ppriv)
176 708
{
177 708
        struct storage *st;
178
179 708
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
180 708
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
181 708
        AN(ppriv);
182
183 708
        if (size > UINT_MAX)
184 0
                return (NULL);
185 708
        st = objallocwithnuke(wrk, stv, size, 0);
186 708
        if (st == NULL)
187 0
                return (NULL);
188 708
        assert(st->space >= size);
189 708
        st->flags = STORAGE_F_BUFFER;
190 708
        st->len = size;
191 708
        *ppriv = (uintptr_t)st;
192 708
        return (st->ptr);
193 708
}
194
195
void v_matchproto_(storage_freebuf_t)
196
SML_FreeBuf(struct worker *wrk, const struct stevedore *stv, uintptr_t priv)
197 708
{
198 708
        struct storage *st;
199
200 708
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
201 708
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
202
203 708
        CAST_OBJ_NOTNULL(st, (void *)priv, STORAGE_MAGIC);
204 708
        assert(st->flags == STORAGE_F_BUFFER);
205 708
        sml_stv_free(stv, st);
206 708
}
207
208
/*---------------------------------------------------------------------
209
 */
210
211
static struct object *
212
sml_getobj(struct worker *wrk, struct objcore *oc)
213 200030
{
214 200030
        const struct stevedore *stv;
215 200030
        struct object *o;
216
217 200030
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
218 200030
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
219 200030
        stv = oc->stobj->stevedore;
220 200030
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
221 200030
        if (stv->sml_getobj != NULL)
222 256
                return (stv->sml_getobj(wrk, oc));
223 199774
        if (oc->stobj->priv == NULL)
224 0
                return (NULL);
225 199774
        CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
226 199774
        return (o);
227 199774
}
228
229
static void v_matchproto_(objslim_f)
230
sml_slim(struct worker *wrk, struct objcore *oc)
231 6787
{
232 6787
        const struct stevedore *stv;
233 6787
        struct object *o;
234 6787
        struct storage *st, *stn;
235
236 6787
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
237
238 6787
        stv = oc->stobj->stevedore;
239 6787
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
240 6787
        o = sml_getobj(wrk, oc);
241 6787
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
242
243 6787
#define OBJ_AUXATTR(U, l)                                       \
244 6787
        do {                                                    \
245 6787
                if (o->aa_##l != NULL) {                        \
246 349
                        sml_stv_free(stv, o->aa_##l);           \
247 349
                        o->aa_##l = NULL;                       \
248 349
                }                                               \
249 6787
        } while (0);
250 6787
#include "tbl/obj_attr.h"
251
252 6787
        VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) {
253 3364
                CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
254 3364
                VTAILQ_REMOVE(&o->list, st, list);
255 3364
                sml_stv_free(stv, st);
256 3364
        }
257 6787
}
258
259
static void
260
sml_bocfini(const struct stevedore *stv, struct boc *boc)
261 4761
{
262 4761
        struct storage *st;
263
264 4761
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
265 4761
        CHECK_OBJ_NOTNULL(boc, BOC_MAGIC);
266
267 4761
        if (boc->stevedore_priv == NULL ||
268 4761
            boc->stevedore_priv == trim_once)
269 4476
                return;
270
271
        /* Free any leftovers from Trim */
272 285
        TAKE_OBJ_NOTNULL(st, &boc->stevedore_priv, STORAGE_MAGIC);
273 285
        sml_stv_free(stv, st);
274 285
}
275
276
/*
277
 * called in two cases:
278
 * - oc->boc == NULL: cache object on LRU freed
279
 * - oc->boc != NULL: cache object replaced for backend error
280
 */
281
static void v_matchproto_(objfree_f)
282
sml_objfree(struct worker *wrk, struct objcore *oc)
283 3681
{
284 3681
        const struct stevedore *stv;
285 3681
        struct storage *st;
286 3681
        struct object *o;
287
288 3681
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
289 3681
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
290 3681
        stv = oc->stobj->stevedore;
291 3681
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
292 3681
        CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
293
294 3681
        sml_slim(wrk, oc);
295 3681
        st = o->objstore;
296 3681
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
297 3681
        FINI_OBJ(o);
298
299 3681
        if (oc->boc != NULL)
300 25
                sml_bocfini(stv, oc->boc);
301 3656
        else if (stv->lru != NULL)
302 3655
                LRU_Remove(oc);
303
304 3681
        sml_stv_free(stv, st);
305
306 3681
        memset(oc->stobj, 0, sizeof oc->stobj);
307
308 3681
        wrk->stats->n_object--;
309 3681
}
310
311
// kept for reviewers - XXX remove later
312
#undef VAI_DBG
313
314
struct sml_hdl {
315
        struct vai_hdl_preamble preamble;
316
#define SML_HDL_MAGIC           0x37dfd996
317
        struct vai_qe           qe;
318
        struct pool_task        task;   // unfortunate
319
        struct ws               *ws;    // NULL is malloc()
320
        struct objcore          *oc;
321
        struct object           *obj;
322
        const struct stevedore  *stv;
323
        struct boc              *boc;
324
325
        struct storage          *st;    // updated by _lease()
326
327
        // only for _lease_boc()
328
        uint64_t                st_off; // already returned fragment of current st
329
        uint64_t                avail, returned;
330
        struct storage          *last;  // to resume
331
};
332
333
static inline void
334
sml_ai_viov_fill(struct viov *viov, struct storage *st)
335 4722
{
336 4722
        viov->iov.iov_base = TRUST_ME(st->ptr);
337 4722
        viov->iov.iov_len = st->len;
338 4722
        viov->lease = ptr2lease(st);
339 4722
        VAI_ASSERT_LEASE(viov->lease);
340 4722
}
341
342
// sml has no mechanism to notify "I got free space again now"
343
// (we could add that, but because storage.h is used in mgt, a first attempt
344
//  looks at least like this would cause some include spill for vai_q_head or
345
//  something similar)
346
//
347
// So anyway, to get ahead we just implement a pretty stupid "call the notify
348
// some time later" on a thread
349
static void
350
sml_ai_later_task(struct worker *wrk, void *priv)
351 0
{
352 0
        struct sml_hdl *hdl;
353 0
        const vtim_dur dur = 0.0042;
354
355 0
        (void)wrk;
356 0
        VTIM_sleep(dur);
357 0
        CAST_VAI_HDL_NOTNULL(hdl, priv, SML_HDL_MAGIC);
358 0
        memset(&hdl->task, 0, sizeof hdl->task);
359 0
        hdl->qe.cb(hdl, hdl->qe.priv);
360 0
}
361
static void
362
sml_ai_later(struct worker *wrk, struct sml_hdl *hdl)
363 0
{
364 0
        AZ(hdl->task.func);
365 0
        AZ(hdl->task.priv);
366 0
        hdl->task.func = sml_ai_later_task;
367 0
        hdl->task.priv = hdl;
368 0
        AZ(Pool_Task(wrk->pool, &hdl->task, TASK_QUEUE_BG));
369 0
}
370
371
372
static int
373
sml_ai_buffer(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
374 16
{
375 16
        const struct stevedore *stv;
376 16
        struct sml_hdl *hdl;
377 16
        struct storage *st;
378 16
        struct viov *vio;
379 16
        int r = 0;
380
381 16
        (void) wrk;
382 16
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
383 16
        stv = hdl->stv;
384 16
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
385
386 16
        VSCARAB_FOREACH(vio, scarab)
387 16
                if (vio->iov.iov_len > UINT_MAX)
388 0
                        return (-EINVAL);
389
390 16
        VSCARAB_FOREACH(vio, scarab) {
391 16
                st = objallocwithnuke(wrk, stv, vio->iov.iov_len, 0);
392 16
                if (st == NULL)
393 0
                        break;
394 16
                assert(st->space >= vio->iov.iov_len);
395 16
                st->flags = STORAGE_F_BUFFER;
396 16
                st->len = st->space;
397
398 16
                sml_ai_viov_fill(vio, st);
399 16
                r++;
400 16
        }
401 16
        if (r == 0) {
402 0
                sml_ai_later(wrk, hdl);
403 0
                r = -EAGAIN;
404 0
        }
405 16
        return (r);
406 16
}
407
408
static int
409
sml_ai_lease_simple(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
410 8309
{
411 8309
        struct storage *st;
412 8309
        struct sml_hdl *hdl;
413 8309
        struct viov *viov;
414 8309
        int r = 0;
415
416 8309
        (void) wrk;
417 8309
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
418 8309
        VSCARAB_CHECK_NOTNULL(scarab);
419
420 8309
        AZ(hdl->st_off);
421 8309
        st = hdl->st;
422 13014
        while (st != NULL && (viov = VSCARAB_GET(scarab)) != NULL) {
423 4705
                CHECK_OBJ(st, STORAGE_MAGIC);
424 4705
                sml_ai_viov_fill(viov, st);
425 4705
                r++;
426 4705
                st = VTAILQ_PREV(st, storagehead, list);
427 4705
        }
428 8309
        hdl->st = st;
429 8309
        if (st == NULL)
430 8308
                scarab->flags |= VSCARAB_F_END;
431 8309
        return (r);
432 8309
}
433
434
static int
435
sml_ai_lease_fail(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
436 0
{
437 0
        (void) wrk;
438 0
        (void) vhdl;
439 0
        (void) scarab;
440
441 0
        return (-EPIPE);
442 0
}
443
444
/*
445
 * on leases while streaming (with a boc):
446
 *
447
 * SML uses the lease return facility to implement the "free behind" for
448
 * OC_F_TRANSIENT objects. When streaming, we also return leases on
449
 * fragments of sts, but we must only "free behind" when we are done with the
450
 * last fragment.
451
 *
452
 * So we use a magic lease to signal "this is only a fragment", which we ignore
453
 * on returns
454
 */
455
456
static int
457
sml_ai_lease_boc(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
458 5476
{
459 5476
        enum boc_state_e state = BOS_INVALID;
460 5476
        struct storage *next;
461 5476
        struct sml_hdl *hdl;
462 5476
        struct viov *viov;
463 5476
        int r = 0;
464
465 5476
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
466 5476
        VSCARAB_CHECK_NOTNULL(scarab);
467 5476
        assert(hdl->boc == hdl->oc->boc);
468
469 5476
        if (hdl->avail == hdl->returned) {
470 5206
                hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned,
471 5206
                    &state, &hdl->qe);
472 5206
                assert(state >= BOS_STREAM);
473 5206
                if (state == BOS_FAILED) {
474 22
                        hdl->last = NULL;
475 22
                        return (-EPIPE);
476 22
                }
477 5184
                else if (state == BOS_FINISHED)
478 933
                        (void)0;
479 4251
                else if (hdl->avail == hdl->returned) {
480
                        // ObjVAIGetExtend() has scheduled a notification
481 2113
                        if (hdl->boc->transit_buffer > 0)
482 255
                                return (-ENOBUFS);
483 1858
                        else
484 1858
                                return (-EAGAIN);
485 2113
                }
486 2138
                else
487 2138
                        assert(state < BOS_FINISHED);
488 5206
        }
489 3341
        Lck_Lock(&hdl->boc->mtx);
490 3341
        if (hdl->st == NULL && hdl->last != NULL)
491 1048
                hdl->st = VTAILQ_PREV(hdl->last, storagehead, list);
492 3341
        if (hdl->last != NULL && state < BOS_FINISHED) {
493 157
                viov = VSCARAB_GET(scarab);
494 157
                AN(viov);
495 157
                viov->iov = IOV_NIL;
496 157
                viov->lease = ptr2lease(hdl->last);
497 157
                r++;
498 157
        }
499 3341
        if (hdl->last != NULL)
500 1048
                hdl->last = NULL;
501 3341
        if (hdl->st == NULL && hdl->returned == 0)
502 953
                hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead);
503 3341
        if (hdl->st == NULL)
504 886
                assert(hdl->avail == hdl->returned);
505
506 5946
        while (hdl->avail > hdl->returned && (viov = VSCARAB_GET(scarab)) != NULL) {
507 2605
                CHECK_OBJ_NOTNULL(hdl->st, STORAGE_MAGIC); // ObjVAIGetExtend ensures
508 2605
                assert(hdl->boc == hdl->oc->boc);
509 2605
                assert(hdl->st_off <= hdl->st->space);
510 2605
                size_t av = hdl->avail - hdl->returned;
511 2605
                size_t l = hdl->st->space - hdl->st_off;
512 2605
                AN(l);
513 2605
                if (l > av)
514 1140
                        l = av;
515 2605
                viov->iov.iov_base = TRUST_ME(hdl->st->ptr + hdl->st_off);
516 2605
                viov->iov.iov_len = l;
517 2605
                if (hdl->st_off + l == hdl->st->space) {
518 1465
                        next = VTAILQ_PREV(hdl->st, storagehead, list);
519 1465
                        AZ(hdl->last);
520 1465
                        if (next == NULL) {
521 1062
                                hdl->last = hdl->st;
522 1062
                                viov->lease = VAI_LEASE_NORET;
523 1062
                        }
524 403
                        else {
525 403
                                CHECK_OBJ(next, STORAGE_MAGIC);
526 403
                                viov->lease = ptr2lease(hdl->st);
527 403
                        }
528
#ifdef VAI_DBG
529
                        if (wrk->vsl)
530
                                VSLb(wrk->vsl, SLT_Debug, "off %zu + l %zu == space st %p next st %p stvprv %p",
531
                                    hdl->st_off, l, hdl->st, next, hdl->boc->stevedore_priv);
532
#endif
533 1465
                        hdl->st_off = 0;
534 1465
                        hdl->st = next;
535 1465
                }
536 1140
                else {
537 1140
                        viov->lease = VAI_LEASE_NORET;
538 1140
                        hdl->st_off += l;
539 1140
                }
540 2605
                hdl->returned += l;
541 2605
                VAI_ASSERT_LEASE(viov->lease);
542 2605
                r++;
543 2605
        }
544
545 3341
        Lck_Unlock(&hdl->boc->mtx);
546 3341
        if (state != BOS_FINISHED && hdl->avail == hdl->returned) {
547 2412
                hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned,
548 2412
                    &state, &hdl->qe);
549 2412
        }
550 3341
        if (state == BOS_FINISHED && hdl->avail == hdl->returned)
551 933
                scarab->flags |= VSCARAB_F_END;
552 3341
        return (r);
553 3341
}
554
555
// return only buffers, used if object is not streaming
556
static void v_matchproto_(vai_return_f)
557
sml_ai_return_buffers(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret)
558 4943
{
559 4943
        struct storage *st;
560 4943
        struct sml_hdl *hdl;
561 4943
        viov_lease_t *p;
562
563 4943
        (void) wrk;
564 4943
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
565
566 5432
        VSCARET_FOREACH(p, scaret) {
567 5432
                if (*p == VAI_LEASE_NORET)
568 643
                        continue;
569 4789
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
570 4789
                if ((st->flags & STORAGE_F_BUFFER) == 0)
571 4773
                        continue;
572 16
                sml_stv_free(hdl->stv, st);
573 16
        }
574 4943
        VSCARET_INIT(scaret, scaret->capacity);
575 4943
}
576
577
// generic return for buffers and object leases, used when streaming
578
static void v_matchproto_(vai_return_f)
579
sml_ai_return(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret)
580 1409
{
581 1409
        struct storage *st;
582 1409
        struct sml_hdl *hdl;
583 1409
        viov_lease_t *p;
584
585 1409
        (void) wrk;
586 1409
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
587 1409
        VSCARET_CHECK_NOTNULL(scaret);
588 1409
        if (scaret->used == 0)
589 0
                return;
590
591
        // callback is only registered if needed
592 1409
        assert(hdl->boc != NULL && (hdl->oc->flags & OC_F_TRANSIENT) != 0);
593
594
        // filter noret and last
595 1409
        VSCARET_LOCAL(todo, scaret->used);
596 2083
        VSCARET_FOREACH(p, scaret) {
597 2083
                if (*p == VAI_LEASE_NORET)
598 1594
                        continue;
599 489
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
600 489
                VSCARET_ADD(todo, *p);
601 489
        }
602 1409
        VSCARET_INIT(scaret, scaret->capacity);
603
604 1409
        Lck_Lock(&hdl->boc->mtx);
605 1409
        VSCARET_FOREACH(p, todo) {
606 489
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
607 489
                if ((st->flags & STORAGE_F_BUFFER) != 0)
608 0
                        continue;
609 489
                VTAILQ_REMOVE(&hdl->obj->list, st, list);
610 489
                if (st == hdl->boc->stevedore_priv)
611 0
                        hdl->boc->stevedore_priv = trim_once;
612 489
        }
613 1409
        Lck_Unlock(&hdl->boc->mtx);
614
615 1409
        VSCARET_FOREACH(p, todo) {
616 489
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
617
#ifdef VAI_DBG
618
                if (wrk->vsl != NULL)
619
                        VSLb(wrk->vsl, SLT_Debug, "ret %p", st);
620
#endif
621 489
                sml_stv_free(hdl->stv, st);
622 489
        }
623 1409
}
624
625
// call the notify_cb to allow VDPs to use -EAGAIN also
626
static void v_matchproto_(vai_return_f)
627
sml_ai_notify(struct worker *wrk, vai_hdl vhdl)
628 128
{
629 128
        struct sml_hdl *hdl;
630
631 128
        (void) wrk;
632 128
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
633 128
        CHECK_OBJ(&hdl->qe, VAI_Q_MAGIC);
634 128
        hdl->qe.cb(vhdl, hdl->qe.priv);
635 128
}
636
637
static void v_matchproto_(vai_fini_f)
638
sml_ai_fini(struct worker *wrk, vai_hdl *vai_hdlp)
639 5150
{
640 5150
        struct sml_hdl *hdl;
641
642 5150
        AN(vai_hdlp);
643 5150
        CAST_VAI_HDL_NOTNULL(hdl, *vai_hdlp, SML_HDL_MAGIC);
644 5150
        *vai_hdlp = NULL;
645
646 5150
        if (hdl->boc != NULL) {
647 962
                ObjVAICancel(wrk, hdl->boc, &hdl->qe);
648 962
                HSH_DerefBoc(wrk, hdl->oc);
649 962
                hdl->boc = NULL;
650 962
        }
651
652 5150
        if (hdl->ws != NULL)
653 32
                WS_Release(hdl->ws, 0);
654 5118
        else
655 5118
                free(hdl);
656 5150
}
657
658
static vai_hdl v_matchproto_(vai_init_f)
659
sml_ai_init(struct worker *wrk, struct objcore *oc, struct ws *ws,
660
    vai_notify_cb *notify, void *notify_priv)
661 5151
{
662 5151
        struct sml_hdl *hdl;
663 5151
        const size_t sz = sizeof *hdl;
664
665 5151
        if (ws != NULL && WS_ReserveSize(ws, (unsigned)sz))
666 32
                hdl = WS_Reservation(ws);
667 5119
        else {
668 5119
                hdl = malloc(sz);
669 5119
                ws = NULL;
670 5119
        }
671
672 5151
        AN(hdl);
673 5151
        INIT_VAI_HDL(hdl, SML_HDL_MAGIC);
674 5151
        hdl->preamble.vai_lease = sml_ai_lease_simple;
675 5151
        hdl->preamble.vai_buffer = sml_ai_buffer;
676 5151
        hdl->preamble.vai_return = sml_ai_return_buffers;
677 5151
        hdl->preamble.vai_notify = sml_ai_notify;
678 5151
        hdl->preamble.vai_fini = sml_ai_fini;
679 5151
        hdl->ws = ws;
680
681 5151
        hdl->oc = oc;
682 5151
        hdl->obj = sml_getobj(wrk, oc);
683 5151
        CHECK_OBJ_NOTNULL(hdl->obj, OBJECT_MAGIC);
684 5151
        hdl->stv = oc->stobj->stevedore;
685 5151
        CHECK_OBJ_NOTNULL(hdl->stv, STEVEDORE_MAGIC);
686
687 5151
        hdl->qe.magic = VAI_Q_MAGIC;
688 5151
        hdl->qe.cb = notify;
689 5151
        hdl->qe.hdl = hdl;
690 5151
        hdl->qe.priv = notify_priv;
691
692 5151
        hdl->boc = HSH_RefBoc(oc);
693 5151
        if (hdl->boc == NULL && oc->flags & OC_F_FAILED) {
694 0
                hdl->preamble.vai_lease = sml_ai_lease_fail;
695 0
                return (hdl);
696 0
        }
697 5151
        if (hdl->boc == NULL) {
698 4189
                hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead);
699 4189
                CHECK_OBJ_ORNULL(hdl->st, STORAGE_MAGIC);
700 4189
                return (hdl);
701 4189
        }
702
703 962
        hdl->preamble.vai_lease = sml_ai_lease_boc;
704 962
        if ((hdl->oc->flags & OC_F_TRANSIENT) != 0)
705 499
                hdl->preamble.vai_return = sml_ai_return;
706 962
        return (hdl);
707 5151
}
708
709
/*
710
 * trivial notification to allow the iterator to simply block
711
 */
712
struct sml_notify {
713
        unsigned                magic;
714
#define SML_NOTIFY_MAGIC        0x4589af31
715
        unsigned                hasmore;
716
        pthread_mutex_t         mtx;
717
        pthread_cond_t          cond;
718
};
719
720
static void
721
sml_notify_init(struct sml_notify *sn)
722 7097
{
723
724 7097
        INIT_OBJ(sn, SML_NOTIFY_MAGIC);
725 7097
        PTOK(pthread_mutex_init(&sn->mtx, NULL));
726 7097
        PTOK(pthread_cond_init(&sn->cond, NULL));
727 7097
}
728
729
static void
730
sml_notify_fini(struct sml_notify *sn)
731 7095
{
732
733 7095
        CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC);
734 7095
        PTOK(pthread_mutex_destroy(&sn->mtx));
735 7096
        PTOK(pthread_cond_destroy(&sn->cond));
736 7096
}
737
738
static void v_matchproto_(vai_notify_cb)
739
sml_notify(vai_hdl hdl, void *priv)
740 2233
{
741 2233
        struct sml_notify *sn;
742
743 2233
        (void) hdl;
744 2233
        CAST_OBJ_NOTNULL(sn, priv, SML_NOTIFY_MAGIC);
745 2233
        PTOK(pthread_mutex_lock(&sn->mtx));
746 2233
        sn->hasmore = 1;
747 2233
        PTOK(pthread_cond_signal(&sn->cond));
748 2233
        PTOK(pthread_mutex_unlock(&sn->mtx));
749
750 2233
}
751
752
static void
753
sml_notify_wait(struct sml_notify *sn)
754 2100
{
755
756 2100
        CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC);
757 2100
        PTOK(pthread_mutex_lock(&sn->mtx));
758 3246
        while (sn->hasmore == 0)
759 2100
                PTOK(pthread_cond_wait(&sn->cond, &sn->mtx));
760 2100
        AN(sn->hasmore);
761 2100
        sn->hasmore = 0;
762 2100
        PTOK(pthread_mutex_unlock(&sn->mtx));
763 2100
}
764
765
int v_matchproto_(objiterator_f)
766
SML_iterator(struct worker *wrk, struct objcore *oc,
767
    void *priv, objiterate_f *func, int final)
768 7097
{
769 7097
        struct sml_notify sn;
770 7097
        struct viov *vio, *last;
771 7097
        unsigned u, uu;
772 7097
        vai_hdl hdl;
773 7097
        int nn, r, r2, islast;
774
775 7097
        VSCARAB_LOCAL(scarab, 16);
776 7097
        VSCARET_LOCAL(scaret, 16);
777
778 7097
        (void) final; // phase out?
779 7097
        sml_notify_init(&sn);
780 7097
        hdl = ObjVAIinit(wrk, oc, NULL, sml_notify, &sn);
781 7097
        AN(hdl);
782
783 7097
        r = u = 0;
784
785 15301
        do {
786 17711
                do {
787 17711
                        nn = ObjVAIlease(wrk, hdl, scarab);
788 17711
                        if (nn <= 0 || scarab->flags & VSCARAB_F_END)
789 15295
                                break;
790 17711
                } while (scarab->used < scarab->capacity);
791
792
                /*
793
                 * nn is the wait/return action or 0
794
                 * nn tells us if to flush
795
                 */
796 0
                uu = u;
797 15301
                last = VSCARAB_LAST(scarab);
798 17435
                VSCARAB_FOREACH(vio, scarab) {
799 17435
                        islast = vio == last;
800 17435
                        AZ(u & OBJ_ITER_END);
801 17435
                        if (islast && scarab->flags & VSCARAB_F_END)
802 6143
                                u |= OBJ_ITER_END;
803
804
                        // flush if it is the scarab's last IOV and we will block next
805
                        // or if we need space in the return leases array
806 17435
                        uu = u;
807 17435
                        if ((islast && nn < 0) || scaret->used == scaret->capacity - 1)
808 2007
                                uu |= OBJ_ITER_FLUSH;
809
810
                        // null iov with the only purpose to return the resume ptr lease
811
                        // exception needed because assert(len > 0) in VDP_bytes()
812 17435
                        if (vio->iov.iov_base == IOV_NIL.iov_base)
813 157
                                r = 0;
814 17278
                        else
815 17278
                                r = func(priv, uu, vio->iov.iov_base, vio->iov.iov_len);
816 17435
                        if (r != 0)
817 59
                                break;
818
819
                        // sufficient space ensured by capacity check above
820 17376
                        VSCARET_ADD(scaret, vio->lease);
821
822
#ifdef VAI_DBG
823
                        if (wrk->vsl)
824
                                VSLb(wrk->vsl, SLT_Debug, "len %zu scaret %u uu %u",
825
                                    vio->iov.iov_len, scaret->used, uu);
826
#endif
827
828
                        // whenever we have flushed, return leases
829 17376
                        if ((uu & OBJ_ITER_FLUSH) && scaret->used > 0)
830 1994
                                ObjVAIreturn(wrk, hdl, scaret);
831 17376
                }
832
833
                // return leases which we did not use if error (break)
834 15301
                VSCARAB_FOREACH_RESUME(vio, scarab) {
835 88
                        if (scaret->used == scaret->capacity)
836 0
                                ObjVAIreturn(wrk, hdl, scaret);
837 88
                        VSCARET_ADD(scaret, vio->lease);
838 88
                }
839
840
                // we have now completed the scarab
841 15301
                VSCARAB_INIT(scarab, scarab->capacity);
842
843
#ifdef VAI_DBG
844
                if (wrk->vsl)
845
                        VSLb(wrk->vsl, SLT_Debug, "r %d nn %d uu %u",
846
                            r, nn, uu);
847
#endif
848
849
                // flush before blocking, unless flush or end already sent
850 15301
                if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN) && uu == 0) {
851 109
                        r = func(priv, OBJ_ITER_FLUSH, NULL, 0);
852 109
                        if (scaret->used > 0)
853 0
                                ObjVAIreturn(wrk, hdl, scaret);
854 109
                }
855
856 15301
                if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN)) {
857 2100
                        assert(scaret->used <= 1);
858 2100
                        sml_notify_wait(&sn);
859 2100
                }
860 13201
                else if (r == 0 && nn < 0)
861 22
                        r = -1;
862 15301
        } while (nn != 0 && r == 0);
863
864 7097
        if ((u & OBJ_ITER_END) == 0) {
865 953
                r2 = func(priv, OBJ_ITER_END, NULL, 0);
866 953
                if (r == 0)
867 914
                        r = r2;
868 953
        }
869
870 7097
        if (scaret->used > 0)
871 6161
                ObjVAIreturn(wrk, hdl, scaret);
872
873 7097
        ObjVAIfini(wrk, &hdl);
874 7097
        sml_notify_fini(&sn);
875
876 7097
        return (r);
877 7097
}
878
879
/*--------------------------------------------------------------------
880
 */
881
882
static struct storage *
883
objallocwithnuke(struct worker *wrk, const struct stevedore *stv, ssize_t size,
884
    int flags)
885 5855
{
886 5855
        struct storage *st = NULL;
887
888 5855
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
889 5855
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
890
891 5855
        if (size > cache_param->fetch_maxchunksize) {
892 2
                if (!(flags & LESS_MEM_ALLOCED_IS_OK))
893 0
                        return (NULL);
894 2
                size = cache_param->fetch_maxchunksize;
895 2
        }
896
897 5855
        assert(size <= UINT_MAX);       /* field limit in struct storage */
898
899 5863
        do {
900
                /* try to allocate from it */
901 5863
                st = sml_stv_alloc(stv, size, flags);
902 5863
                if (st != NULL)
903 5852
                        break;
904
905
                /* no luck; try to free some space and keep trying */
906 11
                if (stv->lru == NULL)
907 0
                        break;
908 11
        } while (LRU_NukeOne(wrk, stv->lru));
909
910 5855
        CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
911 5855
        return (st);
912 5855
}
913
914
static int v_matchproto_(objgetspace_f)
915
sml_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz,
916
    uint8_t **ptr)
917 68018
{
918 68018
        struct object *o;
919 68018
        struct storage *st;
920
921 68018
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
922 68018
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
923 68018
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
924 68018
        AN(sz);
925 68018
        AN(ptr);
926 68018
        if (*sz == 0)
927 64351
                *sz = cache_param->fetch_chunksize;
928 68018
        assert(*sz > 0);
929 68018
        if (oc->boc->transit_buffer > 0)
930 455
                *sz = vmin_t(ssize_t, *sz, oc->boc->transit_buffer);
931
932 68018
        o = sml_getobj(wrk, oc);
933 68018
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
934 68018
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
935
936 68018
        st = VTAILQ_FIRST(&o->list);
937 68018
        if (st != NULL && st->len < st->space) {
938 63342
                *sz = st->space - st->len;
939 63342
                *ptr = st->ptr + st->len;
940 63342
                assert (*sz > 0);
941 63342
                return (1);
942 63342
        }
943
944 4676
        st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz,
945 4676
            LESS_MEM_ALLOCED_IS_OK);
946 4676
        if (st == NULL)
947 5
                return (0);
948
949 4671
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
950 4671
        Lck_Lock(&oc->boc->mtx);
951 4671
        VTAILQ_INSERT_HEAD(&o->list, st, list);
952 4671
        Lck_Unlock(&oc->boc->mtx);
953
954 4671
        *sz = st->space - st->len;
955 4671
        assert (*sz > 0);
956 4671
        *ptr = st->ptr + st->len;
957 4671
        return (1);
958 4671
}
959
960
static void v_matchproto_(objextend_f)
961
sml_extend(struct worker *wrk, struct objcore *oc, ssize_t l)
962 64383
{
963 64383
        struct object *o;
964 64383
        struct storage *st;
965
966 64383
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
967 64383
        assert(l > 0);
968
969 64383
        o = sml_getobj(wrk, oc);
970 64383
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
971 64383
        st = VTAILQ_FIRST(&o->list);
972 64383
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
973 64383
        assert(st->len + l <= st->space);
974 64383
        st->len += l;
975 64383
}
976
977
static void v_matchproto_(objtrimstore_f)
978
sml_trimstore(struct worker *wrk, struct objcore *oc)
979 2016
{
980 2016
        const struct stevedore *stv;
981 2016
        struct storage *st, *st1;
982 2016
        struct object *o;
983
984 2016
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
985 2016
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
986 2016
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
987
988 2016
        stv = oc->stobj->stevedore;
989 2016
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
990
991 2016
        if (oc->boc->stevedore_priv != NULL)
992 0
                WRONG("sml_trimstore already called");
993 2016
        oc->boc->stevedore_priv = trim_once;
994
995 2016
        o = sml_getobj(wrk, oc);
996 2016
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
997 2016
        st = VTAILQ_FIRST(&o->list);
998
999 2016
        if (st == NULL)
1000 0
                return;
1001
1002 2016
        if (st->len == 0) {
1003 63
                Lck_Lock(&oc->boc->mtx);
1004 63
                VTAILQ_REMOVE(&o->list, st, list);
1005 63
                Lck_Unlock(&oc->boc->mtx);
1006
                /* sml_bocdone frees this */
1007 63
                oc->boc->stevedore_priv = st;
1008 63
                return;
1009 63
        }
1010
1011 1953
        if (stv->sml_free == NULL)
1012 0
                return;
1013
1014 1953
        if (st->space - st->len < 512)
1015 1731
                return;
1016
1017 222
        st1 = sml_stv_alloc(stv, st->len, 0);
1018 222
        if (st1 == NULL)
1019 0
                return;
1020 222
        assert(st1->space >= st->len);
1021
1022 222
        vmemcpy(st1->ptr, st->ptr, st->len);
1023 222
        st1->len = st->len;
1024 222
        Lck_Lock(&oc->boc->mtx);
1025 222
        VTAILQ_REMOVE(&o->list, st, list);
1026 222
        VTAILQ_INSERT_HEAD(&o->list, st1, list);
1027 222
        Lck_Unlock(&oc->boc->mtx);
1028
        /* sml_bocdone frees this */
1029 222
        oc->boc->stevedore_priv = st;
1030 222
}
1031
1032
static void v_matchproto_(objbocdone_f)
1033
sml_bocdone(struct worker *wrk, struct objcore *oc, struct boc *boc)
1034 4736
{
1035 4736
        const struct stevedore *stv;
1036
1037 4736
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1038 4736
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
1039 4736
        CHECK_OBJ_NOTNULL(boc, BOC_MAGIC);
1040 4736
        stv = oc->stobj->stevedore;
1041 4736
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
1042
1043 4736
        sml_bocfini(stv, boc);
1044
1045 4736
        if (stv->lru != NULL) {
1046 4701
                if (isnan(wrk->lastused))
1047 0
                        wrk->lastused = VTIM_real();
1048 4701
                LRU_Add(oc, wrk->lastused);     // approx timestamp is OK
1049 4701
        }
1050 4736
}
1051
1052
static const void * v_matchproto_(objgetattr_f)
1053
sml_getattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
1054
   ssize_t *len)
1055 38083
{
1056 38083
        struct object *o;
1057 38083
        ssize_t dummy;
1058
1059 38083
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1060
1061 38083
        if (len == NULL)
1062 20812
                len = &dummy;
1063 38083
        o = sml_getobj(wrk, oc);
1064 38083
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
1065
1066 38083
        switch (attr) {
1067
                /* Fixed size attributes */
1068 0
#define OBJ_FIXATTR(U, l, s)                                            \
1069 28826
        case OA_##U:                                                    \
1070 28826
                *len = sizeof o->fa_##l;                                \
1071 28826
                return (o->fa_##l);
1072 0
#include "tbl/obj_attr.h"
1073
1074
                /* Variable size attributes */
1075 0
#define OBJ_VARATTR(U, l)                                               \
1076 8706
        case OA_##U:                                                    \
1077 8706
                if (o->va_##l == NULL)                                  \
1078 8697
                        return (NULL);                                  \
1079 8697
                *len = o->va_##l##_len;                                 \
1080 8697
                return (o->va_##l);
1081 2
#include "tbl/obj_attr.h"
1082
1083
                /* Auxiliary attributes */
1084 0
#define OBJ_AUXATTR(U, l)                                               \
1085 563
        case OA_##U:                                                    \
1086 563
                if (o->aa_##l == NULL)                                  \
1087 555
                        return (NULL);                                  \
1088 555
                CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);            \
1089 555
                *len = o->aa_##l->len;                                  \
1090 555
                return (o->aa_##l->ptr);
1091 5383
#include "tbl/obj_attr.h"
1092
1093 0
        default:
1094 0
                break;
1095 38083
        }
1096 0
        WRONG("Unsupported OBJ_ATTR");
1097 0
}
1098
1099
static void * v_matchproto_(objsetattr_f)
1100
sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
1101
    ssize_t len, const void *ptr)
1102 15677
{
1103 15677
        struct object *o;
1104 15677
        void *retval = NULL;
1105 15677
        struct storage *st;
1106
1107 15677
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1108
1109 15677
        o = sml_getobj(wrk, oc);
1110 15677
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
1111 15677
        st = o->objstore;
1112
1113 15677
        switch (attr) {
1114
                /* Fixed size attributes */
1115 0
#define OBJ_FIXATTR(U, l, s)                                            \
1116 12058
        case OA_##U:                                                    \
1117 12058
                assert(len == sizeof o->fa_##l);                        \
1118 12058
                retval = o->fa_##l;                                     \
1119 12058
                break;
1120 0
#include "tbl/obj_attr.h"
1121
1122
                /* Variable size attributes */
1123 0
#define OBJ_VARATTR(U, l)                                               \
1124 3164
        case OA_##U:                                                    \
1125 3164
                if (o->va_##l##_len > 0) {                              \
1126 0
                        AN(o->va_##l);                                  \
1127 0
                        assert(len == o->va_##l##_len);                 \
1128 0
                        retval = o->va_##l;                             \
1129 3164
                } else if (len > 0) {                                   \
1130 3164
                        assert(len <= UINT_MAX);                        \
1131 3164
                        assert(st->len + len <= st->space);             \
1132 3164
                        o->va_##l = st->ptr + st->len;                  \
1133 3164
                        st->len += len;                                 \
1134 3164
                        o->va_##l##_len = len;                          \
1135 3164
                        retval = o->va_##l;                             \
1136 3164
                }                                                       \
1137 3164
                break;
1138 2943
#include "tbl/obj_attr.h"
1139
1140
                /* Auxiliary attributes */
1141 2943
#define OBJ_AUXATTR(U, l)                                               \
1142 2943
        case OA_##U:                                                    \
1143 458
                if (o->aa_##l != NULL) {                                \
1144 0
                        CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);    \
1145 0
                        assert(len == o->aa_##l->len);                  \
1146 0
                        retval = o->aa_##l->ptr;                        \
1147 0
                        break;                                          \
1148 0
                }                                                       \
1149 458
                if (len == 0)                                           \
1150 458
                        break;                                          \
1151 458
                o->aa_##l = objallocwithnuke(wrk, oc->stobj->stevedore, \
1152 458
                    len, 0);                                            \
1153 458
                if (o->aa_##l == NULL)                                  \
1154 458
                        break;                                          \
1155 458
                CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);            \
1156 458
                assert(len <= o->aa_##l->space);                        \
1157 458
                o->aa_##l->len = len;                                   \
1158 458
                retval = o->aa_##l->ptr;                                \
1159 458
                break;
1160 2943
#include "tbl/obj_attr.h"
1161
1162 0
        default:
1163 0
                WRONG("Unsupported OBJ_ATTR");
1164 0
                break;
1165 15677
        }
1166
1167 15674
        if (retval != NULL && ptr != NULL)
1168 741
                vmemcpy(retval, ptr, len);
1169 15673
        return (retval);
1170 15677
}
1171
1172
const struct obj_methods SML_methods = {
1173
        .objfree        = sml_objfree,
1174
        .objiterator    = SML_iterator,
1175
        .objgetspace    = sml_getspace,
1176
        .objextend      = sml_extend,
1177
        .objtrimstore   = sml_trimstore,
1178
        .objbocdone     = sml_bocdone,
1179
        .objslim        = sml_slim,
1180
        .objgetattr     = sml_getattr,
1181
        .objsetattr     = sml_setattr,
1182
        .objtouch       = LRU_Touch,
1183
        .vai_init       = sml_ai_init
1184
};
1185
1186
static void
1187
sml_panic_st(struct vsb *vsb, const char *hd, const struct storage *st)
1188 4
{
1189 4
        VSB_printf(vsb, "%s = %p {priv=%p, ptr=%p, len=%u, space=%u},\n",
1190 4
            hd, st, st->priv, st->ptr, st->len, st->space);
1191 4
}
1192
1193
void
1194
SML_panic(struct vsb *vsb, const struct objcore *oc)
1195 2
{
1196 2
        struct object *o;
1197 2
        struct storage *st;
1198
1199 2
        VSB_printf(vsb, "Simple = %p,\n", oc->stobj->priv);
1200 2
        if (oc->stobj->priv == NULL)
1201 0
                return;
1202 2
        o = oc->stobj->priv;
1203 2
        PAN_CheckMagic(vsb, o, OBJECT_MAGIC);
1204 2
        sml_panic_st(vsb, "Obj", o->objstore);
1205
1206 2
#define OBJ_FIXATTR(U, l, sz) \
1207 10
        VSB_printf(vsb, "%s = ", #U); \
1208 10
        VSB_quote(vsb, (const void*)o->fa_##l, sz, VSB_QUOTE_HEX); \
1209 10
        VSB_printf(vsb, ",\n");
1210
1211 2
#define OBJ_VARATTR(U, l) \
1212 4
        VSB_printf(vsb, "%s = {len=%u, ptr=%p},\n", \
1213 4
            #U, o->va_##l##_len, o->va_##l);
1214
1215 2
#define OBJ_AUXATTR(U, l)                                               \
1216 2
        do {                                                            \
1217 2
                if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l);\
1218 2
        } while(0);
1219
1220 2
#include "tbl/obj_attr.h"
1221
1222 2
        VTAILQ_FOREACH(st, &o->list, list) {
1223 2
                sml_panic_st(vsb, "Body", st);
1224 2
        }
1225 2
}