| Code Coverage: vinyl-cache/bin/vinyld/storage/storage_simple.c |
| |
| 82217aa4fc7094b3610340078afaf78cd95a8ec9 |
| |
| 0 |
|
/*- |
| 1 |
|
* Copyright (c) 2007-2015 Varnish Software AS |
| 2 |
|
* All rights reserved. |
| 3 |
|
* |
| 4 |
|
* Author: Poul-Henning Kamp <phk@phk.freebsd.dk> |
| 5 |
|
* |
| 6 |
|
* SPDX-License-Identifier: BSD-2-Clause |
| 7 |
|
* |
| 8 |
|
* Redistribution and use in source and binary forms, with or without |
| 9 |
|
* modification, are permitted provided that the following conditions |
| 10 |
|
* are met: |
| 11 |
|
* 1. Redistributions of source code must retain the above copyright |
| 12 |
|
* notice, this list of conditions and the following disclaimer. |
| 13 |
|
* 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
|
* notice, this list of conditions and the following disclaimer in the |
| 15 |
|
* documentation and/or other materials provided with the distribution. |
| 16 |
|
* |
| 17 |
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
| 18 |
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 19 |
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 20 |
|
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE |
| 21 |
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 22 |
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 23 |
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 24 |
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 25 |
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 26 |
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 27 |
|
* SUCH DAMAGE. |
| 28 |
|
* |
| 29 |
|
*/ |
| 30 |
|
|
| 31 |
|
#include "config.h" |
| 32 |
|
|
| 33 |
|
#include <stdlib.h> |
| 34 |
|
|
| 35 |
|
#include "cache/cache_int.h" |
| 36 |
|
|
| 37 |
|
#include "cache/cache_obj.h" |
| 38 |
|
#include "cache/cache_objhead.h" |
| 39 |
|
|
| 40 |
|
#include "storage/storage.h" |
| 41 |
|
#include "storage/storage_simple.h" |
| 42 |
|
#include "storage/storage_vai.h" |
| 43 |
|
|
| 44 |
|
#include "vtim.h" |
| 45 |
|
|
| 46 |
|
/* Flags for allocating memory in sml_stv_alloc */ |
| 47 |
10765 |
#define LESS_MEM_ALLOCED_IS_OK 1 |
| 48 |
|
|
| 49 |
|
// marker pointer for sml_trimstore |
| 50 |
|
static void *trim_once = &trim_once; |
| 51 |
|
|
| 52 |
|
/*-------------------------------------------------------------------*/ |
| 53 |
|
|
| 54 |
|
static struct storage * |
| 55 |
|
objallocwithnuke(struct worker *, const struct stevedore *, ssize_t size, |
| 56 |
|
int flags); |
| 57 |
|
|
| 58 |
|
static struct storage * |
| 59 |
|
sml_stv_alloc(const struct stevedore *stv, ssize_t size, int flags) |
| 60 |
6087 |
{ |
| 61 |
6087 |
struct storage *st; |
| 62 |
|
|
| 63 |
6087 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 64 |
6087 |
AN(stv->sml_alloc); |
| 65 |
|
|
| 66 |
6087 |
if (!(flags & LESS_MEM_ALLOCED_IS_OK)) { |
| 67 |
1406 |
if (size > cache_param->fetch_maxchunksize) |
| 68 |
0 |
return (NULL); |
| 69 |
1406 |
else |
| 70 |
1406 |
return (stv->sml_alloc(stv, size)); |
| 71 |
1406 |
} |
| 72 |
|
|
| 73 |
4681 |
if (size > cache_param->fetch_maxchunksize) |
| 74 |
0 |
size = cache_param->fetch_maxchunksize; |
| 75 |
|
|
| 76 |
4681 |
assert(size <= UINT_MAX); /* field limit in struct storage */ |
| 77 |
|
|
| 78 |
4816 |
for (;;) { |
| 79 |
|
/* try to allocate from it */ |
| 80 |
4816 |
assert(size > 0); |
| 81 |
4816 |
st = stv->sml_alloc(stv, size); |
| 82 |
4816 |
if (st != NULL) |
| 83 |
4670 |
break; |
| 84 |
|
|
| 85 |
146 |
if (size <= cache_param->fetch_chunksize) |
| 86 |
11 |
break; |
| 87 |
|
|
| 88 |
135 |
size /= 2; |
| 89 |
135 |
} |
| 90 |
4681 |
CHECK_OBJ_ORNULL(st, STORAGE_MAGIC); |
| 91 |
4681 |
return (st); |
| 92 |
4681 |
} |
| 93 |
|
|
| 94 |
|
static void |
| 95 |
|
sml_stv_free(const struct stevedore *stv, struct storage *st) |
| 96 |
8892 |
{ |
| 97 |
|
|
| 98 |
8892 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 99 |
8892 |
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); |
| 100 |
8892 |
if (stv->sml_free != NULL) |
| 101 |
8892 |
stv->sml_free(st); |
| 102 |
8892 |
} |
| 103 |
|
|
| 104 |
|
/*-------------------------------------------------------------------- |
| 105 |
|
* This function is called by stevedores ->allocobj() method, which |
| 106 |
|
* very often will be SML_allocobj() below, to convert a slab |
| 107 |
|
* of storage into object which the stevedore can then register in its |
| 108 |
|
* internal state, before returning it to STV_NewObject(). |
| 109 |
|
* As you probably guessed: All this for persistence. |
| 110 |
|
*/ |
| 111 |
|
|
| 112 |
|
struct object * |
| 113 |
|
SML_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr) |
| 114 |
4746 |
{ |
| 115 |
4746 |
struct object *o; |
| 116 |
|
|
| 117 |
4746 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 118 |
4746 |
AN(stv->methods); |
| 119 |
4746 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 120 |
|
|
| 121 |
4746 |
assert(PAOK(ptr)); |
| 122 |
|
|
| 123 |
4746 |
o = ptr; |
| 124 |
4746 |
INIT_OBJ(o, OBJECT_MAGIC); |
| 125 |
|
|
| 126 |
4746 |
VTAILQ_INIT(&o->list); |
| 127 |
|
|
| 128 |
4746 |
oc->stobj->stevedore = stv; |
| 129 |
4746 |
oc->stobj->priv = o; |
| 130 |
4746 |
oc->stobj->priv2 = 0; |
| 131 |
4746 |
return (o); |
| 132 |
4746 |
} |
| 133 |
|
|
| 134 |
|
/*-------------------------------------------------------------------- |
| 135 |
|
* This is the default ->allocobj() which all stevedores who do not |
| 136 |
|
* implement persistent storage can rely on. |
| 137 |
|
*/ |
| 138 |
|
|
| 139 |
|
int v_matchproto_(storage_allocobj_f) |
| 140 |
|
SML_allocobj(struct worker *wrk, const struct stevedore *stv, |
| 141 |
|
struct objcore *oc, unsigned wsl) |
| 142 |
4736 |
{ |
| 143 |
4736 |
struct object *o; |
| 144 |
4736 |
struct storage *st = NULL; |
| 145 |
4736 |
unsigned ltot; |
| 146 |
|
|
| 147 |
4736 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 148 |
4736 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 149 |
4736 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 150 |
|
|
| 151 |
4736 |
AN(stv->sml_alloc); |
| 152 |
|
|
| 153 |
4736 |
ltot = sizeof(*o) + PRNDUP(wsl); |
| 154 |
|
|
| 155 |
4739 |
do { |
| 156 |
4739 |
st = stv->sml_alloc(stv, ltot); |
| 157 |
4739 |
if (st != NULL && st->space < ltot) { |
| 158 |
0 |
stv->sml_free(st); |
| 159 |
0 |
st = NULL; |
| 160 |
0 |
} |
| 161 |
4739 |
} while (st == NULL && LRU_NukeOne(wrk, stv->lru)); |
| 162 |
4736 |
if (st == NULL) |
| 163 |
9 |
return (0); |
| 164 |
|
|
| 165 |
4727 |
CHECK_OBJ(st, STORAGE_MAGIC); |
| 166 |
4727 |
o = SML_MkObject(stv, oc, st->ptr); |
| 167 |
4727 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 168 |
4727 |
st->len = sizeof(*o); |
| 169 |
4727 |
o->objstore = st; |
| 170 |
4727 |
return (1); |
| 171 |
4727 |
} |
| 172 |
|
|
| 173 |
|
void * v_matchproto_(storage_allocbuf_t) |
| 174 |
|
SML_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size, |
| 175 |
|
uintptr_t *ppriv) |
| 176 |
708 |
{ |
| 177 |
708 |
struct storage *st; |
| 178 |
|
|
| 179 |
708 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 180 |
708 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 181 |
708 |
AN(ppriv); |
| 182 |
|
|
| 183 |
708 |
if (size > UINT_MAX) |
| 184 |
0 |
return (NULL); |
| 185 |
708 |
st = objallocwithnuke(wrk, stv, size, 0); |
| 186 |
708 |
if (st == NULL) |
| 187 |
0 |
return (NULL); |
| 188 |
708 |
assert(st->space >= size); |
| 189 |
708 |
st->flags = STORAGE_F_BUFFER; |
| 190 |
708 |
st->len = size; |
| 191 |
708 |
*ppriv = (uintptr_t)st; |
| 192 |
708 |
return (st->ptr); |
| 193 |
708 |
} |
| 194 |
|
|
| 195 |
|
void v_matchproto_(storage_freebuf_t) |
| 196 |
|
SML_FreeBuf(struct worker *wrk, const struct stevedore *stv, uintptr_t priv) |
| 197 |
708 |
{ |
| 198 |
708 |
struct storage *st; |
| 199 |
|
|
| 200 |
708 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 201 |
708 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 202 |
|
|
| 203 |
708 |
CAST_OBJ_NOTNULL(st, (void *)priv, STORAGE_MAGIC); |
| 204 |
708 |
assert(st->flags == STORAGE_F_BUFFER); |
| 205 |
708 |
sml_stv_free(stv, st); |
| 206 |
708 |
} |
| 207 |
|
|
| 208 |
|
/*--------------------------------------------------------------------- |
| 209 |
|
*/ |
| 210 |
|
|
| 211 |
|
static struct object * |
| 212 |
|
sml_getobj(struct worker *wrk, struct objcore *oc) |
| 213 |
200030 |
{ |
| 214 |
200030 |
const struct stevedore *stv; |
| 215 |
200030 |
struct object *o; |
| 216 |
|
|
| 217 |
200030 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 218 |
200030 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 219 |
200030 |
stv = oc->stobj->stevedore; |
| 220 |
200030 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 221 |
200030 |
if (stv->sml_getobj != NULL) |
| 222 |
256 |
return (stv->sml_getobj(wrk, oc)); |
| 223 |
199774 |
if (oc->stobj->priv == NULL) |
| 224 |
0 |
return (NULL); |
| 225 |
199774 |
CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC); |
| 226 |
199774 |
return (o); |
| 227 |
199774 |
} |
| 228 |
|
|
| 229 |
|
static void v_matchproto_(objslim_f) |
| 230 |
|
sml_slim(struct worker *wrk, struct objcore *oc) |
| 231 |
6787 |
{ |
| 232 |
6787 |
const struct stevedore *stv; |
| 233 |
6787 |
struct object *o; |
| 234 |
6787 |
struct storage *st, *stn; |
| 235 |
|
|
| 236 |
6787 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 237 |
|
|
| 238 |
6787 |
stv = oc->stobj->stevedore; |
| 239 |
6787 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 240 |
6787 |
o = sml_getobj(wrk, oc); |
| 241 |
6787 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 242 |
|
|
| 243 |
6787 |
#define OBJ_AUXATTR(U, l) \ |
| 244 |
6787 |
do { \ |
| 245 |
6787 |
if (o->aa_##l != NULL) { \ |
| 246 |
349 |
sml_stv_free(stv, o->aa_##l); \ |
| 247 |
349 |
o->aa_##l = NULL; \ |
| 248 |
349 |
} \ |
| 249 |
6787 |
} while (0); |
| 250 |
6787 |
#include "tbl/obj_attr.h" |
| 251 |
|
|
| 252 |
6787 |
VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) { |
| 253 |
3364 |
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); |
| 254 |
3364 |
VTAILQ_REMOVE(&o->list, st, list); |
| 255 |
3364 |
sml_stv_free(stv, st); |
| 256 |
3364 |
} |
| 257 |
6787 |
} |
| 258 |
|
|
| 259 |
|
static void |
| 260 |
|
sml_bocfini(const struct stevedore *stv, struct boc *boc) |
| 261 |
4761 |
{ |
| 262 |
4761 |
struct storage *st; |
| 263 |
|
|
| 264 |
4761 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 265 |
4761 |
CHECK_OBJ_NOTNULL(boc, BOC_MAGIC); |
| 266 |
|
|
| 267 |
4761 |
if (boc->stevedore_priv == NULL || |
| 268 |
4761 |
boc->stevedore_priv == trim_once) |
| 269 |
4476 |
return; |
| 270 |
|
|
| 271 |
|
/* Free any leftovers from Trim */ |
| 272 |
285 |
TAKE_OBJ_NOTNULL(st, &boc->stevedore_priv, STORAGE_MAGIC); |
| 273 |
285 |
sml_stv_free(stv, st); |
| 274 |
285 |
} |
| 275 |
|
|
| 276 |
|
/* |
| 277 |
|
* called in two cases: |
| 278 |
|
* - oc->boc == NULL: cache object on LRU freed |
| 279 |
|
* - oc->boc != NULL: cache object replaced for backend error |
| 280 |
|
*/ |
| 281 |
|
static void v_matchproto_(objfree_f) |
| 282 |
|
sml_objfree(struct worker *wrk, struct objcore *oc) |
| 283 |
3681 |
{ |
| 284 |
3681 |
const struct stevedore *stv; |
| 285 |
3681 |
struct storage *st; |
| 286 |
3681 |
struct object *o; |
| 287 |
|
|
| 288 |
3681 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 289 |
3681 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 290 |
3681 |
stv = oc->stobj->stevedore; |
| 291 |
3681 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 292 |
3681 |
CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC); |
| 293 |
|
|
| 294 |
3681 |
sml_slim(wrk, oc); |
| 295 |
3681 |
st = o->objstore; |
| 296 |
3681 |
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); |
| 297 |
3681 |
FINI_OBJ(o); |
| 298 |
|
|
| 299 |
3681 |
if (oc->boc != NULL) |
| 300 |
25 |
sml_bocfini(stv, oc->boc); |
| 301 |
3656 |
else if (stv->lru != NULL) |
| 302 |
3655 |
LRU_Remove(oc); |
| 303 |
|
|
| 304 |
3681 |
sml_stv_free(stv, st); |
| 305 |
|
|
| 306 |
3681 |
memset(oc->stobj, 0, sizeof oc->stobj); |
| 307 |
|
|
| 308 |
3681 |
wrk->stats->n_object--; |
| 309 |
3681 |
} |
| 310 |
|
|
| 311 |
|
// kept for reviewers - XXX remove later |
| 312 |
|
#undef VAI_DBG |
| 313 |
|
|
| 314 |
|
struct sml_hdl { |
| 315 |
|
struct vai_hdl_preamble preamble; |
| 316 |
|
#define SML_HDL_MAGIC 0x37dfd996 |
| 317 |
|
struct vai_qe qe; |
| 318 |
|
struct pool_task task; // unfortunate |
| 319 |
|
struct ws *ws; // NULL is malloc() |
| 320 |
|
struct objcore *oc; |
| 321 |
|
struct object *obj; |
| 322 |
|
const struct stevedore *stv; |
| 323 |
|
struct boc *boc; |
| 324 |
|
|
| 325 |
|
struct storage *st; // updated by _lease() |
| 326 |
|
|
| 327 |
|
// only for _lease_boc() |
| 328 |
|
uint64_t st_off; // already returned fragment of current st |
| 329 |
|
uint64_t avail, returned; |
| 330 |
|
struct storage *last; // to resume |
| 331 |
|
}; |
| 332 |
|
|
| 333 |
|
static inline void |
| 334 |
|
sml_ai_viov_fill(struct viov *viov, struct storage *st) |
| 335 |
4722 |
{ |
| 336 |
4722 |
viov->iov.iov_base = TRUST_ME(st->ptr); |
| 337 |
4722 |
viov->iov.iov_len = st->len; |
| 338 |
4722 |
viov->lease = ptr2lease(st); |
| 339 |
4722 |
VAI_ASSERT_LEASE(viov->lease); |
| 340 |
4722 |
} |
| 341 |
|
|
| 342 |
|
// sml has no mechanism to notify "I got free space again now" |
| 343 |
|
// (we could add that, but because storage.h is used in mgt, a first attempt |
| 344 |
|
// looks at least like this would cause some include spill for vai_q_head or |
| 345 |
|
// something similar) |
| 346 |
|
// |
| 347 |
|
// So anyway, to get ahead we just implement a pretty stupid "call the notify |
| 348 |
|
// some time later" on a thread |
| 349 |
|
static void |
| 350 |
|
sml_ai_later_task(struct worker *wrk, void *priv) |
| 351 |
0 |
{ |
| 352 |
0 |
struct sml_hdl *hdl; |
| 353 |
0 |
const vtim_dur dur = 0.0042; |
| 354 |
|
|
| 355 |
0 |
(void)wrk; |
| 356 |
0 |
VTIM_sleep(dur); |
| 357 |
0 |
CAST_VAI_HDL_NOTNULL(hdl, priv, SML_HDL_MAGIC); |
| 358 |
0 |
memset(&hdl->task, 0, sizeof hdl->task); |
| 359 |
0 |
hdl->qe.cb(hdl, hdl->qe.priv); |
| 360 |
0 |
} |
| 361 |
|
static void |
| 362 |
|
sml_ai_later(struct worker *wrk, struct sml_hdl *hdl) |
| 363 |
0 |
{ |
| 364 |
0 |
AZ(hdl->task.func); |
| 365 |
0 |
AZ(hdl->task.priv); |
| 366 |
0 |
hdl->task.func = sml_ai_later_task; |
| 367 |
0 |
hdl->task.priv = hdl; |
| 368 |
0 |
AZ(Pool_Task(wrk->pool, &hdl->task, TASK_QUEUE_BG)); |
| 369 |
0 |
} |
| 370 |
|
|
| 371 |
|
|
| 372 |
|
static int |
| 373 |
|
sml_ai_buffer(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab) |
| 374 |
16 |
{ |
| 375 |
16 |
const struct stevedore *stv; |
| 376 |
16 |
struct sml_hdl *hdl; |
| 377 |
16 |
struct storage *st; |
| 378 |
16 |
struct viov *vio; |
| 379 |
16 |
int r = 0; |
| 380 |
|
|
| 381 |
16 |
(void) wrk; |
| 382 |
16 |
CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC); |
| 383 |
16 |
stv = hdl->stv; |
| 384 |
16 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 385 |
|
|
| 386 |
16 |
VSCARAB_FOREACH(vio, scarab) |
| 387 |
16 |
if (vio->iov.iov_len > UINT_MAX) |
| 388 |
0 |
return (-EINVAL); |
| 389 |
|
|
| 390 |
16 |
VSCARAB_FOREACH(vio, scarab) { |
| 391 |
16 |
st = objallocwithnuke(wrk, stv, vio->iov.iov_len, 0); |
| 392 |
16 |
if (st == NULL) |
| 393 |
0 |
break; |
| 394 |
16 |
assert(st->space >= vio->iov.iov_len); |
| 395 |
16 |
st->flags = STORAGE_F_BUFFER; |
| 396 |
16 |
st->len = st->space; |
| 397 |
|
|
| 398 |
16 |
sml_ai_viov_fill(vio, st); |
| 399 |
16 |
r++; |
| 400 |
16 |
} |
| 401 |
16 |
if (r == 0) { |
| 402 |
0 |
sml_ai_later(wrk, hdl); |
| 403 |
0 |
r = -EAGAIN; |
| 404 |
0 |
} |
| 405 |
16 |
return (r); |
| 406 |
16 |
} |
| 407 |
|
|
| 408 |
|
static int |
| 409 |
|
sml_ai_lease_simple(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab) |
| 410 |
8309 |
{ |
| 411 |
8309 |
struct storage *st; |
| 412 |
8309 |
struct sml_hdl *hdl; |
| 413 |
8309 |
struct viov *viov; |
| 414 |
8309 |
int r = 0; |
| 415 |
|
|
| 416 |
8309 |
(void) wrk; |
| 417 |
8309 |
CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC); |
| 418 |
8309 |
VSCARAB_CHECK_NOTNULL(scarab); |
| 419 |
|
|
| 420 |
8309 |
AZ(hdl->st_off); |
| 421 |
8309 |
st = hdl->st; |
| 422 |
13014 |
while (st != NULL && (viov = VSCARAB_GET(scarab)) != NULL) { |
| 423 |
4705 |
CHECK_OBJ(st, STORAGE_MAGIC); |
| 424 |
4705 |
sml_ai_viov_fill(viov, st); |
| 425 |
4705 |
r++; |
| 426 |
4705 |
st = VTAILQ_PREV(st, storagehead, list); |
| 427 |
4705 |
} |
| 428 |
8309 |
hdl->st = st; |
| 429 |
8309 |
if (st == NULL) |
| 430 |
8308 |
scarab->flags |= VSCARAB_F_END; |
| 431 |
8309 |
return (r); |
| 432 |
8309 |
} |
| 433 |
|
|
| 434 |
|
static int |
| 435 |
|
sml_ai_lease_fail(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab) |
| 436 |
0 |
{ |
| 437 |
0 |
(void) wrk; |
| 438 |
0 |
(void) vhdl; |
| 439 |
0 |
(void) scarab; |
| 440 |
|
|
| 441 |
0 |
return (-EPIPE); |
| 442 |
0 |
} |
| 443 |
|
|
| 444 |
|
/* |
| 445 |
|
* on leases while streaming (with a boc): |
| 446 |
|
* |
| 447 |
|
* SML uses the lease return facility to implement the "free behind" for |
| 448 |
|
* OC_F_TRANSIENT objects. When streaming, we also return leases on |
| 449 |
|
* fragments of sts, but we must only "free behind" when we are done with the |
| 450 |
|
* last fragment. |
| 451 |
|
* |
| 452 |
|
* So we use a magic lease to signal "this is only a fragment", which we ignore |
| 453 |
|
* on returns |
| 454 |
|
*/ |
| 455 |
|
|
| 456 |
|
static int |
| 457 |
|
sml_ai_lease_boc(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab) |
| 458 |
5476 |
{ |
| 459 |
5476 |
enum boc_state_e state = BOS_INVALID; |
| 460 |
5476 |
struct storage *next; |
| 461 |
5476 |
struct sml_hdl *hdl; |
| 462 |
5476 |
struct viov *viov; |
| 463 |
5476 |
int r = 0; |
| 464 |
|
|
| 465 |
5476 |
CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC); |
| 466 |
5476 |
VSCARAB_CHECK_NOTNULL(scarab); |
| 467 |
5476 |
assert(hdl->boc == hdl->oc->boc); |
| 468 |
|
|
| 469 |
5476 |
if (hdl->avail == hdl->returned) { |
| 470 |
5206 |
hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned, |
| 471 |
5206 |
&state, &hdl->qe); |
| 472 |
5206 |
assert(state >= BOS_STREAM); |
| 473 |
5206 |
if (state == BOS_FAILED) { |
| 474 |
22 |
hdl->last = NULL; |
| 475 |
22 |
return (-EPIPE); |
| 476 |
22 |
} |
| 477 |
5184 |
else if (state == BOS_FINISHED) |
| 478 |
933 |
(void)0; |
| 479 |
4251 |
else if (hdl->avail == hdl->returned) { |
| 480 |
|
// ObjVAIGetExtend() has scheduled a notification |
| 481 |
2113 |
if (hdl->boc->transit_buffer > 0) |
| 482 |
255 |
return (-ENOBUFS); |
| 483 |
1858 |
else |
| 484 |
1858 |
return (-EAGAIN); |
| 485 |
2113 |
} |
| 486 |
2138 |
else |
| 487 |
2138 |
assert(state < BOS_FINISHED); |
| 488 |
5206 |
} |
| 489 |
3341 |
Lck_Lock(&hdl->boc->mtx); |
| 490 |
3341 |
if (hdl->st == NULL && hdl->last != NULL) |
| 491 |
1048 |
hdl->st = VTAILQ_PREV(hdl->last, storagehead, list); |
| 492 |
3341 |
if (hdl->last != NULL && state < BOS_FINISHED) { |
| 493 |
157 |
viov = VSCARAB_GET(scarab); |
| 494 |
157 |
AN(viov); |
| 495 |
157 |
viov->iov = IOV_NIL; |
| 496 |
157 |
viov->lease = ptr2lease(hdl->last); |
| 497 |
157 |
r++; |
| 498 |
157 |
} |
| 499 |
3341 |
if (hdl->last != NULL) |
| 500 |
1048 |
hdl->last = NULL; |
| 501 |
3341 |
if (hdl->st == NULL && hdl->returned == 0) |
| 502 |
953 |
hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead); |
| 503 |
3341 |
if (hdl->st == NULL) |
| 504 |
886 |
assert(hdl->avail == hdl->returned); |
| 505 |
|
|
| 506 |
5946 |
while (hdl->avail > hdl->returned && (viov = VSCARAB_GET(scarab)) != NULL) { |
| 507 |
2605 |
CHECK_OBJ_NOTNULL(hdl->st, STORAGE_MAGIC); // ObjVAIGetExtend ensures |
| 508 |
2605 |
assert(hdl->boc == hdl->oc->boc); |
| 509 |
2605 |
assert(hdl->st_off <= hdl->st->space); |
| 510 |
2605 |
size_t av = hdl->avail - hdl->returned; |
| 511 |
2605 |
size_t l = hdl->st->space - hdl->st_off; |
| 512 |
2605 |
AN(l); |
| 513 |
2605 |
if (l > av) |
| 514 |
1140 |
l = av; |
| 515 |
2605 |
viov->iov.iov_base = TRUST_ME(hdl->st->ptr + hdl->st_off); |
| 516 |
2605 |
viov->iov.iov_len = l; |
| 517 |
2605 |
if (hdl->st_off + l == hdl->st->space) { |
| 518 |
1465 |
next = VTAILQ_PREV(hdl->st, storagehead, list); |
| 519 |
1465 |
AZ(hdl->last); |
| 520 |
1465 |
if (next == NULL) { |
| 521 |
1062 |
hdl->last = hdl->st; |
| 522 |
1062 |
viov->lease = VAI_LEASE_NORET; |
| 523 |
1062 |
} |
| 524 |
403 |
else { |
| 525 |
403 |
CHECK_OBJ(next, STORAGE_MAGIC); |
| 526 |
403 |
viov->lease = ptr2lease(hdl->st); |
| 527 |
403 |
} |
| 528 |
|
#ifdef VAI_DBG |
| 529 |
|
if (wrk->vsl) |
| 530 |
|
VSLb(wrk->vsl, SLT_Debug, "off %zu + l %zu == space st %p next st %p stvprv %p", |
| 531 |
|
hdl->st_off, l, hdl->st, next, hdl->boc->stevedore_priv); |
| 532 |
|
#endif |
| 533 |
1465 |
hdl->st_off = 0; |
| 534 |
1465 |
hdl->st = next; |
| 535 |
1465 |
} |
| 536 |
1140 |
else { |
| 537 |
1140 |
viov->lease = VAI_LEASE_NORET; |
| 538 |
1140 |
hdl->st_off += l; |
| 539 |
1140 |
} |
| 540 |
2605 |
hdl->returned += l; |
| 541 |
2605 |
VAI_ASSERT_LEASE(viov->lease); |
| 542 |
2605 |
r++; |
| 543 |
2605 |
} |
| 544 |
|
|
| 545 |
3341 |
Lck_Unlock(&hdl->boc->mtx); |
| 546 |
3341 |
if (state != BOS_FINISHED && hdl->avail == hdl->returned) { |
| 547 |
2412 |
hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned, |
| 548 |
2412 |
&state, &hdl->qe); |
| 549 |
2412 |
} |
| 550 |
3341 |
if (state == BOS_FINISHED && hdl->avail == hdl->returned) |
| 551 |
933 |
scarab->flags |= VSCARAB_F_END; |
| 552 |
3341 |
return (r); |
| 553 |
3341 |
} |
| 554 |
|
|
| 555 |
|
// return only buffers, used if object is not streaming |
| 556 |
|
static void v_matchproto_(vai_return_f) |
| 557 |
|
sml_ai_return_buffers(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret) |
| 558 |
4943 |
{ |
| 559 |
4943 |
struct storage *st; |
| 560 |
4943 |
struct sml_hdl *hdl; |
| 561 |
4943 |
viov_lease_t *p; |
| 562 |
|
|
| 563 |
4943 |
(void) wrk; |
| 564 |
4943 |
CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC); |
| 565 |
|
|
| 566 |
5432 |
VSCARET_FOREACH(p, scaret) { |
| 567 |
5432 |
if (*p == VAI_LEASE_NORET) |
| 568 |
643 |
continue; |
| 569 |
4789 |
CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC); |
| 570 |
4789 |
if ((st->flags & STORAGE_F_BUFFER) == 0) |
| 571 |
4773 |
continue; |
| 572 |
16 |
sml_stv_free(hdl->stv, st); |
| 573 |
16 |
} |
| 574 |
4943 |
VSCARET_INIT(scaret, scaret->capacity); |
| 575 |
4943 |
} |
| 576 |
|
|
| 577 |
|
// generic return for buffers and object leases, used when streaming |
| 578 |
|
static void v_matchproto_(vai_return_f) |
| 579 |
|
sml_ai_return(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret) |
| 580 |
1409 |
{ |
| 581 |
1409 |
struct storage *st; |
| 582 |
1409 |
struct sml_hdl *hdl; |
| 583 |
1409 |
viov_lease_t *p; |
| 584 |
|
|
| 585 |
1409 |
(void) wrk; |
| 586 |
1409 |
CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC); |
| 587 |
1409 |
VSCARET_CHECK_NOTNULL(scaret); |
| 588 |
1409 |
if (scaret->used == 0) |
| 589 |
0 |
return; |
| 590 |
|
|
| 591 |
|
// callback is only registered if needed |
| 592 |
1409 |
assert(hdl->boc != NULL && (hdl->oc->flags & OC_F_TRANSIENT) != 0); |
| 593 |
|
|
| 594 |
|
// filter noret and last |
| 595 |
1409 |
VSCARET_LOCAL(todo, scaret->used); |
| 596 |
2083 |
VSCARET_FOREACH(p, scaret) { |
| 597 |
2083 |
if (*p == VAI_LEASE_NORET) |
| 598 |
1594 |
continue; |
| 599 |
489 |
CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC); |
| 600 |
489 |
VSCARET_ADD(todo, *p); |
| 601 |
489 |
} |
| 602 |
1409 |
VSCARET_INIT(scaret, scaret->capacity); |
| 603 |
|
|
| 604 |
1409 |
Lck_Lock(&hdl->boc->mtx); |
| 605 |
1409 |
VSCARET_FOREACH(p, todo) { |
| 606 |
489 |
CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC); |
| 607 |
489 |
if ((st->flags & STORAGE_F_BUFFER) != 0) |
| 608 |
0 |
continue; |
| 609 |
489 |
VTAILQ_REMOVE(&hdl->obj->list, st, list); |
| 610 |
489 |
if (st == hdl->boc->stevedore_priv) |
| 611 |
0 |
hdl->boc->stevedore_priv = trim_once; |
| 612 |
489 |
} |
| 613 |
1409 |
Lck_Unlock(&hdl->boc->mtx); |
| 614 |
|
|
| 615 |
1409 |
VSCARET_FOREACH(p, todo) { |
| 616 |
489 |
CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC); |
| 617 |
|
#ifdef VAI_DBG |
| 618 |
|
if (wrk->vsl != NULL) |
| 619 |
|
VSLb(wrk->vsl, SLT_Debug, "ret %p", st); |
| 620 |
|
#endif |
| 621 |
489 |
sml_stv_free(hdl->stv, st); |
| 622 |
489 |
} |
| 623 |
1409 |
} |
| 624 |
|
|
| 625 |
|
// call the notify_cb to allow VDPs to use -EAGAIN also |
| 626 |
|
static void v_matchproto_(vai_return_f) |
| 627 |
|
sml_ai_notify(struct worker *wrk, vai_hdl vhdl) |
| 628 |
128 |
{ |
| 629 |
128 |
struct sml_hdl *hdl; |
| 630 |
|
|
| 631 |
128 |
(void) wrk; |
| 632 |
128 |
CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC); |
| 633 |
128 |
CHECK_OBJ(&hdl->qe, VAI_Q_MAGIC); |
| 634 |
128 |
hdl->qe.cb(vhdl, hdl->qe.priv); |
| 635 |
128 |
} |
| 636 |
|
|
| 637 |
|
static void v_matchproto_(vai_fini_f) |
| 638 |
|
sml_ai_fini(struct worker *wrk, vai_hdl *vai_hdlp) |
| 639 |
5150 |
{ |
| 640 |
5150 |
struct sml_hdl *hdl; |
| 641 |
|
|
| 642 |
5150 |
AN(vai_hdlp); |
| 643 |
5150 |
CAST_VAI_HDL_NOTNULL(hdl, *vai_hdlp, SML_HDL_MAGIC); |
| 644 |
5150 |
*vai_hdlp = NULL; |
| 645 |
|
|
| 646 |
5150 |
if (hdl->boc != NULL) { |
| 647 |
962 |
ObjVAICancel(wrk, hdl->boc, &hdl->qe); |
| 648 |
962 |
HSH_DerefBoc(wrk, hdl->oc); |
| 649 |
962 |
hdl->boc = NULL; |
| 650 |
962 |
} |
| 651 |
|
|
| 652 |
5150 |
if (hdl->ws != NULL) |
| 653 |
32 |
WS_Release(hdl->ws, 0); |
| 654 |
5118 |
else |
| 655 |
5118 |
free(hdl); |
| 656 |
5150 |
} |
| 657 |
|
|
| 658 |
|
static vai_hdl v_matchproto_(vai_init_f) |
| 659 |
|
sml_ai_init(struct worker *wrk, struct objcore *oc, struct ws *ws, |
| 660 |
|
vai_notify_cb *notify, void *notify_priv) |
| 661 |
5151 |
{ |
| 662 |
5151 |
struct sml_hdl *hdl; |
| 663 |
5151 |
const size_t sz = sizeof *hdl; |
| 664 |
|
|
| 665 |
5151 |
if (ws != NULL && WS_ReserveSize(ws, (unsigned)sz)) |
| 666 |
32 |
hdl = WS_Reservation(ws); |
| 667 |
5119 |
else { |
| 668 |
5119 |
hdl = malloc(sz); |
| 669 |
5119 |
ws = NULL; |
| 670 |
5119 |
} |
| 671 |
|
|
| 672 |
5151 |
AN(hdl); |
| 673 |
5151 |
INIT_VAI_HDL(hdl, SML_HDL_MAGIC); |
| 674 |
5151 |
hdl->preamble.vai_lease = sml_ai_lease_simple; |
| 675 |
5151 |
hdl->preamble.vai_buffer = sml_ai_buffer; |
| 676 |
5151 |
hdl->preamble.vai_return = sml_ai_return_buffers; |
| 677 |
5151 |
hdl->preamble.vai_notify = sml_ai_notify; |
| 678 |
5151 |
hdl->preamble.vai_fini = sml_ai_fini; |
| 679 |
5151 |
hdl->ws = ws; |
| 680 |
|
|
| 681 |
5151 |
hdl->oc = oc; |
| 682 |
5151 |
hdl->obj = sml_getobj(wrk, oc); |
| 683 |
5151 |
CHECK_OBJ_NOTNULL(hdl->obj, OBJECT_MAGIC); |
| 684 |
5151 |
hdl->stv = oc->stobj->stevedore; |
| 685 |
5151 |
CHECK_OBJ_NOTNULL(hdl->stv, STEVEDORE_MAGIC); |
| 686 |
|
|
| 687 |
5151 |
hdl->qe.magic = VAI_Q_MAGIC; |
| 688 |
5151 |
hdl->qe.cb = notify; |
| 689 |
5151 |
hdl->qe.hdl = hdl; |
| 690 |
5151 |
hdl->qe.priv = notify_priv; |
| 691 |
|
|
| 692 |
5151 |
hdl->boc = HSH_RefBoc(oc); |
| 693 |
5151 |
if (hdl->boc == NULL && oc->flags & OC_F_FAILED) { |
| 694 |
0 |
hdl->preamble.vai_lease = sml_ai_lease_fail; |
| 695 |
0 |
return (hdl); |
| 696 |
0 |
} |
| 697 |
5151 |
if (hdl->boc == NULL) { |
| 698 |
4189 |
hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead); |
| 699 |
4189 |
CHECK_OBJ_ORNULL(hdl->st, STORAGE_MAGIC); |
| 700 |
4189 |
return (hdl); |
| 701 |
4189 |
} |
| 702 |
|
|
| 703 |
962 |
hdl->preamble.vai_lease = sml_ai_lease_boc; |
| 704 |
962 |
if ((hdl->oc->flags & OC_F_TRANSIENT) != 0) |
| 705 |
499 |
hdl->preamble.vai_return = sml_ai_return; |
| 706 |
962 |
return (hdl); |
| 707 |
5151 |
} |
| 708 |
|
|
| 709 |
|
/* |
| 710 |
|
* trivial notification to allow the iterator to simply block |
| 711 |
|
*/ |
| 712 |
|
struct sml_notify { |
| 713 |
|
unsigned magic; |
| 714 |
|
#define SML_NOTIFY_MAGIC 0x4589af31 |
| 715 |
|
unsigned hasmore; |
| 716 |
|
pthread_mutex_t mtx; |
| 717 |
|
pthread_cond_t cond; |
| 718 |
|
}; |
| 719 |
|
|
| 720 |
|
static void |
| 721 |
|
sml_notify_init(struct sml_notify *sn) |
| 722 |
7097 |
{ |
| 723 |
|
|
| 724 |
7097 |
INIT_OBJ(sn, SML_NOTIFY_MAGIC); |
| 725 |
7097 |
PTOK(pthread_mutex_init(&sn->mtx, NULL)); |
| 726 |
7097 |
PTOK(pthread_cond_init(&sn->cond, NULL)); |
| 727 |
7097 |
} |
| 728 |
|
|
| 729 |
|
static void |
| 730 |
|
sml_notify_fini(struct sml_notify *sn) |
| 731 |
7095 |
{ |
| 732 |
|
|
| 733 |
7095 |
CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC); |
| 734 |
7095 |
PTOK(pthread_mutex_destroy(&sn->mtx)); |
| 735 |
7096 |
PTOK(pthread_cond_destroy(&sn->cond)); |
| 736 |
7096 |
} |
| 737 |
|
|
| 738 |
|
static void v_matchproto_(vai_notify_cb) |
| 739 |
|
sml_notify(vai_hdl hdl, void *priv) |
| 740 |
2233 |
{ |
| 741 |
2233 |
struct sml_notify *sn; |
| 742 |
|
|
| 743 |
2233 |
(void) hdl; |
| 744 |
2233 |
CAST_OBJ_NOTNULL(sn, priv, SML_NOTIFY_MAGIC); |
| 745 |
2233 |
PTOK(pthread_mutex_lock(&sn->mtx)); |
| 746 |
2233 |
sn->hasmore = 1; |
| 747 |
2233 |
PTOK(pthread_cond_signal(&sn->cond)); |
| 748 |
2233 |
PTOK(pthread_mutex_unlock(&sn->mtx)); |
| 749 |
|
|
| 750 |
2233 |
} |
| 751 |
|
|
| 752 |
|
static void |
| 753 |
|
sml_notify_wait(struct sml_notify *sn) |
| 754 |
2100 |
{ |
| 755 |
|
|
| 756 |
2100 |
CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC); |
| 757 |
2100 |
PTOK(pthread_mutex_lock(&sn->mtx)); |
| 758 |
3246 |
while (sn->hasmore == 0) |
| 759 |
2100 |
PTOK(pthread_cond_wait(&sn->cond, &sn->mtx)); |
| 760 |
2100 |
AN(sn->hasmore); |
| 761 |
2100 |
sn->hasmore = 0; |
| 762 |
2100 |
PTOK(pthread_mutex_unlock(&sn->mtx)); |
| 763 |
2100 |
} |
| 764 |
|
|
| 765 |
|
int v_matchproto_(objiterator_f) |
| 766 |
|
SML_iterator(struct worker *wrk, struct objcore *oc, |
| 767 |
|
void *priv, objiterate_f *func, int final) |
| 768 |
7097 |
{ |
| 769 |
7097 |
struct sml_notify sn; |
| 770 |
7097 |
struct viov *vio, *last; |
| 771 |
7097 |
unsigned u, uu; |
| 772 |
7097 |
vai_hdl hdl; |
| 773 |
7097 |
int nn, r, r2, islast; |
| 774 |
|
|
| 775 |
7097 |
VSCARAB_LOCAL(scarab, 16); |
| 776 |
7097 |
VSCARET_LOCAL(scaret, 16); |
| 777 |
|
|
| 778 |
7097 |
(void) final; // phase out? |
| 779 |
7097 |
sml_notify_init(&sn); |
| 780 |
7097 |
hdl = ObjVAIinit(wrk, oc, NULL, sml_notify, &sn); |
| 781 |
7097 |
AN(hdl); |
| 782 |
|
|
| 783 |
7097 |
r = u = 0; |
| 784 |
|
|
| 785 |
15301 |
do { |
| 786 |
17711 |
do { |
| 787 |
17711 |
nn = ObjVAIlease(wrk, hdl, scarab); |
| 788 |
17711 |
if (nn <= 0 || scarab->flags & VSCARAB_F_END) |
| 789 |
15295 |
break; |
| 790 |
17711 |
} while (scarab->used < scarab->capacity); |
| 791 |
|
|
| 792 |
|
/* |
| 793 |
|
* nn is the wait/return action or 0 |
| 794 |
|
* nn tells us if to flush |
| 795 |
|
*/ |
| 796 |
0 |
uu = u; |
| 797 |
15301 |
last = VSCARAB_LAST(scarab); |
| 798 |
17435 |
VSCARAB_FOREACH(vio, scarab) { |
| 799 |
17435 |
islast = vio == last; |
| 800 |
17435 |
AZ(u & OBJ_ITER_END); |
| 801 |
17435 |
if (islast && scarab->flags & VSCARAB_F_END) |
| 802 |
6143 |
u |= OBJ_ITER_END; |
| 803 |
|
|
| 804 |
|
// flush if it is the scarab's last IOV and we will block next |
| 805 |
|
// or if we need space in the return leases array |
| 806 |
17435 |
uu = u; |
| 807 |
17435 |
if ((islast && nn < 0) || scaret->used == scaret->capacity - 1) |
| 808 |
2007 |
uu |= OBJ_ITER_FLUSH; |
| 809 |
|
|
| 810 |
|
// null iov with the only purpose to return the resume ptr lease |
| 811 |
|
// exception needed because assert(len > 0) in VDP_bytes() |
| 812 |
17435 |
if (vio->iov.iov_base == IOV_NIL.iov_base) |
| 813 |
157 |
r = 0; |
| 814 |
17278 |
else |
| 815 |
17278 |
r = func(priv, uu, vio->iov.iov_base, vio->iov.iov_len); |
| 816 |
17435 |
if (r != 0) |
| 817 |
59 |
break; |
| 818 |
|
|
| 819 |
|
// sufficient space ensured by capacity check above |
| 820 |
17376 |
VSCARET_ADD(scaret, vio->lease); |
| 821 |
|
|
| 822 |
|
#ifdef VAI_DBG |
| 823 |
|
if (wrk->vsl) |
| 824 |
|
VSLb(wrk->vsl, SLT_Debug, "len %zu scaret %u uu %u", |
| 825 |
|
vio->iov.iov_len, scaret->used, uu); |
| 826 |
|
#endif |
| 827 |
|
|
| 828 |
|
// whenever we have flushed, return leases |
| 829 |
17376 |
if ((uu & OBJ_ITER_FLUSH) && scaret->used > 0) |
| 830 |
1994 |
ObjVAIreturn(wrk, hdl, scaret); |
| 831 |
17376 |
} |
| 832 |
|
|
| 833 |
|
// return leases which we did not use if error (break) |
| 834 |
15301 |
VSCARAB_FOREACH_RESUME(vio, scarab) { |
| 835 |
88 |
if (scaret->used == scaret->capacity) |
| 836 |
0 |
ObjVAIreturn(wrk, hdl, scaret); |
| 837 |
88 |
VSCARET_ADD(scaret, vio->lease); |
| 838 |
88 |
} |
| 839 |
|
|
| 840 |
|
// we have now completed the scarab |
| 841 |
15301 |
VSCARAB_INIT(scarab, scarab->capacity); |
| 842 |
|
|
| 843 |
|
#ifdef VAI_DBG |
| 844 |
|
if (wrk->vsl) |
| 845 |
|
VSLb(wrk->vsl, SLT_Debug, "r %d nn %d uu %u", |
| 846 |
|
r, nn, uu); |
| 847 |
|
#endif |
| 848 |
|
|
| 849 |
|
// flush before blocking, unless flush or end already sent |
| 850 |
15301 |
if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN) && uu == 0) { |
| 851 |
109 |
r = func(priv, OBJ_ITER_FLUSH, NULL, 0); |
| 852 |
109 |
if (scaret->used > 0) |
| 853 |
0 |
ObjVAIreturn(wrk, hdl, scaret); |
| 854 |
109 |
} |
| 855 |
|
|
| 856 |
15301 |
if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN)) { |
| 857 |
2100 |
assert(scaret->used <= 1); |
| 858 |
2100 |
sml_notify_wait(&sn); |
| 859 |
2100 |
} |
| 860 |
13201 |
else if (r == 0 && nn < 0) |
| 861 |
22 |
r = -1; |
| 862 |
15301 |
} while (nn != 0 && r == 0); |
| 863 |
|
|
| 864 |
7097 |
if ((u & OBJ_ITER_END) == 0) { |
| 865 |
953 |
r2 = func(priv, OBJ_ITER_END, NULL, 0); |
| 866 |
953 |
if (r == 0) |
| 867 |
914 |
r = r2; |
| 868 |
953 |
} |
| 869 |
|
|
| 870 |
7097 |
if (scaret->used > 0) |
| 871 |
6161 |
ObjVAIreturn(wrk, hdl, scaret); |
| 872 |
|
|
| 873 |
7097 |
ObjVAIfini(wrk, &hdl); |
| 874 |
7097 |
sml_notify_fini(&sn); |
| 875 |
|
|
| 876 |
7097 |
return (r); |
| 877 |
7097 |
} |
| 878 |
|
|
| 879 |
|
/*-------------------------------------------------------------------- |
| 880 |
|
*/ |
| 881 |
|
|
| 882 |
|
static struct storage * |
| 883 |
|
objallocwithnuke(struct worker *wrk, const struct stevedore *stv, ssize_t size, |
| 884 |
|
int flags) |
| 885 |
5855 |
{ |
| 886 |
5855 |
struct storage *st = NULL; |
| 887 |
|
|
| 888 |
5855 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 889 |
5855 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 890 |
|
|
| 891 |
5855 |
if (size > cache_param->fetch_maxchunksize) { |
| 892 |
2 |
if (!(flags & LESS_MEM_ALLOCED_IS_OK)) |
| 893 |
0 |
return (NULL); |
| 894 |
2 |
size = cache_param->fetch_maxchunksize; |
| 895 |
2 |
} |
| 896 |
|
|
| 897 |
5855 |
assert(size <= UINT_MAX); /* field limit in struct storage */ |
| 898 |
|
|
| 899 |
5863 |
do { |
| 900 |
|
/* try to allocate from it */ |
| 901 |
5863 |
st = sml_stv_alloc(stv, size, flags); |
| 902 |
5863 |
if (st != NULL) |
| 903 |
5852 |
break; |
| 904 |
|
|
| 905 |
|
/* no luck; try to free some space and keep trying */ |
| 906 |
11 |
if (stv->lru == NULL) |
| 907 |
0 |
break; |
| 908 |
11 |
} while (LRU_NukeOne(wrk, stv->lru)); |
| 909 |
|
|
| 910 |
5855 |
CHECK_OBJ_ORNULL(st, STORAGE_MAGIC); |
| 911 |
5855 |
return (st); |
| 912 |
5855 |
} |
| 913 |
|
|
| 914 |
|
static int v_matchproto_(objgetspace_f) |
| 915 |
|
sml_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz, |
| 916 |
|
uint8_t **ptr) |
| 917 |
68018 |
{ |
| 918 |
68018 |
struct object *o; |
| 919 |
68018 |
struct storage *st; |
| 920 |
|
|
| 921 |
68018 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 922 |
68018 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 923 |
68018 |
CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); |
| 924 |
68018 |
AN(sz); |
| 925 |
68018 |
AN(ptr); |
| 926 |
68018 |
if (*sz == 0) |
| 927 |
64351 |
*sz = cache_param->fetch_chunksize; |
| 928 |
68018 |
assert(*sz > 0); |
| 929 |
68018 |
if (oc->boc->transit_buffer > 0) |
| 930 |
455 |
*sz = vmin_t(ssize_t, *sz, oc->boc->transit_buffer); |
| 931 |
|
|
| 932 |
68018 |
o = sml_getobj(wrk, oc); |
| 933 |
68018 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 934 |
68018 |
CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); |
| 935 |
|
|
| 936 |
68018 |
st = VTAILQ_FIRST(&o->list); |
| 937 |
68018 |
if (st != NULL && st->len < st->space) { |
| 938 |
63342 |
*sz = st->space - st->len; |
| 939 |
63342 |
*ptr = st->ptr + st->len; |
| 940 |
63342 |
assert (*sz > 0); |
| 941 |
63342 |
return (1); |
| 942 |
63342 |
} |
| 943 |
|
|
| 944 |
4676 |
st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz, |
| 945 |
4676 |
LESS_MEM_ALLOCED_IS_OK); |
| 946 |
4676 |
if (st == NULL) |
| 947 |
5 |
return (0); |
| 948 |
|
|
| 949 |
4671 |
CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); |
| 950 |
4671 |
Lck_Lock(&oc->boc->mtx); |
| 951 |
4671 |
VTAILQ_INSERT_HEAD(&o->list, st, list); |
| 952 |
4671 |
Lck_Unlock(&oc->boc->mtx); |
| 953 |
|
|
| 954 |
4671 |
*sz = st->space - st->len; |
| 955 |
4671 |
assert (*sz > 0); |
| 956 |
4671 |
*ptr = st->ptr + st->len; |
| 957 |
4671 |
return (1); |
| 958 |
4671 |
} |
| 959 |
|
|
| 960 |
|
static void v_matchproto_(objextend_f) |
| 961 |
|
sml_extend(struct worker *wrk, struct objcore *oc, ssize_t l) |
| 962 |
64383 |
{ |
| 963 |
64383 |
struct object *o; |
| 964 |
64383 |
struct storage *st; |
| 965 |
|
|
| 966 |
64383 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 967 |
64383 |
assert(l > 0); |
| 968 |
|
|
| 969 |
64383 |
o = sml_getobj(wrk, oc); |
| 970 |
64383 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 971 |
64383 |
st = VTAILQ_FIRST(&o->list); |
| 972 |
64383 |
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); |
| 973 |
64383 |
assert(st->len + l <= st->space); |
| 974 |
64383 |
st->len += l; |
| 975 |
64383 |
} |
| 976 |
|
|
| 977 |
|
static void v_matchproto_(objtrimstore_f) |
| 978 |
|
sml_trimstore(struct worker *wrk, struct objcore *oc) |
| 979 |
2016 |
{ |
| 980 |
2016 |
const struct stevedore *stv; |
| 981 |
2016 |
struct storage *st, *st1; |
| 982 |
2016 |
struct object *o; |
| 983 |
|
|
| 984 |
2016 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 985 |
2016 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 986 |
2016 |
CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); |
| 987 |
|
|
| 988 |
2016 |
stv = oc->stobj->stevedore; |
| 989 |
2016 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 990 |
|
|
| 991 |
2016 |
if (oc->boc->stevedore_priv != NULL) |
| 992 |
0 |
WRONG("sml_trimstore already called"); |
| 993 |
2016 |
oc->boc->stevedore_priv = trim_once; |
| 994 |
|
|
| 995 |
2016 |
o = sml_getobj(wrk, oc); |
| 996 |
2016 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 997 |
2016 |
st = VTAILQ_FIRST(&o->list); |
| 998 |
|
|
| 999 |
2016 |
if (st == NULL) |
| 1000 |
0 |
return; |
| 1001 |
|
|
| 1002 |
2016 |
if (st->len == 0) { |
| 1003 |
63 |
Lck_Lock(&oc->boc->mtx); |
| 1004 |
63 |
VTAILQ_REMOVE(&o->list, st, list); |
| 1005 |
63 |
Lck_Unlock(&oc->boc->mtx); |
| 1006 |
|
/* sml_bocdone frees this */ |
| 1007 |
63 |
oc->boc->stevedore_priv = st; |
| 1008 |
63 |
return; |
| 1009 |
63 |
} |
| 1010 |
|
|
| 1011 |
1953 |
if (stv->sml_free == NULL) |
| 1012 |
0 |
return; |
| 1013 |
|
|
| 1014 |
1953 |
if (st->space - st->len < 512) |
| 1015 |
1731 |
return; |
| 1016 |
|
|
| 1017 |
222 |
st1 = sml_stv_alloc(stv, st->len, 0); |
| 1018 |
222 |
if (st1 == NULL) |
| 1019 |
0 |
return; |
| 1020 |
222 |
assert(st1->space >= st->len); |
| 1021 |
|
|
| 1022 |
222 |
vmemcpy(st1->ptr, st->ptr, st->len); |
| 1023 |
222 |
st1->len = st->len; |
| 1024 |
222 |
Lck_Lock(&oc->boc->mtx); |
| 1025 |
222 |
VTAILQ_REMOVE(&o->list, st, list); |
| 1026 |
222 |
VTAILQ_INSERT_HEAD(&o->list, st1, list); |
| 1027 |
222 |
Lck_Unlock(&oc->boc->mtx); |
| 1028 |
|
/* sml_bocdone frees this */ |
| 1029 |
222 |
oc->boc->stevedore_priv = st; |
| 1030 |
222 |
} |
| 1031 |
|
|
| 1032 |
|
static void v_matchproto_(objbocdone_f) |
| 1033 |
|
sml_bocdone(struct worker *wrk, struct objcore *oc, struct boc *boc) |
| 1034 |
4736 |
{ |
| 1035 |
4736 |
const struct stevedore *stv; |
| 1036 |
|
|
| 1037 |
4736 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 1038 |
4736 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
| 1039 |
4736 |
CHECK_OBJ_NOTNULL(boc, BOC_MAGIC); |
| 1040 |
4736 |
stv = oc->stobj->stevedore; |
| 1041 |
4736 |
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); |
| 1042 |
|
|
| 1043 |
4736 |
sml_bocfini(stv, boc); |
| 1044 |
|
|
| 1045 |
4736 |
if (stv->lru != NULL) { |
| 1046 |
4701 |
if (isnan(wrk->lastused)) |
| 1047 |
0 |
wrk->lastused = VTIM_real(); |
| 1048 |
4701 |
LRU_Add(oc, wrk->lastused); // approx timestamp is OK |
| 1049 |
4701 |
} |
| 1050 |
4736 |
} |
| 1051 |
|
|
| 1052 |
|
static const void * v_matchproto_(objgetattr_f) |
| 1053 |
|
sml_getattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, |
| 1054 |
|
ssize_t *len) |
| 1055 |
38083 |
{ |
| 1056 |
38083 |
struct object *o; |
| 1057 |
38083 |
ssize_t dummy; |
| 1058 |
|
|
| 1059 |
38083 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 1060 |
|
|
| 1061 |
38083 |
if (len == NULL) |
| 1062 |
20812 |
len = &dummy; |
| 1063 |
38083 |
o = sml_getobj(wrk, oc); |
| 1064 |
38083 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 1065 |
|
|
| 1066 |
38083 |
switch (attr) { |
| 1067 |
|
/* Fixed size attributes */ |
| 1068 |
0 |
#define OBJ_FIXATTR(U, l, s) \ |
| 1069 |
28826 |
case OA_##U: \ |
| 1070 |
28826 |
*len = sizeof o->fa_##l; \ |
| 1071 |
28826 |
return (o->fa_##l); |
| 1072 |
0 |
#include "tbl/obj_attr.h" |
| 1073 |
|
|
| 1074 |
|
/* Variable size attributes */ |
| 1075 |
0 |
#define OBJ_VARATTR(U, l) \ |
| 1076 |
8706 |
case OA_##U: \ |
| 1077 |
8706 |
if (o->va_##l == NULL) \ |
| 1078 |
8697 |
return (NULL); \ |
| 1079 |
8697 |
*len = o->va_##l##_len; \ |
| 1080 |
8697 |
return (o->va_##l); |
| 1081 |
2 |
#include "tbl/obj_attr.h" |
| 1082 |
|
|
| 1083 |
|
/* Auxiliary attributes */ |
| 1084 |
0 |
#define OBJ_AUXATTR(U, l) \ |
| 1085 |
563 |
case OA_##U: \ |
| 1086 |
563 |
if (o->aa_##l == NULL) \ |
| 1087 |
555 |
return (NULL); \ |
| 1088 |
555 |
CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC); \ |
| 1089 |
555 |
*len = o->aa_##l->len; \ |
| 1090 |
555 |
return (o->aa_##l->ptr); |
| 1091 |
5383 |
#include "tbl/obj_attr.h" |
| 1092 |
|
|
| 1093 |
0 |
default: |
| 1094 |
0 |
break; |
| 1095 |
38083 |
} |
| 1096 |
0 |
WRONG("Unsupported OBJ_ATTR"); |
| 1097 |
0 |
} |
| 1098 |
|
|
| 1099 |
|
static void * v_matchproto_(objsetattr_f) |
| 1100 |
|
sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, |
| 1101 |
|
ssize_t len, const void *ptr) |
| 1102 |
15677 |
{ |
| 1103 |
15677 |
struct object *o; |
| 1104 |
15677 |
void *retval = NULL; |
| 1105 |
15677 |
struct storage *st; |
| 1106 |
|
|
| 1107 |
15677 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
| 1108 |
|
|
| 1109 |
15677 |
o = sml_getobj(wrk, oc); |
| 1110 |
15677 |
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); |
| 1111 |
15677 |
st = o->objstore; |
| 1112 |
|
|
| 1113 |
15677 |
switch (attr) { |
| 1114 |
|
/* Fixed size attributes */ |
| 1115 |
0 |
#define OBJ_FIXATTR(U, l, s) \ |
| 1116 |
12058 |
case OA_##U: \ |
| 1117 |
12058 |
assert(len == sizeof o->fa_##l); \ |
| 1118 |
12058 |
retval = o->fa_##l; \ |
| 1119 |
12058 |
break; |
| 1120 |
0 |
#include "tbl/obj_attr.h" |
| 1121 |
|
|
| 1122 |
|
/* Variable size attributes */ |
| 1123 |
0 |
#define OBJ_VARATTR(U, l) \ |
| 1124 |
3164 |
case OA_##U: \ |
| 1125 |
3164 |
if (o->va_##l##_len > 0) { \ |
| 1126 |
0 |
AN(o->va_##l); \ |
| 1127 |
0 |
assert(len == o->va_##l##_len); \ |
| 1128 |
0 |
retval = o->va_##l; \ |
| 1129 |
3164 |
} else if (len > 0) { \ |
| 1130 |
3164 |
assert(len <= UINT_MAX); \ |
| 1131 |
3164 |
assert(st->len + len <= st->space); \ |
| 1132 |
3164 |
o->va_##l = st->ptr + st->len; \ |
| 1133 |
3164 |
st->len += len; \ |
| 1134 |
3164 |
o->va_##l##_len = len; \ |
| 1135 |
3164 |
retval = o->va_##l; \ |
| 1136 |
3164 |
} \ |
| 1137 |
3164 |
break; |
| 1138 |
2943 |
#include "tbl/obj_attr.h" |
| 1139 |
|
|
| 1140 |
|
/* Auxiliary attributes */ |
| 1141 |
2943 |
#define OBJ_AUXATTR(U, l) \ |
| 1142 |
2943 |
case OA_##U: \ |
| 1143 |
458 |
if (o->aa_##l != NULL) { \ |
| 1144 |
0 |
CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC); \ |
| 1145 |
0 |
assert(len == o->aa_##l->len); \ |
| 1146 |
0 |
retval = o->aa_##l->ptr; \ |
| 1147 |
0 |
break; \ |
| 1148 |
0 |
} \ |
| 1149 |
458 |
if (len == 0) \ |
| 1150 |
458 |
break; \ |
| 1151 |
458 |
o->aa_##l = objallocwithnuke(wrk, oc->stobj->stevedore, \ |
| 1152 |
458 |
len, 0); \ |
| 1153 |
458 |
if (o->aa_##l == NULL) \ |
| 1154 |
458 |
break; \ |
| 1155 |
458 |
CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC); \ |
| 1156 |
458 |
assert(len <= o->aa_##l->space); \ |
| 1157 |
458 |
o->aa_##l->len = len; \ |
| 1158 |
458 |
retval = o->aa_##l->ptr; \ |
| 1159 |
458 |
break; |
| 1160 |
2943 |
#include "tbl/obj_attr.h" |
| 1161 |
|
|
| 1162 |
0 |
default: |
| 1163 |
0 |
WRONG("Unsupported OBJ_ATTR"); |
| 1164 |
0 |
break; |
| 1165 |
15677 |
} |
| 1166 |
|
|
| 1167 |
15674 |
if (retval != NULL && ptr != NULL) |
| 1168 |
741 |
vmemcpy(retval, ptr, len); |
| 1169 |
15673 |
return (retval); |
| 1170 |
15677 |
} |
| 1171 |
|
|
| 1172 |
|
const struct obj_methods SML_methods = { |
| 1173 |
|
.objfree = sml_objfree, |
| 1174 |
|
.objiterator = SML_iterator, |
| 1175 |
|
.objgetspace = sml_getspace, |
| 1176 |
|
.objextend = sml_extend, |
| 1177 |
|
.objtrimstore = sml_trimstore, |
| 1178 |
|
.objbocdone = sml_bocdone, |
| 1179 |
|
.objslim = sml_slim, |
| 1180 |
|
.objgetattr = sml_getattr, |
| 1181 |
|
.objsetattr = sml_setattr, |
| 1182 |
|
.objtouch = LRU_Touch, |
| 1183 |
|
.vai_init = sml_ai_init |
| 1184 |
|
}; |
| 1185 |
|
|
| 1186 |
|
static void |
| 1187 |
|
sml_panic_st(struct vsb *vsb, const char *hd, const struct storage *st) |
| 1188 |
4 |
{ |
| 1189 |
4 |
VSB_printf(vsb, "%s = %p {priv=%p, ptr=%p, len=%u, space=%u},\n", |
| 1190 |
4 |
hd, st, st->priv, st->ptr, st->len, st->space); |
| 1191 |
4 |
} |
| 1192 |
|
|
| 1193 |
|
void |
| 1194 |
|
SML_panic(struct vsb *vsb, const struct objcore *oc) |
| 1195 |
2 |
{ |
| 1196 |
2 |
struct object *o; |
| 1197 |
2 |
struct storage *st; |
| 1198 |
|
|
| 1199 |
2 |
VSB_printf(vsb, "Simple = %p,\n", oc->stobj->priv); |
| 1200 |
2 |
if (oc->stobj->priv == NULL) |
| 1201 |
0 |
return; |
| 1202 |
2 |
o = oc->stobj->priv; |
| 1203 |
2 |
PAN_CheckMagic(vsb, o, OBJECT_MAGIC); |
| 1204 |
2 |
sml_panic_st(vsb, "Obj", o->objstore); |
| 1205 |
|
|
| 1206 |
2 |
#define OBJ_FIXATTR(U, l, sz) \ |
| 1207 |
10 |
VSB_printf(vsb, "%s = ", #U); \ |
| 1208 |
10 |
VSB_quote(vsb, (const void*)o->fa_##l, sz, VSB_QUOTE_HEX); \ |
| 1209 |
10 |
VSB_printf(vsb, ",\n"); |
| 1210 |
|
|
| 1211 |
2 |
#define OBJ_VARATTR(U, l) \ |
| 1212 |
4 |
VSB_printf(vsb, "%s = {len=%u, ptr=%p},\n", \ |
| 1213 |
4 |
#U, o->va_##l##_len, o->va_##l); |
| 1214 |
|
|
| 1215 |
2 |
#define OBJ_AUXATTR(U, l) \ |
| 1216 |
2 |
do { \ |
| 1217 |
2 |
if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l);\ |
| 1218 |
2 |
} while(0); |
| 1219 |
|
|
| 1220 |
2 |
#include "tbl/obj_attr.h" |
| 1221 |
|
|
| 1222 |
2 |
VTAILQ_FOREACH(st, &o->list, list) { |
| 1223 |
2 |
sml_panic_st(vsb, "Body", st); |
| 1224 |
2 |
} |
| 1225 |
2 |
} |