Skip to content

V
varnish-cache
Commit 2c76f5b2 authored by Poul-Henning Kamp's avatar Poul-Henning Kamp
Browse files
Options

Collect all the stuff which implements objects on top of "simple" 

stevedores and call it "SML".

I bet this is going to surprise Martin :-)
parent 15aad353
Show whitespace changes
Inline Side-by-side
Showing with 559 additions and 455 deletions
bin/varnishd/Makefile.am
View file @ 2c76f5b2
......@@ -113,6 +113,7 @@ noinst_HEADERS = \
mgt/mgt_cli.h \
mgt/mgt_param.h \
storage/storage.h \
storage/storage_simple.h \
storage/storage_persistent.h \
waiter/waiter_priv.h \
waiter/mgt_waiter.h
......
bin/varnishd/cache/cache_obj.c
View file @ 2c76f5b2
......@@ -57,18 +57,6 @@ obj_getmethods(const struct objcore *oc)
return (oc->stobj->stevedore->methods);
}
static struct object *
obj_getobj(struct worker *wrk, struct objcore *oc)
{
const struct storeobj_methods *m;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
m = obj_getmethods(oc);
AN(m->getobj);
return (m->getobj(wrk, oc));
}
/*====================================================================
* ObjIterate()
*
......@@ -78,127 +66,12 @@ int
ObjIterate(struct worker *wrk, struct objcore *oc,
void *priv, objiterate_f *func)
{
struct busyobj *bo;
struct object *obj;
struct storage *st;
struct storage *checkpoint = NULL;
ssize_t checkpoint_len = 0;
ssize_t len = 0;
int ret = 0;
ssize_t ol;
ssize_t nl;
ssize_t sl;
void *p;
ssize_t l;
const struct storeobj_methods *om = obj_getmethods(oc);
if (om->objiterator != NULL)
return (om->objiterator(wrk, oc, priv, func));
obj = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo = HSH_RefBusy(oc);
if (bo == NULL) {
VTAILQ_FOREACH(st, &obj->list, list)
if (func(priv, 0, st->ptr, st->len))
return (-1);
return (0);
}
p = NULL;
l = 0;
while (1) {
ol = len;
nl = VBO_waitlen(wrk, bo, ol);
if (bo->state == BOS_FAILED) {
ret = -1;
break;
}
if (nl == ol) {
if (bo->state == BOS_FINISHED)
break;
continue;
}
Lck_Lock(&bo->mtx);
AZ(VTAILQ_EMPTY(&obj->list));
if (checkpoint == NULL) {
st = VTAILQ_FIRST(&obj->list);
sl = 0;
} else {
st = checkpoint;
sl = checkpoint_len;
ol -= checkpoint_len;
}
while (st != NULL) {
if (st->len > ol) {
p = st->ptr + ol;
l = st->len - ol;
len += l;
break;
}
ol -= st->len;
assert(ol >= 0);
nl -= st->len;
assert(nl > 0);
sl += st->len;
st = VTAILQ_NEXT(st, list);
if (VTAILQ_NEXT(st, list) != NULL) {
checkpoint = st;
checkpoint_len = sl;
}
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
st = VTAILQ_NEXT(st, list);
if (st != NULL && st->len == 0)
st = NULL;
Lck_Unlock(&bo->mtx);
assert(l > 0 || bo->state == BOS_FINISHED);
if (func(priv, st != NULL ? 0 : 1, p, l)) {
ret = -1;
break;
}
}
if (oc->flags & OC_F_PASS)
bo->abandon = 1;
VBO_DerefBusyObj(wrk, &bo);
return (ret);
}
/*--------------------------------------------------------------------
*/
static struct storage *
objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
{
struct storage *st = NULL;
unsigned fail;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
if (size > cache_param->fetch_maxchunksize)
size = cache_param->fetch_maxchunksize;
assert(size <= UINT_MAX); /* field limit in struct storage */
for (fail = 0; fail <= cache_param->nuke_limit; fail++) {
/* try to allocate from it */
AN(stv->alloc);
st = STV_alloc(stv, size);
if (st != NULL)
break;
/* no luck; try to free some space and keep trying */
if (fail < cache_param->nuke_limit &&
EXP_NukeOne(wrk, stv->lru) == -1)
break;
}
CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
return (st);
AN(func);
AN(om->objiterator);
return (om->objiterator(wrk, oc, priv, func));
}
/*====================================================================
......@@ -213,8 +86,6 @@ objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
int
ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr)
{
struct object *o;
struct storage *st;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
......@@ -222,37 +93,8 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr)
AN(ptr);
assert(*sz > 0);
if (om->objgetspace != NULL)
AN(om->objgetspace);
return (om->objgetspace(wrk, oc, sz, ptr));
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = VTAILQ_LAST(&o->list, storagehead);
if (st != NULL && st->len < st->space) {
*sz = st->space - st->len;
*ptr = st->ptr + st->len;
assert (*sz > 0);
return (1);
}
st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz);
if (st == NULL)
return (0);
if (oc->busyobj != NULL) {
CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
Lck_Lock(&oc->busyobj->mtx);
VTAILQ_INSERT_TAIL(&o->list, st, list);
Lck_Unlock(&oc->busyobj->mtx);
} else {
AN(oc->flags & (OC_F_PRIVATE));
VTAILQ_INSERT_TAIL(&o->list, st, list);
}
*sz = st->space - st->len;
assert (*sz > 0);
*ptr = st->ptr + st->len;
return (1);
}
/*====================================================================
......@@ -265,25 +107,13 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr)
void
ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l)
{
struct object *o;
struct storage *st;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
assert(l > 0);
if (om->objextend != NULL) {
AN(om->objextend);
om->objextend(wrk, oc, l);
return;
}
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = VTAILQ_LAST(&o->list, storagehead);
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
assert(st->len + l <= st->space);
st->len += l;
o->len += l;
}
/*====================================================================
......@@ -295,20 +125,14 @@ ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l)
uint64_t
ObjGetLen(struct worker *wrk, struct objcore *oc)
{
struct object *o;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (om->objgetlen != NULL)
AN(om->objgetlen);
return (om->objgetlen(wrk, oc));
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
return (o->len);
}
/*====================================================================
* ObjTrimStore()
*
......@@ -319,31 +143,13 @@ ObjGetLen(struct worker *wrk, struct objcore *oc)
void
ObjTrimStore(struct worker *wrk, struct objcore *oc)
{
const struct stevedore *stv;
struct storage *st;
struct object *o;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (om->objtrimstore != NULL) {
if (om->objtrimstore != NULL)
om->objtrimstore(wrk, oc);
return;
}
stv = oc->stobj->stevedore;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = VTAILQ_LAST(&o->list, storagehead);
if (st == NULL)
return;
if (st->len == 0) {
VTAILQ_REMOVE(&o->list, st, list);
STV_free(stv, st);
} else if (st->len < st->space) {
STV_trim(stv, st, st->len, 1);
}
}
/*====================================================================
......@@ -356,32 +162,12 @@ ObjTrimStore(struct worker *wrk, struct objcore *oc)
void
ObjSlim(struct worker *wrk, struct objcore *oc)
{
const struct stevedore *stv;
struct object *o;
struct storage *st, *stn;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (om->objslim != NULL) {
if (om->objslim != NULL)
om->objslim(wrk, oc);
return;
}
stv = oc->stobj->stevedore;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
if (o->esidata != NULL) {
STV_free(stv, o->esidata);
o->esidata = NULL;
}
VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) {
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
VTAILQ_REMOVE(&o->list, st, list);
STV_free(stv, st);
}
}
/*====================================================================
......@@ -393,8 +179,8 @@ ObjUpdateMeta(struct worker *wrk, struct objcore *oc)
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (m->updatemeta != NULL)
m->updatemeta(wrk, oc);
if (m->objupdatemeta != NULL)
m->objupdatemeta(wrk, oc);
}
/*====================================================================
......@@ -406,8 +192,8 @@ ObjFreeObj(struct worker *wrk, struct objcore *oc)
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
AN(m->freeobj);
m->freeobj(wrk, oc);
AN(m->objfree);
m->objfree(wrk, oc);
AZ(oc->stobj->stevedore);
}
......@@ -418,8 +204,8 @@ ObjGetLRU(const struct objcore *oc)
{
const struct storeobj_methods *m = obj_getmethods(oc);
AN(m->getlru);
return (m->getlru(oc));
AN(m->objgetlru);
return (m->objgetlru(oc));
}
/*====================================================================
......@@ -432,47 +218,12 @@ void *
ObjGetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
ssize_t *len)
{
struct object *o;
ssize_t dummy;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (om->objgetattr != NULL)
AN(om->objgetattr);
return (om->objgetattr(wrk, oc, attr, len));
if (len == NULL)
len = &dummy;
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
switch (attr) {
case OA_ESIDATA:
if (o->esidata == NULL)
return (NULL);
*len = o->esidata->len;
return (o->esidata->ptr);
case OA_FLAGS:
*len = sizeof o->oa_flags;
return (o->oa_flags);
case OA_GZIPBITS:
*len = sizeof o->oa_gzipbits;
return (o->oa_gzipbits);
case OA_HEADERS:
*len = 0; // XXX: hack
return (o->oa_http);
case OA_LASTMODIFIED:
*len = sizeof o->oa_lastmodified;
return (o->oa_lastmodified);
case OA_VARY:
*len = 4; // XXX: hack
return (o->oa_vary);
case OA_VXID:
*len = sizeof o->oa_vxid;
return (o->oa_vxid);
default:
break;
}
WRONG("Unsupported OBJ_ATTR");
}
/*====================================================================
......@@ -486,64 +237,12 @@ void *
ObjSetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
ssize_t len, const void *ptr)
{
struct object *o;
void *retval = NULL;
struct storage *st;
const struct storeobj_methods *om = obj_getmethods(oc);
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (om->objsetattr != NULL)
AN(om->objsetattr);
return (om->objsetattr(wrk, oc, attr, len, ptr));
o = obj_getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = o->objstore;
switch (attr) {
case OA_ESIDATA:
o->esidata = objallocwithnuke(wrk, oc->stobj->stevedore, len);
if (o->esidata == NULL)
return (NULL);
o->esidata->len = len;
retval = o->esidata->ptr;
break;
case OA_FLAGS:
assert(len == sizeof o->oa_flags);
retval = o->oa_flags;
break;
case OA_GZIPBITS:
assert(len == sizeof o->oa_gzipbits);
retval = o->oa_gzipbits;
break;
case OA_HEADERS:
len = PRNDUP(len);
assert(st->len + len <= st->space);
o->oa_http = (void*)(st->ptr + st->len);
st->len += len;
retval = o->oa_http;
break;
case OA_LASTMODIFIED:
assert(len == sizeof o->oa_lastmodified);
retval = o->oa_lastmodified;
break;
case OA_VARY:
len = PRNDUP(len);
assert(st->len + len <= st->space);
o->oa_vary = (void*)(st->ptr + st->len);
st->len += len;
retval = o->oa_vary;
break;
case OA_VXID:
assert(len == sizeof o->oa_vxid);
retval = o->oa_vxid;
break;
default:
WRONG("Unsupported OBJ_ATTR");
break;
}
if (ptr != NULL)
memcpy(retval, ptr, len);
return (retval);
}
/*====================================================================
......
bin/varnishd/storage/mgt_stevedore.c
View file @ 2c76f5b2
......@@ -183,8 +183,8 @@ STV_Config(const char *spec)
ARGV_ERR("(-s%s) too many arguments\n", stv->name);
AN(stv->alloc);
if (stv->allocobj == NULL)
stv->allocobj = stv_default_allocobj;
AN(stv->allocobj);
AN(stv->methods);
if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
stv->transient = 1;
......
bin/varnishd/storage/mgt_storage_persistent.c
View file @ 2c76f5b2
......@@ -44,6 +44,7 @@
#include "cache/cache.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
#include "vsha256.h"
......
bin/varnishd/storage/stevedore.c
View file @ 2c76f5b2
......@@ -131,65 +131,6 @@ STV_alloc(const struct stevedore *stv, size_t size)
return (st);
}
/*--------------------------------------------------------------------
* This function is called by stevedores ->allocobj() method, which
* very often will be stv_default_allocobj() below, to convert a slab
* of storage into object which the stevedore can then register in its
* internal state, before returning it to STV_NewObject().
* As you probably guessed: All this for persistence.
*/
struct object *
STV_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr)
{
struct object *o;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
assert(PAOK(ptr));
o = ptr;
INIT_OBJ(o, OBJECT_MAGIC);
VTAILQ_INIT(&o->list);
oc->stobj->magic = STOREOBJ_MAGIC;
oc->stobj->stevedore = stv;
AN(stv->methods);
oc->stobj->priv = o;
return (o);
}
/*--------------------------------------------------------------------
* This is the default ->allocobj() which all stevedores who do not
* implement persistent storage can rely on.
*/
int
stv_default_allocobj(const struct stevedore *stv, struct objcore *oc,
unsigned wsl)
{
struct object *o;
struct storage *st;
unsigned ltot;
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
ltot = sizeof(struct object) + PRNDUP(wsl);
st = stv->alloc(stv, ltot);
if (st == NULL)
return (0);
if (st->space < ltot) {
stv->free(st);
return (0);
}
o = STV_MkObject(stv, oc, st->ptr);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st->len = sizeof(*o);
o->objstore = st;
return (1);
}
/*-------------------------------------------------------------------
* Allocate storage for an object, based on the header information.
* XXX: If we know (a hint of) the length, we could allocate space
......
bin/varnishd/storage/storage.h
View file @ 2c76f5b2
......@@ -55,46 +55,17 @@ struct storage {
unsigned space;
};
/* Object ------------------------------------------------------------*/
VTAILQ_HEAD(storagehead, storage);
struct object {
unsigned magic;
#define OBJECT_MAGIC 0x32851d42
struct storage *objstore;
char oa_vxid[4];
uint8_t *oa_vary;
uint8_t *oa_http;
uint8_t oa_flags[1];
char oa_gzipbits[32];
char oa_lastmodified[8];
struct storagehead list;
ssize_t len;
struct storage *esidata;
};
/* Methods on objcore ------------------------------------------------*/
#ifdef VARNISH_CACHE_CHILD
typedef void updatemeta_f(struct worker *, struct objcore *oc);
typedef void freeobj_f(struct worker *, struct objcore *oc);
typedef struct lru *getlru_f(const struct objcore *oc);
/*
* Stevedores can either be simple, and provide just this method:
*/
typedef void objupdatemeta_f(struct worker *, struct objcore *oc);
typedef void objfree_f(struct worker *, struct objcore *oc);
typedef struct lru *objgetlru_f(const struct objcore *oc);
typedef struct object *getobj_f(struct worker *, struct objcore *oc);
/* This method is only used by SML (...to get to persistent) */
typedef struct object *sml_getobj_f(struct worker *, struct objcore *oc);
/*
* Or the can be "complex" and provide all of these methods:
* (Described in comments in cache_obj.c)
*/
typedef int objiterator_f(struct worker *, struct objcore *oc,
void *priv, objiterate_f *func);
typedef int objgetspace_f(struct worker *, struct objcore *,
......@@ -109,11 +80,11 @@ typedef void *objsetattr_f(struct worker *, struct objcore *,
typedef uint64_t objgetlen_f(struct worker *, struct objcore *);
struct storeobj_methods {
freeobj_f *freeobj;
getlru_f *getlru;
updatemeta_f *updatemeta;
objfree_f *objfree;
objgetlru_f *objgetlru;
objupdatemeta_f *objupdatemeta;
getobj_f *getobj;
sml_getobj_f *sml_getobj;
objiterator_f *objiterator;
objgetspace_f *objgetspace;
......@@ -152,10 +123,6 @@ typedef void storage_banexport_f(const struct stevedore *, const uint8_t *bans,
#include "tbl/vrt_stv_var.h"
#undef VRTSTVTYPE
extern storage_allocobj_f stv_default_allocobj;
extern const struct storeobj_methods default_oc_methods;
/*--------------------------------------------------------------------*/
struct stevedore {
......@@ -199,8 +166,6 @@ extern struct stevedore *stv_transient;
int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
const char *ctx);
struct object *STV_MkObject(const struct stevedore *, struct objcore *,
void *ptr);
struct lru *LRU_Alloc(void);
void LRU_Free(struct lru *lru);
......
bin/varnishd/storage/storage_file.c
View file @ 2c76f5b2
......@@ -39,6 +39,7 @@
#include "cache/cache.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
#include "vnum.h"
#include "vfil.h"
......@@ -535,7 +536,8 @@ const struct stevedore smf_stevedore = {
.alloc = smf_alloc,
.trim = smf_trim,
.free = smf_free,
.methods = &default_oc_methods,
.allocobj = SML_allocobj,
.methods = &SML_methods,
};
#ifdef INCLUDE_TEST_DRIVER
......
bin/varnishd/storage/storage_malloc.c
View file @ 2c76f5b2
......@@ -36,6 +36,7 @@
#include "cache/cache.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
#include "vnum.h"
......@@ -251,7 +252,8 @@ const struct stevedore sma_stevedore = {
.alloc = sma_alloc,
.free = sma_free,
.trim = sma_trim,
.methods = &default_oc_methods,
.allocobj = SML_allocobj,
.methods = &SML_methods,
.var_free_space = sma_free_space,
.var_used_space = sma_used_space,
};
bin/varnishd/storage/storage_persistent.c
View file @ 2c76f5b2
......@@ -44,6 +44,7 @@
#include "cache/cache.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
#include "hash/hash_slinger.h"
#include "vcli.h"
......@@ -53,6 +54,8 @@
#include "storage/storage_persistent.h"
static struct storeobj_methods smp_oc_realmethods;
/*--------------------------------------------------------------------*/
/*
......@@ -526,7 +529,7 @@ smp_allocobj(const struct stevedore *stv, struct objcore *oc, unsigned wsl)
assert(st->space >= ltot);
o = STV_MkObject(stv, oc, st->ptr);
o = SML_MkObject(stv, oc, st->ptr);
AN(oc->stobj->stevedore);
assert(oc->stobj->stevedore == stv);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
......@@ -592,7 +595,7 @@ const struct stevedore smp_stevedore = {
.signal_close = smp_signal_close,
.baninfo = smp_baninfo,
.banexport = smp_banexport,
.methods = &smp_oc_methods,
.methods = &smp_oc_realmethods,
};
/*--------------------------------------------------------------------
......@@ -682,6 +685,11 @@ void
SMP_Init(void)
{
CLI_AddFuncs(debug_cmds);
smp_oc_realmethods = SML_methods;
smp_oc_realmethods.sml_getobj = smp_oc_methods.sml_getobj;
smp_oc_realmethods.objupdatemeta = smp_oc_methods.objupdatemeta;
smp_oc_realmethods.objfree = smp_oc_methods.objfree;
smp_oc_realmethods.objgetlru = smp_oc_methods.objgetlru;
}
/*--------------------------------------------------------------------
......
bin/varnishd/storage/storage_persistent_silo.c
View file @ 2c76f5b2
......@@ -40,6 +40,7 @@
#include "cache/cache.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
#include "hash/hash_slinger.h"
#include "vsha256.h"
......@@ -387,7 +388,7 @@ smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg,
*/
static struct object *
smp_oc_getobj(struct worker *wrk, struct objcore *oc)
smp_oc_sml_getobj(struct worker *wrk, struct objcore *oc)
{
struct object *o;
struct smp_seg *sg;
......@@ -401,7 +402,7 @@ smp_oc_getobj(struct worker *wrk, struct objcore *oc)
AN(oc->stobj->stevedore);
/* Some calls are direct, but they should match anyway */
assert(oc->stobj->stevedore->methods->getobj == smp_oc_getobj);
assert(oc->stobj->stevedore->methods->sml_getobj == smp_oc_sml_getobj);
CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC);
so = smp_find_so(sg, oc->stobj->priv2);
......@@ -455,7 +456,7 @@ smp_oc_getobj(struct worker *wrk, struct objcore *oc)
}
static void
smp_oc_updatemeta(struct worker *wrk, struct objcore *oc)
smp_oc_objupdatemeta(struct worker *wrk, struct objcore *oc)
{
struct object *o;
struct smp_seg *sg;
......@@ -463,7 +464,7 @@ smp_oc_updatemeta(struct worker *wrk, struct objcore *oc)
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
o = smp_oc_getobj(wrk, oc);
o = smp_oc_sml_getobj(wrk, oc);
AN(o);
CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC);
......@@ -482,8 +483,8 @@ smp_oc_updatemeta(struct worker *wrk, struct objcore *oc)
}
}
static void __match_proto__(freeobj_f)
smp_oc_freeobj(struct worker *wrk, struct objcore *oc)
static void __match_proto__(objfree_f)
smp_oc_objfree(struct worker *wrk, struct objcore *oc)
{
struct smp_seg *sg;
struct smp_object *so;
......@@ -516,8 +517,8 @@ smp_oc_freeobj(struct worker *wrk, struct objcore *oc)
* Find the per-segment lru list for this object
*/
static struct lru * __match_proto__(getlru_f)
smp_oc_getlru(const struct objcore *oc)
static struct lru * __match_proto__(objgetlru_f)
smp_oc_objgetlru(const struct objcore *oc)
{
struct smp_seg *sg;
......@@ -526,10 +527,10 @@ smp_oc_getlru(const struct objcore *oc)
}
const struct storeobj_methods smp_oc_methods = {
.getobj = smp_oc_getobj,
.updatemeta = smp_oc_updatemeta,
.freeobj = smp_oc_freeobj,
.getlru = smp_oc_getlru,
.sml_getobj = smp_oc_sml_getobj,
.objupdatemeta = smp_oc_objupdatemeta,
.objfree = smp_oc_objfree,
.objgetlru = smp_oc_objgetlru,
};
/*--------------------------------------------------------------------*/
......
bin/varnishd/storage/storage_simple.c
View file @ 2c76f5b2
......@@ -33,27 +33,92 @@
#include <stdlib.h>
#include "cache/cache.h"
#include "hash/hash_slinger.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
/*--------------------------------------------------------------------
* This function is called by stevedores ->allocobj() method, which
* very often will be SML_allocobj() below, to convert a slab
* of storage into object which the stevedore can then register in its
* internal state, before returning it to STV_NewObject().
* As you probably guessed: All this for persistence.
*/
struct object *
SML_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr)
{
struct object *o;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
assert(PAOK(ptr));
o = ptr;
INIT_OBJ(o, OBJECT_MAGIC);
VTAILQ_INIT(&o->list);
oc->stobj->magic = STOREOBJ_MAGIC;
oc->stobj->stevedore = stv;
AN(stv->methods);
oc->stobj->priv = o;
return (o);
}
/*--------------------------------------------------------------------
* This is the default ->allocobj() which all stevedores who do not
* implement persistent storage can rely on.
*/
int
SML_allocobj(const struct stevedore *stv, struct objcore *oc,
unsigned wsl)
{
struct object *o;
struct storage *st;
unsigned ltot;
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
ltot = sizeof(struct object) + PRNDUP(wsl);
st = stv->alloc(stv, ltot);
if (st == NULL)
return (0);
if (st->space < ltot) {
stv->free(st);
return (0);
}
o = SML_MkObject(stv, oc, st->ptr);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st->len = sizeof(*o);
o->objstore = st;
return (1);
}
/*---------------------------------------------------------------------
*/
static struct object * __match_proto__(getobj_f)
default_oc_getobj(struct worker *wrk, struct objcore *oc)
static struct object *
getobj(struct worker *wrk, struct objcore *oc)
{
const struct storeobj_methods *m;
struct object *o;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
m = oc->stobj->stevedore->methods;
if (m->sml_getobj != NULL)
return (m->sml_getobj(wrk, oc));
if (oc->stobj->priv == NULL)
return (NULL);
CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
return (o);
}
static void __match_proto__(freeobj_f)
default_oc_freeobj(struct worker *wrk, struct objcore *oc)
static void __match_proto__(objfree_f)
sml_objfree(struct worker *wrk, struct objcore *oc)
{
struct object *o;
......@@ -71,8 +136,8 @@ default_oc_freeobj(struct worker *wrk, struct objcore *oc)
wrk->stats->n_object--;
}
static struct lru * __match_proto__(getlru_f)
default_oc_getlru(const struct objcore *oc)
static struct lru * __match_proto__(objgetlru_f)
sml_objgetlru(const struct objcore *oc)
{
const struct stevedore *stv;
......@@ -81,8 +146,363 @@ default_oc_getlru(const struct objcore *oc)
return (stv->lru);
}
const struct storeobj_methods default_oc_methods = {
.getobj = default_oc_getobj,
.freeobj = default_oc_freeobj,
.getlru = default_oc_getlru,
static int __match_proto__(objiterate_f)
sml_iterator(struct worker *wrk, struct objcore *oc,
void *priv, objiterate_f *func)
{
struct busyobj *bo;
struct object *obj;
struct storage *st;
struct storage *checkpoint = NULL;
ssize_t checkpoint_len = 0;
ssize_t len = 0;
int ret = 0;
ssize_t ol;
ssize_t nl;
ssize_t sl;
void *p;
ssize_t l;
obj = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo = HSH_RefBusy(oc);
if (bo == NULL) {
VTAILQ_FOREACH(st, &obj->list, list)
if (func(priv, 0, st->ptr, st->len))
return (-1);
return (0);
}
p = NULL;
l = 0;
while (1) {
ol = len;
nl = VBO_waitlen(wrk, bo, ol);
if (bo->state == BOS_FAILED) {
ret = -1;
break;
}
if (nl == ol) {
if (bo->state == BOS_FINISHED)
break;
continue;
}
Lck_Lock(&bo->mtx);
AZ(VTAILQ_EMPTY(&obj->list));
if (checkpoint == NULL) {
st = VTAILQ_FIRST(&obj->list);
sl = 0;
} else {
st = checkpoint;
sl = checkpoint_len;
ol -= checkpoint_len;
}
while (st != NULL) {
if (st->len > ol) {
p = st->ptr + ol;
l = st->len - ol;
len += l;
break;
}
ol -= st->len;
assert(ol >= 0);
nl -= st->len;
assert(nl > 0);
sl += st->len;
st = VTAILQ_NEXT(st, list);
if (VTAILQ_NEXT(st, list) != NULL) {
checkpoint = st;
checkpoint_len = sl;
}
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
st = VTAILQ_NEXT(st, list);
if (st != NULL && st->len == 0)
st = NULL;
Lck_Unlock(&bo->mtx);
assert(l > 0 || bo->state == BOS_FINISHED);
if (func(priv, st != NULL ? 0 : 1, p, l)) {
ret = -1;
break;
}
}
if (oc->flags & OC_F_PASS)
bo->abandon = 1;
VBO_DerefBusyObj(wrk, &bo);
return (ret);
}
/*--------------------------------------------------------------------
*/
static struct storage *
objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
{
struct storage *st = NULL;
unsigned fail;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
if (size > cache_param->fetch_maxchunksize)
size = cache_param->fetch_maxchunksize;
assert(size <= UINT_MAX); /* field limit in struct storage */
for (fail = 0; fail <= cache_param->nuke_limit; fail++) {
/* try to allocate from it */
AN(stv->alloc);
st = STV_alloc(stv, size);
if (st != NULL)
break;
/* no luck; try to free some space and keep trying */
if (fail < cache_param->nuke_limit &&
EXP_NukeOne(wrk, stv->lru) == -1)
break;
}
CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
return (st);
}
static int __match_proto__(objgetspace_f)
sml_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz,
uint8_t **ptr)
{
struct object *o;
struct storage *st;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
AN(sz);
AN(ptr);
assert(*sz > 0);
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = VTAILQ_LAST(&o->list, storagehead);
if (st != NULL && st->len < st->space) {
*sz = st->space - st->len;
*ptr = st->ptr + st->len;
assert (*sz > 0);
return (1);
}
st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz);
if (st == NULL)
return (0);
if (oc->busyobj != NULL) {
CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
Lck_Lock(&oc->busyobj->mtx);
VTAILQ_INSERT_TAIL(&o->list, st, list);
Lck_Unlock(&oc->busyobj->mtx);
} else {
AN(oc->flags & (OC_F_PRIVATE));
VTAILQ_INSERT_TAIL(&o->list, st, list);
}
*sz = st->space - st->len;
assert (*sz > 0);
*ptr = st->ptr + st->len;
return (1);
}
static void __match_proto__(objextend_f)
sml_extend(struct worker *wrk, struct objcore *oc, ssize_t l)
{
struct object *o;
struct storage *st;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
assert(l > 0);
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = VTAILQ_LAST(&o->list, storagehead);
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
assert(st->len + l <= st->space);
st->len += l;
o->len += l;
}
static uint64_t __match_proto__(objgetlen_f)
sml_getlen(struct worker *wrk, struct objcore *oc)
{
struct object *o;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
return (o->len);
}
static void __match_proto__(objtrimstore_f)
sml_trimstore(struct worker *wrk, struct objcore *oc)
{
const struct stevedore *stv;
struct storage *st;
struct object *o;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
stv = oc->stobj->stevedore;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = VTAILQ_LAST(&o->list, storagehead);
if (st == NULL)
return;
if (st->len == 0) {
VTAILQ_REMOVE(&o->list, st, list);
STV_free(stv, st);
} else if (st->len < st->space) {
STV_trim(stv, st, st->len, 1);
}
}
static void __match_proto__(objslim_f)
sml_slim(struct worker *wrk, struct objcore *oc)
{
const struct stevedore *stv;
struct object *o;
struct storage *st, *stn;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
stv = oc->stobj->stevedore;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
if (o->esidata != NULL) {
STV_free(stv, o->esidata);
o->esidata = NULL;
}
VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) {
CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
VTAILQ_REMOVE(&o->list, st, list);
STV_free(stv, st);
}
}
static void * __match_proto__(objgetattr_f)
sml_getattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
ssize_t *len)
{
struct object *o;
ssize_t dummy;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (len == NULL)
len = &dummy;
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
switch (attr) {
case OA_ESIDATA:
if (o->esidata == NULL)
return (NULL);
*len = o->esidata->len;
return (o->esidata->ptr);
case OA_FLAGS:
*len = sizeof o->oa_flags;
return (o->oa_flags);
case OA_GZIPBITS:
*len = sizeof o->oa_gzipbits;
return (o->oa_gzipbits);
case OA_HEADERS:
*len = 0; // XXX: hack
return (o->oa_http);
case OA_LASTMODIFIED:
*len = sizeof o->oa_lastmodified;
return (o->oa_lastmodified);
case OA_VARY:
*len = 4; // XXX: hack
return (o->oa_vary);
case OA_VXID:
*len = sizeof o->oa_vxid;
return (o->oa_vxid);
default:
break;
}
WRONG("Unsupported OBJ_ATTR");
}
static void * __match_proto__(objsetattr_f)
sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
ssize_t len, const void *ptr)
{
struct object *o;
void *retval = NULL;
struct storage *st;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
o = getobj(wrk, oc);
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
st = o->objstore;
switch (attr) {
case OA_ESIDATA:
o->esidata = objallocwithnuke(wrk, oc->stobj->stevedore, len);
if (o->esidata == NULL)
return (NULL);
o->esidata->len = len;
retval = o->esidata->ptr;
break;
case OA_FLAGS:
assert(len == sizeof o->oa_flags);
retval = o->oa_flags;
break;
case OA_GZIPBITS:
assert(len == sizeof o->oa_gzipbits);
retval = o->oa_gzipbits;
break;
case OA_HEADERS:
len = PRNDUP(len);
assert(st->len + len <= st->space);
o->oa_http = (void*)(st->ptr + st->len);
st->len += len;
retval = o->oa_http;
break;
case OA_LASTMODIFIED:
assert(len == sizeof o->oa_lastmodified);
retval = o->oa_lastmodified;
break;
case OA_VARY:
len = PRNDUP(len);
assert(st->len + len <= st->space);
o->oa_vary = (void*)(st->ptr + st->len);
st->len += len;
retval = o->oa_vary;
break;
case OA_VXID:
assert(len == sizeof o->oa_vxid);
retval = o->oa_vxid;
break;
default:
WRONG("Unsupported OBJ_ATTR");
break;
}
if (ptr != NULL)
memcpy(retval, ptr, len);
return (retval);
}
const struct storeobj_methods SML_methods = {
.objfree = sml_objfree,
.objgetlru = sml_objgetlru,
.objiterator = sml_iterator,
.objgetspace = sml_getspace,
.objextend = sml_extend,
.objgetlen = sml_getlen,
.objtrimstore = sml_trimstore,
.objslim = sml_slim,
.objgetattr = sml_getattr,
.objsetattr = sml_setattr,
};
bin/varnishd/storage/storage_simple.h 0 → 100644
View file @ 2c76f5b2
/*-
* Copyright (c) 2006 Verdens Gang AS
* Copyright (c) 2006-2011 Varnish Software AS
* All rights reserved.
*
* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* SML is a set of methods for simple stevedores which just do simple
* memory allocation and leave all the high-level stuff to SML.
*
*/
/* Object ------------------------------------------------------------*/
VTAILQ_HEAD(storagehead, storage);
struct object {
unsigned magic;
#define OBJECT_MAGIC 0x32851d42
struct storage *objstore;
char oa_vxid[4];
uint8_t *oa_vary;
uint8_t *oa_http;
uint8_t oa_flags[1];
char oa_gzipbits[32];
char oa_lastmodified[8];
struct storagehead list;
ssize_t len;
struct storage *esidata;
};
extern const struct storeobj_methods SML_methods;
struct object *SML_MkObject(const struct stevedore *, struct objcore *,
void *ptr);
storage_allocobj_f SML_allocobj;
bin/varnishd/storage/storage_umem.c
View file @ 2c76f5b2
......@@ -41,6 +41,7 @@
#include "cache/cache.h"
#include "storage/storage.h"
#include "storage/storage_simple.h"
static size_t smu_max = SIZE_MAX;
static MTX smu_mtx;
......@@ -161,7 +162,8 @@ const struct stevedore smu_stevedore = {
.alloc = smu_alloc,
.free = smu_free,
.trim = smu_trim,
.methods = &default_oc_methods,
.allocobj = SML_allocobj,
.methods = &SML_methods,
};
#endif /* HAVE_UMEM_H */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment