Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
L
libvdp-pesi
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Jobs
Commits
Open sidebar
uplex-varnish
libvdp-pesi
Commits
6524e3bb
Commit
6524e3bb
authored
Aug 02, 2019
by
Nils Goroll
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
vmod / vdp segregation
much remains to be done
parent
829f8627
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
670 additions
and
527 deletions
+670
-527
Makefile.am
src/Makefile.am
+2
-1
node.h
src/node.h
+172
-0
vdp_pesi.c
src/vdp_pesi.c
+17
-526
vdp_pesi.h
src/vdp_pesi.h
+69
-0
vmod_pesi.c
src/vmod_pesi.c
+410
-0
No files found.
src/Makefile.am
View file @
6524e3bb
...
...
@@ -8,6 +8,7 @@ AM_LDFLAGS = $(VARNISHAPI_LIBS) $(VMOD_LDFLAGS) -ldl
vmod_LTLIBRARIES
=
libvmod_pesi.la
libvmod_pesi_la_SOURCES
=
\
vmod_pesi.c
\
vdp_pesi.c
\
tbl_set_parameter.h
\
foreign/qdef.h
\
...
...
@@ -24,7 +25,7 @@ dist_man_MANS = vdp_pesi.3
@BUILD_VSC_PESI@
v
dp_pesi.lo
:
vcc_if.h VSC_pesi.c VSC_pesi.h
v
mod_pesi.c
:
vcc_if.h VSC_pesi.c VSC_pesi.h
vcc_if.h vmod_pesi.rst vmod_pesi.man.rst
:
vcc_if.c
...
...
src/node.h
0 → 100644
View file @
6524e3bb
/*-
* Copyright 2019 UPLEX Nils Goroll Systemoptimierung
* All rights reserved
*
* Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
* Nils Goroll <nils.goroll@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* node interface
*
* XXX mempool vs. struct node
*/
enum
n_type
{
T_INVALID
=
0
,
T_NEXUS
,
// can change into T_SUBREQ / T_FINAL / T_DATA
T_DATA
,
T_CRC
,
T_SUBREQ
,
T_FINAL
// non-ESI pass / hfm / hfp
};
/*
* see state.dot:
*
* ST_DATA: may never have any children
*
* T_DATA / T_CRC / T_SUBREQ / T_FINAL
*
* ST_PRIVATE: may receive pushes creating children
* unpending must not yet touch it
* children can be created unlocked
* can change into other types
*
* T_NEXUS only
*
* ST_OPEN: may receive pushes creating children
* owning thread may run front delivery
* any changes below locked only
*
* T_NEXUS only
*
*
* ST_CLOSED: the request is done, no pushes can occur,
* unpending can proceed upwards
*
* T_NEXUS only
*
* ST_UNPENDING: in the process of being pushed to the client
*
* T_DATA / T_CRC / T_SUBREQ / T_FINAL
*
* ST_DELIVERED: We have pushed data up
*
* any type
*
* for T_NEXUS, means "anything below is delivered"
*/
enum
n_state
{
ST_INVALID
=
0
,
ST_DATA
,
ST_PRIVATE
,
ST_OPEN
,
ST_CLOSED
,
ST_UNPENDING
,
ST_DELIVERED
,
};
VSTAILQ_HEAD
(
node_head
,
node
);
struct
node_nexus
{
struct
node_head
children
;
struct
objcore
*
oc
;
struct
req
*
req
;
const
struct
worker
*
owner
;
// ST_OPEN only
/* number of nodes pending under this node while state == ST_PRIVATE */
int
npending_private
;
/*
* crc for data nodes immediately below
* updated by push_crc, pretendgzip and gzgz
*/
struct
nexus_gzip
gzip
;
// in foreign
};
enum
t_crc
{
INVALID
=
0
,
UPDATE
,
FINAL
// combine with parent or gen tail
};
struct
node_crc
{
enum
t_crc
ctype
;
uint32_t
icrc
;
ssize_t
l_icrc
;
};
struct
node_data
{
const
void
*
ptr
;
struct
storage
*
st
;
ssize_t
len
;
enum
vdp_action
act
;
};
/*
* we transfer the sub-request, boc and oc to the topreq thread, delivery
* happens there
*/
struct
node_subreq
{
struct
req
*
req
;
struct
boc
*
boc
;
// oc is NULL if already transferred back into req
struct
objcore
*
oc
;
// subreq to topreq delivery
int
done
;
pthread_cond_t
cond
;
};
// sub-state for node_final while in ST_DATA
enum
fi_state
{
FI_READY
=
0
,
FI_GO
,
// topreq signalling req to deliver
FI_DONE
,
// req signalling topreq it is done
FI_DESTROYED
// cond/mtx destroyed (fini_final())
};
/* we block the sub-thread when it's ready for delivery and continue when the
* topreqp tells it to */
struct
node_final
{
enum
fi_state
fi_state
;
pthread_mutex_t
fi_mtx
;
pthread_cond_t
fi_cond
;
};
struct
node
{
unsigned
magic
;
#define NODE_MAGIC 0xe31edef3
enum
n_type
type
;
enum
n_state
state
;
VSTAILQ_ENTRY
(
node
)
sibling
;
VSTAILQ_ENTRY
(
node
)
unpend
;
struct
node
*
parent
;
union
{
struct
node_nexus
nexus
;
// T_NEXUS
struct
node_data
data
;
// T_DATA
struct
node_subreq
subreq
;
// T_SUBREQ
struct
node_final
final
;
// T_FINAL
struct
node_crc
crc
;
// T_CRC
};
};
src/vdp_pesi.c
View file @
6524e3bb
...
...
@@ -58,20 +58,20 @@
#include "vcc_if.h"
#include "VSC_pesi.h"
#
define VFAIL(ctx, fmt, ...) \
VRT_fail((ctx), "vdp pesi failure: " fmt, __VA_ARGS__)
#
include "vdp_pesi.h"
#include "node.h"
/*
* whether to use blocking threads for final (private/hfm/fpm) objects
*
* if false, buffering will be used
*/
static
int
block_final
=
0
;
int
block_final
=
0
;
/*
* whether to push bytes early
*
*/
static
int
front_push
=
0
;
int
front_push
=
0
;
#ifdef DEBUG
#define VSLdbgv(req, fmt, ...) \
...
...
@@ -105,145 +105,6 @@ static const uint8_t gzip_hdr[] = {
#define OC_F_FINAL (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)
enum
n_type
{
T_INVALID
=
0
,
T_NEXUS
,
// can change into T_SUBREQ / T_FINAL / T_DATA
T_DATA
,
T_CRC
,
T_SUBREQ
,
T_FINAL
// non-ESI pass / hfm / hfp
};
/*
* see state.dot:
*
* ST_DATA: may never have any children
*
* T_DATA / T_CRC / T_SUBREQ / T_FINAL
*
* ST_PRIVATE: may receive pushes creating children
* unpending must not yet touch it
* children can be created unlocked
* can change into other types
*
* T_NEXUS only
*
* ST_OPEN: may receive pushes creating children
* owning thread may run front delivery
* any changes below locked only
*
* T_NEXUS only
*
*
* ST_CLOSED: the request is done, no pushes can occur,
* unpending can proceed upwards
*
* T_NEXUS only
*
* ST_UNPENDING: in the process of being pushed to the client
*
* T_DATA / T_CRC / T_SUBREQ / T_FINAL
*
* ST_DELIVERED: We have pushed data up
*
* any type
*
* for T_NEXUS, means "anything below is delivered"
*/
enum
n_state
{
ST_INVALID
=
0
,
ST_DATA
,
ST_PRIVATE
,
ST_OPEN
,
ST_CLOSED
,
ST_UNPENDING
,
ST_DELIVERED
,
};
VSTAILQ_HEAD
(
node_head
,
node
);
struct
node_nexus
{
struct
node_head
children
;
struct
objcore
*
oc
;
struct
req
*
req
;
const
struct
worker
*
owner
;
// ST_OPEN only
/* number of nodes pending under this node while state == ST_PRIVATE */
int
npending_private
;
/*
* crc for data nodes immediately below
* updated by push_crc, pretendgzip and gzgz
*/
struct
nexus_gzip
gzip
;
// in foreign
};
enum
t_crc
{
INVALID
=
0
,
UPDATE
,
FINAL
// combine with parent or gen tail
};
struct
node_crc
{
enum
t_crc
ctype
;
uint32_t
icrc
;
ssize_t
l_icrc
;
};
struct
node_data
{
const
void
*
ptr
;
struct
storage
*
st
;
ssize_t
len
;
enum
vdp_action
act
;
};
/*
* we transfer the sub-request, boc and oc to the topreq thread, delivery
* happens there
*/
struct
node_subreq
{
struct
req
*
req
;
struct
boc
*
boc
;
// oc is NULL if already transferred back into req
struct
objcore
*
oc
;
// subreq to topreq delivery
int
done
;
pthread_cond_t
cond
;
};
// sub-state for node_final while in ST_DATA
enum
fi_state
{
FI_READY
=
0
,
FI_GO
,
// topreq signalling req to deliver
FI_DONE
,
// req signalling topreq it is done
FI_DESTROYED
// cond/mtx destroyed (fini_final())
};
/* we block the sub-thread when it's ready for delivery and continue when the
* topreqp tells it to */
struct
node_final
{
enum
fi_state
fi_state
;
pthread_mutex_t
fi_mtx
;
pthread_cond_t
fi_cond
;
};
struct
node
{
unsigned
magic
;
#define NODE_MAGIC 0xe31edef3
enum
n_type
type
;
enum
n_state
state
;
VSTAILQ_ENTRY
(
node
)
sibling
;
VSTAILQ_ENTRY
(
node
)
unpend
;
struct
node
*
parent
;
union
{
struct
node_nexus
nexus
;
// T_NEXUS
struct
node_data
data
;
// T_DATA
struct
node_subreq
subreq
;
// T_SUBREQ
struct
node_final
final
;
// T_FINAL
struct
node_crc
crc
;
// T_CRC
};
};
struct
bytes_tree
{
unsigned
magic
;
#define BYTES_TREE_MAGIC 0x49c59d46
...
...
@@ -280,26 +141,6 @@ struct pesi_tree {
int
task_finishing
;
};
#define PF_HAS_TASK 1U
/* vcl-controlled flags */
#define PF_CFG_SERIAL (1U<<1)
#define PF_CFG_THREAD (1U<<2)
/* undocumented for now */
#define PF_CFG_BLOCK_FINAL (1U<<3)
#define PF_CFG_FRONT_PUSH (1U<<4)
#define PF_CFG_DEFAULT \
( PF_CFG_THREAD \
| (block_final ? PF_CFG_BLOCK_FINAL : 0) \
| (front_push ? PF_CFG_FRONT_PUSH : 0) \
)
#define PF_MASK_CFG \
( PF_CFG_SERIAL \
| PF_CFG_THREAD \
| PF_CFG_BLOCK_FINAL \
| PF_CFG_FRONT_PUSH \
)
/*
* per request state
*
...
...
@@ -346,17 +187,11 @@ static void fini_final(struct req *, struct node *);
static
void
fini_subreq
(
struct
req
*
,
struct
node
*
);
static
void
fini_data
(
struct
req
*
,
struct
node
*
);
/* id for PRIV_TASK */
const
void
*
const
priv_task_id_cfg
=
&
priv_task_id_cfg
;
/* shared object globals */
static
unsigned
loadcnt
=
0
,
warmcnt
=
0
;
static
struct
VSC_lck
*
lck_bytes_tree
,
*
lck_pesi_tree
,
*
lck_stats
;
static
struct
vsc_seg
*
vsc_seg
=
NULL
,
*
pesi_vsc_seg
=
NULL
;
static
struct
mempool
*
mempool
=
NULL
;
static
unsigned
node_alloc_sz
;
static
struct
VSC_pesi
*
stats
;
static
struct
lock
stats_lock
;
/* shared with vmod code */
struct
lock
stats_lock
;
struct
VSC_pesi
*
stats
;
struct
VSC_lck
*
lck_bytes_tree
,
*
lck_pesi_tree
;
struct
mempool
*
mempool
=
NULL
;
/* also used by our version of cache_esi_deliver.c */
...
...
@@ -502,6 +337,12 @@ task_fini(struct pesi_tree *pesi_tree, struct pesi *pesi)
* tree related
*/
size_t
node_size
()
{
return
(
sizeof
(
struct
node
));
}
static
inline
struct
node
*
node_alloc
(
void
)
{
...
...
@@ -2158,30 +1999,6 @@ push_vdps(struct req *req, struct nexus_gzip *gz)
return
(
0
);
}
static
void
vcl_cfg
(
struct
req
*
req
,
unsigned
*
flags
)
{
struct
vmod_priv
*
priv_task
;
struct
vrt_ctx
dummy_ctx
[
1
];
unsigned
vclflags
;
INIT_OBJ
(
dummy_ctx
,
VRT_CTX_MAGIC
);
dummy_ctx
->
req
=
req
;
dummy_ctx
->
ws
=
req
->
ws
;
/* zero length in the priv task == not configured from vcl */
priv_task
=
VRT_priv_task
(
dummy_ctx
,
priv_task_id_cfg
);
if
(
priv_task
==
NULL
||
priv_task
->
len
==
0
)
return
;
assert
(
priv_task
->
len
==
1
);
AZ
(
priv_task
->
free
);
vclflags
=
(
unsigned
)(
uintptr_t
)
priv_task
->
priv
;
AZ
(
vclflags
&
~
PF_MASK_CFG
);
*
flags
=
(
*
flags
&
~
PF_MASK_CFG
)
|
vclflags
;
}
static
int
v_matchproto_
(
vdp_init_f
)
vdp_pesi_init
(
struct
req
*
req
,
void
**
priv
)
{
...
...
@@ -2201,7 +2018,7 @@ vdp_pesi_init(struct req *req, void **priv)
*
priv
=
pesi
;
WS_Assert_Allocated
(
req
->
ws
,
pesi
,
sizeof
*
pesi
);
vcl
_cfg
(
req
,
&
pesi
->
flags
);
get_task
_cfg
(
req
,
&
pesi
->
flags
);
return
(
0
);
}
...
...
@@ -2234,7 +2051,7 @@ vdp_pesi_init(struct req *req, void **priv)
"Cannot allocate workspace for parallel ESI data"
);
return
(
-
1
);
}
vcl
_cfg
(
req
,
&
pesi
->
flags
);
get_task
_cfg
(
req
,
&
pesi
->
flags
);
pecx
=
pesi
->
pecx
;
*
priv
=
pesi
;
RFC2616_Weaken_Etag
(
req
->
resp
);
...
...
@@ -2859,329 +2676,3 @@ static const struct transport VPED_transport = {
.
reembark
=
vped_reembark
,
.
minimal_response
=
vped_minimal_response
,
};
/* VMOD functions */
static
volatile
struct
poolparam
poolparam
=
{
.
min_pool
=
10
,
.
max_pool
=
100
,
.
max_age
=
10
,
};
VCL_VOID
vmod_pool
(
VRT_CTX
,
VCL_INT
min
,
VCL_INT
max
,
VCL_DURATION
max_age
)
{
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
/* cf. tweak_poolparam() in mgt_param_tweak.c */
if
(
min
<=
0
)
{
VFAIL
(
ctx
,
"min (%jd) must be > 0 in pool()"
,
min
);
return
;
}
if
(
max
<=
0
)
{
VFAIL
(
ctx
,
"max (%jd) must be > 0 in pool()"
,
max
);
return
;
}
if
(
max
<
min
)
{
VFAIL
(
ctx
,
"max (%jd) < min (%jd) in pool()"
,
max
,
min
);
return
;
}
if
(
max_age
<
0
.)
{
VFAIL
(
ctx
,
"max_age (%.0fs) < 0s in pool()"
,
max_age
);
return
;
}
if
(
max_age
>
1e6
)
{
VFAIL
(
ctx
,
"max_age (%.0fs) out of range in pool() (max 10^6s)"
,
max_age
);
return
;
}
poolparam
.
min_pool
=
min
;
poolparam
.
max_pool
=
max
;
poolparam
.
max_age
=
max_age
;
}
/*
* pesi_filter_on_ws() and pesi_resp_default_filter_list()
* taken from cache_vrt_filter.c
*
*/
typedef
void
pesi_filter_list_t
(
void
*
,
struct
vsb
*
vsb
);
static
const
char
*
pesi_filter_on_ws
(
struct
ws
*
ws
,
pesi_filter_list_t
*
func
,
void
*
arg
)
{
unsigned
u
;
struct
vsb
vsb
[
1
];
AN
(
func
);
AN
(
arg
);
u
=
WS_ReserveAll
(
ws
);
if
(
u
==
0
)
{
WS_Release
(
ws
,
0
);
WS_MarkOverflow
(
ws
);
return
(
NULL
);
}
AN
(
VSB_new
(
vsb
,
ws
->
f
,
u
,
VSB_FIXEDLEN
));
func
(
arg
,
vsb
);
if
(
VSB_finish
(
vsb
))
{
WS_Release
(
ws
,
0
);
WS_MarkOverflow
(
ws
);
return
(
NULL
);
}
if
(
VSB_len
(
vsb
))
{
WS_Release
(
ws
,
VSB_len
(
vsb
)
+
1
);
return
(
VSB_data
(
vsb
)
+
1
);
}
WS_Release
(
ws
,
0
);
return
(
""
);
}
static
void
v_matchproto_
(
pesi_filter_list_t
)
pesi_resp_default_filter_list
(
void
*
arg
,
struct
vsb
*
vsb
)
{
struct
req
*
req
;
CAST_OBJ_NOTNULL
(
req
,
arg
,
REQ_MAGIC
);
/*
* the req->resp_len check has been removed because it does not work for
* busy objects
*
* pesi will still do the right thing if the response really is empty
*/
if
(
!
req
->
disable_esi
&&
/* req->resp_len != 0 && */
ObjHasAttr
(
req
->
wrk
,
req
->
objcore
,
OA_ESIDATA
))
VSB_cat
(
vsb
,
" pesi"
);
if
(
cache_param
->
http_gzip_support
&&
ObjCheckFlag
(
req
->
wrk
,
req
->
objcore
,
OF_GZIPED
)
&&
!
RFC2616_Req_Gzip
(
req
->
http
))
VSB_cat
(
vsb
,
" gunzip"
);
if
(
cache_param
->
http_range_support
&&
http_GetStatus
(
req
->
resp
)
==
200
&&
http_GetHdr
(
req
->
http
,
H_Range
,
NULL
))
VSB_cat
(
vsb
,
" range"
);
}
VCL_VOID
vmod_activate
(
VRT_CTX
)
{
struct
req
*
req
;
const
char
*
filters
;
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
if
(
ctx
->
method
!=
VCL_MET_DELIVER
)
{
VRT_fail
(
ctx
,
"pesi.activate() may only be called "
"from vcl_deliver{}"
);
return
;
}
req
=
ctx
->
req
;
CHECK_OBJ_NOTNULL
(
req
,
REQ_MAGIC
);
filters
=
pesi_filter_on_ws
(
req
->
ws
,
pesi_resp_default_filter_list
,
req
);
if
(
filters
==
NULL
)
WS_MarkOverflow
(
req
->
ws
);
else
req
->
filter_list
=
filters
;
}
static
unsigned
vmod_set_param_flag
(
VCL_ENUM
e
)
{
#define VMODENUM(p,f) if (e == VENUM(p)) return(f);
#include "tbl_set_parameter.h"
WRONG
(
"illegal enum"
);
}
VCL_VOID
vmod_set
(
VRT_CTX
,
struct
VARGS
(
set
)
*
args
)
{
struct
vmod_priv
*
priv_task
;
unsigned
f
,
vclflags
;
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
if
(
ctx
->
method
!=
VCL_MET_DELIVER
)
{
VRT_fail
(
ctx
,
"pesi.set() may only be called "
"from vcl_deliver{}"
);
return
;
}
/* as of now, all parameters require a bool parameter */
if
(
args
->
valid_bool
==
0
)
{
VRT_fail
(
ctx
,
"pesi.set(%s) requires a bool "
"parameter"
,
args
->
parameter
);
return
;
}
/* get current flags from priv_task */
priv_task
=
VRT_priv_task
(
ctx
,
priv_task_id_cfg
);
if
(
priv_task
==
NULL
)
{
VRT_fail
(
ctx
,
"no priv_task"
);
return
;
}
assert
(
sizeof
priv_task
->
priv
>=
sizeof
(
unsigned
));
if
(
priv_task
->
len
==
0
)
{
vclflags
=
PF_CFG_DEFAULT
;
priv_task
->
len
=
1
;
}
else
{
vclflags
=
(
unsigned
)(
uintptr_t
)
priv_task
->
priv
;
assert
(
priv_task
->
len
==
1
);
}
/* set by args */
f
=
vmod_set_param_flag
(
args
->
parameter
);
vclflags
&=
~
f
;
if
(
args
->
bool
)
vclflags
|=
f
;
AZ
(
vclflags
&
~
PF_MASK_CFG
);
priv_task
->
priv
=
(
void
*
)(
uintptr_t
)
vclflags
;
}
/* Event function */
/*
* MPL embeds its own struct memitem into the allocation and blindly reduces the
* requested size by the memitem size.
*
* Yet it fails to expose the struct memitem size such that any client to that
* API requiring a fixed size cannot directly determine the required allocation
* size
*
* So we request a memitem and determine the struct memitem size indirectly
*
* as long as sizeof(struct node) + sizeof(struct memitem) > MEMITEM_GUESS
* + sizeof (struct memitem), the too small memitem will be freed again and all
* memitems will be exactly the right size. Otherwise the too large memitem will
* eventually die for timeout, but the overhead does not matter anyway
*
* (gdb) print sizeof(struct memitem)
* $1 = 32
*/
#define MEMITEM_GUESS 32
static
struct
mempool
*
mpl_init
()
{
struct
mempool
*
mpl
;
unsigned
sz
;
void
*
test
;
node_alloc_sz
=
MEMITEM_GUESS
*
2
;
mpl
=
MPL_New
(
"pesi"
,
&
poolparam
,
&
node_alloc_sz
);
test
=
MPL_Get
(
mpl
,
&
sz
);
AN
(
test
);
assert
(
sz
<=
node_alloc_sz
);
// no unsigned underflow
node_alloc_sz
+=
sizeof
(
struct
node
);
node_alloc_sz
-=
sz
;
MPL_Free
(
mpl
,
test
);
return
(
mpl
);
}
/*
* ... and another shortcoming of MPL: mpl uses a guard thread, yet
* MPL_Destroy() does not wait for it to finish. So after MPL_Destroy() has
* returned the struct mempool will still be accessed by the guard thread.
*
* we got no other option than to wait for that thread to hopefully have
* finished.
*
* sigh...
*/
static
void
mpl_fini
(
struct
mempool
**
mplp
)
{
MPL_Destroy
(
mplp
);
VTIM_sleep
(
0
.
814
*
2
);
/* max mpl_slp in MPL code * 2 */
}
int
v_matchproto_
(
vmod_event_f
)
vmod_event
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
enum
vcl_event_e
e
)
{
ASSERT_CLI
();
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
AN
(
priv
);
switch
(
e
)
{
case
VCL_EVENT_LOAD
:
if
(
loadcnt
++
==
0
)
{
AZ
(
vsc_seg
);
lck_bytes_tree
=
Lck_CreateClass
(
&
vsc_seg
,
"pesi.bytes_tree"
);
lck_pesi_tree
=
Lck_CreateClass
(
&
vsc_seg
,
"pesi.pesi_tree"
);
lck_stats
=
Lck_CreateClass
(
&
vsc_seg
,
"pesi.stats"
);
AN
(
lck_bytes_tree
);
AN
(
lck_pesi_tree
);
AZ
(
pesi_vsc_seg
);
stats
=
VSC_pesi_New
(
NULL
,
&
pesi_vsc_seg
,
""
);
AN
(
stats
);
AN
(
pesi_vsc_seg
);
Lck_New
(
&
stats_lock
,
lck_stats
);
}
VRT_AddVDP
(
ctx
,
&
VDP_pesi
);
break
;
case
VCL_EVENT_DISCARD
:
VRT_RemoveVDP
(
ctx
,
&
VDP_pesi
);
AN
(
loadcnt
);
if
(
--
loadcnt
==
0
)
{
Lck_Delete
(
&
stats_lock
);
Lck_DestroyClass
(
&
vsc_seg
);
VSC_pesi_Destroy
(
&
pesi_vsc_seg
);
}
break
;
case
VCL_EVENT_WARM
:
if
(
warmcnt
++
==
0
)
{
AZ
(
mempool
);
mempool
=
mpl_init
();
AN
(
mempool
);
VRT_VSC_Reveal
(
pesi_vsc_seg
);
}
break
;
case
VCL_EVENT_COLD
:
AN
(
warmcnt
);
if
(
--
warmcnt
==
0
)
{
AN
(
mempool
);
mpl_fini
(
&
mempool
);
AZ
(
mempool
);
VRT_VSC_Hide
(
pesi_vsc_seg
);
}
break
;
default:
WRONG
(
"Illegal event enum"
);
}
return
(
0
);
}
/* Functions */
VCL_STRING
vmod_version
(
VRT_CTX
)
{
(
void
)
ctx
;
return
VERSION
;
}
src/vdp_pesi.h
0 → 100644
View file @
6524e3bb
/*-
* Copyright 2019 UPLEX Nils Goroll Systemoptimierung
* All rights reserved
*
* Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
* Nils Goroll <nils.goroll@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* interfaces shared between vdp_pesi.c and vmod_pesi.c
*/
#define VFAIL(ctx, fmt, ...) \
VRT_fail((ctx), "vdp pesi failure: " fmt, __VA_ARGS__)
extern
struct
lock
stats_lock
;
extern
struct
VSC_pesi
*
stats
;
extern
struct
VSC_lck
*
lck_bytes_tree
,
*
lck_pesi_tree
;
extern
const
struct
vdp
VDP_pesi
;
extern
struct
mempool
*
mempool
;
size_t
node_size
();
/* ------------------------------------------------------------
* task config
*/
#define PF_HAS_TASK 1U
/* vcl-controlled flags */
#define PF_CFG_SERIAL (1U<<1)
#define PF_CFG_THREAD (1U<<2)
/* undocumented for now */
#define PF_CFG_BLOCK_FINAL (1U<<3)
#define PF_CFG_FRONT_PUSH (1U<<4)
#define PF_CFG_DEFAULT \
( PF_CFG_THREAD \
| (block_final ? PF_CFG_BLOCK_FINAL : 0) \
| (front_push ? PF_CFG_FRONT_PUSH : 0) \
)
#define PF_MASK_CFG \
( PF_CFG_SERIAL \
| PF_CFG_THREAD \
| PF_CFG_BLOCK_FINAL \
| PF_CFG_FRONT_PUSH \
)
void
get_task_cfg
(
struct
req
*
,
unsigned
*
);
extern
int
block_final
,
front_push
;
// XXX
src/vmod_pesi.c
0 → 100644
View file @
6524e3bb
/*-
* Copyright 2019 UPLEX Nils Goroll Systemoptimierung
* All rights reserved
*
* Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
* Nils Goroll <nils.goroll@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include "cache/cache_varnishd.h"
//#include <cache/cache.h>
#include <vcl.h>
#include "vtim.h"
#include "vcc_if.h"
#include "VSC_pesi.h"
#include "vdp_pesi.h"
#include "cache/cache_filter.h"
/* VMOD shared object globals */
static
unsigned
loadcnt
=
0
,
warmcnt
=
0
;
static
struct
vsc_seg
*
vsc_seg
=
NULL
,
*
pesi_vsc_seg
=
NULL
;
static
struct
VSC_lck
*
lck_stats
;
/* mempool */
static
unsigned
node_alloc_sz
;
/* id for PRIV_TASK */
const
void
*
const
priv_task_id_cfg
=
&
priv_task_id_cfg
;
/* ------------------------------------------------------------
* mempool
*/
volatile
struct
poolparam
poolparam
=
{
.
min_pool
=
10
,
.
max_pool
=
100
,
.
max_age
=
10
,
};
/*
* MPL embeds its own struct memitem into the allocation and blindly reduces the
* requested size by the memitem size.
*
* Yet it fails to expose the struct memitem size such that any client to that
* API requiring a fixed size cannot directly determine the required allocation
* size
*
* So we request a memitem and determine the struct memitem size indirectly
*
* as long as sizeof(struct node) + sizeof(struct memitem) > MEMITEM_GUESS
* + sizeof (struct memitem), the too small memitem will be freed again and all
* memitems will be exactly the right size. Otherwise the too large memitem will
* eventually die for timeout, but the overhead does not matter anyway
*
* (gdb) print sizeof(struct memitem)
* $1 = 32
*/
#define MEMITEM_GUESS 32
static
struct
mempool
*
mpl_init
()
{
struct
mempool
*
mpl
;
unsigned
sz
;
void
*
test
;
node_alloc_sz
=
MEMITEM_GUESS
*
2
;
mpl
=
MPL_New
(
"pesi"
,
&
poolparam
,
&
node_alloc_sz
);
test
=
MPL_Get
(
mpl
,
&
sz
);
AN
(
test
);
assert
(
sz
<=
node_alloc_sz
);
// no unsigned underflow
node_alloc_sz
+=
node_size
();
node_alloc_sz
-=
sz
;
MPL_Free
(
mpl
,
test
);
return
(
mpl
);
}
/*
* ... and another shortcoming of MPL: mpl uses a guard thread, yet
* MPL_Destroy() does not wait for it to finish. So after MPL_Destroy() has
* returned the struct mempool will still be accessed by the guard thread.
*
* we got no other option than to wait for that thread to hopefully have
* finished.
*
* sigh...
*/
static
void
mpl_fini
(
struct
mempool
**
mplp
)
{
MPL_Destroy
(
mplp
);
VTIM_sleep
(
0
.
814
*
2
);
/* max mpl_slp in MPL code * 2 */
}
/* ------------------------------------------------------------
* VMOD interface
*/
VCL_VOID
vmod_pool
(
VRT_CTX
,
VCL_INT
min
,
VCL_INT
max
,
VCL_DURATION
max_age
)
{
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
/* cf. tweak_poolparam() in mgt_param_tweak.c */
if
(
min
<=
0
)
{
VFAIL
(
ctx
,
"min (%jd) must be > 0 in pool()"
,
min
);
return
;
}
if
(
max
<=
0
)
{
VFAIL
(
ctx
,
"max (%jd) must be > 0 in pool()"
,
max
);
return
;
}
if
(
max
<
min
)
{
VFAIL
(
ctx
,
"max (%jd) < min (%jd) in pool()"
,
max
,
min
);
return
;
}
if
(
max_age
<
0
.)
{
VFAIL
(
ctx
,
"max_age (%.0fs) < 0s in pool()"
,
max_age
);
return
;
}
if
(
max_age
>
1e6
)
{
VFAIL
(
ctx
,
"max_age (%.0fs) out of range in pool() (max 10^6s)"
,
max_age
);
return
;
}
poolparam
.
min_pool
=
min
;
poolparam
.
max_pool
=
max
;
poolparam
.
max_age
=
max_age
;
}
/*
* pesi_filter_on_ws() and pesi_resp_default_filter_list()
* taken from cache_vrt_filter.c
*
*/
typedef
void
pesi_filter_list_t
(
void
*
,
struct
vsb
*
vsb
);
static
const
char
*
pesi_filter_on_ws
(
struct
ws
*
ws
,
pesi_filter_list_t
*
func
,
void
*
arg
)
{
unsigned
u
;
struct
vsb
vsb
[
1
];
AN
(
func
);
AN
(
arg
);
u
=
WS_ReserveAll
(
ws
);
if
(
u
==
0
)
{
WS_Release
(
ws
,
0
);
WS_MarkOverflow
(
ws
);
return
(
NULL
);
}
AN
(
VSB_new
(
vsb
,
ws
->
f
,
u
,
VSB_FIXEDLEN
));
func
(
arg
,
vsb
);
if
(
VSB_finish
(
vsb
))
{
WS_Release
(
ws
,
0
);
WS_MarkOverflow
(
ws
);
return
(
NULL
);
}
if
(
VSB_len
(
vsb
))
{
WS_Release
(
ws
,
VSB_len
(
vsb
)
+
1
);
return
(
VSB_data
(
vsb
)
+
1
);
}
WS_Release
(
ws
,
0
);
return
(
""
);
}
static
void
v_matchproto_
(
pesi_filter_list_t
)
pesi_resp_default_filter_list
(
void
*
arg
,
struct
vsb
*
vsb
)
{
struct
req
*
req
;
CAST_OBJ_NOTNULL
(
req
,
arg
,
REQ_MAGIC
);
/*
* the req->resp_len check has been removed because it does not work for
* busy objects
*
* pesi will still do the right thing if the response really is empty
*/
if
(
!
req
->
disable_esi
&&
/* req->resp_len != 0 && */
ObjHasAttr
(
req
->
wrk
,
req
->
objcore
,
OA_ESIDATA
))
VSB_cat
(
vsb
,
" pesi"
);
if
(
cache_param
->
http_gzip_support
&&
ObjCheckFlag
(
req
->
wrk
,
req
->
objcore
,
OF_GZIPED
)
&&
!
RFC2616_Req_Gzip
(
req
->
http
))
VSB_cat
(
vsb
,
" gunzip"
);
if
(
cache_param
->
http_range_support
&&
http_GetStatus
(
req
->
resp
)
==
200
&&
http_GetHdr
(
req
->
http
,
H_Range
,
NULL
))
VSB_cat
(
vsb
,
" range"
);
}
VCL_VOID
vmod_activate
(
VRT_CTX
)
{
struct
req
*
req
;
const
char
*
filters
;
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
if
(
ctx
->
method
!=
VCL_MET_DELIVER
)
{
VRT_fail
(
ctx
,
"pesi.activate() may only be called "
"from vcl_deliver{}"
);
return
;
}
req
=
ctx
->
req
;
CHECK_OBJ_NOTNULL
(
req
,
REQ_MAGIC
);
filters
=
pesi_filter_on_ws
(
req
->
ws
,
pesi_resp_default_filter_list
,
req
);
if
(
filters
==
NULL
)
WS_MarkOverflow
(
req
->
ws
);
else
req
->
filter_list
=
filters
;
}
static
unsigned
vmod_set_param_flag
(
VCL_ENUM
e
)
{
#define VMODENUM(p,f) if (e == VENUM(p)) return(f);
#include "tbl_set_parameter.h"
WRONG
(
"illegal enum"
);
}
/* VDP's access to the cfg */
void
get_task_cfg
(
struct
req
*
req
,
unsigned
*
flags
)
{
struct
vmod_priv
*
priv_task
;
struct
vrt_ctx
dummy_ctx
[
1
];
unsigned
vclflags
;
INIT_OBJ
(
dummy_ctx
,
VRT_CTX_MAGIC
);
dummy_ctx
->
req
=
req
;
dummy_ctx
->
ws
=
req
->
ws
;
/* zero length in the priv task == not configured from vcl */
priv_task
=
VRT_priv_task
(
dummy_ctx
,
priv_task_id_cfg
);
if
(
priv_task
==
NULL
||
priv_task
->
len
==
0
)
return
;
assert
(
priv_task
->
len
==
1
);
AZ
(
priv_task
->
free
);
vclflags
=
(
unsigned
)(
uintptr_t
)
priv_task
->
priv
;
AZ
(
vclflags
&
~
PF_MASK_CFG
);
*
flags
=
(
*
flags
&
~
PF_MASK_CFG
)
|
vclflags
;
}
VCL_VOID
vmod_set
(
VRT_CTX
,
struct
VARGS
(
set
)
*
args
)
{
struct
vmod_priv
*
priv_task
;
unsigned
f
,
vclflags
;
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
if
(
ctx
->
method
!=
VCL_MET_DELIVER
)
{
VRT_fail
(
ctx
,
"pesi.set() may only be called "
"from vcl_deliver{}"
);
return
;
}
/* as of now, all parameters require a bool parameter */
if
(
args
->
valid_bool
==
0
)
{
VRT_fail
(
ctx
,
"pesi.set(%s) requires a bool "
"parameter"
,
args
->
parameter
);
return
;
}
/* get current flags from priv_task */
priv_task
=
VRT_priv_task
(
ctx
,
priv_task_id_cfg
);
if
(
priv_task
==
NULL
)
{
VRT_fail
(
ctx
,
"no priv_task"
);
return
;
}
assert
(
sizeof
priv_task
->
priv
>=
sizeof
(
unsigned
));
if
(
priv_task
->
len
==
0
)
{
vclflags
=
PF_CFG_DEFAULT
;
priv_task
->
len
=
1
;
}
else
{
vclflags
=
(
unsigned
)(
uintptr_t
)
priv_task
->
priv
;
assert
(
priv_task
->
len
==
1
);
}
/* set by args */
f
=
vmod_set_param_flag
(
args
->
parameter
);
vclflags
&=
~
f
;
if
(
args
->
bool
)
vclflags
|=
f
;
AZ
(
vclflags
&
~
PF_MASK_CFG
);
priv_task
->
priv
=
(
void
*
)(
uintptr_t
)
vclflags
;
}
/* Functions */
VCL_STRING
vmod_version
(
VRT_CTX
)
{
(
void
)
ctx
;
return
VERSION
;
}
int
v_matchproto_
(
vmod_event_f
)
vmod_event
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
enum
vcl_event_e
e
)
{
ASSERT_CLI
();
CHECK_OBJ_NOTNULL
(
ctx
,
VRT_CTX_MAGIC
);
AN
(
priv
);
switch
(
e
)
{
case
VCL_EVENT_LOAD
:
if
(
loadcnt
++
==
0
)
{
AZ
(
vsc_seg
);
lck_bytes_tree
=
Lck_CreateClass
(
&
vsc_seg
,
"pesi.bytes_tree"
);
lck_pesi_tree
=
Lck_CreateClass
(
&
vsc_seg
,
"pesi.pesi_tree"
);
lck_stats
=
Lck_CreateClass
(
&
vsc_seg
,
"pesi.stats"
);
AN
(
lck_bytes_tree
);
AN
(
lck_pesi_tree
);
AZ
(
pesi_vsc_seg
);
stats
=
VSC_pesi_New
(
NULL
,
&
pesi_vsc_seg
,
""
);
AN
(
stats
);
AN
(
pesi_vsc_seg
);
Lck_New
(
&
stats_lock
,
lck_stats
);
}
VRT_AddVDP
(
ctx
,
&
VDP_pesi
);
break
;
case
VCL_EVENT_DISCARD
:
VRT_RemoveVDP
(
ctx
,
&
VDP_pesi
);
AN
(
loadcnt
);
if
(
--
loadcnt
==
0
)
{
Lck_Delete
(
&
stats_lock
);
Lck_DestroyClass
(
&
vsc_seg
);
VSC_pesi_Destroy
(
&
pesi_vsc_seg
);
}
break
;
case
VCL_EVENT_WARM
:
if
(
warmcnt
++
==
0
)
{
AZ
(
mempool
);
mempool
=
mpl_init
();
AN
(
mempool
);
VRT_VSC_Reveal
(
pesi_vsc_seg
);
}
break
;
case
VCL_EVENT_COLD
:
AN
(
warmcnt
);
if
(
--
warmcnt
==
0
)
{
AN
(
mempool
);
mpl_fini
(
&
mempool
);
AZ
(
mempool
);
VRT_VSC_Hide
(
pesi_vsc_seg
);
}
break
;
default:
WRONG
(
"Illegal event enum"
);
}
return
(
0
);
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment