vmod / vdp segregation

much remains to be done

src/Makefile.am

@@ -8,6 +8,7 @@ AM_LDFLAGS = $(VARNISHAPI_LIBS) $(VMOD_LDFLAGS) -ldl
 vmod_LTLIBRARIES = libvmod_pesi.la
 
 libvmod_pesi_la_SOURCES = \
+	vmod_pesi.c \
 	vdp_pesi.c \
 	tbl_set_parameter.h \
 	foreign/qdef.h \
@@ -24,7 +25,7 @@ dist_man_MANS = vdp_pesi.3
 
 @BUILD_VSC_PESI@
 
-vdp_pesi.lo: vcc_if.h VSC_pesi.c VSC_pesi.h
+vmod_pesi.c: vcc_if.h VSC_pesi.c VSC_pesi.h
 
 vcc_if.h vmod_pesi.rst vmod_pesi.man.rst: vcc_if.c
 

src/node.h

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * node interface
+ *
+ * XXX mempool vs. struct node
+ */
+
+enum n_type {
+	T_INVALID = 0,
+	T_NEXUS,	// can change into T_SUBREQ / T_FINAL / T_DATA
+	T_DATA,
+	T_CRC,
+	T_SUBREQ,
+	T_FINAL	// non-ESI pass / hfm / hfp
+};
+
+/*
+ * see state.dot:
+ *
+ * ST_DATA:    may never have any children
+ *
+ *	       T_DATA / T_CRC / T_SUBREQ / T_FINAL
+ *
+ * ST_PRIVATE: may receive pushes creating children
+ *             unpending must not yet touch it
+ *             children can be created unlocked
+ *             can change into other types
+ *
+ *             T_NEXUS only
+ *
+ * ST_OPEN:    may receive pushes creating children
+ *             owning thread may run front delivery
+ *             any changes below locked only
+ *
+ *             T_NEXUS only
+ *
+ *
+ * ST_CLOSED:  the request is done, no pushes can occur,
+ *             unpending can proceed upwards
+ *
+ *             T_NEXUS only
+ *
+ * ST_UNPENDING: in the process of being pushed to the client
+ *
+ *	         T_DATA / T_CRC / T_SUBREQ / T_FINAL
+ *
+ * ST_DELIVERED: We have pushed data up
+ *
+ *               any type
+ *
+ *		 for T_NEXUS, means "anything below is delivered"
+ */
+enum n_state {
+	ST_INVALID = 0,
+	ST_DATA,
+	ST_PRIVATE,
+	ST_OPEN,
+	ST_CLOSED,
+	ST_UNPENDING,
+	ST_DELIVERED,
+};
+
+VSTAILQ_HEAD(node_head, node);
+
+struct node_nexus {
+	struct node_head	children;
+	struct objcore		*oc;
+	struct req		*req;
+	const struct worker	*owner; // ST_OPEN only
+
+	/* number of nodes pending under this node while state == ST_PRIVATE */
+	int			npending_private;
+	/*
+	 * crc for data nodes immediately below
+	 * updated by push_crc, pretendgzip and gzgz
+	 */
+	struct nexus_gzip	gzip;	// in foreign
+};
+
+enum t_crc {
+	INVALID = 0,
+	UPDATE,
+	FINAL	// combine with parent or gen tail
+};
+
+struct node_crc {
+	enum t_crc	ctype;
+	uint32_t	icrc;
+	ssize_t		l_icrc;
+};
+
+struct node_data {
+	const void		*ptr;
+	struct storage		*st;
+	ssize_t			len;
+	enum vdp_action		act;
+};
+
+/*
+ * we transfer the sub-request, boc and oc to the topreq thread, delivery
+ * happens there
+ */
+struct node_subreq {
+	struct req		*req;
+	struct boc		*boc;
+	// oc is NULL if already transferred back into req
+	struct objcore		*oc;
+	// subreq to topreq delivery
+	int			done;
+	pthread_cond_t		cond;
+};
+
+// sub-state for node_final while in ST_DATA
+enum fi_state {
+	FI_READY	= 0,
+	FI_GO,			// topreq signalling req to deliver
+	FI_DONE,		// req signalling topreq it is done
+	FI_DESTROYED		// cond/mtx destroyed (fini_final())
+};
+
+/* we block the sub-thread when it's ready for delivery and continue when the
+ * topreqp tells it to */
+struct node_final {
+	enum fi_state		fi_state;
+	pthread_mutex_t	fi_mtx;
+	pthread_cond_t		fi_cond;
+};
+
+struct node {
+	unsigned		magic;
+#define NODE_MAGIC 0xe31edef3
+	enum n_type		type;
+	enum n_state		state;
+	VSTAILQ_ENTRY(node)	sibling;
+	VSTAILQ_ENTRY(node)	unpend;
+	struct node		*parent;
+
+	union {
+		struct node_nexus	nexus;	// T_NEXUS
+		struct node_data	data;	// T_DATA
+		struct node_subreq	subreq; // T_SUBREQ
+		struct node_final	final;	// T_FINAL
+		struct node_crc	crc;	// T_CRC
+	};
+};
+

src/vdp_pesi.h

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * interfaces shared between vdp_pesi.c and vmod_pesi.c
+ */
+
+#define VFAIL(ctx, fmt, ...)					\
+	VRT_fail((ctx), "vdp pesi failure: " fmt, __VA_ARGS__)
+
+extern struct lock stats_lock;
+extern struct VSC_pesi *stats;
+extern struct VSC_lck *lck_bytes_tree, *lck_pesi_tree;
+extern const struct vdp VDP_pesi;
+extern struct mempool *mempool;
+size_t node_size();
+
+
+/* ------------------------------------------------------------
+ * task config
+ */
+#define PF_HAS_TASK		1U
+/* vcl-controlled flags */
+#define PF_CFG_SERIAL		(1U<<1)
+#define PF_CFG_THREAD		(1U<<2)
+/* undocumented for now */
+#define PF_CFG_BLOCK_FINAL	(1U<<3)
+#define PF_CFG_FRONT_PUSH	(1U<<4)
+
+#define PF_CFG_DEFAULT					\
+	( PF_CFG_THREAD				\
+	| (block_final ? PF_CFG_BLOCK_FINAL : 0)	\
+	| (front_push ? PF_CFG_FRONT_PUSH : 0)		\
+	)
+
+#define PF_MASK_CFG		\
+	( PF_CFG_SERIAL	\
+	| PF_CFG_THREAD	\
+	| PF_CFG_BLOCK_FINAL	\
+	| PF_CFG_FRONT_PUSH	\
+	)
+
+void get_task_cfg(struct req *, unsigned *);
+
+extern int block_final, front_push; // XXX

src/vmod_pesi.c

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include "cache/cache_varnishd.h"
+//#include <cache/cache.h>
+#include <vcl.h>
+#include "vtim.h"
+
+#include "vcc_if.h"
+#include "VSC_pesi.h"
+
+#include "vdp_pesi.h"
+
+#include "cache/cache_filter.h"
+
+/* VMOD shared object globals */
+static unsigned loadcnt = 0, warmcnt = 0;
+static struct vsc_seg  *vsc_seg = NULL, *pesi_vsc_seg = NULL;
+static struct VSC_lck *lck_stats;
+
+/* mempool */
+static unsigned node_alloc_sz;
+
+/* id for PRIV_TASK */
+const void * const priv_task_id_cfg = &priv_task_id_cfg;
+
+
+/* ------------------------------------------------------------
+ * mempool
+ */
+
+volatile struct poolparam poolparam = {
+	.min_pool = 10,
+	.max_pool = 100,
+	.max_age = 10,
+};
+
+/*
+ * MPL embeds its own struct memitem into the allocation and blindly reduces the
+ * requested size by the memitem size.
+ *
+ * Yet it fails to expose the struct memitem size such that any client to that
+ * API requiring a fixed size cannot directly determine the required allocation
+ * size
+ *
+ * So we request a memitem and determine the struct memitem size indirectly
+ *
+ * as long as sizeof(struct node) + sizeof(struct memitem) > MEMITEM_GUESS
+ * + sizeof (struct memitem), the too small memitem will be freed again and all
+ * memitems will be exactly the right size. Otherwise the too large memitem will
+ * eventually die for timeout, but the overhead does not matter anyway
+ *
+ * (gdb) print sizeof(struct memitem)
+ * $1 = 32
+ */
+#define MEMITEM_GUESS 32
+
+static struct mempool *
+mpl_init()
+{
+	struct mempool *mpl;
+	unsigned sz;
+	void *test;
+
+	node_alloc_sz = MEMITEM_GUESS * 2;
+	mpl = MPL_New("pesi", &poolparam, &node_alloc_sz);
+
+	test = MPL_Get(mpl, &sz);
+	AN(test);
+	assert(sz <= node_alloc_sz);	// no unsigned underflow
+
+	node_alloc_sz += node_size();
+	node_alloc_sz -= sz;
+	MPL_Free(mpl, test);
+
+	return (mpl);
+}
+
+/*
+ * ... and another shortcoming of MPL: mpl uses a guard thread, yet
+ * MPL_Destroy() does not wait for it to finish. So after MPL_Destroy() has
+ * returned the struct mempool will still be accessed by the guard thread.
+ *
+ * we got no other option than to wait for that thread to hopefully have
+ * finished.
+ *
+ * sigh...
+ */
+
+static void
+mpl_fini(struct mempool **mplp)
+{
+	MPL_Destroy(mplp);
+	VTIM_sleep(0.814 * 2); /* max mpl_slp in MPL code * 2 */
+}
+
+/* ------------------------------------------------------------
+ * VMOD interface
+ */
+VCL_VOID
+vmod_pool(VRT_CTX, VCL_INT min, VCL_INT max, VCL_DURATION max_age)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	/* cf. tweak_poolparam() in mgt_param_tweak.c */
+	if (min <= 0) {
+		VFAIL(ctx, "min (%jd) must be > 0 in pool()", min);
+		return;
+	}
+	if (max <= 0) {
+		VFAIL(ctx, "max (%jd) must be > 0 in pool()", max);
+		return;
+	}
+	if (max < min) {
+		VFAIL(ctx, "max (%jd) < min (%jd) in pool()", max, min);
+		return;
+	}
+	if (max_age < 0.) {
+		VFAIL(ctx, "max_age (%.0fs) < 0s in pool()", max_age);
+		return;
+	}
+	if (max_age > 1e6) {
+		VFAIL(ctx, "max_age (%.0fs) out of range in pool() (max 10^6s)",
+		      max_age);
+		return;
+	}
+
+	poolparam.min_pool = min;
+	poolparam.max_pool = max;
+	poolparam.max_age = max_age;
+}
+
+/*
+ * pesi_filter_on_ws() and pesi_resp_default_filter_list()
+ * taken from cache_vrt_filter.c
+ *
+ */
+
+typedef void pesi_filter_list_t(void *, struct vsb *vsb);
+
+static const char *
+pesi_filter_on_ws(struct ws *ws, pesi_filter_list_t *func, void *arg)
+{
+	unsigned u;
+	struct vsb vsb[1];
+
+	AN(func);
+	AN(arg);
+	u = WS_ReserveAll(ws);
+	if (u == 0) {
+		WS_Release(ws, 0);
+		WS_MarkOverflow(ws);
+		return (NULL);
+	}
+	AN(VSB_new(vsb, ws->f, u, VSB_FIXEDLEN));
+	func(arg, vsb);
+	if (VSB_finish(vsb)) {
+		WS_Release(ws, 0);
+		WS_MarkOverflow(ws);
+		return (NULL);
+	}
+	if (VSB_len(vsb)) {
+		WS_Release(ws, VSB_len(vsb) + 1);
+		return (VSB_data(vsb) + 1);
+	}
+	WS_Release(ws, 0);
+	return ("");
+}
+
+static void v_matchproto_(pesi_filter_list_t)
+pesi_resp_default_filter_list(void *arg, struct vsb *vsb)
+{
+	struct req *req;
+
+	CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC);
+
+	/*
+	 * the req->resp_len check has been removed because it does not work for
+	 * busy objects
+	 *
+	 * pesi will still do the right thing if the response really is empty
+	 */
+
+	if (!req->disable_esi && /* req->resp_len != 0 && */
+	    ObjHasAttr(req->wrk, req->objcore, OA_ESIDATA))
+		VSB_cat(vsb, " pesi");
+
+	if (cache_param->http_gzip_support &&
+	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
+	    !RFC2616_Req_Gzip(req->http))
+		VSB_cat(vsb, " gunzip");
+
+	if (cache_param->http_range_support &&
+	    http_GetStatus(req->resp) == 200 &&
+	    http_GetHdr(req->http, H_Range, NULL))
+		VSB_cat(vsb, " range");
+}
+
+VCL_VOID
+vmod_activate(VRT_CTX)
+{
+	struct req *req;
+	const char *filters;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (ctx->method != VCL_MET_DELIVER) {
+		VRT_fail(ctx, "pesi.activate() may only be called "
+		    "from vcl_deliver{}");
+		return;
+	}
+
+	req = ctx->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	filters = pesi_filter_on_ws(req->ws,
+	    pesi_resp_default_filter_list, req);
+
+	if (filters == NULL)
+		WS_MarkOverflow(req->ws);
+	else
+		req->filter_list = filters;
+}
+
+static unsigned
+vmod_set_param_flag(VCL_ENUM e)
+{
+#define VMODENUM(p,f) if (e == VENUM(p)) return(f);
+#include "tbl_set_parameter.h"
+       WRONG("illegal enum");
+}
+
+/* VDP's access to the cfg */
+void
+get_task_cfg(struct req *req, unsigned *flags)
+{
+	struct vmod_priv *priv_task;
+	struct vrt_ctx dummy_ctx[1];
+	unsigned vclflags;
+
+	INIT_OBJ(dummy_ctx, VRT_CTX_MAGIC);
+	dummy_ctx->req = req;
+	dummy_ctx->ws = req->ws;
+
+	/* zero length in the priv task == not configured from vcl */
+	priv_task = VRT_priv_task(dummy_ctx, priv_task_id_cfg);
+	if (priv_task == NULL || priv_task->len == 0)
+		return;
+
+	assert(priv_task->len == 1);
+	AZ(priv_task->free);
+	vclflags = (unsigned)(uintptr_t)priv_task->priv;
+	AZ(vclflags & ~PF_MASK_CFG);
+
+	*flags = (*flags & ~PF_MASK_CFG) | vclflags;
+}
+
+VCL_VOID
+vmod_set(VRT_CTX, struct VARGS(set) *args)
+{
+	struct vmod_priv *priv_task;
+	unsigned f, vclflags;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (ctx->method != VCL_MET_DELIVER) {
+		VRT_fail(ctx, "pesi.set() may only be called "
+		    "from vcl_deliver{}");
+		return;
+	}
+
+	/* as of now, all parameters require a bool parameter */
+	if (args->valid_bool == 0) {
+		VRT_fail(ctx, "pesi.set(%s) requires a bool "
+		    "parameter", args->parameter);
+		return;
+	}
+
+	/* get current flags from priv_task */
+
+	priv_task = VRT_priv_task(ctx, priv_task_id_cfg);
+
+	if (priv_task == NULL) {
+		VRT_fail(ctx, "no priv_task");
+		return;
+	}
+
+	assert(sizeof priv_task->priv >= sizeof(unsigned));
+
+	if (priv_task->len == 0) {
+		vclflags = PF_CFG_DEFAULT;
+		priv_task->len = 1;
+	}
+	else {
+		vclflags = (unsigned)(uintptr_t)priv_task->priv;
+		assert(priv_task->len == 1);
+	}
+
+	/* set by args */
+
+	f = vmod_set_param_flag(args->parameter);
+
+	vclflags &= ~f;
+	if (args->bool)
+		vclflags |= f;
+
+	AZ(vclflags & ~PF_MASK_CFG);
+
+	priv_task->priv = (void *)(uintptr_t)vclflags;
+}
+
+/* Functions */
+
+VCL_STRING
+vmod_version(VRT_CTX)
+{
+	(void) ctx;
+	return VERSION;
+}
+
+int v_matchproto_(vmod_event_f)
+vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
+{
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(priv);
+
+	switch (e) {
+	case VCL_EVENT_LOAD:
+		if (loadcnt++ == 0) {
+			AZ(vsc_seg);
+			lck_bytes_tree = Lck_CreateClass(&vsc_seg,
+			    "pesi.bytes_tree");
+			lck_pesi_tree = Lck_CreateClass(&vsc_seg,
+			    "pesi.pesi_tree");
+			lck_stats = Lck_CreateClass(&vsc_seg,
+			    "pesi.stats");
+			AN(lck_bytes_tree);
+			AN(lck_pesi_tree);
+			AZ(pesi_vsc_seg);
+			stats = VSC_pesi_New(NULL, &pesi_vsc_seg, "");
+			AN(stats);
+			AN(pesi_vsc_seg);
+			Lck_New(&stats_lock, lck_stats);
+		}
+
+		VRT_AddVDP(ctx, &VDP_pesi);
+		break;
+	case VCL_EVENT_DISCARD:
+		VRT_RemoveVDP(ctx, &VDP_pesi);
+
+		AN(loadcnt);
+		if (--loadcnt == 0) {
+			Lck_Delete(&stats_lock);
+			Lck_DestroyClass(&vsc_seg);
+			VSC_pesi_Destroy(&pesi_vsc_seg);
+		}
+		break;
+	case VCL_EVENT_WARM:
+		if (warmcnt++ == 0) {
+			AZ(mempool);
+			mempool = mpl_init();
+			AN(mempool);
+			VRT_VSC_Reveal(pesi_vsc_seg);
+		}
+		break;
+	case VCL_EVENT_COLD:
+		AN(warmcnt);
+		if (--warmcnt == 0) {
+			AN(mempool);
+			mpl_fini(&mempool);
+			AZ(mempool);
+			VRT_VSC_Hide(pesi_vsc_seg);
+		}
+		break;
+	default:
+		WRONG("Illegal event enum");
+	}
+	return (0);
+}