Initial public release

.dir-locals.el

+((nil . ((indent-tabs-mode . t)))
+ (c-mode . ((c-file-style . "BSD"))))

.gitignore

+Makefile
+Makefile.in
+.deps/
+.libs/
+*.o
+*.lo
+*.la
+*~
+*.[1-9]
+*.gcda
+*.gcno
+.dirstamp
+
+/aclocal.m4
+/ar-lib
+/autom4te.cache/
+/compile
+/config.guess
+/config.h
+/config.h.in
+/config.log
+/config.status
+/config.sub
+/configure
+/depcomp
+/install-sh
+/libtool
+/ltmain.sh
+/missing
+/stamp-h1
+/test-driver
+/m4/
+
+/src/vcc_pesi_debug_if.[ch]
+/src/vcc_pesi_if.[ch]
+/src/vmod_*rst
+/src/VSC_pesi.c
+/src/VSC_pesi.h
+
+/src/tests/*.log
+/src/tests/*.trs
+/src/test-suite.log
+
+/src/coverage

CONTRIBUTING.rst

+CONTRIBUTING
+============
+
+To contribute code or documentation, submit a pull request at the
+`source repository website
+<https://gitlab.com/uplex/varnish/libvdp-pesi>`_.
+
+If you have a problem or discover a bug, you can post an `issue
+<https://gitlab.com/uplex/varnish/libvdp-pesi/issues>`_ at
+the website. You can also write to <varnish-support@uplex.de>.
+
+For developers
+--------------
+
+The VDP depends heavily on function calls that are internal to
+Varnish, and are not part of the public APIs provided for VMODs and
+delivery or fetch processors. This is why the build requires a Varnish
+source tree, identified with the ``VARNISHSRC`` variable in the
+invocation of ``configure``, as described in `INSTALL.rst
+<INSTALL.rst>`_. Some of the function and struct definitions that are
+declared static in the Varnish source are copied in the
+``src/foreign`` directory of the VDP source.
+
+Elements of the Varnish code that are not public may be changed
+without notice in future versions. In that case, the VDP code will
+have to be updated as the need arises.
+
+The VDP source code is in C, and compilation has been tested with gcc
+and clang. The code MUST always compile successfully with both of
+them.
+
+The build specifies C99 conformance for C sources (``-std=c99``). All
+compiler warnings are turned on, and all warnings are considered
+errors (``-Werror -Wall -Wextra``).  The code MUST always build
+without warnings or errors under these constraints.
+
+By default, ``CFLAGS`` is set to ``-g -O2``, so that symbols are
+included in the shared library, and optimization is at level
+``O2``. To change or disable these options, set ``CFLAGS`` explicitly
+before calling ``configure`` (it may be set to the empty string).
+
+For development/debugging cycles, the ``configure`` option
+``--enable-debugging`` is recommended (off by default). This will turn
+off optimizations and function inlining, so that a debugger will step
+through the code as expected.
+
+Experience has shown that adding ``-ggdb3`` to ``CFLAGS`` is
+beneficial if you need to examine the VDP with the gdb debugger. The
+shared object for a VDP (as for VMODs) is loaded from a directory
+relative to the Varnish home directory (by default
+``/usr/local/var/$INSTANCE`` for development builds). A debugger needs
+to locate the shared object from that relative path to load its
+symbols, so the Varnish home directory should be the current working
+directory when the debugger is run. For example::
+
+  # To run gdb and examine a coredump
+  $ cd /usr/local/var/myinstance
+  $ gdb /usr/local/sbin/varnishd /path/to/coredump
+
+By default, the VDP is built with the stack protector enabled
+(compile option ``-fstack-protector``), but it can be disabled with
+the ``configure`` option ``--disable-stack-protector``.

COPYING

+See LICENSE for details.
+

INSTALL.rst

+INSTALLATION
+============
+
+RPMs
+~~~~
+
+Binary, debuginfo and source RPMs for the VDP are available at:
+
+	https://pkg.uplex.de/
+
+The packages are built for Enterprise Linux 7 (el7), and hence will
+run on compatible distros (such as RHEL7, Fedora, CentOS 7 and Amazon
+Linux).
+
+To add the repo to your YUM configuration::
+
+  yum-config-manager --add-repo https://pkg.uplex.de/rpm/7/uplex-varnish/x86_64/
+
+The RPM for the VDP requires a Varnish installation from the official
+packages:
+
+	https://packagecloud.io/varnishcache
+
+You can then install the VDP with::
+
+  yum install --nogpgcheck vdp-pesi
+
+If you have problems or questions concerning the RPMs, post an issue
+to the source repository web site for the VDP, or contact
+<varnish-support@uplex.de>.
+
+Building from source
+~~~~~~~~~~~~~~~~~~~~
+
+The VDP is built against both a Varnish development installation *and*
+a Varnish source tree. The Varnish source tree must be built, so that
+generated sources are present; that is, its build process must be
+carried out at least as far the execution of ``make``.  The build
+version of installed Varnish must match the version of the source
+tree; this means that the "commit ID" portion of the version string of
+installed Varnish must match the HEAD of the source tree.  A simple
+solution is to build and install Varnish from the source tree.  If
+Varnish is installed by other means (for example from package), then
+the commit IDs must match.
+
+For the installation, the autotools use ``pkg-config(1)`` to locate
+the necessary header files and other resources for Varnish. The root
+directory of the source tree is identified by the variable
+``VARNISHSRC`` in the invocation of ``configure``.
+
+This sequence will install the VDP::
+
+  > ./autogen.sh	# for builds from the git repo
+  > ./configure VARNISHSRC=/path/to/source/tree
+  > make
+  > make check		# to run unit tests in src/tests/*.vtc
+  > make distcheck	# run check and prepare a distribution tarball
+  > sudo make install
+
+See `CONTRIBUTING.rst <CONTRIBUTING.rst>`_ for notes about building
+from source.
+
+If you have installed Varnish in non-standard directories, call
+``autogen.sh`` and ``configure`` with the ``PKG_CONFIG_PATH``
+environment variable set to include the paths where the ``.pc`` file
+can be located for ``varnishapi``.  For example, when varnishd
+configure was called with ``--prefix=$PREFIX``, use::
+
+  > PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig
+  > export PKG_CONFIG_PATH
+
+By default, the ``configure`` script installs the VDP in the VMOD
+directory for Varnish, determined via ``pkg-config(1)``. The VMOD
+installation directory can be overridden by passing the ``VMOD_DIR``
+variable to ``configure``.
+
+Other files such as the man-page are installed in the locations
+determined by ``configure``, which inherits its default ``--prefix``
+setting from Varnish.

LICENSE

+Copyright 2019 - 2021 UPLEX Nils Goroll Systemoptimierung
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.

Makefile.am

+ACLOCAL_AMFLAGS = -I m4 -I ${VARNISHAPI_DATAROOTDIR}/aclocal
+
+SUBDIRS = src
+
+DISTCHECK_CONFIGURE_FLAGS = \
+	VMOD_DIR='$${libdir}/varnish/vmods'
+
+EXTRA_DIST = README.rst LICENSE COPYING CONTRIBUTING.rst INSTALL.rst
+
+doc_DATA = README.rst LICENSE COPYING CONTRIBUTING.rst INSTALL.rst
+
+README.rst: src/vdp_pesi.vcc
+	$(MAKE) $(AM_MAKEFLAGS) -C src vmod_pesi.man.rst
+	cp src/vmod_pesi.man.rst README.rst
+
+coverage:
+	$(MAKE) $(AM_MAKEFLAGS) -C src coverage
+
+%.1 %.2 %.3 %.4 %.5 %.6 %.7 %.8 %.9:
+if HAVE_RST2MAN
+	${RST2MAN} $< $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif

autogen.sh

+#!/bin/sh
+
+warn() {
+	echo "WARNING: $@" 1>&2
+}
+
+case `uname -s` in
+Darwin)
+	LIBTOOLIZE=glibtoolize
+	;;
+FreeBSD)
+	LIBTOOLIZE=libtoolize
+	;;
+Linux)
+	LIBTOOLIZE=libtoolize
+	;;
+SunOS)
+	LIBTOOLIZE=libtoolize
+	;;
+*)
+	warn "unrecognized platform:" `uname -s`
+	LIBTOOLIZE=libtoolize
+esac
+
+automake_version=`automake --version | tr ' ' '\n' | egrep '^[0-9]\.[0-9a-z.-]+'`
+if [ -z "$automake_version" ] ; then
+	warn "unable to determine automake version"
+else
+	case $automake_version in
+		0.*|1.[0-8]|1.[0-8][.-]*)
+			warn "automake ($automake_version) detected; 1.9 or newer recommended"
+			;;
+		*)
+			;;
+	esac
+fi
+
+# check for varnishapi.m4 in custom paths
+dataroot=$(pkg-config --variable=datarootdir varnishapi 2>/dev/null)
+if [ -z "$dataroot" ] ; then
+	cat >&2 <<'EOF'
+Package varnishapi was not found in the pkg-config search path.
+Perhaps you should add the directory containing `varnishapi.pc'
+to the PKG_CONFIG_PATH environment variable
+EOF
+	exit 1
+fi
+set -ex
+aclocal -I m4 -I ${dataroot}/aclocal
+$LIBTOOLIZE --copy --force
+autoheader
+automake --add-missing --copy --foreign
+autoconf

configure.ac

+AC_PREREQ(2.68)
+AC_COPYRIGHT([Copyright 2019 - 2021  UPLEX - Nils Goroll Systemoptimierung])
+AC_INIT([libvdp-pesi], [trunk], [varnish-support@uplex.de], [vdp-pesi])
+AC_CONFIG_MACRO_DIR([m4])
+AC_CONFIG_SRCDIR(src/vdp_pesi.vcc)
+AM_CONFIG_HEADER(config.h)
+AC_CANONICAL_SYSTEM
+AC_LANG(C)
+
+AM_INIT_AUTOMAKE([1.12 -Wall -Werror foreign parallel-tests])
+AM_SILENT_RULES([yes])
+AM_PROG_AR
+
+LT_PREREQ([2.2.6])
+LT_INIT([dlopen disable-static])
+
+AC_PROG_CC
+AC_PROG_CC_STDC
+if test "x$ac_cv_prog_cc_c99" = xno; then
+	AC_MSG_ERROR([Could not find a C99 compatible compiler])
+fi
+AC_PROG_CPP
+
+AX_PTHREAD(,[AC_MSG_ERROR([Could not configure pthreads support])])
+
+LIBS="$PTHREAD_LIBS $LIBS"
+CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+CC="$PTHREAD_CC"
+
+AC_ARG_WITH([rst2man],
+	AS_HELP_STRING(
+		[--with-rst2man=PATH],
+		[Location of rst2man (auto)]),
+	[RST2MAN="$withval"],
+	AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], []))
+AM_CONDITIONAL(HAVE_RST2MAN, [test "x$RST2MAN" != "xno"])
+
+AC_ARG_WITH([lcov],
+	AS_HELP_STRING(
+		[--with-lcov=PATH],
+		[Location of lcov to generate coverage data (auto)]),
+	[LCOV="$withval"],
+	AC_CHECK_PROGS(LCOV, [lcov], []))
+AM_CONDITIONAL(HAVE_LCOV, [test -n "$LCOV"])
+
+AC_ARG_WITH([genhtml],
+	AS_HELP_STRING(
+		[--with-genhtml=PATH],
+		[Location of genhtml to generate coverage reports (auto)]),
+	[GENHTML="$withval"],
+	AC_CHECK_PROGS(GENHTML, [genhtml], []))
+AM_CONDITIONAL(HAVE_GENHTML, [test -n "$GENHTML"])
+
+m4_ifndef([VARNISH_PREREQ], AC_MSG_ERROR([Need varnish.m4 -- see README.rst]))
+
+VARNISH_PREREQ([trunk])
+VARNISH_VMODS([pesi pesi_debug])
+VARNISH_COUNTERS([pesi])
+
+VMOD_TESTS="$(cd $srcdir/src && echo tests/*.vtc)"
+AC_SUBST(VMOD_TESTS)
+
+PKG_CHECK_VAR([LIBVARNISHAPI_LIBDIR], [varnishapi], [libdir])
+AC_SUBST([VARNISH_LIBRARY_PATH],
+	[$LIBVARNISHAPI_LIBDIR:$LIBVARNISHAPI_LIBDIR/varnish])
+
+# Checks for C functions
+AC_CHECK_FUNCS([memset])
+AC_CHECK_FUNCS([strchr])
+
+# C headers
+AC_CHECK_HEADERS([limits.h])
+AC_CHECK_HEADERS([stddef.h])
+AC_CHECK_HEADER_STDBOOL
+
+# C types
+AC_TYPE_SIZE_T
+AC_TYPE_SSIZE_T
+AC_TYPE_UINT8_T
+AC_TYPE_UINT16_T
+AC_TYPE_UINT32_T
+AC_TYPE_UINT64_T
+
+AC_C_INLINE
+
+AC_ARG_VAR([VARNISHSRC], [path to Varnish source tree (REQUIRED)])
+if test "${VARNISHSRC+set}" != set; then
+   AC_MSG_ERROR([VARNISHSRC must be set (path to Varnish source tree)])
+fi
+AC_CHECK_FILE(["$VARNISHSRC/include/miniobj.h"],
+	[AC_MSG_RESULT([VARNISHSRC (Varnish source dir)... $VARNISHSRC])],
+	[AC_MSG_ERROR([VARNISHSRC ($VARNISHSRC) is not a Varnish source dir])])
+
+# --enable-stack-protector
+AC_ARG_ENABLE(stack-protector,
+       AS_HELP_STRING([--enable-stack-protector],[enable stack protector (default is YES)]),
+       [],
+       [enable_stack_protector=yes])
+
+if test "x$enable_stack_protector" != "xno"; then
+	AX_CHECK_COMPILE_FLAG([-fstack-protector],
+		AX_CHECK_LINK_FLAG([-fstack-protector],
+			[CFLAGS="${CFLAGS} -fstack-protector"], [], []),
+		[], [])
+fi
+
+# --enable-debugging
+AC_ARG_ENABLE(debugging,
+	AS_HELP_STRING([--enable-debugging],[enable debugging (default is NO)]),
+	[],
+	[enable_debugging=no])
+
+# AC_PROG_CC sets CFLAGS to '-g -O2' unless already set, so there's no
+# need to add -g. Disable or change by explicitly setting CFLAGS. If
+# this option is enabled, then -Og or -O0 becomes the last
+# optimization option, and hence takes precedence.
+if test "x$enable_debugging" != "xno"; then
+	CFLAGS="${CFLAGS} -fno-inline"
+	AX_CHECK_COMPILE_FLAG([-Og],
+		[CFLAGS="${CFLAGS} -O0 -Og"],
+		[CFLAGS="${CFLAGS} -O0"],
+		[])
+fi
+
+AC_CONFIG_FILES([
+	Makefile
+	src/Makefile
+])
+AC_OUTPUT

src/Makefile.am

+AUTOMAKE_OPTIONS = subdir-objects
+
+AM_CFLAGS = $(VARNISHAPI_CFLAGS) -I$(VARNISHSRC)/include \
+	-I$(VARNISHSRC)/bin/varnishd -I$(VARNISHSRC)/lib/libvgz \
+	-Wall -Werror -Wextra -std=c99
+AM_LDFLAGS = $(VARNISHAPI_LIBS) $(VMOD_LDFLAGS) -ldl
+
+vmod_LTLIBRARIES = libvmod_pesi.la libvmod_pesi_debug.la
+
+libvmod_pesi_la_SOURCES = \
+	misc.h \
+	misc.c \
+	pesi.h \
+	pesi_flags.h \
+	pesi_tree.h \
+	pesi.c \
+	node.h \
+	node_assert.h \
+	node_head.h \
+	node_mempool.h \
+	node.c \
+	vmod_pesi.c \
+	vdp_pesi.h \
+	vdp_pesi.c \
+	debug.h \
+	tbl_set_parameter.h \
+	foreign/qdef.h \
+	foreign/from_cache_esi_deliver.h \
+	foreign/from_cache_esi_deliver.c
+
+nodist_libvmod_pesi_la_SOURCES = \
+	vcc_pesi_if.c \
+	vcc_pesi_if.h \
+	VSC_pesi.c \
+	VSC_pesi.h
+
+libvmod_pesi_debug_la_SOURCES = \
+	vmod_pesi_debug.c
+
+nodist_libvmod_pesi_debug_la_SOURCES = \
+	vcc_pesi_debug_if.c \
+	vcc_pesi_debug_if.h
+
+dist_man_MANS = vdp_pesi.3
+
+@BUILD_VSC_PESI@
+
+# -- pesi
+
+vmod_pesi.lo: vcc_pesi_if.h VSC_pesi.c VSC_pesi.h misc.h node_mempool.h \
+	pesi_flags.h vdp_pesi.h tbl_set_parameter.h
+
+vdp_pesi.lo: debug.h vdp_pesi.h node.h node_assert.h pesi.h pesi_flags.h \
+	pesi_tree.h misc.h
+
+misc.lo: misc.h debug.h
+
+node.lo: debug.h node.h node_assert.h node_mempool.h pesi.h misc.h vcc_pesi_if.h
+
+pesi.lo: debug.h VSC_pesi.h pesi_tree.h pesi_flags.h pesi.h
+
+vcc_pesi_if.h vmod_pesi.rst vmod_pesi.man.rst: vcc_pesi_if.c
+
+vcc_pesi_if.c: vdp_pesi.vcc
+	$(AM_V_VMODTOOL) $(PYTHON) $(VMODTOOL) -o vcc_pesi_if $(srcdir)/vdp_pesi.vcc
+
+vdp_pesi.3: vmod_pesi.man.rst
+	$(AM_V_GEN) $(RST2MAN) vmod_pesi.man.rst vdp_pesi.3
+
+# -- pesi_debug
+
+vcc_pesi_debug_if.h vmod_pesi_debug.rst vmod_pesi_debug.man.rst: vcc_pesi_debug_if.c
+
+vcc_pesi_debug_if.c: vmod_pesi_debug.vcc
+	$(AM_V_VMODTOOL) $(PYTHON) $(VMODTOOL) -o vcc_pesi_debug_if $(srcdir)/vmod_pesi_debug.vcc
+
+AM_TESTS_ENVIRONMENT = \
+	PATH="$(VMOD_TEST_PATH)" \
+	LD_LIBRARY_PATH="$(VARNISH_LIBRARY_PATH)"
+TEST_EXTENSIONS = .vtc
+TESTFLAGS = -v -t 20
+VTC_LOG_COMPILER = varnishtest ${TESTFLAGS}
+AM_VTC_LOG_FLAGS = -Dvmod_pesi="$(VMOD_PESI)" \
+	-Dvmod_pesi_debug="$(VMOD_PESI_DEBUG)" \
+	-p vcl_path=$(abs_srcdir)/tests
+
+# To test an individual VTC test named test.vtc:
+# $ make check TESTS=tests/test.vtc
+
+TESTS = @VMOD_TESTS@
+
+gcov: clean
+	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) CC=gcc \
+		CFLAGS="${AM_CFLAGS} --coverage -fno-inline -g -O0" check
+
+# Set QUIET=-q for non-verbose builds, otherwise set to empty.
+QUIET_0 = -q
+QUIET_ = $(QUIET_@AM_DEFAULT_V@)
+QUIET = $(QUIET_@AM_V@)
+
+coverage/lcov.info: gcov
+if HAVE_LCOV
+	$(AM_V_at)@mkdir $(builddir)/coverage
+	$(AM_V_GEN) $(LCOV) $(QUIET) -c -d . -o  $(builddir)/coverage/lcov.info
+else
+	@echo "================================================="
+	@echo "You need lcov installed to generate coverage data"
+	@echo "================================================="
+	@false
+endif
+
+coverage: coverage/lcov.info
+if HAVE_GENHTML
+	$(AM_V_GEN) $(GENHTML) $(QUIET) $(builddir)/coverage/lcov.info \
+		-o  $(builddir)/coverage
+else
+	@echo "======================================================="
+	@echo "You need genhtml installed to generate coverage reports"
+	@echo "======================================================="
+	@false
+endif
+
+EXTRA_DIST = \
+	vdp_pesi.vcc \
+	vmod_pesi_debug.vcc \
+	pesi.vsc \
+	$(VMOD_TESTS) \
+	$(builddir)/tests/debug.inc.vcl
+
+CLEANFILES = \
+	$(builddir)/vcc_pesi_if.c \
+	$(builddir)/vcc_pesi_if.h \
+	$(builddir)/vmod_pesi.rst \
+	$(builddir)/vmod_pesi.man.rst \
+	$(builddir)/vcc_pesi_debug_if.c \
+	$(builddir)/vcc_pesi_debug_if.h \
+	$(builddir)/vmod_pesi_debug.rst \
+	$(builddir)/vmod_pesi_debug.man.rst \
+	$(builddir)/vdp_pesi.3 \
+	$(builddir)/VSC_pesi.c \
+	$(builddir)/VSC_pesi.h \
+	$(builddir)/*.gcda \
+	$(builddir)/*.gcno
+
+clean-local:
+	@rm -rf $(builddir)/coverage

src/debug.h

+#ifdef DEBUG
+#include <stdio.h>
+#include "vtim.h"
+#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+#define VSLdbgv(tng, fmt, ...)						\
+	do {								\
+		VSL(SLT_Debug, 0, "xid=%u t=%.6f " fmt,			\
+		    VXID((tng)->vsl->wid), VTIM_real(), __VA_ARGS__);	\
+		VSLb((tng)->vsl, SLT_Debug, fmt, __VA_ARGS__);		\
+	} while(0)
+
+#define VSLdbg(tng, msg)				\
+	do {						\
+		VSL(SLT_Debug, 0, "xid=%u t=%.6f " msg,	\
+		    VXID((tng)->vsl->wid), VTIM_real());	\
+		VSLb((tng)->vsl, SLT_Debug, msg);		\
+	} while(0)
+#define VSL0dbg(...) VSL(SLT_Debug, 0, __VA_ARGS__)
+#else
+#define VSLdbgv(tng, fmt, ...)	(void)0
+#define VSLdbg(tng, msg)	(void)0
+#define VSL0dbg(...)	(void)0
+#define Debug(fmt, ...) /**/
+#endif

src/flint.lnt

+-emacro(655, OC_F_FINAL)
+-efile(766, "../config.h")
+// we do not lint the VSC files
+-esym(526, VSC_pesi_New, VSC_pesi_Destroy)
+-esym(755, VSC_pesi_size)
+-esym(769, *INVALID)
\ No newline at end of file

src/flint.sh

+#!/bin/sh
+
+if [ "x$1" = "x-ok" -a -f _.fl ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+	exit 0
+fi
+
+flexelint \
+	-D__FLEXELINT__ \
+	${VARNISHSRC}/flint.lnt \
+	flint.lnt \
+	-zero \
+	-I.. \
+	-I${VARNISHSRC}/include \
+	-I${VARNISHSRC}/bin/varnishd \
+	-I${VARNISHSRC}/lib/libvgz \
+	$(ls *.c | grep -v VSC_) \
+	foreign/*.c \
+	2>&1 | tee _.fl
+
+if [ -f _.fl.old ] ; then
+	diff -u _.fl.old _.fl
+fi
+
+if [ "x$1" = "x-ok" ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+fi

src/foreign/from_cache_esi_deliver.c

+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VPED - Varnish Parallel Esi Delivery
+ *
+ * modded version for parallel PESI
+ *
+ * actual esi handling is in vdp_pesi.c, this file contains VDPs we keep
+ * functionally equivalent to varnish-cache built-in esi, just with changed
+ * signatures & data structures
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "cache/cache.h"
+#include "cache/cache_filter.h"
+
+//#include "vtim.h"
+//#include "cache_esi.h"
+#include "vend.h"
+#include "vgz.h"
+
+#include "from_cache_esi_deliver.h"
+
+/*---------------------------------------------------------------------
+ * If a gzip'ed ESI object includes a ungzip'ed object, we need to make
+ * it looked like a gzip'ed data stream.  The official way to do so would
+ * be to fire up libvgz and gzip it, but we don't, we fake it.
+ *
+ * First, we cannot know if it is ungzip'ed on purpose, the admin may
+ * know something we don't.
+ *
+ * What do you mean "BS ?"
+ *
+ * All right then...
+ *
+ * The matter of the fact is that we simply will not fire up a gzip in
+ * the output path because it costs too much memory and CPU, so we simply
+ * wrap the data in very convenient "gzip copy-blocks" and send it down
+ * the stream with a bit more overhead.
+ */
+
+static int v_matchproto_(vdp_fini_f)
+vped_pretend_gzip_fini(struct vdp_ctx *vdc, void **priv)
+{
+	(void)vdc;
+	*priv = NULL;
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
+vped_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv,
+    const void *pv, ssize_t l)
+{
+	uint8_t buf1[5], buf2[5];
+	const uint8_t *p;
+	uint16_t lx;
+	struct nexus_gzip *gz;
+
+	CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC);
+	CAST_OBJ_NOTNULL(gz, *priv, NEXUS_GZIP_MAGIC);
+
+	if (l == 0)
+		return (VDP_bytes(vdx, act, pv, l));
+
+	p = pv;
+
+	AN (gz->is);
+	gz->crc = crc32(gz->crc, p, l);
+	gz->l_crc += l;
+
+	lx = 65535;
+	buf1[0] = 0;
+	vle16enc(buf1 + 1, lx);
+	vle16enc(buf1 + 3, ~lx);
+
+	while (l > 0) {
+		if (l >= 65535) {
+			lx = 65535;
+			if (VDP_bytes(vdx, VDP_NULL, buf1, sizeof buf1))
+				return (-1);
+		} else {
+			lx = (uint16_t)l;
+			buf2[0] = 0;
+			vle16enc(buf2 + 1, lx);
+			vle16enc(buf2 + 3, ~lx);
+			if (VDP_bytes(vdx, VDP_NULL, buf2, sizeof buf2))
+				return (-1);
+		}
+		if (VDP_bytes(vdx, VDP_NULL, p, lx))
+			return (-1);
+		l -= lx;
+		p += lx;
+	}
+	/* buf1 & buf2 is local, have to flush */
+	return (VDP_bytes(vdx, VDP_FLUSH, NULL, 0));
+}
+
+const struct vdp vped_pretend_gz = {
+	.name =		"PPGZ",
+	.bytes =	vped_pretend_gzip_bytes,
+	.fini =		vped_pretend_gzip_fini,
+};
+
+/*---------------------------------------------------------------------
+ * Include a gzip'ed object in a gzip'ed ESI object delivery
+ *
+ * This is the interesting case: Deliver all the deflate blocks, stripping
+ * the "LAST" bit of the last one and padding it, as necessary, to a byte
+ * boundary.
+ *
+ */
+
+static int v_matchproto_(vdp_fini_f)
+vped_gzgz_init(struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+{
+	struct req *req;
+	ssize_t l;
+	const char *p;
+	struct vped_gzgz_priv *foo;
+
+	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
+	(void)oc;
+	req = vdc->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(foo, *priv, VPED_GZGZ_PRIV_MAGIC);
+	CHECK_OBJ_NOTNULL(foo->objcore, OBJCORE_MAGIC);
+
+	memset(foo->tailbuf, 0xdd, sizeof foo->tailbuf);
+
+	AN(ObjCheckFlag(req->wrk, foo->objcore, OF_GZIPED));
+
+	p = ObjGetAttr(req->wrk, foo->objcore, OA_GZIPBITS, &l);
+	AN(p);
+	assert(l == 32);
+	foo->start = vbe64dec(p);
+	foo->last = vbe64dec(p + 8);
+	foo->stop = vbe64dec(p + 16);
+	foo->olen = ObjGetLen(req->wrk, foo->objcore);
+	assert(foo->start > 0 && foo->start < foo->olen * 8);
+	assert(foo->last > 0 && foo->last < foo->olen * 8);
+	assert(foo->stop > 0 && foo->stop < foo->olen * 8);
+	assert(foo->last >= foo->start);
+	assert(foo->last < foo->stop);
+
+	/* The start bit must be byte aligned. */
+	AZ(foo->start & 7);
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
+vped_gzgz_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv,
+    const void *ptr, ssize_t len)
+{
+	struct vped_gzgz_priv *foo;
+	const uint8_t *pp;
+	ssize_t dl;
+	ssize_t l;
+
+	// XXX would like to know which is the last
+	// segment sent to push FLUSH/END with it
+
+	CAST_OBJ_NOTNULL(foo, *priv, VPED_GZGZ_PRIV_MAGIC);
+	pp = ptr;
+	if (len > 0) {
+		/* Skip over the GZIP header */
+		dl = foo->start / 8 - foo->ll;
+		if (dl > 0) {
+			/* Before foo.start, skip */
+			if (dl > len)
+				dl = len;
+			foo->ll += dl;
+			len -= dl;
+			pp += dl;
+		}
+	}
+	if (len > 0) {
+		/* The main body of the object */
+		dl = foo->last / 8 - foo->ll;
+		if (dl > 0) {
+			if (dl > len)
+				dl = len;
+			if (VDP_bytes(vdx, VDP_NULL, pp, dl))
+				return(-1);
+			foo->ll += dl;
+			len -= dl;
+			pp += dl;
+		}
+	}
+	if (len > 0 && (unsigned)foo->ll == foo->last / 8) {
+		/* Remove the "LAST" bit */
+		foo->dbits[0] = *pp;
+		foo->dbits[0] &= ~(1U << (foo->last & 7));
+		if (VDP_bytes(vdx, VDP_NULL, foo->dbits, 1))
+			return (-1);
+		foo->ll++;
+		len--;
+		pp++;
+	}
+	if (len > 0) {
+		/* Last block */
+		dl = foo->stop / 8 - foo->ll;
+		if (dl > 0) {
+			if (dl > len)
+				dl = len;
+			if (VDP_bytes(vdx, VDP_NULL, pp, dl))
+				return (-1);
+			foo->ll += dl;
+			len -= dl;
+			pp += dl;
+		}
+	}
+	if (len > 0 && (foo->stop & 7) && (unsigned)foo->ll == foo->stop / 8) {
+		/* Add alignment to byte boundary */
+		foo->dbits[1] = *pp;
+		foo->ll++;
+		len--;
+		pp++;
+		switch ((int)(foo->stop & 7)) {
+		case 1: /*
+			 * x000....
+			 * 00000000 00000000 11111111 11111111
+			 */
+		case 3: /*
+			 * xxx000..
+			 * 00000000 00000000 11111111 11111111
+			 */
+		case 5: /*
+			 * xxxxx000
+			 * 00000000 00000000 11111111 11111111
+			 */
+			foo->dbits[2] = 0x00; foo->dbits[3] = 0x00;
+			foo->dbits[4] = 0xff; foo->dbits[5] = 0xff;
+			foo->lpad = 5;
+			break;
+		case 2: /* xx010000 00000100 00000001 00000000 */
+			foo->dbits[1] |= 0x08;
+			foo->dbits[2] = 0x20;
+			foo->dbits[3] = 0x80;
+			foo->dbits[4] = 0x00;
+			foo->lpad = 4;
+			break;
+		case 4: /* xxxx0100 00000001 00000000 */
+			foo->dbits[1] |= 0x20;
+			foo->dbits[2] = 0x80;
+			foo->dbits[3] = 0x00;
+			foo->lpad = 3;
+			break;
+		case 6: /* xxxxxx01 00000000 */
+			foo->dbits[1] |= 0x80;
+			foo->dbits[2] = 0x00;
+			foo->lpad = 2;
+			break;
+		case 7:	/*
+			 * xxxxxxx0
+			 * 00......
+			 * 00000000 00000000 11111111 11111111
+			 */
+			foo->dbits[2] = 0x00;
+			foo->dbits[3] = 0x00; foo->dbits[4] = 0x00;
+			foo->dbits[5] = 0xff; foo->dbits[6] = 0xff;
+			foo->lpad = 6;
+			break;
+		case 0: /* xxxxxxxx */
+		default:
+			WRONG("compiler must be broken");
+		}
+		if (VDP_bytes(vdx, VDP_NULL, foo->dbits + 1, foo->lpad))
+			return (-1);
+	}
+	if (len > 0) {
+		/* Recover GZIP tail */
+		dl = foo->olen - foo->ll;
+		assert(dl >= 0);
+		if (dl > len)
+			dl = len;
+		if (dl > 0) {
+			assert(dl <= 8);
+			l = foo->ll - (foo->olen - 8);
+			assert(l >= 0);
+			assert(l <= 8);
+			assert(l + dl <= 8);
+			memcpy(foo->tailbuf + l, pp, dl);
+			foo->ll += dl;
+			len -= dl;
+		}
+	}
+	assert(len == 0);
+	if (act != VDP_END)
+		act = VDP_FLUSH;
+	return (VDP_bytes(vdx, act, NULL, 0));
+}
+
+static int v_matchproto_(vdp_fini_f)
+vped_gzgz_fini(struct vdp_ctx *vdc, void **priv)
+{
+	uint32_t icrc;
+	uint32_t ilen;
+	struct vped_gzgz_priv *vgzgz;
+
+	(void) vdc;
+
+	CAST_OBJ_NOTNULL(vgzgz, *priv, VPED_GZGZ_PRIV_MAGIC);
+	*priv = NULL;
+
+	icrc = vle32dec(vgzgz->tailbuf);
+	ilen = vle32dec(vgzgz->tailbuf + 4);
+	vgzgz->gz->crc = crc32_combine(vgzgz->gz->crc, icrc, ilen);
+	vgzgz->gz->l_crc += ilen;
+
+	return (0);
+}
+
+const struct vdp vped_gzgz = {
+	.name =         "PVZZ",
+	.init =         vped_gzgz_init,
+	.bytes =        vped_gzgz_bytes,
+	.fini =         vped_gzgz_fini,
+};

src/foreign/from_cache_esi_deliver.h

+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VED - Varnish Esi Delivery
+ */
+
+extern const struct vdp vped_pretend_gz;
+extern const struct vdp vped_gzgz;
+
+struct nexus_gzip {
+	unsigned		magic;
+#define NEXUS_GZIP_MAGIC	0xb71c17a8
+	unsigned		is;
+	uint32_t		crc;
+	ssize_t			l_crc;
+	struct nexus_gzip	*up;
+};
+
+struct vped_gzgz_priv {
+	unsigned		magic;
+#define VPED_GZGZ_PRIV_MAGIC	0xaa9cd734
+	struct nexus_gzip	*gz;
+	struct objcore		*objcore;
+	uint64_t		start, last, stop, lpad;
+	ssize_t			ll;
+	uint64_t		olen;
+	uint8_t			dbits[8];
+	uint8_t			tailbuf[8];
+};

src/foreign/qdef.h

+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1991, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * Berkeley Software Design, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)cdefs.h	8.8 (Berkeley) 1/9/95
+ * $FreeBSD$
+ */
+
+/*
+ * vqueue.h presupposes the __containerof() macro for VSTAILQ_LAST(),
+ * which is in FreeBSD cdefs.h. Include this after vdef.h, which defines
+ * __GNUC_PREREQ__, and before vqueue.h.
+ *
+ * This differs from cdefs.h in that uintptr_t is taken from stdint.h, and
+ * offsetof() is taken from stddef.h.
+ */
+
+#ifndef	__DEQUALIFY
+#define	__DEQUALIFY(type, var)	((type)(uintptr_t)(const volatile void *)(var))
+#endif
+
+/*
+ * Given the pointer x to the member m of the struct s, return
+ * a pointer to the containing structure.  When using GCC, we first
+ * assign pointer x to a local variable, to check that its type is
+ * compatible with member m.
+ */
+#if __GNUC_PREREQ__(3, 1)
+#define	__containerof(x, s, m) ({					\
+	const volatile __typeof(((s *)0)->m) *__x = (x);		\
+	__DEQUALIFY(s *, (const volatile char *)__x - offsetof(s, m));  \
+})
+#else
+#define	__containerof(x, s, m)						\
+	__DEQUALIFY(s *, (const volatile char *)(x) - offsetof(s, m))
+#endif

src/misc.c

+/*-
+ * Copyright 2019 - 2020 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * utility function(s) used by node.c and vdp_pesi.c
+ */
+
+#include "config.h"
+
+#include "cache/cache_varnishd.h"
+
+#include "misc.h"
+#include "debug.h"
+
+void
+req_fini(struct req **reqp, struct worker *wrk)
+{
+	struct req *req;
+
+	AN(reqp);
+	if (*reqp == NULL)
+		return;
+
+	TAKE_OBJ_NOTNULL(req, reqp, REQ_MAGIC);
+
+	VSLdbg(req, "req_fini called");
+
+	req->acct.resp_bodybytes += VDP_Close(req->vdc);
+	Req_Cleanup(req->sp, wrk, req);
+	Req_Release(req);
+}

src/misc.h

+#define OC_F_FINAL (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)
+
+void req_fini(struct req **, struct worker *);
+
+#define VFAIL(ctx, fmt, ...)					\
+	VRT_fail((ctx), "vdp pesi failure: " fmt, __VA_ARGS__)

src/node.h

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * node interface
+ */
+
+#include "node_head.h"
+
+struct bytes_tree {
+	unsigned		magic;
+#define BYTES_TREE_MAGIC 0x49c59d46
+	struct lock		tree_lock;
+	// esi_level > 1 signalling new data
+	pthread_cond_t		cond;
+	struct node		*root;
+	struct node		*front;
+	const struct worker	*front_owner;
+	const struct worker	*unpend_owner;
+	// candidate for last ever node
+	struct node		*end;
+	int			end_sent;
+	int			npending;
+	int			retval;
+};
+
+enum n_type {
+	T_INVALID = 0,
+	T_NEXUS,	// can change into T_SUBREQ / T_FINAL / T_DATA
+	T_DATA,
+	T_CRC,
+	T_SUBREQ,
+	T_FINAL	// non-ESI pass / hfm / hfp
+} __attribute__ ((__packed__));
+
+/*
+ * see state.dot:
+ *
+ * ST_DATA:    may never have any children
+ *
+ *	       T_DATA / T_CRC / T_SUBREQ / T_FINAL
+ *
+ * ST_PRIVATE: may receive pushes creating children
+ *             unpending must not yet touch it
+ *             children can be created unlocked
+ *             can change into other types
+ *
+ *             T_NEXUS only
+ *
+ * ST_OPEN:    may receive pushes creating children
+ *             owning thread may run front delivery
+ *             any changes below locked only
+ *
+ *             T_NEXUS only
+ *
+ *
+ * ST_CLOSED:  the request is done, no pushes can occur,
+ *             unpending can proceed upwards
+ *
+ *             T_NEXUS only
+ *
+ * ST_UNPENDING: in the process of being pushed to the client
+ *
+ *	         T_DATA / T_CRC / T_SUBREQ / T_FINAL
+ *
+ * ST_DELIVERED: We have pushed data up
+ *
+ *               any type
+ *
+ *		 for T_NEXUS, means "anything below is delivered"
+ *
+ * ST_PRUNED: nodes below have been freed, no attempt must
+ *            be made to access any children of this node
+ *            (similar to ST_PRIVATE)
+ *
+ *            T_NEXUS only
+ */
+enum n_state {
+	ST_INVALID = 0,
+	ST_DATA,
+	ST_PRIVATE,
+	ST_OPEN,
+	ST_CLOSED,
+	ST_UNPENDING,
+	ST_DELIVERED,
+	ST_PRUNED
+} __attribute__ ((__packed__));
+
+struct node_nexus {
+	struct node_head	children;
+	struct objcore		*oc;
+	struct req		*req;
+	const struct worker	*owner; // ST_OPEN only
+
+	/* number of nodes pending under this node while state == ST_PRIVATE */
+	int			npending_private;
+	/*
+	 * crc for data nodes immediately below
+	 * updated by push_crc, pretendgzip and gzgz
+	 */
+	struct nexus_gzip	gzip;	// in foreign
+};
+
+enum t_crc {
+	INVALID = 0,
+	GZIP_HDR,
+	UPDATE,
+	FINAL	// combine with parent or gen tail
+} __attribute__ ((__packed__));
+
+struct node_crc {
+	enum t_crc	ctype;
+	uint32_t	icrc;
+	ssize_t		l_icrc;
+};
+
+struct node_data {
+	const void		*ptr;
+	struct storage		*st;
+	ssize_t			len;
+	enum vdp_action		act;
+};
+
+/*
+ * we transfer the sub-request, boc and oc to the topreq thread, delivery
+ * happens there
+ */
+struct node_subreq {
+	struct req		*req;
+	struct boc		*boc;
+	// oc is NULL if already transferred back into req
+	struct objcore		*oc;
+	// subreq to topreq delivery
+	int			done;
+	pthread_cond_t		cond;
+};
+
+// sub-state for node_final while in ST_DATA
+enum fi_state {
+	FI_READY	= 0,
+	FI_GO,			// topreq signalling req to deliver
+	FI_DONE,		// req signalling topreq it is done
+	FI_DESTROYED		// cond/mtx destroyed (fini_final())
+} __attribute__ ((__packed__));
+
+/* we block the sub-thread when it's ready for delivery and continue when the
+ * topreqp tells it to */
+struct node_final {
+	enum fi_state		fi_state;
+	pthread_mutex_t	fi_mtx;
+	pthread_cond_t		fi_cond;
+};
+
+enum n_alloc {
+	NA_INVALID	= 0,
+	NA_WS,
+	NA_MPL
+} __attribute__ ((__packed__));
+
+struct node {
+	unsigned		magic;
+#define NODE_MAGIC 0xe31edef3
+	enum n_type		type;
+	enum n_state		state;
+	enum n_alloc		allocator;
+	unsigned		is_end:1;
+	VSTAILQ_ENTRY(node)	sibling;
+	VSTAILQ_ENTRY(node)	unpend;
+	struct node		*parent;
+
+	union {
+		struct node_nexus	nexus;	// T_NEXUS
+		struct node_data	data;	// T_DATA
+		struct node_subreq	subreq; // T_SUBREQ
+		struct node_final	final;	// T_FINAL
+		struct node_crc	crc;	// T_CRC
+	};
+};
+
+/*
+ * node mutation: turn an existing nexus into something else
+ *
+ * howto:
+ *
+ * 0) node must be T_NEXUS / ST_PRIVATE, untouched
+ *
+ * 1) call node_mutate_prep()
+ *
+ * 2) modify data in the destination-type specific struct
+ *
+ * 3) call node_mutate_lock() with destination type/state
+ *
+ * 4) optionally do more things under the lock
+ *
+ * 5) call node_mutate_unlock()
+ *
+ */
+
+static inline void
+node_mutate_prep(const struct bytes_tree *tree, struct node *node)
+{
+	CHECK_OBJ_NOTNULL(node, NODE_MAGIC);
+	CHECK_OBJ_NOTNULL(node->parent, NODE_MAGIC);
+
+	assert(node->state == ST_PRIVATE);
+	assert(node->type == T_NEXUS);
+	assert(node != tree->root);
+
+	AZ(node->nexus.npending_private);
+	AZ(node->nexus.oc);
+
+	memset(&node->nexus, 0, sizeof node->nexus);
+}
+
+static inline void
+node_mutate_lock(struct bytes_tree *tree, struct node *node,
+		 enum n_type type, enum n_state state)
+{
+	/* these checks can be relexed when needed */
+	assert(type == T_DATA ||
+	       type == T_SUBREQ ||
+	       type == T_FINAL);
+	assert(state == ST_DATA);
+
+	Lck_Lock(&tree->tree_lock);
+	node->type = type;
+	node->state = state;
+	if (node->parent->state != ST_PRIVATE)
+		AZ(pthread_cond_signal(&tree->cond));
+}
+
+static inline void
+node_mutate_unlock(struct bytes_tree *tree)
+{
+	Lck_Unlock(&tree->tree_lock);
+}
+
+//--------------
+
+void node_fill_nodestock(struct ws *, struct node_head *);
+
+//--------------
+
+struct pesi;
+struct node *node_alloc(struct pesi *);
+void node_insert(struct bytes_tree *, struct node *, struct node *);
+void set_open(struct bytes_tree *, struct node *, const struct worker *);
+void set_closed(struct bytes_tree *, struct node *, const struct worker *);
+
+//--------------
+
+void tree_prune(struct vdp_ctx *, struct node *);
+void tree_free(struct vdp_ctx *, struct node *);
+
+//--------------
+
+void tree_deliver(struct vdp_ctx *, struct bytes_tree *);

src/node_assert.h

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * tree integrity check
+ *
+ * CHK_ORDER: within child list can only transition from ST_DELIVERED -> !
+ * ST_DELIVERED iow, once a child is found pending, all others must be also
+ */
+
+enum check_state {
+	CHK_ANY = 0,
+	CHK_PEND,	// ! ST_DELIVERED && ! ST_UNPENDING
+	CHK_DELI,	// ST_DELIVERED
+	CHK_ORDER
+};
+
+/*
+ * keep in prod builds for now to gain trust in tree sanity
+ *
+ * XXX enable only for DEBUG later?
+ */
+#ifdef NO_ASSERT_NODE
+#define assert_node(n, c) (void)0
+#define assert_nexus(n, c) (void)0
+#else
+static inline void assert_node(const struct node *node, enum check_state check);
+
+static inline void
+assert_nexus(const struct node *node, enum check_state nexcheck)
+{
+	struct node *child;
+	enum check_state check;
+
+	CHECK_OBJ_NOTNULL(node, NODE_MAGIC);
+	assert(node->type == T_NEXUS);
+
+	check = nexcheck;
+	VSTAILQ_FOREACH(child, &node->nexus.children, sibling) {
+		if (child->type == T_NEXUS) {
+			/*
+			 * actually, we would need to communicate up from the
+			 * subtree the fact that any <= ST_CLOSED was found and
+			 * set check accordingly for CHK_ORDER as below, but
+			 * still this is only an assertion....
+			 */
+			assert_node(child, nexcheck);
+			continue;
+		}
+		assert_node(child, check);
+		if (check == CHK_ORDER && child->state <= ST_CLOSED)
+			check = CHK_PEND;
+	}
+}
+
+static inline void
+assert_node(const struct node *node, enum check_state check)
+{
+	CHECK_OBJ_NOTNULL(node, NODE_MAGIC);
+
+	switch (check) {
+	case CHK_ANY:
+	case CHK_ORDER:
+		break;
+	case CHK_PEND:
+		// not ST_UNPENDING ST_DELIVERED ST_PRUNED
+		assert(node->state <= ST_CLOSED);
+		break;
+	case CHK_DELI:
+		if (node->state == ST_PRUNED)
+			assert(node->type == T_NEXUS);
+		else
+			assert(node->state == ST_DELIVERED);
+		break;
+	default:
+		INCOMPL();
+	}
+
+	switch (node->state) {
+	case ST_PRIVATE:
+	case ST_PRUNED:
+		assert(node->type == T_NEXUS);
+		/* hands off all nexus fields */
+		break;
+	case ST_DATA:
+	case ST_UNPENDING:
+		assert(node->type != T_NEXUS);
+		break;
+	case ST_OPEN:
+		AN(node->nexus.owner);
+		AZ(node->nexus.npending_private);
+		assert_nexus(node, CHK_ORDER);
+		break;
+	case ST_CLOSED:
+		AZ(node->nexus.owner);
+		AZ(node->nexus.npending_private);
+		assert_nexus(node, CHK_ORDER);
+		break;
+	case ST_DELIVERED:
+		if (node->type == T_NEXUS) {
+			assert_nexus(node, CHK_DELI);
+			AZ(node->nexus.oc);
+		} else if (node->type == T_DATA) {
+			AZ(node->data.st);
+		}
+		break;
+	default:
+		INCOMPL();
+	}
+}
+#endif

src/node_head.h

+/*
+ * shared between pesi.h and node.h
+ * because of (struct pesi).nodestock
+ */
+
+#ifndef PESI_NODE_HEAD_H
+#define PESI_NODE_HEAD_H
+struct node;
+VSTAILQ_HEAD(node_head, node);
+void node_init_nodestock(struct node_head *);
+#endif

src/node_mempool.h

+extern struct mempool *mempool;
+size_t node_size();

src/pesi.c

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * PESI per request state and task management
+ */
+
+#include "config.h"
+
+#include <string.h>
+
+#include "cache/cache.h"
+
+#include "VSC_pesi.h"
+
+#include "pesi_tree.h"
+#include "pesi_flags.h"
+#include "node_head.h"
+#include "pesi.h"
+
+/* shared with vmod code */
+extern struct lock stats_lock;
+extern struct VSC_pesi *stats;
+struct lock stats_lock;
+struct VSC_pesi *stats = NULL;
+
+/* ------------------------------------------------------------
+ * pesi tasks
+ *
+ * in the top request, we need to wait for all pesis to finish because, via
+ * req->topreq and req->sp, they all reference the top request
+ *
+ * for this purpose, simple refcounting would suffice, but putting all the pesis
+ * on a list does not add any relevant amount of complexity and might help with
+ * troubleshooting and through additional assertions
+ *
+ * pesi_new always happens in a task but the destruction can happen in any
+ * order:
+ * - for T_SUBREQ, pesi outlives the task
+ * - otherwise, pesi gets destroyed before the task
+ */
+struct pesi *
+pesi_new(struct ws *ws, struct pesi_tree *pesi_tree)
+{
+	struct pesi *pesi;
+
+	pesi = WS_Alloc(ws, sizeof *pesi);
+	if (pesi == NULL)
+		return (NULL);
+
+	INIT_OBJ(pesi, PESI_MAGIC);
+	pesi->pecx->magic = PECX_MAGIC;
+	pesi->pesi_tree = pesi_tree;
+	pesi->flags = PF_HAS_TASK | PF_CFG_DEFAULT;
+
+	node_init_nodestock(&pesi->nodestock);
+
+	Lck_Lock(&pesi_tree->task_lock);
+	VTAILQ_INSERT_TAIL(&pesi_tree->task_head, pesi, list);
+	assert(pesi_tree->task_running >= 0);
+	pesi_tree->task_running++;
+	Lck_Unlock(&pesi_tree->task_lock);
+	return (pesi);
+}
+
+/*
+ * shutting down a pesi request is a two stage process, because Req_Cleanup() in
+ * req_fini() called from vped_task() zeroes the workspace including our
+ * per-request struct pesi, so removing the registration from
+ * pesi_tree->task_head need to happen before Req_Cleanup().
+ *
+ * But notification of the topreq needs to happen after Req_Cleanup(), because
+ * it still references topreq and sp, thus the topreq may not finish until all
+ * pesi tasks have finished.
+ *
+ * on the implementation:
+ *
+ * - in vdp_pesi_fini() we call pesi_destroy() to clean up the per-request
+ *   struct pesi and remove it from the global list.
+ *
+ *   For possible debuggung purposes, we record the fact that a pesi request is
+ *   finishing
+ *
+ * - in vped_task(), we call task_fini() to decrement the global counter and
+ *   notify the top request
+ *
+ * for esi_level == 0, all cleanup happens in vdp_pesi_fini
+ */
+
+void
+pesi_destroy(struct pesi **pesip)
+{
+	struct pesi *pesi;
+	struct pesi_tree *pesi_tree;
+
+	TAKE_OBJ_NOTNULL(pesi, pesip, PESI_MAGIC);
+	CHECK_OBJ_NOTNULL(pesi->pecx, PECX_MAGIC);
+	TAKE_OBJ_NOTNULL(pesi_tree, &pesi->pesi_tree, PESI_TREE_MAGIC);
+
+	if (pesi->no_thread != 0) {
+		Lck_Lock(&stats_lock);
+		AN(stats);
+		stats->no_thread += pesi->no_thread;
+		Lck_Unlock(&stats_lock);
+	}
+
+	Lck_Lock(&pesi_tree->task_lock);
+	VTAILQ_REMOVE(&pesi_tree->task_head, pesi, list);
+
+	assert(pesi_tree->task_running >= 0);
+
+	if (pesi->flags & PF_HAS_TASK)
+		pesi_tree->task_finishing++;
+	Lck_Unlock(&pesi_tree->task_lock);
+
+	memset(pesi, 0, sizeof *pesi);
+}
+
+
+void
+task_fini(struct pesi_tree *pesi_tree, struct pesi *pesi)
+{
+	Lck_Lock(&pesi_tree->task_lock);
+	assert(pesi_tree->task_running > 0);
+	if (pesi == NULL) {
+		assert(pesi_tree->task_finishing > 0);
+		pesi_tree->task_finishing--;
+	}
+	else {
+		AN(pesi->flags & PF_HAS_TASK);
+		pesi->flags &= ~PF_HAS_TASK;
+	}
+	pesi_tree->task_running--;
+	if (pesi_tree->task_running == 0) {
+		AZ(pesi_tree->task_finishing);
+		AZ(pthread_cond_signal(&pesi_tree->task_cond));
+	}
+	Lck_Unlock(&pesi_tree->task_lock);
+}

src/pesi.h

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * PESI per request state
+ */
+
+struct pesi_tree;
+
+struct pesi * pesi_new(struct ws *ws, struct pesi_tree *pesi_tree);
+void pesi_destroy(struct pesi **pesip);
+void task_fini(struct pesi_tree *pesi_tree, struct pesi *pesi);
+
+/*
+ * per request state
+ *
+ * pecx: ESI parser
+ * pesi: any request
+ *
+ */
+struct pecx {
+	unsigned		magic;
+#define PECX_MAGIC		0x5d8cd06d
+	const uint8_t		*p;
+	const uint8_t		*e;
+
+	ssize_t			l;
+	int			state;
+};
+
+struct pesi {
+	unsigned		magic;
+#define PESI_MAGIC 0xa6ba54a0
+	unsigned		flags;
+
+	struct pesi_tree	*pesi_tree;
+	struct worker		*wrk;
+	struct node		*node;
+	int			woken;
+
+	struct pecx		pecx[1];
+	VTAILQ_ENTRY(pesi)	list;
+	struct node_head	nodestock;
+
+	unsigned		no_thread;
+
+#ifdef DEBUG_PESI_WS
+	uintptr_t		ws_snap;
+#endif
+};

src/pesi.vsc

+..
+	This is *NOT* a RST file but the syntax has been chosen so
+	that it may become an RST file at some later date.
+
+.. varnish_vsc_begin::  pesi
+	:oneliner:      VDP pesi stats
+	:order:         90
+
+.. varnish_vsc:: no_thread
+	:type:  counter
+	:oneliner:      No threads available for parallel ESI
+
+	Number of times no threads were available for parallel ESI.
+
+.. varnish_vsc_end::    pesi

src/pesi_flags.h

+extern int block_final, front_push;
+
+#define PF_HAS_TASK		1U
+/* vcl-controlled flags */
+#define PF_CFG_SERIAL		(1U<<1)
+#define PF_CFG_THREAD		(1U<<2)
+/* undocumented for now */
+#define PF_CFG_BLOCK_FINAL	(1U<<3)
+#define PF_CFG_FRONT_PUSH	(1U<<4)
+
+#define PF_CFG_DEFAULT					\
+	( PF_CFG_THREAD				\
+	| (block_final ? PF_CFG_BLOCK_FINAL : 0)	\
+	| (front_push ? PF_CFG_FRONT_PUSH : 0)		\
+	)
+
+#define PF_MASK_CFG		\
+	( PF_CFG_SERIAL	\
+	| PF_CFG_THREAD	\
+	| PF_CFG_BLOCK_FINAL	\
+	| PF_CFG_FRONT_PUSH	\
+	)

src/pesi_tree.h

+/*-
+ * Copyright 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Authors: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *	    Nils Goroll <nils.goroll@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * pesi_tree is a global context for the entire ESI tree started at level 0.
+ *
+ * XXX merge pesi_tree with bytes_tree ?
+ */
+
+struct pesi_tree {
+	unsigned		magic;
+#define PESI_TREE_MAGIC 0xe8ce8adb
+	struct bytes_tree	*tree;
+
+	/* all pesis (struct pesi) we started */
+	struct lock		task_lock;
+	pthread_cond_t		task_cond;
+	VTAILQ_HEAD(,pesi)	task_head;
+	int			task_running;
+	int			task_finishing;
+};

src/state.dot

+digraph bytes_node_state {
+	node_insert [label="node_insert()"]
+	node_insert -> ST_DATA
+
+	vdp_pesi_init [label="vdp_pesi_init()\nroot only"]
+	node_insert -> ST_PRIVATE
+	vdp_pesi_init -> ST_PRIVATE
+
+	subgraph cluster_T_NEXUS {
+		label="T_NEXUS"
+
+		ST_PRIVATE -> ST_OPEN		[label=" set_open()"]
+		ST_PRIVATE -> ST_CLOSED	[label="set_closed()"]
+
+		ST_OPEN -> ST_CLOSED		[label=" set_closed()"]
+
+	}
+
+	subgraph cluster_leaf {
+		label="leaf nodes:\nT_DATA\nT_CRC\nT_SUBREQ\nT_FINAL"
+
+		ST_DATA -> ST_UNPENDING	[label="set_unpending()"]
+	}
+
+	ST_CLOSED -> ST_DELIVERED	[label="set_delivered()\n"]
+	ST_UNPENDING -> ST_DELIVERED	[label="set_delivered()"]
+
+	ST_DATA -> ST_DELIVERED	[label="set_delivered()\nif NULL data"]
+
+	ST_DELIVERED -> ST_PRUNED	[label="  tree_prune()"]
+
+	# mutate
+	ST_PRIVATE -> ST_DATA		[xlabel="node_\nmutate_*()"]
+
+
+}
\ No newline at end of file

src/tbl_set_parameter.h

+VMODENUM(serial, PF_CFG_SERIAL)
+VMODENUM(thread, PF_CFG_THREAD)
+#undef VMODENUM

src/tests/00basic.vtc

+# looks like -*- vcl -*-
+
+varnishtest "vcl.use and .discard, and version string"
+
+varnish v1 -vcl {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+	backend b { .host = "${bad_ip}"; }
+} -start
+
+varnish v1 -vsc MEMPOOL.pesi.*
+varnish v1 -expect MEMPOOL.pesi.pool > 0
+varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
+varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
+
+varnish v1 -vsc PESI.*
+varnish v1 -expect PESI.no_thread == 0
+
+varnish v1 -vsc LCK.pesi.*
+varnish v1 -expect LCK.pesi.stats.creat == 1
+varnish v1 -expect LCK.pesi.stats.locks == 0
+varnish v1 -expect LCK.pesi.buf.creat == 0
+varnish v1 -expect LCK.pesi.buf.locks == 0
+varnish v1 -expect LCK.pesi.tasks.creat == 0
+varnish v1 -expect LCK.pesi.tasks.locks == 0
+
+varnish v1 -vcl {backend b { .host = "${bad_ip}"; }}
+
+varnish v1 -vsc MEMPOOL.pesi.*
+varnish v1 -expect MEMPOOL.pesi.pool > 0
+varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
+varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
+
+varnish v1 -cli "vcl.list"
+varnish v1 -cli "vcl.use vcl1"
+
+varnish v1 -vsc MEMPOOL.pesi.*
+varnish v1 -expect MEMPOOL.pesi.pool > 0
+varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
+varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
+
+varnish v1 -cli "vcl.use vcl2"
+varnish v1 -cli "vcl.use vcl1"
+
+varnish v1 -vsc MEMPOOL.pesi.*
+varnish v1 -expect MEMPOOL.pesi.pool > 0
+varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
+varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
+
+varnish v1 -cli "vcl.state vcl2 cold"
+
+varnish v1 -vsc MEMPOOL.pesi.*
+varnish v1 -expect MEMPOOL.pesi.pool > 0
+varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
+varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
+
+varnish v1 -vsc PESI.*
+varnish v1 -expect PESI.no_thread == 0
+
+varnish v1 -cli "vcl.state vcl2 warm"
+
+varnish v1 -vsc MEMPOOL.pesi.*
+varnish v1 -expect MEMPOOL.pesi.pool > 0
+varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
+varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
+
+varnish v1 -vsc PESI.*
+varnish v1 -expect PESI.no_thread == 0
+
+varnish v1 -vsc LCK.pesi.*
+varnish v1 -expect LCK.pesi.stats.creat == 1
+varnish v1 -expect LCK.pesi.stats.locks == 0
+varnish v1 -expect LCK.pesi.buf.creat == 0
+varnish v1 -expect LCK.pesi.buf.locks == 0
+varnish v1 -expect LCK.pesi.tasks.creat == 0
+varnish v1 -expect LCK.pesi.tasks.locks == 0
+
+varnish v1 -cli "vcl.show vcl1"
+varnish v1 -cli "vcl.use vcl2"
+varnish v1 -cli "vcl.discard vcl1"
+varnish v1 -cli "vcl.list"
+
+varnish v1 -vcl {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.x-version = pesi.version();
+		if (!resp.http.x-version) {
+			set resp.status = 500;
+		}
+		return(deliver);
+	}
+}
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.x-version ~ "^.+$"
+} -run

src/tests/backend_timeout.vtc

+varnishtest "like e00003.vtc but backend timeout at ESI level 1"
+
+
+server s1 {
+	rxreq
+	expect req.http.esi0 == "foo"
+	txresp -body {
+		<html>
+		Before include
+		<esi:include src="/body" sr="foo"/>
+		After include
+		</html>
+	}
+} -start
+
+server s2 {
+	rxreq
+	expect req.url == "/body1"
+	expect req.http.esi0 != "foo"
+	delay 0.6
+	txresp -body {
+		Included file
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_recv {
+		if (req.esi_level > 0) {
+			set req.url = req.url + req.esi_level;
+		} else {
+			set req.http.esi0 = "foo";
+		}
+	}
+
+	sub vcl_backend_fetch {
+		if (bereq.url == "/") {
+			set bereq.backend = s1;
+		}
+		else {
+			set bereq.backend = s2;
+			set bereq.first_byte_timeout = 0.5s;
+		}
+	}
+
+	sub vcl_backend_response {
+		if (bereq.url == "/") {
+			set beresp.do_esi = true;
+		}
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+
+	sub vcl_backend_error {
+		set beresp.body = "ouch!";
+		return(deliver);
+	}
+} -start
+
+client c1 {
+	txreq -hdr "Host: foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 62
+	expect resp.body == {
+		<html>
+		Before include
+		ouch!
+		After include
+		</html>
+	}
+} -run

src/tests/client_close.vtc

+varnishtest "ESI include - like e00003.vtc but early client close"
+
+
+server s1 {
+	rxreq
+	expect req.http.esi0 == "foo"
+	txresp -body {
+		<html>
+		Before include
+		<esi:include src="/body" sr="foo"/>
+		After include
+		</html>
+	}
+} -start
+
+server s2 {
+	rxreq
+	expect req.url == "/body1"
+	expect req.http.esi0 != "foo"
+	txresp -body {
+		Included file
+	}
+} -start
+
+varnish v1 -arg "-p debug=+syncvsl" -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_recv {
+		if (req.esi_level > 0) {
+			set req.url = req.url + req.esi_level;
+		} else {
+			set req.http.esi0 = "foo";
+		}
+	}
+
+	sub vcl_backend_fetch {
+		if (bereq.url == "/") {
+			set bereq.backend = s1;
+		}
+		else {
+			set bereq.backend = s2;
+		}
+	}
+
+	sub vcl_backend_response {
+		if (bereq.url == "/") {
+			set beresp.do_esi = true;
+		}
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+} -start
+
+client c1 {
+	txreq -hdr "Host: foo"
+	rxresphdrs
+	rxchunk
+} -run

src/tests/debug.inc.vcl

+sub vcl_recv {
+        pesi_debug.register_privs();
+}
+sub vcl_deliver {
+        pesi_debug.check_privs();
+}
+sub vcl_synth {
+        # return(fail) implies a rollback - PRIVs lost
+        if (resp.status != 503 || resp.reason != "VCL failed") {
+                pesi_debug.check_privs();
+        }
+}

src/tests/disable_esi.vtc

+varnishtest "disable ESI with standard VCL (resp.do_esi or req.esi)"
+
+server s1 {
+	rxreq
+	txresp -body {<esi:include src="/body"/>}
+} -start
+
+server s2 {
+	rxreq
+	expect req.url == "/body"
+	txresp -body {foo}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_fetch {
+		if (bereq.url == "/") {
+			set bereq.backend = s1;
+		}
+		else {
+			set bereq.backend = s2;
+		}
+	}
+
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+
+	sub vcl_deliver {
+		if (req.http.Disable) {
+			set resp.do_esi = false;
+		}
+		pesi.activate();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+
+	txreq -hdr "Disable: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.body == {<esi:include src="/body"/>}
+} -run
+
+varnish v1 -syntax 4.0 -vcl {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_deliver {
+		if (req.http.Disable) {
+			set req.esi = false;
+		}
+		pesi.activate();
+	}
+}
+
+client c1 -run

src/tests/e00000.vtc

+varnishtest "pESI test with no ESI content"
+
+
+server s1 {
+	rxreq
+	txresp -body {
+		-This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+
+	sub vcl_deliver {
+		set resp.http.can_esi = obj.can_esi;
+		pesi.activate();
+	}
+} -start
+
+logexpect l1 -v v1 -g raw {
+	expect * * ESI_xmlerror esi_disable_xml_check
+	expect * 1001 End
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 33
+	expect resp.http.can_esi == "false"
+	expect resp.body == {
+		-This is a test: Hello world
+	}
+}
+
+client c1 -run
+logexpect l1 -wait
+varnish v1 -expect esi_errors == 0
+varnish v1 -expect MAIN.s_resp_bodybytes == 33
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 33
+		expect resp.http.can_esi == "false"
+		expect resp.body == {
+		-This is a test: Hello world
+	}
+	} -run
+} -run

src/tests/e00001.vtc

+varnishtest "pESI:remove"
+
+
+server s1 {
+	rxreq
+	txresp -body {
+		<esi:remove>
+		This is a test: Unseen University
+		<esi:include src="trick question">
+		<!--esi XXX -->
+		</esi:remove>
+		<esX>This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+		set resp.http.filters = resp.filters;
+		set resp.http.can_esi = obj.can_esi;
+	}
+} -start
+
+logexpect l1 -v v1 -g raw {
+	expect * * ESI_xmlerror {^ERR after 3 ESI 1.0 <esi:include> element nested in <esi:remove>}
+	expect 0 = ESI_xmlerror {^ERR after 3 ESI 1.0 Nested <!--esi element in <esi:remove>}
+	expect * 1001 End
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 40
+	expect resp.http.can_esi == "true"
+	expect resp.body == {
+		
+		<esX>This is a test: Hello world
+	}
+}
+
+client c1 -run
+logexpect l1 -wait
+varnish v1 -expect esi_errors == 2
+varnish v1 -expect MAIN.s_resp_bodybytes == 40
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 40
+		expect resp.body == {
+		
+		<esX>This is a test: Hello world
+	}
+	} -run
+} -run

src/tests/e00002.vtc

+varnishtest "ESI CDATA"
+
+
+server s1 {
+	rxreq
+	txresp -body {
+		<esi:remove>
+		<![CDATA[</esi:remove>]]>
+		This is a test: Unseen University
+		</esi:remove>
+		This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 35
+	expect resp.body == {
+		
+		This is a test: Hello world
+	}
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
+varnish v1 -expect MAIN.s_resp_bodybytes == 35
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 35
+		expect resp.body == {
+		
+		This is a test: Hello world
+	}
+	} -run
+} -run

src/tests/e00003.vtc

+varnishtest "ESI include"
+
+
+server s1 {
+	rxreq
+	expect req.http.esi0 == "foo"
+	txresp -body {
+		<html>
+		Before include
+		<esi:include src="/body" sr="foo"/>
+		After include
+		</html>
+	}
+} -start
+
+server s2 {
+	rxreq
+	expect req.url == "/body1"
+	expect req.http.esi0 != "foo"
+	txresp -body {
+		Included file
+	}
+} -start
+
+varnish v1 -arg "-p debug=+syncvsl" -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_recv {
+		if (req.esi_level > 0) {
+			set req.url = req.url + req.esi_level;
+		} else {
+			set req.http.esi0 = "foo";
+		}
+	}
+
+	sub vcl_backend_fetch {
+		if (bereq.url == "/") {
+			set bereq.backend = s1;
+		}
+		else {
+			set bereq.backend = s2;
+		}
+	}
+
+	sub vcl_backend_response {
+		if (bereq.url == "/") {
+			set beresp.do_esi = true;
+		}
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+		set resp.http.can_esi = obj.can_esi;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "Host: foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 75
+	expect resp.http.can_esi == "true"
+	expect resp.body == {
+		<html>
+		Before include
+		
+		Included file
+	
+		After include
+		</html>
+	}
+
+	delay .1
+	# test that there is no difference on miss/hit
+	txreq -hdr "Host: foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 75
+	expect resp.http.can_esi == "true"
+	expect resp.body == {
+		<html>
+		Before include
+		
+		Included file
+	
+		After include
+		</html>
+	}
+}
+
+# Because ReqAcct includes chunk headers, depending on the order of
+# events and whether or not we use (partial) sequential delivery (for
+# example, when no threads are available), ReqAcct adds n x 8 to the
+# net data size.
+#
+# in this test case we see either one or two chunk headers in addition
+# to the end chunk.
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
+varnish v1 -expect MAIN.s_resp_bodybytes == 150
+delay 1
+
+logexpect l1 -v v1 -d 1 -g vxid -q "vxid == 1001" {
+	expect 0 1001   Begin   "^req .* rxreq"
+	expect * =	ReqAcct	"^29 0 29 202 75 277$"
+	expect 0 =      End
+} -run
+
+logexpect l2 -v v1 -d 1 -g vxid -q "vxid == 1002" {
+	expect * 1002   Begin   "^bereq "
+	expect * =      End
+} -run
+
+logexpect l3 -v v1 -d 1 -g vxid -q "vxid == 1003" {
+	expect * 1003   Begin   "^req .* esi"
+	expect * =	ReqAcct	"^0 0 0 0 18 18$"
+	expect 0 =      End
+} -run
+
+logexpect l4 -v v1 -d 1 -g vxid -q "vxid == 1004" {
+	expect * 1004   Begin   "^bereq "
+	expect * =      End
+} -run
+
+
+logexpect l5 -v v1 -d 1 -g vxid -q "vxid == 1005" {
+	expect * 1005   Begin   "^req .* rxreq"
+	# Header bytes is 5 larger than in l1 due to two item X-Varnish hdr
+	expect * =	ReqAcct	"^29 0 29 207 75 282$"
+	expect 0 =      End
+} -run
+
+logexpect l6 -v v1 -d 1 -g vxid -q "vxid == 1006" {
+	expect * 1006   Begin   "^req .* esi"
+	expect * =	ReqAcct	"^0 0 0 0 18 18$"
+	expect 0 =      End
+} -run
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq -hdr host foo
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 75
+		expect resp.body == {
+		<html>
+		Before include
+		
+		Included file
+	
+		After include
+		</html>
+	}
+	} -run
+} -run

src/tests/e00004.vtc

+varnishtest "ESI commented include"
+
+
+server s1 {
+	rxreq
+	txresp -body {
+		<html>
+		Before include
+		<!--esi <esi:include src="/body"/> -->
+		After include
+	}
+	rxreq
+	expect req.url == "/body"
+	txresp -body {
+		Included file
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_response {
+		if (bereq.url != "/body") {
+			set beresp.do_esi = true;
+		}
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 67
+	expect resp.body == {
+		<html>
+		Before include
+		 
+		Included file
+	 
+		After include
+	}
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
+varnish v1 -expect MAIN.s_resp_bodybytes == 67
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 67
+		expect resp.body == {
+		<html>
+		Before include
+		 
+		Included file
+	 
+		After include
+	}
+	} -run
+} -run

src/tests/e00005.vtc

+varnishtest "ESI relative include"
+
+
+server s1 {
+	rxreq
+	expect req.url == "/foo/bar"
+	txresp -body {
+		<html>
+		Before include
+		<!--esi <esi:include src="body"/> -->
+		After include
+	}
+	rxreq
+	expect req.url == "/foo/body"
+	txresp -body {
+		Included file
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_response {
+		if (bereq.url != "/foo/body") {
+			set beresp.do_esi = true;
+		}
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+} -start
+
+client c1 {
+	txreq -url /foo/bar
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 67
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
+varnish v1 -expect MAIN.s_resp_bodybytes == 67
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq -url /foo/bar
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 67
+		expect resp.body == {
+		<html>
+		Before include
+		 
+		Included file
+	 
+		After include
+	}
+	} -run
+} -run

src/tests/e00006.vtc

+varnishtest "ESI include with http://"
+
+
+server s1 {
+	rxreq
+	expect req.url == "/foo/bar"
+	txresp -body {
+		<html>
+		Before include
+		<!--esi <esi:include src="http://bozz/body"/> -->
+		After include
+	}
+} -start
+
+
+server s2 {
+	rxreq
+	expect req.url == "/body"
+	txresp -body {
+		<pre>Included file</pre>
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_fetch {
+		if (bereq.http.host == "bozz") {
+			set bereq.backend = s2;
+		} else {
+			set bereq.backend = s1;
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+} -start
+
+client c1 {
+	txreq -url /foo/bar -hdr "Host: froboz"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 78
+	expect resp.body == {
+		<html>
+		Before include
+		 
+		<pre>Included file</pre>
+	 
+		After include
+	}
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
+varnish v1 -expect MAIN.s_resp_bodybytes == 78
+
+# Now try with invalid URLs
+
+server s1 {
+	rxreq
+	expect req.url == /http
+	txresp -body {<esi:include src="http://foobar" />1234}
+	rxreq
+	expect req.url == /https
+	txresp -body {<esi:include src="https://foobar" />123456}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		set req.backend_hint = s2;
+		set req.backend_hint = s1;
+	}
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+}
+
+varnish v1 -cliok "param.set feature +esi_ignore_https"
+
+logexpect l1 -v v1 -g raw {
+	expect * * ESI_xmlerror "ERR after 0 ESI 1.0 <esi:include> invalid src= URL"
+	expect * * ESI_xmlerror "WARN after 0 ESI 1.0 <esi:include> https:// treated as http://"
+	expect * * ESI_xmlerror "ERR after 0 ESI 1.0 <esi:include> invalid src= URL"
+} -start
+
+client c1 {
+	txreq -url /http
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+	expect resp.body == "1234"
+} -run
+
+varnish v1 -expect esi_errors == 1
+varnish v1 -expect MAIN.s_resp_bodybytes == 82
+
+client c1 {
+	txreq -url /https
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.body == "123456"
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -expect esi_errors == 2
+varnish v1 -expect MAIN.s_resp_bodybytes == 88
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq -url /foo/bar -hdr host froboz
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 78
+		expect resp.body == {
+		<html>
+		Before include
+		 
+		<pre>Included file</pre>
+	 
+		After include
+	}
+	} -run
+	delay .1
+	stream 3 {
+		txreq -url /http
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 4
+		expect resp.body == "1234"
+	} -run
+	delay .1
+	stream 5 {
+		txreq -url /https
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 6
+		expect resp.body == "123456"
+	} -run
+} -run

src/tests/e00013.vtc

+varnishtest "All white-space object, in multiple storage segments"
+
+server s1 {
+	loop 2 {
+		rxreq
+		expect req.url ~ {^/foo\d$}
+		txresp -nolen -hdr "Transfer-Encoding: chunked"
+		chunked {<x>                                      }
+		chunkedlen 0
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_pesi};
+	import ${vmod_pesi_debug};
+	include "debug.inc.vcl";
+
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+
+	sub vcl_deliver {
+		pesi.activate();
+	}
+} -start
+
+varnish v1 -cliok "debug.fragfetch 4"
+
+client c1 {
+	txreq -url /foo1
+	rxresp
+	expect resp.bodylen == 41
+	expect resp.body == {<x>                                      }
+} -run
+
+varnish v1 -expect esi_errors == 0
+
+## HTTP/2
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq -url /foo2
+		rxresp
+		expect resp.status == 200
+		expect resp.bodylen == 41
+		expect resp.body == {<x>                                      }
+	} -run
+} -run