initial public release

parent 86cdd8ce
((c-mode . ((indent-tabs-mode . t)
(c-file-style . "BSD"))))
......@@ -38,3 +38,10 @@ vmod_*.rst
*_options.rst
*_synopsis.rst
vmod_*.3
# extracted headers
src/vmod_blob.h
src/vmod_blobdigest.h
# header extractor
src/vmod_get_h
========
vmod-tus
========
This version is for Varnish-Cache master as of 2020-08-17
(post-6.4.0).
DESCRIPTION
===========
.. _tus: https://tus.io/protocols/resumable-upload.html
.. _vmod_blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest
This vmod implements a `tus`_ proxy which collects uploads on the
varnish server to send them to a backend in one go for storage. It
does not implement any permanent storage itself.
Besides the basic resumable uploads specified as the `tus`_ core
protocol, all currently defined extensions are supported, in
particular concatenation uploads.
See vmod documentation/man page for details on how to use it.
INSTALLATION
============
The source tree is based on autotools to configure the building, and
does also have the necessary bits in place to do functional unit tests
using the ``varnishtest`` tool.
Besides the Varnish header files (as located via ``pkg-config``), this
vmod also requires
* The ``VARNISHSRC`` Variable to point to the varnish-cache sources
used for building the installed varnish version.
* `vmod_blobdigest`_ to be installed before building this vmod if
hashes are to be used.
This vmod will need to be rebuild specifically for each version of
varnish and `vmod_blobdigest`_ as it requires internal interfaces.
Usage::
export VARNISHSRC=/path/to/your/varnish/sources
./autogen.sh
./configure
If you have installed Varnish to a non-standard directory, call
``autogen.sh`` and ``configure`` with ``PKG_CONFIG_PATH`` pointing to
the appropriate path. For instance, when varnishd configure was called
with ``--prefix=$PREFIX``, use
::
export PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig
export ACLOCAL_PATH=${PREFIX}/share/aclocal
The module will inherit its prefix from Varnish, unless you specify a
different ``--prefix`` when running the ``configure`` script for this
module.
Make targets:
* make - builds the vmod.
* make install - installs your vmod.
* make check - runs the unit tests in ``src/vtc/*.vtc``.
* make distcheck - run check and prepare a tarball of the vmod.
If you build a dist tarball, you don't need any of the autotools or
pkg-config. You can build the module simply by running::
./configure
make
Installation directories
------------------------
By default, the vmod ``configure`` script installs the built vmod in the
directory relevant to the prefix. The vmod installation directory can be
overridden by passing the ``vmoddir`` variable to ``make install``.
......@@ -3,6 +3,7 @@ AC_INIT([libvmod-tus], [0.1])
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_AUX_DIR([build-aux])
AC_CONFIG_HEADER([config.h])
AC_GNU_SOURCE
AM_INIT_AUTOMAKE([1.12 -Wall -Werror foreign parallel-tests])
AM_SILENT_RULES([yes])
......@@ -20,6 +21,22 @@ AC_ARG_WITH([rst2man],
VARNISH_PREREQ([6.0.0])
VARNISH_VMODS([tus])
AC_ARG_VAR([VARNISHSRC], [path to Varnish source])
if test "x$VARNISHSRC" = x; then
AC_MSG_FAILURE([Need VARNISHSRC])
fi
VARNISHAPI_CFLAGS="$VARNISHAPI_CFLAGS -I$VARNISHSRC/bin/varnishd"
AC_PATH_TOOL(VMOD_BLOB, [libvmod_blob.so],
[AC_MSG_WARN([vmod_blob not found, checksum support will be disabled])],
[${VARNISHAPI_VMODDIR}:${vmoddir}])
AC_SUBST(VMOD_BLOB)
AC_PATH_TOOL(VMOD_BLOBDIGEST, [libvmod_blobdigest.so],
[AC_MSG_WARN([vmod_blob not found, checksum support will be disabled])],
[${VARNISHAPI_VMODDIR}:${vmoddir}])
AC_SUBST(VMOD_BLOBDIGEST)
AM_CONDITIONAL(CHKSUM, test x$VMOD_BLOB != x && test x$VMOD_BLOBDIGEST != x)
AC_CONFIG_FILES([
Makefile
......@@ -40,4 +57,7 @@ AS_ECHO("
compiler: $CC
cflags: $CFLAGS
ldflags: $LDFLAGS
blob: $VMOD_BLOB
blobdigest: $VMOD_BLOBDIGEST
")
AM_CFLAGS = $(VARNISHAPI_CFLAGS)
# Modules
# ----------------------------------------
# get header from ext vmod
bin_PROGRAMS = vmod_get_h
vmod_get_h_SOURCES = \
vmod_get_h.c
vmod_get_h_LDADD = -ldl
# ----------------------------------------
# ext vmods
if CHKSUM
vmod_blob.h: vmod_get_h $(VMOD_BLOB)
$(builddir)/vmod_get_h blob $(VMOD_BLOB) >$@
vmod_blobdigest.h: vmod_get_h $(VMOD_BLOBDIGEST)
$(builddir)/vmod_get_h blobdigest $(VMOD_BLOBDIGEST) >$@
tus_blob.c: vmod_blob.h vmod_blobdigest.h
libvmod_tus_la_CFLAGS = $(AM_CFLAGS) -DHAVE_CHKSUM=1
endif
# ----------------------------------------
# vmod
vmod_LTLIBRARIES = \
libvmod_tus.la
libvmod_tus_la_LDFLAGS = $(VMOD_LDFLAGS)
libvmod_tus_la_SOURCES = vmod_tus.c
libvmod_tus_la_SOURCES = \
tbl_hash_enum.h \
tbl_method.h \
tus_b64.h \
tus_blob.c \
tus_blob.h \
tus_concat.h \
tus_file.c \
tus_file.h \
tus_file_exp.c \
tus_file_exp.h \
tus_hdr.c \
tus_hdr.h \
tus_hex.h \
tus_request.c \
tus_request.h \
tus_response.c \
tus_response.h \
tus_server.h \
tus_servers.c \
tus_servers.h \
tus_stv.c \
tus_stv.h \
vmod_tus.c
nodist_libvmod_tus_la_SOURCES = \
vcc_tus_if.c \
vcc_tus_if.h
......@@ -19,13 +66,16 @@ AM_TESTS_ENVIRONMENT = \
PATH="$(abs_builddir):$(VARNISH_TEST_PATH):$(PATH)" \
LD_LIBRARY_PATH="$(VARNISH_LIBRARY_PATH)"
TEST_EXTENSIONS = .vtc
VTC_LOG_COMPILER = varnishtest -v
VTC_LOG_COMPILER = varnishtest -v -t 10
AM_VTC_LOG_FLAGS = \
-p vcl_path="$(abs_top_srcdir)/vcl:$(VARNISHAPI_VCLDIR)" \
-p vmod_path="$(abs_builddir)/.libs:$(vmoddir):$(VARNISHAPI_VMODDIR)"
TESTS = \
vtc/vmod_tus.vtc
vtc/plain.vtc \
vtc/name_hash.vtc \
vtc/expires.vtc \
vtc/done.vtc
# Documentation
......
// HENUM(tusname, blobdigestenum)
HENUM(crc32, CRC32)
HENUM(icrc32, ICRC32)
HENUM(md5, MD5)
HENUM(rs, RS)
HENUM(sha1, SHA1)
HENUM(sha224, SHA224)
HENUM(sha256, SHA256)
HENUM(sha384, SHA384)
HENUM(sha3_224, SHA3_224)
HENUM(sha3_256, SHA3_256)
HENUM(sha3_384, SHA3_384)
HENUM(sha3_512, SHA3_512)
HENUM(sha512, SHA512)
#undef HENUM
MET(OPTIONS)
MET(POST)
MET(HEAD)
MET(PATCH)
MET(DELETE)
MET(GET)
#undef MET
VCL_BLOB tus_b64_decode(VRT_CTX, const char *s, VCL_INT l);
#include "config.h"
#include <dlfcn.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "cache/cache.h"
#include "vsb.h"
#include "vmod_blob.h"
#include "vmod_blobdigest.h"
#include "tus_b64.h"
#include "tus_blob.h"
static const struct Vmod_vmod_blob_Func *vmod_blob = NULL;
static const struct Vmod_vmod_blobdigest_Func *vmod_blobdigest = NULL;
static void *dl_vmod_blob = NULL;
static void *dl_vmod_blobdigest = NULL;
/* ------------------------------------------------------------
* linkage to blob and blobdigest vmods
*/
static void
load_vmod(const struct vmod_data *data, void **hdl, const void **func)
{
const struct vmod_data *d;
void *dlhdl;
char buf[256];
bprintf(buf, "./vmod_cache/_vmod_%s.%s",
data->name, data->file_id);
dlhdl = dlopen(buf, RTLD_LAZY | RTLD_LOCAL);
if (dlhdl == NULL) {
fprintf(stderr, "dlopen vmod_%s: %s\n", data->name, dlerror());
return;
}
bprintf(buf, "Vmod_%s_Data", data->name);
d = dlsym(dlhdl, buf);
if (d == NULL) {
fprintf(stderr, "dlsym vmod_%s: %s\n", data->name, dlerror());
(void) dlclose(dlhdl);
return;
}
if (strcmp(d->name, data->name) ||
strcmp(d->file_id, data->file_id) ||
strcmp(d->abi, data->abi)) {
fprintf(stderr, "vmod_%s: name/id/abi mismatch\n", data->name);
(void) dlclose(dlhdl);
return;
}
*hdl = dlhdl;
*func = d->func;
fprintf(stderr, "vmod_%s OK\n", data->name);
}
#define HENUM(t, b) static struct vmod_blobdigest_digest *digest_ ## t = NULL;
#include "tbl_hash_enum.h"
static unsigned enabled = 0;
static char *hashes = NULL;
int
tus_chksum_init(VRT_CTX) {
struct vsb names[1];
char buf[512];
int first = 1;
AN(VSB_new(names, buf, sizeof buf, VSB_FIXEDLEN));
load_vmod(&import_Vmod_blob_Data, &dl_vmod_blob,
(const void **)&vmod_blob);
load_vmod(&import_Vmod_blobdigest_Data, &dl_vmod_blobdigest,
(const void **)&vmod_blobdigest);
if (vmod_blobdigest != NULL && vmod_blob != NULL)
enabled = 1;
else
return (0);
fprintf(stderr, "enabled %u\n", enabled);
#define HENUM(t, b) do { \
vmod_blobdigest->digest__init(ctx, &digest_ ## t, \
"tus_digest_" #t, *vmod_blobdigest->enum_ ## b, NULL, \
*vmod_blobdigest->enum_TASK); \
if (digest_ ## t == NULL) \
break; \
if (first == 0) \
VSB_cat(names, ","); \
VSB_cat(names, #t); \
first = 0; \
} while (0);
#include "tbl_hash_enum.h"
AZ(VSB_finish(names));
REPLACE(hashes, VSB_data(names));
fprintf(stderr, "digests OK\n");
// dummy ref to appease compiler
memset(&Vmod_vmod_blobdigest_Func, 0, sizeof Vmod_vmod_blobdigest_Func);
memset(&Vmod_vmod_blob_Func, 0, sizeof Vmod_vmod_blob_Func);
return (0);
}
int
tus_chksum_fini(VRT_CTX) {
#define HENUM(t, b) if (digest_ ## t != NULL) \
vmod_blobdigest->digest__fini(&digest_ ## t);
#include "tbl_hash_enum.h"
REPLACE(hashes, NULL);
if (dl_vmod_blob != NULL)
(void) dlclose(dl_vmod_blob);
if (dl_vmod_blobdigest != NULL)
(void) dlclose(dl_vmod_blobdigest);
return (0);
}
/* ------------------------------------------------------------
* wrap base64
*/
VCL_BLOB
tus_b64_decode(VRT_CTX, const char *s, VCL_INT l)
{
struct strands st = {.n = 1, .p = &s};
if (! enabled)
return (NULL);
AN(vmod_blob);
AN(vmod_blob->decode);
AN(vmod_blob->enum_BASE64);
return (vmod_blob->decode(ctx, *vmod_blob->enum_BASE64, l, &st));
}
/* ------------------------------------------------------------
* enabled checksums
*/
const char *
tus_checksums(void)
{
return (enabled ? hashes : NULL);
}
/* ------------------------------------------------------------
* handling of tus checksums
*/
struct tus_chksum {
unsigned magic;
#define VMOD_TUS_CHKSUM_MAGIC 0x105c6650
const struct vmod_blobdigest_digest *digest;
VCL_BLOB expect;
VCL_BLOB final;
};
/* tus hash names to blobdiget object */
const struct vmod_blobdigest_digest *
tus_hash(const char *s, size_t l)
{
if (enabled == 0 || s == NULL)
return (NULL);
if (l == 0)
l = strlen(s);
#define HENUM(t, b) \
if (strncmp(s, #t, l) == 0) return (digest_ ## t);
#include "tbl_hash_enum.h"
return (NULL);
}
struct tus_chksum *
tus_chksum_new(VRT_CTX, const struct vmod_blobdigest_digest *d)
{
struct tus_chksum *r;
AN(d);
r = WS_Alloc(ctx->ws, sizeof *r);
if (r == NULL) {
VRT_fail(ctx, "WS_Alloc failed");
return (NULL);
}
INIT_OBJ(r, VMOD_TUS_CHKSUM_MAGIC);
r->digest = d;
return (r);
}
struct tus_chksum *
tus_chksum_hdr(VRT_CTX, const char *hdr)
{
const struct vmod_blobdigest_digest *d;
struct tus_chksum *r;
VCL_BLOB expect;
const char *sep;
if (enabled == 0 || hdr == NULL)
return (NULL);
sep = strchr(hdr, ' ');
if (sep == NULL)
return (NULL);
d = tus_hash(hdr, (unsigned)(sep - hdr));
if (d == NULL)
return (NULL);
r = tus_chksum_new(ctx, d);
if (r == NULL)
return (NULL);
sep++;
expect = tus_b64_decode(ctx, sep, 0);
if (expect == NULL)
return (NULL);
r->expect = expect;
return (r);
}
void
tus_chksum_update(VRT_CTX, struct tus_chksum *c,
const void *ptr, size_t l)
{
struct vrt_blob b;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(c, VMOD_TUS_CHKSUM_MAGIC);
AN(enabled);
b.type = 0x105;
b.len = l;
b.blob = ptr;
(void) vmod_blobdigest->digest_update(ctx, c->digest, &b);
}
VCL_BLOB
tus_chksum_final(VRT_CTX, struct tus_chksum *c)
{
AN(enabled);
if (c->final == NULL)
c->final = vmod_blobdigest->digest_final(ctx, c->digest);
return (c->final);
}
VCL_BOOL
tus_chksum_equal(VRT_CTX, struct tus_chksum *c)
{
(void) tus_chksum_final(ctx, c);
return (vmod_blob->equal(ctx, c->expect, c->final));
}
// ------------------------------------------------------------
// not using vmod_blob because upload filenames should work without it
static inline char
hexnibble(unsigned char n)
{
return (n < 0xa ? '0' + n : 'a' + n - 0xa);
}
VCL_STRING
tus_hex_buf(char * buf, size_t bufl, VCL_BLOB b)
{
VCL_STRING r = buf;
const unsigned char *p;
unsigned char n;
size_t l;
if (b == NULL || b->len == 0)
return ("");
p = b->blob;
l = b->len;
while (l > 0) {
if (bufl < 3)
return (NULL);
n = *p;
buf[1] = hexnibble(n & 0xf);
n >>= 4;
buf[0] = hexnibble(n & 0xf);
buf += 2;
bufl -= 2;
p++;
l--;
}
AN(bufl);
buf[0] = '\0';
return (r);
}
VCL_STRING
tus_hex(VRT_CTX, VCL_BLOB b)
{
char * buf;
size_t bufl;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
if (b == NULL || b->len == 0)
return ("");
bufl = b->len * 2 + 1;
buf = WS_Alloc(ctx->ws, bufl);
if (buf == NULL)
return (NULL);
return (tus_hex_buf(buf, bufl, b));
}
void
tus_vsbhex(struct vsb *vsb, VCL_BLOB b)
{
size_t l = b->len * 2 + 1;
char buf[l];
VSB_cat(vsb, tus_hex_buf(buf, l, b));
}
int tus_chksum_init(VRT_CTX);
int tus_chksum_fini(VRT_CTX);
const struct vmod_blobdigest_digest *tus_hash(const char *, size_t);
struct tus_chksum *tus_chksum_new(VRT_CTX,
const struct vmod_blobdigest_digest *);
struct tus_chksum *tus_chksum_hdr(VRT_CTX, const char *);
void tus_chksum_update(VRT_CTX, struct tus_chksum *c,
const void *ptr, size_t l);
VCL_BLOB tus_chksum_final(VRT_CTX, struct tus_chksum *c);
VCL_BOOL tus_chksum_equal(VRT_CTX, struct tus_chksum *c);
#define MAX_CONCAT 64
#define READY_TIMEOUT 60 // parameter?
struct tus_concat {
unsigned magic;
#define TUS_CONCAT_MAGIC 0x105c0ca7
unsigned n;
struct tus_file_core *cores[];
// dynamically sized
};
This diff is collapsed.
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include <vtree.h>
#include <vtim.h>
#include "tus_server.h"
#define TUS_PATH_MAX PATH_MAX
#define TUS_METADATA_MAX 2048u // ballpark of AWS S3 metadata
enum tus_f_type {
TUS_SINGLE = 0,
TUS_PARTIAL,
TUS_FINAL,
TUS_REDIRECT
};
// header of disk file - mmaped
struct tus_file_disk {
unsigned magic;
#define VMOD_TUS_FILE_DISK_MAGIC 0x105f11ed
unsigned version;
char url_path[TUS_PATH_MAX];
unsigned guard_magic;
unsigned url_path_length;
// for TUS_REDIRECT, repurposed for location
char metadata[TUS_METADATA_MAX];
unsigned guard2_magic;
unsigned metadata_length;
ssize_t upload_length; // -1 == defer
ssize_t upload_offset;
VCL_TIME upload_expires;
enum tus_f_type type;
};
struct tus_file_core {
unsigned magic;
#define VMOD_TUS_FILE_CORE_MAGIC 0x105f11e0
int fd;
char *filename; // at basedir
struct VPFX(tus_server) *server;
VSPLAY_ENTRY(tus_file_core) entry;
pthread_mutex_t mtx;
pthread_cond_t cond;
struct tus_file_disk *disk;
unsigned refcnt;
unsigned exp_idx;
/* final mmap or concats list */
void *ptr;
size_t len;
};
static inline const char *
tus_file_disk_err(const struct tus_file_disk *d)
{
if (d->magic != VMOD_TUS_FILE_DISK_MAGIC)
return ("bad magic");
if (d->guard_magic != VMOD_TUS_FILE_DISK_MAGIC)
return ("bad guard_magic");
if (d->guard2_magic != VMOD_TUS_FILE_DISK_MAGIC)
return ("bad guard2_magic");
if (d->version != 1)
return ("version != 1");
if (strnlen(d->url_path, TUS_PATH_MAX) != d->url_path_length)
return ("url_path_length mismatch");
if (strnlen(d->metadata, TUS_METADATA_MAX) != d->metadata_length)
return ("metadata_length mismatch");
return (NULL);
}
#define CHECK_TUS_FILE_DISK(x) do { \
const char *err; \
AN(x); \
err = tus_file_disk_err(x); \
if (err != NULL) \
WRONG(err); \
} while(0);
static inline int
tus_file_cmp(const struct tus_file_core *a,
const struct tus_file_core *b) {
const struct tus_file_disk *aa, *bb;
CHECK_OBJ_NOTNULL(a, VMOD_TUS_FILE_CORE_MAGIC);
CHECK_OBJ_NOTNULL(b, VMOD_TUS_FILE_CORE_MAGIC);
aa = a->disk;
bb = b->disk;
CHECK_TUS_FILE_DISK(aa);
CHECK_TUS_FILE_DISK(bb);
return (strcmp(aa->url_path, bb->url_path));
};
VSPLAY_HEAD(tus_files, tus_file_core);
VSPLAY_PROTOTYPE(tus_files, tus_file_core, entry, tus_file_cmp);
// tus_file.c
void tus_file_init(void);
void tus_file_load(struct VPFX(tus_server) *);
void tus_file_shutdown(struct VPFX(tus_server) *srv);
void tus_file_del(struct tus_file_core **);
struct tus_file_core *tus_file_new(VRT_CTX, struct VPFX(tus_server) *,
enum tus_f_type, const char *, const char *, const char *);
struct tus_file_core *tus_file_lookup(struct VPFX(tus_server) *, const char *);
unsigned tus_body_to_file(VRT_CTX, struct tus_file_core *);
void tus_file_mmap(struct tus_file_core *);
unsigned tus_file_ref(struct tus_file_core *);
unsigned tus_file_unref(struct tus_file_core *);
#ifdef BAD_IDEA
void
tus_file_rename_base(struct tus_file_core *fcore, const char *basename);
#endif
const char *
tus_file_final_urls(VRT_CTX, const struct tus_file_core *fcore,
const char *pfx);
struct concat_embryo {
unsigned magic;
#define CONCAT_EMBRYO_MAGIC 0x150c05e5
// to be written to fcore->fd once we have it
struct VPFX(tus_server) *srv;
struct vsb *spec_vsb;
struct tus_concat *concat;
size_t concat_l;
ssize_t upload_length;
};
struct concat_embryo * tus_file_final_concat(struct VPFX(tus_server) *srv,
struct concat_embryo *embryo, const char *spec);
VCL_BLOB tus_concat_hash(VRT_CTX, const struct VPFX(tus_server) *,
const struct tus_concat *);
struct tus_file_core *
tus_file_final_birth(struct tus_file_core *fcore, struct concat_embryo *embryo);
void
tus_file_final_abort(struct concat_embryo *embryo);
VCL_BOOL
tus_file_done(struct tus_file_core *, struct tus_file_disk *, const char *);
#include "config.h"
#include <pthread.h>
#include <math.h>
#include "vdef.h"
#include "vrt.h"
#include "miniobj.h"
#include "vas.h"
#include "vbh.h"
#include "tus_file.h"
#include "tus_file_exp.h"
struct tus_exp {
unsigned magic;
#define TUS_EXP_MAGIC 0x105e8900
unsigned die;
struct vbh *heap;
pthread_mutex_t mtx;
pthread_cond_t cond;
pthread_t thread;
};
static inline vtim_real
fcore_when(const void *p)
{
const struct tus_file_core *fcore;
const struct tus_file_disk *fdisk;
CAST_OBJ_NOTNULL(fcore, p, VMOD_TUS_FILE_CORE_MAGIC);
fdisk = fcore->disk;
CHECK_OBJ_NOTNULL(fdisk, VMOD_TUS_FILE_DISK_MAGIC);
return (fdisk->upload_expires);
}
void *
tus_exp_thread(void *p)
{
struct VPFX(tus_server) *srv;
struct tus_file_core *fcore;
struct timespec ts;
struct tus_exp *e;
vtim_real now, when, t;
CAST_OBJ_NOTNULL(e, p, TUS_EXP_MAGIC);
pthread_mutex_lock(&e->mtx);
while (! e->die) {
fcore = VBH_root(e->heap);
if (fcore == NULL) {
pthread_cond_wait(&e->cond, &e->mtx);
continue;
}
when = fcore_when(fcore);
now = VTIM_real();
if (when > now) {
assert(when > 1e9);
ts.tv_nsec = (long)(modf(when, &t) * 1e9);
ts.tv_sec = (long)t;
assert(ts.tv_nsec >= 0 && ts.tv_nsec < 999999999);
errno = pthread_cond_timedwait(&e->cond, &e->mtx, &ts);
assert(errno == 0 || errno == ETIMEDOUT ||
errno == EINTR);
continue;
}
pthread_mutex_unlock(&e->mtx);
srv = fcore->server;
tus_server_lock(srv);
AZ(pthread_mutex_lock(&fcore->mtx));
tus_file_del(&fcore);
tus_server_unlock(srv);
pthread_mutex_lock(&e->mtx);
}
pthread_mutex_unlock(&e->mtx);
}
void
tus_exp_delete(const struct tus_file_core *fcore)
{
struct tus_exp *e = tus_server_exp(fcore->server);
CHECK_OBJ_NOTNULL(e, TUS_EXP_MAGIC);
pthread_mutex_lock(&e->mtx);
assert(fcore->exp_idx != VBH_NOIDX);
VBH_delete(e->heap, fcore->exp_idx);
pthread_mutex_unlock(&e->mtx);
}
void
tus_exp_insert(struct tus_file_core *fcore)
{
struct tus_exp *e = tus_server_exp(fcore->server);
CHECK_OBJ_NOTNULL(e, TUS_EXP_MAGIC);
pthread_mutex_lock(&e->mtx);
VBH_insert(e->heap, fcore);
if (VBH_root(e->heap) == fcore)
pthread_cond_signal(&e->cond);
pthread_mutex_unlock(&e->mtx);
}
void
tus_exp_touch(const struct tus_file_core *fcore)
{
struct tus_exp *e = tus_server_exp(fcore->server);
CHECK_OBJ_NOTNULL(e, TUS_EXP_MAGIC);
pthread_mutex_lock(&e->mtx);
VBH_reorder(e->heap, fcore->exp_idx);
pthread_mutex_unlock(&e->mtx);
}
static int
tus_exp_cmp(void *priv, const void *a, const void *b) {
(void) priv;
return (fcore_when(a) < fcore_when(b));
}
static void
tus_exp_update(void *priv, void *p, unsigned u)
{
struct tus_file_core *fcore;
(void) priv;
CAST_OBJ_NOTNULL(fcore, p, VMOD_TUS_FILE_CORE_MAGIC);
fcore->exp_idx = u;
}
struct tus_exp *
tus_file_exp_new(void)
{
struct tus_exp *e;
pthread_attr_t attr;
ALLOC_OBJ(e, TUS_EXP_MAGIC);
AN(e);
e->heap = VBH_new(NULL, tus_exp_cmp, tus_exp_update);
AN(e->heap);
AZ(pthread_mutex_init(&e->mtx, NULL));
AZ(pthread_cond_init(&e->cond, NULL));
AZ(pthread_attr_init(&attr));
AZ(pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN));
AZ(pthread_create(&e->thread, &attr, tus_exp_thread, e));
AZ(pthread_attr_destroy(&attr));
return (e);
}
void
tus_file_exp_destroy(struct tus_exp **ep)
{
struct tus_exp *e;
TAKE_OBJ_NOTNULL(e, ep, TUS_EXP_MAGIC);
AN(e->heap);
AZ(VBH_root(e->heap));
e->die = 1;
AZ(pthread_cond_signal(&e->cond));
AZ(pthread_join(e->thread, NULL));
AZ(pthread_cond_destroy(&e->cond));
AZ(pthread_mutex_destroy(&e->mtx));
free(e);
}
void tus_exp_delete(const struct tus_file_core *fcore);
void tus_exp_insert(struct tus_file_core *fcore);
void tus_exp_touch(const struct tus_file_core *fcore);
const char * const hdr_resum = "\016Tus-Resumable:";
const char * const hdr_vers = "\014Tus-Version:";
const char * const hdr_ext = "\016Tus-Extension:";
const char * const hdr_chkalg = "\027Tus-Checksum-Algorithm:";
const char * const hdr_chksum = "\020Upload-Checksum:";
const char * const hdr_concat = "\016Upload-Concat:";
const char * const hdr_max = "\015Tus-Max-Size:";
const char * const hdr_defer = "\024Upload-Defer-Length:";
const char * const hdr_meta = "\020Upload-Metadata:";
const char * const hdr_exp = "\017Upload-Expires:";
const char * const hdr_loc = "\011Location:";
const char * const hdr_method = "\027X-HTTP-Method-Override:";
const char * const hdr_off = "\016Upload-Offset:";
const char * const hdr_len = "\016Upload-Length:";
const char * const hdr_origin = "\007Origin:";
const char * const hdr_acah = "\035Access-Control-Allow-Headers:";
const char * const hdr_acam = "\035Access-Control-Allow-Methods:";
const char * const hdr_acma = "\027Access-Control-Max-Age:";
const char * const hdr_acao = "\034Access-Control-Allow-Origin:";
const char * const hdr_aceh = "\036Access-Control-Expose-Headers:";
const char * const hdr_allow = "\006Allow:";
const char * const hdr_vtc = "\015VTC-Filename:";
const char * const hdr_resum;
const char * const hdr_vers;
const char * const hdr_ext;
const char * const hdr_concat;
const char * const hdr_chkalg;
const char * const hdr_chksum;
const char * const hdr_max;
const char * const hdr_defer;
const char * const hdr_meta;
const char * const hdr_exp;
const char * const hdr_loc;
const char * const hdr_method;
const char * const hdr_off;
const char * const hdr_len;
const char * const hdr_origin;
const char * const hdr_acah;
const char * const hdr_acam;
const char * const hdr_acma;
const char * const hdr_acao;
const char * const hdr_aceh;
const char * const hdr_allow;
const char * const hdr_vtc;
VCL_STRING tus_hex_buf(char *, size_t, VCL_BLOB);
VCL_STRING tus_hex(VRT_CTX, VCL_BLOB);
void tus_vsbhex(struct vsb *, VCL_BLOB);
This diff is collapsed.
VCL_BOOL
tus_request(VRT_CTX, struct VPFX(tus_server) *tussrv,
struct tus_response *r, const char *url, const char *id);
#include "config.h"
#include <sys/statvfs.h>
#include <cache/cache.h>
#include <vrt_obj.h>
#include "vcc_tus_if.h"
#include "tus_b64.h"
#include "tus_file.h"
#include "tus_servers.h"
#include "tus_hdr.h"
#include "tus_response.h"
/* tus_blob.c */
const char * tus_checksums(void);
const unsigned s_MULT = 1000;
const unsigned s_OPTIONS = 1;
const unsigned s_HEAD = 2;
const char * const allow = "POST, GET, HEAD, PATCH, DELETE, OPTIONS";
static VCL_BYTES
tus_upload_length(const struct VPFX(tus_server) *tussrv,
const struct tus_file_core *fcore)
{
VCL_BYTES max;
struct statvfs fs;
CHECK_OBJ_NOTNULL(tussrv, VMOD_TUS_SERVER_MAGIC);
if (fcore != NULL && fcore->fd >= 0) {
if (fstatvfs(fcore->fd, &fs))
return (tussrv->max);
} else if (statvfs(tussrv->basedir, &fs))
return (tussrv->max);
max = fs.f_bavail * fs.f_frsize;
return (max < tussrv->max ? max : tussrv->max);
}
static void
tus_cors(VCL_HTTP r, unsigned status, const char *origin)
{
http_ForceHeader(r, hdr_acao, origin);
// from tus go server pkg/handler/unrouted_handler.go
if (status / s_MULT == s_OPTIONS) {
http_ForceHeader(r, hdr_acam, allow);
http_ForceHeader(r, hdr_acah,
"Authorization, Origin, X-Requested-With, X-Request-ID, "
"X-HTTP-Method-Override, Content-Type, Upload-Length, "
"Upload-Offset, Tus-Resumable, Upload-Metadata, "
"Upload-Defer-Length, Upload-Concat");
http_ForceHeader(r, hdr_acma, "86400");
} else {
http_ForceHeader(r, hdr_aceh,
"Upload-Offset, Location, Upload-Length, Tus-Version, "
"Tus-Resumable, Tus-Max-Size, Tus-Extension, "
"Upload-Metadata, Upload-Defer-Length, Upload-Concat");
}
}
void
tus_response(VRT_CTX, const struct VPFX(tus_server) *tussrv,
struct tus_response *resp)
{
const struct tus_file_disk *fdisk = NULL;
const struct tus_file_core *fcore = NULL;
const char *chksums = tus_checksums();
VCL_HTTP r;
char t[VTIM_FORMAT_SIZE];
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(tussrv, VMOD_TUS_SERVER_MAGIC);
CHECK_OBJ_NOTNULL(resp, VMOD_TUS_RESPONSE_MAGIC);
if (resp->fcore) {
fcore = resp->fcore;
fdisk = fcore->disk;
AN(fdisk);
}
CHECK_OBJ_ORNULL(fdisk, VMOD_TUS_FILE_DISK_MAGIC);
r = ctx->http_resp;
CHECK_OBJ_NOTNULL(r, HTTP_MAGIC);
http_Unset(r, hdr_max);
http_Unset(r, hdr_off);
http_Unset(r, hdr_len);
http_Unset(r, hdr_defer);
http_Unset(r, hdr_meta);
http_Unset(r, hdr_exp);
http_Unset(r, hdr_loc);
if (resp->status == 301) {
AN(fdisk);
if (fdisk->metadata_length == 0)
resp->status = 410;
else if (fdisk->metadata[0] == '/')
http_PrintfHeader(r, "Location: %s%s", resp->schemeauth,
fdisk->metadata);
else
http_ForceHeader(r, hdr_loc, fdisk->metadata);
VRT_l_resp_status(ctx, resp->status);
return;
}
VRT_l_resp_status(ctx, resp->status);
if (resp->status == 405) {
http_ForceHeader(r, hdr_allow, allow);
return;
}
http_ForceHeader(r, hdr_resum, "1.0.0");
http_ForceHeader(r, hdr_vers, "1.0.0");
if (chksums != NULL) {
http_ForceHeader(r, hdr_ext, "creation,creation-with-upload,"
"expiration,termination,concatenation,checksum");
http_ForceHeader(r, hdr_chkalg, chksums);
} else {
http_ForceHeader(r, hdr_ext, "creation,creation-with-upload,"
"expiration,termination,concatenation");
}
if (resp->origin != NULL && *resp->origin != '\0')
tus_cors(r, resp->status, resp->origin);
if (tussrv) {
http_PrintfHeader(r, "Tus-Max-Size: %ju",
tus_upload_length(tussrv, fcore));
}
while (fdisk) {
AN(fcore);
if (fdisk->upload_offset >= 0)
http_PrintfHeader(r, "Upload-Offset: %zi",
fdisk->upload_offset);
if (fdisk->upload_length >= 0)
http_PrintfHeader(r, "Upload-Length: %zi",
fdisk->upload_length);
else if (fdisk->upload_length == -1)
http_ForceHeader(r, hdr_defer, "1");
else
INCOMPL();
if (fdisk->metadata_length != 0)
http_ForceHeader(r, hdr_meta, fdisk->metadata);
VTIM_format(fdisk->upload_expires, t);
http_ForceHeader(r, hdr_exp, t);
if (resp->status == 201) {
AN(resp->schemeauth);
http_PrintfHeader(r, "Location: %s%s", resp->schemeauth,
fdisk->url_path);
}
switch (fdisk->type) {
case TUS_SINGLE:
break;
case TUS_FINAL:
if (fcore->ptr == NULL)
break;
AN(resp->schemeauth);
http_ForceHeader(r, hdr_concat,
tus_file_final_urls(ctx,
resp->fcore, resp->schemeauth));
break;
case TUS_PARTIAL:
http_ForceHeader(r, hdr_concat, "partial");
break;
}
break;
}
}
VCL_BOOL
tus_done(VRT_CTX, const struct VPFX(tus_server) *tussrv,
const struct tus_response *resp, const char *url)
{
struct tus_file_core *fcore;
struct tus_file_disk *fdisk;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(tussrv, VMOD_TUS_SERVER_MAGIC);
CHECK_OBJ_NOTNULL(resp, VMOD_TUS_RESPONSE_MAGIC);
fcore = resp->fcore;
if (fcore == NULL)
return (0);
CHECK_OBJ_NOTNULL(fcore, VMOD_TUS_FILE_CORE_MAGIC);
fdisk = fcore->disk;
CHECK_OBJ_NOTNULL(fdisk, VMOD_TUS_FILE_DISK_MAGIC);
if (fdisk->type != TUS_SINGLE && fdisk->type != TUS_FINAL)
return (0);
return (tus_file_done(fcore, fdisk, url));
}
static inline const char *
tus_meta_find(const char *meta, const char *key, size_t l,
const char **vp, size_t *vlp)
{
const char *v, *ve;
while (meta != NULL && *meta != '\0') {
if (strncmp(meta, key, l) == 0) {
v = meta + l;
if (*v == '\0' || *v == ',' ||
(v[0] == ' ' && v[1] == ','))
return (meta);
if (*v == ' ') {
v++;
if (vp != NULL)
*vp = v;
if (vlp != NULL) {
ve = strchr(v, ',');
if (ve != NULL)
*vlp = ve - v;
else
*vlp = strlen(v);
}
return (meta);
}
}
meta = strchr(meta, ',');
if (meta != NULL)
meta++;
}
return (NULL);
}
VCL_BOOL
tus_meta(VRT_CTX, const struct tus_response *resp, const char *k, VCL_BLOB *b)
{
const struct tus_file_core *fcore;
const struct tus_file_disk *fdisk;
const char *meta, *v = NULL;
size_t kl = strlen(k), vl = 0;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(resp, VMOD_TUS_RESPONSE_MAGIC);
fcore = resp->fcore;
if (fcore == NULL)
return (0);
CHECK_OBJ_NOTNULL(fcore, VMOD_TUS_FILE_CORE_MAGIC);
fdisk = fcore->disk;
CHECK_OBJ_NOTNULL(fdisk, VMOD_TUS_FILE_DISK_MAGIC);
meta = fdisk->metadata;
if (b == NULL)
return (tus_meta_find(meta, k, kl, NULL, NULL) != NULL);
meta = tus_meta_find(meta, k, kl, &v, &vl);
if (meta == NULL)
return (0);
*b = tus_b64_decode(ctx, v, vl);
return (1);
}
struct tus_response {
unsigned magic;
#define VMOD_TUS_RESPONSE_MAGIC 0x1054e570
unsigned status;
const char *schemeauth; // from tussrv
const char *origin; // CORS
// if set, mtx is helt
struct tus_file_core *fcore;
};
void
tus_response(VRT_CTX, const struct VPFX(tus_server) *tussrv,
struct tus_response *resp);
VCL_BOOL
tus_done(VRT_CTX, const struct VPFX(tus_server) *tussrv,
const struct tus_response *resp, const char *url);
VCL_BOOL
tus_meta(VRT_CTX, const struct tus_response *resp, const char *k, VCL_BLOB *b);
extern const unsigned s_MULT;
extern const unsigned s_OPTIONS;
extern const unsigned s_HEAD;
// pseudo-opaque declaration
#ifndef VPFX
#define VPFX(x) tus_ ## x
#endif
struct VPFX(tus_server);
struct tus_exp;
struct vmod_blobdigest_digest;
// tus_servers.c
const char * tus_server_name(struct VPFX(tus_server) *);
struct tus_files *tus_server_files(struct VPFX(tus_server) *);
int tus_server_basefd(const struct VPFX(tus_server) *);
const char * tus_server_basedir(const struct VPFX(tus_server) *);
VCL_DURATION tus_server_expires(const struct VPFX(tus_server) *);
void tus_server_lock(struct VPFX(tus_server) *);
void tus_server_unlock(struct VPFX(tus_server) *);
VCL_BYTES tus_server_max(const struct VPFX(tus_server) *s);
const struct vmod_blobdigest_digest * tus_server_digest(
const struct VPFX(tus_server) *s);
struct tus_exp *tus_server_exp(const struct VPFX(tus_server) *);
#include "config.h"
#include <cache/cache.h>
#include "vcc_tus_if.h"
#include "tus_file.h"
#include "tus_server.h"
#include "tus_servers.h"
struct tus_servers tus_servers[1] = { VSPLAY_INITIALIZER(tus_servers) };
VSPLAY_GENERATE(tus_servers, VPFX(tus_server), entry, tus_server_cmp);
const char *
tus_server_name(struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->vcl_name);
}
struct tus_files *
tus_server_files(struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->files);
}
int
tus_server_basefd(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->basefd);
}
const char *
tus_server_basedir(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->basedir);
}
VCL_DURATION
tus_server_expires(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->expires);
}
void
tus_server_lock(struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
AZ(pthread_mutex_lock(&s->mtx));
}
void
tus_server_unlock(struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
AZ(pthread_mutex_unlock(&s->mtx));
}
VCL_BYTES
tus_server_max(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->max);
}
const struct vmod_blobdigest_digest *
tus_server_digest(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->digest);
}
struct tus_exp *
tus_server_exp(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->exp);
}
#include <string.h>
#include <stdlib.h>
#include <vtree.h>
struct vmod_blobdigest_digest;
struct tus_exp;
/* ============================================================
* global server splay tree
*/
struct VPFX(tus_server) {
unsigned magic;
#define VMOD_TUS_SERVER_MAGIC 0x1055e47e
unsigned refcnt;
VSPLAY_ENTRY(VPFX(tus_server)) entry;
pthread_mutex_t mtx;
char *vcl_name;
char *basedir;
char *schemeauth;
VCL_BYTES max;
VCL_DURATION expires;
int basefd;
struct tus_files files[1];
const struct vmod_blobdigest_digest *digest;
struct tus_exp *exp;
};
static inline int
tus_server_cmp(const struct VPFX(tus_server) *a,
const struct VPFX(tus_server) *b) {
CHECK_OBJ_NOTNULL(a, VMOD_TUS_SERVER_MAGIC);
CHECK_OBJ_NOTNULL(b, VMOD_TUS_SERVER_MAGIC);
return (strcmp(a->vcl_name, b->vcl_name));
}
VSPLAY_HEAD(tus_servers, VPFX(tus_server));
struct tus_servers tus_servers[1];
VSPLAY_PROTOTYPE(tus_servers, VPFX(tus_server), entry, tus_server_cmp);
#include "config.h"
#include <sys/mman.h>
#include <cache/cache.h>
#include <cache/cache_obj.h>
#include <cache/cache_objhead.h>
#include <storage/storage.h>
#include "tus_file.h"
#include "tus_concat.h"
#include "tus_stv.h"
static void
tus_objfree(struct worker *wrk, struct objcore *oc);
static int
tus_objiterator(struct worker *wrk, struct objcore *oc,
void *priv, objiterate_f *func, int final);
static int
tus_objgetspace(struct worker *wrk, struct objcore *oc,
ssize_t *sz, uint8_t **ptr);
static void
tus_objextend(struct worker *wrk, struct objcore *oc, ssize_t l);
static const void *
tus_objgetattr(struct worker *wrk, struct objcore *oc,
enum obj_attr attr, ssize_t *len);
static void *
tus_objsetattr(struct worker *wrk, struct objcore *oc,
enum obj_attr attr, ssize_t len, const void *ptr);
const struct obj_methods obj_tus = {
.objfree = tus_objfree,
.objiterator = tus_objiterator,
.objgetspace = tus_objgetspace,
.objextend = tus_objextend,
.objgetattr = tus_objgetattr,
.objsetattr = tus_objsetattr
};
// ------------------------------------------------------------
static int
tus_allocobj(struct worker *wrk, const struct stevedore *stv,
struct objcore *oc, unsigned l)
{
(void) wrk;
(void) stv;
(void) oc;
(void) l;
INCOMPL();
return (0); // USELESS
}
const struct stevedore stv_tus = {
.magic = STEVEDORE_MAGIC,
.name = "tus",
.allocobj = tus_allocobj,
.methods = &obj_tus,
.ident = "tus",
.vclname = "tus"
};
/*
* ============================================================
*
*/
// instead of STV_NewObject()
//
// assign an existing tus file to an objcore
void
tus_body_assign(struct req *req, struct tus_concat *c)
{
struct objcore *oc;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(c, TUS_CONCAT_MAGIC);
if (req->body_oc != NULL)
AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
AZ(req->body_oc);
req->body_oc = oc = HSH_Private(req->wrk);
AN(oc);
AZ(oc->stobj->stevedore);
AZ(oc->stobj->priv);
oc->oa_present = 0;
oc->stobj->stevedore = &stv_tus;
oc->stobj->priv = c;
//oc->stobj->priv2 = 0;
req->req_body_status = BS_CACHED;
HSH_DerefBoc(req->wrk, oc);
}
static void
tus_objfree(struct worker *wrk, struct objcore *oc)
{
(void) wrk;
// not actually freeing it, the tus object gets expired / refcounted
oc->stobj->stevedore = NULL;
oc->stobj->priv = 0;
}
static int
tus_objiterator(struct worker *wrk, struct objcore *oc,
void *priv, objiterate_f *func, int final)
{
// sensible upper limit taking into account tcp windows
const unsigned max_write = 16 * 1024 * 1024;
struct tus_concat *c;
struct tus_file_core *fcore;
unsigned i, off, len;
void *p;
int r = -1;
(void) wrk;
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
assert(oc->stobj->stevedore == &stv_tus);
CAST_OBJ_NOTNULL(c, oc->stobj->priv, TUS_CONCAT_MAGIC);
for (i = 0; i < c->n; i++) {
fcore = c->cores[i];
CHECK_OBJ_NOTNULL(fcore, VMOD_TUS_FILE_CORE_MAGIC);
if (fcore->len == 0)
continue;
AN(fcore->ptr);
p = fcore->ptr;
len = fcore->len;
AZ(posix_madvise(p, len, POSIX_MADV_SEQUENTIAL));
AZ(posix_madvise(p, len, POSIX_MADV_WILLNEED));
off = 0;
do {
len = fcore->len - off;
if (len > max_write)
len = max_write;
p = (char *)fcore->ptr + off;
r = func(priv, OBJ_ITER_FINAL | OBJ_ITER_FLUSH, p, len);
if (r < 0)
return (r);
AZ(posix_madvise(p, len, POSIX_MADV_DONTNEED));
off += len;
} while (off < fcore->len);
}
return (r);
}
static int
tus_objgetspace(struct worker *wrk, struct objcore *oc,
ssize_t *sz, uint8_t **ptr)
{
(void) wrk;
(void) oc;
(void) sz;
(void) ptr;
INCOMPL();
return (0); // USELESS
}
static void
tus_objextend(struct worker *wrk, struct objcore *oc, ssize_t l)
{
(void) wrk;
(void) oc;
(void) l;
INCOMPL();
}
static const void *
tus_objgetattr(struct worker *wrk, struct objcore *oc,
enum obj_attr attr, ssize_t *len)
{
(void) wrk;
(void) oc;
(void) attr;
(void) len;
INCOMPL();
return (NULL); // USELESS
}
static void *
tus_objsetattr(struct worker *wrk, struct objcore *oc,
enum obj_attr attr, ssize_t len, const void *ptr)
{
(void) wrk;
(void) oc;
(void) attr;
(void) len;
(void) ptr;
INCOMPL();
return (NULL); // USELESS
}
// helper
struct tus_concat *
tus_body_single(struct req *req, struct tus_file_core *fcore)
{
struct tus_concat *c;
c = WS_Alloc(req->ws, sizeof *c + sizeof fcore);
if (c == NULL)
return (NULL);
INIT_OBJ(c, TUS_CONCAT_MAGIC);
c->n = 1;
c->cores[0] = fcore;
return (c);
}
struct tus_concat;
void tus_body_assign(struct req *req, struct tus_concat *concat);
struct tus_concat *tus_body_single(struct req *req, struct tus_file_core *fcore);
#include "config.h"
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include "vdef.h"
#include "vrt.h"
int
main(int argc, char **argv) {
void *dlhdl;
char buf[256];
const struct vmod_data *d;
const char *name = argv[1];
const char *file = argv[2];
dlhdl = dlopen(file, RTLD_LAZY | RTLD_LOCAL);
if (dlhdl == NULL) {
fprintf(stderr, "dlopen: %s\n", dlerror());
return (1);
}
bprintf(buf, "Vmod_%s_Data", name);
d = dlsym(dlhdl, buf);
if (d == NULL) {
fprintf(stderr, "dlsym: %s\n", dlerror());
return (1);
}
printf(d->proto);
printf("\nstatic const struct vmod_data import_%s = {\n", buf);
printf("\t.name =\t\t\"%s\",\n", d->name);
printf("\t.file_id =\t\"%s\",\n", d->file_id);
printf("\t.abi =\t\t\"%s\",\n", d->abi);
printf("};\n");
return (0);
}
This diff is collapsed.
$Module tus 3 "Varnish tus Module"
$Event event
$Prefix tus
DESCRIPTION
===========
This VCC file was generated by VCDK, it is used to for both the VMOD
interface and its manual using reStructuredText.
.. _tus: https://tus.io/protocols/resumable-upload.html
XXX: document vmod-tus
This vmod implements a `tus`_ proxy which collects uploads on the
varnish server to send them to a backend in one go for storage. It
does not implement any permanent storage itself.
Example
::
Besides the basic resumable uploads specified as the `tus`_ core
protocol, all currently defined extensions are supported, in
particular concatenation uploads.
import tus;
This vmod implements methods to be called from VCL and requires a
common call pattern for normal operation. When used this way, this
vmod handles all client interaction via synthetic responses until an
upload is complete. For the client request concluding the upload, it
modifies the request into a single ``PUT`` and provides a single
request body to be sent to a backend.
sub vcl_deliver {
set resp.http.Hello = tus.hello();
}
By this VCL integration, http behaviour remains highly customizable.
XXX: define vmod-tus interface
CORS
----
$Function STRING hello()
The `xserver.synth()`_ and `xserver.deliver()`_ methods will set the
right CORS headers allowing all Origins, that is,
``Access-Control-Allow-Origin`` set to ``Origin``. As any other
header, the CORS headers can be changes after the call to
`xserver.synth()`_, and should be to restrict Origin access.
Description
Hello world for vmod-tus
Hashes
------
.. _vmod_blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest
This vmod supports checking and generating content hashes if
`vmod_blob` (bundled with varnish-cache) and `vmod_blobdigest`_ are
* installed at build time
* installed in exactly the same version at run time
* and imported to the VCL before ``tus``, like so::
import blob;
import blobdigest;
import tus;
Given these preconditions are true, thus vmod supports the following
hashes as ``Tus-Checksum-Algorithm`` and as the ``name_hash`` argument
to `tus.server()`_:
* crc32
* icrc32
* md5
* rs
* sha1
* sha224
* sha256
* sha384
* sha3_224
* sha3_256
* sha3_384
* sha3_512
* sha512
$Object server(STRING schemeauth, BYTES max=1073741824,
DURATION expires=86400, [STRING basedir],
[STRING name_hash])
Declare a tus server at the given ``schemeauth``, which is the
protocol and servername, like ``https://tus.io``.
Tus servers are shared across VCLs. Each server has its own namespace.
The ``max`` argument (defaulting to 1GB) specifies the maximum upload
size supported by the server. The actual maximum upload size supported
as reportde in the ``Tus-Max-Size`` header will be that value capped
by the available disk space in the ``basedir``. Notice that as the
available space might vary inbetween requests, so will the reported
space, and uploads might still fail. Thus the ``max`` argument is
recommended to be chosen sufficiently small compared to the available
disk space and disk space be monitored.
The ``expires`` argument (defaulting to 1 day) specifies the duration
until unfinished uploads expire and are removed. The expiry gets
extended by that duration with every operation on a file.
.. XXX warm event? per VCL object?
The ``schemeauth``, ``max`` and ``expires`` values for an already defined
server *can* be changed. The value used in the vcl *loaded* last
applies.
The optional ``basedir`` argument can be used to define a directory
under which this server's objects are stored. This directory or its
parent directory should exist.
Sharing basedirs between servers is not supported. An attempting to
change the ``basedir`` for a server already defined through a
different VCL is an error.
The optional ``name_hash`` argument can be used to instruct the tus
server to use the hex encoding of a hash over the content the upload
as the last part of the url when constructing the backend ``PUT``. For
``Upload-Concat: final`` uploads, this name is also used as the
``Location`` returned to the client.
$Method BOOL .recv([STRING url], [STRING id])
Process a tus request.
If the `url` argument is not given, it is taken from ``req.url``.
The optional `id` argument allows to override the id assigned by the
tus server for a create (HTTP ``POST``). Notice that, while id is
normally guaranteed to be unique for the currently active uploads, a
``POST`` may fail for a duplicate `id` argument. `id` has no relevance
for any request but ``POST``.
If the return value is ``true``, the request body has been replaced
with an object ready to be sent to the backend, so a backend call
should be made to send data to storage.
If the reuturn value is ``false``, a synthetic reponse has to be
generated, so ``return(synth(code))`` should be returned such that
`xserver.synth()`_ will be called for ``code``.
Thus, the common call pattern is::
sub vcl_recv {
if (mytus.recv()) {
return (pass);
} else {
return (synth(4200));
}
}
Must be called from ``vcl_recv {}``
$Method BOOL .deliver()
Generate a response to a tus request for which content was
stored. Must be called from ``vcl_deliver {}`` after `xserver.recv()`_
was called from ``vcl_recv {}``.
$Method BOOL .synth()
Generate a synthetic response to a tus request. Must be called from
``vcl_synth {}`` after `xserver.recv()`_ was called from ``vcl_recv {}``.
For the example above, the code would be::
sub vcl_synth {
if (resp.status == 4200) {
mytus.synth();
return (deliver);
}
}
$Method BOOL .done([STRING location])
Mark the upload as done: This frees all storage except for information
on the upload URL itself.
If the optional *location* string is provided, a 301 redirect to
*location* will be generated for any access attempt. Otherwise, a 410
response will be generated.
May only be called from client methods. For anything but final concat
or single (non-concat) uploads, this operation is a noop.
$Method BOOL .has_metadata(STRING key)
Return true if *key* is present in the metadata.
Only available if vmod_blob is available (see `Hashes` _) and on the
client side after `xserver.recv()`_ was called from ``vcl_recv {}``.
$Method BLOB .metadata(STRING key)
Extract *key* from metadata and return the corresponding value decoded.
Only available if vmod_blob is available (see `Hashes` _) and on the
client side after `xserver.recv()`_ was called from ``vcl_recv {}``.
SEE ALSO
========vcl\(7),varnishd\(1)
varnishtest "test vmod-tus set done redirect"
server s1 {
rxreq
txresp
expect req.method == PUT
expect req.bodylen == 100
} -start
varnish v1 -vcl+backend {
import blob;
import blobdigest;
import tus;
sub vcl_init {
new tmp = tus.server("http://localhost",
basedir="/tmp/tus", max = 3145B);
}
sub vcl_backend_fetch {
if (bereq.url ~ "^/id") {
set bereq.backend = s1;
} else {
return (abandon);
}
}
sub vcl_recv {
if (tmp.recv(id=req.http.id)) {
return(pass);
} else {
return(synth(4200));
}
}
sub vcl_synth {
if (resp.status == 4200) {
tmp.synth();
return (deliver);
}
}
sub vcl_deliver {
tmp.deliver();
tmp.done(req.url);
}
} -start
# dynamic file name complete post
client c1 {
txreq -method "DELETE" -url "/id" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Checksum: sha256 H98ak0Maj3gjAqfjx2KIqrwAil7CHeKsnJwyzLAXEu8=" \
-hdr "Id: id" \
-bodylen 100
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation,checksum"
expect resp.http.Tus-Checksum-Algorithm == "crc32,icrc32,md5,rs,sha1,sha224,sha256,sha384,sha3_224,sha3_256,sha3_384,sha3_512,sha512"
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 100
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "http://localhost/id"
txreq -url "/id" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 301
expect resp.http.Location == "http://localhost/id"
} -run
varnishtest "test expiring files"
varnish v1 -vcl {
import tus;
backend dummy None;
sub vcl_init {
new tmp = tus.server("https://my.origin",
basedir="/tmp/tusexp", expires=1s);
}
sub vcl_recv {
if (tmp.recv(id=req.http.id)) {
return(fail);
} else {
return(synth(4200));
}
}
sub vcl_synth {
if (resp.status == 4200) {
tmp.synth();
return (deliver);
}
}
sub vcl_deliver {
tmp.deliver();
}
} -start
client c1 {
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Checksum: sha256 H98ak0Maj3gjAqfjx2KIqrwAil7CHeKsnJwyzLAXEu8=" \
-hdr "Id: c1" \
-hdr "Content-Length: 0" \
-nolen
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "https://my.origin/c1"
txreq -method HEAD -url "/c1" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
delay 2
txreq -method HEAD -url "/c1" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 404
} -start
client c2 {
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Checksum: sha256 H98ak0Maj3gjAqfjx2KIqrwAil7CHeKsnJwyzLAXEu8=" \
-hdr "Id: c2" \
-hdr "Content-Length: 0" \
-nolen
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "https://my.origin/c2"
txreq -method HEAD -url "/c2" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
delay 2
txreq -method HEAD -url "/c2" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 404
} -start
client c3 {
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Checksum: sha256 H98ak0Maj3gjAqfjx2KIqrwAil7CHeKsnJwyzLAXEu8=" \
-hdr "Id: c3" \
-hdr "Content-Length: 0" \
-nolen
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "https://my.origin/c3"
txreq -method HEAD -url "/c3" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
delay 2
txreq -method HEAD -url "/c3" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 404
} -start
client c4 {
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Checksum: sha256 H98ak0Maj3gjAqfjx2KIqrwAil7CHeKsnJwyzLAXEu8=" \
-hdr "Id: c4" \
-hdr "Content-Length: 0" \
-nolen
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "https://my.origin/c4"
txreq -method HEAD -url "/c4" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
delay 2
txreq -method HEAD -url "/c4" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 404
} -start
client c1 -wait
client c2 -wait
client c3 -wait
client c4 -wait
This diff is collapsed.
This diff is collapsed.
varnishtest "test vmod-tus"
server s1 {
rxreq
txresp
} -start
varnish v1 -vcl+backend {
import tus;
sub vcl_deliver {
set resp.http.Hello = tus.hello();
}
} -start
client c1 {
txreq
rxresp
expect resp.status == 200
expect resp.http.Hello == "vmod-tus"
} -run
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment