Early mmap for basic multipart support

parent f59c6eb0
......@@ -14,6 +14,7 @@ vmod_get_h_LDADD = -ldl
TESTS = \
vtc/plain.vtc \
vtc/plain_cors.vtc \
vtc/multipart.vtc \
vtc/name_hash.vtc \
vtc/expires.vtc \
vtc/done.vtc \
......
......@@ -1097,11 +1097,19 @@ struct tus_suck_fd {
struct tus_suck_common sc;
};
struct tus_suck_mmap {
unsigned magic;
#define TUS_SUCK_MMAP_MAGIC 0x11ffbafa
void *ptr;
struct tus_suck_common sc;
};
typedef int suck_truncate_f(void *, off_t);
struct tus_suck {
union {
struct tus_suck_fd sfd;
struct tus_suck_fd sfd;
struct tus_suck_mmap sm;
} priv;
objiterate_f *func;
suck_truncate_f *trunc;
......@@ -1158,7 +1166,7 @@ tus_suck_fd_truncate_f(void *priv, off_t length)
CAST_OBJ_NOTNULL(sfd, priv, TUS_SUCK_FD_MAGIC);
return (ftruncate(sfd->fd, length));
return (ftruncate(sfd->fd, header_size + length));
}
static inline struct tus_suck_common *
......@@ -1179,6 +1187,73 @@ tus_suck_fd_init(struct tus_suck *suck, struct tus_file_core *fcore)
* suck to mmapped
*/
static int v_matchproto_(objiterate_f)
tus_suck_mmap_f(void *priv, unsigned flush, const void *ptr, ssize_t len)
{
struct tus_suck_mmap *sm;
struct tus_suck_common *sc;
void *wptr;
CAST_OBJ_NOTNULL(sm, priv, TUS_SUCK_MMAP_MAGIC);
sc = &sm->sc;
(void) flush;
if (len == 0)
return (0);
assert(len > 0);
if (*sc->upload_offset + len > sc->max) {
errno = EFBIG;
return (-1);
}
wptr = (char *)sm->ptr + *sc->upload_offset;
memcpy(wptr, ptr, len);
/* XXX write ordering with fdisk flush ?
*
* maybe we can just truncate all trailing NULs
* when reading?
*/
return (tus_suck_finish(sc, len, ptr, len));
}
static int
tus_suck_mmap_truncate_f(void *priv, off_t length)
{
struct tus_suck_mmap *sm;
struct tus_suck_common *sc;
void *wptr;
CAST_OBJ_NOTNULL(sm, priv, TUS_SUCK_MMAP_MAGIC);
sc = &sm->sc;
assert(length <= sc->max);
if (length >= *sc->upload_offset)
return (0);
wptr = (char *)sm->ptr + length;
memset(wptr, 0, *sc->upload_offset - length);
}
static inline struct tus_suck_common *
tus_suck_mmap_init(struct tus_suck *suck, struct tus_file_core *fcore)
{
struct tus_suck_mmap *sm = &suck->priv.sm;
tus_file_mmap(fcore);
sm->magic = TUS_SUCK_MMAP_MAGIC;
sm->ptr = fcore->ptr;
AN(sm->ptr);
suck->func = tus_suck_mmap_f;
suck->trunc = tus_suck_mmap_truncate_f;
return (&sm->sc);
}
/* ------------------------------------------------------------
* top level sucking
*/
......@@ -1192,9 +1267,17 @@ tus_suck_trunc(struct tus_suck *suck, off_t length)
}
static inline struct tus_suck_common *
tus_suck_init(struct tus_suck *suck, struct tus_file_core *fcore)
tus_suck_init(struct tus_suck *suck, struct tus_file_core *fcore,
struct tus_file_disk *fdisk)
{
return (tus_suck_fd_init(suck, fcore));
VCL_BYTES bm;
bm = tus_server_multipart(fcore->server);
if (bm > 0 && fdisk->upload_offset >= bm)
return (tus_suck_mmap_init(suck, fcore));
else
return (tus_suck_fd_init(suck, fcore));
}
unsigned
......@@ -1215,10 +1298,10 @@ tus_body_to_file(VRT_CTX, struct tus_file_core *fcore)
memset(&suck, 0, sizeof suck);
sc = tus_suck_init(&suck, fcore);
sc = tus_suck_init(&suck, fcore, fdisk);
AN(sc);
offset_saved = fcore->disk->upload_offset;
offset_saved = fdisk->upload_offset;
sc->ctx = ctx;
sc->max = tus_server_max(fcore->server);
......@@ -1238,8 +1321,8 @@ tus_body_to_file(VRT_CTX, struct tus_file_core *fcore)
suck.func, &suck.priv);
if (written < 0 && errno == EFBIG) {
tus_suck_trunc(&suck, sc->max);
fdisk->upload_offset = fdisk->upload_length = sc->max;
tus_suck_trunc(&suck, header_size + sc->max);
return (413);
}
......@@ -1253,8 +1336,8 @@ tus_body_to_file(VRT_CTX, struct tus_file_core *fcore)
if (tus_chksum_equal(ctx, sc->chksum))
return (204);
tus_suck_trunc(&suck, offset_saved);
fcore->disk->upload_offset = offset_saved;
tus_suck_trunc(&suck, header_size + offset_saved);
return (460);
}
......
......@@ -44,6 +44,7 @@ VCL_DURATION tus_server_expires(const struct VPFX(tus_server) *);
void tus_server_lock(struct VPFX(tus_server) *);
void tus_server_unlock(struct VPFX(tus_server) *);
VCL_BYTES tus_server_max(const struct VPFX(tus_server) *s);
VCL_BYTES tus_server_multipart(const struct VPFX(tus_server) *s);
struct vmod_blobdigest_digest * tus_server_digest(
const struct VPFX(tus_server) *s);
struct tus_exp *tus_server_exp(const struct VPFX(tus_server) *);
......@@ -103,6 +103,14 @@ tus_server_max(const struct VPFX(tus_server) *s)
return (s->max);
}
VCL_BYTES
tus_server_multipart(const struct VPFX(tus_server) *s)
{
CHECK_OBJ_NOTNULL(s, VMOD_TUS_SERVER_MAGIC);
return (s->multipart);
}
struct vmod_blobdigest_digest *
tus_server_digest(const struct VPFX(tus_server) *s)
{
......
varnishtest "test vmod-tus with ids"
barrier b1 cond 2
barrier b2 cond 2
server s1 {
rxreq
txresp
expect req.method == PUT
expect req.bodylen == 100
} -start
server s2 {
rxreq
txresp
expect req.method == PUT
expect req.url == /vtc
expect req.bodylen == 100
} -start
server s3 {
rxreq
txresp
expect req.method == PUT
expect req.url == /concat
expect req.bodylen == 200
} -start
server s4 {
rxreq
txresp
expect req.method == PUT
expect req.url == /empty-single
expect req.bodylen == 0
} -start
varnish v1 -vcl+backend {
import tus;
sub vcl_init {
new test = tus.server("https://my.origin");
new tmp = tus.server("http://localhost",
basedir="/tmp/tus", max = 3145B,
multipart = 10B);
}
sub vcl_backend_fetch {
if (bereq.url ~ "^/tus") {
set bereq.backend = s1;
} else if (bereq.url == "/vtc") {
set bereq.backend = s2;
} else if (bereq.url == "/concat") {
set bereq.backend = s3;
} else if (bereq.url == "/empty-single") {
set bereq.backend = s4;
} else {
return (abandon);
}
}
sub vcl_recv {
if (tmp.recv(id=req.http.id)) {
return(pass);
} else {
return(synth(4200));
}
}
sub meta {
set resp.http.has-filename = tmp.has_metadata("filename");
set resp.http.has-is_confidential =
tmp.has_metadata("is_confidential");
set resp.http.has-filenamee = tmp.has_metadata("filenamee");
set resp.http.has-filenam = tmp.has_metadata("filenam");
}
sub vcl_synth {
if (resp.status == 4200) {
call meta;
tmp.synth();
tmp.sync();
return (deliver);
}
}
sub vcl_deliver {
call meta;
tmp.deliver();
}
} -start
varnish v1 -errvcl "attempt to change basedir" {
import tus;
backend dummy None;
sub vcl_init {
new test = tus.server("http://localhost", basedir="different");
}
}
varnish v1 -errvcl "Argument 'schemeauth' missing" {
import tus;
backend dummy None;
sub vcl_init {
new test = tus.server(basedir="different");
}
}
varnish v1 -errvcl "opening basedir" {
import tus;
backend dummy None;
sub vcl_init {
new no_way = tus.server(schemeauth="http://localhost",
basedir="/dev/null/foo");
}
}
# dynamic file name complete post
client c1 {
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-bodylen 100
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 100
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location ~ "^http://localhost/tus"
} -start
# POST/PATCH test with vtc-provided ID
client c2 {
# clean out
txreq -url "/vtc" \
-method DELETE \
-hdr "Content-Length: 0" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
# partial Upload with create
txreq -method POST \
-hdr "Upload-Metadata: filename d29ybGRfZG9taW5hdGlvbl9wbGFuLnBkZg==,is_confidential" \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Id: vtc" \
-bodylen 40
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 40
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
## XXX only for HEAD?
expect resp.http.Upload-Metadata == "filename d29ybGRfZG9taW5hdGlvbl9wbGFuLnBkZg==,is_confidential"
expect resp.http.Location == "http://localhost/vtc"
expect resp.http.has-filename == true
expect resp.http.has-is_confidential == true
expect resp.http.has-filenamee == false
expect resp.http.has-filenam == false
# failing get on upload in progress
txreq -url "/vtc"
rxresp
expect resp.status == 400
# finish upload
txreq -method PATCH -url "/vtc" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Length: 100" \
-hdr "Upload-Offset: 40" \
-bodylen 60
rxresp
expect resp.status == 204
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 100
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
## XXX only for HEAD?
expect resp.http.Upload-Metadata == "filename d29ybGRfZG9taW5hdGlvbl9wbGFuLnBkZg==,is_confidential"
expect resp.http.has-filename == true
expect resp.http.has-is_confidential == true
expect resp.http.has-filenamee == false
expect resp.http.has-filenam == false
} -start
# part1
client c3 {
# clean out
txreq -url "/part1" \
-method DELETE \
-hdr "Content-Length: 0" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Id: part1" \
-hdr "Upload-Concat: partial" \
-bodylen 40
rxresp
barrier b1 sync
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 40
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "http://localhost/part1"
txreq -method HEAD -url "/part1" \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 40
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
# finish upload
txreq -method PATCH -url "/part1" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Length: 100" \
-hdr "Upload-Offset: 40" \
-bodylen 60
rxresp
expect resp.status == 204
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 100
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
} -start
# part2
client c4 {
# clean out
txreq -url "/part2" \
-method DELETE \
-hdr "Content-Length: 0" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
txreq -method POST \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Id: part2" \
-hdr "Upload-Defer-Length: 1" \
-hdr "Upload-Concat: partial"
rxresp
barrier b2 sync
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Defer-Length == 1
expect resp.http.Upload-Length == <undef>
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "http://localhost/part2"
txreq -method HEAD -url "/part2" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Defer-Length == 1
expect resp.http.Upload-Length == <undef>
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Upload-Concat == partial
# upload
txreq -method PATCH -url "/part2" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Length: 100" \
-hdr "Upload-Offset: 0" \
-bodylen 100
rxresp
expect resp.status == 204
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 100
expect resp.http.Upload-Length == 100
expect resp.http.Upload-Expires ~ "GMT$"
} -start
# concat
client c5 {
# clean out
txreq -url "/concat" \
-method DELETE \
-hdr "Content-Length: 0" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
barrier b1 sync
barrier b2 sync
txreq -url / \
-method POST \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Id: concat" \
-hdr "Upload-Concat: final; /part1 /part2"
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 200
expect resp.http.Upload-Length == 200
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location == "http://localhost/concat"
txreq -url /concat \
-method HEAD \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 200
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 200
expect resp.http.Upload-Length == 200
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Upload-Concat == "final;http://localhost/part1 http://localhost/part2"
# fail patch on concat
txreq -url /concat \
-method PATCH \
-hdr "Tus-Resumable: 1.0.0"
rxresp
expect resp.status == 403
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Upload-Offset == <undef>
expect resp.http.Upload-Length == <undef>
expect resp.http.Upload-Expires == <undef>
expect resp.http.Location == <undef>
} -start
# bad concat
client c6 {
txreq -url / \
-method POST \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Id:badconcat" \
-hdr "Upload-Concat: final;/foo"
rxresp
expect resp.status == 400
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == <undef>
expect resp.http.Upload-Length == <undef>
expect resp.http.Upload-Expires == <undef>
expect resp.http.Location == <undef>
txreq -url / \
-method POST \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Id:badconcat2" \
-hdr "Upload-Concat: final;"
rxresp
expect resp.status == 400
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == <undef>
expect resp.http.Upload-Length == <undef>
expect resp.http.Upload-Expires == <undef>
expect resp.http.Location == <undef>
} -start
# exceeding max size
client c7 {
txreq -method POST \
-hdr "Upload-Length: 3146" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-bodylen 3146
rxresp
expect resp.status == 413
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
## XXX correct?
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == <undef>
expect resp.http.Upload-Length == <undef>
expect resp.http.Upload-Expires == <undef>
expect resp.http.Location == <undef>
} -start
# checksum despite not supported
client c8 {
txreq -method POST \
-hdr "Upload-Length: 100" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Upload-Checksum: md5 ABCD" \
-bodylen 100
rxresp
expect resp.status == 400
} -start
client c9 {
# clean out
txreq -url "/empty-single" \
-method DELETE \
-hdr "Content-Length: 0" \
-hdr "Tus-Resumable: 1.0.0"
rxresp
txreq -method POST \
-hdr "Upload-Length: 0" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Id: empty-single" \
-hdr "Content-Type: application/offset+octet-stream" \
-hdr "Content-Length: 0"
rxresp
expect resp.status == 201
expect resp.http.Tus-Resumable == "1.0.0"
expect resp.http.Tus-Version == "1.0.0"
expect resp.http.Tus-Extension == "creation,creation-with-upload,expiration,termination,concatenation"
expect resp.http.Tus-Checksum-Algorithm == <undef>
expect resp.http.Tus-Max-Size == 3145
expect resp.http.Upload-Offset == 0
expect resp.http.Upload-Length == 0
expect resp.http.Upload-Expires ~ "GMT$"
expect resp.http.Location ~ "^http://localhost/empty-single"
} -start
# bad metadata fmt
client c10 {
txreq -method POST \
-hdr "Upload-Metadata: filename d29y!GRfZG9taW5hdGlvbl9wbGFuLnBkZg==,is_confidential" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream"
rxresp
expect resp.status == 400
txreq -method POST \
-hdr "Upload-Metadata: filename,filename" \
-hdr "Tus-Resumable: 1.0.0" \
-hdr "Content-Type: application/offset+octet-stream"
rxresp
expect resp.status == 400
} -start
# missing / bad version
client c11 {
txreq -method POST \
-hdr "Tus-Resumable: 1.0.1" \
-hdr "Content-Type: application/offset+octet-stream"
rxresp
expect resp.status == 412
txreq -method POST \
-hdr "Content-Type: application/offset+octet-stream"
rxresp
expect resp.status == 412
} -start
client c1 -wait
client c2 -wait
client c3 -wait
client c4 -wait
client c5 -wait
client c6 -wait
client c7 -wait
client c8 -wait
client c9 -wait
client c10 -wait
client c11 -wait
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment