Skip to content

L
libvdfp-pipe
Commit 104b0069 authored by Geoff Simmons's avatar Geoff Simmons
Browse files
Options

Bugfixing for large outputs and parallel tests. 

- Close the unused ends of pipes in the parent process.

- Don't stop reading from stdout if waitpid(2) shows that
  the process has exited, as there may still be pending
  output. We now call waitpid(2) only in vdp_fini.

- Close stdin as early as possible on VDP_END.

- Read stdout and stderr until they close.

- Don't use /bin/true and /bin/false in the tests, as
  they actually don't consume stdin, and may close
  stdin before we get to poll(2), resulting in POLLERR.
  This happened with varnishtest -jN, with j > about 5.

- Index VDPs with vdp and vcl pointers in the RB tree.
  This did not turn out to be a bugfix as I thought, but
  it should be more reliable, since VCL and object names
  may be identical in varnishtest -j tests.
parent 5c03356c
Hide whitespace changes
Inline Side-by-side
Showing with 117 additions and 68 deletions
src/tests/vdp.vtc
View file @ 104b0069
......@@ -5,6 +5,40 @@ varnishtest "vdp object"
server s1 {
rxreq
txresp -body {foo bar baz quux}
rxreq
txresp -body {Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.}
rxreq
txresp -body {Lorem ipsum dolor sit amet, leo ornare nulla, eu malesuada. Cras turpis pretium consectetuer mauris urna sodales, aliquet morbi malesuada potenti velit hendrerit, ut rhoncus lorem vitae neque sed, feugiat nec tellus gravida magna sit. Pretium ac massa arcu viverra, diam velit in aliqua adipiscing vehicula, aliquet iaculis a lobortis enim sit, ligula nisl ultricies, magna volutpat eros suspendisse massa. Ut mauris, purus mi arcu faucibus pretium elit. Pulvinar enim eu, et orci leo tortor parturient magna, commodo ut pellentesque, justo lorem vestibulum non lacinia odio. Consequatur diam volutpat.
Lacus donec libero venenatis venenatis, egestas pretium ut urna vehicula, ac eget ac, amet maecenas turpis viverra blandit. Ullamcorper massa sed in, nec velit nec aenean justo, nullam cras, ullamcorper eros pharetra magna urna metus, ultricies rutrum neque nunc eleifend. Pretium consequat leo in, amet varius integer, eros non et ut sit ac, magnis quis nibh dolor ornare sed. Turpis nec morbi, nunc a mauris faucibus molestie, dolor velit tincidunt, amet ac erat eros massa pede. Vitae venenatis volutpat eget vestibulum. In nam pharetra in vivamus pretium, nulla nunc sociosqu, aliquam nullam ligula, ut sed vestibulum quam morbi turpis mattis, wisi at per lorem aliquam ante sed. Mauris taciti ligula venenatis vehicula, morbi eget nec. Amet id sodales eos, diam suspendisse pulvinar, at vestibulum torquent nec eget risus est. Mauris sapien tortor eget nesciunt.
Ridiculus congue erat velit, nulla felis, viverra non tempor cum nulla, quis quisque enim at sociosqu vestibulum sit. Donec molestie. Cursus pretium donec phasellus lacinia velit nulla, quis sapien dapibus suscipit viverra magni commodo, viverra vel eros lectus tristique lorem vivamus. Tortor mi ante, aut elit, metus ante voluptatem nisl aenean eget amet, felis at et. Posuere lorem ridiculus, et leo viverra pellentesque neque, ullamcorper neque volutpat pretium amet laoreet. Sapien at, phasellus ullamco wisi, arcu fermentum fringilla nam turpis mi. Congue et donec rutrum, voluptatem et duis neque eget elit. Mus sit luctus est, nullam fringilla magna vel massa, vivamus vel adipiscing quis dolor quam. Risus laoreet consequat fusce suspendisse vitae, felis aliquet.
Enim per accumsan, augue id maecenas bibendum ullamcorper in fermentum, platea fermentum. Convallis pulvinar tortor eros sit cras nascetur, quam dis elit sed aliquam ac, nulla morbi suscipit egestas placerat pretium. Vehicula elementum duis, turpis wisi nascetur velit. Senectus aenean ipsum blandit, eleifend leo donec nunc, ut vel sagittis sed nam, ut aliquam quam aliquam at tellus. Vivamus donec libero feugiat amet gravida, felis id aliquam nullam consequat, aut eget suspendisse rutrum in. Orci tellus. A sapien scelerisque diam, ornare laoreet lobortis blandit in. Aliquam at suscipit, id risus. Leo lacus quam mollis erat morbi vel, ut adipiscing cillum dui. Elit bibendum tristique et non eros libero, egestas lectus turpis, quis gravida dolor volutpat, purus eu, consectetuer pharetra per vehicula quam tellus.}
rxreq
txresp -bodylen 65536
rxreq
txresp -nolen -hdr "Transfer-encoding: chunked"
chunkedlen 8
chunkedlen 8
chunkedlen 16
chunkedlen 32
chunkedlen 64
chunkedlen 128
chunkedlen 256
chunkedlen 512
chunkedlen 1024
chunkedlen 2048
chunkedlen 4096
chunkedlen 8192
chunkedlen 16384
chunkedlen 32768
chunkedlen 65536
chunkedlen 0
} -start
varnish v1 -vcl+backend {
......@@ -12,60 +46,54 @@ varnish v1 -vcl+backend {
sub vcl_init {
new cat = pipe.vdp(name="cat", path="/bin/cat");
new t = pipe.vdp(name="true", path="/bin/true");
new f = pipe.vdp(name="false", path="/bin/false");
}
sub vcl_backend_response {
set beresp.uncacheable = true;
}
sub vcl_deliver {
set resp.filters = req.http.X-Filters;
set resp.filters = "cat";
}
} -start
client c1 {
txreq -hdr "X-Filters: cat"
txreq
rxresp
expect resp.status == 200
expect resp.body == "foo bar baz quux"
txreq -hdr "X-Filters: cat true"
txreq
rxresp
expect resp.status == 200
expect resp.body == ""
expect resp.body == {Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.}
# Only read response headers, the body may fail.
txreq -hdr "X-Filters: cat false"
rxresp -no_obj
txreq
rxresp
expect resp.status == 200
} -run
expect resp.body == {Lorem ipsum dolor sit amet, leo ornare nulla, eu malesuada. Cras turpis pretium consectetuer mauris urna sodales, aliquet morbi malesuada potenti velit hendrerit, ut rhoncus lorem vitae neque sed, feugiat nec tellus gravida magna sit. Pretium ac massa arcu viverra, diam velit in aliqua adipiscing vehicula, aliquet iaculis a lobortis enim sit, ligula nisl ultricies, magna volutpat eros suspendisse massa. Ut mauris, purus mi arcu faucibus pretium elit. Pulvinar enim eu, et orci leo tortor parturient magna, commodo ut pellentesque, justo lorem vestibulum non lacinia odio. Consequatur diam volutpat.
logexpect l1 -v v1 -g vxid -d 1 -q {Notice ~ "^vdfp_pipe: vdp cat:"} {
expect 0 * Begin {^req \d+ rxreq$}
expect * = Notice {^vdfp_pipe: vdp cat: exec'd /bin/cat as pid \d+$}
expect * = Notice {^vdfp_pipe: vdp cat: /bin/cat exited with status 0$}
expect * = End
Lacus donec libero venenatis venenatis, egestas pretium ut urna vehicula, ac eget ac, amet maecenas turpis viverra blandit. Ullamcorper massa sed in, nec velit nec aenean justo, nullam cras, ullamcorper eros pharetra magna urna metus, ultricies rutrum neque nunc eleifend. Pretium consequat leo in, amet varius integer, eros non et ut sit ac, magnis quis nibh dolor ornare sed. Turpis nec morbi, nunc a mauris faucibus molestie, dolor velit tincidunt, amet ac erat eros massa pede. Vitae venenatis volutpat eget vestibulum. In nam pharetra in vivamus pretium, nulla nunc sociosqu, aliquam nullam ligula, ut sed vestibulum quam morbi turpis mattis, wisi at per lorem aliquam ante sed. Mauris taciti ligula venenatis vehicula, morbi eget nec. Amet id sodales eos, diam suspendisse pulvinar, at vestibulum torquent nec eget risus est. Mauris sapien tortor eget nesciunt.
expect 0 * Begin {^req \d+ rxreq$}
expect * = Notice {^vdfp_pipe: vdp cat: exec'd /bin/cat as pid \d+$}
expect * = Notice {^vdfp_pipe: vdp cat: /bin/cat exited with status 0$}
expect * = End
Ridiculus congue erat velit, nulla felis, viverra non tempor cum nulla, quis quisque enim at sociosqu vestibulum sit. Donec molestie. Cursus pretium donec phasellus lacinia velit nulla, quis sapien dapibus suscipit viverra magni commodo, viverra vel eros lectus tristique lorem vivamus. Tortor mi ante, aut elit, metus ante voluptatem nisl aenean eget amet, felis at et. Posuere lorem ridiculus, et leo viverra pellentesque neque, ullamcorper neque volutpat pretium amet laoreet. Sapien at, phasellus ullamco wisi, arcu fermentum fringilla nam turpis mi. Congue et donec rutrum, voluptatem et duis neque eget elit. Mus sit luctus est, nullam fringilla magna vel massa, vivamus vel adipiscing quis dolor quam. Risus laoreet consequat fusce suspendisse vitae, felis aliquet.
expect 0 * Begin {^req \d+ rxreq$}
expect * = Notice {^vdfp_pipe: vdp cat: exec'd /bin/cat as pid \d+$}
expect * = Notice {^vdfp_pipe: vdp cat: /bin/cat exited with status 0$}
expect * = End
} -run
Enim per accumsan, augue id maecenas bibendum ullamcorper in fermentum, platea fermentum. Convallis pulvinar tortor eros sit cras nascetur, quam dis elit sed aliquam ac, nulla morbi suscipit egestas placerat pretium. Vehicula elementum duis, turpis wisi nascetur velit. Senectus aenean ipsum blandit, eleifend leo donec nunc, ut vel sagittis sed nam, ut aliquam quam aliquam at tellus. Vivamus donec libero feugiat amet gravida, felis id aliquam nullam consequat, aut eget suspendisse rutrum in. Orci tellus. A sapien scelerisque diam, ornare laoreet lobortis blandit in. Aliquam at suscipit, id risus. Leo lacus quam mollis erat morbi vel, ut adipiscing cillum dui. Elit bibendum tristique et non eros libero, egestas lectus turpis, quis gravida dolor volutpat, purus eu, consectetuer pharetra per vehicula quam tellus.}
logexpect l1 -v v1 -g vxid -d 1 -q {Notice ~ "^vdfp_pipe: vdp t:"} {
expect 0 * Begin {^req \d+ rxreq$}
expect * = Notice {^vdfp_pipe: vdp t: exec'd /bin/true as pid \d+$}
expect * = Notice {^vdfp_pipe: vdp t: /bin/true exited with status 0$}
expect * = End
txreq
rxresp
expect resp.status == 200
expect resp.bodylen == 65536
txreq
rxresp
expect resp.status == 200
expect resp.bodylen == 131072
} -run
logexpect l1 -v v1 -g vxid -d 1 -q {Error ~ "^vdfp_pipe: vdp f:" or Notice ~ "^vdfp_pipe: vdp f"} {
logexpect l1 -v v1 -g vxid -d 1 -q {Notice ~ "^vdfp_pipe: vdp cat:"} {
expect 0 * Begin {^req \d+ rxreq$}
expect * = Notice {^vdfp_pipe: vdp f: exec'd /bin/false as pid \d+$}
expect * = Error {^vdfp_pipe: vdp f: /bin/false exited with status \d+$}
expect * = Notice {^vdfp_pipe: vdp cat: exec'd /bin/cat as pid \d+$}
expect * = Notice {^vdfp_pipe: vdp cat: /bin/cat exited with status 0$}
expect * = End
} -run
......
src/vdfp_pipe.c
View file @ 104b0069
......@@ -67,18 +67,23 @@ struct vdp_map {
unsigned magic;
#define PIPE_VDP_MAP_MAGIC 0x87a66950
VRBT_ENTRY(vdp_map) entry;
const char *vcl_name;
const char *vdp_name;
struct vcl *vcl;
const struct vdp *vdp;
struct VPFX(pipe_vdp) *obj;
};
static inline int
map_cmp(const struct vdp_map *v1, const struct vdp_map *v2)
{
int vclcmp = strcmp(v1->vcl_name, v2->vcl_name);
if (vclcmp != 0)
return (vclcmp);
return (strcmp(v1->vdp_name, v2->vdp_name));
if (v1->vcl < v2->vcl)
return (-1);
if (v1->vcl > v2->vcl)
return (1);
if (v1->vdp == v2->vdp)
return (0);
if (v1->vdp < v2->vdp)
return (-1);
return (1);
}
VRBT_HEAD(vdp_tree, vdp_map);
......@@ -148,9 +153,8 @@ vdp_init(struct req *req, void **priv)
vdpe = VTAILQ_LAST(&req->vdc->vdp, vdp_entry_s);
CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC);
AN(vdpe->vdp);
AN(vdpe->vdp->name);
map_entry.vdp_name = vdpe->vdp->name;
map_entry.vcl_name = VCL_Name(req->vcl);
map_entry.vdp = vdpe->vdp;
map_entry.vcl = req->vcl;
map = VRBT_FIND(vdp_tree, &tree_h, &map_entry);
CHECK_OBJ_NOTNULL(map, PIPE_VDP_MAP_MAGIC);
CHECK_OBJ_NOTNULL(map->obj, PIPE_VDP_MAGIC);
......@@ -221,6 +225,9 @@ vdp_init(struct req *req, void **priv)
VSLb(req->vsl, SLT_Notice, "vdfp_pipe: vdp %s: exec'd %s as pid %jd",
obj->name, obj->path, (intmax_t)state->chldpid);
closefd(&in[0]);
closefd(&out[1]);
closefd(&err[1]);
state->chldin = in[1];
state->chldout = out[0];
state->chlderr = err[0];
......@@ -314,11 +321,13 @@ rw_child(struct req *req, struct vdp_state *state, enum vdp_action act,
AN(state->obj->name);
AN(state->obj->path);
if (act == VDP_END && len == 0)
closefd(&state->chldin);
for (;;) {
if ((retval = check_pid(state, req->vsl, WNOHANG)) != 0)
return (retval);
nfds = 0;
if (ptr != NULL && len > 0 && state->chldin != -1) {
if (len > 0 && state->chldin != -1) {
AN(ptr);
fds[0].fd = state->chldin;
fds[0].events = POLLOUT;
fds[0].revents = 0;
......@@ -338,6 +347,7 @@ rw_child(struct req *req, struct vdp_state *state, enum vdp_action act,
}
if (nfds == 0)
break;
errno = 0;
retval = poll(fds, nfds, TIMEOUT_MS);
if (retval < 0) {
......@@ -391,9 +401,16 @@ rw_child(struct req *req, struct vdp_state *state, enum vdp_action act,
ptr += nbytes;
continue;
}
AN(fds[i].revents & POLLIN);
assert(fds[i].fd == state->chldout
|| fds[i].fd == state->chlderr);
if (!(fds[i].revents & POLLIN)) {
AN(fds[i].revents & POLLHUP);
if (fds[i].fd == state->chldout)
closefd(&state->chldout);
else
closefd(&state->chlderr);
continue;
}
errno = 0;
nbytes = read(fds[i].fd, state->buf, default_bufsz);
if (nbytes < 0) {
......@@ -407,8 +424,11 @@ rw_child(struct req *req, struct vdp_state *state, enum vdp_action act,
return (-1);
}
if (nbytes == 0) {
closefd(&fds[i].fd);
break;
if (fds[i].fd == state->chldout)
closefd(&state->chldout);
else
closefd(&state->chlderr);
continue;
}
if (fds[i].fd == state->chldout) {
retval = VDP_bytes(req, VDP_FLUSH, state->buf,
......@@ -424,12 +444,17 @@ rw_child(struct req *req, struct vdp_state *state, enum vdp_action act,
"stderr ...", state->obj->name, state->obj->path);
VSLb_bin(req->vsl, SLT_Error, nbytes, state->buf);
}
if (act == VDP_END) {
if (state->chldin != -1)
closefd(&state->chldin);
if (state->chldout != -1 || state->chlderr != -1)
continue;
break;
}
if (len == 0)
break;
}
AZ(len);
if (act == VDP_END && state->chldin != -1)
closefd(&state->chldin);
return (0);
}
......@@ -500,13 +525,9 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
while (map != NULL) {
CHECK_OBJ(map, PIPE_VDP_MAP_MAGIC);
prev_map = NULL;
if (map->vcl_name != NULL &&
strcmp(map->vcl_name, VCL_Name(ctx->vcl)) == 0) {
CHECK_OBJ_NOTNULL(map->obj, PIPE_VDP_MAGIC);
if (map->obj->vdp != NULL) {
VRT_RemoveVDP(ctx, map->obj->vdp);
free(map->obj->vdp);
}
if (map->vcl == ctx->vcl && map->vdp != NULL) {
VRT_RemoveVDP(ctx, map->vdp);
free(TRUST_ME(map->vdp));
prev_map = map;
}
map = VRBT_NEXT(vdp_tree, &tree_h, map);
......@@ -584,18 +605,6 @@ vmod_vdp__init(VRT_CTX, struct VPFX(pipe_vdp) **vdpp, const char *obj_name,
vdp_obj->name = strdup(obj_name);
vdp_obj->path = strdup(path);
errno = 0;
ALLOC_OBJ(map, PIPE_VDP_MAP_MAGIC);
if (map == NULL) {
VDPFAIL(ctx, "new %s: cannot allocate space for map entry: %s",
obj_name, vstrerror(errno));
return;
}
map->vdp_name = strdup(vdp_name);
map->vcl_name = strdup(VCL_Name(ctx->vcl));
map->obj = vdp_obj;
AZ(VRBT_INSERT(vdp_tree, &tree_h, map));
errno = 0;
vdp = malloc(sizeof(*vdp));
if (vdp == NULL) {
......@@ -610,6 +619,18 @@ vmod_vdp__init(VRT_CTX, struct VPFX(pipe_vdp) **vdpp, const char *obj_name,
VRT_AddVDP(ctx, vdp);
vdp_obj->vdp = vdp;
errno = 0;
ALLOC_OBJ(map, PIPE_VDP_MAP_MAGIC);
if (map == NULL) {
VDPFAIL(ctx, "new %s: cannot allocate space for map entry: %s",
obj_name, vstrerror(errno));
return;
}
map->vdp = vdp;
map->vcl = ctx->vcl;
map->obj = vdp_obj;
AZ(VRBT_INSERT(vdp_tree, &tree_h, map));
return;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment