Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
V
varnish-cache
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Commits
Open sidebar
varnishcache
varnish-cache
Commits
599f2e3f
Commit
599f2e3f
authored
Jun 17, 2013
by
Poul-Henning Kamp
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Split the HTTP1 specific parts of the backend fetch code into a
separate source file.
parent
1252ddf2
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
462 additions
and
413 deletions
+462
-413
Makefile.am
bin/varnishd/Makefile.am
+1
-0
cache.h
bin/varnishd/cache/cache.h
+4
-0
cache_fetch.c
bin/varnishd/cache/cache_fetch.c
+7
-413
cache_http1_fetch.c
bin/varnishd/cache/cache_http1_fetch.c
+450
-0
No files found.
bin/varnishd/Makefile.am
View file @
599f2e3f
...
@@ -27,6 +27,7 @@ varnishd_SOURCES = \
...
@@ -27,6 +27,7 @@ varnishd_SOURCES = \
cache/cache_gzip.c
\
cache/cache_gzip.c
\
cache/cache_hash.c
\
cache/cache_hash.c
\
cache/cache_http.c
\
cache/cache_http.c
\
cache/cache_http1_fetch.c
\
cache/cache_http1_fsm.c
\
cache/cache_http1_fsm.c
\
cache/cache_http1_proto.c
\
cache/cache_http1_proto.c
\
cache/cache_lck.c
\
cache/cache_lck.c
\
...
...
bin/varnishd/cache/cache.h
View file @
599f2e3f
...
@@ -784,6 +784,10 @@ void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
...
@@ -784,6 +784,10 @@ void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
void
VBO_Free
(
struct
busyobj
**
vbo
);
void
VBO_Free
(
struct
busyobj
**
vbo
);
void
VBO_extend
(
const
struct
busyobj
*
,
ssize_t
);
void
VBO_extend
(
const
struct
busyobj
*
,
ssize_t
);
/* cache_http1_fetch.c [V1F] */
int
V1F_fetch_hdr
(
struct
worker
*
wrk
,
struct
busyobj
*
bo
,
struct
req
*
req
);
void
V1F_fetch_body
(
struct
worker
*
wrk
,
struct
busyobj
*
bo
);
/* cache_http1_fsm.c [HTTP1] */
/* cache_http1_fsm.c [HTTP1] */
typedef
int
(
req_body_iter_f
)(
struct
req
*
,
void
*
priv
,
void
*
ptr
,
size_t
);
typedef
int
(
req_body_iter_f
)(
struct
req
*
,
void
*
priv
,
void
*
ptr
,
size_t
);
void
HTTP1_Session
(
struct
worker
*
,
struct
req
*
);
void
HTTP1_Session
(
struct
worker
*
,
struct
req
*
);
...
...
bin/varnishd/cache/cache_fetch.c
View file @
599f2e3f
...
@@ -41,8 +41,6 @@
...
@@ -41,8 +41,6 @@
#include "cache_backend.h"
#include "cache_backend.h"
#include "vcli_priv.h"
#include "vcli_priv.h"
#include "vcl.h"
#include "vcl.h"
#include "vct.h"
#include "vtcp.h"
#include "vtim.h"
#include "vtim.h"
static
unsigned
fetchfrag
;
static
unsigned
fetchfrag
;
...
@@ -210,414 +208,6 @@ VBF_GetStorage(struct busyobj *bo, ssize_t sz)
...
@@ -210,414 +208,6 @@ VBF_GetStorage(struct busyobj *bo, ssize_t sz)
return
(
st
);
return
(
st
);
}
}
/*--------------------------------------------------------------------
* Convert a string to a size_t safely
*/
static
ssize_t
vbf_fetch_number
(
const
char
*
nbr
,
int
radix
)
{
uintmax_t
cll
;
ssize_t
cl
;
char
*
q
;
if
(
*
nbr
==
'\0'
)
return
(
-
1
);
cll
=
strtoumax
(
nbr
,
&
q
,
radix
);
if
(
q
==
NULL
||
*
q
!=
'\0'
)
return
(
-
1
);
cl
=
(
ssize_t
)
cll
;
if
((
uintmax_t
)
cl
!=
cll
)
/* Protect against bogusly large values */
return
(
-
1
);
return
(
cl
);
}
/*--------------------------------------------------------------------*/
static
int
vbf_fetch_straight
(
struct
busyobj
*
bo
,
struct
http_conn
*
htc
,
ssize_t
cl
)
{
int
i
;
assert
(
htc
->
body_status
==
BS_LENGTH
);
if
(
cl
<
0
)
{
return
(
VBF_Error
(
bo
,
"straight length field bogus"
));
}
else
if
(
cl
==
0
)
return
(
0
);
i
=
bo
->
vfp
->
bytes
(
bo
,
htc
,
cl
);
if
(
i
<=
0
)
return
(
VBF_Error
(
bo
,
"straight insufficient bytes"
));
return
(
0
);
}
/*--------------------------------------------------------------------
* Read a chunked HTTP object.
*
* XXX: Reading one byte at a time is pretty pessimal.
*/
static
int
vbf_fetch_chunked
(
struct
busyobj
*
bo
,
struct
http_conn
*
htc
)
{
int
i
;
char
buf
[
20
];
/* XXX: 20 is arbitrary */
unsigned
u
;
ssize_t
cl
;
assert
(
htc
->
body_status
==
BS_CHUNKED
);
do
{
/* Skip leading whitespace */
do
{
if
(
HTTP1_Read
(
htc
,
buf
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
}
while
(
vct_islws
(
buf
[
0
]));
if
(
!
vct_ishex
(
buf
[
0
]))
return
(
VBF_Error
(
bo
,
"chunked header non-hex"
));
/* Collect hex digits, skipping leading zeros */
for
(
u
=
1
;
u
<
sizeof
buf
;
u
++
)
{
do
{
if
(
HTTP1_Read
(
htc
,
buf
+
u
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
}
while
(
u
==
1
&&
buf
[
0
]
==
'0'
&&
buf
[
u
]
==
'0'
);
if
(
!
vct_ishex
(
buf
[
u
]))
break
;
}
if
(
u
>=
sizeof
buf
)
return
(
VBF_Error
(
bo
,
"chunked header too long"
));
/* Skip trailing white space */
while
(
vct_islws
(
buf
[
u
])
&&
buf
[
u
]
!=
'\n'
)
if
(
HTTP1_Read
(
htc
,
buf
+
u
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
if
(
buf
[
u
]
!=
'\n'
)
return
(
VBF_Error
(
bo
,
"chunked header no NL"
));
buf
[
u
]
=
'\0'
;
cl
=
vbf_fetch_number
(
buf
,
16
);
if
(
cl
<
0
)
return
(
VBF_Error
(
bo
,
"chunked header number syntax"
));
if
(
cl
>
0
&&
bo
->
vfp
->
bytes
(
bo
,
htc
,
cl
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
i
=
HTTP1_Read
(
htc
,
buf
,
1
);
if
(
i
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
if
(
buf
[
0
]
==
'\r'
&&
HTTP1_Read
(
htc
,
buf
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
if
(
buf
[
0
]
!=
'\n'
)
return
(
VBF_Error
(
bo
,
"chunked tail no NL"
));
}
while
(
cl
>
0
);
return
(
0
);
}
/*--------------------------------------------------------------------*/
static
void
vbf_fetch_eof
(
struct
busyobj
*
bo
,
struct
http_conn
*
htc
)
{
assert
(
htc
->
body_status
==
BS_EOF
);
if
(
bo
->
vfp
->
bytes
(
bo
,
htc
,
SSIZE_MAX
)
<
0
)
(
void
)
VBF_Error
(
bo
,
"eof socket fail"
);
}
/*--------------------------------------------------------------------
* Pass the request body to the backend
*/
static
int
__match_proto__
(
req_body_iter_f
)
vbf_iter_req_body
(
struct
req
*
req
,
void
*
priv
,
void
*
ptr
,
size_t
l
)
{
struct
worker
*
wrk
;
CHECK_OBJ_NOTNULL
(
req
,
REQ_MAGIC
);
CAST_OBJ_NOTNULL
(
wrk
,
priv
,
WORKER_MAGIC
);
if
(
l
>
0
)
{
(
void
)
WRW_Write
(
wrk
,
ptr
,
l
);
if
(
WRW_Flush
(
wrk
))
return
(
-
1
);
}
return
(
0
);
}
/*--------------------------------------------------------------------
* Send request, and receive the HTTP protocol response, but not the
* response body.
*
* Return value:
* -1 failure, not retryable
* 0 success
* 1 failure which can be retried.
*/
static
int
vbf_fetch_hdr
(
struct
worker
*
wrk
,
struct
busyobj
*
bo
,
struct
req
*
req
)
{
struct
vbc
*
vc
;
struct
http
*
hp
;
enum
htc_status_e
hs
;
int
retry
=
-
1
;
int
i
,
first
;
struct
http_conn
*
htc
;
CHECK_OBJ_NOTNULL
(
wrk
,
WORKER_MAGIC
);
CHECK_OBJ_ORNULL
(
req
,
REQ_MAGIC
);
CHECK_OBJ_NOTNULL
(
bo
,
BUSYOBJ_MAGIC
);
htc
=
&
bo
->
htc
;
AN
(
bo
->
director
);
hp
=
bo
->
bereq
;
bo
->
vbc
=
VDI_GetFd
(
NULL
,
bo
);
if
(
bo
->
vbc
==
NULL
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"no backend connection"
);
return
(
-
1
);
}
vc
=
bo
->
vbc
;
if
(
vc
->
recycled
)
retry
=
1
;
/*
* Now that we know our backend, we can set a default Host:
* header if one is necessary. This cannot be done in the VCL
* because the backend may be chosen by a director.
*/
if
(
!
http_GetHdr
(
bo
->
bereq
,
H_Host
,
NULL
))
VDI_AddHostHeader
(
bo
->
bereq
,
vc
);
(
void
)
VTCP_blocking
(
vc
->
fd
);
/* XXX: we should timeout instead */
WRW_Reserve
(
wrk
,
&
vc
->
fd
,
bo
->
vsl
,
bo
->
t_fetch
);
(
void
)
HTTP1_Write
(
wrk
,
hp
,
0
);
/* XXX: stats ? */
/* Deal with any message-body the request might (still) have */
i
=
0
;
if
(
req
!=
NULL
)
{
i
=
HTTP1_IterateReqBody
(
req
,
vbf_iter_req_body
,
wrk
);
if
(
req
->
req_body_status
==
REQ_BODY_DONE
)
retry
=
-
1
;
}
if
(
WRW_FlushRelease
(
wrk
)
||
i
!=
0
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"backend write error: %d (%s)"
,
errno
,
strerror
(
errno
));
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
retry
);
}
/* XXX is this the right place? */
VSC_C_main
->
backend_req
++
;
/* Receive response */
HTTP1_Init
(
htc
,
bo
->
ws
,
vc
->
fd
,
vc
->
vsl
,
cache_param
->
http_resp_size
,
cache_param
->
http_resp_hdr_len
);
VTCP_set_read_timeout
(
vc
->
fd
,
vc
->
first_byte_timeout
);
first
=
1
;
do
{
hs
=
HTTP1_Rx
(
htc
);
if
(
hs
==
HTTP1_OVERFLOW
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"http %sread error: overflow"
,
first
?
"first "
:
""
);
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
-
1
);
}
if
(
hs
==
HTTP1_ERROR_EOF
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"http %sread error: EOF"
,
first
?
"first "
:
""
);
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
retry
);
}
if
(
first
)
{
retry
=
-
1
;
first
=
0
;
VTCP_set_read_timeout
(
vc
->
fd
,
vc
->
between_bytes_timeout
);
}
}
while
(
hs
!=
HTTP1_COMPLETE
);
hp
=
bo
->
beresp
;
if
(
HTTP1_DissectResponse
(
hp
,
htc
))
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"http format error"
);
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
-
1
);
}
return
(
0
);
}
/*--------------------------------------------------------------------
* This function is either called by the requesting thread OR by a
* dedicated body-fetch work-thread.
*
* We get passed the busyobj in the priv arg, and we inherit a
* refcount on it, which we must release, when done fetching.
*/
static
void
vbf_fetch_body
(
struct
worker
*
wrk
,
struct
busyobj
*
bo
)
{
int
cls
;
struct
storage
*
st
;
int
mklen
;
ssize_t
cl
;
struct
http_conn
*
htc
;
struct
object
*
obj
;
CHECK_OBJ_NOTNULL
(
wrk
,
WORKER_MAGIC
);
CHECK_OBJ_NOTNULL
(
bo
,
BUSYOBJ_MAGIC
);
htc
=
&
bo
->
htc
;
CHECK_OBJ_ORNULL
(
bo
->
vbc
,
VBC_MAGIC
);
obj
=
bo
->
fetch_obj
;
CHECK_OBJ_NOTNULL
(
obj
,
OBJECT_MAGIC
);
CHECK_OBJ_NOTNULL
(
obj
->
http
,
HTTP_MAGIC
);
assert
(
bo
->
state
==
BOS_INVALID
);
/*
* XXX: The busyobj needs a dstat, but it is not obvious which one
* XXX: it should be (own/borrowed). For now borrow the wrk's.
*/
AZ
(
bo
->
stats
);
bo
->
stats
=
&
wrk
->
stats
;
if
(
bo
->
vfp
==
NULL
)
bo
->
vfp
=
&
vfp_nop
;
AN
(
bo
->
vfp
);
AZ
(
bo
->
vgz_rx
);
AZ
(
VTAILQ_FIRST
(
&
obj
->
store
));
bo
->
state
=
BOS_FETCHING
;
/* XXX: pick up estimate from objdr ? */
cl
=
0
;
cls
=
bo
->
should_close
;
switch
(
htc
->
body_status
)
{
case
BS_NONE
:
mklen
=
0
;
break
;
case
BS_ZERO
:
mklen
=
1
;
break
;
case
BS_LENGTH
:
cl
=
vbf_fetch_number
(
bo
->
h_content_length
,
10
);
bo
->
vfp
->
begin
(
bo
,
cl
);
if
(
bo
->
state
==
BOS_FETCHING
&&
cl
>
0
)
cls
|=
vbf_fetch_straight
(
bo
,
htc
,
cl
);
mklen
=
1
;
if
(
bo
->
vfp
->
end
(
bo
))
assert
(
bo
->
state
==
BOS_FAILED
);
break
;
case
BS_CHUNKED
:
bo
->
vfp
->
begin
(
bo
,
cl
>
0
?
cl
:
0
);
if
(
bo
->
state
==
BOS_FETCHING
)
cls
|=
vbf_fetch_chunked
(
bo
,
htc
);
mklen
=
1
;
if
(
bo
->
vfp
->
end
(
bo
))
assert
(
bo
->
state
==
BOS_FAILED
);
break
;
case
BS_EOF
:
bo
->
vfp
->
begin
(
bo
,
cl
>
0
?
cl
:
0
);
if
(
bo
->
state
==
BOS_FETCHING
)
vbf_fetch_eof
(
bo
,
htc
);
mklen
=
1
;
cls
=
1
;
if
(
bo
->
vfp
->
end
(
bo
))
assert
(
bo
->
state
==
BOS_FAILED
);
break
;
case
BS_ERROR
:
cls
|=
VBF_Error
(
bo
,
"error incompatible Transfer-Encoding"
);
mklen
=
0
;
break
;
default:
mklen
=
0
;
INCOMPL
();
}
AZ
(
bo
->
vgz_rx
);
/*
* We always call vfp_nop_end() to ditch or trim the last storage
* segment, to avoid having to replicate that code in all vfp's.
*/
AZ
(
vfp_nop_end
(
bo
));
bo
->
vfp
=
NULL
;
VSLb
(
bo
->
vsl
,
SLT_Fetch_Body
,
"%u(%s) cls %d mklen %d"
,
htc
->
body_status
,
body_status_2str
(
htc
->
body_status
),
cls
,
mklen
);
http_Teardown
(
bo
->
bereq
);
http_Teardown
(
bo
->
beresp
);
if
(
bo
->
vbc
!=
NULL
)
{
if
(
cls
)
VDI_CloseFd
(
&
bo
->
vbc
);
else
VDI_RecycleFd
(
&
bo
->
vbc
);
}
if
(
bo
->
state
==
BOS_FAILED
)
{
wrk
->
stats
.
fetch_failed
++
;
obj
->
len
=
0
;
EXP_Clr
(
&
obj
->
exp
);
EXP_Rearm
(
obj
);
}
else
{
assert
(
bo
->
state
==
BOS_FETCHING
);
VSLb
(
bo
->
vsl
,
SLT_Length
,
"%zd"
,
obj
->
len
);
{
/* Sanity check fetch methods accounting */
ssize_t
uu
;
uu
=
0
;
VTAILQ_FOREACH
(
st
,
&
obj
->
store
,
list
)
uu
+=
st
->
len
;
if
(
bo
->
do_stream
)
/* Streaming might have started freeing stuff */
assert
(
uu
<=
obj
->
len
);
else
assert
(
uu
==
obj
->
len
);
}
if
(
mklen
>
0
)
{
http_Unset
(
obj
->
http
,
H_Content_Length
);
http_PrintfHeader
(
obj
->
http
,
"Content-Length: %zd"
,
obj
->
len
);
}
/* XXX: Atomic assignment, needs volatile/membar ? */
bo
->
state
=
BOS_FINISHED
;
}
if
(
obj
->
objcore
->
objhead
!=
NULL
)
HSH_Complete
(
obj
->
objcore
);
bo
->
stats
=
NULL
;
}
/*--------------------------------------------------------------------
/*--------------------------------------------------------------------
* Copy req->bereq and run it by VCL::vcl_backend_fetch{}
* Copy req->bereq and run it by VCL::vcl_backend_fetch{}
*/
*/
...
@@ -683,7 +273,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo, struct req **reqp)
...
@@ -683,7 +273,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo, struct req **reqp)
if
(
!
bo
->
do_pass
)
if
(
!
bo
->
do_pass
)
*
reqp
=
NULL
;
*
reqp
=
NULL
;
i
=
vbf
_fetch_hdr
(
wrk
,
bo
,
*
reqp
);
i
=
V1F
_fetch_hdr
(
wrk
,
bo
,
*
reqp
);
/*
/*
* If we recycle a backend connection, there is a finite chance
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* that the backend closed it before we get a request to it.
...
@@ -691,7 +281,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo, struct req **reqp)
...
@@ -691,7 +281,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo, struct req **reqp)
*/
*/
if
(
i
==
1
)
{
if
(
i
==
1
)
{
VSC_C_main
->
backend_retry
++
;
VSC_C_main
->
backend_retry
++
;
i
=
vbf
_fetch_hdr
(
wrk
,
bo
,
*
reqp
);
i
=
V1F
_fetch_hdr
(
wrk
,
bo
,
*
reqp
);
}
}
if
(
bo
->
do_pass
)
if
(
bo
->
do_pass
)
...
@@ -967,7 +557,10 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
...
@@ -967,7 +557,10 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
HSH_Unbusy
(
&
wrk
->
stats
,
obj
->
objcore
);
HSH_Unbusy
(
&
wrk
->
stats
,
obj
->
objcore
);
}
}
vbf_fetch_body
(
wrk
,
bo
);
if
(
bo
->
vfp
==
NULL
)
bo
->
vfp
=
&
vfp_nop
;
V1F_fetch_body
(
wrk
,
bo
);
assert
(
bo
->
refcount
>=
1
);
assert
(
bo
->
refcount
>=
1
);
...
@@ -1110,6 +703,7 @@ static struct cli_proto debug_cmds[] = {
...
@@ -1110,6 +703,7 @@ static struct cli_proto debug_cmds[] = {
{
NULL
}
{
NULL
}
};
};
/*--------------------------------------------------------------------
/*--------------------------------------------------------------------
*
*
*/
*/
...
...
bin/varnishd/cache/cache_http1_fetch.c
0 → 100644
View file @
599f2e3f
/*-
* Copyright (c) 2006 Verdens Gang AS
* Copyright (c) 2006-2011 Varnish Software AS
* All rights reserved.
*
* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "config.h"
#include <inttypes.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include "cache.h"
#include "hash/hash_slinger.h"
#include "cache_backend.h"
#include "vcli_priv.h"
#include "vct.h"
#include "vtcp.h"
/*--------------------------------------------------------------------
* Convert a string to a size_t safely
*/
static
ssize_t
vbf_fetch_number
(
const
char
*
nbr
,
int
radix
)
{
uintmax_t
cll
;
ssize_t
cl
;
char
*
q
;
if
(
*
nbr
==
'\0'
)
return
(
-
1
);
cll
=
strtoumax
(
nbr
,
&
q
,
radix
);
if
(
q
==
NULL
||
*
q
!=
'\0'
)
return
(
-
1
);
cl
=
(
ssize_t
)
cll
;
if
((
uintmax_t
)
cl
!=
cll
)
/* Protect against bogusly large values */
return
(
-
1
);
return
(
cl
);
}
/*--------------------------------------------------------------------*/
static
int
vbf_fetch_straight
(
struct
busyobj
*
bo
,
struct
http_conn
*
htc
,
ssize_t
cl
)
{
int
i
;
assert
(
htc
->
body_status
==
BS_LENGTH
);
if
(
cl
<
0
)
{
return
(
VBF_Error
(
bo
,
"straight length field bogus"
));
}
else
if
(
cl
==
0
)
return
(
0
);
i
=
bo
->
vfp
->
bytes
(
bo
,
htc
,
cl
);
if
(
i
<=
0
)
return
(
VBF_Error
(
bo
,
"straight insufficient bytes"
));
return
(
0
);
}
/*--------------------------------------------------------------------
* Read a chunked HTTP object.
*
* XXX: Reading one byte at a time is pretty pessimal.
*/
static
int
vbf_fetch_chunked
(
struct
busyobj
*
bo
,
struct
http_conn
*
htc
)
{
int
i
;
char
buf
[
20
];
/* XXX: 20 is arbitrary */
unsigned
u
;
ssize_t
cl
;
assert
(
htc
->
body_status
==
BS_CHUNKED
);
do
{
/* Skip leading whitespace */
do
{
if
(
HTTP1_Read
(
htc
,
buf
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
}
while
(
vct_islws
(
buf
[
0
]));
if
(
!
vct_ishex
(
buf
[
0
]))
return
(
VBF_Error
(
bo
,
"chunked header non-hex"
));
/* Collect hex digits, skipping leading zeros */
for
(
u
=
1
;
u
<
sizeof
buf
;
u
++
)
{
do
{
if
(
HTTP1_Read
(
htc
,
buf
+
u
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
}
while
(
u
==
1
&&
buf
[
0
]
==
'0'
&&
buf
[
u
]
==
'0'
);
if
(
!
vct_ishex
(
buf
[
u
]))
break
;
}
if
(
u
>=
sizeof
buf
)
return
(
VBF_Error
(
bo
,
"chunked header too long"
));
/* Skip trailing white space */
while
(
vct_islws
(
buf
[
u
])
&&
buf
[
u
]
!=
'\n'
)
if
(
HTTP1_Read
(
htc
,
buf
+
u
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
if
(
buf
[
u
]
!=
'\n'
)
return
(
VBF_Error
(
bo
,
"chunked header no NL"
));
buf
[
u
]
=
'\0'
;
cl
=
vbf_fetch_number
(
buf
,
16
);
if
(
cl
<
0
)
return
(
VBF_Error
(
bo
,
"chunked header number syntax"
));
if
(
cl
>
0
&&
bo
->
vfp
->
bytes
(
bo
,
htc
,
cl
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
i
=
HTTP1_Read
(
htc
,
buf
,
1
);
if
(
i
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
if
(
buf
[
0
]
==
'\r'
&&
HTTP1_Read
(
htc
,
buf
,
1
)
<=
0
)
return
(
VBF_Error
(
bo
,
"chunked read err"
));
if
(
buf
[
0
]
!=
'\n'
)
return
(
VBF_Error
(
bo
,
"chunked tail no NL"
));
}
while
(
cl
>
0
);
return
(
0
);
}
/*--------------------------------------------------------------------*/
static
void
vbf_fetch_eof
(
struct
busyobj
*
bo
,
struct
http_conn
*
htc
)
{
assert
(
htc
->
body_status
==
BS_EOF
);
if
(
bo
->
vfp
->
bytes
(
bo
,
htc
,
SSIZE_MAX
)
<
0
)
(
void
)
VBF_Error
(
bo
,
"eof socket fail"
);
}
/*--------------------------------------------------------------------
* Pass the request body to the backend
*/
static
int
__match_proto__
(
req_body_iter_f
)
vbf_iter_req_body
(
struct
req
*
req
,
void
*
priv
,
void
*
ptr
,
size_t
l
)
{
struct
worker
*
wrk
;
CHECK_OBJ_NOTNULL
(
req
,
REQ_MAGIC
);
CAST_OBJ_NOTNULL
(
wrk
,
priv
,
WORKER_MAGIC
);
if
(
l
>
0
)
{
(
void
)
WRW_Write
(
wrk
,
ptr
,
l
);
if
(
WRW_Flush
(
wrk
))
return
(
-
1
);
}
return
(
0
);
}
/*--------------------------------------------------------------------
* Send request, and receive the HTTP protocol response, but not the
* response body.
*
* Return value:
* -1 failure, not retryable
* 0 success
* 1 failure which can be retried.
*/
int
V1F_fetch_hdr
(
struct
worker
*
wrk
,
struct
busyobj
*
bo
,
struct
req
*
req
)
{
struct
vbc
*
vc
;
struct
http
*
hp
;
enum
htc_status_e
hs
;
int
retry
=
-
1
;
int
i
,
first
;
struct
http_conn
*
htc
;
CHECK_OBJ_NOTNULL
(
wrk
,
WORKER_MAGIC
);
CHECK_OBJ_ORNULL
(
req
,
REQ_MAGIC
);
CHECK_OBJ_NOTNULL
(
bo
,
BUSYOBJ_MAGIC
);
htc
=
&
bo
->
htc
;
AN
(
bo
->
director
);
hp
=
bo
->
bereq
;
bo
->
vbc
=
VDI_GetFd
(
NULL
,
bo
);
if
(
bo
->
vbc
==
NULL
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"no backend connection"
);
return
(
-
1
);
}
vc
=
bo
->
vbc
;
if
(
vc
->
recycled
)
retry
=
1
;
/*
* Now that we know our backend, we can set a default Host:
* header if one is necessary. This cannot be done in the VCL
* because the backend may be chosen by a director.
*/
if
(
!
http_GetHdr
(
bo
->
bereq
,
H_Host
,
NULL
))
VDI_AddHostHeader
(
bo
->
bereq
,
vc
);
(
void
)
VTCP_blocking
(
vc
->
fd
);
/* XXX: we should timeout instead */
WRW_Reserve
(
wrk
,
&
vc
->
fd
,
bo
->
vsl
,
bo
->
t_fetch
);
(
void
)
HTTP1_Write
(
wrk
,
hp
,
0
);
/* XXX: stats ? */
/* Deal with any message-body the request might (still) have */
i
=
0
;
if
(
req
!=
NULL
)
{
i
=
HTTP1_IterateReqBody
(
req
,
vbf_iter_req_body
,
wrk
);
if
(
req
->
req_body_status
==
REQ_BODY_DONE
)
retry
=
-
1
;
}
if
(
WRW_FlushRelease
(
wrk
)
||
i
!=
0
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"backend write error: %d (%s)"
,
errno
,
strerror
(
errno
));
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
retry
);
}
/* XXX is this the right place? */
VSC_C_main
->
backend_req
++
;
/* Receive response */
HTTP1_Init
(
htc
,
bo
->
ws
,
vc
->
fd
,
vc
->
vsl
,
cache_param
->
http_resp_size
,
cache_param
->
http_resp_hdr_len
);
VTCP_set_read_timeout
(
vc
->
fd
,
vc
->
first_byte_timeout
);
first
=
1
;
do
{
hs
=
HTTP1_Rx
(
htc
);
if
(
hs
==
HTTP1_OVERFLOW
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"http %sread error: overflow"
,
first
?
"first "
:
""
);
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
-
1
);
}
if
(
hs
==
HTTP1_ERROR_EOF
)
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"http %sread error: EOF"
,
first
?
"first "
:
""
);
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
retry
);
}
if
(
first
)
{
retry
=
-
1
;
first
=
0
;
VTCP_set_read_timeout
(
vc
->
fd
,
vc
->
between_bytes_timeout
);
}
}
while
(
hs
!=
HTTP1_COMPLETE
);
hp
=
bo
->
beresp
;
if
(
HTTP1_DissectResponse
(
hp
,
htc
))
{
VSLb
(
bo
->
vsl
,
SLT_FetchError
,
"http format error"
);
VDI_CloseFd
(
&
bo
->
vbc
);
/* XXX: other cleanup ? */
return
(
-
1
);
}
return
(
0
);
}
/*--------------------------------------------------------------------
* This function is either called by the requesting thread OR by a
* dedicated body-fetch work-thread.
*
* We get passed the busyobj in the priv arg, and we inherit a
* refcount on it, which we must release, when done fetching.
*/
void
V1F_fetch_body
(
struct
worker
*
wrk
,
struct
busyobj
*
bo
)
{
int
cls
;
struct
storage
*
st
;
int
mklen
;
ssize_t
cl
;
struct
http_conn
*
htc
;
struct
object
*
obj
;
CHECK_OBJ_NOTNULL
(
wrk
,
WORKER_MAGIC
);
CHECK_OBJ_NOTNULL
(
bo
,
BUSYOBJ_MAGIC
);
htc
=
&
bo
->
htc
;
CHECK_OBJ_ORNULL
(
bo
->
vbc
,
VBC_MAGIC
);
obj
=
bo
->
fetch_obj
;
CHECK_OBJ_NOTNULL
(
obj
,
OBJECT_MAGIC
);
CHECK_OBJ_NOTNULL
(
obj
->
http
,
HTTP_MAGIC
);
assert
(
bo
->
state
==
BOS_INVALID
);
/*
* XXX: The busyobj needs a dstat, but it is not obvious which one
* XXX: it should be (own/borrowed). For now borrow the wrk's.
*/
AZ
(
bo
->
stats
);
bo
->
stats
=
&
wrk
->
stats
;
AN
(
bo
->
vfp
);
AZ
(
bo
->
vgz_rx
);
AZ
(
VTAILQ_FIRST
(
&
obj
->
store
));
bo
->
state
=
BOS_FETCHING
;
/* XXX: pick up estimate from objdr ? */
cl
=
0
;
cls
=
bo
->
should_close
;
switch
(
htc
->
body_status
)
{
case
BS_NONE
:
mklen
=
0
;
break
;
case
BS_ZERO
:
mklen
=
1
;
break
;
case
BS_LENGTH
:
cl
=
vbf_fetch_number
(
bo
->
h_content_length
,
10
);
bo
->
vfp
->
begin
(
bo
,
cl
);
if
(
bo
->
state
==
BOS_FETCHING
&&
cl
>
0
)
cls
|=
vbf_fetch_straight
(
bo
,
htc
,
cl
);
mklen
=
1
;
if
(
bo
->
vfp
->
end
(
bo
))
assert
(
bo
->
state
==
BOS_FAILED
);
break
;
case
BS_CHUNKED
:
bo
->
vfp
->
begin
(
bo
,
cl
>
0
?
cl
:
0
);
if
(
bo
->
state
==
BOS_FETCHING
)
cls
|=
vbf_fetch_chunked
(
bo
,
htc
);
mklen
=
1
;
if
(
bo
->
vfp
->
end
(
bo
))
assert
(
bo
->
state
==
BOS_FAILED
);
break
;
case
BS_EOF
:
bo
->
vfp
->
begin
(
bo
,
cl
>
0
?
cl
:
0
);
if
(
bo
->
state
==
BOS_FETCHING
)
vbf_fetch_eof
(
bo
,
htc
);
mklen
=
1
;
cls
=
1
;
if
(
bo
->
vfp
->
end
(
bo
))
assert
(
bo
->
state
==
BOS_FAILED
);
break
;
case
BS_ERROR
:
cls
|=
VBF_Error
(
bo
,
"error incompatible Transfer-Encoding"
);
mklen
=
0
;
break
;
default:
mklen
=
0
;
INCOMPL
();
}
AZ
(
bo
->
vgz_rx
);
#if 0
/*
* We always call vfp_nop_end() to ditch or trim the last storage
* segment, to avoid having to replicate that code in all vfp's.
*/
AZ(vfp_nop_end(bo));
#endif
bo
->
vfp
=
NULL
;
VSLb
(
bo
->
vsl
,
SLT_Fetch_Body
,
"%u(%s) cls %d mklen %d"
,
htc
->
body_status
,
body_status_2str
(
htc
->
body_status
),
cls
,
mklen
);
http_Teardown
(
bo
->
bereq
);
http_Teardown
(
bo
->
beresp
);
if
(
bo
->
vbc
!=
NULL
)
{
if
(
cls
)
VDI_CloseFd
(
&
bo
->
vbc
);
else
VDI_RecycleFd
(
&
bo
->
vbc
);
}
if
(
bo
->
state
==
BOS_FAILED
)
{
wrk
->
stats
.
fetch_failed
++
;
obj
->
len
=
0
;
EXP_Clr
(
&
obj
->
exp
);
EXP_Rearm
(
obj
);
}
else
{
assert
(
bo
->
state
==
BOS_FETCHING
);
VSLb
(
bo
->
vsl
,
SLT_Length
,
"%zd"
,
obj
->
len
);
{
/* Sanity check fetch methods accounting */
ssize_t
uu
;
uu
=
0
;
VTAILQ_FOREACH
(
st
,
&
obj
->
store
,
list
)
uu
+=
st
->
len
;
if
(
bo
->
do_stream
)
/* Streaming might have started freeing stuff */
assert
(
uu
<=
obj
->
len
);
else
assert
(
uu
==
obj
->
len
);
}
if
(
mklen
>
0
)
{
http_Unset
(
obj
->
http
,
H_Content_Length
);
http_PrintfHeader
(
obj
->
http
,
"Content-Length: %zd"
,
obj
->
len
);
}
/* XXX: Atomic assignment, needs volatile/membar ? */
bo
->
state
=
BOS_FINISHED
;
}
if
(
obj
->
objcore
->
objhead
!=
NULL
)
HSH_Complete
(
obj
->
objcore
);
bo
->
stats
=
NULL
;
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment