Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
V
varnish-cache
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Commits
Open sidebar
varnishcache
varnish-cache
Commits
4094cf5b
Commit
4094cf5b
authored
Dec 10, 2017
by
Nils Goroll
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
retire shard .key() and .reconfigure() algorithm choice
Ref: #2500
parent
b4d6b714
Changes
18
Show whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
147 additions
and
330 deletions
+147
-330
d00017.vtc
bin/varnishtest/tests/d00017.vtc
+7
-5
d00019.vtc
bin/varnishtest/tests/d00019.vtc
+1
-1
d00020.vtc
bin/varnishtest/tests/d00020.vtc
+0
-50
d00021.vtc
bin/varnishtest/tests/d00021.vtc
+0
-81
d00022.vtc
bin/varnishtest/tests/d00022.vtc
+1
-1
d00023.vtc
bin/varnishtest/tests/d00023.vtc
+1
-3
d00024.vtc
bin/varnishtest/tests/d00024.vtc
+1
-1
d00026.vtc
bin/varnishtest/tests/d00026.vtc
+8
-6
changes.rst
doc/changes.rst
+45
-0
Makefile.am
lib/libvmod_directors/Makefile.am
+0
-2
shard_cfg.c
lib/libvmod_directors/shard_cfg.c
+7
-11
shard_cfg.h
lib/libvmod_directors/shard_cfg.h
+1
-1
shard_dir.c
lib/libvmod_directors/shard_dir.c
+43
-0
shard_dir.h
lib/libvmod_directors/shard_dir.h
+2
-0
shard_hash.c
lib/libvmod_directors/shard_hash.c
+0
-110
shard_hash.h
lib/libvmod_directors/shard_hash.h
+0
-30
vmod.vcc
lib/libvmod_directors/vmod.vcc
+16
-10
vmod_shard.c
lib/libvmod_directors/vmod_shard.c
+14
-18
No files found.
bin/varnishtest/tests/d00017.vtc
View file @
4094cf5b
...
@@ -25,6 +25,7 @@ server s3 {
...
@@ -25,6 +25,7 @@ server s3 {
varnish v1 -vcl+backend {
varnish v1 -vcl+backend {
import std;
import std;
import directors;
import directors;
import blob;
sub vcl_init {
sub vcl_init {
new vd = directors.shard();
new vd = directors.shard();
...
@@ -44,8 +45,9 @@ varnish v1 -vcl+backend {
...
@@ -44,8 +45,9 @@ varnish v1 -vcl+backend {
}
}
sub vcl_recv {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
set req.backend_hint = vd.backend(by=BLOB,
key=vd.key(req.url, CRC32));
key_blob=blob.decode(HEX, encoded=
regsub(req.url, "^/", "")));
return(pass);
return(pass);
}
}
...
@@ -142,15 +144,15 @@ logexpect l1 -v v1 -g raw -d 1 {
...
@@ -142,15 +144,15 @@ logexpect l1 -v v1 -g raw -d 1 {
} -start
} -start
client c1 {
client c1 {
txreq -url /
eishoSu2
txreq -url /
68b902f7
rxresp
rxresp
expect resp.body == "ech3Ooj"
expect resp.body == "ech3Ooj"
txreq -url /
Zainao9d
txreq -url /
39dc4613
rxresp
rxresp
expect resp.body == "ieQu2qua"
expect resp.body == "ieQu2qua"
txreq -url /
Aunah3uo
txreq -url /
c7793505
rxresp
rxresp
expect resp.body == "xiuFi3Pe"
expect resp.body == "xiuFi3Pe"
} -run
} -run
...
...
bin/varnishtest/tests/d00019.vtc
View file @
4094cf5b
varnishtest "shard director
SHA256
(default)"
varnishtest "shard director
by req.url
(default)"
server s1 {
server s1 {
rxreq
rxreq
...
...
bin/varnishtest/tests/d00020.vtc
deleted
100644 → 0
View file @
b4d6b714
varnishtest "shard director RS"
server s1 {
rxreq
txresp -body "ech3Ooj"
} -start
server s2 {
rxreq
txresp -body "ieQu2qua"
} -start
server s3 {
rxreq
txresp -body "xiuFi3Pe"
} -start
varnish v1 -vcl+backend {
import directors;
sub vcl_init {
new vd = directors.shard();
vd.add_backend(s1);
vd.add_backend(s2);
vd.add_backend(s3);
vd.reconfigure(replicas=25);
}
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
key=vd.key(req.url, alg=RS));
return(pass);
}
} -start
client c1 {
txreq -url /we0eeTho
rxresp
expect resp.body == "ech3Ooj"
txreq -url /mae8ooNu
rxresp
expect resp.body == "ieQu2qua"
txreq -url /oob3dahS
rxresp
expect resp.body == "xiuFi3Pe"
} -run
bin/varnishtest/tests/d00021.vtc
deleted
100644 → 0
View file @
b4d6b714
varnishtest "shard director key function"
server s1 {
rxreq
txresp -body "ech3Ooj"
rxreq
txresp -body "ech3Ooj"
rxreq
txresp -body "ech3Ooj"
} -start
server s2 {
rxreq
txresp -body "ieQu2qua"
} -start
server s3 {
rxreq
txresp -body "xiuFi3Pe"
} -start
varnish v1 -vcl+backend {
import directors;
sub vcl_init {
new vd = directors.shard();
vd.add_backend(s1);
vd.add_backend(s2);
vd.add_backend(s3);
vd.reconfigure(25);
}
sub recv_sub {
set req.backend_hint = vd.backend(by=KEY,
key=vd.key(req.http.X-Hash, RS));
}
sub vcl_recv {
if (req.url == "/1") {
set req.backend_hint = vd.backend(by=KEY,
key=vd.key(alg=CRC32, string="/eishoSu2"));
} else if (req.url == "/2") {
set req.backend_hint = vd.backend(by=KEY,
key=vd.key("/eishoSu2"));
} else if (req.url == "/3") {
set req.http.X-Hash = "/oob3dahS";
call recv_sub;
} else if (req.url == "/null_by_string") {
set req.backend_hint = vd.backend(by=KEY,
key=vd.key(req.http.NonExistent));
} else if (req.url == "/null_by_string_hash") {
set req.backend_hint = vd.backend(by=KEY,
key=vd.key(req.http.NonExistent, SHA256));
}
return(pass);
}
} -start
client c1 {
txreq -url /1
rxresp
expect resp.body == "ech3Ooj"
txreq -url /2
rxresp
expect resp.body == "ieQu2qua"
txreq -url /3
rxresp
expect resp.body == "xiuFi3Pe"
txreq -url /null_by_string
rxresp
expect resp.body == "ech3Ooj"
txreq -url /null_by_string_hash
rxresp
expect resp.body == "ech3Ooj"
} -run
bin/varnishtest/tests/d00022.vtc
View file @
4094cf5b
...
@@ -40,7 +40,7 @@ varnish v1 -vcl+backend {
...
@@ -40,7 +40,7 @@ varnish v1 -vcl+backend {
sub vcl_recv {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
set req.backend_hint = vd.backend(by=KEY,
key=
vd.key("/eishoSu2", CRC32)
,
key=
1756955383
,
alt=req.restarts,
alt=req.restarts,
healthy=ALL);
healthy=ALL);
...
...
bin/varnishtest/tests/d00023.vtc
View file @
4094cf5b
varnishtest "shard director Unhealthy"
varnishtest "shard director Unhealthy"
server s1 {
server s1 {
rxreq
txresp -body "ech3Ooj"
} -start
} -start
server s2 {
server s2 {
...
@@ -32,7 +30,7 @@ varnish v1 -vcl+backend {
...
@@ -32,7 +30,7 @@ varnish v1 -vcl+backend {
sub vcl_recv {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
set req.backend_hint = vd.backend(by=KEY,
key=
vd.key("/eishoSu2", CRC32)
);
key=
1756955383
);
set req.http.healthy = std.healthy(req.backend_hint);
set req.http.healthy = std.healthy(req.backend_hint);
return(pass);
return(pass);
}
}
...
...
bin/varnishtest/tests/d00024.vtc
View file @
4094cf5b
...
@@ -35,7 +35,7 @@ varnish v1 -vcl+backend {
...
@@ -35,7 +35,7 @@ varnish v1 -vcl+backend {
sub vcl_recv {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
set req.backend_hint = vd.backend(by=KEY,
key=
vd.key(alg=CRC32, string="/eishoSu2")
);
key=
1756955383
);
return(pass);
return(pass);
}
}
} -start
} -start
...
...
bin/varnishtest/tests/d00026.vtc
View file @
4094cf5b
varnishtest "shard director - same as
v01000
.vtc but setting backend in fetch"
varnishtest "shard director - same as
d00017
.vtc but setting backend in fetch"
server s1 {
server s1 {
rxreq
rxreq
...
@@ -17,6 +17,7 @@ server s3 {
...
@@ -17,6 +17,7 @@ server s3 {
varnish v1 -vcl+backend {
varnish v1 -vcl+backend {
import directors;
import directors;
import blob;
sub vcl_init {
sub vcl_init {
new vd = directors.shard();
new vd = directors.shard();
...
@@ -27,8 +28,9 @@ varnish v1 -vcl+backend {
...
@@ -27,8 +28,9 @@ varnish v1 -vcl+backend {
}
}
sub vcl_backend_fetch {
sub vcl_backend_fetch {
set bereq.backend = vd.backend(by=KEY,
set bereq.backend = vd.backend(by=BLOB,
key=vd.key(bereq.url, CRC32));
key_blob=blob.decode(HEX, encoded=
regsub(bereq.url, "^/", "")));
return(fetch);
return(fetch);
}
}
...
@@ -36,15 +38,15 @@ varnish v1 -vcl+backend {
...
@@ -36,15 +38,15 @@ varnish v1 -vcl+backend {
client c1 {
client c1 {
txreq -url /
eishoSu2
txreq -url /
68b902f7
rxresp
rxresp
expect resp.body == "ech3Ooj"
expect resp.body == "ech3Ooj"
txreq -url /
Zainao9d
txreq -url /
39dc4613
rxresp
rxresp
expect resp.body == "ieQu2qua"
expect resp.body == "ieQu2qua"
txreq -url /
Aunah3uo
txreq -url /
c7793505
rxresp
rxresp
expect resp.body == "xiuFi3Pe"
expect resp.body == "xiuFi3Pe"
} -run
} -run
doc/changes.rst
View file @
4094cf5b
...
@@ -59,6 +59,51 @@ VCL and bundled VMODs
...
@@ -59,6 +59,51 @@ VCL and bundled VMODs
* added ``return(restart)`` from ``vcl_recv{}``
* added ``return(restart)`` from ``vcl_recv{}``
* The ``alg`` argument of the ``shard`` director ``.reconfigure()``
method has been removed - the consistent hashing ring is now always
generated using the last 32 bits of a SHA256 hash of ``"ident%d"``
as with ``alg=SHA256`` or the default.
We believe that the other algorithms did not yield sufficiently
dispersed placement of backends on the consistent hashing ring and
thus retire this option without replacement.
Users of ``.reconfigure(alg=CRC32)`` or ``.reconfigure(alg=RS)`` be
advised that when upgrading and removing the ``alg`` argument,
consistent hashing values for all backends will change once and only
once.
* The ``alg`` argument of the ``shard`` director ``.key()`` method has
been removed - it now always hashes its arguments using SHA256 and
returns the last 32 bits for use as a shard key.
Backwards compatibility is provided through `vmod blobdigest`_ with
the ``key_blob`` argument of the ``shard`` director ``.backend()``
method:
* for ``alg=CRC32``, replace::
<dir>.backend(by=KEY, key=<dir>.key(<string>, CRC32))
with::
<dir>.backend(by=BLOB, key_blob=blobdigest.hash(ICRC32,
blob.decode(encoded=<string>)))
`Note:` The `vmod blobdigest`_ hash method corresponding to the
shard director CRC32 method is called **I**\ CRC32
.. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst
* for ``alg=RS``, replace::
<dir>.backend(by=KEY, key=<dir>.key(<string>, RS))
with::
<dir>.backend(by=BLOB, key_blob=blobdigest.hash(RS,
blob.decode(encoded=<string>)))
Logging / statistics
Logging / statistics
--------------------
--------------------
...
...
lib/libvmod_directors/Makefile.am
View file @
4094cf5b
...
@@ -12,8 +12,6 @@ libvmod_directors_la_SOURCES = \
...
@@ -12,8 +12,6 @@ libvmod_directors_la_SOURCES = \
shard_cfg.h
\
shard_cfg.h
\
shard_dir.c
\
shard_dir.c
\
shard_dir.h
\
shard_dir.h
\
shard_hash.c
\
shard_hash.h
\
shard_parse_vcc_enums.h
\
shard_parse_vcc_enums.h
\
shard_parse_vcc_enums.c
shard_parse_vcc_enums.c
...
...
lib/libvmod_directors/shard_cfg.c
View file @
4094cf5b
...
@@ -39,7 +39,6 @@
...
@@ -39,7 +39,6 @@
#include "shard_dir.h"
#include "shard_dir.h"
#include "shard_cfg.h"
#include "shard_cfg.h"
#include "shard_hash.h"
/*lint -esym(749, shard_change_task_e::*) */
/*lint -esym(749, shard_change_task_e::*) */
enum
shard_change_task_e
{
enum
shard_change_task_e
{
...
@@ -232,11 +231,12 @@ circlepoint_compare(const struct shard_circlepoint *a,
...
@@ -232,11 +231,12 @@ circlepoint_compare(const struct shard_circlepoint *a,
}
}
static
void
static
void
shardcfg_hashcircle
(
struct
sharddir
*
shardd
,
VCL_INT
replicas
,
enum
alg_e
alg
)
shardcfg_hashcircle
(
struct
sharddir
*
shardd
,
VCL_INT
replicas
)
{
{
int
i
,
j
;
int
i
,
j
;
const
char
*
ident
;
const
char
*
ident
;
int
len
;
const
int
len
=
12
;
// log10(UINT32_MAX) + 2;
char
s
[
len
];
CHECK_OBJ_NOTNULL
(
shardd
,
SHARDDIR_MAGIC
);
CHECK_OBJ_NOTNULL
(
shardd
,
SHARDDIR_MAGIC
);
AZ
(
shardd
->
hashcircle
);
AZ
(
shardd
->
hashcircle
);
...
@@ -259,14 +259,10 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg)
...
@@ -259,14 +259,10 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg)
assert
(
ident
[
0
]
!=
'\0'
);
assert
(
ident
[
0
]
!=
'\0'
);
len
=
strlen
(
ident
)
+
12
;
// log10(UINT32_MAX) + 2;
char
s
[
len
];
for
(
j
=
0
;
j
<
replicas
;
j
++
)
{
for
(
j
=
0
;
j
<
replicas
;
j
++
)
{
assert
(
snprintf
(
s
,
len
,
"%
s%d"
,
ident
,
j
)
<
len
);
assert
(
snprintf
(
s
,
len
,
"%
d"
,
j
)
<
len
);
shardd
->
hashcircle
[
i
*
replicas
+
j
].
point
=
shardd
->
hashcircle
[
i
*
replicas
+
j
].
point
=
shard_hash_f
[
alg
](
s
);
sharddir_sha256
(
ident
,
s
,
vrt_magic_string_end
);
shardd
->
hashcircle
[
i
*
replicas
+
j
].
host
=
i
;
shardd
->
hashcircle
[
i
*
replicas
+
j
].
host
=
i
;
}
}
/* not used in current interface */
/* not used in current interface */
...
@@ -574,7 +570,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
...
@@ -574,7 +570,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
VCL_BOOL
VCL_BOOL
shardcfg_reconfigure
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
shardcfg_reconfigure
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
struct
sharddir
*
shardd
,
VCL_INT
replicas
,
enum
alg_e
alg
)
struct
sharddir
*
shardd
,
VCL_INT
replicas
)
{
{
struct
shard_change
*
change
;
struct
shard_change
*
change
;
...
@@ -607,7 +603,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
...
@@ -607,7 +603,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
return
0
;
return
0
;
}
}
shardcfg_hashcircle
(
shardd
,
replicas
,
alg
);
shardcfg_hashcircle
(
shardd
,
replicas
);
sharddir_unlock
(
shardd
);
sharddir_unlock
(
shardd
);
return
(
1
);
return
(
1
);
}
}
...
...
lib/libvmod_directors/shard_cfg.h
View file @
4094cf5b
...
@@ -34,7 +34,7 @@ VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
...
@@ -34,7 +34,7 @@ VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
VCL_BOOL
shardcfg_clear
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
VCL_BOOL
shardcfg_clear
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
const
struct
sharddir
*
shardd
);
const
struct
sharddir
*
shardd
);
VCL_BOOL
shardcfg_reconfigure
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
VCL_BOOL
shardcfg_reconfigure
(
VRT_CTX
,
struct
vmod_priv
*
priv
,
struct
sharddir
*
shardd
,
VCL_INT
replicas
,
enum
alg_e
alg_e
);
struct
sharddir
*
shardd
,
VCL_INT
replicas
);
VCL_VOID
shardcfg_set_warmup
(
struct
sharddir
*
shardd
,
VCL_REAL
ratio
);
VCL_VOID
shardcfg_set_warmup
(
struct
sharddir
*
shardd
,
VCL_REAL
ratio
);
VCL_VOID
shardcfg_set_rampup
(
struct
sharddir
*
shardd
,
VCL_VOID
shardcfg_set_rampup
(
struct
sharddir
*
shardd
,
VCL_DURATION
duration
);
VCL_DURATION
duration
);
lib/libvmod_directors/shard_dir.c
View file @
4094cf5b
...
@@ -41,6 +41,8 @@
...
@@ -41,6 +41,8 @@
#include "vbm.h"
#include "vbm.h"
#include "vrnd.h"
#include "vrnd.h"
#include "vsha256.h"
#include "vend.h"
#include "shard_dir.h"
#include "shard_dir.h"
...
@@ -87,6 +89,47 @@ sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...)
...
@@ -87,6 +89,47 @@ sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...)
va_end
(
ap
);
va_end
(
ap
);
}
}
uint32_t
sharddir_sha256v
(
const
char
*
s
,
va_list
ap
)
{
struct
VSHA256Context
sha256
;
union
{
unsigned
char
digest
[
32
];
uint32_t
uint32_digest
[
8
];
}
sha256_digest
;
uint32_t
r
;
const
char
*
p
;
VSHA256_Init
(
&
sha256
);
p
=
s
;
while
(
p
!=
vrt_magic_string_end
)
{
if
(
p
!=
NULL
&&
*
p
!=
'\0'
)
VSHA256_Update
(
&
sha256
,
p
,
strlen
(
p
));
p
=
va_arg
(
ap
,
const
char
*
);
}
VSHA256_Final
(
sha256_digest
.
digest
,
&
sha256
);
/*
* use low 32 bits only
* XXX: Are these the best bits to pick?
*/
vle32enc
(
&
r
,
sha256_digest
.
uint32_digest
[
7
]);
return
(
r
);
}
uint32_t
sharddir_sha256
(
const
char
*
s
,
...)
{
va_list
ap
;
uint32_t
r
;
va_start
(
ap
,
s
);
r
=
sharddir_sha256v
(
s
,
ap
);
va_end
(
ap
);
return
(
r
);
}
static
int
static
int
shard_lookup
(
const
struct
sharddir
*
shardd
,
const
uint32_t
key
)
shard_lookup
(
const
struct
sharddir
*
shardd
,
const
uint32_t
key
)
{
{
...
...
lib/libvmod_directors/shard_dir.h
View file @
4094cf5b
...
@@ -103,6 +103,8 @@ sharddir_backend_ident(const struct sharddir *shardd, int host)
...
@@ -103,6 +103,8 @@ sharddir_backend_ident(const struct sharddir *shardd, int host)
void
sharddir_debug
(
struct
sharddir
*
shardd
,
const
uint32_t
flags
);
void
sharddir_debug
(
struct
sharddir
*
shardd
,
const
uint32_t
flags
);
void
sharddir_err
(
VRT_CTX
,
enum
VSL_tag_e
tag
,
const
char
*
fmt
,
...);
void
sharddir_err
(
VRT_CTX
,
enum
VSL_tag_e
tag
,
const
char
*
fmt
,
...);
uint32_t
sharddir_sha256v
(
const
char
*
s
,
va_list
ap
);
uint32_t
sharddir_sha256
(
const
char
*
s
,
...);
void
sharddir_new
(
struct
sharddir
**
sharddp
,
const
char
*
vcl_name
);
void
sharddir_new
(
struct
sharddir
**
sharddp
,
const
char
*
vcl_name
);
void
sharddir_delete
(
struct
sharddir
**
sharddp
);
void
sharddir_delete
(
struct
sharddir
**
sharddp
);
void
sharddir_wrlock
(
struct
sharddir
*
shardd
);
void
sharddir_wrlock
(
struct
sharddir
*
shardd
);
...
...
lib/libvmod_directors/shard_hash.c
deleted
100644 → 0
View file @
b4d6b714
/*-
* Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung
* All rights reserved.
*
* Authors: Nils Goroll <nils.goroll@uplex.de>
* Geoffrey Simmons <geoff.simmons@uplex.de>
* Julian Wiesener <jw@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "config.h"
#include <string.h>
#include "cache/cache.h"
#include "vsha256.h"
#include "vend.h"
#include "shard_parse_vcc_enums.h"
#include "shard_hash.h"
/*
* XXX use the crc32 from libvgz, but declare it here to avoid an include
* dependency nightmare (at least for now)
*/
unsigned
long
crc32
(
unsigned
long
,
const
unsigned
char
*
buf
,
unsigned
len
);
static
uint32_t
v_matchproto_
(
hash_func
)
shard_hash_crc32
(
VCL_STRING
s
)
{
uint32_t
crc
;
crc
=
crc32
(
~
0U
,
(
const
unsigned
char
*
)
s
,
strlen
(
s
));
crc
^=
~
0U
;
return
(
crc
);
}
static
uint32_t
v_matchproto_
(
hash_func
)
shard_hash_sha256
(
VCL_STRING
s
)
{
struct
VSHA256Context
sha256
;
union
{
unsigned
char
digest
[
32
];
uint32_t
uint32_digest
[
8
];
}
sha256_digest
;
uint32_t
r
;
VSHA256_Init
(
&
sha256
);
VSHA256_Update
(
&
sha256
,
s
,
strlen
(
s
));
VSHA256_Final
(
sha256_digest
.
digest
,
&
sha256
);
/*
* use low 32 bits only
* XXX: Are these the best bits to pick?
*/
vle32enc
(
&
r
,
sha256_digest
.
uint32_digest
[
7
]);
return
(
r
);
}
static
uint32_t
v_matchproto_
(
hash_func
)
shard_hash_rs
(
VCL_STRING
s
)
{
uint32_t
res
=
0
;
/* hash function from Robert Sedgwicks 'Algorithms in C' book */
const
uint32_t
b
=
378551
;
uint32_t
a
=
63689
;
while
(
*
s
)
{
res
=
res
*
a
+
(
*
s
++
);
a
*=
b
;
}
return
(
res
);
}
static
uint32_t
v_matchproto_
(
hash_func
)
_shard_hash_invalid
(
VCL_STRING
s
)
{
(
void
)
s
;
WRONG
(
"invalid hash fp _ALG_E_ENVALID"
);
NEEDLESS
(
return
(
0
));
}
const
hash_func
shard_hash_f
[
_ALG_E_MAX
]
=
{
[
_ALG_E_INVALID
]
=
_shard_hash_invalid
,
[
CRC32
]
=
shard_hash_crc32
,
[
SHA256
]
=
shard_hash_sha256
,
[
RS
]
=
shard_hash_rs
};
lib/libvmod_directors/shard_hash.h
deleted
100644 → 0
View file @
b4d6b714
/*-
* Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung
* All rights reserved.
*
* Author: Julian Wiesener <jw@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
typedef
uint32_t
(
*
hash_func
)(
VCL_STRING
);
extern
const
hash_func
shard_hash_f
[
_ALG_E_MAX
];
lib/libvmod_directors/vmod.vcc
View file @
4094cf5b
...
@@ -307,10 +307,11 @@ Method
...
@@ -307,10 +307,11 @@ Method
``````
``````
When ``.reconfigure()`` is called, a consistent hashing circular data
When ``.reconfigure()`` is called, a consistent hashing circular data
structure gets built from hash values of "ident%d" (default ident
structure gets built from the last 32 bits of SHA256 hash values of
being the backend name) for each backend and for a running number from
`<ident>`\ `<n>` (default `ident` being the backend name) for each
1 to n (n is the number of `replicas`). Hashing creates the seemingly
backend and for a running number `n` from 1 to `replicas`. Hashing
random order for placement of backends on the consistent hashing ring.
creates the seemingly random order for placement of backends on the
consistent hashing ring.
When ``.backend()`` is called, a load balancing key gets generated
When ``.backend()`` is called, a load balancing key gets generated
unless provided. The smallest hash value in the circle is looked up
unless provided. The smallest hash value in the circle is looked up
...
@@ -388,19 +389,24 @@ Remove all backends from the director.
...
@@ -388,19 +389,24 @@ Remove all backends from the director.
NOTE: Backend changes need to be finalized with `shard.reconfigure()`
NOTE: Backend changes need to be finalized with `shard.reconfigure()`
and are only supported on one shard director at a time.
and are only supported on one shard director at a time.
$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67,
$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67)
ENUM { CRC32, SHA256, RS } alg="SHA256")
Reconfigure the consistent hashing ring to reflect backend changes.
Reconfigure the consistent hashing ring to reflect backend changes.
This method must be called at least once before the director can be
This method must be called at least once before the director can be
used.
used.
$Method INT .key(STRING
string, ENUM { CRC32, SHA256, RS } alg="SHA256"
)
$Method INT .key(STRING
_LIST
)
Utility method to generate a sharding key for use with the
Convenience method to generate a sharding key for use with the `key`
``shard.backend()`` method by hashing `string` with hash algorithm
argument to the ``shard.backend()`` method by hashing the given string
`alg`.
with SHA256.
To generate sharding keys using other hashes, use a custom vmod like
`vmod blobdigest`_ with the `key_blob` argument of the
``shard.backend()`` method.
.. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst
$Method BACKEND .backend(
$Method BACKEND .backend(
ENUM {HASH, URL, KEY, BLOB} by="HASH",
ENUM {HASH, URL, KEY, BLOB} by="HASH",
...
...
lib/libvmod_directors/vmod_shard.c
View file @
4094cf5b
...
@@ -40,7 +40,6 @@
...
@@ -40,7 +40,6 @@
#include "vcc_if.h"
#include "vcc_if.h"
#include "shard_dir.h"
#include "shard_dir.h"
#include "shard_cfg.h"
#include "shard_cfg.h"
#include "shard_hash.h"
struct
vmod_directors_shard
{
struct
vmod_directors_shard
{
unsigned
magic
;
unsigned
magic
;
...
@@ -56,7 +55,7 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp,
...
@@ -56,7 +55,7 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp,
VCL_INT
t1
;
VCL_INT
t1
;
uint32_t
t2a
,
t2b
;
uint32_t
t2a
,
t2b
;
/*
see vmod_key comment
*/
/*
we put our uint32 key in a VCL_INT container
*/
assert
(
sizeof
(
VCL_INT
)
>=
sizeof
(
uint32_t
));
assert
(
sizeof
(
VCL_INT
)
>=
sizeof
(
uint32_t
));
t2a
=
UINT32_MAX
;
t2a
=
UINT32_MAX
;
t1
=
(
VCL_INT
)
t2a
;
t1
=
(
VCL_INT
)
t2a
;
...
@@ -84,22 +83,20 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp)
...
@@ -84,22 +83,20 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp)
FREE_OBJ
(
vshard
);
FREE_OBJ
(
vshard
);
}
}
/*
* our key is a uint32_t, but VCL_INT is a (signed) long. We cast back and
* forth, asserting in vmod_shard__init() that VCL_INT is a large enough
* container
*/
VCL_INT
v_matchproto_
(
td_directors_shard_key
)
VCL_INT
v_matchproto_
(
td_directors_shard_key
)
vmod_shard_key
(
VRT_CTX
,
struct
vmod_directors_shard
*
vshard
,
vmod_shard_key
(
VRT_CTX
,
struct
vmod_directors_shard
*
vshard
,
const
char
*
s
,
...)
VCL_STRING
s
,
VCL_ENUM
alg_s
)
{
{
enum
alg_e
alg
=
parse_alg_e
(
alg_s
)
;
va_list
ap
;
hash_func
hash_fp
=
shard_hash_f
[
alg
]
;
uint32_t
r
;
(
void
)
ctx
;
(
void
)
ctx
;
(
void
)
vshard
;;
(
void
)
vshard
;
va_start
(
ap
,
s
);
r
=
sharddir_sha256v
(
s
,
ap
);
va_end
(
ap
);
return
(
VCL_INT
)
hash_fp
(
s
?
s
:
""
);
return
(
(
VCL_INT
)
r
);
}
}
VCL_VOID
v_matchproto_
(
td_directors_set_warmup
)
VCL_VOID
v_matchproto_
(
td_directors_set_warmup
)
...
@@ -169,11 +166,9 @@ vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard,
...
@@ -169,11 +166,9 @@ vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard,
VCL_BOOL
v_matchproto_
(
td_directors_shard_reconfigure
)
VCL_BOOL
v_matchproto_
(
td_directors_shard_reconfigure
)
vmod_shard_reconfigure
(
VRT_CTX
,
struct
vmod_directors_shard
*
vshard
,
vmod_shard_reconfigure
(
VRT_CTX
,
struct
vmod_directors_shard
*
vshard
,
struct
vmod_priv
*
priv
,
VCL_INT
replicas
,
VCL_ENUM
alg_s
)
struct
vmod_priv
*
priv
,
VCL_INT
replicas
)
{
{
enum
alg_e
alg
=
parse_alg_e
(
alg_s
);
return
shardcfg_reconfigure
(
ctx
,
priv
,
vshard
->
shardd
,
replicas
);
return
shardcfg_reconfigure
(
ctx
,
priv
,
vshard
->
shardd
,
replicas
,
alg
);
}
}
static
inline
uint32_t
static
inline
uint32_t
...
@@ -198,7 +193,8 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob)
...
@@ -198,7 +193,8 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob)
AN
(
ctx
->
http_bereq
);
AN
(
ctx
->
http_bereq
);
AN
(
http
=
ctx
->
http_bereq
);
AN
(
http
=
ctx
->
http_bereq
);
}
}
return
(
shard_hash_f
[
SHA256
](
http
->
hd
[
HTTP_HDR_URL
].
b
));
return
(
sharddir_sha256
(
http
->
hd
[
HTTP_HDR_URL
].
b
,
vrt_magic_string_end
));
case
BY_KEY
:
case
BY_KEY
:
return
((
uint32_t
)
key_int
);
return
((
uint32_t
)
key_int
);
case
BY_BLOB
:
case
BY_BLOB
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment