Skip to content

k8s-ingress
k8s-ingress
Commit f2f71758 authored by Geoff Simmons's avatar Geoff Simmons
Browse files
Options

Another breaking change to fix & complete self-sharding with conditions. 

The self-sharding field in VarnishConfig now has a mandatory field
rules. This is an array of at least one object with a mandatory
field shard and an optional field conditions.

spec.self-sharding.rules[n].shard configures sharding in the fields
key, digest and primaryOnly (all optional).

spec.self-sharding.rules[n].conditions, if specified, is an array
of at least one condition, which specifies a boolean expression.
It is the same as conditions used for other contexts such as
rewrites.

The rules array is evaluated in order as an if-elsif-else sequence.
This means that if any element of the array specifies no conditions,
it MUST be the last rule in the array. In an array of length > 1,
it becomes the else case. This constraint is trivially satisfied
if there is only one rule. It is currently *not* enforced (would be
suitable for a webhook validation).

The fields probe and max2ndttl are now top-level fields under
self-sharding, both optional.
parent 9f4f9f47
Hide whitespace changes
Inline Side-by-side
Showing with 787 additions and 496 deletions
charts/viking-controller/crds/varnishcfg-crd.yaml
View file @ f2f71758
......@@ -33,108 +33,118 @@ spec:
type: string
self-sharding:
type: object
required:
- rules
properties:
shard:
probe:
type: object
properties:
max-secondary-ttl:
timeout:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
primaryOnly:
type: boolean
key:
interval:
type: string
pattern: "^client\\.identity$|^req\\.(url|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
digest:
type: string
enum:
- CRC32
- ICRC32
- MD5
- RS
- SHA1
- SHA224
- SHA256
- SHA384
- SHA512
- SHA3_224
- SHA3_256
- SHA3_512
probe:
type: object
properties:
timeout:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
interval:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
initial:
type: integer
minimum: 0
window:
type: integer
minimum: 0
maximum: 64
threshold:
type: integer
minimum: 0
maximum: 64
conditions:
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
initial:
type: integer
minimum: 0
window:
type: integer
minimum: 0
maximum: 64
threshold:
type: integer
minimum: 0
maximum: 64
max-secondary-ttl:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
rules:
type: array
minItems: 1
items:
type: object
required:
- comparand
- shard
properties:
comparand:
type: string
pattern: "^req\\.(url|method|proto|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
compare:
type: string
enum:
- equal
- not-equal
- match
- not-match
- prefix
- not-prefix
- exists
- not-exists
values:
type: array
minItems: 1
items:
type: string
match-flags:
shard:
type: object
properties:
max-mem:
type: integer
minimum: 0
anchor:
primaryOnly:
type: boolean
key:
type: string
pattern: "^client\\.identity$|^req\\.(url|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
digest:
type: string
enum:
- none
- start
- both
utf8:
type: boolean
posix-syntax:
type: boolean
longest-match:
type: boolean
literal:
type: boolean
never-capture:
type: boolean
case-sensitive:
type: boolean
perl-classes:
type: boolean
word-boundary:
type: boolean
- CRC32
- ICRC32
- MD5
- RS
- SHA1
- SHA224
- SHA256
- SHA384
- SHA512
- SHA3_224
- SHA3_256
- SHA3_512
conditions:
type: array
minItems: 1
items:
type: object
required:
- comparand
properties:
comparand:
type: string
pattern: "^req\\.(url|method|proto|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
compare:
type: string
enum:
- equal
- not-equal
- match
- not-match
- prefix
- not-prefix
- exists
- not-exists
values:
type: array
minItems: 1
items:
type: string
match-flags:
type: object
properties:
max-mem:
type: integer
minimum: 0
anchor:
type: string
enum:
- none
- start
- both
utf8:
type: boolean
posix-syntax:
type: boolean
longest-match:
type: boolean
literal:
type: boolean
never-capture:
type: boolean
case-sensitive:
type: boolean
perl-classes:
type: boolean
word-boundary:
type: boolean
auth:
type: array
minItems: 1
......
deploy/varnishcfg-crd.yaml
View file @ f2f71758
......@@ -33,108 +33,118 @@ spec:
type: string
self-sharding:
type: object
required:
- rules
properties:
shard:
probe:
type: object
properties:
max-secondary-ttl:
timeout:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
primaryOnly:
type: boolean
key:
interval:
type: string
pattern: "^client\\.identity$|^req\\.(url|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
digest:
type: string
enum:
- CRC32
- ICRC32
- MD5
- RS
- SHA1
- SHA224
- SHA256
- SHA384
- SHA512
- SHA3_224
- SHA3_256
- SHA3_512
probe:
type: object
properties:
timeout:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
interval:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
initial:
type: integer
minimum: 0
window:
type: integer
minimum: 0
maximum: 64
threshold:
type: integer
minimum: 0
maximum: 64
conditions:
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
initial:
type: integer
minimum: 0
window:
type: integer
minimum: 0
maximum: 64
threshold:
type: integer
minimum: 0
maximum: 64
max-secondary-ttl:
type: string
pattern: '^\d+(\.\d+)?(ms|[smhdwy])$'
rules:
type: array
minItems: 1
items:
type: object
required:
- comparand
- shard
properties:
comparand:
type: string
pattern: "^req\\.(url|method|proto|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
compare:
type: string
enum:
- equal
- not-equal
- match
- not-match
- prefix
- not-prefix
- exists
- not-exists
values:
type: array
minItems: 1
items:
type: string
match-flags:
shard:
type: object
properties:
max-mem:
type: integer
minimum: 0
anchor:
primaryOnly:
type: boolean
key:
type: string
pattern: "^client\\.identity$|^req\\.(url|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
digest:
type: string
enum:
- none
- start
- both
utf8:
type: boolean
posix-syntax:
type: boolean
longest-match:
type: boolean
literal:
type: boolean
never-capture:
type: boolean
case-sensitive:
type: boolean
perl-classes:
type: boolean
word-boundary:
type: boolean
- CRC32
- ICRC32
- MD5
- RS
- SHA1
- SHA224
- SHA256
- SHA384
- SHA512
- SHA3_224
- SHA3_256
- SHA3_512
conditions:
type: array
minItems: 1
items:
type: object
required:
- comparand
properties:
comparand:
type: string
pattern: "^req\\.(url|method|proto|http\\.[a-zA-Z0-9!#$%&'*+.^_`|~-]+)$"
compare:
type: string
enum:
- equal
- not-equal
- match
- not-match
- prefix
- not-prefix
- exists
- not-exists
values:
type: array
minItems: 1
items:
type: string
match-flags:
type: object
properties:
max-mem:
type: integer
minimum: 0
anchor:
type: string
enum:
- none
- start
- both
utf8:
type: boolean
posix-syntax:
type: boolean
longest-match:
type: boolean
literal:
type: boolean
never-capture:
type: boolean
case-sensitive:
type: boolean
perl-classes:
type: boolean
word-boundary:
type: boolean
auth:
type: array
minItems: 1
......
examples/self-sharding/primary-only-by-clientid.yaml
View file @ f2f71758
......@@ -19,12 +19,13 @@ spec:
# self-sharding: {}
#
self-sharding:
shard:
primaryOnly: true
key: client.identity
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- shard:
primaryOnly: true
key: client.identity
examples/self-sharding/primary-only.yaml
View file @ f2f71758
......@@ -19,11 +19,12 @@ spec:
# self-sharding: {}
#
self-sharding:
shard:
primaryOnly: true
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- shard:
primaryOnly: true
examples/self-sharding/self-sharding-cfg.yaml
View file @ f2f71758
......@@ -19,11 +19,12 @@ spec:
# self-sharding: {}
#
self-sharding:
shard:
max-secondary-ttl: 2m
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
max-secondary-ttl: 2m
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- shard: {}
examples/self-sharding/shard-by-digest.yaml
View file @ f2f71758
......@@ -19,7 +19,8 @@ spec:
# self-sharding: {}
#
self-sharding:
shard:
primaryOnly: true
key: req.http.Host
digest: SHA3_512
rules:
- shard:
primaryOnly: true
key: req.http.Host
digest: SHA3_512
examples/self-sharding/shard-by-key.yaml
View file @ f2f71758
......@@ -19,6 +19,7 @@ spec:
# self-sharding: {}
#
self-sharding:
shard:
primaryOnly: true
key: req.http.Host
rules:
- shard:
primaryOnly: true
key: req.http.Host
examples/self-sharding/shard-by-url.yaml
View file @ f2f71758
......@@ -19,6 +19,7 @@ spec:
# self-sharding: {}
#
self-sharding:
shard:
primaryOnly: true
key: req.url
rules:
- shard:
primaryOnly: true
key: req.url
examples/self-sharding/shard-conditions.yaml
View file @ f2f71758
......@@ -19,17 +19,20 @@ spec:
# self-sharding: {}
#
self-sharding:
conditions:
- comparand: req.url
compare: prefix
values:
- /foo/
shard:
primaryOnly: true
key: client.identity
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- conditions:
- comparand: req.url
compare: prefix
values:
- /foo/
shard:
primaryOnly: true
key: client.identity
- shard:
key: client.identity
examples/self-sharding/values-primary-only-by-clientid.yaml
View file @ f2f71758
......@@ -19,12 +19,13 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
shard:
primaryOnly: true
key: client.identity
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- shard:
primaryOnly: true
key: client.identity
examples/self-sharding/values-primary-only.yaml
View file @ f2f71758
......@@ -19,11 +19,12 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
shard:
primaryOnly: true
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- shard:
primaryOnly: true
examples/self-sharding/values-self-sharding.yaml
View file @ f2f71758
......@@ -19,11 +19,12 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
shard:
max-secondary-ttl: 2m
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
max-secondary-ttl: 2m
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- shard: {}
examples/self-sharding/values-shard-by-digest.yaml
View file @ f2f71758
......@@ -19,7 +19,8 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
shard:
primaryOnly: true
key: req.http.Host
digest: SHA3_512
rules:
- shard:
primaryOnly: true
key: req.http.Host
digest: SHA3_512
examples/self-sharding/values-shard-by-key.yaml
View file @ f2f71758
......@@ -19,6 +19,7 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
shard:
primaryOnly: true
key: req.http.Host
rules:
- shard:
primaryOnly: true
key: req.http.Host
examples/self-sharding/values-shard-by-url.yaml
View file @ f2f71758
......@@ -19,6 +19,7 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
shard:
primaryOnly: true
key: req.url
rules:
- shard:
primaryOnly: true
key: req.url
examples/self-sharding/values-shard-conditions.yaml
View file @ f2f71758
......@@ -19,17 +19,20 @@ ingress:
vikingAdmSvc: varnish-ingress-admin
selfSharding:
conditions:
- comparand: req.url
compare: prefix
values:
- /foo/
shard:
primaryOnly: true
key: client.identity
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
probe:
timeout: 6s
interval: 6s
initial: 2
window: 4
threshold: 3
rules:
- conditions:
- comparand: req.url
compare: prefix
values:
- /foo/
shard:
primaryOnly: true
key: client.identity
- shard:
key: client.identity
go.mod
View file @ f2f71758
......@@ -21,4 +21,5 @@ require (
k8s.io/api v0.16.4
k8s.io/apimachinery v0.16.4
k8s.io/client-go v0.16.4
k8s.io/code-generator v0.16.5-beta.1 // indirect
)
go.sum
View file @ f2f71758
......@@ -11,6 +11,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
......@@ -40,6 +41,7 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
......@@ -226,6 +228,7 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
......@@ -263,9 +266,13 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k=
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
......@@ -300,6 +307,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
......@@ -317,10 +325,17 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
......@@ -356,7 +371,11 @@ k8s.io/apimachinery v0.16.4 h1:+VNiyTcctUvBBRUxfulwL2I6TGratkR1oAoULuas/HI=
k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ=
k8s.io/client-go v0.16.4 h1:sf+FEZXYhJNjpTZapQDLvvN+0kBeUTxCYxlXcVdhv2E=
k8s.io/client-go v0.16.4/go.mod h1:ZgxhFDxSnoKY0J0U2/Y1C8obKDdlhGPZwA7oHH863Ok=
k8s.io/code-generator v0.16.5-beta.1 h1:+zWxMQH3a6fd8lZe6utWyW/V7nmG2ZMXwtovSJI2p+0=
k8s.io/code-generator v0.16.5-beta.1/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM=
k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ=
......@@ -365,6 +384,11 @@ k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJD
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
k8s.io/utils v0.0.0-20190801114015-581e00157fb1 h1:+ySTxfHnfzZb9ys375PXNlLhkJPLKgHajBU0N62BDvE=
k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
pkg/apis/varnishingress/v1alpha1/types.go
View file @ f2f71758
......@@ -61,17 +61,23 @@ type VarnishConfigSpec struct {
// SelfShardSpec specifies self-sharding in a Varnish cluster.
// see: https://code.uplex.de/uplex-varnish/k8s-ingress/blob/master/docs/self-sharding.md
type SelfShardSpec struct {
Probe *ProbeSpec `json:"probe,omitempty"`
Rules []ShardRule `json:"rules"`
Max2ndTTL string `json:"max-secondary-ttl,omitempty"`
}
// ShardRule specifies a sharding configuration, and optional
// conditions under which the configuration holds.
type ShardRule struct {
Conditions []ReqCondition `json:"conditions,omitempty"`
Sharding ShardSpec `json:"shard"`
}
// ShardSpec specifies the configuration details for sharding.
type ShardSpec struct {
Probe *ProbeSpec `json:"probe,omitempty"`
Key string `json:"key,omitempty"`
Digest string `json:"digest,omitempty"`
Max2ndTTL string `json:"max-secondary-ttl,omitempty"`
PrimaryOnly bool `json:"primaryOnly,omitempty"`
Key string `json:"key,omitempty"`
Digest string `json:"digest,omitempty"`
PrimaryOnly bool `json:"primaryOnly,omitempty"`
}
// ProbeSpec specifies health probes for self-sharding and BackendConfig.
......
pkg/apis/varnishingress/v1alpha1/zz_generated.deepcopy.go
View file @ f2f71758
......@@ -466,14 +466,18 @@ func (in *RewriteSpec) DeepCopy() *RewriteSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SelfShardSpec) DeepCopyInto(out *SelfShardSpec) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]ReqCondition, len(*in))
if in.Probe != nil {
in, out := &in.Probe, &out.Probe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.Rules != nil {
in, out := &in.Rules, &out.Rules
*out = make([]ShardRule, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
in.Sharding.DeepCopyInto(&out.Sharding)
return
}
......@@ -488,13 +492,32 @@ func (in *SelfShardSpec) DeepCopy() *SelfShardSpec {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ShardSpec) DeepCopyInto(out *ShardSpec) {
func (in *ShardRule) DeepCopyInto(out *ShardRule) {
*out = *in
if in.Probe != nil {
in, out := &in.Probe, &out.Probe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]ReqCondition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
out.Sharding = in.Sharding
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardRule.
func (in *ShardRule) DeepCopy() *ShardRule {
if in == nil {
return nil
}
out := new(ShardRule)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ShardSpec) DeepCopyInto(out *ShardSpec) {
*out = *in
return
}
......
pkg/controller/varnishconfig.go
View file @ f2f71758
......@@ -426,31 +426,38 @@ func (worker *NamespaceWorker) configSharding(spec *vcl.Spec,
"%s/%s: %+v", svc.Namespace, svc.Name, spec.ShardCluster.Nodes)
cfgSpec := vcfg.Spec.SelfSharding
spec.ShardCluster.Conditions = reqConds2vclConds(cfgSpec.Conditions)
spec.ShardCluster.Probe = getVCLProbe(cfgSpec.Sharding.Probe)
spec.ShardCluster.PrimaryOnly = cfgSpec.Sharding.PrimaryOnly
spec.ShardCluster.MaxSecondaryTTL = defMax2ndTTL
spec.ShardCluster.By = vcl.ByHash
if cfgSpec.Sharding.Max2ndTTL != "" {
spec.ShardCluster.MaxSecondaryTTL = cfgSpec.Sharding.Max2ndTTL
spec.ShardCluster.Probe = getVCLProbe(cfgSpec.Probe)
if cfgSpec.Max2ndTTL != "" {
spec.ShardCluster.MaxSecondaryTTL = cfgSpec.Max2ndTTL
} else {
spec.ShardCluster.MaxSecondaryTTL = defMax2ndTTL
}
if cfgSpec.Sharding.Digest != "" &&
cfgSpec.Sharding.Digest != "SHA256" {
spec.ShardCluster.By = vcl.Blob
spec.ShardCluster.Algo = getHashAlgo(cfgSpec.Sharding.Digest)
if cfgSpec.Sharding.Key != "" {
spec.ShardCluster.Key = cfgSpec.Sharding.Key
} else {
spec.ShardCluster.Key = "req.url"
spec.ShardCluster.Rules = make([]vcl.ShardRule, len(cfgSpec.Rules))
for i, rule := range cfgSpec.Rules {
vclRule := vcl.ShardRule{
Conditions: reqConds2vclConds(rule.Conditions),
PrimaryOnly: rule.Sharding.PrimaryOnly,
By: vcl.ByHash,
}
} else if cfgSpec.Sharding.Key != "" {
spec.ShardCluster.Key = cfgSpec.Sharding.Key
if cfgSpec.Sharding.Key == "req.url" {
spec.ShardCluster.By = vcl.URL
} else {
spec.ShardCluster.By = vcl.Key
if rule.Sharding.Digest != "" &&
rule.Sharding.Digest != "SHA256" {
vclRule.By = vcl.Blob
vclRule.Algo = getHashAlgo(rule.Sharding.Digest)
if rule.Sharding.Key != "" {
vclRule.Key = rule.Sharding.Key
} else {
vclRule.Key = "req.url"
}
} else if rule.Sharding.Key != "" {
vclRule.Key = rule.Sharding.Key
if rule.Sharding.Key == "req.url" {
vclRule.By = vcl.URL
} else {
vclRule.By = vcl.Key
}
}
spec.ShardCluster.Rules[i] = vclRule
}
worker.log.Tracef("Spec configuration for self-sharding in Service "+
......@@ -477,7 +484,7 @@ func (worker *NamespaceWorker) syncVcfg(key string) update.Status {
}
if vcfg.Spec.SelfSharding != nil {
if err = validateProbe(vcfg.Spec.SelfSharding.Sharding.Probe); err != nil {
if err = validateProbe(vcfg.Spec.SelfSharding.Probe); err != nil {
return update.MakeFatal(
"VarnishConfig %s/%s invalid sharding spec: %v",
vcfg.Namespace, vcfg.Name, err)
......
pkg/varnish/vcl/self_shard_tmpl.go
View file @ f2f71758
......@@ -89,30 +89,34 @@ sub vcl_init {
vk8s_cluster.add_backend({{nodeName $node}});
{{end -}}
vk8s_cluster.reconfigure();
{{- digest_init . }}
{{ if .PrimaryOnly -}}
new vk8s_cluster_forward = taskvar.bool();
{{ if hasPrimary . -}}
new vk8s_cluster_primary = taskvar.backend();
{{- end }}
{{- range $cidx, $c := .Conditions -}}
{{- range $ridx, $rule := .Rules -}}
{{- digest_init $rule }}
{{- range $cidx, $c := $rule.Conditions -}}
{{- if condNeedsMatcher $c }}
new {{condMatcher $cidx}} = {{vmod $c.Compare}}.set({{flags $c}});
new {{condMatcher $ridx $cidx}} = {{vmod $c.Compare}}.set({{flags $c}});
{{- range $val := $c.Values}}
{{condMatcher $cidx}}.add("{{$val}}");
{{condMatcher $ridx $cidx}}.add("{{$val}}");
{{- end -}}
{{end -}}
{{- end }}
{{- end }}
}
sub vcl_recv {
{{ if .PrimaryOnly -}}
{{ if .Conditions -}}
{{ range $ridx, $rule := .Rules -}}
{{ if $rule.PrimaryOnly -}}
{{ if $rule.Conditions -}}
if (
{{- range $cidx, $cond := .Conditions}}
{{- range $cidx, $cond := $rule.Conditions}}
{{- if ne $cidx 0}} &&
{{end}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $cidx}}.{{match .Compare}}({{.Comparand}})
{{- condMatcher $ridx $cidx}}.{{match .Compare}}({{.Comparand}})
{{- else if exists .Compare}}
{{- .Comparand}}
{{- else}}
......@@ -121,15 +125,16 @@ sub vcl_recv {
{{- end -}}
) {
{{ end -}}
{{- digest_update 'c' . }}
{{- digest_update 'c' $rule }}
vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW
{{- key 'c' .}}));
{{- key 'c' $rule }}));
if (remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster_primary.get() != server.identity) {
set req.backend_hint = vk8s_cluster_primary.get();
return (pipe);
}
{{ if .Conditions }}}
{{ if $rule.Conditions }}}
{{ end -}}
{{ end -}}
{{ end -}}
if (remote.ip ~ vk8s_cluster_acl) {
......@@ -152,31 +157,45 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
{{- digest_update 'b' . }}
vk8s_cluster_param.set({{ key 'b' .}});
if (
{{- range $cidx, $cond := .Conditions}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $cidx}}.{{match .Compare}}({{ctx 'b' .Comparand}})
{{- else if exists .Compare}}
{{- ctx 'b' .Comparand}}
{{- else}}
{{- ctx 'b' .Comparand}} {{cmpRelation .Compare .Negate}} {{value $cond}}
{{- end}}
&&
{{ end -}}
bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
{{ range $ridx, $rule := .Rules -}}
{{ ifElseIf $ridx (len $.Rules) $rule }}
{{ if not $rule.PrimaryOnly }}
{{- range $cidx, $cond := $rule.Conditions}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $ridx $cidx}}.{{match .Compare}}({{ctx 'b' .Comparand}})
{{- else if exists .Compare}}
{{- ctx 'b' .Comparand}}
{{- else}}
{{- ctx 'b' .Comparand}} {{cmpRelation .Compare .Negate}} {{value $cond}}
{{- end}}
{{- end}}
{{ closeIf $rule }}
{{- digest_update 'b' $rule }}
vk8s_cluster_param.set({{ key 'b' $rule }});
vk8s_cluster_forward.set(true);
{{ closeRule $rule }}
{{- end }}{{/* if not PrimaryOnly */}}
{{- end }}{{/* range .Rules */}}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......@@ -203,7 +222,7 @@ sub vcl_backend_error {
sub vcl_deliver {
unset resp.http.VK8S-Cluster-TTL;
if (remote.ip ~ vk8s_cluster_acl
{{- if .PrimaryOnly }} && ! vk8s_cluster_primary.defined()
{{- if hasPrimary . }} && ! vk8s_cluster_primary.defined()
{{- end }}) {
if (! obj.uncacheable) {
set resp.http.VK8S-Cluster-TTL = obj.ttl;
......@@ -222,37 +241,73 @@ func context(ctx rune, key string) string {
return reqMatch.ReplaceAllLiteralString(key, "bereq")
}
func keyParams(ctx rune, shard ShardCluster) string {
func keyParams(ctx rune, rule ShardRule) string {
pfx := ""
if ctx == 'c' {
pfx = ", "
}
switch shard.By {
switch rule.By {
case ByHash:
return ""
case URL:
return pfx + "by=URL"
case Key:
return pfx + "by=KEY, key=vk8s_cluster.key(" +
context(ctx, shard.Key) + ")"
context(ctx, rule.Key) + ")"
}
return pfx + "by=BLOB, key_blob=vk8s_shard_digest.final()"
}
func digestInit(shard ShardCluster) string {
if shard.By != Blob {
func digestInit(rule ShardRule) string {
if rule.By != Blob {
return ""
}
return "\n\tnew vk8s_shard_digest = blobdigest.digest(" +
shard.Algo.String() + ");"
rule.Algo.String() + ");"
}
func digestUpdate(ctx rune, shard ShardCluster) string {
if shard.By != Blob {
func digestUpdate(ctx rune, rule ShardRule) string {
if rule.By != Blob {
return ""
}
return "\n\tvk8s_shard_digest.update(blob.decode(encoded=" +
context(ctx, shard.Key) + "));"
context(ctx, rule.Key) + "));"
}
func hasPrimary(shard ShardCluster) bool {
for _, rule := range shard.Rules {
if rule.PrimaryOnly {
return true
}
}
return false
}
func ifElseIf(ridx, rlen int, rule ShardRule) string {
if rule.PrimaryOnly || len(rule.Conditions) == 0 {
return ""
}
if ridx == 1 {
return "if"
}
if ridx == rlen-1 {
return "else"
}
return "elsif"
}
func closeIf(rule ShardRule) string {
if len(rule.Conditions) == 0 {
return ""
}
return ") {"
}
func closeRule(rule ShardRule) string {
if len(rule.Conditions) == 0 {
return ""
}
return "}"
}
const selfShardName = "self-sharding"
......@@ -261,6 +316,10 @@ var shardFuncMap = template.FuncMap{
"key": keyParams,
"digest_init": digestInit,
"digest_update": digestUpdate,
"hasPrimary": hasPrimary,
"ifElseIf": ifElseIf,
"closeIf": closeIf,
"closeRule": closeRule,
"nodeName": func(svc Service) string {
// MUST match -i setting in the varnishd invocaton
// (server.identity)
......@@ -278,8 +337,8 @@ var shardFuncMap = template.FuncMap{
"condNeedsMatcher": func(cond Condition) bool {
return reqNeedsMatcher(cond)
},
"condMatcher": func(cidx int) string {
return fmt.Sprintf("vk8s_selfshard_cond_%d", cidx)
"condMatcher": func(ridx, cidx int) string {
return fmt.Sprintf("vk8s_selfshard_cond_%d_%d", ridx, cidx)
},
"ctx": context,
}
......
pkg/varnish/vcl/shard_test.go
View file @ f2f71758
......@@ -69,6 +69,7 @@ var varnishCluster = ShardCluster{
Threshold: "3",
},
MaxSecondaryTTL: "5m",
Rules: []ShardRule{{}},
}
func TestShardTemplate(t *testing.T) {
......@@ -76,14 +77,14 @@ func TestShardTemplate(t *testing.T) {
}
func TestPrimaryOnlyShardTemplate(t *testing.T) {
varnishCluster.PrimaryOnly = true
varnishCluster.Rules[0].PrimaryOnly = true
templateTest(t, shardTmpl, varnishCluster, "primaryonly_shard.golden")
}
func TestGetSrc(t *testing.T) {
gold := "ingress_shard.golden"
cafeSpec.ShardCluster = varnishCluster
cafeSpec.ShardCluster.PrimaryOnly = false
cafeSpec.ShardCluster.Rules[0].PrimaryOnly = false
src, err := cafeSpec.GetSrc()
if err != nil {
t.Error("Spec.GetSrc():", err)
......@@ -107,33 +108,34 @@ func TestGetSrc(t *testing.T) {
}
func TestShardByURL(t *testing.T) {
varnishCluster.PrimaryOnly = true
varnishCluster.By = URL
varnishCluster.Rules[0].PrimaryOnly = true
varnishCluster.Rules[0].By = URL
templateTest(t, shardTmpl, varnishCluster, "shard_by_url.golden")
}
func TestShardByKey(t *testing.T) {
varnishCluster.PrimaryOnly = true
varnishCluster.By = Key
varnishCluster.Key = "req.http.Host"
varnishCluster.Rules[0].PrimaryOnly = true
varnishCluster.Rules[0].By = Key
varnishCluster.Rules[0].Key = "req.http.Host"
templateTest(t, shardTmpl, varnishCluster, "shard_by_key.golden")
}
func TestShardByDigest(t *testing.T) {
varnishCluster.PrimaryOnly = true
varnishCluster.By = Blob
varnishCluster.Key = "req.http.Host"
varnishCluster.Algo = Sha3_512
varnishCluster.Rules[0].PrimaryOnly = true
varnishCluster.Rules[0].By = Blob
varnishCluster.Rules[0].Key = "req.http.Host"
varnishCluster.Rules[0].Algo = Sha3_512
templateTest(t, shardTmpl, varnishCluster, "shard_by_digest.golden")
}
func TestShardConditions(t *testing.T) {
varnishCluster.Conditions = []Condition{
varnishCluster.Rules[0].Conditions = []Condition{
{
Comparand: "req.url",
Compare: Prefix,
Values: []string{"/foo/"},
},
}
varnishCluster.Rules = append(varnishCluster.Rules, ShardRule{})
templateTest(t, shardTmpl, varnishCluster, "shard_conditions.golden")
}
pkg/varnish/vcl/spec.go
View file @ f2f71758
......@@ -336,35 +336,43 @@ const (
Blob
)
// ShardRule represents a sharding configuration, and optional
// conditions under which the configuration holds.
type ShardRule struct {
Conditions []Condition
Key string
By KeyBy
Algo HashAlgo
PrimaryOnly bool
}
// ShardCluster represents the configuration for self-sharding derived
// from the VarnishConfig Custom Resource.
type ShardCluster struct {
Nodes []Service
Conditions []Condition
Rules []ShardRule
Probe *Probe
MaxSecondaryTTL string
Key string
By KeyBy
Algo HashAlgo
PrimaryOnly bool
}
func (shard ShardCluster) hash(hash hash.Hash) {
for _, node := range shard.Nodes {
node.hash(hash)
}
for _, cond := range shard.Conditions {
cond.hash(hash)
}
if shard.Probe != nil {
shard.Probe.hash(hash)
}
hash.Write([]byte(shard.MaxSecondaryTTL))
hash.Write([]byte(shard.Key))
hash.Write([]byte{byte(shard.By)})
hash.Write([]byte{byte(shard.Algo)})
if shard.PrimaryOnly {
hash.Write([]byte{1})
for _, rule := range shard.Rules {
for _, cond := range rule.Conditions {
cond.hash(hash)
}
hash.Write([]byte(rule.Key))
hash.Write([]byte{byte(rule.By)})
hash.Write([]byte{byte(rule.Algo)})
if rule.PrimaryOnly {
hash.Write([]byte{1})
}
}
}
......
pkg/varnish/vcl/testdata/ingress_shard.golden
View file @ f2f71758
......@@ -122,6 +122,7 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_cluster_forward = taskvar.bool();
}
......@@ -146,18 +147,31 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_cluster_param.set();
if (bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
vk8s_cluster_param.set();
vk8s_cluster_forward.set(true);
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
pkg/varnish/vcl/testdata/primaryonly_shard.golden
View file @ f2f71758
......@@ -50,6 +50,7 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_cluster_forward = taskvar.bool();
new vk8s_cluster_primary = taskvar.backend();
}
......@@ -81,18 +82,27 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_cluster_param.set();
if (bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
pkg/varnish/vcl/testdata/shard.golden
View file @ f2f71758
......@@ -50,6 +50,7 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_cluster_forward = taskvar.bool();
}
......@@ -74,18 +75,31 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_cluster_param.set();
if (bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
vk8s_cluster_param.set();
vk8s_cluster_forward.set(true);
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
pkg/varnish/vcl/testdata/shard_by_digest.golden
View file @ f2f71758
......@@ -50,8 +50,9 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_shard_digest = blobdigest.digest(SHA3_512);
new vk8s_cluster_forward = taskvar.bool();
new vk8s_cluster_primary = taskvar.backend();
new vk8s_shard_digest = blobdigest.digest(SHA3_512);
}
sub vcl_recv {
......@@ -83,19 +84,27 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_shard_digest.update(blob.decode(encoded=bereq.http.Host));
vk8s_cluster_param.set(by=BLOB, key_blob=vk8s_shard_digest.final());
if (bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
pkg/varnish/vcl/testdata/shard_by_key.golden
View file @ f2f71758
......@@ -50,6 +50,7 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_cluster_forward = taskvar.bool();
new vk8s_cluster_primary = taskvar.backend();
}
......@@ -81,18 +82,27 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_cluster_param.set(by=KEY, key=vk8s_cluster.key(bereq.http.Host));
if (bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
pkg/varnish/vcl/testdata/shard_by_url.golden
View file @ f2f71758
......@@ -50,6 +50,7 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_cluster_forward = taskvar.bool();
new vk8s_cluster_primary = taskvar.backend();
}
......@@ -81,18 +82,27 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_cluster_param.set(by=URL);
if (bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
pkg/varnish/vcl/testdata/shard_conditions.golden
View file @ f2f71758
......@@ -50,14 +50,15 @@ sub vcl_init {
vk8s_cluster.add_backend(vk8s__);
vk8s_cluster.add_backend(vk8s_default_varnish-8445d4f7f-ldljf);
vk8s_cluster.reconfigure();
new vk8s_shard_digest = blobdigest.digest(SHA3_512);
new vk8s_cluster_forward = taskvar.bool();
new vk8s_cluster_primary = taskvar.backend();
new vk8s_selfshard_cond_0 = selector.set(case_sensitive=false);
vk8s_selfshard_cond_0.add("/foo/");
new vk8s_shard_digest = blobdigest.digest(SHA3_512);
new vk8s_selfshard_cond_0_0 = selector.set(case_sensitive=false);
vk8s_selfshard_cond_0_0.add("/foo/");
}
sub vcl_recv {
if (vk8s_selfshard_cond_0.hasprefix(req.url)) {
if (vk8s_selfshard_cond_0_0.hasprefix(req.url)) {
vk8s_shard_digest.update(blob.decode(encoded=req.http.Host));
vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW, by=BLOB, key_blob=vk8s_shard_digest.final()));
......@@ -87,21 +88,32 @@ sub vcl_recv {
}
}
sub vcl_backend_fetch {
vk8s_shard_digest.update(blob.decode(encoded=bereq.http.Host));
vk8s_cluster_param.set(by=BLOB, key_blob=vk8s_shard_digest.final());
if (vk8s_selfshard_cond_0.hasprefix(bereq.url)
&&
bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
vk8s_cluster_param.set();
vk8s_cluster_forward.set(true);
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
if (beresp.http.VK8S-Cluster-TTL) {
......
test/e2e/tmplcfg/tmplcfg.yaml
View file @ f2f71758
......@@ -476,30 +476,34 @@ spec:
vk8s_cluster.add_backend({{nodeName $node}});
{{end -}}
vk8s_cluster.reconfigure();
{{- digest_init . }}
{{ if .PrimaryOnly -}}
new vk8s_cluster_forward = taskvar.bool();
{{ if hasPrimary . -}}
new vk8s_cluster_primary = taskvar.backend();
{{- end }}
{{range $cidx, $c := .Conditions -}}
{{if condNeedsMatcher $c -}}
new {{condMatcher $cidx}} = {{vmod $c.Compare}}.set({{flags $c}});
{{- range $ridx, $rule := .Rules -}}
{{- digest_init $rule }}
{{- range $cidx, $c := $rule.Conditions -}}
{{- if condNeedsMatcher $c }}
new {{condMatcher $ridx $cidx}} = {{vmod $c.Compare}}.set({{flags $c}});
{{- range $val := $c.Values}}
{{condMatcher $cidx}}.add("{{$val}}");
{{condMatcher $ridx $cidx}}.add("{{$val}}");
{{- end -}}
{{end -}}
{{end -}}
{{- end }}
{{- end }}
}
sub vcl_recv {
{{ if .PrimaryOnly -}}
{{ if .Conditions -}}
{{ range $ridx, $rule := .Rules -}}
{{ if $rule.PrimaryOnly -}}
{{ if $rule.Conditions -}}
if (
{{- range $cidx, $cond := .Conditions}}
{{- range $cidx, $cond := $rule.Conditions}}
{{- if ne $cidx 0}} &&
{{end}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $cidx}}.{{match .Compare}}({{.Comparand}})
{{- condMatcher $ridx $cidx}}.{{match .Compare}}({{.Comparand}})
{{- else if exists .Compare}}
{{- .Comparand}}
{{- else}}
......@@ -508,15 +512,17 @@ spec:
{{- end -}}
) {
{{ end -}}
{{- digest_update 'c' . }}
{{- digest_update 'c' $rule }}
vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW
{{- key 'c' .}}));
{{- key 'c' $rule }}));
if (remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster_primary.get() != server.identity) {
set req.backend_hint = vk8s_cluster_primary.get();
return (pipe);
}
{{ if .Conditions }}}{{ end -}}
{{ if $rule.Conditions }}}
{{ end -}}
{{ end -}}
{{ end -}}
if (remote.ip ~ vk8s_cluster_acl) {
if (req.http.Host == "vk8s_cluster") {
......@@ -538,31 +544,45 @@ spec:
}
}
sub vcl_backend_fetch {
{{- digest_update 'b' . }}
vk8s_cluster_param.set({{ key 'b' .}});
if (
{{- range $cidx, $cond := .Conditions}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $cidx}}.{{match .Compare}}({{.Comparand}})
{{- else if exists .Compare}}
{{- .Comparand}}
{{- else}}
{{- .Comparand}} {{cmpRelation .Compare .Negate}} {{value $cond}}
{{- end}}
&&
{{ end -}}
bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
{{ range $ridx, $rule := .Rules -}}
{{ ifElseIf $ridx (len $.Rules) $rule }}
{{ if not $rule.PrimaryOnly }}
{{- range $cidx, $cond := $rule.Conditions}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $ridx $cidx}}.{{match .Compare}}({{ctx 'b' .Comparand}})
{{- else if exists .Compare}}
{{- ctx 'b' .Comparand}}
{{- else}}
{{- ctx 'b' .Comparand}} {{cmpRelation .Compare .Negate}} {{value $cond}}
{{- end}}
{{- end}}
{{ closeIf $rule }}
{{- digest_update 'b' $rule }}
vk8s_cluster_param.set({{ key 'b' $rule }});
vk8s_cluster_forward.set(true);
{{ closeRule $rule }}
{{- end }}{{/* if not PrimaryOnly */}}
{{- end }}{{/* range .Rules */}}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
set beresp.http.Shard-Template = "override";
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
......@@ -589,7 +609,7 @@ spec:
sub vcl_deliver {
if (remote.ip ~ vk8s_cluster_acl
{{- if .PrimaryOnly }} && ! vk8s_cluster_primary.defined()
{{- if hasPrimary . }} && ! vk8s_cluster_primary.defined()
{{- end }}) {
if (! obj.uncacheable) {
set resp.http.VK8S-Cluster-TTL = obj.ttl;
......
test/e2e/tmplcfg/values.yaml
View file @ f2f71758
......@@ -121,7 +121,9 @@ rewrites:
source: req.http.X-Cache
method: replace
selfSharding: {}
selfSharding:
rules:
- shard: {}
templates:
ingress: |
......@@ -596,30 +598,34 @@ templates:
vk8s_cluster.add_backend({{nodeName $node}});
{{end -}}
vk8s_cluster.reconfigure();
{{- digest_init . }}
{{ if .PrimaryOnly -}}
new vk8s_cluster_forward = taskvar.bool();
{{ if hasPrimary . -}}
new vk8s_cluster_primary = taskvar.backend();
{{- end }}
{{range $cidx, $c := .Conditions -}}
{{if condNeedsMatcher $c -}}
new {{condMatcher $cidx}} = {{vmod $c.Compare}}.set({{flags $c}});
{{- range $ridx, $rule := .Rules -}}
{{- digest_init $rule }}
{{- range $cidx, $c := $rule.Conditions -}}
{{- if condNeedsMatcher $c }}
new {{condMatcher $ridx $cidx}} = {{vmod $c.Compare}}.set({{flags $c}});
{{- range $val := $c.Values}}
{{condMatcher $cidx}}.add("{{$val}}");
{{condMatcher $ridx $cidx}}.add("{{$val}}");
{{- end -}}
{{end -}}
{{end -}}
{{- end }}
{{- end }}
}
sub vcl_recv {
{{ if .PrimaryOnly -}}
{{ if .Conditions -}}
{{ range $ridx, $rule := .Rules -}}
{{ if $rule.PrimaryOnly -}}
{{ if $rule.Conditions -}}
if (
{{- range $cidx, $cond := .Conditions}}
{{- range $cidx, $cond := $rule.Conditions}}
{{- if ne $cidx 0}} &&
{{end}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $cidx}}.{{match .Compare}}({{.Comparand}})
{{- condMatcher $ridx $cidx}}.{{match .Compare}}({{.Comparand}})
{{- else if exists .Compare}}
{{- .Comparand}}
{{- else}}
......@@ -628,15 +634,17 @@ templates:
{{- end -}}
) {
{{ end -}}
{{- digest_update 'c' . }}
{{- digest_update 'c' $rule }}
vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW
{{- key 'c' .}}));
{{- key 'c' $rule }}));
if (remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster_primary.get() != server.identity) {
set req.backend_hint = vk8s_cluster_primary.get();
return (pipe);
}
{{ if .Conditions }}}{{ end -}}
{{ if $rule.Conditions }}}
{{ end -}}
{{ end -}}
{{ end -}}
if (remote.ip ~ vk8s_cluster_acl) {
if (req.http.Host == "vk8s_cluster") {
......@@ -658,31 +666,45 @@ templates:
}
}
sub vcl_backend_fetch {
{{- digest_update 'b' . }}
vk8s_cluster_param.set({{ key 'b' .}});
if (
{{- range $cidx, $cond := .Conditions}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $cidx}}.{{match .Compare}}({{.Comparand}})
{{- else if exists .Compare}}
{{- .Comparand}}
{{- else}}
{{- .Comparand}} {{cmpRelation .Compare .Negate}} {{value $cond}}
{{- end}}
&&
{{ end -}}
bereq.retries == 0
&& !bereq.uncacheable
&& remote.ip !~ vk8s_cluster_acl
&& "" + vk8s_cluster.backend(resolve=NOW) != server.identity) {
sub vk8s_cluster_fetch {
if (bereq.retries > 0
|| bereq.uncacheable
|| remote.ip ~ vk8s_cluster_acl
|| "" + vk8s_cluster.backend(resolve=NOW) == server.identity) {
return;
}
{{ range $ridx, $rule := .Rules -}}
{{ ifElseIf $ridx (len $.Rules) $rule }}
{{ if not $rule.PrimaryOnly }}
{{- range $cidx, $cond := $rule.Conditions}}
{{- if .Negate}}! {{end}}
{{- if condNeedsMatcher $cond}}
{{- condMatcher $ridx $cidx}}.{{match .Compare}}({{ctx 'b' .Comparand}})
{{- else if exists .Compare}}
{{- ctx 'b' .Comparand}}
{{- else}}
{{- ctx 'b' .Comparand}} {{cmpRelation .Compare .Negate}} {{value $cond}}
{{- end}}
{{- end}}
{{ closeIf $rule }}
{{- digest_update 'b' $rule }}
vk8s_cluster_param.set({{ key 'b' $rule }});
vk8s_cluster_forward.set(true);
{{ closeRule $rule }}
{{- end }}{{/* if not PrimaryOnly */}}
{{- end }}{{/* range .Rules */}}
if (vk8s_cluster_forward.get(fallback=false)) {
set bereq.backend = vk8s_cluster.backend(resolve=LAZY);
set bereq.http.VK8S-Is-Bgfetch = bereq.is_bgfetch;
return (fetch);
}
}
sub vcl_backend_fetch {
call vk8s_cluster_fetch;
}
sub vcl_backend_response {
set beresp.http.Shard-Template = "override";
if (bereq.backend == vk8s_cluster.backend(resolve=LAZY)) {
......@@ -709,7 +731,7 @@ templates:
sub vcl_deliver {
if (remote.ip ~ vk8s_cluster_acl
{{- if .PrimaryOnly }} && ! vk8s_cluster_primary.defined()
{{- if hasPrimary . }} && ! vk8s_cluster_primary.defined()
{{- end }}) {
if (! obj.uncacheable) {
set resp.http.VK8S-Cluster-TTL = obj.ttl;
......
test/e2e/tmplcfg/vcfg.yaml
View file @ f2f71758
......@@ -103,4 +103,6 @@ spec:
source: req.http.X-Cache
method: replace
self-sharding: {}
self-sharding:
rules:
- shard: {}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment