WIP #28 fellow_logs_autotune

This commit introduces auto-tuning of objsize_hint based on a rough
metric of the average object size when the storage is between 25% and
75% full. A change in objsize_hint triggers resizing of the
pre-allocated log-regions, which is no issue when they shrink (because
of increased objsize_hint).

But increasing the log-regions requires relevant contigious storage
regions, for which LRU might need to remove a lot of objects.

So I am not sure yet if this is a good idea. We could maybe merge the
feature, but leave it disabled for now...

src/fellow_log.c

@@ -5215,6 +5215,75 @@ fellow_logs_rewriting(const struct fellow_fd *ffd)
 	return (ffd->rewriting > 0);
 }
 
+static void
+fellow_logs_autotune(struct fellow_fd *ffd)
+{
+	const struct dle_stats *s;
+	struct stvfe_tune *tune;
+	size_t obj, del, spc, sz;
+	double f;
+	unsigned o, n;
+
+	spc = buddy_space(ffd->dskbuddy, 1);
+	sz  = buddy_size(ffd->dskbuddy);
+
+	if (sz == 0)
+		return;
+
+	/*
+	 * autotune only active between 25% and 75% full
+	 * to get reasonable numbers and also avoid
+	 * interfering too much with log region allocations
+	 */
+
+	f = (sz - spc);
+	f /= sz;
+
+	ffd->diag("fellow: approx %.2f%% full", f * 100);
+
+	if (f < .25 || f > .75)
+		return;
+
+	tune = ffd->tune;
+	if (tune->objsize_hint_auto == 0)
+		return;
+
+	s = &ffd->logbuf->dle_stats;
+
+	obj = s->obj_add;
+	del = s->obj_del_free;
+
+	if (obj > del)
+		obj -= del;
+
+	if (obj == 0)
+		return;
+
+	sz = sz / obj;
+
+	o = log2down(tune->objsize_hint);
+	n = log2down(sz);
+	if (n < MIN_FELLOW_BITS)
+		n = MIN_FELLOW_BITS;
+
+	if (n < o - 2) {
+		ffd->diag(
+		    "fellow: AUTOTUNE: config objsize_hint %zu (2^%u) "
+		    "way bigger than actual average %zu (2^%u) "
+		    "-> lowering to %zu\n",
+		    tune->objsize_hint, o, sz, n, (size_t)1 << n);
+		tune->objsize_hint     = (size_t)1 << n;
+		tune->objsize_hint_max = (size_t)1 << (n + 1);
+	} else
+	if (n > o + 3) {
+		ffd->diag(
+		    "fellow: tuning advice: objsize hint %zu (2^%u) much "
+		    "smaller than actual average %zu (2^%u)\n"
+		    "fellow: consider increasing it carefully.\n",
+		    tune->objsize_hint, o, sz, n);
+	}
+}
+
 static void
 fellow_logs_rewrite(struct fellow_fd *ffd,
     struct buddy_off_extent *new_log_fdr,
@@ -5484,6 +5553,7 @@ fellow_logs_rewrite(struct fellow_fd *ffd,
 	case FP_INIT:
 		AN(lbuf);
 		assert(lbuf->state == LBUF_LOGREG || lbuf->state == LBUF_MEM);
+		fellow_logs_autotune(ffd);
 		lbuf->state = LBUF_OPEN;
 		fellow_signal_open(ffd);
 
@@ -5550,6 +5620,8 @@ fellow_logs_rewrite(struct fellow_fd *ffd,
 
 			fellow_fd_ioctx_return(&fdil);
 
+			fellow_logs_autotune(ffd);
+
 			break;
 		case FP_FINI:
 			regionlist_free(&tofree, ffd->dskbuddy);

src/fellow_tune.c

@@ -127,6 +127,12 @@ stvfe_tune_check(struct stvfe_tune *tune)
 	if (tune->objsize_max == 0)
 		tune->objsize_max = (tune->dsksz >> 2);
 
+	if (tune->objsize_hint_max > 0 &&
+	    tune->objsize_hint > tune->objsize_hint_max) {
+		fprintf(stderr,"fellow: objsize_hint limited to %zu\n",
+		    tune->objsize_hint_max);
+		tune->objsize_hint = tune->objsize_hint_max;
+	}
 //lint --e{685,568} misc const comparisons
 #define TUNE(t, n, d, min, max)						\
 	if (tune->n < (min))						\

src/fellow_tune.h

@@ -34,6 +34,9 @@ struct stvfe_tune {
 
 	size_t			memsz;
 	size_t			dsksz;
+
+	// dynamic limit set by fellow_logs_autotune()
+	size_t			objsize_hint_max;
 };
 
 const char * stvfe_tune_init(struct stvfe_tune *tune,

src/tbl/fellow_tunables.h

@@ -46,6 +46,7 @@ TUNE(uint8_t, lru_exponent, 0, 0, 6);
 TUNE(unsigned, dsk_reserve_chunks, 4, 2, UINT_MAX);
 TUNE(unsigned, mem_reserve_chunks, 1, 0, UINT_MAX);
 TUNE(size_t, objsize_hint, 256 * 1024, 4096, SIZE_MAX);
+TUNE(unsigned, objsize_hint_auto, 1, 0, 1);
 TUNE(size_t, objsize_max, 0, 0, SIZE_MAX);
 TUNE(size_t, discard_immediate, 256 * 1024, 4096, SIZE_MAX);
 // 31 is safe max for stack usage, further limited by memsz

src/vmod_slash.man.rst

@@ -51,7 +51,7 @@ vcl storage objects and methods
 
   new xfellow = slash.fellow(STRING filename, BYTES dsksize, BYTES memsize, BYTES objsize_hint, BOOL delete)
 
-      STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
+      STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BOOL objsize_hint_auto], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
 
       STEVEDORE xfellow.storage()
 
@@ -68,7 +68,7 @@ vcl functions
 
   STRING tune_buddy(STEVEDORE storage, [INT chunk_exponent], [BYTES chunk_bytes], [INT reserve_chunks], [INT cram])
 
-  STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
+  STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BOOL objsize_hint_auto], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
 
 vcl loadmasters (storage routers)
 ---------------------------------
@@ -472,17 +472,18 @@ actual figures are output at startup as ``fellow: metadata (bitmap)
 memory``.
 
 *objsize_hint* (default 256KB) is used to sanity check *memsize* in
-relation to *dsksize*. It should be set to a value lower than the
-average object size (actual or expected). If *memsize* is configured
-too low with respect to *dsksize* and *objsize_hint*, a higher value
-will be used (which might fail of insufficient memory is available).
+relation to *dsksize* and to size the transaction logs. It should be
+set to a value lower than the average object size (actual or
+expected). If *memsize* is configured too low with respect to
+*dsksize* and *objsize_hint*, a higher value will be used (which might
+fail of insufficient memory is available).
 
 *delete* specifies if the storage is to be emptied.
 
 .. _xfellow.tune():
 
-STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT wait_table_exponent], [INT lru_exponent], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT wait_table_exponent], [INT lru_exponent], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BOOL objsize_hint_auto], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
 ::
 
@@ -497,6 +498,7 @@ STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [
             [INT dsk_reserve_chunks],
             [INT mem_reserve_chunks],
             [BYTES objsize_hint],
+            [BOOL objsize_hint_auto],
             [BYTES objsize_max],
             [INT cram],
             [INT readahead],
@@ -655,6 +657,22 @@ fellow storage can be fine tuned:
   used to (re)size the fixed log space. The value should be chosen as
   a safe *lower* bound of average object size.
 
+  Because setting this value too high can cause performance issues, it
+  is automatically lowered and limited based on the actual object
+  sizes used. See next parameter.
+
+* *objsize_hint_auto*
+
+  - unit: boolean
+  - default: true
+  - minimum: false
+
+  Can be used to turn off automatic adjustments of objsize_hint as
+  described above.
+
+  It is strongly recommended to leave this switch ``true``, unless
+  recommended by a developer or based on a very good reason.
+
 * *objsize_max*
 
   - unit: bytes
@@ -871,8 +889,8 @@ Restricted to: ``vcl_init``.
 
 .. _slash.tune_fellow():
 
-STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT wait_table_exponent], [INT lru_exponent], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT wait_table_exponent], [INT lru_exponent], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BOOL objsize_hint_auto], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
 ::
 
@@ -888,6 +906,7 @@ STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_
       [INT dsk_reserve_chunks],
       [INT mem_reserve_chunks],
       [BYTES objsize_hint],
+      [BOOL objsize_hint_auto],
       [BYTES objsize_max],
       [INT cram],
       [INT readahead],
@@ -974,6 +993,8 @@ overhead and fragmentation.
 The higher ``reserve_chunks`` is set, the more agressively LRU will
 pre-evict objects in order to have space available for new requests.
 
+.. _diag:
+
 FELLOW DIAGNOSTICS
 ==================
 

src/vmod_slash.vcc

@@ -36,7 +36,7 @@ vcl storage objects and methods
 
   new xfellow = slash.fellow(STRING filename, BYTES dsksize, BYTES memsize, BYTES objsize_hint, BOOL delete)
 
-      STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
+      STRING xfellow.tune([INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BOOL objsize_hint_auto], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
 
       STEVEDORE xfellow.storage()
 
@@ -53,7 +53,7 @@ vcl functions
 
   STRING tune_buddy(STEVEDORE storage, [INT chunk_exponent], [BYTES chunk_bytes], [INT reserve_chunks], [INT cram])
 
-  STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
+  STRING tune_fellow(STEVEDORE storage, [INT logbuffer_size], [DURATION logbuffer_flush_interval], [REAL log_rewrite_ratio], [INT chunk_exponent], [BYTES chunk_bytes], [INT dsk_reserve_chunks], [INT mem_reserve_chunks], [BYTES objsize_hint], [BOOL objsize_hint_auto], [BYTES objsize_max], [INT cram], [INT readahead], [BYTES discard_immediate], [INT io_batch_min], [INT io_batch_max], [ENUM hash_obj], [ENUM hash_log], [ENUM ioerr_obj], [ENUM ioerr_log], [ENUM allocerr_obj], [ENUM allocerr_log])
 
 vcl loadmasters (storage routers)
 ---------------------------------
@@ -416,10 +416,11 @@ actual figures are output at startup as ``fellow: metadata (bitmap)
 memory``.
 
 *objsize_hint* (default 256KB) is used to sanity check *memsize* in
-relation to *dsksize*. It should be set to a value lower than the
-average object size (actual or expected). If *memsize* is configured
-too low with respect to *dsksize* and *objsize_hint*, a higher value
-will be used (which might fail of insufficient memory is available).
+relation to *dsksize* and to size the transaction logs. It should be
+set to a value lower than the average object size (actual or
+expected). If *memsize* is configured too low with respect to
+*dsksize* and *objsize_hint*, a higher value will be used (which might
+fail of insufficient memory is available).
 
 *delete* specifies if the storage is to be emptied.
 
@@ -434,6 +435,7 @@ $Method STRING .tune(
 	[ INT		dsk_reserve_chunks ],
 	[ INT		mem_reserve_chunks ],
 	[ BYTES		objsize_hint ],
+	[ BOOL		objsize_hint_auto ],
 	[ BYTES		objsize_max ],
 	[ INT		cram ],
 	[ INT		readahead ],
@@ -591,6 +593,22 @@ fellow storage can be fine tuned:
   used to (re)size the fixed log space. The value should be chosen as
   a safe *lower* bound of average object size.
 
+  Because setting this value too high can cause performance issues, it
+  is automatically lowered and limited based on the actual object
+  sizes used. See next parameter.
+
+* *objsize_hint_auto*
+
+  - unit: boolean
+  - default: true
+  - minimum: false
+
+  Can be used to turn off automatic adjustments of objsize_hint as
+  described above.
+
+  It is strongly recommended to leave this switch ``true``, unless
+  recommended by a developer or based on a very good reason.
+
 * *objsize_max*
 
   - unit: bytes
@@ -804,6 +822,7 @@ $Function STRING tune_fellow(
 	[ INT		dsk_reserve_chunks ],
 	[ INT		mem_reserve_chunks ],
 	[ BYTES		objsize_hint ],
+	[ BOOL		objsize_hint_auto ],
 	[ BYTES		objsize_max ],
 	[ INT		cram ],
 	[ INT		readahead ],
@@ -889,6 +908,8 @@ overhead and fragmentation.
 The higher ``reserve_chunks`` is set, the more agressively LRU will
 pre-evict objects in order to have space available for new requests.
 
+.. _diag:
+
 FELLOW DIAGNOSTICS
 ==================