fellow_cache: Improve the seglist sizing strategy

previously, we just sized the disk seglist to fit.

Now we chose the size such that the memory seglist corresponding to the
disk seglist fits a power of two page.

src/fellow_cache.c

@@ -320,6 +320,8 @@ struct fellow_disk_seglist {
  * rounded down to a fellow block
  * minus the seglist
  * devided by size of the seg
+ *
+ * see t_seglist_sizes() output
  */
 
 #define FELLOW_DISK_SEGLIST_MAX_SEGS	/* 65534 */			\
@@ -328,8 +330,22 @@ struct fellow_disk_seglist {
 	sizeof(struct fellow_disk_seglist)) /				\
 	sizeof(struct fellow_disk_seg))
 
-#define fellow_disk_seglist_size(fdsl, lsegs)		\
-	(sizeof *fdsl + lsegs * sizeof *fdsl->segs)
+#define SEGLIST_SIZE(xdsl, lsegs)		\
+	(sizeof *xdsl + lsegs * sizeof *xdsl->segs)
+
+#define SEGLIST_FIT_FUNC(what)						\
+static inline uint16_t							\
+	fellow_ ## what ## _seglist_fit(size_t sz)			\
+{									\
+	assert(sz >= sizeof(struct fellow_ ## what ## _seglist));	\
+	sz -= sizeof(struct fellow_ ## what ## _seglist);		\
+	sz /= sizeof(struct fellow_ ## what ## _seg);			\
+	if (sz > UINT16_MAX)						\
+		sz = UINT16_MAX;					\
+	return (uint16_t)sz;						\
+}
+
+SEGLIST_FIT_FUNC(disk)
 
 struct fellow_cache_obj;
 struct fellow_cache_seg {
@@ -374,8 +390,7 @@ struct fellow_cache_seglist {
 	struct fellow_cache_seg	segs[];
 };
 
-#define fellow_cache_seglist_size(fcsl, lsegs)		\
-	(sizeof *fcsl + lsegs * sizeof *fcsl->segs)
+SEGLIST_FIT_FUNC(cache)
 
 struct fellow_disk_obj_attr {
 	uint32_t	aoff;	// from struct fellow_disk_obj
@@ -552,8 +567,19 @@ assert_oa_fdoa(void)
  * struct fellow_disk_obj	disk_obj
  * uint8_t			varattr_data[variable]
  * fellow_disk_seglist		seglist
+ *
+ * new memory allocation layout IF FITS IN POW2
+ *
+ * struct fellow_disk_obj	disk_obj
+ * uint8_t			varattr_data[variable]
+ * fellow_disk_seglist		seglist
+ * ... variable length disk segs
+ * fellow_cache_seglist
+ * ... variable length cache segs
+ * fellow_cache_obj
  */
 
+
 struct fellow_disk_obj {
 	uint32_t			magic;
 #define FELLOW_DISK_OBJ_MAGIC		0x50728fbd
@@ -657,6 +683,31 @@ struct fellow_cache_obj {
 	struct fellow_cache_seglist	fcsl_embed;
 };
 
+/*
+ * we write fellow_disk_object(s) with the embedded fellow_disk_seglist such
+ * that, when reading the object, the fdo can either be fully embedded in an
+ * allocation of 4KB, or at least the fcsl_embed can hold all of the
+ * fellow_disk_seglist.
+ *
+ * we want the disk format to be stable, but we might need to grow the
+ * fellow_cache_obj, so add some extra headroom
+ *
+ * starting point on linux
+ *
+ * (gdb) p ((1<<12) - sizeof(struct fellow_cache_obj)) / sizeof(struct fellow_cache_seg)
+ * $4 = 58
+ * (gdb) p sizeof(struct fellow_cache_obj) + 58 * sizeof(struct fellow_cache_seg)
+ * $5 = 4072
+ *
+ * -> 24 bytes left
+ *
+ * mutex_t and cond_t can be bigger on other platforms and might change in size,
+ * so this optimization may always fail. no problem if it does, we just waste
+ * some space when reading back objects
+ */
+
+//#define FCO_MAX_SIZE
+
 #define fellow_cache_obj_size(fco)		\
 	(sizeof *fco + (fco)->fcsl_embed.lsegs * sizeof *(fco)->fcsl_embed.segs)
 
@@ -1977,7 +2028,7 @@ fellow_cache_seglists_load(const struct fellow_cache *fc,
 		ofcsl = fcsl;
 
 		fcsl_mem = buddy_alloc1_ptr_extent_wait(fc->membuddy, FEP_META,
-		    fellow_cache_seglist_size(fcsl, fdsl->nsegs), 0);
+		    SEGLIST_SIZE(fcsl, fdsl->nsegs), 0);
 
 		if (FC_INJ || fcsl_mem.ptr == NULL) {
 			err = FC_ERRSTR("disk seglist fcsl alloc failed");
@@ -2412,6 +2463,28 @@ fellow_busy_region_free(struct fellow_busy *fbo, struct buddy_off_extent *fdr)
 static uint16_t XXX_LIMIT_LDSEGS = 0;
 #endif
 
+static uint16_t
+fellow_busy_body_seglist_size_strategy(size_t segs, unsigned chunk_exponent)
+{
+	struct fellow_cache_seglist *fcsl = NULL;
+	unsigned bits;
+
+	// size of the fcsl, round down to power2
+	bits = log2down(SEGLIST_SIZE(fcsl, segs));
+	(void) fcsl;	// flexelint
+
+	if (bits < MIN_FELLOW_BITS)
+		bits = MIN_FELLOW_BITS;
+	else if (bits > chunk_exponent)
+		bits = chunk_exponent;
+
+	segs = fellow_cache_seglist_fit((size_t)1 << bits);
+	if (segs > FELLOW_DISK_SEGLIST_MAX_SEGS)
+		segs = FELLOW_DISK_SEGLIST_MAX_SEGS;
+	assert(segs <= UINT16_MAX);
+	return ((uint16_t)segs);
+}
+
 static struct fellow_cache_res
 fellow_disk_seglist_alloc(struct fellow_busy *fbo,
     struct fellow_cache_seglist *ofcsl, uint16_t ldsegs, uint8_t fht);
@@ -2434,17 +2507,24 @@ fellow_busy_body_seglist_alloc(struct fellow_busy *fbo,
 
 	assert(fbo->sz_estimate > fbo->sz_returned);
 
+	/* sz is number of segments */
 	if (fbo->grown) {
 		sz = fbo->fc->tune->objsize_max - fbo->sz_dskalloc;
+		sz += (((size_t)1 << MIN_FELLOW_BITS) - 1);
 		sz >>= MIN_FELLOW_BITS;
+		// can not apply strategy because we could hit FCO_MAX_REGIONS
+		if (sz > FELLOW_DISK_SEGLIST_MAX_SEGS)
+			sz = FELLOW_DISK_SEGLIST_MAX_SEGS;
+		ldsegs = (uint16_t)sz;
 	} else {
 		sz = fbo->sz_estimate - fbo->sz_returned;
+		sz += (((size_t)1 << chunk_exponent) - 1);
 		sz >>= chunk_exponent;
+		ldsegs = fellow_busy_body_seglist_size_strategy(sz,
+		    chunk_exponent);
 	}
-	sz++;
-	if (sz > FELLOW_DISK_SEGLIST_MAX_SEGS)
-		sz = FELLOW_DISK_SEGLIST_MAX_SEGS;
-	ldsegs = (uint16_t)sz;
+
+
 #ifdef TEST_DRIVER
 	if (XXX_LIMIT_LDSEGS > 0)
 		ldsegs = XXX_LIMIT_LDSEGS;
@@ -2476,25 +2556,19 @@ fellow_disk_seglist_alloc(struct fellow_busy *fbo,
 	AZ(ofdsl->next.off);
 	AZ(ofdsl->next.size);
 
-	sz = fellow_disk_seglist_size(fdsl, ldsegs);
+	sz = SEGLIST_SIZE(fdsl, ldsegs);
 
 	fdr = fellow_busy_region_alloc(fbo, sz, 0);
 	if (fdr == NULL)
 		return (FCR_ALLOCFAIL("seglist disk region"));
 	ofdsl->next = *fdr;
 
-	sz = fdr->size;
-	assert(sz > sizeof *fdsl);
-	sz -= sizeof *fdsl;
-	sz /= sizeof *fdsl->segs;
-	if (sz > UINT16_MAX)
-		sz = UINT16_MAX;
-	ldsegs = (uint16_t)sz;
+	ldsegs = fellow_disk_seglist_fit(fdr->size);
 
 	reqs = BUDDY_REQS_STK(fbo->fc->membuddy, 2);
 	BUDDY_REQS_PRI(reqs, FEP_META);
 	AN(buddy_req_extent(reqs, fdr->size, 0));
-	AN(buddy_req_extent(reqs, fellow_cache_seglist_size(fcsl, ldsegs), 0));
+	AN(buddy_req_extent(reqs, SEGLIST_SIZE(fcsl, ldsegs), 0));
 	u = buddy_alloc_wait(reqs);
 	if (FC_INJ || u != 2) {
 		buddy_alloc_wait_done(reqs);
@@ -5434,7 +5508,7 @@ struct objcore **ocp, uintptr_t priv2, unsigned crit)
 	else {
 		// dup fellow_cache_seglists_load()
 		fcsl_mem = buddy_alloc1_ptr_extent_wait(fc->membuddy, FEP_META,
-		    fellow_cache_seglist_size(fco->fcsl, fdsl->nsegs), 0);
+		    SEGLIST_SIZE(fco->fcsl, fdsl->nsegs), 0);
 		if (FC_INJ || fcsl_mem.ptr == NULL) {
 			err = FC_ERRSTR("first disk seglist fcsl alloc failed");
 			goto err;
@@ -6924,6 +6998,42 @@ t_tailq_concat(void)
 	AZ(VTAILQ_NEXT(fcs, lru_list));
 }
 
+/*
+ * if this fails, the strategy in fellow_busy_body_seglist_alloc() needs to be
+ * adjusted
+ */
+static void
+t_seglist_sizes(void)
+{
+	struct fellow_disk_seglist *fdsl;
+	unsigned u, fcslsegs, fdslsegs, fdslcap;
+	uint16_t strategy;
+	size_t fdslsz;
+
+	for (u = 12; u < 30; u++) {
+		fcslsegs = fellow_cache_seglist_fit((size_t)1 << u);
+		fdslsegs = fellow_disk_seglist_fit((size_t)1 << u);
+		assert(fcslsegs <= fdslsegs);
+		fdslsz = rup_min(SEGLIST_SIZE(fdsl, fcslsegs),
+		    MIN_FELLOW_BITS);
+		fdslcap = fellow_disk_seglist_fit(fdslsz);
+		strategy = fellow_busy_body_seglist_size_strategy(fcslsegs + 1, 30);
+		fprintf(stderr, "seglist 1<<%2u fcsl %6u (strat %6u) fdsl %6u "
+		    "-> fdsl_size %7zu (%3zu blocks) cap %6u waste %6u\n",
+		    u, fcslsegs, strategy, fdslsegs,
+		    fdslsz, fdslsz >> MIN_FELLOW_BITS,
+		    fdslcap, fdslcap - fcslsegs);
+		// special case because limit in *seglist_fit
+		if (strategy == FELLOW_DISK_SEGLIST_MAX_SEGS)
+			assert(fcslsegs == 65535);
+		else
+			assert(strategy == fcslsegs);
+		if (fcslsegs >= FELLOW_DISK_SEGLIST_MAX_SEGS ||
+		    fdslsegs >= FELLOW_DISK_SEGLIST_MAX_SEGS)
+			break;
+	}
+}
+
 int
 main(int argc, char **argv)
 {
@@ -6936,6 +7046,8 @@ main(int argc, char **argv)
 	DBG("fellow_busy %zu", sizeof(struct fellow_busy));
 	assert(sizeof(struct fellow_busy) <= 2 * MIN_FELLOW_BLOCK);
 
+	t_seglist_sizes();
+
 	t_tailq_concat();
 
 #ifdef HAVE_XXHASH_H