fellow_cache: Change embedded fcsl to slim fco for external fcsl

src/fellow_cache.c

@@ -401,6 +401,9 @@ struct fellow_cache_seglist {
 
 SEGLIST_FIT_FUNC(cache)
 
+#define fellow_cache_seglist_size(n) (sizeof(struct fellow_cache_seglist) + \
+	(n) * sizeof(struct fellow_cache_seg))
+
 struct fellow_disk_obj_attr {
 	uint32_t	aoff;	// from struct fellow_disk_obj
 	uint32_t	alen;
@@ -730,42 +733,49 @@ struct fellow_cache_obj {
 #include "tbl/fellow_obj_attr.h"
 
 	struct fellow_cache_seglist	*fcsl;
-	struct fellow_cache_seglist	fcsl_embed;
+	// *2) below
 };
 
 /* comments on the above:
  *
- * *1) this is redundant to region_fdb(&fdo->fdo_fds.seg) an is duplicated here
- *     for efficiency of the fdb lookup (see fco_cmp())
- */
-
-/*
- * we write fellow_disk_object(s) with the embedded fellow_disk_seglist such
- * that, when reading the object, the fcsl_embed can hold all of the
- * fellow_disk_seglist.
+ * *1) this is redundant to region_fdb(&fdo->fdo_fds.seg) for efficiency of the
+ *     fdb lookup (see fco_cmp())
+ *
+ * *2) fcsl can be a separate allocation or immediately follow the fco. Also,
+ *     the fellow_disk_obj may be copied into the same allocation. So the
+ *     structure is
+ *
+ *     fellow_cache_obj
+ *     [fellow_cache_seglist + n * fellow_cache_seg]
+ *     [fellow_disk_obj]
+ *
+ *     we write fellow_disk_object(s) with the embedded fellow_disk_seglist such
+ *     that, when reading the object, the fco allocation can also hold all of
+ *     the fellow_cache_seglist required for the fellow_disk_seglist.
  *
- * the numbers on linux:
+ *     the numbers on linux:
  *
- * (gdb) p ((1<<12) - sizeof(struct fellow_cache_obj)) / sizeof(struct fellow_cache_seg)
- * $4 = 58
- * (gdb) p sizeof(struct fellow_cache_obj) + 58 * sizeof(struct fellow_cache_seg)
- * $5 = 4072
+ *     (gdb) p ((1<<12) - sizeof(struct fellow_cache_obj) - sizeof(struct fellow_cache_seglist)) / sizeof(struct fellow_cache_seg)
+ *     $1 = 58
+ *     (gdb) p sizeof(struct fellow_cache_obj) + sizeof(struct fellow_cache_seglist) + 58 * sizeof(struct fellow_cache_seg)
+ *     $2 = 4072
  *
- * -> 24 bytes left
+ *     -> 24 bytes left
  *
- * mutex_t and cond_t can be bigger on other platforms and might change in size,
- * so this optimization may always fail when transporting storage to other
- * platforms. no problem if it does, we just waste some space when reading back
- * objects
+ *     mutex_t and cond_t can be bigger on other platforms and might change in
+ *     size, so this optimization may always fail when transporting storage to
+ *     other platforms. no problem if it does, we might just waste some space
+ *     when reading back objects
  */
 
+#define fellow_cache_obj_with_seglist_size(n)				\
+	(sizeof(struct fellow_cache_obj) + fellow_cache_seglist_size(n))
+
 #define FDO_MAX_EMBED_SEGS \
-	(unsigned)((MIN_FELLOW_BLOCK - sizeof(struct fellow_cache_obj))	\
+	(unsigned)(((MIN_FELLOW_BLOCK - sizeof(struct fellow_cache_obj)) \
+	   - sizeof(struct fellow_cache_seglist))			\
 	 / sizeof(struct fellow_cache_seg))
 
-#define fellow_cache_obj_size(fco)		\
-	(sizeof *fco + (fco)->fcsl_embed.lsegs * sizeof *(fco)->fcsl_embed.segs)
-
 #define FCR_OK(p)\
 	(struct fellow_cache_res){.r.ptr = (p), .status = fcr_ok}
 #define FCR_ALLOCERR(str)\
@@ -2318,7 +2328,7 @@ fellow_cache_obj_new(
 	struct buddy_reqs *reqs;
 	struct buddy_ptr_extent fco_mem = buddy_ptr_extent_nil;
 	struct buddy_ptr_extent fdo_mem = buddy_ptr_extent_nil;
-	size_t mem_sz, ssz;
+	size_t mem_sz;
 	unsigned u;
 
 	DBG("arg dsk_sz %zu nseg_guess %u", dsk_sz, nseg_guess);
@@ -2326,7 +2336,8 @@ fellow_cache_obj_new(
 	AN(nseg_guess);
 
 	// for busy objects, mem_sz may be less than MIN_FELLOW_BLOCK
-	mem_sz = (size_t)1 << log2up(sizeof *fco + nseg_guess * sizeof *fcs);
+	mem_sz = (size_t)1 << log2up(
+	    fellow_cache_obj_with_seglist_size(nseg_guess));
 	assert(mem_sz <= MIN_FELLOW_BLOCK);
 
 	// dowry is largest, so we allocate it first
@@ -2375,10 +2386,6 @@ fellow_cache_obj_new(
 	AZ(pthread_mutex_init(&fco->mtx, &fc_mtxattr_errorcheck));
 	AZ(pthread_cond_init(&fco->cond, NULL));
 
-	ssz = fco_mem.size - sizeof *fco;
-
-	(void) fellow_cache_seglist_init(&fco->fcsl_embed, ssz, fco);
-
 #define FDO_AUXATTR(U, l)						\
 	fellow_cache_seg_init(&fco->aa_##l##_seg, fco, FCS_INIT);
 #include "tbl/fellow_obj_attr.h"
@@ -2556,8 +2563,13 @@ fellow_busy_obj_alloc(struct fellow_cache *fc,
 
 	fdsl = fellow_disk_seglist_init(fellow_disk_obj_fdsl(fdo),
 	    ldsegs, fc->tune->hash_obj);
-	fellow_cache_seglist_associate(&fco->fcsl_embed, fdsl, FCS_USABLE);
-	fco->fcsl = &fco->fcsl_embed;
+
+	// FDO_MAX_EMBED_SEGS ensures this
+	sz = fellow_cache_seglist_size(ldsegs);
+	assert(sizeof *fco + sz <= fco->fco_mem.size);
+	fco->fcsl = fellow_cache_seglist_init((void *)(fco + 1), sz, fco);
+	AN(fco->fcsl);
+	fellow_cache_seglist_associate(fco->fcsl, fdsl, FCS_USABLE);
 	fco->seglstate = SEGL_DONE;
 //	DBG("fdsl lsegs %u fcsl lsegs %u", fdsl->lsegs, fco->fcsl->lsegs);
 
@@ -3211,7 +3223,7 @@ fellow_cache_obj_trim(const struct fellow_cache *fc,
 	// no trim if fdo is embedded - XXX cleaner way?
 	assert(FCO_FDO(fco) == FCO_FCS(fco)->alloc.ptr);
 
-	fcsl = &fco->fcsl_embed;
+	fcsl = fco->fcsl;
 	CHECK_OBJ(fcsl, FELLOW_CACHE_SEGLIST_MAGIC);
 	fdsl = fcsl->fdsl;
 	CHECK_OBJ_ORNULL(fdsl, FELLOW_DISK_SEGLIST_MAGIC);
@@ -3220,13 +3232,14 @@ fellow_cache_obj_trim(const struct fellow_cache *fc,
 	else
 		n = fdsl->nsegs;
 
+	// for a busy object, fcsl and fdsl are embedded in fco/fdo
 	AZ(fcsl->fcsl_sz);
 	AZ(fcsl->fdsl_sz);
 
 	assert(fcsl->lsegs >= n);
 	fcsl->lsegs = n;
 	buddy_trim1_ptr_extent(fc->membuddy, &fco->fco_mem,
-	    fellow_cache_obj_size(fco));
+	    fellow_cache_obj_with_seglist_size(n));
 }
 
 /* XXX can save some io for inlog == 0 */
@@ -5113,8 +5126,8 @@ fellow_busy_obj_trim_seglists(struct fellow_busy *fbo)
 	// only to be called if anything to trim at all
 	AZ(fbo->body_seglist->fdsl->nsegs);
 
-	// not for the embedded seglist
-	if (fbo->body_seglist == &fco->fcsl_embed)
+	// not for the first (embedded) seglist
+	if (fbo->body_seglist == fco->fcsl)
 		return;
 
 	fcsl = fco->fcsl;
@@ -5381,7 +5394,6 @@ fellow_busy_done(struct fellow_busy *fbo, struct objcore *oc, unsigned inlog)
 
 	// the first disk seglist is always embedded
 	fcsl = fco->fcsl;
-	assert(fcsl == &fco->fcsl_embed);
 	AN(fcsl);
 	if (fcsl->next) {
 		AN(fcsl->fdsl);
@@ -5747,10 +5759,20 @@ struct objcore **ocp, uintptr_t priv2, unsigned crit)
 	if (err != NULL)
 		goto err;
 
-	// use the embedded fcsl if possible
-	if (fco->fcsl_embed.lsegs >= fdsl->nsegs) {
-		fco->fcsl = &fco->fcsl_embed;
-		fco->fcsl_embed.lsegs = fdsl->nsegs;
+	ptr = (void *)(fco + 1);
+	nsz = sizeof *fco;
+	spc = fco->fco_mem.size - nsz;
+	sz = fellow_cache_seglist_size(fdsl->nsegs);
+
+	assert(PAOK(ptr));
+
+	// embed fcsl if possible
+	if (spc >= sz) {
+		fco->fcsl = fellow_cache_seglist_init((void *)ptr, sz, fco);
+		AN(fco->fcsl);
+		ptr += sz;
+		nsz += sz;
+		spc -= sz;
 	}
 	else {
 		// dup fellow_cache_seglists_load()
@@ -5764,28 +5786,13 @@ struct objcore **ocp, uintptr_t priv2, unsigned crit)
 		    fcsl_mem.size - sizeof *fco->fcsl, fco);
 		AN(fco->fcsl);
 		fco->fcsl->fcsl_sz = fcsl_mem.size;
-		fco->fcsl_embed.lsegs = 0;
 	}
-
-	/* can we fit the fdo?
-	 *
-	 * XXX: for the extra fcsl case above, we are wasting
-	 * sizeof(struct fellow_cache_seglist)
-	 */
-	AN(fco->fco_mem.ptr);
-	AN(fco->fco_mem.size);
-	sz = fellow_cache_obj_size(fco);
-	assert(sz <= fco->fco_mem.size);
-	ptr = (char *)fco + sz;
 	assert(PAOK(ptr));
-	nsz = sz;
-
-	spc = fco->fco_mem.size - sz;
 
 	fdsl->lsegs = fdsl->nsegs;
 	sz = fellow_disk_obj_size(fdo, fdsl);
 
-	if (sz <= spc) {
+	if (spc >= sz) {
 		memcpy(ptr, fdo, sz);
 		fdo = fcs->u.fco_fdo = (void *)ptr;