fellow_cache: embed fdo in fco allocation if possible

Conflicts:
	src/fellow_cache.c

src/fellow_cache.c

@@ -323,7 +323,7 @@ struct fellow_disk_seglist {
  */
 
 #define FELLOW_DISK_SEGLIST_MAX_SEGS	/* 65534 */			\
-	(((((sizeof(struct fellow_disk_seg) << 16) +			\
+	(unsigned)(((((sizeof(struct fellow_disk_seg) << 16) +		\
 	sizeof(struct fellow_disk_seglist)) & ~FELLOW_BLOCK_ALIGN) -	\
 	sizeof(struct fellow_disk_seglist)) /				\
 	sizeof(struct fellow_disk_seg))
@@ -704,7 +704,7 @@ struct fellow_cache_obj {
  */
 
 #define FDO_MAX_EMBED_SEGS \
-	((MIN_FELLOW_BLOCK - sizeof(struct fellow_cache_obj)) \
+	(unsigned)((MIN_FELLOW_BLOCK - sizeof(struct fellow_cache_obj))	\
 	 / sizeof(struct fellow_cache_seg))
 
 #define fellow_cache_obj_size(fco)		\
@@ -1158,6 +1158,9 @@ fellow_disk_obj_trim(const struct fellow_cache *fc,
 	fdo = fellow_disk_obj(fcs);
 	CHECK_OBJ_NOTNULL(fdo, FELLOW_DISK_OBJ_MAGIC);
 
+	// not if already relocated
+	assert(fdo == fcs->alloc.ptr);
+
 	fdsl = fellow_disk_obj_fdsl(fdo);
 	CHECK_OBJ(fdsl, FELLOW_DISK_SEGLIST_MAGIC);
 
@@ -2085,16 +2088,16 @@ fellow_cache_obj_new(
 	struct buddy_reqs *reqs;
 	struct buddy_ptr_extent fco_mem = buddy_ptr_extent_nil;
 	struct buddy_ptr_extent fdo_mem = buddy_ptr_extent_nil;
-	size_t asz, mem_sz, ssz;
+	size_t mem_sz, ssz;
 	unsigned u;
 
 	DBG("arg dsk_sz %zu nseg_guess %u", dsk_sz, nseg_guess);
 
-	if (nseg_guess == 0)
-		nseg_guess = 1;
+	AN(nseg_guess);
 
+	// for busy objects, mem_sz may be less than MIN_FELLOW_BLOCK
 	mem_sz = (size_t)1 << log2up(sizeof *fco + nseg_guess * sizeof *fcs);
-	asz = (size_t)1 << log2up(dsk_sz);
+	assert(mem_sz <= MIN_FELLOW_BLOCK);
 
 	// dowry is largest, so we allocate it first
 	reqs = BUDDY_REQS_STK(fc->membuddy, 4);
@@ -2103,14 +2106,12 @@ fellow_cache_obj_new(
 		AN(buddy_req_page(reqs, fc->tune->chunk_exponent, 0));
 	if (fbo_mem != NULL)
 		AN(buddy_req_extent(reqs, sizeof(struct fellow_busy), 0));
-	if (mem_sz > asz) {
-		AN(buddy_req_extent(reqs, mem_sz, 0));
-		AN(buddy_req_extent(reqs, asz, 0));
-	}
-	else {
-		AN(buddy_req_extent(reqs, asz, 0));
-		AN(buddy_req_extent(reqs, mem_sz, 0));
-	}
+
+	assert(dsk_sz == fellow_rndup(fc->ffd, dsk_sz));
+	assert(dsk_sz >= mem_sz);
+	AN(buddy_req_extent(reqs, dsk_sz, 0));
+	AN(buddy_req_extent(reqs, mem_sz, 0));
+
 	u = buddy_alloc_wait(reqs);
 	if (FC_INJ || u != 2 + (dowry ? 1 : 0) + (fbo_mem ? 1 : 0)) {
 		buddy_alloc_wait_done(reqs);
@@ -2125,14 +2126,8 @@ fellow_cache_obj_new(
 	if (fbo_mem != NULL)
 		*fbo_mem = buddy_get_next_ptr_extent(reqs);
 
-	if (mem_sz > asz) {
-		fco_mem = buddy_get_next_ptr_extent(reqs);
-		fdo_mem = buddy_get_next_ptr_extent(reqs);
-	}
-	else {
-		fdo_mem = buddy_get_next_ptr_extent(reqs);
-		fco_mem = buddy_get_next_ptr_extent(reqs);
-	}
+	fdo_mem = buddy_get_next_ptr_extent(reqs);
+	fco_mem = buddy_get_next_ptr_extent(reqs);
 
 	buddy_alloc_wait_done(reqs);
 
@@ -2972,6 +2967,12 @@ fellow_cache_obj_trim(const struct fellow_cache *fc,
 
 	CHECK_OBJ_NOTNULL(fc, FELLOW_CACHE_MAGIC);
 	CHECK_OBJ_NOTNULL(fco, FELLOW_CACHE_OBJ_MAGIC);
+
+	// no trim if fdo is embedded - XXX cleaner way?
+	if (FCO_FCS(fco)->alloc.ptr == NULL)
+		return;
+	assert(FCO_FDO(fco) == FCO_FCS(fco)->alloc.ptr);
+
 	fcsl = &fco->fcsl_embed;
 	CHECK_OBJ(fcsl, FELLOW_CACHE_SEGLIST_MAGIC);
 	fdsl = fcsl->fdsl;
@@ -5253,7 +5254,7 @@ fellow_cache_obj_prepread(struct fellow_cache *fc, fellow_disk_block fdba,
 	if (! PAOK(sz))
 		return (FCR_IOFAIL("bad size"));
 
-	fcr = fellow_cache_obj_new(fc, sz, 0, NULL,
+	fcr = fellow_cache_obj_new(fc, sz, FDO_MAX_EMBED_SEGS, NULL,
 	    crit ? NULL : &dowry,
 	    crit ? FEP_OHLCK : FEP_GET);
 	if (fcr.status != fcr_ok)
@@ -5381,6 +5382,8 @@ struct objcore **ocp, uintptr_t priv2, unsigned crit)
 	fellow_disk_block fdba;
 	unsigned oref = 0;
 	const char *err;
+	char *ptr;
+	size_t spc, sz;
 	int r;
 
 	CHECK_OBJ_NOTNULL(fc, FELLOW_CACHE_MAGIC);
@@ -5481,28 +5484,24 @@ struct objcore **ocp, uintptr_t priv2, unsigned crit)
 	if (err != NULL)
 		goto err;
 
-	// or because oa_inlog_bit
+	// |= is because of oa_inlog_bit
 	if (fco->oc)
 		fco->oc->oa_present |= fdoa2oa_present(fdo->fdoa_present);
 
-	fdsl = fellow_disk_obj_fdsl(fdo);
 	/* the check of the fdo-embedded fdsl needs to happen before trim,
 	 * because both trim and relocation change data after the used part of
 	 * the fdsl
 	 */
+	fdsl = fellow_disk_obj_fdsl(fdo);
 	err = fellow_disk_seglist_check(fdsl);
 	if (err != NULL)
 		goto err;
 
-	/* trim may relocate, so get the new fdo and fdsl pointer */
-	fdo = fellow_disk_obj_trim(fc, fcs);
-	fdsl = fellow_disk_obj_fdsl(fdo);
-	assert(PAOK(fdsl));
-	CHECK_OBJ_NOTNULL(fdsl, FELLOW_DISK_SEGLIST_MAGIC);
-
-	// the fco-embedded fcsl may or may not fit
-	if (fco->fcsl_embed.lsegs >= fdsl->nsegs)
+	// use the embedded fcsl if possible
+	if (fco->fcsl_embed.lsegs >= fdsl->nsegs) {
 		fco->fcsl = &fco->fcsl_embed;
+		fco->fcsl_embed.lsegs = fdsl->nsegs;
+	}
 	else {
 		// dup fellow_cache_seglists_load()
 		fcsl_mem = buddy_alloc1_ptr_extent_wait(fc->membuddy, FEP_META,
@@ -5516,7 +5515,38 @@ struct objcore **ocp, uintptr_t priv2, unsigned crit)
 		AN(fco->fcsl);
 		fco->fcsl->fcsl_sz = fcsl_mem.size;
 		fcsl_mem = buddy_ptr_extent_nil;
+		fco->fcsl_embed.lsegs = 0;
+	}
+
+	/* can we fit the fdo?
+	 *
+	 * XXX: for the extra fcsl case above, we are wasting
+	 * sizeof(struct fellow_cache_seglist)
+	 */
+	AN(fco->fco_mem.ptr);
+	AN(fco->fco_mem.size);
+	sz = fellow_cache_obj_size(fco);
+	assert(sz <= fco->fco_mem.size);
+	ptr = (char *)fco + sz;
+	assert(PAOK(ptr));
+
+	spc = fco->fco_mem.size - sz;
+
+	fdsl->lsegs = fdsl->nsegs;
+	sz = fellow_disk_obj_size(fdo, fdsl);
+
+	if (sz <= spc) {
+		memcpy(ptr, fdo, sz);
+		fdo = fcs->u.fco_fdo = (void *)ptr;
+
+		buddy_return1_ptr_extent(fc->membuddy, &fcs->alloc);
 	}
+	else
+		fdo = fellow_disk_obj_trim(fc, fcs);
+
+	fdsl = fellow_disk_obj_fdsl(fdo);
+	assert(PAOK(fdsl));
+	CHECK_OBJ_NOTNULL(fdsl, FELLOW_DISK_SEGLIST_MAGIC);
 
 	// XXX load of folow-up seglists could be async
 	// should not be a common case though, we try to make the first
@@ -6657,7 +6687,7 @@ t_cache(const char *fn, unsigned chksum)
 
 	DBGSZ(fellow_disk_seg);
 	DBGSZ(fellow_disk_seglist);
-	DBG("FELLOW_DISK_SEGLIST_MAX_SEGS %zu", FELLOW_DISK_SEGLIST_MAX_SEGS);
+	DBG("FELLOW_DISK_SEGLIST_MAX_SEGS %u", FELLOW_DISK_SEGLIST_MAX_SEGS);
 	assert(FELLOW_DISK_SEGLIST_MAX_SEGS <= UINT16_MAX);
 	DBGSZ(fellow_disk_obj);