Fix logic error in fellow_busy_region_free & add test

The code accepted the right pointer, the right offset _OR_ the right
size. Which lead to the wrong (usually the last) segment being freed.

Fixes #39

src/fellow_cache.c

@@ -2193,9 +2193,7 @@ fellow_busy_region_free(struct fellow_busy *fbo, struct buddy_off_extent *fdr)
 	AN(fbo->nregion);
 	u = fbo->nregion;
 	while (u--) {
-		if (fdr != &fbo->region[u] &&
-		    fdr->off != fbo->region[u].off &&
-		    fdr->size != fbo->region[u].size)
+		if (fdr != &fbo->region[u])
 			continue;
 		fellow_region_free(fc->ffd, fdr);
 		fbo->nregion--;
@@ -4395,7 +4393,7 @@ fellow_busy_obj_trimstore(struct fellow_busy *fbo)
 	fbo->body_seglist = NULL;
 }
 
-#ifdef XXXDEBUG
+#ifdef TEST_DRIVER
 static int
 fdr_compar(const void *aa, const void *bb)
 {
@@ -4448,7 +4446,11 @@ fellow_busy_log_submit(const struct fellow_busy *fbo)
 	assert(r <= ndle);
 	fellow_log_dle_submit(fbo->fc->ffd, dles, r);
 
-#ifdef XXXDEBUG
+#ifdef TEST_DRIVER
+	/*
+	 * additional sanity check that we get back the same regions via
+	 * fellow_obj_regions which we have created originally
+	 */
 	struct buddy_off_extent region[FCO_MAX_REGIONS];
 	unsigned n;
 
@@ -5974,6 +5976,84 @@ t_cache(unsigned chksum)
 	test_bocdone(fbo, TRUST_ME(hash), 1);
 	fellow_cache_obj_deref(fc, fco);
 
+	// === max out region alloc
+	fbo = fellow_busy_obj_alloc(fc, &fco, &priv2, 1234).r.ptr;
+	CHECK_OBJ_NOTNULL(fbo, FELLOW_BUSY_MAGIC);
+
+	u = 0;
+	while (fbo->nregion < FCO_MAX_REGIONS - 1) {
+		struct fellow_cache_seglist *fcsl;
+		struct fellow_body_region *fbr;
+		struct buddy_off_extent *fdr;
+		struct fellow_cache_seg *fcs;
+
+		/* allocate segments until seglist is full
+		 *
+		 * skip body region alloc heuristic in
+		 * fellow_busy_body_seg_alloc() by making sure we always have an
+		 * empty region
+		 */
+		fcsl = fbo->body_seglist;
+		fbr = &fbo->body_region;
+		while (fbo->nregion < FCO_MAX_REGIONS - 1 &&
+		    fcsl->fdsl->nsegs < fcsl->fdsl->lsegs) {
+			fdr = fbr->reg;
+			if (fdr == NULL || fbr->len == fdr->size) {
+				memset(fbr, 0, sizeof *fbr);
+				fdr = fellow_busy_region_alloc(fbo, 1234,
+				    INT8_MAX);
+				fbr->reg = fdr;
+				AZ(fbr->len);
+			}
+			AN(fdr);
+			assert(fbr->len < fdr->size);
+			assert(fcsl->segs[fcsl->fdsl->nsegs].state ==
+			    FCS_USABLE);
+			fcs = &fcsl->segs[fcsl->fdsl->nsegs++];
+			AN(fellow_busy_body_seg_alloc(fbo, fcs->disk_seg,
+				SIZE_MAX >> 1));
+			// pretend - not actually written
+			fcs->state = FCS_DISK;
+			fbo->body_seg = fcs;
+		}
+
+		if (fbo->nregion == FCO_MAX_REGIONS - 1)
+			break;
+
+		fcsl = fbo->body_seglist;
+		fcr = fellow_disk_seglist_alloc(fbo, fcsl, 1, FH_SHA256);
+		assert(fcr.status == fcr_ok);
+		CAST_OBJ_NOTNULL(fcsl, fcr.r.ptr, FELLOW_CACHE_SEGLIST_MAGIC);
+		u++;
+
+		fbo->body_seglist = fcsl;
+		assert(fcsl->fdsl->lsegs >= 3);
+		if (FCO_MAX_REGIONS - fbo->nregion <= 4)
+			fcsl->fdsl->lsegs = 3;
+		else
+			fcsl->fdsl->lsegs = 1;
+	}
+	DBG("seglists created %u", u);
+
+	for (u = 0; u < OA__MAX; u++) {
+		AN(fellow_busy_setattr(fbo, u, strlen(oatest[u]), oatest[u]));
+	}
+	t_getattr(fc, fco);
+
+	/*
+	 * #39
+	 * test fellow_busy_obj_trimstore() returning
+	 * the second last segment (ESI attr are last)
+	 */
+	AN(fbo->body_seg);
+	assert(fbo->body_seg->state == FCS_DISK);
+	fbo->body_seg->state = FCS_USABLE;
+	AN(fellow_busy_seg_memalloc(membuddy, fbo->body_seg, 0));
+
+	assert(fbo->nregion == FCO_MAX_REGIONS);
+	test_bocdone(fbo, TRUST_ME(hash), 1);
+	fellow_cache_obj_delete(fc, fco, hash);
+
 	// === alloc space, dont use
 	fbo = fellow_busy_obj_alloc(fc, &fco, &priv2, 1234).r.ptr;
 	CHECK_OBJ_NOTNULL(fbo, FELLOW_BUSY_MAGIC);