Generalize the "pooled" async allocation idea from logbuffer dskreqs

A pool tries to always have allocations ready. There are two buddy_reqs,
and when one is depleted, allocations are taken from the other, while
the empty reqs are being filled again.

src/buddy.c

@@ -2800,8 +2800,69 @@ t_log2(size_t min, size_t max)
 	fprintf(stderr, "log2 %zu..%zu OK\n", min, max);
 }
 
+BUDDY_POOL(t_pool_s, 4);
+
+static void
+t_pool_fill(struct buddy_reqs *reqs, const void *priv)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(reqs, BUDDY_REQS_MAGIC);
+
+	(void)priv;
+
+	BUDDY_REQS_PRI(reqs, 3);	// 3 is arbitrary
+#ifdef FREEPAGE_WHEN
+	for (i = 0; i < reqs->space; i++)
+		AN(buddywhen_req_page(reqs, 8, 0, 42.0));
+#else
+	for (i = 0; i < reqs->space; i++)
+		AN(buddy_req_page(reqs, 8, 0));
+#endif
+}
+
+// defines t_pool_s_get()
+BUDDY_POOL_GET_FUNC(t_pool_s, static)
+
+/*
+ * get allocations from a pool and return them
+ */
+static void
+t_pool(void)
+{
+	unsigned u;
+	buddy_t bspc;
+	buddy_t *buddy = &bspc;
+	struct t_pool_s pool[1];
+	struct buddy_reqs *reqs;
+	struct buddy_returns *rets;
+	struct buddy_ptr_page alloc;
+
+	BUDDYF(init)(buddy, MIN_BUDDY_BITS, 1 << 14,
+	    BUDDYF(mmap), NULL,
+	    BUDDYF(mmap), NULL);
+	rets = BUDDY_RETURNS_STK(buddy, 8);
+
+	BUDDY_POOL_INIT(pool, buddy, t_pool_fill, NULL);
+
+	for (u = 0; u < 42; u++) {
+		reqs = t_pool_s_get(pool, NULL);
+		alloc = buddy_get_next_ptr_page(reqs);
+		assert(alloc.ptr > 0);
+		do {
+			BUDDYF(return_ptr_page)(rets, &alloc);
+			alloc = buddy_get_next_ptr_page(reqs);
+		} while (alloc.ptr > 0);
+	}
+	BUDDYF(return)(rets);
+	BUDDY_POOL_FINI(pool);
+
+	BUDDYF(fini)(&buddy, BUDDYF(unmap), NULL, NULL, NULL);
+}
+
 int main(void)
 {
+	t_pool();
 	t_cramlimit();
 	t_log2(1, (size_t)1 << 30);
 	t_log2(SIZE_MAX - 5, SIZE_MAX - 1);

src/buddy.h

@@ -572,6 +572,75 @@ struct foo {
 };
 */
 
+/*
+ * a reqs pair which acts as an async, (hopefully) always filled reserve: one
+ * req is active (allocations are taken from it), while the other is filling
+ *
+ * size is per reqs, so the overall size is twice that
+ */
+
+typedef void buddy_pool_fill_f(struct buddy_reqs *, const void *);
+
+#define BUDDY_POOL_MAGIC 0x729c0ffd
+
+#define BUDDY_POOL(name, size)				\
+	struct name {					\
+		unsigned magic;				\
+		unsigned active;			\
+		buddy_pool_fill_f *fill;		\
+		BUDDY_REQS(,size) reqs[2];		\
+	}
+
+
+/*
+ * i < 3: If both reqs are empty, at iteration 3 we
+ * must hit the first filled alloc
+ */
+#define BUDDY_POOL_GET_FUNC(name, decl)			\
+decl struct buddy_reqs *					\
+name ## _get(struct name *poolp, const void *priv)		\
+{								\
+	struct buddy_reqs *reqs;				\
+	int i;							\
+								\
+	CHECK_OBJ_NOTNULL(poolp, BUDDY_POOL_MAGIC);		\
+	AN(poolp->fill);					\
+	for (i = 0; i < 3; i++) {				\
+		AZ(poolp->active & ~1);				\
+		reqs = &((poolp)->reqs[poolp->active].reqs);	\
+		CHECK_OBJ_NOTNULL(reqs, BUDDY_REQS_MAGIC);	\
+		if (buddy_reqs_next_ready(reqs))		\
+			return (reqs);				\
+		(void) BUDDYF(alloc_async_wait)(reqs);		\
+		if (buddy_reqs_next_ready(reqs))		\
+			return (reqs);				\
+		BUDDYF(alloc_wait_done)(reqs);			\
+		poolp->fill(reqs, priv);			\
+		(void) BUDDYF(alloc_async)(reqs);		\
+								\
+		poolp->active = ! poolp->active;		\
+	}							\
+	WRONG("Expected second return() to be hit");		\
+}
+
+#define BUDDY_POOL_INIT(poolp, b, f, p) do {				\
+	INIT_OBJ(poolp, BUDDY_POOL_MAGIC);				\
+	poolp->fill = f;						\
+	BUDDY_REQS_INIT(&(poolp)->reqs[0], b);				\
+	BUDDY_REQS_INIT(&(poolp)->reqs[1], b);				\
+	f(&(poolp)->reqs[0].reqs, p);					\
+	f(&(poolp)->reqs[1].reqs, p);					\
+	(void) BUDDYF(alloc_async)(&(poolp)->reqs[0].reqs);		\
+	(void) BUDDYF(alloc_async)(&(poolp)->reqs[1].reqs);		\
+} while (0)
+
+#define BUDDY_POOL_FINI(poolp) do {					\
+	CHECK_OBJ_NOTNULL(poolp, BUDDY_POOL_MAGIC);			\
+	BUDDYF(alloc_wait_done)(&((poolp)->reqs[0].reqs));		\
+	BUDDYF(alloc_wait_done)(&((poolp)->reqs[1].reqs));		\
+	memset(poolp, 0, sizeof *poolp);				\
+} while (0);
+
 #define BUDDY_REQS_LIT(b, size)						\
 {							\
 	.magic = BUDDY_REQS_MAGIC,					\