Buddy: reprioritize requests before waiting

src/buddy.c

@@ -1747,6 +1747,51 @@ buddy_reqs_wait_cancel(struct buddy_reqs *reqs)
 	//lint -e{454} locked mutex not unlocked
 }
 
+/*
+ * if necessary, adjust the priority of a waiting allocation
+ * to that set in the reqs
+ */
+static inline void
+buddy_reqs_wait_prio_adjust(struct buddy_reqs *reqs)
+{
+	struct buddy_reqs_head *ohead, *nhead;
+	struct i_wait *w;
+	buddy_t *buddy;
+
+	CHECK_OBJ_NOTNULL(reqs, BUDDY_REQS_MAGIC);
+	w = &reqs->i_wait;
+	CHECK_OBJ_NOTNULL(w, I_WAIT_MAGIC);
+
+	// unlocked checks for noop
+	if (w->state != IW_WAITING ||
+	    w->pri == reqs->pri)
+		return;
+
+	buddy = reqs->buddy;
+	ohead = &buddy->reqs_head[w->pri];
+	nhead = &buddy->reqs_head[reqs->pri];
+
+	AZ(pthread_mutex_lock(&reqs->buddy->map_mtx));
+	AZ(pthread_mutex_lock(&w->wait_mtx));
+
+	if (w->state == IW_WAITING) {
+		AN(buddy->waiting);
+		/* _INSERT_HEAD? _INSERT_TAIL?
+		 *
+		 * _HEAD sounds right, because the request either cam from a
+		 * higher priority, or it was promoted because it is urgent
+		 */
+		VTAILQ_REMOVE(ohead, reqs, i_wait.list);
+		VTAILQ_INSERT_HEAD(nhead, reqs, i_wait.list);
+		w->pri = reqs->pri;
+	}
+	AZ(pthread_mutex_unlock(&w->wait_mtx));
+
+	if (buddy->waiting && buddy->wait_pri == w->pri)
+		buddy_wait_work(buddy);
+	AZ(pthread_mutex_unlock(&reqs->buddy->map_mtx));
+}
+
 static uint8_t
 buddy_reqs_wait_allocated(struct i_wait *w)
 {
@@ -1938,6 +1983,7 @@ BUDDYF(alloc_async_wait)(struct buddy_reqs *reqs)
 	if (reqs->n == 0)
 		return (0);
 
+	buddy_reqs_wait_prio_adjust(reqs);
 	alloced = buddy_reqs_wait_allocated(w);
 	buddy_reqs_fini(reqs, &alloced);
 
@@ -2322,6 +2368,15 @@ wrap_alloc1_ptr_page(buddy_t *buddy, unsigned bits, int cram)
 #endif
 }
 
+static int
+wrap_req_page(struct buddy_reqs *reqs, unsigned bits, int8_t cram)
+{
+#ifdef FREEPAGE_WHEN
+	return (buddywhen_req_page(reqs, bits, cram, drand48()));
+#else
+	return (buddy_req_page(reqs, bits, cram));
+#endif
+}
 
 static void *
 wrap_alloc1_ptr_extent(buddy_t *buddy, size_t *sizep)
@@ -2679,6 +2734,44 @@ t_async(buddy_t *buddy)
 	BUDDYF(alloc_async_done)(reqs);
 }
 
+static void
+t_prio(buddy_t *buddy)
+{
+	struct buddy_reqs *reqs_block = BUDDY_REQS_STK(buddy, 1);
+	struct buddy_reqs *reqs_raise = BUDDY_REQS_STK(buddy, 1);
+	struct buddy_ptr_page hog[2];
+
+	// allocate (more than) half the buddy plus a bit
+	hog[0] = wrap_alloc1_ptr_page(buddy, log2up(buddy->map->size) - 1, 0);
+	AN(hog[0].ptr);
+	hog[1] = wrap_alloc1_ptr_page(buddy, MIN_BUDDY_BITS, 0);
+	AN(hog[1].ptr);
+
+	// try to allocate another half at prio one, this must block
+	AN(wrap_req_page(reqs_block, log2up(buddy->map->size) - 1, 0));
+	BUDDY_REQS_PRI(reqs_block, 1);
+	AZ(BUDDYF(alloc_async)(reqs_block));
+
+	// try to allocate min bits at prio 0, this must also block
+	AN(wrap_req_page(reqs_raise, MIN_BUDDY_BITS, 0));
+	BUDDY_REQS_PRI(reqs_raise, 0);
+	AZ(BUDDYF(alloc_async)(reqs_raise));
+
+	// raise the priority and wait, this should return.
+	// note: because of _INSERT_HEAD in the wait, prio 1
+	// is sufficient
+	BUDDY_REQS_PRI(reqs_raise, 1);
+	AN(BUDDYF(alloc_async_wait)(reqs_raise));
+
+	// no need to get the actual allocations, the fact that we got
+	// here tests that the prio raise has been successful
+
+	BUDDYF(alloc_async_done)(reqs_raise);
+	BUDDYF(alloc_async_done)(reqs_block);
+	BUDDYF(return1_ptr_page)(buddy, &hog[0]);
+	BUDDYF(return1_ptr_page)(buddy, &hog[1]);
+}
+
 static void
 t_page(buddy_t *buddy, size_t free)
 {
@@ -2763,6 +2856,9 @@ t_buddy(size_t free)
 	t_async(buddy);
 	assert(free == map->size);
 
+	t_prio(buddy);
+	assert(free == map->size);
+
 	t_page(buddy, free);
 	assert(free == map->size);