Introduce a flush finish state

it is planned to replace the "inuse" tri-state and might turn
out helpful for debugging.

src/fellow_log.c

@@ -262,12 +262,28 @@ struct fellow_fd_ioctx_lease {
 	fellow_ioctx_t		*ioctx;
 };
 
+#define FF_TRANSITION(ff, from, to) do {	\
+		assert((ff)->state == from);	\
+		(ff)->state = to;		\
+	} while(0)
+
+enum ff_state_e {
+	FF_INVAL = 0,
+	FF_SCHEDULED,
+	FF_WAIT_OUTSTANDING,
+	FF_HEAD,
+	FF_HDR,
+	FF_FREE,
+	FF_DONE
+};
+
 // flush finish
 struct fellow_logbuffer_ff {
 	unsigned			magic;
 #define FELLOW_LOGBUFFER_FF_MAGIC	0xcb1341d3
 	unsigned			inuse;
 	unsigned			can;
+	enum ff_state_e			state;
 #ifdef DEBUG
 	vtim_mono			t[2];
 #endif
@@ -2704,6 +2720,8 @@ logbuffer_flush_finish(struct fellow_fd *ffd,
 	ff->t[1] = VTIM_mono();
 #endif
 
+	FF_TRANSITION(ff, FF_INVAL, FF_SCHEDULED);
+
 	if (doclose ||
 	    ffd->taskrun(logbuffer_flush_finish_work, ff, &ff->taskstate))
 		logbuffer_flush_finish_work(NULL, ff);
@@ -2725,6 +2743,8 @@ logbuffer_flush_finish_work(struct worker *wrk, void *priv)
 	phase_mtx = &ff->ffd->phase_mtx;
 	phase_cond = &ff->ffd->phase_cond;
 
+	FF_TRANSITION(ff, FF_SCHEDULED, FF_WAIT_OUTSTANDING);
+
 	assert(ff->inuse == 1);
 	ff->inuse = 2;
 
@@ -2746,6 +2766,8 @@ logbuffer_flush_finish_work(struct worker *wrk, void *priv)
 	(void) n;
 	(void) v;
 
+	FF_TRANSITION(ff, FF_WAIT_OUTSTANDING, FF_HEAD);
+
 	if (ff->head.block) {
 		AN(ff->active_off);
 		AN(ff->head.off);
@@ -2767,8 +2789,12 @@ logbuffer_flush_finish_work(struct worker *wrk, void *priv)
 	FDBG(D_LOG_FLUSH, "ref active %zu / %zu", ff->ffd->log_info.logblk_off,
 	    ff->ffd->log_info.pendblk_off);
 
+	FF_TRANSITION(ff, FF_HEAD, FF_HDR);
+
 	AZ(fellow_io_write_hdr(ff->ffd));
 
+	FF_TRANSITION(ff, FF_HDR, FF_FREE);
+
 	if (ff->regions_to_free) {
 		logbuffer_flush_finish_need_ioctx(ff);
 		regionlist_discard(ff->ffd, ff->fdil.ioctx,
@@ -2798,12 +2824,15 @@ logbuffer_flush_finish_work(struct worker *wrk, void *priv)
 	    (t2 - ff->t[1]) * 100 / dt,
 	    (t3 - t2) * 100 / dt);
 
+	FF_TRANSITION(ff, FF_FREE, FF_DONE);
+
 	if (ff->ffd->phase != FP_OPEN) {
 		ff->inuse = 0;
 		return;
 	}
 
 	AZ(pthread_mutex_lock(phase_mtx));
+	assert(ff->state == FF_DONE);
 	assert(ff->inuse == 2);
 	ff->inuse = 0;
 	AZ(pthread_cond_broadcast(phase_cond));