Refactor LRU as a separate object

src/fellow_cache.h

@@ -50,7 +50,7 @@ struct fellow_cache_res {
 struct fellow_cache *
 fellow_cache_init(struct fellow_fd *, buddy_t *, struct stvfe_tune *,
     fellow_task_run_t);
-void fellow_cache_fini(struct fellow_cache **, const pthread_t *);
+void fellow_cache_fini(struct fellow_cache **);
 
 void fellow_cache_obj_delete(struct fellow_cache *fc,
     struct fellow_cache_obj *fco, const uint8_t hash[32]);
@@ -65,8 +65,7 @@ fellow_cache_obj_iter(struct fellow_cache *fc, struct fellow_cache_obj *fco,
 void *fellow_cache_obj_getattr(struct fellow_cache *fc,
     struct fellow_cache_obj *fco,
     enum obj_attr attr, size_t *len);
-int fellow_cache_obj_lru_touch(struct fellow_cache *fc,
-    struct fellow_cache_obj *fco);
+int fellow_cache_obj_lru_touch(struct fellow_cache_obj *fco);
 
 struct fellow_cache_res
 fellow_busy_obj_alloc(struct fellow_cache *fc,

src/fellow_cache_storage.h

@@ -20,14 +20,16 @@
  * electronic and paper mail.
  */
 
-void fellow_cache_obj_evict_mutate(struct fellow_cache *fc,
+struct fellow_cache_lru;
+void fellow_cache_obj_evict_mutate(struct fellow_cache_lru *lru,
     struct fellow_cache_obj *fco);
-void fellow_cache_obj_slim(struct fellow_cache *fc,
+void fellow_cache_obj_slim(const struct fellow_cache *fc,
     struct fellow_cache_obj *fco);
 
 void stvfe_oc_log_removed(struct objcore *oc);
 void stvfe_oc_log_submitted(struct objcore *oc);
 void stvfe_oc_dle_obj(struct objcore *oc, struct fellow_dle *e);
 
-int stvfe_mutate(struct worker *wrk, struct objcore *oc);
+int stvfe_mutate(struct worker *wrk, struct fellow_cache_lru *lru,
+    struct objcore *oc);
 void stvfe_sumstat(struct worker *wrk); // wraps Pool_Sumstat(wrk);

src/fellow_storage.c

@@ -296,7 +296,6 @@ struct stvfe {
 	buddy_t		my_membuddy;
 	buddy_t		*membuddy;
 
-	pthread_t			mem_lru_thread;
 	pthread_t			dsk_lru_thread;
 
 	uintptr_t			oev;
@@ -1034,7 +1033,7 @@ sfemem_getattr(struct worker *wrk, struct objcore *memoc, enum obj_attr attr,
 		*slen = (ssize_t)len;
 
 	if (attr == OA_VARY)
-		(void) fellow_cache_obj_lru_touch(stvfe->fc, fco);
+		(void) fellow_cache_obj_lru_touch(fco);
 
 	return (r);
 }
@@ -1109,7 +1108,7 @@ sfemem_touch(struct worker *wrk, struct objcore *memoc, vtim_real now)
 	 * last_lru. Does it matter?
 	 */
 
-	(void) fellow_cache_obj_lru_touch(stvfe->fc, fco);
+	(void) fellow_cache_obj_lru_touch(fco);
 	LRU_Touch(wrk, memoc, now);
 }
 
@@ -1781,7 +1780,8 @@ stvfe_sumstat(struct worker *wrk)
  * fco->mtx held!
  */
 int
-stvfe_mutate(struct worker *wrk, struct objcore *oc)
+stvfe_mutate(struct worker *wrk, struct fellow_cache_lru *lru,
+    struct objcore *oc)
 {
 	const struct stevedore *stv;
 	const struct stvfe *stvfe;
@@ -1809,8 +1809,8 @@ stvfe_mutate(struct worker *wrk, struct objcore *oc)
 		AN(oc->stobj->priv);
 		AN(oc->stobj->priv2);
 
-		fellow_cache_obj_evict_mutate(stvfe->fc,
-		    oc->stobj->priv);
+		// XXX LRU with change, reduce LRU mtx scope
+		fellow_cache_obj_evict_mutate(lru, oc->stobj->priv);
 
 		oc->stobj->priv = NULL;
 		oc->stobj->stevedore = stvfe->dskstv;
@@ -2327,6 +2327,7 @@ sfe_open_scope(struct stevedore *stv)
 	struct sfe_resurrect_priv sfer[1];
 	struct sfe_open_worker_priv sow[1];
 	const char *err = NULL;
+	pthread_t mem_lru_thread;
 
 	ASSERT_CLI();
 	CAST_OBJ_NOTNULL(stvfe, stv->priv, STVFE_MAGIC);
@@ -2412,8 +2413,9 @@ sfe_open_scope(struct stevedore *stv)
 		goto err;
 	}
 
-	WRK_BgThread(&stvfe->mem_lru_thread, "sfe-mem-lru",
+	WRK_BgThread(&mem_lru_thread, "sfe-mem-lru",
 	    fellow_cache_lru_thread, stvfe->fc);
+	AN(mem_lru_thread);
 	WRK_BgThread(&stvfe->dsk_lru_thread, "sfe-dsk-lru",
 	    sfedsk_lru_thread, stvfe->dskstv);
 
@@ -2430,7 +2432,7 @@ sfe_open_scope(struct stevedore *stv)
 	}
 
 	if (fc != NULL)
-		fellow_cache_fini(&fc, NULL);
+		fellow_cache_fini(&fc);
 	AZ(fc);
 
 	if (ffd != NULL)
@@ -2530,7 +2532,7 @@ sfe_close_real(struct stvfe *stvfe)
 	sfe_ban_space_return(stvfe);
 	buddy_alloc_async_done(&stvfe->ban_reqs->reqs);
 
-	fellow_cache_fini(&stvfe->fc, &stvfe->mem_lru_thread);
+	fellow_cache_fini(&stvfe->fc);
 	AZ(stvfe->fc);
 
 	fellow_log_close(&stvfe->ffd);