Allow gaining an fcore reference without the fcode mutex

in tus_file_final_concat(), we needed to get an fcore reference while holding
the srv lock for lookup.

Thus the srv lock would be kept until the fcore could get locked.

To avoid this situation, we add the option to also gain refcounts under the
server mutex.

src/tus_file.c

@@ -54,7 +54,7 @@ VSPLAY_GENERATE(tus_files, tus_file_core, entry, tus_file_cmp)
 #define TUS_FILE_SUFF "_XXXXXX"
 
 static uintmax_t header_size;
-static unsigned tus_file_ref(struct tus_file_core *);
+static unsigned tus_file_srvref(struct tus_file_core *);
 static unsigned tus_file_unref(struct tus_file_core *);
 
 
@@ -140,7 +140,7 @@ tus_file_final_concat(struct VPFX(tus_server) *srv,
 		if (pdisk == NULL ||
 		    pdisk->type != TUS_PARTIAL)
 			goto lookup_err;
-		(void) tus_file_ref(part);
+		(void) tus_file_srvref(part);
 		parts[n++] = part;
 		if (n == MAX_CONCAT)
 			goto lookup_err;
@@ -388,7 +388,7 @@ tus_file_core_new(struct VPFX(tus_server) *srv,
 	AZ(pthread_mutex_init(&fcore->mtx, NULL));
 	AZ(pthread_cond_init(&fcore->cond, NULL));
 	fcore->disk = fdisk;
-	fcore->refcnt = 1;
+	fcore->srvref = 1;
 
 	if (VSPLAY_INSERT(tus_files, tus_server_files(srv), fcore) == NULL) {
 		tus_exp_insert(fcore);
@@ -514,6 +514,16 @@ tus_file_unlock(struct tus_file_core **fcorep)
 	AZ(pthread_mutex_unlock(&fcore->mtx));
 }
 
+/* under borh fcore and server mtx */
+static inline void
+del_ref_transfer(struct tus_file_core *fcore)
+{
+	assert(fcore->srvref > 0);
+	fcore->ref += fcore->srvref;
+	fcore->srvref = 0;
+	assert(fcore->ref > 0);
+}
+
 /* caller holds tus_server mtx */
 static void
 tus_file_del_locked(struct tus_file_core **fcorep)
@@ -525,6 +535,7 @@ tus_file_del_locked(struct tus_file_core **fcorep)
 
 	rm = VSPLAY_REMOVE(tus_files, tus_server_files(fcore->server),
 	    fcore);
+	del_ref_transfer(fcore);
 
 	assert (rm == fcore);
 	tus_exp_delete(fcore);
@@ -545,6 +556,7 @@ tus_file_del(struct tus_file_core **fcorep)
 
 	tus_server_lock(srv);
 	rm = VSPLAY_REMOVE(tus_files, tus_server_files(srv), fcore);
+	del_ref_transfer(fcore);
 	tus_server_unlock(srv);
 
 	assert (rm == fcore);
@@ -606,27 +618,44 @@ tus_file_rename_base(struct tus_file_core *fcore, const char *basename)
 }
 #endif
 
+/* under server mutex */
+static unsigned
+tus_file_srvref(struct tus_file_core *fcore)
+{
+	int r;
+
+	CHECK_OBJ_NOTNULL(fcore, VMOD_TUS_FILE_CORE_MAGIC);
+	r = fcore->srvref++;
+	// as long as the object can be found, it has one srvref at least
+	assert(r > 0);
+	return (r);
+}
+
+#ifdef UNUSED
 static unsigned
 tus_file_ref(struct tus_file_core *fcore)
 {
-	unsigned r;
+	int r;
 
 	CHECK_OBJ_NOTNULL(fcore, VMOD_TUS_FILE_CORE_MAGIC);
 	AZ(pthread_mutex_lock(&fcore->mtx));
-	r = fcore->refcnt++;
+	r = fcore->ref++ + fcore->srvref;
 	AZ(pthread_mutex_unlock(&fcore->mtx));
+	assert(r >= 0);
 	return (r);
 }
+#endif
 
 static unsigned
 tus_file_unref_locked(struct tus_file_core *fcore)
 {
-	unsigned r;
+	int r;
 
 	CHECK_OBJ_NOTNULL(fcore, VMOD_TUS_FILE_CORE_MAGIC);
 
-	r = --fcore->refcnt;
+	r = --fcore->ref + fcore->srvref;
 	AZ(pthread_mutex_unlock(&fcore->mtx));
+	assert (r >= 0);
 	if (r == 0)
 		tus_file_fini(fcore);
 	return (r);

src/tus_file.h

@@ -77,9 +77,24 @@ struct tus_file_core {
 	pthread_mutex_t		mtx;
 	pthread_cond_t			cond;
 	struct tus_file_disk		*disk;
-	unsigned			refcnt;
 	unsigned			exp_idx;
 
+	/* srvrefs can be gained under the server mutex
+	 * during lookup
+	 *
+	 * one srvref is owned by exp as long as the file
+	 * is on the index (can be found)
+	 *
+	 * once the file gets deleted / expired, srvref
+	 * gets transferred to ref.
+	 */
+	unsigned			srvref;
+	/* additional references, e.g. by concats
+	 * because concats can also take srvrefs, but
+	 * return refs, ref can become negative.
+	 */
+	int				ref;
+
 	/* final mmap or concats list */
 	void				*ptr;
 	size_t				len;