Add a segmented bitfields implementation

.gitignore

@@ -61,12 +61,7 @@ slashmap.rst
 
 slashmap
 slash
-buddy_test
-buddy_test_witness
-buddy_test_when
-fellow_log_test
-fellow_log_test_ndebug
-fellow_cache_test
+*_test*
 fellow_log_dbg
 
 # coverage

src/Makefile.am

@@ -83,7 +83,8 @@ vmod_slash.c: vcc_slash_if.h
 bin_PROGRAMS = \
 	slashmap \
 	buddy_test \
-	buddy_test_when
+	buddy_test_when \
+	bitf_segmentation_test
 
 slashmap_LDFLAGS = $(VARNISHAPI_LIBS) @CURSES_LIBS@
 slashmap_CFLAGS = $(AM_CFLAGS) @CURSES_CFLAGS@ $(DEV_CFLAGS)
@@ -115,10 +116,17 @@ buddy_test_when_CFLAGS = \
 buddy_test_when_SOURCES = \
 	$(buddy_sources)
 
+bitf_segmentation_test_CFLAGS = \
+	$(AM_CFLAGS) $(DEV_CFLAGS)
+bitf_segmentation_test_SOURCES = \
+	bitf_segmentation.h \
+	bitf_segmentation_test.c
+
 noinst_PROGRAMS = \
 	buddy_test \
 	buddy_test_witness \
-	buddy_test_when
+	buddy_test_when \
+	bitf_segmentation_test
 
 # Test suite
 
@@ -135,7 +143,8 @@ AM_VTC_LOG_FLAGS = \
 TESTS = \
 	buddy_test \
 	buddy_test_witness \
-	buddy_test_when
+	buddy_test_when \
+	bitf_segmentation_test
 
 VTC_TESTS = \
 	vtc/buddy_vcl.vtc \
@@ -478,4 +487,4 @@ dist_man_MANS = \
 # XXX TODO flexelint fellow_io_*
 flint:
 	flexelint $(VARNISHSRC_CFLAGS) -I .. flint.lnt \
-		`ls *.c | grep -vE 'fellow_io_aio|fellow_io_threads|fellow_log_test.c|slashmap.c|aux_'`
+		`ls *.c | grep -vE 'fellow_io_aio|fellow_io_threads|fellow_log_test.c|slashmap.c|aux_|bitf_segmentation_test.c'`

src/bitf_segmentation.h

+/*-
+ * SPDX-License-Identifier: LGPL-2.1-only
+ * Copyright 2023 UPLEX Nils Goroll Systemoptimierung. All rights reserved.
+ * Author: Nils Goroll <nils.goroll@uplex.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA Also add information on how to contact you by
+ * electronic and paper mail.
+ */
+
+#include "bitf.h"
+
+/*
+ * a struct bitf is always contigious, and allocating a long region of memory is
+ * an issue for memory allocators, which usally are quite fragmented.
+ *
+ * this data structure trades some overhead for bitf segmentation, that is, an
+ * overall bitfield comprised of several bitfs.
+ */
+
+struct bitfss {
+	size_t		off;
+	struct bitf	*bitf;
+};
+
+#define bitfs_sz(n) (sizeof(struct bitfs) + n * sizeof(struct bitfss))
+
+struct bitfs {
+	unsigned	magic;
+#define BITFS_MAGIC	0xc18b98bd
+	uint16_t	nbitfss;	// length of flexarray (used)
+	uint16_t	lbitfss;	// space for flexarray
+	size_t		nbits;
+	size_t		nset;		// count of set bits
+	struct bitfss	s[];
+};
+
+#define bitfs_size(n) (sizeof(struct bitfs) + (n + 1) * sizeof(struct bitfss))
+
+static inline struct bitfs *
+bitfs_init(void *p, size_t space)
+{
+	struct bitfs *bitfs;
+
+	AN(p);
+	assert(space > sizeof *bitfs);
+	memset(p, 0, space);
+	bitfs = p;
+	bitfs->magic = BITFS_MAGIC;
+
+	space -= sizeof *bitfs;
+	space /= sizeof *bitfs->s;
+	assert(space > 1);
+	assert(space <= UINT16_MAX);
+
+	// last element is next offset for simplicity of search
+	bitfs->lbitfss = (uint16_t)space - 1;
+
+	return (bitfs);
+}
+
+static inline void
+bitfs_add(struct bitfs *bitfs, struct bitf *bitf)
+{
+	struct bitfss *s;
+
+	CHECK_OBJ_NOTNULL(bitfs, BITFS_MAGIC);
+	CHECK_OBJ_NOTNULL(bitf, BITF_MAGIC);
+
+	assert(bitfs->nbitfss < bitfs->lbitfss);
+	assert(bitfs->nset == 0);
+
+	assert(bitf_nset(bitf) == 0);
+	assert(bitf_nbits(bitf) > 0);
+
+	s = &bitfs->s[bitfs->nbitfss++];
+	assert(s->off == bitfs->nbits);
+	AZ(s->bitf);
+	s->bitf = bitf;
+
+	bitfs->nbits += bitf_nbits(bitf);
+	s[1].off = bitfs->nbits;
+}
+
+static inline uint16_t
+bitfs_idx(const struct bitfs *bitfs, size_t bit)
+{
+	const struct bitfss *s;
+	uint16_t d, lo, hi;
+
+	CHECK_OBJ_NOTNULL(bitfs, BITFS_MAGIC);
+	assert(bit < bitfs->nbits);
+	s = bitfs->s;
+
+	lo = 0;
+	hi = bitfs->nbitfss;
+	assert(bit < s[hi].off);
+
+	while (! (s[lo].off <= bit && s[lo + 1].off > bit)) {
+		d = (hi + lo) / 2;
+		if (s[d].off <= bit)
+			lo = d;
+		else if (s[d].off > bit)
+			hi = d;
+		else
+			assert(0);	//lint !e506 const
+	}
+	return (lo);
+}
+
+static inline struct bitfss *
+bitfs_lookup(struct bitfs *bitfs, size_t bit)
+{
+
+	CHECK_OBJ_NOTNULL(bitfs, BITFS_MAGIC);
+	return (&bitfs->s[bitfs_idx(bitfs, bit)]);
+}
+
+static inline const struct bitfss *
+bitfs_lookupc(const struct bitfs *bitfs, size_t bit)
+{
+
+	CHECK_OBJ_NOTNULL(bitfs, BITFS_MAGIC);
+	return (&bitfs->s[bitfs_idx(bitfs, bit)]);
+}
+
+static inline size_t
+bitfs_nset(const struct bitfs *bitfs)
+{
+
+	CHECK_OBJ_NOTNULL(bitfs, BITFS_MAGIC);
+	return (bitfs->nset);
+}
+
+// number of bit set
+static inline size_t
+bitfs_nbits(const struct bitfs *bitfs)
+{
+	CHECK_OBJ_NOTNULL(bitfs, BITFS_MAGIC);
+	return (bitfs->nbits);
+}
+
+// return if bit was set (1: was zero, 0: was already one)
+static inline unsigned
+bitfs_set(struct bitfs *bitfs, size_t bit)
+{
+	struct bitfss *s;
+	unsigned r;
+
+	s = bitfs_lookup(bitfs, bit);
+	r = bitf_set(s->bitf, bit - s->off);
+	bitfs->nset += r;
+	return (r);
+}
+
+static inline unsigned
+bitfs_clr(struct bitfs *bitfs, size_t bit)
+{
+	struct bitfss *s;
+	unsigned r;
+
+	s = bitfs_lookup(bitfs, bit);
+	r = bitf_clr(s->bitf, bit - s->off);
+	AN(bitfs->nset);
+	bitfs->nset -= r;
+	return (r);
+}
+
+static inline int
+bitfs_get(const struct bitfs *bitfs, size_t bit)
+{
+	const struct bitfss *s;
+
+	s = bitfs_lookupc(bitfs, bit);
+	return(bitf_get(s->bitf, bit - s->off));
+}
+
+/* NO FFS - inefficient ! */

src/bitf_segmentation_test.c

+/*-
+ * SPDX-License-Identifier: LGPL-2.1-only
+ * Copyright 2023 UPLEX Nils Goroll Systemoptimierung. All rights reserved.
+ * Author: Nils Goroll <nils.goroll@uplex.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA Also add information on how to contact you by
+ * electronic and paper mail.
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+
+// varnish-cache
+#include <vdef.h>
+#include <miniobj.h>
+#include <vas.h>
+
+#include "bitf_segmentation.h"
+
+#define NBITF 32
+
+int
+main(void) {
+	struct bitfs *bitfs;
+	struct bitf *bitf[NBITF];
+	size_t o, off[NBITF + 1];
+	char stk[bitfs_size(NBITF)];
+
+	unsigned i, u;
+	size_t sz, n;
+	char *p;
+
+	bitfs = bitfs_init(stk, sizeof stk);
+
+	for (u = 0, o = 0; u < NBITF; u++) {
+		sz = (size_t)(drand48() * 8192);
+		if (sz < bitf_sz(1))
+			sz = bitf_sz(1);
+		p = malloc(sz);
+		AN(p);
+		n = sz_nbits(sz);
+		bitf[u] = bitf_init(p, n, sz);
+		AN(bitf[u]);
+		off[u] = o;
+		o += n;
+
+		bitfs_add(bitfs, bitf[u]);
+		AZ(bitfs_nset(bitfs));
+		assert(bitfs_nbits(bitfs) == o);
+	}
+	off[u] = o;
+
+	// limits
+	for (u = 0; u < NBITF; u++) {
+		assert(off[u + 1] > off[u]);
+		assert(bitfs_lookup(bitfs, off[u])->bitf ==
+		    bitf[u]);
+		assert(bitfs_lookup(bitfs, off[u + 1] - 1)->bitf ==
+		    bitf[u]);
+	}
+
+	// some random bits
+	for (i = 0; i < 10240; i++) {
+		for (u = 0; u < NBITF; u++) {
+			n = off[u] + (size_t)
+			    (drand48() * (off[u + 1] - off[u]));
+			assert(bitfs_lookup(bitfs, n)->bitf ==
+			    bitf[u]);
+		}
+	}
+
+
+	for (u = 0; u < NBITF; u++)
+		free(bitf[u]);
+
+
+	printf("OK\n");
+	return (0);
+}