fixes #1740 Public ID hash API

This includes a manual page documenting the entire set of
functions in one step.  The hash is 64-bit based for now, to
be maximally flexible.  An internal 32-bit convenience for the
common internal use is also provided (not public).

The public API includes a test suite.

cmake/NNGOptions.cmake

@@ -141,3 +141,8 @@ if (NNG_TRANSPORT_WS OR NNG_TRANSPORT_WSS)
     set(NNG_SUPP_BASE64 ON)
     set(NNG_SUPP_SHA1 ON)
 endif()
+
+# ID hash API is small wrapper around core, probably should always be enabled unless memory
+# is extraordinarily constrained.
+option(NNG_SUPP_IDHASH "Enable application IDHASH API" ON)
+mark_as_advanced(NNG_SUPP_IDHASH)

docs/man/CMakeLists.txt

@@ -295,6 +295,7 @@ if (NNG_ENABLE_DOC)
             nng_cv_wait
             nng_cv_wake
             nng_cv_wake1
+            nng_id_map
             nng_msleep
             nng_mtx_alloc
             nng_mtx_free

docs/man/libnng.3.adoc

@@ -298,6 +298,7 @@ as a convenience to aid in creating portable applications.
 |xref:nng_cv_wait.3supp.adoc[nng_cv_wait()]|wait for condition
 |xref:nng_cv_wake.3supp.adoc[nng_cv_wake()]|wake all waiters
 |xref:nng_cv_wake1.3supp.adoc[nng_cv_wake1()]|wake one waiter
+|xref:nng_id_map.3supp.adoc[nng_id_map]|identifier based mapping table
 |xref:nng_msleep.3supp.adoc[nng_msleep()]|sleep for milliseconds
 |xref:nng_mtx_alloc.3supp.adoc[nng_mtx_alloc()]|allocate mutex
 |xref:nng_mtx_free.3supp.adoc[nng_mtx_free()]|free mutex

docs/man/nng_id_map.3supp.adoc

@@ -0,0 +1,100 @@
+= nng_id_map(3supp)
+//
+// Copyright 2023 Staysail Systems, Inc. <[email protected]>
+//
+// This document is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+== NAME
+
+nng_id_map - identifier based mapping table
+
+== SYNOPSIS
+
+[source, c]
+----
+#include <nng/nng.h>
+#include <nng/supplemental/idhash/idhash.h>
+
+typedef struct nng_id_map_s nng_id_map;
+
+#define NNG_MAP_RANDOM 1
+
+int   nng_id_map_alloc(nng_id_map **map_p, uint64_t lo, uint64_t hi, int flags);
+void  nng_id_map_free(nng_id_map *map);
+void *nng_id_get(nng_id_map *map, uint64_t id);
+int   nng_id_set(nng_id_map *map, uint64_t, void *value);
+int   nng_id_alloc(nng_id_map *map, uint64_t *id_p, void *value);
+int   nng_id_remove(nng_id_map *map, uint64_t id);
+
+----
+
+== DESCRIPTION
+
+These functions provide support for managing tables of data based on
+identifiers, ensuring that identifiers are allocated uniquely and within
+specified range limits.
+
+The table stores data pointers (which must not be `NULL`) at a logical numeric index.
+It does so efficiently, even if large gaps exist, and it provides a means to efficiently
+allocate a numeric identifier from a pool of unused identifiers.
+
+Identifiers are allocated in increasing order, without reusing old identifiers until the
+largest possible identifier is allocated.  After wrapping, only identifiers that are no longer
+in use will be considered.
+No effort to order the availability of identifiers based on when they were freed is made.
+
+An initial table is allocated with `nng_id_map_alloc()`, which takes the lowest legal identifier in _lo_,
+and the largest legal identifier in _hi_.
+The new table is returned in _map_p_, and should be used as the _map_ argument to the rest of these functions.
+
+****
+As a special convenience, if these are specified as zero, then a full range of 32-bit identifiers is assumed.
+If identifiers larger than or equal to 2^32^ are required, then both _lo_ and _hi_ must be specified with the
+exact values desired.
+****
+
+The _flags_ argument is a bit mask of flags for the table.
+If `NNG_MAP_RANDOM` is specified, then the starting point for allocations is randomized, but subsequent allocations will then be monotonically increasing.
+This is useful to reduce the odds of different instances of an application using the same identifiers at the same time.
+
+The `nng_id_get()` function returns the value previously stored with the given identifier.
+If no value is currently associated with the identifer, it returns `NULL`.
+
+The `nng_id_set()` function sets the value with the associated identifier.
+This can be used to replace a previously allocated identifier.
+If the identifier was not previously allocated, then it is allocated as part of the call.
+This function does not necessarily honor the identifier range limits set for the map when it was allocated.
+
+The `nng_id_alloc()` function allocates a new identifier from the range for the map, and associates it with
+the supplied _value_.
+
+The `nng_id_remove()` function removes the identifier and its associated value from the table.
+
+NOTE: These functions are limited to storing at most 2^32^ identifiers, even though the identifers may
+themselves be larger than 2^32^.
+
+IMPORTANT: These functions are *not* thread-safe.
+Callers should use a xref:nng_mtx_lock.3supp[mutex] or similar approach when thread-safety is needed.
+
+== RETURN VALUES
+
+The `nng_id_map_alloc()`, `nng_id_set()`, `nng_id_alloc()`, and `nng_id_remove()` functions
+return 0 on success, or -1 on failure.
+
+The `nng_id_map_get()` function returns the requested data pointer, or `NULL` if the identifier was not found.
+
+== ERRORS
+
+[horizontal]
+`NNG_ENOENT`:: The _id_ does not exist in the table.
+`NNG_ENOMEM`:: Insufficient memory is available, or the table is full.
+
+== SEE ALSO
+
+[.text-left]
+xref:nng_mtx_lock.3supp.adoc[nng(7)]
+xref:nng.7.adoc[nng(7)]

src/core/dialer.c

@@ -259,7 +259,7 @@ nni_dialer_create(nni_dialer **dp, nni_sock *s, const char *url_str)
 	nni_aio_init(&d->d_tmo_aio, dialer_timer_cb, d);
 
 	nni_mtx_lock(&dialers_lk);
-	rv = nni_id_alloc(&dialers, &d->d_id, d);
+	rv = nni_id_alloc32(&dialers, &d->d_id, d);
 	nni_mtx_unlock(&dialers_lk);
 
 #ifdef NNG_ENABLE_STATS

src/core/id_test.c

@@ -1,5 +1,5 @@
 //
-// Copyright 2021 Staysail Systems, Inc. <[email protected]>
+// Copyright 2023 Staysail Systems, Inc. <[email protected]>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -39,7 +39,7 @@ void
 test_random(void)
 {
 	int      i;
-	uint32_t id;
+	uint64_t id;
 	for (i = 0; i < 2; i++) {
 		nni_id_map m;
 		nni_id_map_init(&m, 0, 0, true);
@@ -94,7 +94,7 @@ void
 test_not_found(void)
 {
 	nni_id_map m;
-	uint32_t   id;
+	uint64_t   id;
 	nni_id_map_init(&m, 0, 0, false);
 
 	NUTS_PASS(nni_id_alloc(&m, &id, &id));
@@ -137,7 +137,7 @@ test_dynamic(void)
 {
 	nni_id_map m;
 	int        expect[5];
-	uint32_t   id;
+	uint64_t   id;
 
 	nni_id_map_init(&m, 10, 13, false);
 
@@ -175,7 +175,7 @@ test_set_out_of_range(void)
 	// We can insert outside the range forcibly.
 	NUTS_PASS(nni_id_set(&m, 1, &x));
 	NUTS_PASS(nni_id_set(&m, 100, &x));
-	NUTS_PASS(nni_id_alloc(&m, &id, &x));
+	NUTS_PASS(nni_id_alloc32(&m, &id, &x));
 	NUTS_TRUE(id == 10);
 	nni_id_map_fini(&m);
 }

src/core/idhash.c

@@ -1,5 +1,5 @@
 //
-// Copyright 2021 Staysail Systems, Inc. <[email protected]>
+// Copyright 2023 Staysail Systems, Inc. <[email protected]>
 // Copyright 2018 Capitar IT Group BV <[email protected]>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -13,7 +13,7 @@
 #include <string.h>
 
 struct nni_id_entry {
-	uint32_t key;
+	uint64_t key;
 	uint32_t skips;
 	void    *val;
 };
@@ -24,7 +24,7 @@ static nni_id_map **id_reg_map = NULL;
 static nni_mtx      id_reg_mtx = NNI_MTX_INITIALIZER;
 
 void
-nni_id_map_init(nni_id_map *m, uint32_t lo, uint32_t hi, bool randomize)
+nni_id_map_init(nni_id_map *m, uint64_t lo, uint64_t hi, bool randomize)
 {
 	if (lo == 0) {
 		lo = 1;
@@ -68,7 +68,7 @@ nni_id_map_fini(nni_id_map *m)
 #define ID_INDEX(m, j) ((j) & (m->id_cap - 1))
 
 static size_t
-id_find(nni_id_map *m, uint32_t id)
+id_find(nni_id_map *m, uint64_t id)
 {
 	size_t index;
 	size_t start;
@@ -98,7 +98,7 @@ id_find(nni_id_map *m, uint32_t id)
 }
 
 void *
-nni_id_get(nni_id_map *m, uint32_t id)
+nni_id_get(nni_id_map *m, uint64_t id)
 {
 	size_t index;
 	if ((index = id_find(m, id)) == (size_t) -1) {
@@ -130,7 +130,8 @@ id_map_register(nni_id_map *m)
 		}
 		id_reg_len = len;
 		if (id_reg_map != NULL)
-			memcpy(mr, id_reg_map, id_reg_num * sizeof(nni_id_map *));
+			memcpy(
+			    mr, id_reg_map, id_reg_num * sizeof(nni_id_map *));
 		id_reg_map = mr;
 	}
 	id_reg_map[id_reg_num++] = m;
@@ -233,7 +234,7 @@ id_resize(nni_id_map *m)
 }
 
 int
-nni_id_remove(nni_id_map *m, uint32_t id)
+nni_id_remove(nni_id_map *m, uint64_t id)
 {
 	size_t index;
 	size_t probe;
@@ -251,7 +252,7 @@ nni_id_remove(nni_id_map *m, uint32_t id)
 		nni_id_entry *entry;
 
 		// The load was increased once each hashing operation we used
-		// to place the the item.  Decrement it accordingly.
+		// to place the item.  Decrement it accordingly.
 		m->id_load--;
 		entry = &m->id_entries[probe];
 		if (probe == index) {
@@ -273,7 +274,7 @@ nni_id_remove(nni_id_map *m, uint32_t id)
 }
 
 int
-nni_id_set(nni_id_map *m, uint32_t id, void *val)
+nni_id_set(nni_id_map *m, uint64_t id, void *val)
 {
 	size_t        index;
 	nni_id_entry *ent;
@@ -314,9 +315,9 @@ nni_id_set(nni_id_map *m, uint32_t id, void *val)
 }
 
 int
-nni_id_alloc(nni_id_map *m, uint32_t *idp, void *val)
+nni_id_alloc(nni_id_map *m, uint64_t *idp, void *val)
 {
-	uint32_t id;
+	uint64_t id;
 	int      rv;
 
 	NNI_ASSERT(val != NULL);
@@ -355,3 +356,14 @@ nni_id_alloc(nni_id_map *m, uint32_t *idp, void *val)
 	}
 	return (rv);
 }
+
+int
+nni_id_alloc32(nni_id_map *m, uint32_t *idp, void *val)
+{
+	uint64_t id;
+	int      rv;
+	rv = nni_id_alloc(m, &id, val);
+	NNI_ASSERT(id < (1ULL << 32));
+	*idp = (uint32_t) id;
+	return (rv);
+}

src/core/idhash.h

@@ -1,5 +1,5 @@
 //
-// Copyright 2021 Staysail Systems, Inc. <[email protected]>
+// Copyright 2023 Staysail Systems, Inc. <[email protected]>
 // Copyright 2018 Capitar IT Group BV <[email protected]>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -28,29 +28,31 @@ typedef struct nni_id_entry nni_id_entry;
 
 // NB: These details are entirely private to the hash implementation.
 // They are provided here to facilitate inlining in structures.
+// We can support at most 2^32 ~ 4 billion ~ entries.
 struct nni_id_map {
+	uint32_t      id_flags;
 	uint32_t      id_cap;
 	uint32_t      id_count;
 	uint32_t      id_load;
 	uint32_t      id_min_load; // considers placeholders
 	uint32_t      id_max_load;
-	uint32_t      id_min_val;
-	uint32_t      id_max_val;
-	uint32_t      id_dyn_val;
-	uint32_t      id_flags;
+	uint64_t      id_min_val;
+	uint64_t      id_max_val;
+	uint64_t      id_dyn_val;
 	nni_id_entry *id_entries;
 };
 
 #define NNI_ID_FLAG_STATIC 1   // allocated statically
 #define NNI_ID_FLAG_RANDOM 2   // start at a random value
 #define NNI_ID_FLAG_REGISTER 4 // map is registered for finalization
 
-extern void  nni_id_map_init(nni_id_map *, uint32_t, uint32_t, bool);
+extern void  nni_id_map_init(nni_id_map *, uint64_t, uint64_t, bool);
 extern void  nni_id_map_fini(nni_id_map *);
-extern void *nni_id_get(nni_id_map *, uint32_t);
-extern int   nni_id_set(nni_id_map *, uint32_t, void *);
-extern int   nni_id_alloc(nni_id_map *, uint32_t *, void *);
-extern int   nni_id_remove(nni_id_map *, uint32_t);
+extern void *nni_id_get(nni_id_map *, uint64_t);
+extern int   nni_id_set(nni_id_map *, uint64_t, void *);
+extern int   nni_id_alloc(nni_id_map *, uint64_t *, void *);
+extern int   nni_id_alloc32(nni_id_map *, uint32_t *, void *);
+extern int   nni_id_remove(nni_id_map *, uint64_t);
 extern void  nni_id_map_sys_fini(void);
 
 #define NNI_ID_MAP_INITIALIZER(min, max, flags)            \

src/core/listener.c

@@ -243,7 +243,7 @@ nni_listener_create(nni_listener **lp, nni_sock *s, const char *url_str)
 	nni_aio_init(&l->l_tmo_aio, listener_timer_cb, l);
 
 	nni_mtx_lock(&listeners_lk);
-	rv = nni_id_alloc(&listeners, &l->l_id, l);
+	rv = nni_id_alloc32(&listeners, &l->l_id, l);
 	nni_mtx_unlock(&listeners_lk);
 
 #ifdef NNG_ENABLE_STATS

src/core/pipe.c

@@ -259,7 +259,7 @@ pipe_create(nni_pipe **pp, nni_sock *sock, nni_sp_tran *tran, void *tran_data)
 	nni_cv_init(&p->p_cv, &pipes_lk);
 
 	nni_mtx_lock(&pipes_lk);
-	rv = nni_id_alloc(&pipes, &p->p_id, p);
+	rv = nni_id_alloc32(&pipes, &p->p_id, p);
 	nni_mtx_unlock(&pipes_lk);
 
 #ifdef NNG_ENABLE_STATS

src/core/socket.c

@@ -641,7 +641,7 @@ nni_sock_open(nni_sock **sockp, const nni_proto *proto)
 	}
 
 	nni_mtx_lock(&sock_lk);
-	if ((rv = nni_id_alloc(&sock_ids, &s->s_id, s)) != 0) {
+	if ((rv = nni_id_alloc32(&sock_ids, &s->s_id, s)) != 0) {
 		nni_mtx_unlock(&sock_lk);
 		sock_destroy(s);
 		return (rv);
@@ -1343,7 +1343,7 @@ nni_ctx_open(nni_ctx **ctxp, nni_sock *sock)
 		nni_free(ctx, ctx->c_size);
 		return (NNG_ECLOSED);
 	}
-	if ((rv = nni_id_alloc(&ctx_ids, &ctx->c_id, ctx)) != 0) {
+	if ((rv = nni_id_alloc32(&ctx_ids, &ctx->c_id, ctx)) != 0) {
 		nni_mtx_unlock(&sock_lk);
 		nni_free(ctx, ctx->c_size);
 		return (rv);

src/sp/protocol/reqrep0/req.c

@@ -717,7 +717,7 @@ req0_ctx_send(void *arg, nni_aio *aio)
 	req0_ctx_reset(ctx);
 
 	// Insert us on the per ID hash list, so that receives can find us.
-	if ((rv = nni_id_alloc(&s->requests, &ctx->request_id, ctx)) != 0) {
+	if ((rv = nni_id_alloc32(&s->requests, &ctx->request_id, ctx)) != 0) {
 		nni_mtx_unlock(&s->mtx);
 		nni_aio_finish_error(aio, rv);
 		return;

src/sp/protocol/survey0/survey.c

@@ -220,7 +220,7 @@ surv0_ctx_send(void *arg, nni_aio *aio)
 	surv0_ctx_abort(ctx, NNG_ECANCELED);
 
 	// Allocate the new ID.
-	if ((rv = nni_id_alloc(&sock->surveys, &ctx->survey_id, ctx)) != 0) {
+	if ((rv = nni_id_alloc32(&sock->surveys, &ctx->survey_id, ctx)) != 0) {
 		nni_mtx_unlock(&sock->mtx);
 		nni_aio_finish_error(aio, rv);
 		return;

src/supplemental/CMakeLists.txt

@@ -11,6 +11,7 @@ nng_directory(supplemental)
 
 add_subdirectory(base64)
 add_subdirectory(http)
+add_subdirectory(idhash)
 add_subdirectory(sha1)
 add_subdirectory(tls)
 add_subdirectory(util)