Add xlookup.c, and test prog

Makefile

@@ -65,7 +65,7 @@ distclean: clean
 # The nitty-gritty stuff below
 # ----------------------------------------------------------------------------
 
-SOURCES = $(SRC)/xchange.c $(SRC)/xstruct.c $(SRC)/xjson.c
+SOURCES = $(SRC)/xchange.c $(SRC)/xstruct.c $(SRC)/xlookup.c $(SRC)/xjson.c
 
 # Generate a list of object (obj/*.o) files from the input sources
 OBJECTS := $(subst $(SRC),$(OBJ),$(SOURCES))

README.md

@@ -279,6 +279,43 @@ You can also remove existing fields from structures using `xRemoveField()`, e.g.
   xDestroyField(xRemoveField(s, "blah"));
 ```
 
+#### Large structures
+
+The normal `xGetField()` and `xGetSubstruct()` functions have computational costs that scale linearly with the number 
+of direct fields in the structure. It is not much of an issue for structures that contain dozens of fields (per layer). 
+For much larger structures, which have a fixed layout, there is an option for significantly faster hash-based lookup
+also. E.g. instead of `xGetField()` you may use `xLookupField()`:
+
+```c
+  XStructure *s = ...
+
+  // Let's create a lookup table for all fields of 's' and all its substructures.
+  XLookupTable *l = xCreateLookupTable(s, TRUE);
+
+  XField *f = xLookupField(l, "subsystem:property");
+  ...
+
+  // Once done with the lookup, destroy it.
+  xDestroyLookup(l);
+```
+
+#### Iterating over elements
+
+You can easily iterate over the elements also. This is one application where you may want to know the internal layout
+of `XStructure`, namely that it contains a simple linked-list of `XField` fields. One way to iterate over a strucures
+elements is with a `for` loop, e.g.:
+
+```c
+  XStructure *s = ...
+  XField *f;
+  
+  for (f = s->firstField; f != NULL; f = f->next) {
+    // Process each field 'f' here...
+    ...
+  }
+```
+
+
 -----------------------------------------------------------------------------
 
 <a name="json-interchange"></a>

include/xchange.h

@@ -157,6 +157,19 @@ typedef struct XStructure {
   struct XStructure *parent;    ///< Reference to parent structure (if any)
 } XStructure;
 
+/**
+ * A fast lookup table for fields in large XStructures.
+ *
+ * @sa xCreateLookup()
+ * @sa xLookupField()
+ * @sa xDestroyLookup()
+ */
+typedef struct {
+  void *priv;                   ///< Private data, not exposed to users
+} XLookupTable;
+
+
+
 /**
  * Static initializer for an XStructure data structure.
  */
@@ -189,11 +202,17 @@ XField *xRemoveField(XStructure *s, const char *name);
 boolean xIsFieldValid(const XField *f);
 int xGetFieldCount(const XField *f);
 int xCountFields(const XStructure *s);
+long xDeepCountFields(const XStructure *s);
 XStructure *xGetSubstruct(const XStructure *s, const char *id);
 XField *xSetSubstruct(XStructure *s, const char *name, XStructure *substruct);
 int xReduceDims(int *ndim, int *sizes);
 int xReduceAllDims(XStructure *s);
 
+// Fast field lookup
+XLookupTable *xCreateLookup(const XStructure *s, boolean recursive);
+XField *xLookupField(const XLookupTable *tab, const char *id);
+void xDestroyLookup(XLookupTable *tab);
+
 // Convenience field creator methods
 XField *xCreateDoubleField(const char *name, double value);
 XField *xCreateIntField(const char *name, int value);

src/xlookup.c

@@ -0,0 +1,221 @@
+/**
+ * @file
+ *
+ * @date Created  on Aug 30, 2024
+ * @author Attila Kovacs
+ *
+ *   Lookup table for faster field access in large fixed-layout structures.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "xchange.h"
+
+/// \cond PRIVATE
+
+typedef struct LookupEntry {
+  long hash;
+  char *key;                ///< dynamically allocated.
+  XField *value;
+  struct LookupEntry *next;
+} XLookupEntry;
+
+typedef struct {
+  XLookupEntry **table;
+  int nBins;
+  int nEntries;
+} XLookupPrivate;
+
+/// \endcond
+
+static long xGetHash(const char *str) {
+  int i;
+  long hash = 0;
+  if(!str) return 0;
+  for(i=0; str[i]; i++) hash += str[i] ^ i;
+  return hash;
+}
+
+
+static XLookupEntry *xGetLookupEntry(const XLookupTable *tab, const char *key, long hash) {
+  const XLookupPrivate *p;
+  XLookupEntry *e;
+
+  p = (XLookupPrivate *) tab->priv;
+
+  for(e = p->table[(int) (hash % p->nBins)]; e != NULL; e=e->next) if(strcmp(e->key, key) == 0) return e;
+
+  return NULL;
+}
+
+static int xLookupAddAsync(XLookupTable *tab, const char *prefix, const XField *field, void **oldValue) {
+  XLookupPrivate *p = (XLookupPrivate *) tab->priv;
+  const char *id = prefix ? xGetAggregateID(prefix, field->name) : strdup(field->name);
+  long hash = xGetHash(id);
+  XLookupEntry *e = xGetLookupEntry(tab, id, hash);
+  int idx;
+
+  if(e) {
+    if(oldValue) *oldValue = e->value;
+    e->value = (XField *) field;
+    return 1;
+  }
+
+  e = (XLookupEntry *) calloc(1, sizeof(XLookupEntry));
+  e->hash = hash;
+  e->key = (char *) id;
+  e->value = (XField *) field;
+
+  idx = hash % p->nBins;
+
+  e->next = p->table[idx];
+  p->table[idx] = e;
+  p->nEntries++;
+
+  return 0;
+}
+
+static void xLookupAddAllAsync(XLookupTable *tab, const char *prefix, const XStructure *s, boolean recursive) {
+  XField *f;
+  int lp = prefix ? strlen(prefix) : 0;
+
+  for(f = s->firstField; f != NULL; f = f->next) {
+    xLookupAddAsync(tab, prefix, f, NULL);
+
+    if(f->type == X_STRUCT && recursive) {
+      XStructure *sub = (XStructure *) f->value;
+      char *p1 = (char *) malloc(lp + strlen(f->value) + 2 * sizeof(X_SEP) + 12);
+
+      int count = xGetFieldCount(f);
+      while(--count >= 0) {
+        int n = 0;
+        if(prefix) n = sprintf(p1, "%s" X_SEP, prefix);
+
+        if(f->ndim == 0) sprintf(&p1[n], "%s", f->name);
+        else sprintf(&p1[n], "%s" X_SEP "%d", f->name, (count + 1));
+
+        xLookupAddAllAsync(tab, p1, &sub[count], TRUE);
+      }
+    }
+  }
+}
+
+static XLookupTable *xAllocLookup(int size) {
+  XLookupTable *tab;
+  XLookupPrivate *p;
+
+  unsigned int n = 1;
+  while(n < size) n <<= 1;
+
+  p = (XLookupPrivate *) calloc(1, sizeof(XLookupPrivate));
+  p->table = (XLookupEntry **) calloc(n, sizeof(XLookupEntry *));
+  if(!p->table) {
+    perror("ERROR! xAllocLookup");
+    free(p);
+    return NULL;
+  }
+
+  p->nBins = p->table ? n : 0;
+
+  tab = (XLookupTable *) calloc(1, sizeof(XLookupTable));
+  tab->priv = p;
+
+
+  return tab;
+}
+
+/**
+ * Creates a fast name lookup table for the fields of structure, with or without including fields of embedded
+ * substructures also. For structures with a large number of fields, such a lookup can significantly
+ * improve access times for retrieving specific fields from a structure. However, the lookup will not
+ * track fields added or removed after its creation, and so it is suited for accessing structures with a
+ * fixed layout only.
+ *
+ * Since the lookup table contains references to the structure, you should not destroy the structure as long
+ * as the lookup table is used.
+ *
+ * Once the lookup table is no longer used, the caller should explicitly destroy it with `xDestroyLookup()`
+ *
+ * @param s           Pointer to a structure, for which to create a field lookup.
+ * @param recursive   Whether to include fields from substructures recursively in the lookup.
+ * @return            The lookup table, or NULL if there was an error (errno will inform about the type of
+ *                    error).
+ *
+ * @sa xLookupField()
+ * @sa xDestroyLookup()
+ */
+XLookupTable *xCreateLookup(const XStructure *s, boolean recursive) {
+  XLookupTable *l;
+
+  if(s == NULL) {
+    errno = EINVAL;
+    return NULL;
+  }
+
+  l = xAllocLookup(recursive ? xDeepCountFields(s) : xCountFields(s));
+  if(!l) return NULL;
+
+  xLookupAddAllAsync(l, NULL, s, recursive);
+
+  return l;
+}
+
+/**
+ * Returns a named field from a lookup table.
+ *
+ * @param tab       Pointer to the lookup table
+ * @param id        The aggregate ID of the field to find.
+ * @return          The corresponding field or NULL if no such field exists or tab was NULL.
+ *
+ * @sa xCreateLookup()
+ * @sa xGetField()
+ */
+XField *xLookupField(const XLookupTable *tab, const char *id) {
+  XLookupEntry *e;
+
+  if(!tab || !id) {
+    errno = EINVAL;
+    return NULL;
+  }
+
+  e = xGetLookupEntry(tab, id, xGetHash(id));
+  return e ? e->value : NULL;
+}
+
+/**
+ * Destroys a lookup table, freeing up it's in-memory resources.
+ *
+ * @param tab     Pointer to the lookup table to destroy.
+ *
+ * @sa xCreateLookup()
+ */
+void xDestroyLookup(XLookupTable *tab) {
+  XLookupPrivate *p;
+
+  if(!tab) return;
+
+  p = (XLookupPrivate *) tab->priv;
+  p->nEntries = 0;
+  p->nBins = 0;
+
+  if(p->table) {
+    int i;
+    for (i = 0; i < p->nBins; i++) {
+      XLookupEntry *e = p->table[i];
+
+      while(e != NULL) {
+        XLookupEntry *next = e->next;
+        if(e->key) free(e->key);
+        free(e);
+        e = next;
+      }
+    }
+
+    free(p->table);
+  }
+
+  free(tab);
+}

src/xstruct.c

@@ -171,6 +171,7 @@ XField *xCopyOfField(const XField *f) {
  * \return      Matching field from the structure or NULL if there is no match or one of
  *              the arguments is NULL.
  *
+ * \sa xLookupField()
  * \sa xSetField()
  * \sa xGetSubstruct()
  */
@@ -597,6 +598,8 @@ XField *xSetField(XStructure *s, XField *f) {
  *
  * @param s     Pointer to the structure to investigate
  * @return      the number of fields cotnained in the structure (but not counting fields in sub-structures).
+ *
+ * @sa xDeepCountFields()
  */
 int xCountFields(const XStructure *s) {
   XField *f;
@@ -914,5 +917,38 @@ int xSplitID(char *id, char **pKey) {
   return X_SUCCESS;
 }
 
+/**
+ * Counts the number of fields in a structure, including the field count for all embedded
+ * substructures also recursively.
+ *
+ * @param s           Pointer to a structure
+ * @return            The total number of fields present in the structure and all its sub-structures.
+ *
+ * @sa xCountFields()
+ *
+ */
+long xDeepCountFields(const XStructure *s) {
+  XField *f;
+  long n = 0;
+
+  if(!s) return 0;
+
+  for(f = s->firstField; f != NULL; f = f->next) {
+    n++;
+
+    if(f->type == X_STRUCT) {
+      XStructure *sub = (XStructure *) f->value;
+      int count = xGetFieldCount(f);
+      while(--count >= 0) {
+        long m = xDeepCountFields(&sub[count]);
+        if(m < 0) return -1;
+        n += m;
+      }
+    }
+  }
+
+  return n;
+}
+