From b86259969cb6cd8fd4f27ce7fbeffc9fcc7fbaa2 Mon Sep 17 00:00:00 2001 From: Attila Kovacs Date: Fri, 30 Aug 2024 22:44:49 +0200 Subject: [PATCH] Add xlookup.c, and test prog --- Makefile | 2 +- README.md | 37 ++++++++ include/xchange.h | 19 ++++ src/xlookup.c | 221 +++++++++++++++++++++++++++++++++++++++++++++ src/xstruct.c | 36 ++++++++ test/Makefile | 24 +++-- test/test-json.c | 11 ++- test/test-lookup.c | 53 +++++++++++ 8 files changed, 392 insertions(+), 11 deletions(-) create mode 100644 src/xlookup.c create mode 100644 test/test-lookup.c diff --git a/Makefile b/Makefile index 73a5f34..a36cea9 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ distclean: clean # The nitty-gritty stuff below # ---------------------------------------------------------------------------- -SOURCES = $(SRC)/xchange.c $(SRC)/xstruct.c $(SRC)/xjson.c +SOURCES = $(SRC)/xchange.c $(SRC)/xstruct.c $(SRC)/xlookup.c $(SRC)/xjson.c # Generate a list of object (obj/*.o) files from the input sources OBJECTS := $(subst $(SRC),$(OBJ),$(SOURCES)) diff --git a/README.md b/README.md index c6fc986..1b00236 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,43 @@ You can also remove existing fields from structures using `xRemoveField()`, e.g. xDestroyField(xRemoveField(s, "blah")); ``` +#### Large structures + +The normal `xGetField()` and `xGetSubstruct()` functions have computational costs that scale linearly with the number +of direct fields in the structure. It is not much of an issue for structures that contain dozens of fields (per layer). +For much larger structures, which have a fixed layout, there is an option for significantly faster hash-based lookup +also. E.g. instead of `xGetField()` you may use `xLookupField()`: + +```c + XStructure *s = ... + + // Let's create a lookup table for all fields of 's' and all its substructures. + XLookupTable *l = xCreateLookupTable(s, TRUE); + + XField *f = xLookupField(l, "subsystem:property"); + ... + + // Once done with the lookup, destroy it. + xDestroyLookup(l); +``` + +#### Iterating over elements + +You can easily iterate over the elements also. This is one application where you may want to know the internal layout +of `XStructure`, namely that it contains a simple linked-list of `XField` fields. One way to iterate over a strucures +elements is with a `for` loop, e.g.: + +```c + XStructure *s = ... + XField *f; + + for (f = s->firstField; f != NULL; f = f->next) { + // Process each field 'f' here... + ... + } +``` + + ----------------------------------------------------------------------------- diff --git a/include/xchange.h b/include/xchange.h index 48cd419..f3454d9 100644 --- a/include/xchange.h +++ b/include/xchange.h @@ -157,6 +157,19 @@ typedef struct XStructure { struct XStructure *parent; ///< Reference to parent structure (if any) } XStructure; +/** + * A fast lookup table for fields in large XStructures. + * + * @sa xCreateLookup() + * @sa xLookupField() + * @sa xDestroyLookup() + */ +typedef struct { + void *priv; ///< Private data, not exposed to users +} XLookupTable; + + + /** * Static initializer for an XStructure data structure. */ @@ -189,11 +202,17 @@ XField *xRemoveField(XStructure *s, const char *name); boolean xIsFieldValid(const XField *f); int xGetFieldCount(const XField *f); int xCountFields(const XStructure *s); +long xDeepCountFields(const XStructure *s); XStructure *xGetSubstruct(const XStructure *s, const char *id); XField *xSetSubstruct(XStructure *s, const char *name, XStructure *substruct); int xReduceDims(int *ndim, int *sizes); int xReduceAllDims(XStructure *s); +// Fast field lookup +XLookupTable *xCreateLookup(const XStructure *s, boolean recursive); +XField *xLookupField(const XLookupTable *tab, const char *id); +void xDestroyLookup(XLookupTable *tab); + // Convenience field creator methods XField *xCreateDoubleField(const char *name, double value); XField *xCreateIntField(const char *name, int value); diff --git a/src/xlookup.c b/src/xlookup.c new file mode 100644 index 0000000..bcf9bbb --- /dev/null +++ b/src/xlookup.c @@ -0,0 +1,221 @@ +/** + * @file + * + * @date Created on Aug 30, 2024 + * @author Attila Kovacs + * + * Lookup table for faster field access in large fixed-layout structures. + */ + +#include +#include +#include +#include + +#include "xchange.h" + +/// \cond PRIVATE + +typedef struct LookupEntry { + long hash; + char *key; ///< dynamically allocated. + XField *value; + struct LookupEntry *next; +} XLookupEntry; + +typedef struct { + XLookupEntry **table; + int nBins; + int nEntries; +} XLookupPrivate; + +/// \endcond + +static long xGetHash(const char *str) { + int i; + long hash = 0; + if(!str) return 0; + for(i=0; str[i]; i++) hash += str[i] ^ i; + return hash; +} + + +static XLookupEntry *xGetLookupEntry(const XLookupTable *tab, const char *key, long hash) { + const XLookupPrivate *p; + XLookupEntry *e; + + p = (XLookupPrivate *) tab->priv; + + for(e = p->table[(int) (hash % p->nBins)]; e != NULL; e=e->next) if(strcmp(e->key, key) == 0) return e; + + return NULL; +} + +static int xLookupAddAsync(XLookupTable *tab, const char *prefix, const XField *field, void **oldValue) { + XLookupPrivate *p = (XLookupPrivate *) tab->priv; + const char *id = prefix ? xGetAggregateID(prefix, field->name) : strdup(field->name); + long hash = xGetHash(id); + XLookupEntry *e = xGetLookupEntry(tab, id, hash); + int idx; + + if(e) { + if(oldValue) *oldValue = e->value; + e->value = (XField *) field; + return 1; + } + + e = (XLookupEntry *) calloc(1, sizeof(XLookupEntry)); + e->hash = hash; + e->key = (char *) id; + e->value = (XField *) field; + + idx = hash % p->nBins; + + e->next = p->table[idx]; + p->table[idx] = e; + p->nEntries++; + + return 0; +} + +static void xLookupAddAllAsync(XLookupTable *tab, const char *prefix, const XStructure *s, boolean recursive) { + XField *f; + int lp = prefix ? strlen(prefix) : 0; + + for(f = s->firstField; f != NULL; f = f->next) { + xLookupAddAsync(tab, prefix, f, NULL); + + if(f->type == X_STRUCT && recursive) { + XStructure *sub = (XStructure *) f->value; + char *p1 = (char *) malloc(lp + strlen(f->value) + 2 * sizeof(X_SEP) + 12); + + int count = xGetFieldCount(f); + while(--count >= 0) { + int n = 0; + if(prefix) n = sprintf(p1, "%s" X_SEP, prefix); + + if(f->ndim == 0) sprintf(&p1[n], "%s", f->name); + else sprintf(&p1[n], "%s" X_SEP "%d", f->name, (count + 1)); + + xLookupAddAllAsync(tab, p1, &sub[count], TRUE); + } + } + } +} + +static XLookupTable *xAllocLookup(int size) { + XLookupTable *tab; + XLookupPrivate *p; + + unsigned int n = 1; + while(n < size) n <<= 1; + + p = (XLookupPrivate *) calloc(1, sizeof(XLookupPrivate)); + p->table = (XLookupEntry **) calloc(n, sizeof(XLookupEntry *)); + if(!p->table) { + perror("ERROR! xAllocLookup"); + free(p); + return NULL; + } + + p->nBins = p->table ? n : 0; + + tab = (XLookupTable *) calloc(1, sizeof(XLookupTable)); + tab->priv = p; + + + return tab; +} + +/** + * Creates a fast name lookup table for the fields of structure, with or without including fields of embedded + * substructures also. For structures with a large number of fields, such a lookup can significantly + * improve access times for retrieving specific fields from a structure. However, the lookup will not + * track fields added or removed after its creation, and so it is suited for accessing structures with a + * fixed layout only. + * + * Since the lookup table contains references to the structure, you should not destroy the structure as long + * as the lookup table is used. + * + * Once the lookup table is no longer used, the caller should explicitly destroy it with `xDestroyLookup()` + * + * @param s Pointer to a structure, for which to create a field lookup. + * @param recursive Whether to include fields from substructures recursively in the lookup. + * @return The lookup table, or NULL if there was an error (errno will inform about the type of + * error). + * + * @sa xLookupField() + * @sa xDestroyLookup() + */ +XLookupTable *xCreateLookup(const XStructure *s, boolean recursive) { + XLookupTable *l; + + if(s == NULL) { + errno = EINVAL; + return NULL; + } + + l = xAllocLookup(recursive ? xDeepCountFields(s) : xCountFields(s)); + if(!l) return NULL; + + xLookupAddAllAsync(l, NULL, s, recursive); + + return l; +} + +/** + * Returns a named field from a lookup table. + * + * @param tab Pointer to the lookup table + * @param id The aggregate ID of the field to find. + * @return The corresponding field or NULL if no such field exists or tab was NULL. + * + * @sa xCreateLookup() + * @sa xGetField() + */ +XField *xLookupField(const XLookupTable *tab, const char *id) { + XLookupEntry *e; + + if(!tab || !id) { + errno = EINVAL; + return NULL; + } + + e = xGetLookupEntry(tab, id, xGetHash(id)); + return e ? e->value : NULL; +} + +/** + * Destroys a lookup table, freeing up it's in-memory resources. + * + * @param tab Pointer to the lookup table to destroy. + * + * @sa xCreateLookup() + */ +void xDestroyLookup(XLookupTable *tab) { + XLookupPrivate *p; + + if(!tab) return; + + p = (XLookupPrivate *) tab->priv; + p->nEntries = 0; + p->nBins = 0; + + if(p->table) { + int i; + for (i = 0; i < p->nBins; i++) { + XLookupEntry *e = p->table[i]; + + while(e != NULL) { + XLookupEntry *next = e->next; + if(e->key) free(e->key); + free(e); + e = next; + } + } + + free(p->table); + } + + free(tab); +} diff --git a/src/xstruct.c b/src/xstruct.c index 2888565..b0640ee 100644 --- a/src/xstruct.c +++ b/src/xstruct.c @@ -171,6 +171,7 @@ XField *xCopyOfField(const XField *f) { * \return Matching field from the structure or NULL if there is no match or one of * the arguments is NULL. * + * \sa xLookupField() * \sa xSetField() * \sa xGetSubstruct() */ @@ -597,6 +598,8 @@ XField *xSetField(XStructure *s, XField *f) { * * @param s Pointer to the structure to investigate * @return the number of fields cotnained in the structure (but not counting fields in sub-structures). + * + * @sa xDeepCountFields() */ int xCountFields(const XStructure *s) { XField *f; @@ -914,5 +917,38 @@ int xSplitID(char *id, char **pKey) { return X_SUCCESS; } +/** + * Counts the number of fields in a structure, including the field count for all embedded + * substructures also recursively. + * + * @param s Pointer to a structure + * @return The total number of fields present in the structure and all its sub-structures. + * + * @sa xCountFields() + * + */ +long xDeepCountFields(const XStructure *s) { + XField *f; + long n = 0; + + if(!s) return 0; + + for(f = s->firstField; f != NULL; f = f->next) { + n++; + + if(f->type == X_STRUCT) { + XStructure *sub = (XStructure *) f->value; + int count = xGetFieldCount(f); + while(--count >= 0) { + long m = xDeepCountFields(&sub[count]); + if(m < 0) return -1; + n += m; + } + } + } + + return n; +} + diff --git a/test/Makefile b/test/Makefile index 2e90f34..34ae176 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,19 +1,29 @@ +# Use the parent directories for libraries and headers. +LIB = ../lib INC = ../include +# Load the common Makefile definitions... include ../config.mk CFLAGS += -g +export CFLAGS -.PHONY: all -all: bin/test-json +all: $(BIN)/test-json $(BIN)/test-lookup -bin/test-json: test-json.c ../lib/libxchange.a +$(BIN)/test-%: $(OBJ)/test-%.o $(LIB)/libxchange.a | $(BIN) + $(CC) -o $@ $^ $(LDFLAGS) -../lib/libxchange.a: - @make -C .. static +$(LIB)/libxchange.a: + make -C .. static +$(BIN): + mkdir $(BIN) -bin/%: | bin - $(CC) -o $@ $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) +.PHONY: clean-test +clean-test: + rm -rf bin + +clean: clean-test +# Finally, the standard generic rules and targets... include ../build.mk diff --git a/test/test-json.c b/test/test-json.c index 2589e74..2dc96b1 100644 --- a/test/test-json.c +++ b/test/test-json.c @@ -7,6 +7,7 @@ #include #include +#include #include "xchange.h" #include "xjson.h" @@ -41,15 +42,19 @@ static XStructure *createStruct() { int main(int argc, char *argv[]) { XStructure *s = createStruct(), *s1; - char *str = xjsonToString(s), *next; + char *str = xjsonToString(s), *next, *str1; printf("%s\n\n", str); next = str; s1 = xjsonParseAt(&next, NULL); - printf("%s\n\n", xjsonToString(s1)); - + str1 = xjsonToString(s1); + if(strcmp(str1, str) != 0) { + fprintf(stderr, "ERROR! str1 != str:\n\n"); + printf("%s\n\n", str1); + return 1; + } return 0; } diff --git a/test/test-lookup.c b/test/test-lookup.c new file mode 100644 index 0000000..dd6f48d --- /dev/null +++ b/test/test-lookup.c @@ -0,0 +1,53 @@ +/** + * @file + * + * @date Created on Aug 30, 2024 + * @author Attila Kovacs + */ + +#include +#include + +#include "xchange.h" + +int main(int argc, char *argv[]) { + XStructure *s, *sys, *sub; + XLookupTable *l; + XField *f; + + s = xCreateStruct(); + + sys = xCreateStruct(); + xSetSubstruct(s, "system", sys); + + sub = xCreateStruct(); + xSetSubstruct(sys, "subsystem", sub); + + xSetField(sub, xCreateIntField("property", 1)); + + l = xCreateLookup(s, FALSE); + f = xLookupField(l, "system"); + if(!f) { + fprintf(stderr, "ERROR! system not found\n"); + return 1; + } + else if(f->value != (void *) sys) { + fprintf(stderr, "ERROR! l1: %p != %p\n", f->value, sys); + return 1; + } + xDestroyLookup(l); + + l = xCreateLookup(s, TRUE); + f = xLookupField(l, "system:subsystem"); + if(!f) { + fprintf(stderr, "ERROR! subsystem not found\n"); + return 1; + } + else if(f->value != (void *) sub) { + fprintf(stderr, "ERROR! l1: %p != %p\n", f->value, sub); + return 1; + } + xDestroyLookup(l); + + return 0; +}