From 12578aaab11829fe3630ca065523288f4dbcb20a Mon Sep 17 00:00:00 2001 From: Joey Date: Wed, 5 Jan 2022 10:57:38 -0800 Subject: [PATCH 01/26] Track rax allocation size in rax header Signed-off-by: Guillaume Koenig --- src/rax.c | 28 ++++++++++++++++++++++++++++ src/rax.h | 8 +++++--- src/rax_malloc.h | 1 + 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/rax.c b/src/rax.c index cee9b714e5..eaf96f8fca 100644 --- a/src/rax.c +++ b/src/rax.c @@ -192,6 +192,7 @@ rax *raxNew(void) { rax->numele = 0; rax->numnodes = 1; rax->head = raxNewNode(0, 0); + rax->alloc = sizeof(rax) + rax_alloc_size(rax->head); if (rax->head == NULL) { rax_free(rax); return NULL; @@ -507,9 +508,12 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** * our key. We have just to reallocate the node and make space for the * data pointer. */ if (i == len && (!h->iscompr || j == 0 /* not in the middle if j is 0 */)) { + size_t oldalloc = 0; + debugf("### Insert: node representing key exists\n"); /* Make space for the value pointer if needed. */ if (!h->iskey || (h->isnull && overwrite)) { + oldalloc = rax_alloc_size(h); h = raxReallocForData(h, data); if (h) memcpy(parentlink, &h, sizeof(h)); } @@ -530,6 +534,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** * will set h->iskey. */ raxSetData(h, data); rax->numele++; + rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h); return 1; /* Element inserted. */ } @@ -706,6 +711,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** return 0; } splitnode->data[0] = h->data[j]; + rax->alloc += rax_alloc_size(splitnode); if (j == 0) { /* 3a: Replace the old node with the split node. */ @@ -730,6 +736,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** memcpy(parentlink, &trimmed, sizeof(trimmed)); parentlink = cp; /* Set parentlink to splitnode parent. */ rax->numnodes++; + rax->alloc += rax_alloc_size(trimmed); } /* 4: Create the postfix node: what remains of the original @@ -744,6 +751,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** raxNode **cp = raxNodeLastChildPtr(postfix); memcpy(cp, &next, sizeof(next)); rax->numnodes++; + rax->alloc += rax_alloc_size(postfix); } else { /* 4b: just use next as postfix node. */ postfix = next; @@ -756,6 +764,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** /* 6. Continue insertion: this will cause the splitnode to * get a new child (the non common character at the currently * inserted key). */ + rax->alloc -= rax_alloc_size(h); rax_free(h); h = splitnode; } else if (h->iscompr && i == len) { @@ -794,6 +803,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** raxNode **cp = raxNodeLastChildPtr(postfix); memcpy(cp, &next, sizeof(next)); rax->numnodes++; + rax->alloc += rax_alloc_size(postfix); /* 3: Trim the compressed node. */ trimmed->size = j; @@ -806,6 +816,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** void *aux = raxGetData(h); raxSetData(trimmed, aux); } + rax->alloc += rax_alloc_size(trimmed); /* Fix the trimmed node child pointer to point to * the postfix node. */ @@ -815,6 +826,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** /* Finish! We don't need to continue with the insertion * algorithm for ALGO 2. The key is already inserted. */ rax->numele++; + rax->alloc -= rax_alloc_size(h); rax_free(h); return 1; /* Key inserted. */ } @@ -823,6 +835,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** * chars in our string. We need to insert the missing nodes. */ while (i < len) { raxNode *child; + size_t oldalloc = rax_alloc_size(h); /* If this node is going to have a single child, and there * are other characters, so that that would result in a chain @@ -848,14 +861,17 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** i++; } rax->numnodes++; + rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h) + rax_alloc_size(child); h = child; } + size_t oldalloc = rax_alloc_size(h); raxNode *newh = raxReallocForData(h, data); if (newh == NULL) goto oom; h = newh; if (!h->iskey) rax->numele++; raxSetData(h, data); memcpy(parentlink, &h, sizeof(h)); + rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h); return 1; /* Element inserted. */ oom: @@ -1025,6 +1041,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { child = h; debugf("Freeing child %p [%.*s] key:%d\n", (void *)child, (int)child->size, (char *)child->data, child->iskey); + rax->alloc -= rax_alloc_size(child); rax_free(child); rax->numnodes--; h = raxStackPop(&ts); @@ -1035,6 +1052,8 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { if (child) { debugf("Unlinking child %p from parent %p\n", (void *)child, (void *)h); raxNode *new = raxRemoveChild(h, child); + size_t oldalloc = rax_alloc_size(h); + rax->alloc = rax->alloc - oldalloc + rax_alloc_size(new); if (new != h) { raxNode *parent = raxStackPeek(&ts); raxNode **parentlink; @@ -1046,6 +1065,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { memcpy(parentlink, &new, sizeof(new)); } + /* If after the removal the node has just a single child * and is not a key, we need to try to compress it. */ if (new->size == 1 && new->iskey == 0) { @@ -1151,6 +1171,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { new->iscompr = 1; new->size = comprsize; rax->numnodes++; + rax->alloc += rax_alloc_size(new); /* Scan again, this time to populate the new node content and * to fix the new node child pointer. At the same time we free @@ -1163,6 +1184,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { raxNode **cp = raxNodeLastChildPtr(h); raxNode *tofree = h; memcpy(&h, cp, sizeof(h)); + rax->alloc -= rax_alloc_size(tofree); rax_free(tofree); rax->numnodes--; if (h->iskey || (!h->iscompr && h->size != 1)) break; @@ -1218,6 +1240,7 @@ void raxFreeWithCallback(rax *rax, void (*free_callback)(void *)) { /* Free a whole radix tree. */ void raxFree(rax *rax) { raxFreeWithCallback(rax, NULL); + rax->alloc = sizeof(rax); } /* ------------------------------- Iterator --------------------------------- */ @@ -1764,6 +1787,11 @@ uint64_t raxSize(rax *rax) { return rax->numele; } +/* Return the rax tree allocation size in bytes */ +size_t raxAllocSize(rax *rax) { + return rax->alloc; +} + /* ----------------------------- Introspection ------------------------------ */ /* This function is mostly used for debugging and learning purposes. diff --git a/src/rax.h b/src/rax.h index c03e0303a0..57cc357c62 100644 --- a/src/rax.h +++ b/src/rax.h @@ -131,9 +131,10 @@ typedef struct raxNode { } raxNode; typedef struct rax { - raxNode *head; - uint64_t numele; - uint64_t numnodes; + raxNode *head; /* Pointer to root node of tree */ + uint64_t numele; /* Number of keys in the tree */ + uint64_t numnodes; /* Number of rax nodes in the tree */ + size_t alloc; /* Total allocation size of the tree in bytes */ } rax; /* Stack data structure used by raxLowWalk() in order to, optionally, return @@ -205,6 +206,7 @@ void raxStop(raxIterator *it); int raxEOF(raxIterator *it); void raxShow(rax *rax); uint64_t raxSize(rax *rax); +size_t raxAllocSize(rax *rax); unsigned long raxTouch(raxNode *n); void raxSetDebugMsg(int onoff); diff --git a/src/rax_malloc.h b/src/rax_malloc.h index 9295985c65..357fe04916 100644 --- a/src/rax_malloc.h +++ b/src/rax_malloc.h @@ -41,4 +41,5 @@ #define rax_malloc zmalloc #define rax_realloc zrealloc #define rax_free zfree +#define rax_alloc_size zmalloc_size #endif From aaebb31c37f4606dce0f66050be9fe49482dd7a7 Mon Sep 17 00:00:00 2001 From: Joey Date: Thu, 20 Jan 2022 16:56:48 -0800 Subject: [PATCH 02/26] fix use after free, don't update the alloc field in raxFree Signed-off-by: Guillaume Koenig --- src/rax.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rax.c b/src/rax.c index eaf96f8fca..33a09d8283 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1240,7 +1240,6 @@ void raxFreeWithCallback(rax *rax, void (*free_callback)(void *)) { /* Free a whole radix tree. */ void raxFree(rax *rax) { raxFreeWithCallback(rax, NULL); - rax->alloc = sizeof(rax); } /* ------------------------------- Iterator --------------------------------- */ From e09bef36a9413656061816130b0dc47576ffe543 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Mon, 24 Jun 2024 21:27:03 +0000 Subject: [PATCH 03/26] Fix issues with previous attempt Signed-off-by: Guillaume Koenig --- src/rax.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/rax.c b/src/rax.c index 33a09d8283..03c3d10c7a 100644 --- a/src/rax.c +++ b/src/rax.c @@ -192,7 +192,7 @@ rax *raxNew(void) { rax->numele = 0; rax->numnodes = 1; rax->head = raxNewNode(0, 0); - rax->alloc = sizeof(rax) + rax_alloc_size(rax->head); + rax->alloc = rax_alloc_size(rax) + rax_alloc_size(rax->head); if (rax->head == NULL) { rax_free(rax); return NULL; @@ -515,7 +515,10 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** if (!h->iskey || (h->isnull && overwrite)) { oldalloc = rax_alloc_size(h); h = raxReallocForData(h, data); - if (h) memcpy(parentlink, &h, sizeof(h)); + if (h) { + memcpy(parentlink, &h, sizeof(h)); + rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h); + } } if (h == NULL) { errno = ENOMEM; @@ -534,7 +537,6 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** * will set h->iskey. */ raxSetData(h, data); rax->numele++; - rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h); return 1; /* Element inserted. */ } @@ -1065,7 +1067,6 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { memcpy(parentlink, &new, sizeof(new)); } - /* If after the removal the node has just a single child * and is not a key, we need to try to compress it. */ if (new->size == 1 && new->iskey == 0) { From 1acaa417a9b8530ac4749e0372549dc26ddb8e98 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 31 Jan 2022 18:14:12 -0800 Subject: [PATCH 04/26] Add rax-test from antirez repo into Redis repo Signed-off-by: Guillaume Koenig --- src/rax-test.c | 1069 ++++++++++++++++++++++++++++++++++++++++++++++++ src/server.c | 2 + 2 files changed, 1071 insertions(+) create mode 100644 src/rax-test.c diff --git a/src/rax-test.c b/src/rax-test.c new file mode 100644 index 0000000000..9207bf803a --- /dev/null +++ b/src/rax-test.c @@ -0,0 +1,1069 @@ +/* Rax -- A radix tree implementation. + * + * Copyright (c) 2017-2018, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef REDIS_TEST + +#include +#include +#include +#include +#include +#include +#include + +#include "rax.h" +#include "rc4rand.h" + +uint16_t crc16(const char *buf, int len); /* From crc16.c */ + +/* --------------------------------------------------------------------------- + * Simple hash table implementation, no rehashing, just chaining. This is + * used in order to test the radix tree implementation against something that + * will always "tell the truth" :-) */ + +#define HT_TABLE_SIZE 100000 /* This is huge but we want it fast enough without + * reahshing needed. */ +typedef struct htNode { + uint64_t keylen; + unsigned char *key; + void *data; + struct htNode *next; +} htNode; + +typedef struct ht { + uint64_t numele; + htNode *table[HT_TABLE_SIZE]; +} hashtable; + +/* Create a new hash table. */ +hashtable *htNew(void) { + hashtable *ht = calloc(1,sizeof(*ht)); + ht->numele = 0; + return ht; +} + +/* djb2 hash function. */ +uint32_t htHash(unsigned char *s, size_t len) { + uint32_t hash = 5381; + for (size_t i = 0; i < len; i++) + hash = hash * 33 + s[i]; + return hash % HT_TABLE_SIZE; +} + +/* Low level hash table lookup function. */ +htNode *htRawLookup(hashtable *t, unsigned char *s, size_t len, uint32_t *hash, htNode ***parentlink) { + uint32_t h = htHash(s,len); + if (hash) *hash = h; + htNode *n = t->table[h]; + if (parentlink) *parentlink = &t->table[h]; + while(n) { + if (n->keylen == len && memcmp(n->key,s,len) == 0) return n; + if (parentlink) *parentlink = &n->next; + n = n->next; + } + return NULL; +} + +/* Add an elmenet to the hash table, return 1 if the element is new, + * 0 if it existed and the value was updated to the new one. */ +int htAdd(hashtable *t, unsigned char *s, size_t len, void *data) { + uint32_t hash; + htNode *n = htRawLookup(t,s,len,&hash,NULL); + + if (!n) { + n = malloc(sizeof(*n)); + n->key = malloc(len); + memcpy(n->key,s,len); + n->keylen = len; + n->data = data; + n->next = t->table[hash]; + t->table[hash] = n; + t->numele++; + return 1; + } else { + n->data = data; + return 0; + } +} + +/* Remove the specified element, returns 1 on success, 0 if the element + * was not there already. */ +int htRem(hashtable *t, unsigned char *s, size_t len) { + htNode **parentlink; + htNode *n = htRawLookup(t,s,len,NULL,&parentlink); + + if (!n) return 0; + *parentlink = n->next; + free(n->key); + free(n); + t->numele--; + return 1; +} + +void *htNotFound = (void*)"ht-not-found"; + +/* Find an element inside the hash table. Returns htNotFound if the + * element is not there, otherwise returns the associated value. */ +void *htFind(hashtable *t, unsigned char *s, size_t len) { + htNode *n = htRawLookup(t,s,len,NULL,NULL); + if (!n) return htNotFound; + return n->data; +} + +/* Free the whole hash table including all the linked nodes. */ +void htFree(hashtable *ht) { + for (int j = 0; j < HT_TABLE_SIZE; j++) { + htNode *next = ht->table[j]; + while(next) { + htNode *this = next; + next = this->next; + free(this->key); + free(this); + } + } + free(ht); +} + +/* -------------------------------------------------------------------------- + * Utility functions to generate keys, check time usage and so forth. + * -------------------------------------------------------------------------*/ + +/* This is a simple Feistel network in order to turn every possible + * uint32_t input into another "randomly" looking uint32_t. It is a + * one to one map so there are no repetitions. */ +static uint32_t int2int(uint32_t input) { + uint16_t l = input & 0xffff; + uint16_t r = input >> 16; + for (int i = 0; i < 8; i++) { + uint16_t nl = r; + uint16_t F = (((r * 31) + (r >> 5) + 7 * 371) ^ r) & 0xffff; + r = l ^ F; + l = nl; + } + return (r<<16)|l; +} + +/* Turn an uint32_t integer into an alphanumerical key and return its + * length. This function is used in order to generate keys that have + * a large charset, so that the radix tree can be testsed with many + * children per node. */ +static size_t int2alphakey(char *s, size_t maxlen, uint32_t i) { + const char *set = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789"; + const size_t setlen = 62; + + if (maxlen == 0) return 0; + maxlen--; /* Space for null term char. */ + size_t len = 0; + while(len < maxlen) { + s[len++] = set[i%setlen]; + i /= setlen; + if (i == 0) break; + } + s[len] = '\0'; + return len; +} + +/* Return the UNIX time in microseconds */ +long long ustime(void) { + struct timeval tv; + long long ust; + + gettimeofday(&tv, NULL); + ust = ((long long)tv.tv_sec)*1000000; + ust += tv.tv_usec; + return ust; +} + +/* Turn the integer 'i' into a key according to 'mode'. + * KEY_INT: Just represents the integer as a string. + * KEY_UNIQUE_ALPHA: Turn it into a random-looking alphanumerical string + * according to the int2alphakey() function, so that + * at every integer is mapped a different string. + * KEY_RANDOM: Totally random string up to maxlen bytes. + * KEY_RANDOM_ALPHA: Alphanumerical random string up to maxlen bytes. + * KEY_RANDOM_SMALL_CSET: Small charset random strings. + * KEY_CHAIN: 'i' times the character "A". */ +#define KEY_INT 0 +#define KEY_UNIQUE_ALPHA 1 +#define KEY_RANDOM 2 +#define KEY_RANDOM_ALPHA 3 +#define KEY_RANDOM_SMALL_CSET 4 +#define KEY_CHAIN 5 +static size_t int2key(char *s, size_t maxlen, uint32_t i, int mode) { + if (mode == KEY_INT) { + return snprintf(s,maxlen,"%lu",(unsigned long)i); + } else if (mode == KEY_UNIQUE_ALPHA) { + if (maxlen > 16) maxlen = 16; + i = int2int(i); + return int2alphakey(s,maxlen,i); + } else if (mode == KEY_RANDOM) { + if (maxlen > 16) maxlen = 16; + int r = rc4rand() % maxlen; + for (int i = 0; i < r; i++) s[i] = rc4rand()&0xff; + return r; + } else if (mode == KEY_RANDOM_ALPHA) { + if (maxlen > 16) maxlen = 16; + int r = rc4rand() % maxlen; + for (int i = 0; i < r; i++) s[i] = 'A'+rc4rand()%('z'-'A'+1); + return r; + } else if (mode == KEY_RANDOM_SMALL_CSET) { + if (maxlen > 16) maxlen = 16; + int r = rc4rand() % maxlen; + for (int i = 0; i < r; i++) s[i] = 'A'+rc4rand()%4; + return r; + } else if (mode == KEY_CHAIN) { + if (i > maxlen) i = maxlen; + memset(s,'A',i); + return i; + } else { + return 0; + } +} + +/* -------------------------------------------------------------------------- */ + +/* Perform a fuzz test, returns 0 on success, 1 on error. */ +int fuzzTest(int keymode, size_t count, double addprob, double remprob) { + hashtable *ht = htNew(); + rax *rax = raxNew(); + + printf("Fuzz test in mode %d [%zu]: ", keymode, count); + fflush(stdout); + + /* Perform random operations on both the dictionaries. */ + for (size_t i = 0; i < count; i++) { + unsigned char key[1024]; + uint32_t keylen; + + /* Insert element. */ + if ((double)rc4rand()/RAND_MAX < addprob) { + keylen = int2key((char*)key,sizeof(key),i,keymode); + void *val = (void*)(unsigned long)rc4rand(); + /* Stress NULL values more often, they use a special encoding. */ + if (!(rc4rand() % 100)) val = NULL; + int retval1 = htAdd(ht,key,keylen,val); + int retval2 = raxInsert(rax,key,keylen,val,NULL); + if (retval1 != retval2) { + printf("Fuzz: key insertion reported mismatching value in HT/RAX\n"); + return 1; + } + } + + /* Remove element. */ + if ((double)rc4rand()/RAND_MAX < remprob) { + keylen = int2key((char*)key,sizeof(key),i,keymode); + int retval1 = htRem(ht,key,keylen); + int retval2 = raxRemove(rax,key,keylen,NULL); + if (retval1 != retval2) { + printf("Fuzz: key deletion of '%.*s' reported mismatching " + "value in HT=%d RAX=%d\n", + (int)keylen,(char*)key,retval1, retval2); + printf("%p\n", raxFind(rax,key,keylen)); + printf("%p\n", raxNotFound); + return 1; + } + } + } + + /* Check that count matches. */ + if (ht->numele != raxSize(rax)) { + printf("Fuzz: HT / RAX keys count mismatch: %lu vs %lu\n", + (unsigned long) ht->numele, + (unsigned long) raxSize(rax)); + return 1; + } + printf("%lu elements inserted\n", (unsigned long)ht->numele); + + /* Check that elements match. */ + raxIterator iter; + raxStart(&iter,rax); + raxSeek(&iter,"^",NULL,0); + + size_t numkeys = 0; + while(raxNext(&iter)) { + void *val1 = htFind(ht,iter.key,iter.key_len); + void *val2 = raxFind(rax,iter.key,iter.key_len); + if (val1 != val2) { + printf("Fuzz: HT=%p, RAX=%p value do not match " + "for key %.*s\n", + val1, val2, (int)iter.key_len,(char*)iter.key); + return 1; + } + numkeys++; + } + + /* Check that the iterator reported all the elements. */ + if (ht->numele != numkeys) { + printf("Fuzz: the iterator reported %lu keys instead of %lu\n", + (unsigned long) numkeys, + (unsigned long) ht->numele); + return 1; + } + + raxStop(&iter); + raxFree(rax); + htFree(ht); + return 0; +} + +/* Redis Cluster alike fuzz testing. + * + * This test simulates the radix tree usage made by Redis Cluster in order + * to maintain the hash slot -> keys mappig. The keys are alphanumerical + * but the first two bytes that are binary (and are the key hashed). + * + * In this test there is no comparison with the hash table, the only goal + * is to crash the radix tree implementation, or to trigger Valgrind + * warnings. */ +int fuzzTestCluster(size_t count, double addprob, double remprob) { + unsigned char key[128]; + int keylen = 0; + + printf("Cluster Fuzz test [keys:%zu keylen:%d]: ", count, keylen); + fflush(stdout); + + rax *rax = raxNew(); + + /* This is our template to generate keys. The first two bytes will + * be replaced with the binary redis cluster hash slot. */ + keylen = snprintf((char*)key,sizeof(key),"__geocode:2e68e5df3624"); + char *cset = "0123456789abcdef"; + + for (unsigned long j = 0; j < count; j++) { + /* Generate a random key by altering our template key. */ + + /* With a given probability, let's use a common prefix so that there + * is a subset of keys that have an higher percentage of probability + * of being hit again and again. */ + size_t commonprefix = rc4rand() & 0xf; + if (commonprefix == 0) memcpy(key+10,"2e68e5",6); + + /* Alter a random char in the key. */ + int pos = 10+rc4rand()%12; + key[pos] = cset[rc4rand()%16]; + + /* Compute the Redis Cluster hash slot to set the first two + * binary bytes of the key. */ + int hashslot = crc16((char*)key,keylen) & 0x3FFF; + key[0] = (hashslot >> 8) & 0xff; + key[1] = hashslot & 0xff; + + /* Insert element. */ + if ((double)rc4rand()/RAND_MAX < addprob) { + raxInsert(rax,key,keylen,NULL,NULL); + } + + /* Remove element. */ + if ((double)rc4rand()/RAND_MAX < remprob) { + raxRemove(rax,key,keylen,NULL); + } + } + size_t finalkeys = raxSize(rax); + raxFree(rax); + printf("ok with %zu final keys\n",finalkeys); + return 0; +} + +/* Iterator fuzz testing. Compared the items returned by the Rax iterator with + * a C implementation obtained by sorting the inserted strings in a linear + * array. */ +typedef struct arrayItem { + unsigned char *key; + size_t key_len; +} arrayItem; + +/* Utility functions used with qsort() in order to sort the array of strings + * in the same way Rax sorts keys (which is, lexicographically considering + * every byte an unsigned integer. */ +int compareAB(const unsigned char *keya, size_t lena, const unsigned char *keyb, size_t lenb) { + size_t minlen = (lena <= lenb) ? lena : lenb; + int retval = memcmp(keya,keyb,minlen); + if (lena == lenb || retval != 0) return retval; + return (lena > lenb) ? 1 : -1; +} + +int compareArrayItems(const void *aptr, const void *bptr) { + const arrayItem *a = aptr; + const arrayItem *b = bptr; + return compareAB(a->key,a->key_len,b->key,b->key_len); +} + +/* Seek an element in the array, returning the seek index (the index inside the + * array). If the seek is not possible (== operator and key not found or empty + * array) -1 is returned. */ +int arraySeek(arrayItem *array, int count, unsigned char *key, size_t len, char *op) { + if (count == 0) return -1; + if (op[0] == '^') return 0; + if (op[0] == '$') return count-1; + + int eq = 0, lt = 0, gt = 0; + if (op[1] == '=') eq = 1; + if (op[0] == '<') lt = 1; + if (op[0] == '>') gt = 1; + + int i; + for (i = 0; i < count; i++) { + int cmp = compareAB(array[i].key,array[i].key_len,key,len); + if (eq && !cmp) return i; + if (cmp > 0 && gt) return i; + if (cmp >= 0 && lt) { + i--; + break; + } + } + if (lt && i == count) return count-1; + if (i < 0 || i >= count) return -1; + return i; +} + +int iteratorFuzzTest(int keymode, size_t count) { + count = rc4rand()%count; + rax *rax = raxNew(); + arrayItem *array = malloc(sizeof(arrayItem)*count); + + /* Fill a radix tree and a linear array with some data. */ + unsigned char key[1024]; + size_t j = 0; + for (size_t i = 0; i < count; i++) { + uint32_t keylen = int2key((char*)key,sizeof(key),i,keymode); + void *val = (void*)(unsigned long)htHash(key,keylen); + + if (raxInsert(rax,key,keylen,val,NULL)) { + array[j].key = malloc(keylen); + array[j].key_len = keylen; + memcpy(array[j].key,key,keylen); + j++; + } + } + count = raxSize(rax); + + /* Sort the array. */ + qsort(array,count,sizeof(arrayItem),compareArrayItems); + + /* Perform a random seek operation. */ + uint32_t keylen = int2key((char*)key,sizeof(key), + rc4rand()%(count ? count : 1),keymode); + raxIterator iter; + raxStart(&iter,rax); + char *seekops[] = {"==",">=","<=",">","<","^","$"}; + char *seekop = seekops[rc4rand() % 7]; + raxSeek(&iter,seekop,key,keylen); + int seekidx = arraySeek(array,count,key,keylen,seekop); + + int next = rc4rand() % 2; + int iteration = 0; + while(1) { + int rax_res; + int array_res; + unsigned char *array_key = NULL; + size_t array_key_len = 0; + + array_res = (seekidx == -1) ? 0 : 1; + if (array_res) { + if (next && seekidx == (signed)count) array_res = 0; + if (!next && seekidx == -1) array_res = 0; + if (array_res != 0) { + array_key = array[seekidx].key; + array_key_len = array[seekidx].key_len; + } + } + + if (next) { + rax_res = raxNext(&iter); + if (array_res) seekidx++; + } else { + rax_res = raxPrev(&iter); + if (array_res) seekidx--; + } + + /* Both the iteratos should agree about EOF. */ + if (array_res != rax_res) { + printf("Iter fuzz: iterators do not agree about EOF " + "at iteration %d: " + "array_more=%d rax_more=%d next=%d\n", + iteration, array_res, rax_res, next); + return 1; + } + if (array_res == 0) break; /* End of iteration reached. */ + + /* Check that the returned keys are the same. */ + if (iter.key_len != array_key_len || + memcmp(iter.key,array_key,iter.key_len)) + { + printf("Iter fuzz: returned element %d mismatch\n", iteration); + printf("SEEKOP was %s\n",seekop); + if (keymode != KEY_RANDOM) { + printf("\n"); + printf("BUG SEEKING: %s %.*s\n",seekop,keylen,key); + printf("%.*s (iter) VS %.*s (array) next=%d idx=%d " + "count=%lu keymode=%d\n", + (int)iter.key_len, (char*)iter.key, + (int)array_key_len, (char*)array_key, + next, seekidx, (unsigned long)count, keymode); + if (count < 500) { + printf("\n"); + for (unsigned int j = 0; j < count; j++) { + printf("%d) '%.*s'\n",j, + (int)array[j].key_len, + array[j].key); + } + } + exit(1); + } + return 1; + } + iteration++; + } + + for (unsigned int i = 0; i < count; i++) free(array[i].key); + free(array); + raxStop(&iter); + raxFree(rax); + return 0; +} + +/* Test the random walk function. */ +int randomWalkTest(void) { + rax *t = raxNew(); + char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; + + long numele; + for (numele = 0; toadd[numele] != NULL; numele++) { + raxInsert(t,(unsigned char*)toadd[numele], + strlen(toadd[numele]),(void*)numele,NULL); + } + + raxIterator iter; + raxStart(&iter,t); + raxSeek(&iter,"^",NULL,0); + int maxloops = 100000; + while(raxRandomWalk(&iter,0) && maxloops--) { + int nulls = 0; + for (long i = 0; i < numele; i++) { + if (toadd[i] == NULL) { + nulls++; + continue; + } + if (strlen(toadd[i]) == iter.key_len && + memcmp(toadd[i],iter.key,iter.key_len) == 0) + { + toadd[i] = NULL; + nulls++; + } + } + if (nulls == numele) break; + } + if (maxloops == 0) { + printf("randomWalkTest() is unable to report all the elements " + "after 100k iterations!\n"); + return 1; + } + raxStop(&iter); + raxFree(t); + return 0; +} + +int iteratorUnitTests(void) { + rax *t = raxNew(); + char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; + + for (int x = 0; x < 10000; x++) rc4rand(); + + long items = 0; + while(toadd[items] != NULL) items++; + + for (long i = 0; i < items; i++) + raxInsert(t,(unsigned char*)toadd[i],strlen(toadd[i]),(void*)i,NULL); + + raxIterator iter; + raxStart(&iter,t); + + struct { + char *seek; + size_t seeklen; + char *seekop; + char *expected; + } tests[] = { + /* Seek value. */ /* Expected result. */ + {"rpxxx",5,"<=", "romulus"}, + {"rom",3,">=", "romane"}, + {"rub",3,">=", "rub"}, + {"rub",3,">", "rubens"}, + {"rub",3,"<", "romulus"}, + {"rom",3,">", "romane"}, + {"chro",4,">", "chromodynamic"}, + {"chro",4,"<", "baloon"}, + {"chromz",6,"<", "chromodynamic"}, + {"",0,"^", "alien"}, + {"zorro",5,"<=", "rubicundus"}, + {"zorro",5,"<", "rubicundus"}, + {"zorro",5,"<", "rubicundus"}, + {"",0,"$", "rubicundus"}, + {"ro",2,">=", "romane"}, + {"zo",2,">", NULL}, + {"zo",2,"==", NULL}, + {"romane",6,"==", "romane"} + }; + + for (int i = 0; tests[i].expected != NULL; i++) { + raxSeek(&iter,tests[i].seekop,(unsigned char*)tests[i].seek, + tests[i].seeklen); + int retval = raxNext(&iter); + + if (tests[i].expected != NULL) { + if (strlen(tests[i].expected) != iter.key_len || + memcmp(tests[i].expected,iter.key,iter.key_len) != 0) + { + printf("Iterator unit test error: " + "test %d, %s expected, %.*s reported\n", + i, tests[i].expected, (int)iter.key_len, + (char*)iter.key); + return 1; + } + } else { + if (retval != 0) { + printf("Iterator unit test error: " + "EOF expected in test %d\n", i); + return 1; + } + } + } + raxStop(&iter); + raxFree(t); + return 0; +} + +/* Test that raxInsert() / raxTryInsert() overwrite semantic + * works as expected. */ +int tryInsertUnitTests(void) { + rax *t = raxNew(); + raxInsert(t,(unsigned char*)"FOO",3,(void*)(long)1,NULL); + void *old, *val; + raxTryInsert(t,(unsigned char*)"FOO",3,(void*)(long)2,&old); + if (old != (void*)(long)1) { + printf("Old value not returned correctly by raxTryInsert(): %p", + old); + return 1; + } + + val = raxFind(t,(unsigned char*)"FOO",3); + if (val != (void*)(long)1) { + printf("FOO value mismatch: is %p intead of 1", val); + return 1; + } + + raxInsert(t,(unsigned char*)"FOO",3,(void*)(long)2,NULL); + val = raxFind(t,(unsigned char*)"FOO",3); + if (val != (void*)(long)2) { + printf("FOO value mismatch: is %p intead of 2", val); + return 1; + } + + raxFree(t); + return 0; +} + +/* Regression test #1: Iterator wrong element returned after seek. */ +int regtest1(void) { + rax *rax = raxNew(); + raxInsert(rax,(unsigned char*)"LKE",3,(void*)(long)1,NULL); + raxInsert(rax,(unsigned char*)"TQ",2,(void*)(long)2,NULL); + raxInsert(rax,(unsigned char*)"B",1,(void*)(long)3,NULL); + raxInsert(rax,(unsigned char*)"FY",2,(void*)(long)4,NULL); + raxInsert(rax,(unsigned char*)"WI",2,(void*)(long)5,NULL); + + raxIterator iter; + raxStart(&iter,rax); + raxSeek(&iter,">",(unsigned char*)"FMP",3); + if (raxNext(&iter)) { + if (iter.key_len != 2 || + memcmp(iter.key,"FY",2)) + { + printf("Regression test 1 failed: 'FY' expected, got: '%.*s'\n", + (int)iter.key_len, (char*)iter.key); + return 1; + } + } + + raxStop(&iter); + raxFree(rax); + return 0; +} + +/* Regression test #2: Crash when mixing NULL and not NULL values. */ +int regtest2(void) { + rax *rt = raxNew(); + raxInsert(rt,(unsigned char *)"a",1,(void *)100,NULL); + raxInsert(rt,(unsigned char *)"ab",2,(void *)101,NULL); + raxInsert(rt,(unsigned char *)"abc",3,(void *)NULL,NULL); + raxInsert(rt,(unsigned char *)"abcd",4,(void *)NULL,NULL); + raxInsert(rt,(unsigned char *)"abc",3,(void *)102,NULL); + raxFree(rt); + return 0; +} + +/* Regression test #3: Wrong access at node value in raxRemoveChild() + * when iskey == 1 and isnull == 1: the memmove() was performed including + * the value length regardless of the fact there was no actual value. + * + * Note that this test always returns success but will trigger a + * Valgrind error. */ +int regtest3(void) { + rax *rt = raxNew(); + raxInsert(rt, (unsigned char *)"D",1,(void*)1,NULL); + raxInsert(rt, (unsigned char *)"",0,NULL,NULL); + raxRemove(rt, (unsigned char *)"D",1,NULL); + raxFree(rt); + return 0; +} + +/* Regression test #4: Github issue #8, iterator does not populate the + * data field after seek in case of exact match. The test case is looks odd + * because it is quite indirect: Seeking "^" will result into seeking + * the element >= "", and since we just added "" an exact match happens, + * however we are using the original one from the bug report, since this + * is quite odd and may later protect against different bugs related to + * storing and fetching the empty string key. */ +int regtest4(void) { + rax *rt = raxNew(); + raxIterator iter; + raxInsert(rt, (unsigned char*)"", 0, (void *)-1, NULL); + if (raxFind(rt, (unsigned char*)"", 0) != (void *)-1) { + printf("Regression test 4 failed. Key value mismatch in raxFind()\n"); + return 1; + } + raxStart(&iter,rt); + raxSeek(&iter, "^", NULL, 0); + raxNext(&iter); + if (iter.data != (void *)-1) { + printf("Regression test 4 failed. Key value mismatch in raxNext()\n"); + return 1; + } + raxStop(&iter); + raxFree(rt); + return 0; +} + +/* Less than seek bug when stopping in the middle of a compressed node. */ +int regtest5(void) { + rax *rax = raxNew(); + + raxInsert(rax,(unsigned char*)"b",1,(void*)(long)1,NULL); + raxInsert(rax,(unsigned char*)"ba",2,(void*)(long)2,NULL); + raxInsert(rax,(unsigned char*)"banana",6,(void*)(long)3,NULL); + + raxInsert(rax,(unsigned char*)"f",1,(void*)(long)4,NULL); + raxInsert(rax,(unsigned char*)"foobar",6,(void*)(long)5,NULL); + raxInsert(rax,(unsigned char*)"foobar123",9,(void*)(long)6,NULL); + + raxIterator ri; + raxStart(&ri,rax); + + raxSeek(&ri,"<",(unsigned char*)"foo",3); + raxNext(&ri); + if (ri.key_len != 1 || ri.key[0] != 'f') { + printf("Regression test 4 failed. Key value mismatch in raxNext()\n"); + return 1; + } + + raxStop(&ri); + raxFree(rax); + return 0; +} + +/* Seek may not populate iterator data. See issue #25. */ +int regtest6(void) { + rax *rax = raxNew(); + + char *key1 = "172.17.141.2/adminguide/v5.0/"; + char *key2 = "172.17.141.2/adminguide/v5.0/entitlements-configure.html"; + char *seekpoint = "172.17.141.2/adminguide/v5.0/entitlements"; + + raxInsert(rax, (unsigned char*)key1,strlen(key1),(void*)(long)1234, NULL); + raxInsert(rax, (unsigned char*)key2,strlen(key2),(void*)(long)5678, NULL); + + raxIterator ri; + raxStart(&ri,rax); + raxSeek(&ri,"<=", (unsigned char*)seekpoint, strlen(seekpoint)); + raxPrev(&ri); + if ((long)ri.data != 1234) { + printf("Regression test 6 failed. Key data not populated.\n"); + return 1; + } + + raxStop(&ri); + raxFree(rax); + return 0; +} + +void benchmark(void) { + for (int mode = 0; mode < 2; mode++) { + printf("Benchmark with %s keys:\n", + (mode == 0) ? "integer" : "alphanumerical"); + rax *t = raxNew(); + long long start = ustime(); + for (int i = 0; i < 5000000; i++) { + char buf[64]; + int len = int2key(buf,sizeof(buf),i,mode); + raxInsert(t,(unsigned char*)buf,len,(void*)(long)i,NULL); + } + printf("Insert: %f\n", (double)(ustime()-start)/1000000); + printf("%llu total nodes\n", (unsigned long long)t->numnodes); + printf("%llu total elements\n", (unsigned long long)t->numele); + + start = ustime(); + for (int i = 0; i < 5000000; i++) { + char buf[64]; + int len = int2key(buf,sizeof(buf),i,mode); + void *data = raxFind(t,(unsigned char*)buf,len); + if (data != (void*)(long)i) { + printf("Issue with %s: %p instead of %p\n", buf, + data, (void*)(long)i); + } + } + printf("Linear lookup: %f\n", (double)(ustime()-start)/1000000); + + start = ustime(); + for (int i = 0; i < 5000000; i++) { + char buf[64]; + int r = rc4rand() % 5000000; + int len = int2key(buf,sizeof(buf),r,mode); + void *data = raxFind(t,(unsigned char*)buf,len); + if (data != (void*)(long)r) { + printf("Issue with %s: %p instead of %p\n", buf, + data, (void*)(long)r); + } + } + printf("Random lookup: %f\n", (double)(ustime()-start)/1000000); + + start = ustime(); + for (int i = 0; i < 5000000; i++) { + char buf[64]; + int len = int2key(buf,sizeof(buf),i,mode); + buf[i%len] = '!'; /* "!" is never set into keys. */ + void *data = raxFind(t,(unsigned char*) buf,len); + if (data != raxNotFound) { + printf("** Failed lookup did not reported NOT FOUND!\n"); + } + } + printf("Failed lookup: %f\n", (double)(ustime()-start)/1000000); + + start = ustime(); + raxIterator ri; + raxStart(&ri,t); + raxSeek(&ri,"^",NULL,0); + int iter = 0; + while (raxNext(&ri)) iter++; + if (iter != 5000000) printf("** Warning iteration is incomplete\n"); + raxStop(&ri); + printf("Full iteration: %f\n", (double)(ustime()-start)/1000000); + + start = ustime(); + for (int i = 0; i < 5000000; i++) { + char buf[64]; + int len = int2key(buf,sizeof(buf),i,mode); + int retval = raxRemove(t,(unsigned char*)buf,len,NULL); + assert(retval == 1); + } + printf("Deletion: %f\n", (double)(ustime()-start)/1000000); + + printf("%llu total nodes\n", (unsigned long long)t->numnodes); + printf("%llu total elements\n", (unsigned long long)t->numele); + raxFree(t); + } +} + +/* Compressed nodes can only hold (2^29)-1 characters, so it is important + * to test for keys bigger than this amount, in order to make sure that + * the code to handle this edge case works as expected. + * + * This test is disabled by default because it uses a lot of memory. */ +int testHugeKey(void) { + size_t max_keylen = ((1<<29)-1) + 100; + unsigned char *key = malloc(max_keylen); + if (key == NULL) goto oom; + + memset(key,'a',max_keylen); + key[10] = 'X'; + key[max_keylen-1] = 'Y'; + rax *rax = raxNew(); + int retval = raxInsert(rax,(unsigned char*)"aaabbb",6,(void*)5678L,NULL); + if (retval == 0 && errno == ENOMEM) goto oom; + retval = raxInsert(rax,key,max_keylen,(void*)1234L,NULL); + if (retval == 0 && errno == ENOMEM) goto oom; + void *value1 = raxFind(rax,(unsigned char*)"aaabbb",6); + void *value2 = raxFind(rax,key,max_keylen); + if (value1 != (void*)5678L || value2 != (void*)1234L) { + printf("Huge key test failed\n"); + return 1; + } + raxFree(rax); + return 0; + +oom: + fprintf(stderr,"Sorry, not enough memory to execute --hugekey test."); + exit(1); +} + +int raxTest(int argc, char **argv) { + rc4srand(1234); + + /* Tests to run by default are set here. */ + int do_benchmark = 0; + int do_units = 1; + int do_fuzz_cluster = 0; + int do_fuzz = 1; + int do_regression = 1; + int do_hugekey = 0; + + /* If the user passed arguments, override the tests to run. */ + if (argc > 1) { + do_benchmark = 0; + do_units = 0; + do_fuzz = 0; + do_regression = 0; + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i],"--bench")) { + do_benchmark = 1; + } else if (!strcmp(argv[i],"--fuzz-cluster")) { + do_fuzz_cluster = 1; + } else if (!strcmp(argv[i],"--fuzz")) { + do_fuzz = 1; + } else if (!strcmp(argv[i],"--units")) { + do_units = 1; + } else if (!strcmp(argv[i],"--regression")) { + do_regression = 1; + } else if (!strcmp(argv[i],"--hugekey")) { + do_hugekey = 1; + } else { + fprintf(stderr, "Usage: %s :\n" + " [--bench (default off)]\n" + " [--fuzz-cluster] (default off)\n" + " [--fuzz] (default on)\n" + " [--units] (default on)\n" + " [--regression] (default on)\n" + " [--hugekey (default off)]\n" + "Without options all the default tests will\n" + "be executed.\n", + argv[0]); + exit(1); + } + } + } + + int errors = 0; + + if (do_units) { + printf("Unit tests: "); fflush(stdout); + if (randomWalkTest()) errors++; + if (iteratorUnitTests()) errors++; + if (tryInsertUnitTests()) errors++; + if (errors == 0) printf("OK\n"); + } + + if (do_regression) { + printf("Performing regression tests: "); fflush(stdout); + if (regtest1()) errors++; + if (regtest2()) errors++; + if (regtest3()) errors++; + if (regtest4()) errors++; + if (regtest5()) errors++; + if (regtest6()) errors++; + if (errors == 0) printf("OK\n"); + } + + if (do_hugekey) { + printf("Performing huge key tests: "); fflush(stdout); + if (testHugeKey()) errors++; + } + + if (do_fuzz_cluster) { + for (int i = 0; i < 10; i++) { + double alpha = (double)rc4rand() / RAND_MAX; + double beta = 1-alpha; + if (fuzzTestCluster(rc4rand()%100000000,alpha,beta)) errors++; + } + } + + if (do_fuzz) { + for (int i = 0; i < 10; i++) { + double alpha = (double)rc4rand() / RAND_MAX; + double beta = 1-alpha; + if (fuzzTest(KEY_INT,rc4rand()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA,rc4rand()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM,rc4rand()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA,rc4rand()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET,rc4rand()%10000,alpha,beta)) errors++; + } + + size_t numops = 100000, cycles = 3; + while(cycles--) { + if (fuzzTest(KEY_INT,numops,.7,.3)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA,numops,.7,.3)) errors++; + if (fuzzTest(KEY_RANDOM,numops,.7,.3)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA,numops,.7,.3)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET,numops,.7,.3)) errors++; + numops *= 10; + } + + if (fuzzTest(KEY_CHAIN,1000,.7,.3)) errors++; + printf("Iterator fuzz test: "); fflush(stdout); + for (int i = 0; i < 100000; i++) { + if (iteratorFuzzTest(KEY_INT,100)) errors++; + if (iteratorFuzzTest(KEY_UNIQUE_ALPHA,100)) errors++; + if (iteratorFuzzTest(KEY_RANDOM_ALPHA,1000)) errors++; + if (iteratorFuzzTest(KEY_RANDOM,1000)) errors++; + if (i && !(i % 100)) { + printf("."); + if (!(i % 1000)) { + printf("%d%% done",i/1000); + } + fflush(stdout); + } + } + printf("\n"); + } + + if (do_benchmark) { + benchmark(); + } + + if (errors) { + printf("!!! WARNING !!!: %d errors found\n", errors); + } else { + printf("OK! \\o/\n"); + } + return errors; +} + +#endif // REDIS_TEST diff --git a/src/server.c b/src/server.c index fe522b3e5d..17b3e22ce7 100644 --- a/src/server.c +++ b/src/server.c @@ -6595,6 +6595,7 @@ int iAmPrimary(void) { #ifdef SERVER_TEST #include "testhelp.h" #include "intset.h" /* Compact integer set structure */ +#include "rax-test.c" int __failed_tests = 0; int __test_num = 0; @@ -6612,6 +6613,7 @@ struct serverTest { {"zipmap", zipmapTest}, {"dict", dictTest}, {"listpack", listpackTest}, + {"rax", raxTest}, }; serverTestProc *getTestProcByName(const char *name) { int numtests = sizeof(serverTests) / sizeof(struct serverTest); From 9ac6f88eff9f30b9a7c4ad2aacd247d7a596c5b9 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 31 Jan 2022 18:21:31 -0800 Subject: [PATCH 05/26] replace rc4rand with twin prime generator Signed-off-by: Guillaume Koenig --- src/rax-test.c | 62 +++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/rax-test.c b/src/rax-test.c index 9207bf803a..c286ec7434 100644 --- a/src/rax-test.c +++ b/src/rax-test.c @@ -39,7 +39,7 @@ #include #include "rax.h" -#include "rc4rand.h" +#include "mt19937-64.h" uint16_t crc16(const char *buf, int len); /* From crc16.c */ @@ -227,18 +227,18 @@ static size_t int2key(char *s, size_t maxlen, uint32_t i, int mode) { return int2alphakey(s,maxlen,i); } else if (mode == KEY_RANDOM) { if (maxlen > 16) maxlen = 16; - int r = rc4rand() % maxlen; - for (int i = 0; i < r; i++) s[i] = rc4rand()&0xff; + int r = genrand64_int64() % maxlen; + for (int i = 0; i < r; i++) s[i] = genrand64_int64()&0xff; return r; } else if (mode == KEY_RANDOM_ALPHA) { if (maxlen > 16) maxlen = 16; - int r = rc4rand() % maxlen; - for (int i = 0; i < r; i++) s[i] = 'A'+rc4rand()%('z'-'A'+1); + int r = genrand64_int64() % maxlen; + for (int i = 0; i < r; i++) s[i] = 'A'+genrand64_int64()%('z'-'A'+1); return r; } else if (mode == KEY_RANDOM_SMALL_CSET) { if (maxlen > 16) maxlen = 16; - int r = rc4rand() % maxlen; - for (int i = 0; i < r; i++) s[i] = 'A'+rc4rand()%4; + int r = genrand64_int64() % maxlen; + for (int i = 0; i < r; i++) s[i] = 'A'+genrand64_int64()%4; return r; } else if (mode == KEY_CHAIN) { if (i > maxlen) i = maxlen; @@ -265,11 +265,11 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { uint32_t keylen; /* Insert element. */ - if ((double)rc4rand()/RAND_MAX < addprob) { + if ((double)genrand64_int64()/RAND_MAX < addprob) { keylen = int2key((char*)key,sizeof(key),i,keymode); - void *val = (void*)(unsigned long)rc4rand(); + void *val = (void*)(unsigned long)genrand64_int64(); /* Stress NULL values more often, they use a special encoding. */ - if (!(rc4rand() % 100)) val = NULL; + if (!(genrand64_int64() % 100)) val = NULL; int retval1 = htAdd(ht,key,keylen,val); int retval2 = raxInsert(rax,key,keylen,val,NULL); if (retval1 != retval2) { @@ -279,7 +279,7 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { } /* Remove element. */ - if ((double)rc4rand()/RAND_MAX < remprob) { + if ((double)genrand64_int64()/RAND_MAX < remprob) { keylen = int2key((char*)key,sizeof(key),i,keymode); int retval1 = htRem(ht,key,keylen); int retval2 = raxRemove(rax,key,keylen,NULL); @@ -364,12 +364,12 @@ int fuzzTestCluster(size_t count, double addprob, double remprob) { /* With a given probability, let's use a common prefix so that there * is a subset of keys that have an higher percentage of probability * of being hit again and again. */ - size_t commonprefix = rc4rand() & 0xf; + size_t commonprefix = genrand64_int64() & 0xf; if (commonprefix == 0) memcpy(key+10,"2e68e5",6); /* Alter a random char in the key. */ - int pos = 10+rc4rand()%12; - key[pos] = cset[rc4rand()%16]; + int pos = 10+genrand64_int64()%12; + key[pos] = cset[genrand64_int64()%16]; /* Compute the Redis Cluster hash slot to set the first two * binary bytes of the key. */ @@ -378,12 +378,12 @@ int fuzzTestCluster(size_t count, double addprob, double remprob) { key[1] = hashslot & 0xff; /* Insert element. */ - if ((double)rc4rand()/RAND_MAX < addprob) { + if ((double)genrand64_int64()/RAND_MAX < addprob) { raxInsert(rax,key,keylen,NULL,NULL); } /* Remove element. */ - if ((double)rc4rand()/RAND_MAX < remprob) { + if ((double)genrand64_int64()/RAND_MAX < remprob) { raxRemove(rax,key,keylen,NULL); } } @@ -446,7 +446,7 @@ int arraySeek(arrayItem *array, int count, unsigned char *key, size_t len, char } int iteratorFuzzTest(int keymode, size_t count) { - count = rc4rand()%count; + count = genrand64_int64()%count; rax *rax = raxNew(); arrayItem *array = malloc(sizeof(arrayItem)*count); @@ -471,15 +471,15 @@ int iteratorFuzzTest(int keymode, size_t count) { /* Perform a random seek operation. */ uint32_t keylen = int2key((char*)key,sizeof(key), - rc4rand()%(count ? count : 1),keymode); + genrand64_int64()%(count ? count : 1),keymode); raxIterator iter; raxStart(&iter,rax); char *seekops[] = {"==",">=","<=",">","<","^","$"}; - char *seekop = seekops[rc4rand() % 7]; + char *seekop = seekops[genrand64_int64() % 7]; raxSeek(&iter,seekop,key,keylen); int seekidx = arraySeek(array,count,key,keylen,seekop); - int next = rc4rand() % 2; + int next = genrand64_int64() % 2; int iteration = 0; while(1) { int rax_res; @@ -596,7 +596,7 @@ int iteratorUnitTests(void) { rax *t = raxNew(); char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; - for (int x = 0; x < 10000; x++) rc4rand(); + for (int x = 0; x < 10000; x++) genrand64_int64(); long items = 0; while(toadd[items] != NULL) items++; @@ -855,7 +855,7 @@ void benchmark(void) { start = ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; - int r = rc4rand() % 5000000; + int r = genrand64_int64() % 5000000; int len = int2key(buf,sizeof(buf),r,mode); void *data = raxFind(t,(unsigned char*)buf,len); if (data != (void*)(long)r) { @@ -935,7 +935,7 @@ int testHugeKey(void) { } int raxTest(int argc, char **argv) { - rc4srand(1234); + init_genrand64(1234); /* Tests to run by default are set here. */ int do_benchmark = 0; @@ -1009,21 +1009,21 @@ int raxTest(int argc, char **argv) { if (do_fuzz_cluster) { for (int i = 0; i < 10; i++) { - double alpha = (double)rc4rand() / RAND_MAX; + double alpha = (double)genrand64_int64() / RAND_MAX; double beta = 1-alpha; - if (fuzzTestCluster(rc4rand()%100000000,alpha,beta)) errors++; + if (fuzzTestCluster(genrand64_int64()%100000000,alpha,beta)) errors++; } } if (do_fuzz) { for (int i = 0; i < 10; i++) { - double alpha = (double)rc4rand() / RAND_MAX; + double alpha = (double)genrand64_int64() / RAND_MAX; double beta = 1-alpha; - if (fuzzTest(KEY_INT,rc4rand()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_UNIQUE_ALPHA,rc4rand()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM,rc4rand()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM_ALPHA,rc4rand()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM_SMALL_CSET,rc4rand()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_INT,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET,genrand64_int64()%10000,alpha,beta)) errors++; } size_t numops = 100000, cycles = 3; From 2de1d2e3def423da9a03728bfe166e4cd574b718 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 31 Jan 2022 18:26:25 -0800 Subject: [PATCH 06/26] Replace usage of libc alloc fn calls with zmalloc calls Signed-off-by: Guillaume Koenig --- src/rax-test.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/rax-test.c b/src/rax-test.c index c286ec7434..9ea36155b5 100644 --- a/src/rax-test.c +++ b/src/rax-test.c @@ -40,6 +40,7 @@ #include "rax.h" #include "mt19937-64.h" +#include "zmalloc.h" uint16_t crc16(const char *buf, int len); /* From crc16.c */ @@ -64,7 +65,7 @@ typedef struct ht { /* Create a new hash table. */ hashtable *htNew(void) { - hashtable *ht = calloc(1,sizeof(*ht)); + hashtable *ht = zcalloc(sizeof(*ht)); ht->numele = 0; return ht; } @@ -98,8 +99,8 @@ int htAdd(hashtable *t, unsigned char *s, size_t len, void *data) { htNode *n = htRawLookup(t,s,len,&hash,NULL); if (!n) { - n = malloc(sizeof(*n)); - n->key = malloc(len); + n = zmalloc(sizeof(*n)); + n->key = zmalloc(len); memcpy(n->key,s,len); n->keylen = len; n->data = data; @@ -121,8 +122,8 @@ int htRem(hashtable *t, unsigned char *s, size_t len) { if (!n) return 0; *parentlink = n->next; - free(n->key); - free(n); + zfree(n->key); + zfree(n); t->numele--; return 1; } @@ -144,11 +145,11 @@ void htFree(hashtable *ht) { while(next) { htNode *this = next; next = this->next; - free(this->key); - free(this); + zfree(this->key); + zfree(this); } } - free(ht); + zfree(ht); } /* -------------------------------------------------------------------------- @@ -448,7 +449,7 @@ int arraySeek(arrayItem *array, int count, unsigned char *key, size_t len, char int iteratorFuzzTest(int keymode, size_t count) { count = genrand64_int64()%count; rax *rax = raxNew(); - arrayItem *array = malloc(sizeof(arrayItem)*count); + arrayItem *array = zmalloc(sizeof(arrayItem)*count); /* Fill a radix tree and a linear array with some data. */ unsigned char key[1024]; @@ -458,7 +459,7 @@ int iteratorFuzzTest(int keymode, size_t count) { void *val = (void*)(unsigned long)htHash(key,keylen); if (raxInsert(rax,key,keylen,val,NULL)) { - array[j].key = malloc(keylen); + array[j].key = zmalloc(keylen); array[j].key_len = keylen; memcpy(array[j].key,key,keylen); j++; @@ -544,8 +545,8 @@ int iteratorFuzzTest(int keymode, size_t count) { iteration++; } - for (unsigned int i = 0; i < count; i++) free(array[i].key); - free(array); + for (unsigned int i = 0; i < count; i++) zfree(array[i].key); + zfree(array); raxStop(&iter); raxFree(rax); return 0; @@ -909,7 +910,7 @@ void benchmark(void) { * This test is disabled by default because it uses a lot of memory. */ int testHugeKey(void) { size_t max_keylen = ((1<<29)-1) + 100; - unsigned char *key = malloc(max_keylen); + unsigned char *key = zmalloc(max_keylen); if (key == NULL) goto oom; memset(key,'a',max_keylen); From 4909a1f9d590f5f414bddb9d8844b94ed10f0096 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 31 Jan 2022 18:32:37 -0800 Subject: [PATCH 07/26] Remove time function, remove argv parsing code Signed-off-by: Guillaume Koenig --- src/rax-test.c | 54 +++++++------------------------------------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/src/rax-test.c b/src/rax-test.c index 9ea36155b5..ae9cddc734 100644 --- a/src/rax-test.c +++ b/src/rax-test.c @@ -193,16 +193,6 @@ static size_t int2alphakey(char *s, size_t maxlen, uint32_t i) { return len; } -/* Return the UNIX time in microseconds */ -long long ustime(void) { - struct timeval tv; - long long ust; - - gettimeofday(&tv, NULL); - ust = ((long long)tv.tv_sec)*1000000; - ust += tv.tv_usec; - return ust; -} /* Turn the integer 'i' into a key according to 'mode'. * KEY_INT: Just represents the integer as a string. @@ -935,8 +925,13 @@ int testHugeKey(void) { exit(1); } -int raxTest(int argc, char **argv) { - init_genrand64(1234); +int raxTest(int argc, char **argv, int flags) { + /* If an argument is given, use it as the random seed. */ + if (argc >= 4) { + init_genrand64(atoi(argv[3])); + } else { + init_genrand64(1234); + } /* Tests to run by default are set here. */ int do_benchmark = 0; @@ -946,41 +941,6 @@ int raxTest(int argc, char **argv) { int do_regression = 1; int do_hugekey = 0; - /* If the user passed arguments, override the tests to run. */ - if (argc > 1) { - do_benchmark = 0; - do_units = 0; - do_fuzz = 0; - do_regression = 0; - - for (int i = 1; i < argc; i++) { - if (!strcmp(argv[i],"--bench")) { - do_benchmark = 1; - } else if (!strcmp(argv[i],"--fuzz-cluster")) { - do_fuzz_cluster = 1; - } else if (!strcmp(argv[i],"--fuzz")) { - do_fuzz = 1; - } else if (!strcmp(argv[i],"--units")) { - do_units = 1; - } else if (!strcmp(argv[i],"--regression")) { - do_regression = 1; - } else if (!strcmp(argv[i],"--hugekey")) { - do_hugekey = 1; - } else { - fprintf(stderr, "Usage: %s :\n" - " [--bench (default off)]\n" - " [--fuzz-cluster] (default off)\n" - " [--fuzz] (default on)\n" - " [--units] (default on)\n" - " [--regression] (default on)\n" - " [--hugekey (default off)]\n" - "Without options all the default tests will\n" - "be executed.\n", - argv[0]); - exit(1); - } - } - } int errors = 0; From eeb0f2eec6e91aac2fcb607acb16f8a94066b924 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 31 Jan 2022 18:46:08 -0800 Subject: [PATCH 08/26] Add features of the unit test as flags Signed-off-by: Guillaume Koenig --- src/rax-test.c | 4 ++++ src/server.c | 6 ++++++ src/testhelp.h | 3 +++ 3 files changed, 13 insertions(+) diff --git a/src/rax-test.c b/src/rax-test.c index ae9cddc734..6aca403c3f 100644 --- a/src/rax-test.c +++ b/src/rax-test.c @@ -941,6 +941,10 @@ int raxTest(int argc, char **argv, int flags) { int do_regression = 1; int do_hugekey = 0; + if (flags & REDIS_TEST_BENCHMARK) do_benchmark = 1; + if (flags & REDIS_TEST_FUZZ_CLUSTER) do_fuzz_cluster = 1; + if (flags & REDIS_TEST_HUGE_KEY) do_hugekey = 1; + int errors = 0; diff --git a/src/server.c b/src/server.c index 17b3e22ce7..974145e70e 100644 --- a/src/server.c +++ b/src/server.c @@ -6643,6 +6643,12 @@ int main(int argc, char **argv) { flags |= TEST_LARGE_MEMORY; else if (!strcasecmp(arg, "--valgrind")) flags |= TEST_VALGRIND; + else if (!strcasecmp(arg, "--benchmark")) + flags |= TEST_BENCHMARK; + else if (!strcasecmp(arg, "--fuzz-cluster")) + flags |= TEST_FUZZ_CLUSTER; + else if (!strcasecmp(arg, "--huge-key")) + flags |= TEST_HUGE_KEY; } if (!strcasecmp(argv[2], "all")) { diff --git a/src/testhelp.h b/src/testhelp.h index 07c88cecd8..dc5f32f1a6 100644 --- a/src/testhelp.h +++ b/src/testhelp.h @@ -42,6 +42,9 @@ #define TEST_ACCURATE (1 << 0) #define TEST_LARGE_MEMORY (1 << 1) #define TEST_VALGRIND (1 << 2) +#define TEST_BENCHMARK (1 << 3) +#define TEST_FUZZ_CLUSTER (1 << 4) +#define TEST_HUGE_KEY (1 << 5) extern int __failed_tests; extern int __test_num; From 88cbb16b194b7aad639207a207a72e15db1ede68 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Tue, 25 Jun 2024 21:16:05 +0000 Subject: [PATCH 09/26] Fix rax-test so that it compiles and tests pass Signed-off-by: Guillaume Koenig --- src/rax-test.c | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/rax-test.c b/src/rax-test.c index 6aca403c3f..79c56739a2 100644 --- a/src/rax-test.c +++ b/src/rax-test.c @@ -28,7 +28,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifdef REDIS_TEST +#ifdef SERVER_TEST #include #include @@ -278,8 +278,6 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { printf("Fuzz: key deletion of '%.*s' reported mismatching " "value in HT=%d RAX=%d\n", (int)keylen,(char*)key,retval1, retval2); - printf("%p\n", raxFind(rax,key,keylen)); - printf("%p\n", raxNotFound); return 1; } } @@ -302,7 +300,8 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { size_t numkeys = 0; while(raxNext(&iter)) { void *val1 = htFind(ht,iter.key,iter.key_len); - void *val2 = raxFind(rax,iter.key,iter.key_len); + void *val2 = NULL; + raxFind(rax,iter.key,iter.key_len,&val2); if (val1 != val2) { printf("Fuzz: HT=%p, RAX=%p value do not match " "for key %.*s\n", @@ -666,14 +665,16 @@ int tryInsertUnitTests(void) { return 1; } - val = raxFind(t,(unsigned char*)"FOO",3); + val = NULL; + raxFind(t,(unsigned char*)"FOO",3,&val); if (val != (void*)(long)1) { printf("FOO value mismatch: is %p intead of 1", val); return 1; } raxInsert(t,(unsigned char*)"FOO",3,(void*)(long)2,NULL); - val = raxFind(t,(unsigned char*)"FOO",3); + val = NULL; + raxFind(t,(unsigned char*)"FOO",3,&val); if (val != (void*)(long)2) { printf("FOO value mismatch: is %p intead of 2", val); return 1; @@ -748,7 +749,9 @@ int regtest4(void) { rax *rt = raxNew(); raxIterator iter; raxInsert(rt, (unsigned char*)"", 0, (void *)-1, NULL); - if (raxFind(rt, (unsigned char*)"", 0) != (void *)-1) { + void *val = NULL; + raxFind(rt, (unsigned char*)"", 0, &val); + if (val != (void *)-1) { printf("Regression test 4 failed. Key value mismatch in raxFind()\n"); return 1; } @@ -835,8 +838,8 @@ void benchmark(void) { for (int i = 0; i < 5000000; i++) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); - void *data = raxFind(t,(unsigned char*)buf,len); - if (data != (void*)(long)i) { + void *data; + if (!raxFind(t,(unsigned char*)buf,len,&data) || data != (void*)(long)i) { printf("Issue with %s: %p instead of %p\n", buf, data, (void*)(long)i); } @@ -848,8 +851,8 @@ void benchmark(void) { char buf[64]; int r = genrand64_int64() % 5000000; int len = int2key(buf,sizeof(buf),r,mode); - void *data = raxFind(t,(unsigned char*)buf,len); - if (data != (void*)(long)r) { + void *data; + if (!raxFind(t,(unsigned char*)buf,len,&data) || data != (void*)(long)r) { printf("Issue with %s: %p instead of %p\n", buf, data, (void*)(long)r); } @@ -861,8 +864,7 @@ void benchmark(void) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); buf[i%len] = '!'; /* "!" is never set into keys. */ - void *data = raxFind(t,(unsigned char*) buf,len); - if (data != raxNotFound) { + if (!raxFind(t,(unsigned char*) buf,len,NULL)) { printf("** Failed lookup did not reported NOT FOUND!\n"); } } @@ -911,8 +913,13 @@ int testHugeKey(void) { if (retval == 0 && errno == ENOMEM) goto oom; retval = raxInsert(rax,key,max_keylen,(void*)1234L,NULL); if (retval == 0 && errno == ENOMEM) goto oom; - void *value1 = raxFind(rax,(unsigned char*)"aaabbb",6); - void *value2 = raxFind(rax,key,max_keylen); + void *value1, *value2; + int found1 = raxFind(rax,(unsigned char*)"aaabbb",6,&value1); + int found2 = raxFind(rax,key,max_keylen,&value2); + if (!found1 || !found2) { + printf("Huge key test failed on elementhood\n"); + return 1; + } if (value1 != (void*)5678L || value2 != (void*)1234L) { printf("Huge key test failed\n"); return 1; @@ -941,10 +948,9 @@ int raxTest(int argc, char **argv, int flags) { int do_regression = 1; int do_hugekey = 0; - if (flags & REDIS_TEST_BENCHMARK) do_benchmark = 1; - if (flags & REDIS_TEST_FUZZ_CLUSTER) do_fuzz_cluster = 1; - if (flags & REDIS_TEST_HUGE_KEY) do_hugekey = 1; - + if (flags & TEST_BENCHMARK) do_benchmark = 1; + if (flags & TEST_FUZZ_CLUSTER) do_fuzz_cluster = 1; + if (flags & TEST_HUGE_KEY) do_hugekey = 1; int errors = 0; @@ -1031,4 +1037,4 @@ int raxTest(int argc, char **argv, int flags) { return errors; } -#endif // REDIS_TEST +#endif // SERVER_TEST From f66ba889423eb7d40b17f8bac19e804963c73e28 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Thu, 27 Jun 2024 17:26:35 -0400 Subject: [PATCH 10/26] Move rax-test.c under src/unit/test_rax.c Signed-off-by: Guillaume Koenig --- src/server.c | 8 -------- src/testhelp.h | 3 --- src/{rax-test.c => unit/test_rax.c} | 0 3 files changed, 11 deletions(-) rename src/{rax-test.c => unit/test_rax.c} (100%) diff --git a/src/server.c b/src/server.c index 974145e70e..fe522b3e5d 100644 --- a/src/server.c +++ b/src/server.c @@ -6595,7 +6595,6 @@ int iAmPrimary(void) { #ifdef SERVER_TEST #include "testhelp.h" #include "intset.h" /* Compact integer set structure */ -#include "rax-test.c" int __failed_tests = 0; int __test_num = 0; @@ -6613,7 +6612,6 @@ struct serverTest { {"zipmap", zipmapTest}, {"dict", dictTest}, {"listpack", listpackTest}, - {"rax", raxTest}, }; serverTestProc *getTestProcByName(const char *name) { int numtests = sizeof(serverTests) / sizeof(struct serverTest); @@ -6643,12 +6641,6 @@ int main(int argc, char **argv) { flags |= TEST_LARGE_MEMORY; else if (!strcasecmp(arg, "--valgrind")) flags |= TEST_VALGRIND; - else if (!strcasecmp(arg, "--benchmark")) - flags |= TEST_BENCHMARK; - else if (!strcasecmp(arg, "--fuzz-cluster")) - flags |= TEST_FUZZ_CLUSTER; - else if (!strcasecmp(arg, "--huge-key")) - flags |= TEST_HUGE_KEY; } if (!strcasecmp(argv[2], "all")) { diff --git a/src/testhelp.h b/src/testhelp.h index dc5f32f1a6..07c88cecd8 100644 --- a/src/testhelp.h +++ b/src/testhelp.h @@ -42,9 +42,6 @@ #define TEST_ACCURATE (1 << 0) #define TEST_LARGE_MEMORY (1 << 1) #define TEST_VALGRIND (1 << 2) -#define TEST_BENCHMARK (1 << 3) -#define TEST_FUZZ_CLUSTER (1 << 4) -#define TEST_HUGE_KEY (1 << 5) extern int __failed_tests; extern int __test_num; diff --git a/src/rax-test.c b/src/unit/test_rax.c similarity index 100% rename from src/rax-test.c rename to src/unit/test_rax.c From 77f1b073daac473169033f0eda98887a1d122c04 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Thu, 27 Jun 2024 17:27:32 -0400 Subject: [PATCH 11/26] Adapt rax tests to the new unit test framework Signed-off-by: Guillaume Koenig --- src/unit/test_files.h | 14 +++ src/unit/test_rax.c | 231 +++++++++++++++++++++--------------------- 2 files changed, 131 insertions(+), 114 deletions(-) diff --git a/src/unit/test_files.h b/src/unit/test_files.h index a087e6fe44..ad85570e3b 100644 --- a/src/unit/test_files.h +++ b/src/unit/test_files.h @@ -23,6 +23,18 @@ int test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyDict(int argc, char **argv, in int test_kvstoreIteratorRemoveAllKeysDeleteEmptyDict(int argc, char **argv, int flags); int test_kvstoreDictIteratorRemoveAllKeysNoDeleteEmptyDict(int argc, char **argv, int flags); int test_kvstoreDictIteratorRemoveAllKeysDeleteEmptyDict(int argc, char **argv, int flags); +int test_randomWalkTest(int argc, char **argv, int flags); +int test_iteratorUnitTests(int argc, char **argv, int flags); +int test_tryInsertUnitTests(int argc, char **argv, int flags); +int test_regtest1(int argc, char **argv, int flags); +int test_regtest2(int argc, char **argv, int flags); +int test_regtest3(int argc, char **argv, int flags); +int test_regtest4(int argc, char **argv, int flags); +int test_regtest5(int argc, char **argv, int flags); +int test_regtest6(int argc, char **argv, int flags); +int test_benchmark(int argc, char **argv, int flags); +int test_hugeKey(int argc, char **argv, int flags); +int test_fuzz(int argc, char **argv, int flags); int test_sds(int argc, char **argv, int flags); int test_typesAndAllocSize(int argc, char **argv, int flags); int test_sdsHeaderSizes(int argc, char **argv, int flags); @@ -79,6 +91,7 @@ unitTest __test_crc64combine_c[] = {{"test_crc64combine", test_crc64combine}, {N unitTest __test_endianconv_c[] = {{"test_endianconv", test_endianconv}, {NULL, NULL}}; unitTest __test_intset_c[] = {{"test_intsetValueEncodings", test_intsetValueEncodings}, {"test_intsetBasicAdding", test_intsetBasicAdding}, {"test_intsetLargeNumberRandomAdd", test_intsetLargeNumberRandomAdd}, {"test_intsetUpgradeFromint16Toint32", test_intsetUpgradeFromint16Toint32}, {"test_intsetUpgradeFromint16Toint64", test_intsetUpgradeFromint16Toint64}, {"test_intsetUpgradeFromint32Toint64", test_intsetUpgradeFromint32Toint64}, {"test_intsetStressLookups", test_intsetStressLookups}, {"test_intsetStressAddDelete", test_intsetStressAddDelete}, {NULL, NULL}}; unitTest __test_kvstore_c[] = {{"test_kvstoreAdd16Keys", test_kvstoreAdd16Keys}, {"test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyDict", test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyDict}, {"test_kvstoreIteratorRemoveAllKeysDeleteEmptyDict", test_kvstoreIteratorRemoveAllKeysDeleteEmptyDict}, {"test_kvstoreDictIteratorRemoveAllKeysNoDeleteEmptyDict", test_kvstoreDictIteratorRemoveAllKeysNoDeleteEmptyDict}, {"test_kvstoreDictIteratorRemoveAllKeysDeleteEmptyDict", test_kvstoreDictIteratorRemoveAllKeysDeleteEmptyDict}, {NULL, NULL}}; +unitTest __test_rax_c[] = {{"test_randomWalkTest", test_randomWalkTest}, {"test_iteratorUnitTests", test_iteratorUnitTests}, {"test_tryInsertUnitTests", test_tryInsertUnitTests}, {"test_regtest1", test_regtest1}, {"test_regtest2", test_regtest2}, {"test_regtest3", test_regtest3}, {"test_regtest4", test_regtest4}, {"test_regtest5", test_regtest5}, {"test_regtest6", test_regtest6}, {"test_benchmark", test_benchmark}, {"test_hugeKey", test_hugeKey}, {"test_fuzz", test_fuzz}, {NULL, NULL}}; unitTest __test_sds_c[] = {{"test_sds", test_sds}, {"test_typesAndAllocSize", test_typesAndAllocSize}, {"test_sdsHeaderSizes", test_sdsHeaderSizes}, {NULL, NULL}}; unitTest __test_sha1_c[] = {{"test_sha1", test_sha1}, {NULL, NULL}}; unitTest __test_util_c[] = {{"test_string2ll", test_string2ll}, {"test_string2l", test_string2l}, {"test_ll2string", test_ll2string}, {"test_ld2string", test_ld2string}, {"test_fixedpoint_d2string", test_fixedpoint_d2string}, {"test_version2num", test_version2num}, {"test_reclaimFilePageCache", test_reclaimFilePageCache}, {NULL, NULL}}; @@ -94,6 +107,7 @@ struct unitTestSuite { {"test_endianconv.c", __test_endianconv_c}, {"test_intset.c", __test_intset_c}, {"test_kvstore.c", __test_kvstore_c}, + {"test_rax.c", __test_rax_c}, {"test_sds.c", __test_sds_c}, {"test_sha1.c", __test_sha1_c}, {"test_util.c", __test_util_c}, diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index 79c56739a2..de69910a4f 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -28,21 +28,19 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifdef SERVER_TEST - #include #include #include #include #include -#include #include -#include "rax.h" -#include "mt19937-64.h" -#include "zmalloc.h" +#include "../rax.c" +#include "../mt19937-64.c" +#include "test_help.h" uint16_t crc16(const char *buf, int len); /* From crc16.c */ +long long _ustime(void); /* From test_crc64combine.c */ /* --------------------------------------------------------------------------- * Simple hash table implementation, no rehashing, just chaining. This is @@ -542,7 +540,11 @@ int iteratorFuzzTest(int keymode, size_t count) { } /* Test the random walk function. */ -int randomWalkTest(void) { +int test_randomWalkTest(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *t = raxNew(); char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; @@ -582,7 +584,11 @@ int randomWalkTest(void) { return 0; } -int iteratorUnitTests(void) { +int test_iteratorUnitTests(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *t = raxNew(); char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; @@ -654,7 +660,11 @@ int iteratorUnitTests(void) { /* Test that raxInsert() / raxTryInsert() overwrite semantic * works as expected. */ -int tryInsertUnitTests(void) { +int test_tryInsertUnitTests(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *t = raxNew(); raxInsert(t,(unsigned char*)"FOO",3,(void*)(long)1,NULL); void *old, *val; @@ -685,7 +695,11 @@ int tryInsertUnitTests(void) { } /* Regression test #1: Iterator wrong element returned after seek. */ -int regtest1(void) { +int test_regtest1(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *rax = raxNew(); raxInsert(rax,(unsigned char*)"LKE",3,(void*)(long)1,NULL); raxInsert(rax,(unsigned char*)"TQ",2,(void*)(long)2,NULL); @@ -712,7 +726,11 @@ int regtest1(void) { } /* Regression test #2: Crash when mixing NULL and not NULL values. */ -int regtest2(void) { +int test_regtest2(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *rt = raxNew(); raxInsert(rt,(unsigned char *)"a",1,(void *)100,NULL); raxInsert(rt,(unsigned char *)"ab",2,(void *)101,NULL); @@ -729,7 +747,11 @@ int regtest2(void) { * * Note that this test always returns success but will trigger a * Valgrind error. */ -int regtest3(void) { +int test_regtest3(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *rt = raxNew(); raxInsert(rt, (unsigned char *)"D",1,(void*)1,NULL); raxInsert(rt, (unsigned char *)"",0,NULL,NULL); @@ -745,7 +767,11 @@ int regtest3(void) { * however we are using the original one from the bug report, since this * is quite odd and may later protect against different bugs related to * storing and fetching the empty string key. */ -int regtest4(void) { +int test_regtest4(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *rt = raxNew(); raxIterator iter; raxInsert(rt, (unsigned char*)"", 0, (void *)-1, NULL); @@ -768,7 +794,11 @@ int regtest4(void) { } /* Less than seek bug when stopping in the middle of a compressed node. */ -int regtest5(void) { +int test_regtest5(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *rax = raxNew(); raxInsert(rax,(unsigned char*)"b",1,(void*)(long)1,NULL); @@ -795,7 +825,11 @@ int regtest5(void) { } /* Seek may not populate iterator data. See issue #25. */ -int regtest6(void) { +int test_regtest6(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + rax *rax = raxNew(); char *key1 = "172.17.141.2/adminguide/v5.0/"; @@ -819,22 +853,28 @@ int regtest6(void) { return 0; } -void benchmark(void) { +int test_benchmark(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + + if (!(flags & UNIT_TEST_SINGLE)) + return 0; + for (int mode = 0; mode < 2; mode++) { printf("Benchmark with %s keys:\n", (mode == 0) ? "integer" : "alphanumerical"); rax *t = raxNew(); - long long start = ustime(); + long long start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); raxInsert(t,(unsigned char*)buf,len,(void*)(long)i,NULL); } - printf("Insert: %f\n", (double)(ustime()-start)/1000000); + printf("Insert: %f\n", (double)(_ustime()-start)/1000000); printf("%llu total nodes\n", (unsigned long long)t->numnodes); printf("%llu total elements\n", (unsigned long long)t->numele); - start = ustime(); + start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); @@ -844,9 +884,9 @@ void benchmark(void) { data, (void*)(long)i); } } - printf("Linear lookup: %f\n", (double)(ustime()-start)/1000000); + printf("Linear lookup: %f\n", (double)(_ustime()-start)/1000000); - start = ustime(); + start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; int r = genrand64_int64() % 5000000; @@ -857,9 +897,9 @@ void benchmark(void) { data, (void*)(long)r); } } - printf("Random lookup: %f\n", (double)(ustime()-start)/1000000); + printf("Random lookup: %f\n", (double)(_ustime()-start)/1000000); - start = ustime(); + start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); @@ -868,9 +908,9 @@ void benchmark(void) { printf("** Failed lookup did not reported NOT FOUND!\n"); } } - printf("Failed lookup: %f\n", (double)(ustime()-start)/1000000); + printf("Failed lookup: %f\n", (double)(_ustime()-start)/1000000); - start = ustime(); + start = _ustime(); raxIterator ri; raxStart(&ri,t); raxSeek(&ri,"^",NULL,0); @@ -878,21 +918,23 @@ void benchmark(void) { while (raxNext(&ri)) iter++; if (iter != 5000000) printf("** Warning iteration is incomplete\n"); raxStop(&ri); - printf("Full iteration: %f\n", (double)(ustime()-start)/1000000); + printf("Full iteration: %f\n", (double)(_ustime()-start)/1000000); - start = ustime(); + start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); int retval = raxRemove(t,(unsigned char*)buf,len,NULL); assert(retval == 1); } - printf("Deletion: %f\n", (double)(ustime()-start)/1000000); + printf("Deletion: %f\n", (double)(_ustime()-start)/1000000); printf("%llu total nodes\n", (unsigned long long)t->numnodes); printf("%llu total elements\n", (unsigned long long)t->numele); raxFree(t); } + + return 0; } /* Compressed nodes can only hold (2^29)-1 characters, so it is important @@ -900,7 +942,13 @@ void benchmark(void) { * the code to handle this edge case works as expected. * * This test is disabled by default because it uses a lot of memory. */ -int testHugeKey(void) { +int test_hugeKey(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + + if (!(flags & UNIT_TEST_LARGE_MEMORY)) + return 0; + size_t max_keylen = ((1<<29)-1) + 100; unsigned char *key = zmalloc(max_keylen); if (key == NULL) goto oom; @@ -932,109 +980,64 @@ int testHugeKey(void) { exit(1); } -int raxTest(int argc, char **argv, int flags) { - /* If an argument is given, use it as the random seed. */ - if (argc >= 4) { - init_genrand64(atoi(argv[3])); - } else { - init_genrand64(1234); - } - - /* Tests to run by default are set here. */ - int do_benchmark = 0; - int do_units = 1; - int do_fuzz_cluster = 0; - int do_fuzz = 1; - int do_regression = 1; - int do_hugekey = 0; +int test_fuzz(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); - if (flags & TEST_BENCHMARK) do_benchmark = 1; - if (flags & TEST_FUZZ_CLUSTER) do_fuzz_cluster = 1; - if (flags & TEST_HUGE_KEY) do_hugekey = 1; + if (!(flags & UNIT_TEST_ACCURATE)) + return 0; int errors = 0; - if (do_units) { - printf("Unit tests: "); fflush(stdout); - if (randomWalkTest()) errors++; - if (iteratorUnitTests()) errors++; - if (tryInsertUnitTests()) errors++; - if (errors == 0) printf("OK\n"); - } + init_genrand64(1234); - if (do_regression) { - printf("Performing regression tests: "); fflush(stdout); - if (regtest1()) errors++; - if (regtest2()) errors++; - if (regtest3()) errors++; - if (regtest4()) errors++; - if (regtest5()) errors++; - if (regtest6()) errors++; - if (errors == 0) printf("OK\n"); + for (int i = 0; i < 10; i++) { + double alpha = (double)genrand64_int64() / RAND_MAX; + double beta = 1-alpha; + if (fuzzTestCluster(genrand64_int64()%100000000,alpha,beta)) errors++; } - if (do_hugekey) { - printf("Performing huge key tests: "); fflush(stdout); - if (testHugeKey()) errors++; + for (int i = 0; i < 10; i++) { + double alpha = (double)genrand64_int64() / RAND_MAX; + double beta = 1-alpha; + if (fuzzTest(KEY_INT,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET,genrand64_int64()%10000,alpha,beta)) errors++; } - if (do_fuzz_cluster) { - for (int i = 0; i < 10; i++) { - double alpha = (double)genrand64_int64() / RAND_MAX; - double beta = 1-alpha; - if (fuzzTestCluster(genrand64_int64()%100000000,alpha,beta)) errors++; - } + size_t numops = 100000, cycles = 3; + while(cycles--) { + if (fuzzTest(KEY_INT,numops,.7,.3)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA,numops,.7,.3)) errors++; + if (fuzzTest(KEY_RANDOM,numops,.7,.3)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA,numops,.7,.3)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET,numops,.7,.3)) errors++; + numops *= 10; } - if (do_fuzz) { - for (int i = 0; i < 10; i++) { - double alpha = (double)genrand64_int64() / RAND_MAX; - double beta = 1-alpha; - if (fuzzTest(KEY_INT,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_UNIQUE_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM_SMALL_CSET,genrand64_int64()%10000,alpha,beta)) errors++; - } - - size_t numops = 100000, cycles = 3; - while(cycles--) { - if (fuzzTest(KEY_INT,numops,.7,.3)) errors++; - if (fuzzTest(KEY_UNIQUE_ALPHA,numops,.7,.3)) errors++; - if (fuzzTest(KEY_RANDOM,numops,.7,.3)) errors++; - if (fuzzTest(KEY_RANDOM_ALPHA,numops,.7,.3)) errors++; - if (fuzzTest(KEY_RANDOM_SMALL_CSET,numops,.7,.3)) errors++; - numops *= 10; - } - - if (fuzzTest(KEY_CHAIN,1000,.7,.3)) errors++; - printf("Iterator fuzz test: "); fflush(stdout); - for (int i = 0; i < 100000; i++) { - if (iteratorFuzzTest(KEY_INT,100)) errors++; - if (iteratorFuzzTest(KEY_UNIQUE_ALPHA,100)) errors++; - if (iteratorFuzzTest(KEY_RANDOM_ALPHA,1000)) errors++; - if (iteratorFuzzTest(KEY_RANDOM,1000)) errors++; - if (i && !(i % 100)) { - printf("."); - if (!(i % 1000)) { - printf("%d%% done",i/1000); - } - fflush(stdout); + if (fuzzTest(KEY_CHAIN,1000,.7,.3)) errors++; + printf("Iterator fuzz test: "); fflush(stdout); + for (int i = 0; i < 100000; i++) { + if (iteratorFuzzTest(KEY_INT,100)) errors++; + if (iteratorFuzzTest(KEY_UNIQUE_ALPHA,100)) errors++; + if (iteratorFuzzTest(KEY_RANDOM_ALPHA,1000)) errors++; + if (iteratorFuzzTest(KEY_RANDOM,1000)) errors++; + if (i && !(i % 100)) { + printf("."); + if (!(i % 1000)) { + printf("%d%% done",i/1000); } + fflush(stdout); } - printf("\n"); - } - - if (do_benchmark) { - benchmark(); } + printf("\n"); if (errors) { printf("!!! WARNING !!!: %d errors found\n", errors); } else { printf("OK! \\o/\n"); } - return errors; + return !!errors; } - -#endif // SERVER_TEST From 6cf1d96b5190bd7a6ea58c19e0b794c4615e2e9b Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Thu, 27 Jun 2024 17:34:13 -0400 Subject: [PATCH 12/26] s/rax->alloc/rax->alloc_size/ Signed-off-by: Guillaume Koenig --- src/rax.c | 32 ++++++++++++++++---------------- src/rax.h | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/rax.c b/src/rax.c index 03c3d10c7a..bd1372ae00 100644 --- a/src/rax.c +++ b/src/rax.c @@ -192,7 +192,7 @@ rax *raxNew(void) { rax->numele = 0; rax->numnodes = 1; rax->head = raxNewNode(0, 0); - rax->alloc = rax_alloc_size(rax) + rax_alloc_size(rax->head); + rax->alloc_size = rax_alloc_size(rax) + rax_alloc_size(rax->head); if (rax->head == NULL) { rax_free(rax); return NULL; @@ -517,7 +517,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** h = raxReallocForData(h, data); if (h) { memcpy(parentlink, &h, sizeof(h)); - rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h); + rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(h); } } if (h == NULL) { @@ -713,7 +713,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** return 0; } splitnode->data[0] = h->data[j]; - rax->alloc += rax_alloc_size(splitnode); + rax->alloc_size += rax_alloc_size(splitnode); if (j == 0) { /* 3a: Replace the old node with the split node. */ @@ -738,7 +738,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** memcpy(parentlink, &trimmed, sizeof(trimmed)); parentlink = cp; /* Set parentlink to splitnode parent. */ rax->numnodes++; - rax->alloc += rax_alloc_size(trimmed); + rax->alloc_size += rax_alloc_size(trimmed); } /* 4: Create the postfix node: what remains of the original @@ -753,7 +753,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** raxNode **cp = raxNodeLastChildPtr(postfix); memcpy(cp, &next, sizeof(next)); rax->numnodes++; - rax->alloc += rax_alloc_size(postfix); + rax->alloc_size += rax_alloc_size(postfix); } else { /* 4b: just use next as postfix node. */ postfix = next; @@ -766,7 +766,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** /* 6. Continue insertion: this will cause the splitnode to * get a new child (the non common character at the currently * inserted key). */ - rax->alloc -= rax_alloc_size(h); + rax->alloc_size -= rax_alloc_size(h); rax_free(h); h = splitnode; } else if (h->iscompr && i == len) { @@ -805,7 +805,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** raxNode **cp = raxNodeLastChildPtr(postfix); memcpy(cp, &next, sizeof(next)); rax->numnodes++; - rax->alloc += rax_alloc_size(postfix); + rax->alloc_size += rax_alloc_size(postfix); /* 3: Trim the compressed node. */ trimmed->size = j; @@ -818,7 +818,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** void *aux = raxGetData(h); raxSetData(trimmed, aux); } - rax->alloc += rax_alloc_size(trimmed); + rax->alloc_size += rax_alloc_size(trimmed); /* Fix the trimmed node child pointer to point to * the postfix node. */ @@ -828,7 +828,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** /* Finish! We don't need to continue with the insertion * algorithm for ALGO 2. The key is already inserted. */ rax->numele++; - rax->alloc -= rax_alloc_size(h); + rax->alloc_size -= rax_alloc_size(h); rax_free(h); return 1; /* Key inserted. */ } @@ -863,7 +863,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** i++; } rax->numnodes++; - rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h) + rax_alloc_size(child); + rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(h) + rax_alloc_size(child); h = child; } size_t oldalloc = rax_alloc_size(h); @@ -873,7 +873,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** if (!h->iskey) rax->numele++; raxSetData(h, data); memcpy(parentlink, &h, sizeof(h)); - rax->alloc = rax->alloc - oldalloc + rax_alloc_size(h); + rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(h); return 1; /* Element inserted. */ oom: @@ -1043,7 +1043,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { child = h; debugf("Freeing child %p [%.*s] key:%d\n", (void *)child, (int)child->size, (char *)child->data, child->iskey); - rax->alloc -= rax_alloc_size(child); + rax->alloc_size -= rax_alloc_size(child); rax_free(child); rax->numnodes--; h = raxStackPop(&ts); @@ -1055,7 +1055,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { debugf("Unlinking child %p from parent %p\n", (void *)child, (void *)h); raxNode *new = raxRemoveChild(h, child); size_t oldalloc = rax_alloc_size(h); - rax->alloc = rax->alloc - oldalloc + rax_alloc_size(new); + rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(new); if (new != h) { raxNode *parent = raxStackPeek(&ts); raxNode **parentlink; @@ -1172,7 +1172,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { new->iscompr = 1; new->size = comprsize; rax->numnodes++; - rax->alloc += rax_alloc_size(new); + rax->alloc_size += rax_alloc_size(new); /* Scan again, this time to populate the new node content and * to fix the new node child pointer. At the same time we free @@ -1185,7 +1185,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { raxNode **cp = raxNodeLastChildPtr(h); raxNode *tofree = h; memcpy(&h, cp, sizeof(h)); - rax->alloc -= rax_alloc_size(tofree); + rax->alloc_size -= rax_alloc_size(tofree); rax_free(tofree); rax->numnodes--; if (h->iskey || (!h->iscompr && h->size != 1)) break; @@ -1789,7 +1789,7 @@ uint64_t raxSize(rax *rax) { /* Return the rax tree allocation size in bytes */ size_t raxAllocSize(rax *rax) { - return rax->alloc; + return rax->alloc_size; } /* ----------------------------- Introspection ------------------------------ */ diff --git a/src/rax.h b/src/rax.h index 57cc357c62..7083d435fc 100644 --- a/src/rax.h +++ b/src/rax.h @@ -134,7 +134,7 @@ typedef struct rax { raxNode *head; /* Pointer to root node of tree */ uint64_t numele; /* Number of keys in the tree */ uint64_t numnodes; /* Number of rax nodes in the tree */ - size_t alloc; /* Total allocation size of the tree in bytes */ + size_t alloc_size; /* Total allocation size of the tree in bytes */ } rax; /* Stack data structure used by raxLowWalk() in order to, optionally, return From 11a5a4ff444f573a9618ed1265e81b5774a54fb8 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Fri, 28 Jun 2024 16:14:18 -0400 Subject: [PATCH 13/26] Tentative fix for address sanitizer Signed-off-by: Guillaume Koenig --- src/rax.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rax.c b/src/rax.c index bd1372ae00..d33d691139 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1053,8 +1053,8 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { } if (child) { debugf("Unlinking child %p from parent %p\n", (void *)child, (void *)h); - raxNode *new = raxRemoveChild(h, child); size_t oldalloc = rax_alloc_size(h); + raxNode *new = raxRemoveChild(h, child); rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(new); if (new != h) { raxNode *parent = raxStackPeek(&ts); From da095ac81667ddf0cf7fee09a915e1c750f6c009 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Fri, 28 Jun 2024 16:44:58 -0400 Subject: [PATCH 14/26] Fix missing zfree for src/valkey-unit-tests --large-memory Signed-off-by: Guillaume Koenig --- src/unit/test_rax.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index de69910a4f..21c1247a02 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -964,6 +964,7 @@ int test_hugeKey(int argc, char **argv, int flags) { void *value1, *value2; int found1 = raxFind(rax,(unsigned char*)"aaabbb",6,&value1); int found2 = raxFind(rax,key,max_keylen,&value2); + zfree(key); if (!found1 || !found2) { printf("Huge key test failed on elementhood\n"); return 1; From 581fdbecb9afaa52f27775b0396cc459fc94515c Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Wed, 3 Jul 2024 21:54:55 +0000 Subject: [PATCH 15/26] Reduce scope of variable Signed-off-by: Guillaume Koenig --- src/rax.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rax.c b/src/rax.c index d33d691139..92147072af 100644 --- a/src/rax.c +++ b/src/rax.c @@ -508,12 +508,10 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** * our key. We have just to reallocate the node and make space for the * data pointer. */ if (i == len && (!h->iscompr || j == 0 /* not in the middle if j is 0 */)) { - size_t oldalloc = 0; - debugf("### Insert: node representing key exists\n"); /* Make space for the value pointer if needed. */ if (!h->iskey || (h->isnull && overwrite)) { - oldalloc = rax_alloc_size(h); + size_t oldalloc = rax_alloc_size(h); h = raxReallocForData(h, data); if (h) { memcpy(parentlink, &h, sizeof(h)); From a254c28c8690b61fd5bfb48105713c9153654ba4 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Fri, 5 Jul 2024 22:14:28 +0000 Subject: [PATCH 16/26] s/rax_alloc_size/rax_ptr_alloc_size/ Signed-off-by: Guillaume Koenig --- src/rax.c | 38 +++++++++++++++++++------------------- src/rax_malloc.h | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/rax.c b/src/rax.c index 92147072af..9815b3709b 100644 --- a/src/rax.c +++ b/src/rax.c @@ -192,7 +192,7 @@ rax *raxNew(void) { rax->numele = 0; rax->numnodes = 1; rax->head = raxNewNode(0, 0); - rax->alloc_size = rax_alloc_size(rax) + rax_alloc_size(rax->head); + rax->alloc_size = rax_ptr_alloc_size(rax) + rax_ptr_alloc_size(rax->head); if (rax->head == NULL) { rax_free(rax); return NULL; @@ -511,11 +511,11 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** debugf("### Insert: node representing key exists\n"); /* Make space for the value pointer if needed. */ if (!h->iskey || (h->isnull && overwrite)) { - size_t oldalloc = rax_alloc_size(h); + size_t oldalloc = rax_ptr_alloc_size(h); h = raxReallocForData(h, data); if (h) { memcpy(parentlink, &h, sizeof(h)); - rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(h); + rax->alloc_size = rax->alloc_size - oldalloc + rax_ptr_alloc_size(h); } } if (h == NULL) { @@ -711,7 +711,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** return 0; } splitnode->data[0] = h->data[j]; - rax->alloc_size += rax_alloc_size(splitnode); + rax->alloc_size += rax_ptr_alloc_size(splitnode); if (j == 0) { /* 3a: Replace the old node with the split node. */ @@ -736,7 +736,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** memcpy(parentlink, &trimmed, sizeof(trimmed)); parentlink = cp; /* Set parentlink to splitnode parent. */ rax->numnodes++; - rax->alloc_size += rax_alloc_size(trimmed); + rax->alloc_size += rax_ptr_alloc_size(trimmed); } /* 4: Create the postfix node: what remains of the original @@ -751,7 +751,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** raxNode **cp = raxNodeLastChildPtr(postfix); memcpy(cp, &next, sizeof(next)); rax->numnodes++; - rax->alloc_size += rax_alloc_size(postfix); + rax->alloc_size += rax_ptr_alloc_size(postfix); } else { /* 4b: just use next as postfix node. */ postfix = next; @@ -764,7 +764,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** /* 6. Continue insertion: this will cause the splitnode to * get a new child (the non common character at the currently * inserted key). */ - rax->alloc_size -= rax_alloc_size(h); + rax->alloc_size -= rax_ptr_alloc_size(h); rax_free(h); h = splitnode; } else if (h->iscompr && i == len) { @@ -803,7 +803,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** raxNode **cp = raxNodeLastChildPtr(postfix); memcpy(cp, &next, sizeof(next)); rax->numnodes++; - rax->alloc_size += rax_alloc_size(postfix); + rax->alloc_size += rax_ptr_alloc_size(postfix); /* 3: Trim the compressed node. */ trimmed->size = j; @@ -816,7 +816,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** void *aux = raxGetData(h); raxSetData(trimmed, aux); } - rax->alloc_size += rax_alloc_size(trimmed); + rax->alloc_size += rax_ptr_alloc_size(trimmed); /* Fix the trimmed node child pointer to point to * the postfix node. */ @@ -826,7 +826,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** /* Finish! We don't need to continue with the insertion * algorithm for ALGO 2. The key is already inserted. */ rax->numele++; - rax->alloc_size -= rax_alloc_size(h); + rax->alloc_size -= rax_ptr_alloc_size(h); rax_free(h); return 1; /* Key inserted. */ } @@ -835,7 +835,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** * chars in our string. We need to insert the missing nodes. */ while (i < len) { raxNode *child; - size_t oldalloc = rax_alloc_size(h); + size_t oldalloc = rax_ptr_alloc_size(h); /* If this node is going to have a single child, and there * are other characters, so that that would result in a chain @@ -861,17 +861,17 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void ** i++; } rax->numnodes++; - rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(h) + rax_alloc_size(child); + rax->alloc_size = rax->alloc_size - oldalloc + rax_ptr_alloc_size(h) + rax_ptr_alloc_size(child); h = child; } - size_t oldalloc = rax_alloc_size(h); + size_t oldalloc = rax_ptr_alloc_size(h); raxNode *newh = raxReallocForData(h, data); if (newh == NULL) goto oom; h = newh; if (!h->iskey) rax->numele++; raxSetData(h, data); memcpy(parentlink, &h, sizeof(h)); - rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(h); + rax->alloc_size = rax->alloc_size - oldalloc + rax_ptr_alloc_size(h); return 1; /* Element inserted. */ oom: @@ -1041,7 +1041,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { child = h; debugf("Freeing child %p [%.*s] key:%d\n", (void *)child, (int)child->size, (char *)child->data, child->iskey); - rax->alloc_size -= rax_alloc_size(child); + rax->alloc_size -= rax_ptr_alloc_size(child); rax_free(child); rax->numnodes--; h = raxStackPop(&ts); @@ -1051,9 +1051,9 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { } if (child) { debugf("Unlinking child %p from parent %p\n", (void *)child, (void *)h); - size_t oldalloc = rax_alloc_size(h); + size_t oldalloc = rax_ptr_alloc_size(h); raxNode *new = raxRemoveChild(h, child); - rax->alloc_size = rax->alloc_size - oldalloc + rax_alloc_size(new); + rax->alloc_size = rax->alloc_size - oldalloc + rax_ptr_alloc_size(new); if (new != h) { raxNode *parent = raxStackPeek(&ts); raxNode **parentlink; @@ -1170,7 +1170,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { new->iscompr = 1; new->size = comprsize; rax->numnodes++; - rax->alloc_size += rax_alloc_size(new); + rax->alloc_size += rax_ptr_alloc_size(new); /* Scan again, this time to populate the new node content and * to fix the new node child pointer. At the same time we free @@ -1183,7 +1183,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { raxNode **cp = raxNodeLastChildPtr(h); raxNode *tofree = h; memcpy(&h, cp, sizeof(h)); - rax->alloc_size -= rax_alloc_size(tofree); + rax->alloc_size -= rax_ptr_alloc_size(tofree); rax_free(tofree); rax->numnodes--; if (h->iskey || (!h->iscompr && h->size != 1)) break; diff --git a/src/rax_malloc.h b/src/rax_malloc.h index 357fe04916..ecf45596cc 100644 --- a/src/rax_malloc.h +++ b/src/rax_malloc.h @@ -41,5 +41,5 @@ #define rax_malloc zmalloc #define rax_realloc zrealloc #define rax_free zfree -#define rax_alloc_size zmalloc_size +#define rax_ptr_alloc_size zmalloc_size #endif From 59169c00ac1b4d271d57ca71ae4f7d4d301849d0 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Tue, 9 Jul 2024 11:41:59 -0400 Subject: [PATCH 17/26] Demo checking rax size vs allocator Signed-off-by: Guillaume Koenig --- src/unit/test_rax.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index 21c1247a02..eeb66afa25 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -553,6 +553,7 @@ int test_randomWalkTest(int argc, char **argv, int flags) { raxInsert(t,(unsigned char*)toadd[numele], strlen(toadd[numele]),(void*)numele,NULL); } + TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); raxIterator iter; raxStart(&iter,t); From 540d2cfc24c54fb1a6a82b920e2fb89a562e9857 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Tue, 9 Jul 2024 18:20:11 -0400 Subject: [PATCH 18/26] More testing of raxAllocSize against the allocator's number Signed-off-by: Guillaume Koenig --- src/unit/test_rax.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index eeb66afa25..a71069fcd0 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -368,11 +368,13 @@ int fuzzTestCluster(size_t count, double addprob, double remprob) { /* Insert element. */ if ((double)genrand64_int64()/RAND_MAX < addprob) { raxInsert(rax,key,keylen,NULL,NULL); + TEST_ASSERT(raxAllocSize(rax) == zmalloc_used_memory()); } /* Remove element. */ if ((double)genrand64_int64()/RAND_MAX < remprob) { raxRemove(rax,key,keylen,NULL); + TEST_ASSERT(raxAllocSize(rax) == zmalloc_used_memory()); } } size_t finalkeys = raxSize(rax); @@ -552,8 +554,8 @@ int test_randomWalkTest(int argc, char **argv, int flags) { for (numele = 0; toadd[numele] != NULL; numele++) { raxInsert(t,(unsigned char*)toadd[numele], strlen(toadd[numele]),(void*)numele,NULL); + TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } - TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); raxIterator iter; raxStart(&iter,t); @@ -598,8 +600,10 @@ int test_iteratorUnitTests(int argc, char **argv, int flags) { long items = 0; while(toadd[items] != NULL) items++; - for (long i = 0; i < items; i++) + for (long i = 0; i < items; i++) { raxInsert(t,(unsigned char*)toadd[i],strlen(toadd[i]),(void*)i,NULL); + TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); + } raxIterator iter; raxStart(&iter,t); @@ -870,6 +874,7 @@ int test_benchmark(int argc, char **argv, int flags) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); raxInsert(t,(unsigned char*)buf,len,(void*)(long)i,NULL); + TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } printf("Insert: %f\n", (double)(_ustime()-start)/1000000); printf("%llu total nodes\n", (unsigned long long)t->numnodes); @@ -926,7 +931,8 @@ int test_benchmark(int argc, char **argv, int flags) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); int retval = raxRemove(t,(unsigned char*)buf,len,NULL); - assert(retval == 1); + TEST_ASSERT(retval == 1); + TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } printf("Deletion: %f\n", (double)(_ustime()-start)/1000000); From 212ac684a480cb1cdd125971cce45735f46f0f70 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Tue, 9 Jul 2024 19:09:39 -0400 Subject: [PATCH 19/26] Update some checks into asserts with the new framework Signed-off-by: Guillaume Koenig --- src/unit/test_rax.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index a71069fcd0..b97e18de1d 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -910,9 +910,8 @@ int test_benchmark(int argc, char **argv, int flags) { char buf[64]; int len = int2key(buf,sizeof(buf),i,mode); buf[i%len] = '!'; /* "!" is never set into keys. */ - if (!raxFind(t,(unsigned char*) buf,len,NULL)) { - printf("** Failed lookup did not reported NOT FOUND!\n"); - } + TEST_ASSERT_MESSAGE("Lookup should have failed", + raxFind(t,(unsigned char*) buf,len,NULL)); } printf("Failed lookup: %f\n", (double)(_ustime()-start)/1000000); @@ -922,7 +921,7 @@ int test_benchmark(int argc, char **argv, int flags) { raxSeek(&ri,"^",NULL,0); int iter = 0; while (raxNext(&ri)) iter++; - if (iter != 5000000) printf("** Warning iteration is incomplete\n"); + TEST_ASSERT_MESSAGE("Iteration is incomplete", iter == 5000000); raxStop(&ri); printf("Full iteration: %f\n", (double)(_ustime()-start)/1000000); From 9f94fdb6e39c44cfa6b24ef9771b2db6d5bc87bf Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Wed, 10 Jul 2024 16:10:30 -0400 Subject: [PATCH 20/26] Fix confusion that led to test failing Signed-off-by: Guillaume Koenig --- src/unit/test_rax.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index b97e18de1d..905526f2c9 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -911,7 +911,7 @@ int test_benchmark(int argc, char **argv, int flags) { int len = int2key(buf,sizeof(buf),i,mode); buf[i%len] = '!'; /* "!" is never set into keys. */ TEST_ASSERT_MESSAGE("Lookup should have failed", - raxFind(t,(unsigned char*) buf,len,NULL)); + !raxFind(t,(unsigned char*) buf,len,NULL)); } printf("Failed lookup: %f\n", (double)(_ustime()-start)/1000000); From 5f903feee5c188caa899c6fc8c9ec0e5f923961a Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Wed, 10 Jul 2024 17:49:10 -0400 Subject: [PATCH 21/26] Apply clang-format Signed-off-by: Guillaume Koenig --- src/rax.h | 8 +- src/unit/test_rax.c | 444 +++++++++++++++++++++----------------------- 2 files changed, 214 insertions(+), 238 deletions(-) diff --git a/src/rax.h b/src/rax.h index 7083d435fc..b78e25ce1e 100644 --- a/src/rax.h +++ b/src/rax.h @@ -131,10 +131,10 @@ typedef struct raxNode { } raxNode; typedef struct rax { - raxNode *head; /* Pointer to root node of tree */ - uint64_t numele; /* Number of keys in the tree */ - uint64_t numnodes; /* Number of rax nodes in the tree */ - size_t alloc_size; /* Total allocation size of the tree in bytes */ + raxNode *head; /* Pointer to root node of tree */ + uint64_t numele; /* Number of keys in the tree */ + uint64_t numnodes; /* Number of rax nodes in the tree */ + size_t alloc_size; /* Total allocation size of the tree in bytes */ } rax; /* Stack data structure used by raxLowWalk() in order to, optionally, return diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index 905526f2c9..eb23a9df97 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -40,15 +40,16 @@ #include "test_help.h" uint16_t crc16(const char *buf, int len); /* From crc16.c */ -long long _ustime(void); /* From test_crc64combine.c */ +long long _ustime(void); /* From test_crc64combine.c */ /* --------------------------------------------------------------------------- * Simple hash table implementation, no rehashing, just chaining. This is * used in order to test the radix tree implementation against something that * will always "tell the truth" :-) */ -#define HT_TABLE_SIZE 100000 /* This is huge but we want it fast enough without - * reahshing needed. */ +#define HT_TABLE_SIZE \ + 100000 /* This is huge but we want it fast enough without \ + * reahshing needed. */ typedef struct htNode { uint64_t keylen; unsigned char *key; @@ -71,19 +72,18 @@ hashtable *htNew(void) { /* djb2 hash function. */ uint32_t htHash(unsigned char *s, size_t len) { uint32_t hash = 5381; - for (size_t i = 0; i < len; i++) - hash = hash * 33 + s[i]; + for (size_t i = 0; i < len; i++) hash = hash * 33 + s[i]; return hash % HT_TABLE_SIZE; } /* Low level hash table lookup function. */ htNode *htRawLookup(hashtable *t, unsigned char *s, size_t len, uint32_t *hash, htNode ***parentlink) { - uint32_t h = htHash(s,len); + uint32_t h = htHash(s, len); if (hash) *hash = h; htNode *n = t->table[h]; if (parentlink) *parentlink = &t->table[h]; - while(n) { - if (n->keylen == len && memcmp(n->key,s,len) == 0) return n; + while (n) { + if (n->keylen == len && memcmp(n->key, s, len) == 0) return n; if (parentlink) *parentlink = &n->next; n = n->next; } @@ -94,12 +94,12 @@ htNode *htRawLookup(hashtable *t, unsigned char *s, size_t len, uint32_t *hash, * 0 if it existed and the value was updated to the new one. */ int htAdd(hashtable *t, unsigned char *s, size_t len, void *data) { uint32_t hash; - htNode *n = htRawLookup(t,s,len,&hash,NULL); + htNode *n = htRawLookup(t, s, len, &hash, NULL); if (!n) { n = zmalloc(sizeof(*n)); n->key = zmalloc(len); - memcpy(n->key,s,len); + memcpy(n->key, s, len); n->keylen = len; n->data = data; n->next = t->table[hash]; @@ -116,7 +116,7 @@ int htAdd(hashtable *t, unsigned char *s, size_t len, void *data) { * was not there already. */ int htRem(hashtable *t, unsigned char *s, size_t len) { htNode **parentlink; - htNode *n = htRawLookup(t,s,len,NULL,&parentlink); + htNode *n = htRawLookup(t, s, len, NULL, &parentlink); if (!n) return 0; *parentlink = n->next; @@ -126,12 +126,12 @@ int htRem(hashtable *t, unsigned char *s, size_t len) { return 1; } -void *htNotFound = (void*)"ht-not-found"; +void *htNotFound = (void *)"ht-not-found"; /* Find an element inside the hash table. Returns htNotFound if the * element is not there, otherwise returns the associated value. */ void *htFind(hashtable *t, unsigned char *s, size_t len) { - htNode *n = htRawLookup(t,s,len,NULL,NULL); + htNode *n = htRawLookup(t, s, len, NULL, NULL); if (!n) return htNotFound; return n->data; } @@ -140,7 +140,7 @@ void *htFind(hashtable *t, unsigned char *s, size_t len) { void htFree(hashtable *ht) { for (int j = 0; j < HT_TABLE_SIZE; j++) { htNode *next = ht->table[j]; - while(next) { + while (next) { htNode *this = next; next = this->next; zfree(this->key); @@ -166,7 +166,7 @@ static uint32_t int2int(uint32_t input) { r = l ^ F; l = nl; } - return (r<<16)|l; + return (r << 16) | l; } /* Turn an uint32_t integer into an alphanumerical key and return its @@ -182,8 +182,8 @@ static size_t int2alphakey(char *s, size_t maxlen, uint32_t i) { if (maxlen == 0) return 0; maxlen--; /* Space for null term char. */ size_t len = 0; - while(len < maxlen) { - s[len++] = set[i%setlen]; + while (len < maxlen) { + s[len++] = set[i % setlen]; i /= setlen; if (i == 0) break; } @@ -209,29 +209,29 @@ static size_t int2alphakey(char *s, size_t maxlen, uint32_t i) { #define KEY_CHAIN 5 static size_t int2key(char *s, size_t maxlen, uint32_t i, int mode) { if (mode == KEY_INT) { - return snprintf(s,maxlen,"%lu",(unsigned long)i); + return snprintf(s, maxlen, "%lu", (unsigned long)i); } else if (mode == KEY_UNIQUE_ALPHA) { if (maxlen > 16) maxlen = 16; i = int2int(i); - return int2alphakey(s,maxlen,i); + return int2alphakey(s, maxlen, i); } else if (mode == KEY_RANDOM) { if (maxlen > 16) maxlen = 16; int r = genrand64_int64() % maxlen; - for (int i = 0; i < r; i++) s[i] = genrand64_int64()&0xff; + for (int i = 0; i < r; i++) s[i] = genrand64_int64() & 0xff; return r; } else if (mode == KEY_RANDOM_ALPHA) { if (maxlen > 16) maxlen = 16; int r = genrand64_int64() % maxlen; - for (int i = 0; i < r; i++) s[i] = 'A'+genrand64_int64()%('z'-'A'+1); + for (int i = 0; i < r; i++) s[i] = 'A' + genrand64_int64() % ('z' - 'A' + 1); return r; } else if (mode == KEY_RANDOM_SMALL_CSET) { if (maxlen > 16) maxlen = 16; int r = genrand64_int64() % maxlen; - for (int i = 0; i < r; i++) s[i] = 'A'+genrand64_int64()%4; + for (int i = 0; i < r; i++) s[i] = 'A' + genrand64_int64() % 4; return r; } else if (mode == KEY_CHAIN) { if (i > maxlen) i = maxlen; - memset(s,'A',i); + memset(s, 'A', i); return i; } else { return 0; @@ -254,13 +254,13 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { uint32_t keylen; /* Insert element. */ - if ((double)genrand64_int64()/RAND_MAX < addprob) { - keylen = int2key((char*)key,sizeof(key),i,keymode); - void *val = (void*)(unsigned long)genrand64_int64(); + if ((double)genrand64_int64() / RAND_MAX < addprob) { + keylen = int2key((char *)key, sizeof(key), i, keymode); + void *val = (void *)(unsigned long)genrand64_int64(); /* Stress NULL values more often, they use a special encoding. */ if (!(genrand64_int64() % 100)) val = NULL; - int retval1 = htAdd(ht,key,keylen,val); - int retval2 = raxInsert(rax,key,keylen,val,NULL); + int retval1 = htAdd(ht, key, keylen, val); + int retval2 = raxInsert(rax, key, keylen, val, NULL); if (retval1 != retval2) { printf("Fuzz: key insertion reported mismatching value in HT/RAX\n"); return 1; @@ -268,14 +268,14 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { } /* Remove element. */ - if ((double)genrand64_int64()/RAND_MAX < remprob) { - keylen = int2key((char*)key,sizeof(key),i,keymode); - int retval1 = htRem(ht,key,keylen); - int retval2 = raxRemove(rax,key,keylen,NULL); + if ((double)genrand64_int64() / RAND_MAX < remprob) { + keylen = int2key((char *)key, sizeof(key), i, keymode); + int retval1 = htRem(ht, key, keylen); + int retval2 = raxRemove(rax, key, keylen, NULL); if (retval1 != retval2) { printf("Fuzz: key deletion of '%.*s' reported mismatching " "value in HT=%d RAX=%d\n", - (int)keylen,(char*)key,retval1, retval2); + (int)keylen, (char *)key, retval1, retval2); return 1; } } @@ -283,27 +283,26 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { /* Check that count matches. */ if (ht->numele != raxSize(rax)) { - printf("Fuzz: HT / RAX keys count mismatch: %lu vs %lu\n", - (unsigned long) ht->numele, - (unsigned long) raxSize(rax)); + printf("Fuzz: HT / RAX keys count mismatch: %lu vs %lu\n", (unsigned long)ht->numele, + (unsigned long)raxSize(rax)); return 1; } printf("%lu elements inserted\n", (unsigned long)ht->numele); /* Check that elements match. */ raxIterator iter; - raxStart(&iter,rax); - raxSeek(&iter,"^",NULL,0); + raxStart(&iter, rax); + raxSeek(&iter, "^", NULL, 0); size_t numkeys = 0; - while(raxNext(&iter)) { - void *val1 = htFind(ht,iter.key,iter.key_len); + while (raxNext(&iter)) { + void *val1 = htFind(ht, iter.key, iter.key_len); void *val2 = NULL; - raxFind(rax,iter.key,iter.key_len,&val2); + raxFind(rax, iter.key, iter.key_len, &val2); if (val1 != val2) { printf("Fuzz: HT=%p, RAX=%p value do not match " "for key %.*s\n", - val1, val2, (int)iter.key_len,(char*)iter.key); + val1, val2, (int)iter.key_len, (char *)iter.key); return 1; } numkeys++; @@ -311,9 +310,8 @@ int fuzzTest(int keymode, size_t count, double addprob, double remprob) { /* Check that the iterator reported all the elements. */ if (ht->numele != numkeys) { - printf("Fuzz: the iterator reported %lu keys instead of %lu\n", - (unsigned long) numkeys, - (unsigned long) ht->numele); + printf("Fuzz: the iterator reported %lu keys instead of %lu\n", (unsigned long)numkeys, + (unsigned long)ht->numele); return 1; } @@ -343,7 +341,7 @@ int fuzzTestCluster(size_t count, double addprob, double remprob) { /* This is our template to generate keys. The first two bytes will * be replaced with the binary redis cluster hash slot. */ - keylen = snprintf((char*)key,sizeof(key),"__geocode:2e68e5df3624"); + keylen = snprintf((char *)key, sizeof(key), "__geocode:2e68e5df3624"); char *cset = "0123456789abcdef"; for (unsigned long j = 0; j < count; j++) { @@ -353,33 +351,33 @@ int fuzzTestCluster(size_t count, double addprob, double remprob) { * is a subset of keys that have an higher percentage of probability * of being hit again and again. */ size_t commonprefix = genrand64_int64() & 0xf; - if (commonprefix == 0) memcpy(key+10,"2e68e5",6); + if (commonprefix == 0) memcpy(key + 10, "2e68e5", 6); /* Alter a random char in the key. */ - int pos = 10+genrand64_int64()%12; - key[pos] = cset[genrand64_int64()%16]; + int pos = 10 + genrand64_int64() % 12; + key[pos] = cset[genrand64_int64() % 16]; /* Compute the Redis Cluster hash slot to set the first two * binary bytes of the key. */ - int hashslot = crc16((char*)key,keylen) & 0x3FFF; + int hashslot = crc16((char *)key, keylen) & 0x3FFF; key[0] = (hashslot >> 8) & 0xff; key[1] = hashslot & 0xff; /* Insert element. */ - if ((double)genrand64_int64()/RAND_MAX < addprob) { - raxInsert(rax,key,keylen,NULL,NULL); + if ((double)genrand64_int64() / RAND_MAX < addprob) { + raxInsert(rax, key, keylen, NULL, NULL); TEST_ASSERT(raxAllocSize(rax) == zmalloc_used_memory()); } /* Remove element. */ - if ((double)genrand64_int64()/RAND_MAX < remprob) { - raxRemove(rax,key,keylen,NULL); + if ((double)genrand64_int64() / RAND_MAX < remprob) { + raxRemove(rax, key, keylen, NULL); TEST_ASSERT(raxAllocSize(rax) == zmalloc_used_memory()); } } size_t finalkeys = raxSize(rax); raxFree(rax); - printf("ok with %zu final keys\n",finalkeys); + printf("ok with %zu final keys\n", finalkeys); return 0; } @@ -396,7 +394,7 @@ typedef struct arrayItem { * every byte an unsigned integer. */ int compareAB(const unsigned char *keya, size_t lena, const unsigned char *keyb, size_t lenb) { size_t minlen = (lena <= lenb) ? lena : lenb; - int retval = memcmp(keya,keyb,minlen); + int retval = memcmp(keya, keyb, minlen); if (lena == lenb || retval != 0) return retval; return (lena > lenb) ? 1 : -1; } @@ -404,7 +402,7 @@ int compareAB(const unsigned char *keya, size_t lena, const unsigned char *keyb, int compareArrayItems(const void *aptr, const void *bptr) { const arrayItem *a = aptr; const arrayItem *b = bptr; - return compareAB(a->key,a->key_len,b->key,b->key_len); + return compareAB(a->key, a->key_len, b->key, b->key_len); } /* Seek an element in the array, returning the seek index (the index inside the @@ -413,7 +411,7 @@ int compareArrayItems(const void *aptr, const void *bptr) { int arraySeek(arrayItem *array, int count, unsigned char *key, size_t len, char *op) { if (count == 0) return -1; if (op[0] == '^') return 0; - if (op[0] == '$') return count-1; + if (op[0] == '$') return count - 1; int eq = 0, lt = 0, gt = 0; if (op[1] == '=') eq = 1; @@ -422,7 +420,7 @@ int arraySeek(arrayItem *array, int count, unsigned char *key, size_t len, char int i; for (i = 0; i < count; i++) { - int cmp = compareAB(array[i].key,array[i].key_len,key,len); + int cmp = compareAB(array[i].key, array[i].key_len, key, len); if (eq && !cmp) return i; if (cmp > 0 && gt) return i; if (cmp >= 0 && lt) { @@ -430,48 +428,47 @@ int arraySeek(arrayItem *array, int count, unsigned char *key, size_t len, char break; } } - if (lt && i == count) return count-1; + if (lt && i == count) return count - 1; if (i < 0 || i >= count) return -1; return i; } int iteratorFuzzTest(int keymode, size_t count) { - count = genrand64_int64()%count; + count = genrand64_int64() % count; rax *rax = raxNew(); - arrayItem *array = zmalloc(sizeof(arrayItem)*count); + arrayItem *array = zmalloc(sizeof(arrayItem) * count); /* Fill a radix tree and a linear array with some data. */ unsigned char key[1024]; size_t j = 0; for (size_t i = 0; i < count; i++) { - uint32_t keylen = int2key((char*)key,sizeof(key),i,keymode); - void *val = (void*)(unsigned long)htHash(key,keylen); + uint32_t keylen = int2key((char *)key, sizeof(key), i, keymode); + void *val = (void *)(unsigned long)htHash(key, keylen); - if (raxInsert(rax,key,keylen,val,NULL)) { + if (raxInsert(rax, key, keylen, val, NULL)) { array[j].key = zmalloc(keylen); array[j].key_len = keylen; - memcpy(array[j].key,key,keylen); + memcpy(array[j].key, key, keylen); j++; } } count = raxSize(rax); /* Sort the array. */ - qsort(array,count,sizeof(arrayItem),compareArrayItems); + qsort(array, count, sizeof(arrayItem), compareArrayItems); /* Perform a random seek operation. */ - uint32_t keylen = int2key((char*)key,sizeof(key), - genrand64_int64()%(count ? count : 1),keymode); + uint32_t keylen = int2key((char *)key, sizeof(key), genrand64_int64() % (count ? count : 1), keymode); raxIterator iter; - raxStart(&iter,rax); - char *seekops[] = {"==",">=","<=",">","<","^","$"}; + raxStart(&iter, rax); + char *seekops[] = {"==", ">=", "<=", ">", "<", "^", "$"}; char *seekop = seekops[genrand64_int64() % 7]; - raxSeek(&iter,seekop,key,keylen); - int seekidx = arraySeek(array,count,key,keylen,seekop); + raxSeek(&iter, seekop, key, keylen); + int seekidx = arraySeek(array, count, key, keylen, seekop); int next = genrand64_int64() % 2; int iteration = 0; - while(1) { + while (1) { int rax_res; int array_res; unsigned char *array_key = NULL; @@ -506,25 +503,20 @@ int iteratorFuzzTest(int keymode, size_t count) { if (array_res == 0) break; /* End of iteration reached. */ /* Check that the returned keys are the same. */ - if (iter.key_len != array_key_len || - memcmp(iter.key,array_key,iter.key_len)) - { + if (iter.key_len != array_key_len || memcmp(iter.key, array_key, iter.key_len)) { printf("Iter fuzz: returned element %d mismatch\n", iteration); - printf("SEEKOP was %s\n",seekop); + printf("SEEKOP was %s\n", seekop); if (keymode != KEY_RANDOM) { printf("\n"); - printf("BUG SEEKING: %s %.*s\n",seekop,keylen,key); + printf("BUG SEEKING: %s %.*s\n", seekop, keylen, key); printf("%.*s (iter) VS %.*s (array) next=%d idx=%d " "count=%lu keymode=%d\n", - (int)iter.key_len, (char*)iter.key, - (int)array_key_len, (char*)array_key, - next, seekidx, (unsigned long)count, keymode); + (int)iter.key_len, (char *)iter.key, (int)array_key_len, (char *)array_key, next, seekidx, + (unsigned long)count, keymode); if (count < 500) { printf("\n"); for (unsigned int j = 0; j < count; j++) { - printf("%d) '%.*s'\n",j, - (int)array[j].key_len, - array[j].key); + printf("%d) '%.*s'\n", j, (int)array[j].key_len, array[j].key); } } exit(1); @@ -548,29 +540,27 @@ int test_randomWalkTest(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; + char *toadd[] = {"alligator", "alien", "baloon", "chromodynamic", "romane", "romanus", "romulus", "rubens", + "ruber", "rubicon", "rubicundus", "all", "rub", "ba", NULL}; long numele; for (numele = 0; toadd[numele] != NULL; numele++) { - raxInsert(t,(unsigned char*)toadd[numele], - strlen(toadd[numele]),(void*)numele,NULL); + raxInsert(t, (unsigned char *)toadd[numele], strlen(toadd[numele]), (void *)numele, NULL); TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } raxIterator iter; - raxStart(&iter,t); - raxSeek(&iter,"^",NULL,0); + raxStart(&iter, t); + raxSeek(&iter, "^", NULL, 0); int maxloops = 100000; - while(raxRandomWalk(&iter,0) && maxloops--) { + while (raxRandomWalk(&iter, 0) && maxloops--) { int nulls = 0; for (long i = 0; i < numele; i++) { if (toadd[i] == NULL) { nulls++; continue; } - if (strlen(toadd[i]) == iter.key_len && - memcmp(toadd[i],iter.key,iter.key_len) == 0) - { + if (strlen(toadd[i]) == iter.key_len && memcmp(toadd[i], iter.key, iter.key_len) == 0) { toadd[i] = NULL; nulls++; } @@ -593,67 +583,63 @@ int test_iteratorUnitTests(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - char *toadd[] = {"alligator","alien","baloon","chromodynamic","romane","romanus","romulus","rubens","ruber","rubicon","rubicundus","all","rub","ba",NULL}; + char *toadd[] = {"alligator", "alien", "baloon", "chromodynamic", "romane", "romanus", "romulus", "rubens", + "ruber", "rubicon", "rubicundus", "all", "rub", "ba", NULL}; for (int x = 0; x < 10000; x++) genrand64_int64(); long items = 0; - while(toadd[items] != NULL) items++; + while (toadd[items] != NULL) items++; for (long i = 0; i < items; i++) { - raxInsert(t,(unsigned char*)toadd[i],strlen(toadd[i]),(void*)i,NULL); + raxInsert(t, (unsigned char *)toadd[i], strlen(toadd[i]), (void *)i, NULL); TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } raxIterator iter; - raxStart(&iter,t); + raxStart(&iter, t); struct { char *seek; size_t seeklen; char *seekop; char *expected; - } tests[] = { - /* Seek value. */ /* Expected result. */ - {"rpxxx",5,"<=", "romulus"}, - {"rom",3,">=", "romane"}, - {"rub",3,">=", "rub"}, - {"rub",3,">", "rubens"}, - {"rub",3,"<", "romulus"}, - {"rom",3,">", "romane"}, - {"chro",4,">", "chromodynamic"}, - {"chro",4,"<", "baloon"}, - {"chromz",6,"<", "chromodynamic"}, - {"",0,"^", "alien"}, - {"zorro",5,"<=", "rubicundus"}, - {"zorro",5,"<", "rubicundus"}, - {"zorro",5,"<", "rubicundus"}, - {"",0,"$", "rubicundus"}, - {"ro",2,">=", "romane"}, - {"zo",2,">", NULL}, - {"zo",2,"==", NULL}, - {"romane",6,"==", "romane"} - }; + } tests[] = {/* Seek value. */ /* Expected result. */ + {"rpxxx", 5, "<=", "romulus"}, + {"rom", 3, ">=", "romane"}, + {"rub", 3, ">=", "rub"}, + {"rub", 3, ">", "rubens"}, + {"rub", 3, "<", "romulus"}, + {"rom", 3, ">", "romane"}, + {"chro", 4, ">", "chromodynamic"}, + {"chro", 4, "<", "baloon"}, + {"chromz", 6, "<", "chromodynamic"}, + {"", 0, "^", "alien"}, + {"zorro", 5, "<=", "rubicundus"}, + {"zorro", 5, "<", "rubicundus"}, + {"zorro", 5, "<", "rubicundus"}, + {"", 0, "$", "rubicundus"}, + {"ro", 2, ">=", "romane"}, + {"zo", 2, ">", NULL}, + {"zo", 2, "==", NULL}, + {"romane", 6, "==", "romane"}}; for (int i = 0; tests[i].expected != NULL; i++) { - raxSeek(&iter,tests[i].seekop,(unsigned char*)tests[i].seek, - tests[i].seeklen); + raxSeek(&iter, tests[i].seekop, (unsigned char *)tests[i].seek, tests[i].seeklen); int retval = raxNext(&iter); if (tests[i].expected != NULL) { - if (strlen(tests[i].expected) != iter.key_len || - memcmp(tests[i].expected,iter.key,iter.key_len) != 0) - { + if (strlen(tests[i].expected) != iter.key_len || memcmp(tests[i].expected, iter.key, iter.key_len) != 0) { printf("Iterator unit test error: " "test %d, %s expected, %.*s reported\n", - i, tests[i].expected, (int)iter.key_len, - (char*)iter.key); + i, tests[i].expected, (int)iter.key_len, (char *)iter.key); return 1; } } else { if (retval != 0) { printf("Iterator unit test error: " - "EOF expected in test %d\n", i); + "EOF expected in test %d\n", + i); return 1; } } @@ -671,26 +657,25 @@ int test_tryInsertUnitTests(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - raxInsert(t,(unsigned char*)"FOO",3,(void*)(long)1,NULL); + raxInsert(t, (unsigned char *)"FOO", 3, (void *)(long)1, NULL); void *old, *val; - raxTryInsert(t,(unsigned char*)"FOO",3,(void*)(long)2,&old); - if (old != (void*)(long)1) { - printf("Old value not returned correctly by raxTryInsert(): %p", - old); + raxTryInsert(t, (unsigned char *)"FOO", 3, (void *)(long)2, &old); + if (old != (void *)(long)1) { + printf("Old value not returned correctly by raxTryInsert(): %p", old); return 1; } val = NULL; - raxFind(t,(unsigned char*)"FOO",3,&val); - if (val != (void*)(long)1) { + raxFind(t, (unsigned char *)"FOO", 3, &val); + if (val != (void *)(long)1) { printf("FOO value mismatch: is %p intead of 1", val); return 1; } - raxInsert(t,(unsigned char*)"FOO",3,(void*)(long)2,NULL); + raxInsert(t, (unsigned char *)"FOO", 3, (void *)(long)2, NULL); val = NULL; - raxFind(t,(unsigned char*)"FOO",3,&val); - if (val != (void*)(long)2) { + raxFind(t, (unsigned char *)"FOO", 3, &val); + if (val != (void *)(long)2) { printf("FOO value mismatch: is %p intead of 2", val); return 1; } @@ -706,21 +691,18 @@ int test_regtest1(int argc, char **argv, int flags) { UNUSED(flags); rax *rax = raxNew(); - raxInsert(rax,(unsigned char*)"LKE",3,(void*)(long)1,NULL); - raxInsert(rax,(unsigned char*)"TQ",2,(void*)(long)2,NULL); - raxInsert(rax,(unsigned char*)"B",1,(void*)(long)3,NULL); - raxInsert(rax,(unsigned char*)"FY",2,(void*)(long)4,NULL); - raxInsert(rax,(unsigned char*)"WI",2,(void*)(long)5,NULL); + raxInsert(rax, (unsigned char *)"LKE", 3, (void *)(long)1, NULL); + raxInsert(rax, (unsigned char *)"TQ", 2, (void *)(long)2, NULL); + raxInsert(rax, (unsigned char *)"B", 1, (void *)(long)3, NULL); + raxInsert(rax, (unsigned char *)"FY", 2, (void *)(long)4, NULL); + raxInsert(rax, (unsigned char *)"WI", 2, (void *)(long)5, NULL); raxIterator iter; - raxStart(&iter,rax); - raxSeek(&iter,">",(unsigned char*)"FMP",3); + raxStart(&iter, rax); + raxSeek(&iter, ">", (unsigned char *)"FMP", 3); if (raxNext(&iter)) { - if (iter.key_len != 2 || - memcmp(iter.key,"FY",2)) - { - printf("Regression test 1 failed: 'FY' expected, got: '%.*s'\n", - (int)iter.key_len, (char*)iter.key); + if (iter.key_len != 2 || memcmp(iter.key, "FY", 2)) { + printf("Regression test 1 failed: 'FY' expected, got: '%.*s'\n", (int)iter.key_len, (char *)iter.key); return 1; } } @@ -737,11 +719,11 @@ int test_regtest2(int argc, char **argv, int flags) { UNUSED(flags); rax *rt = raxNew(); - raxInsert(rt,(unsigned char *)"a",1,(void *)100,NULL); - raxInsert(rt,(unsigned char *)"ab",2,(void *)101,NULL); - raxInsert(rt,(unsigned char *)"abc",3,(void *)NULL,NULL); - raxInsert(rt,(unsigned char *)"abcd",4,(void *)NULL,NULL); - raxInsert(rt,(unsigned char *)"abc",3,(void *)102,NULL); + raxInsert(rt, (unsigned char *)"a", 1, (void *)100, NULL); + raxInsert(rt, (unsigned char *)"ab", 2, (void *)101, NULL); + raxInsert(rt, (unsigned char *)"abc", 3, (void *)NULL, NULL); + raxInsert(rt, (unsigned char *)"abcd", 4, (void *)NULL, NULL); + raxInsert(rt, (unsigned char *)"abc", 3, (void *)102, NULL); raxFree(rt); return 0; } @@ -758,9 +740,9 @@ int test_regtest3(int argc, char **argv, int flags) { UNUSED(flags); rax *rt = raxNew(); - raxInsert(rt, (unsigned char *)"D",1,(void*)1,NULL); - raxInsert(rt, (unsigned char *)"",0,NULL,NULL); - raxRemove(rt, (unsigned char *)"D",1,NULL); + raxInsert(rt, (unsigned char *)"D", 1, (void *)1, NULL); + raxInsert(rt, (unsigned char *)"", 0, NULL, NULL); + raxRemove(rt, (unsigned char *)"D", 1, NULL); raxFree(rt); return 0; } @@ -779,14 +761,14 @@ int test_regtest4(int argc, char **argv, int flags) { rax *rt = raxNew(); raxIterator iter; - raxInsert(rt, (unsigned char*)"", 0, (void *)-1, NULL); + raxInsert(rt, (unsigned char *)"", 0, (void *)-1, NULL); void *val = NULL; - raxFind(rt, (unsigned char*)"", 0, &val); + raxFind(rt, (unsigned char *)"", 0, &val); if (val != (void *)-1) { printf("Regression test 4 failed. Key value mismatch in raxFind()\n"); return 1; } - raxStart(&iter,rt); + raxStart(&iter, rt); raxSeek(&iter, "^", NULL, 0); raxNext(&iter); if (iter.data != (void *)-1) { @@ -806,18 +788,18 @@ int test_regtest5(int argc, char **argv, int flags) { rax *rax = raxNew(); - raxInsert(rax,(unsigned char*)"b",1,(void*)(long)1,NULL); - raxInsert(rax,(unsigned char*)"ba",2,(void*)(long)2,NULL); - raxInsert(rax,(unsigned char*)"banana",6,(void*)(long)3,NULL); + raxInsert(rax, (unsigned char *)"b", 1, (void *)(long)1, NULL); + raxInsert(rax, (unsigned char *)"ba", 2, (void *)(long)2, NULL); + raxInsert(rax, (unsigned char *)"banana", 6, (void *)(long)3, NULL); - raxInsert(rax,(unsigned char*)"f",1,(void*)(long)4,NULL); - raxInsert(rax,(unsigned char*)"foobar",6,(void*)(long)5,NULL); - raxInsert(rax,(unsigned char*)"foobar123",9,(void*)(long)6,NULL); + raxInsert(rax, (unsigned char *)"f", 1, (void *)(long)4, NULL); + raxInsert(rax, (unsigned char *)"foobar", 6, (void *)(long)5, NULL); + raxInsert(rax, (unsigned char *)"foobar123", 9, (void *)(long)6, NULL); raxIterator ri; - raxStart(&ri,rax); + raxStart(&ri, rax); - raxSeek(&ri,"<",(unsigned char*)"foo",3); + raxSeek(&ri, "<", (unsigned char *)"foo", 3); raxNext(&ri); if (ri.key_len != 1 || ri.key[0] != 'f') { printf("Regression test 4 failed. Key value mismatch in raxNext()\n"); @@ -841,12 +823,12 @@ int test_regtest6(int argc, char **argv, int flags) { char *key2 = "172.17.141.2/adminguide/v5.0/entitlements-configure.html"; char *seekpoint = "172.17.141.2/adminguide/v5.0/entitlements"; - raxInsert(rax, (unsigned char*)key1,strlen(key1),(void*)(long)1234, NULL); - raxInsert(rax, (unsigned char*)key2,strlen(key2),(void*)(long)5678, NULL); + raxInsert(rax, (unsigned char *)key1, strlen(key1), (void *)(long)1234, NULL); + raxInsert(rax, (unsigned char *)key2, strlen(key2), (void *)(long)5678, NULL); raxIterator ri; - raxStart(&ri,rax); - raxSeek(&ri,"<=", (unsigned char*)seekpoint, strlen(seekpoint)); + raxStart(&ri, rax); + raxSeek(&ri, "<=", (unsigned char *)seekpoint, strlen(seekpoint)); raxPrev(&ri); if ((long)ri.data != 1234) { printf("Regression test 6 failed. Key data not populated.\n"); @@ -862,78 +844,73 @@ int test_benchmark(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); - if (!(flags & UNIT_TEST_SINGLE)) - return 0; + if (!(flags & UNIT_TEST_SINGLE)) return 0; for (int mode = 0; mode < 2; mode++) { - printf("Benchmark with %s keys:\n", - (mode == 0) ? "integer" : "alphanumerical"); + printf("Benchmark with %s keys:\n", (mode == 0) ? "integer" : "alphanumerical"); rax *t = raxNew(); long long start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; - int len = int2key(buf,sizeof(buf),i,mode); - raxInsert(t,(unsigned char*)buf,len,(void*)(long)i,NULL); + int len = int2key(buf, sizeof(buf), i, mode); + raxInsert(t, (unsigned char *)buf, len, (void *)(long)i, NULL); TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } - printf("Insert: %f\n", (double)(_ustime()-start)/1000000); + printf("Insert: %f\n", (double)(_ustime() - start) / 1000000); printf("%llu total nodes\n", (unsigned long long)t->numnodes); printf("%llu total elements\n", (unsigned long long)t->numele); start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; - int len = int2key(buf,sizeof(buf),i,mode); + int len = int2key(buf, sizeof(buf), i, mode); void *data; - if (!raxFind(t,(unsigned char*)buf,len,&data) || data != (void*)(long)i) { - printf("Issue with %s: %p instead of %p\n", buf, - data, (void*)(long)i); + if (!raxFind(t, (unsigned char *)buf, len, &data) || data != (void *)(long)i) { + printf("Issue with %s: %p instead of %p\n", buf, data, (void *)(long)i); } } - printf("Linear lookup: %f\n", (double)(_ustime()-start)/1000000); + printf("Linear lookup: %f\n", (double)(_ustime() - start) / 1000000); start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; int r = genrand64_int64() % 5000000; - int len = int2key(buf,sizeof(buf),r,mode); + int len = int2key(buf, sizeof(buf), r, mode); void *data; - if (!raxFind(t,(unsigned char*)buf,len,&data) || data != (void*)(long)r) { - printf("Issue with %s: %p instead of %p\n", buf, - data, (void*)(long)r); + if (!raxFind(t, (unsigned char *)buf, len, &data) || data != (void *)(long)r) { + printf("Issue with %s: %p instead of %p\n", buf, data, (void *)(long)r); } } - printf("Random lookup: %f\n", (double)(_ustime()-start)/1000000); + printf("Random lookup: %f\n", (double)(_ustime() - start) / 1000000); start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; - int len = int2key(buf,sizeof(buf),i,mode); - buf[i%len] = '!'; /* "!" is never set into keys. */ - TEST_ASSERT_MESSAGE("Lookup should have failed", - !raxFind(t,(unsigned char*) buf,len,NULL)); + int len = int2key(buf, sizeof(buf), i, mode); + buf[i % len] = '!'; /* "!" is never set into keys. */ + TEST_ASSERT_MESSAGE("Lookup should have failed", !raxFind(t, (unsigned char *)buf, len, NULL)); } - printf("Failed lookup: %f\n", (double)(_ustime()-start)/1000000); + printf("Failed lookup: %f\n", (double)(_ustime() - start) / 1000000); start = _ustime(); raxIterator ri; - raxStart(&ri,t); - raxSeek(&ri,"^",NULL,0); + raxStart(&ri, t); + raxSeek(&ri, "^", NULL, 0); int iter = 0; while (raxNext(&ri)) iter++; TEST_ASSERT_MESSAGE("Iteration is incomplete", iter == 5000000); raxStop(&ri); - printf("Full iteration: %f\n", (double)(_ustime()-start)/1000000); + printf("Full iteration: %f\n", (double)(_ustime() - start) / 1000000); start = _ustime(); for (int i = 0; i < 5000000; i++) { char buf[64]; - int len = int2key(buf,sizeof(buf),i,mode); - int retval = raxRemove(t,(unsigned char*)buf,len,NULL); + int len = int2key(buf, sizeof(buf), i, mode); + int retval = raxRemove(t, (unsigned char *)buf, len, NULL); TEST_ASSERT(retval == 1); TEST_ASSERT(raxAllocSize(t) == zmalloc_used_memory()); } - printf("Deletion: %f\n", (double)(_ustime()-start)/1000000); + printf("Deletion: %f\n", (double)(_ustime() - start) / 1000000); printf("%llu total nodes\n", (unsigned long long)t->numnodes); printf("%llu total elements\n", (unsigned long long)t->numele); @@ -952,30 +929,29 @@ int test_hugeKey(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); - if (!(flags & UNIT_TEST_LARGE_MEMORY)) - return 0; + if (!(flags & UNIT_TEST_LARGE_MEMORY)) return 0; - size_t max_keylen = ((1<<29)-1) + 100; + size_t max_keylen = ((1 << 29) - 1) + 100; unsigned char *key = zmalloc(max_keylen); if (key == NULL) goto oom; - memset(key,'a',max_keylen); + memset(key, 'a', max_keylen); key[10] = 'X'; - key[max_keylen-1] = 'Y'; + key[max_keylen - 1] = 'Y'; rax *rax = raxNew(); - int retval = raxInsert(rax,(unsigned char*)"aaabbb",6,(void*)5678L,NULL); + int retval = raxInsert(rax, (unsigned char *)"aaabbb", 6, (void *)5678L, NULL); if (retval == 0 && errno == ENOMEM) goto oom; - retval = raxInsert(rax,key,max_keylen,(void*)1234L,NULL); + retval = raxInsert(rax, key, max_keylen, (void *)1234L, NULL); if (retval == 0 && errno == ENOMEM) goto oom; void *value1, *value2; - int found1 = raxFind(rax,(unsigned char*)"aaabbb",6,&value1); - int found2 = raxFind(rax,key,max_keylen,&value2); + int found1 = raxFind(rax, (unsigned char *)"aaabbb", 6, &value1); + int found2 = raxFind(rax, key, max_keylen, &value2); zfree(key); if (!found1 || !found2) { printf("Huge key test failed on elementhood\n"); return 1; } - if (value1 != (void*)5678L || value2 != (void*)1234L) { + if (value1 != (void *)5678L || value2 != (void *)1234L) { printf("Huge key test failed\n"); return 1; } @@ -983,7 +959,7 @@ int test_hugeKey(int argc, char **argv, int flags) { return 0; oom: - fprintf(stderr,"Sorry, not enough memory to execute --hugekey test."); + fprintf(stderr, "Sorry, not enough memory to execute --hugekey test."); exit(1); } @@ -991,8 +967,7 @@ int test_fuzz(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); - if (!(flags & UNIT_TEST_ACCURATE)) - return 0; + if (!(flags & UNIT_TEST_ACCURATE)) return 0; int errors = 0; @@ -1000,41 +975,42 @@ int test_fuzz(int argc, char **argv, int flags) { for (int i = 0; i < 10; i++) { double alpha = (double)genrand64_int64() / RAND_MAX; - double beta = 1-alpha; - if (fuzzTestCluster(genrand64_int64()%100000000,alpha,beta)) errors++; + double beta = 1 - alpha; + if (fuzzTestCluster(genrand64_int64() % 100000000, alpha, beta)) errors++; } for (int i = 0; i < 10; i++) { double alpha = (double)genrand64_int64() / RAND_MAX; - double beta = 1-alpha; - if (fuzzTest(KEY_INT,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_UNIQUE_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM_ALPHA,genrand64_int64()%10000,alpha,beta)) errors++; - if (fuzzTest(KEY_RANDOM_SMALL_CSET,genrand64_int64()%10000,alpha,beta)) errors++; + double beta = 1 - alpha; + if (fuzzTest(KEY_INT, genrand64_int64() % 10000, alpha, beta)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA, genrand64_int64() % 10000, alpha, beta)) errors++; + if (fuzzTest(KEY_RANDOM, genrand64_int64() % 10000, alpha, beta)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA, genrand64_int64() % 10000, alpha, beta)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET, genrand64_int64() % 10000, alpha, beta)) errors++; } size_t numops = 100000, cycles = 3; - while(cycles--) { - if (fuzzTest(KEY_INT,numops,.7,.3)) errors++; - if (fuzzTest(KEY_UNIQUE_ALPHA,numops,.7,.3)) errors++; - if (fuzzTest(KEY_RANDOM,numops,.7,.3)) errors++; - if (fuzzTest(KEY_RANDOM_ALPHA,numops,.7,.3)) errors++; - if (fuzzTest(KEY_RANDOM_SMALL_CSET,numops,.7,.3)) errors++; + while (cycles--) { + if (fuzzTest(KEY_INT, numops, .7, .3)) errors++; + if (fuzzTest(KEY_UNIQUE_ALPHA, numops, .7, .3)) errors++; + if (fuzzTest(KEY_RANDOM, numops, .7, .3)) errors++; + if (fuzzTest(KEY_RANDOM_ALPHA, numops, .7, .3)) errors++; + if (fuzzTest(KEY_RANDOM_SMALL_CSET, numops, .7, .3)) errors++; numops *= 10; } - if (fuzzTest(KEY_CHAIN,1000,.7,.3)) errors++; - printf("Iterator fuzz test: "); fflush(stdout); + if (fuzzTest(KEY_CHAIN, 1000, .7, .3)) errors++; + printf("Iterator fuzz test: "); + fflush(stdout); for (int i = 0; i < 100000; i++) { - if (iteratorFuzzTest(KEY_INT,100)) errors++; - if (iteratorFuzzTest(KEY_UNIQUE_ALPHA,100)) errors++; - if (iteratorFuzzTest(KEY_RANDOM_ALPHA,1000)) errors++; - if (iteratorFuzzTest(KEY_RANDOM,1000)) errors++; + if (iteratorFuzzTest(KEY_INT, 100)) errors++; + if (iteratorFuzzTest(KEY_UNIQUE_ALPHA, 100)) errors++; + if (iteratorFuzzTest(KEY_RANDOM_ALPHA, 1000)) errors++; + if (iteratorFuzzTest(KEY_RANDOM, 1000)) errors++; if (i && !(i % 100)) { printf("."); if (!(i % 1000)) { - printf("%d%% done",i/1000); + printf("%d%% done", i / 1000); } fflush(stdout); } From ea3e33c3be7df4fe0cb18828d54d6a38cd9c5173 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Wed, 10 Jul 2024 18:45:55 -0400 Subject: [PATCH 22/26] Attempt at fixing spellcheck Signed-off-by: Guillaume Koenig --- .config/typos.toml | 1 + src/unit/test_rax.c | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.config/typos.toml b/.config/typos.toml index d378b5655a..1dc44ea0e9 100644 --- a/.config/typos.toml +++ b/.config/typos.toml @@ -20,6 +20,7 @@ extend-ignore-re = [ "D4C4DAA4", # sha1.c "Georg Nees", "\\[l\\]ist", # eval.c + "LKE", # test_rax.c ] [type.tcl] diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index eb23a9df97..a151740931 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -90,7 +90,7 @@ htNode *htRawLookup(hashtable *t, unsigned char *s, size_t len, uint32_t *hash, return NULL; } -/* Add an elmenet to the hash table, return 1 if the element is new, +/* Add an element to the hash table, return 1 if the element is new, * 0 if it existed and the value was updated to the new one. */ int htAdd(hashtable *t, unsigned char *s, size_t len, void *data) { uint32_t hash; @@ -540,8 +540,8 @@ int test_randomWalkTest(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - char *toadd[] = {"alligator", "alien", "baloon", "chromodynamic", "romane", "romanus", "romulus", "rubens", - "ruber", "rubicon", "rubicundus", "all", "rub", "ba", NULL}; + char *toadd[] = {"alligator", "alien", "byword", "chromodynamic", "romane", "romanus", "romulus", "rubens", + "ruber", "rubicon", "rubicundus", "all", "rub", "by", NULL}; long numele; for (numele = 0; toadd[numele] != NULL; numele++) { @@ -583,8 +583,8 @@ int test_iteratorUnitTests(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - char *toadd[] = {"alligator", "alien", "baloon", "chromodynamic", "romane", "romanus", "romulus", "rubens", - "ruber", "rubicon", "rubicundus", "all", "rub", "ba", NULL}; + char *toadd[] = {"alligator", "alien", "byword", "chromodynamic", "romane", "romanus", "romulus", "rubens", + "ruber", "rubicon", "rubicundus", "all", "rub", "by", NULL}; for (int x = 0; x < 10000; x++) genrand64_int64(); @@ -612,7 +612,7 @@ int test_iteratorUnitTests(int argc, char **argv, int flags) { {"rub", 3, "<", "romulus"}, {"rom", 3, ">", "romane"}, {"chro", 4, ">", "chromodynamic"}, - {"chro", 4, "<", "baloon"}, + {"chro", 4, "<", "byword"}, {"chromz", 6, "<", "chromodynamic"}, {"", 0, "^", "alien"}, {"zorro", 5, "<=", "rubicundus"}, @@ -668,7 +668,7 @@ int test_tryInsertUnitTests(int argc, char **argv, int flags) { val = NULL; raxFind(t, (unsigned char *)"FOO", 3, &val); if (val != (void *)(long)1) { - printf("FOO value mismatch: is %p intead of 1", val); + printf("FOO value mismatch: is %p instead of 1", val); return 1; } @@ -676,7 +676,7 @@ int test_tryInsertUnitTests(int argc, char **argv, int flags) { val = NULL; raxFind(t, (unsigned char *)"FOO", 3, &val); if (val != (void *)(long)2) { - printf("FOO value mismatch: is %p intead of 2", val); + printf("FOO value mismatch: is %p instead of 2", val); return 1; } @@ -789,8 +789,8 @@ int test_regtest5(int argc, char **argv, int flags) { rax *rax = raxNew(); raxInsert(rax, (unsigned char *)"b", 1, (void *)(long)1, NULL); - raxInsert(rax, (unsigned char *)"ba", 2, (void *)(long)2, NULL); - raxInsert(rax, (unsigned char *)"banana", 6, (void *)(long)3, NULL); + raxInsert(rax, (unsigned char *)"by", 2, (void *)(long)2, NULL); + raxInsert(rax, (unsigned char *)"byword", 6, (void *)(long)3, NULL); raxInsert(rax, (unsigned char *)"f", 1, (void *)(long)4, NULL); raxInsert(rax, (unsigned char *)"foobar", 6, (void *)(long)5, NULL); From f25969b05803248baf29444cc89ca9fd24c4200f Mon Sep 17 00:00:00 2001 From: Guillaume Koenig <106696198+knggk@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:08:49 -0400 Subject: [PATCH 23/26] Update src/unit/test_rax.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Viktor Söderqvist Signed-off-by: Guillaume Koenig <106696198+knggk@users.noreply.github.com> --- src/unit/test_rax.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index a151740931..0945875e0a 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -47,9 +47,8 @@ long long _ustime(void); /* From test_crc64combine.c */ * used in order to test the radix tree implementation against something that * will always "tell the truth" :-) */ -#define HT_TABLE_SIZE \ - 100000 /* This is huge but we want it fast enough without \ - * reahshing needed. */ +/* This is huge but we want it fast enough without reahshing needed. */ +#define HT_TABLE_SIZE 100000 typedef struct htNode { uint64_t keylen; unsigned char *key; From 93b6b57e65790b6c5d354ca25b31842f5d5152d0 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Fri, 12 Jul 2024 09:40:03 -0400 Subject: [PATCH 24/26] Add rax into test names for better visualization Signed-off-by: Guillaume Koenig --- src/unit/test_files.h | 26 +++++++++++++------------- src/unit/test_rax.c | 24 ++++++++++++------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/unit/test_files.h b/src/unit/test_files.h index ad85570e3b..0b40224ed0 100644 --- a/src/unit/test_files.h +++ b/src/unit/test_files.h @@ -23,18 +23,18 @@ int test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyDict(int argc, char **argv, in int test_kvstoreIteratorRemoveAllKeysDeleteEmptyDict(int argc, char **argv, int flags); int test_kvstoreDictIteratorRemoveAllKeysNoDeleteEmptyDict(int argc, char **argv, int flags); int test_kvstoreDictIteratorRemoveAllKeysDeleteEmptyDict(int argc, char **argv, int flags); -int test_randomWalkTest(int argc, char **argv, int flags); -int test_iteratorUnitTests(int argc, char **argv, int flags); -int test_tryInsertUnitTests(int argc, char **argv, int flags); -int test_regtest1(int argc, char **argv, int flags); -int test_regtest2(int argc, char **argv, int flags); -int test_regtest3(int argc, char **argv, int flags); -int test_regtest4(int argc, char **argv, int flags); -int test_regtest5(int argc, char **argv, int flags); -int test_regtest6(int argc, char **argv, int flags); -int test_benchmark(int argc, char **argv, int flags); -int test_hugeKey(int argc, char **argv, int flags); -int test_fuzz(int argc, char **argv, int flags); +int test_raxRandomWalk(int argc, char **argv, int flags); +int test_raxIteratorUnitTests(int argc, char **argv, int flags); +int test_raxTryInsertUnitTests(int argc, char **argv, int flags); +int test_raxRegressionTest1(int argc, char **argv, int flags); +int test_raxRegressionTest2(int argc, char **argv, int flags); +int test_raxRegressionTest3(int argc, char **argv, int flags); +int test_raxRegressionTest4(int argc, char **argv, int flags); +int test_raxRegressionTest5(int argc, char **argv, int flags); +int test_raxRegressionTest6(int argc, char **argv, int flags); +int test_raxBenchmark(int argc, char **argv, int flags); +int test_raxHugeKey(int argc, char **argv, int flags); +int test_raxFuzz(int argc, char **argv, int flags); int test_sds(int argc, char **argv, int flags); int test_typesAndAllocSize(int argc, char **argv, int flags); int test_sdsHeaderSizes(int argc, char **argv, int flags); @@ -91,7 +91,7 @@ unitTest __test_crc64combine_c[] = {{"test_crc64combine", test_crc64combine}, {N unitTest __test_endianconv_c[] = {{"test_endianconv", test_endianconv}, {NULL, NULL}}; unitTest __test_intset_c[] = {{"test_intsetValueEncodings", test_intsetValueEncodings}, {"test_intsetBasicAdding", test_intsetBasicAdding}, {"test_intsetLargeNumberRandomAdd", test_intsetLargeNumberRandomAdd}, {"test_intsetUpgradeFromint16Toint32", test_intsetUpgradeFromint16Toint32}, {"test_intsetUpgradeFromint16Toint64", test_intsetUpgradeFromint16Toint64}, {"test_intsetUpgradeFromint32Toint64", test_intsetUpgradeFromint32Toint64}, {"test_intsetStressLookups", test_intsetStressLookups}, {"test_intsetStressAddDelete", test_intsetStressAddDelete}, {NULL, NULL}}; unitTest __test_kvstore_c[] = {{"test_kvstoreAdd16Keys", test_kvstoreAdd16Keys}, {"test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyDict", test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyDict}, {"test_kvstoreIteratorRemoveAllKeysDeleteEmptyDict", test_kvstoreIteratorRemoveAllKeysDeleteEmptyDict}, {"test_kvstoreDictIteratorRemoveAllKeysNoDeleteEmptyDict", test_kvstoreDictIteratorRemoveAllKeysNoDeleteEmptyDict}, {"test_kvstoreDictIteratorRemoveAllKeysDeleteEmptyDict", test_kvstoreDictIteratorRemoveAllKeysDeleteEmptyDict}, {NULL, NULL}}; -unitTest __test_rax_c[] = {{"test_randomWalkTest", test_randomWalkTest}, {"test_iteratorUnitTests", test_iteratorUnitTests}, {"test_tryInsertUnitTests", test_tryInsertUnitTests}, {"test_regtest1", test_regtest1}, {"test_regtest2", test_regtest2}, {"test_regtest3", test_regtest3}, {"test_regtest4", test_regtest4}, {"test_regtest5", test_regtest5}, {"test_regtest6", test_regtest6}, {"test_benchmark", test_benchmark}, {"test_hugeKey", test_hugeKey}, {"test_fuzz", test_fuzz}, {NULL, NULL}}; +unitTest __test_rax_c[] = {{"test_raxRandomWalk", test_raxRandomWalk}, {"test_raxIteratorUnitTests", test_raxIteratorUnitTests}, {"test_raxTryInsertUnitTests", test_raxTryInsertUnitTests}, {"test_raxRegressionTest1", test_raxRegressionTest1}, {"test_raxRegressionTest2", test_raxRegressionTest2}, {"test_raxRegressionTest3", test_raxRegressionTest3}, {"test_raxRegressionTest4", test_raxRegressionTest4}, {"test_raxRegressionTest5", test_raxRegressionTest5}, {"test_raxRegressionTest6", test_raxRegressionTest6}, {"test_raxBenchmark", test_raxBenchmark}, {"test_raxHugeKey", test_raxHugeKey}, {"test_raxFuzz", test_raxFuzz}, {NULL, NULL}}; unitTest __test_sds_c[] = {{"test_sds", test_sds}, {"test_typesAndAllocSize", test_typesAndAllocSize}, {"test_sdsHeaderSizes", test_sdsHeaderSizes}, {NULL, NULL}}; unitTest __test_sha1_c[] = {{"test_sha1", test_sha1}, {NULL, NULL}}; unitTest __test_util_c[] = {{"test_string2ll", test_string2ll}, {"test_string2l", test_string2l}, {"test_ll2string", test_ll2string}, {"test_ld2string", test_ld2string}, {"test_fixedpoint_d2string", test_fixedpoint_d2string}, {"test_version2num", test_version2num}, {"test_reclaimFilePageCache", test_reclaimFilePageCache}, {NULL, NULL}}; diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index 0945875e0a..5dcc4aa7c3 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -533,7 +533,7 @@ int iteratorFuzzTest(int keymode, size_t count) { } /* Test the random walk function. */ -int test_randomWalkTest(int argc, char **argv, int flags) { +int test_raxRandomWalk(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -576,7 +576,7 @@ int test_randomWalkTest(int argc, char **argv, int flags) { return 0; } -int test_iteratorUnitTests(int argc, char **argv, int flags) { +int test_raxIteratorUnitTests(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -650,7 +650,7 @@ int test_iteratorUnitTests(int argc, char **argv, int flags) { /* Test that raxInsert() / raxTryInsert() overwrite semantic * works as expected. */ -int test_tryInsertUnitTests(int argc, char **argv, int flags) { +int test_raxTryInsertUnitTests(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -684,7 +684,7 @@ int test_tryInsertUnitTests(int argc, char **argv, int flags) { } /* Regression test #1: Iterator wrong element returned after seek. */ -int test_regtest1(int argc, char **argv, int flags) { +int test_raxRegressionTest1(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -712,7 +712,7 @@ int test_regtest1(int argc, char **argv, int flags) { } /* Regression test #2: Crash when mixing NULL and not NULL values. */ -int test_regtest2(int argc, char **argv, int flags) { +int test_raxRegressionTest2(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -733,7 +733,7 @@ int test_regtest2(int argc, char **argv, int flags) { * * Note that this test always returns success but will trigger a * Valgrind error. */ -int test_regtest3(int argc, char **argv, int flags) { +int test_raxRegressionTest3(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -753,7 +753,7 @@ int test_regtest3(int argc, char **argv, int flags) { * however we are using the original one from the bug report, since this * is quite odd and may later protect against different bugs related to * storing and fetching the empty string key. */ -int test_regtest4(int argc, char **argv, int flags) { +int test_raxRegressionTest4(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -780,7 +780,7 @@ int test_regtest4(int argc, char **argv, int flags) { } /* Less than seek bug when stopping in the middle of a compressed node. */ -int test_regtest5(int argc, char **argv, int flags) { +int test_raxRegressionTest5(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -811,7 +811,7 @@ int test_regtest5(int argc, char **argv, int flags) { } /* Seek may not populate iterator data. See issue #25. */ -int test_regtest6(int argc, char **argv, int flags) { +int test_raxRegressionTest6(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); UNUSED(flags); @@ -839,7 +839,7 @@ int test_regtest6(int argc, char **argv, int flags) { return 0; } -int test_benchmark(int argc, char **argv, int flags) { +int test_raxBenchmark(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); @@ -924,7 +924,7 @@ int test_benchmark(int argc, char **argv, int flags) { * the code to handle this edge case works as expected. * * This test is disabled by default because it uses a lot of memory. */ -int test_hugeKey(int argc, char **argv, int flags) { +int test_raxHugeKey(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); @@ -962,7 +962,7 @@ int test_hugeKey(int argc, char **argv, int flags) { exit(1); } -int test_fuzz(int argc, char **argv, int flags) { +int test_raxFuzz(int argc, char **argv, int flags) { UNUSED(argc); UNUSED(argv); From b2c3383f16aab6908b3888aa56dac760f958529e Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Fri, 12 Jul 2024 10:31:17 -0400 Subject: [PATCH 25/26] Plug raxAllocSize to mem reporting spots Signed-off-by: Guillaume Koenig --- src/module.c | 6 ++---- src/object.c | 29 +++-------------------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/src/module.c b/src/module.c index c98c837a5c..6337ee7eb3 100644 --- a/src/module.c +++ b/src/module.c @@ -10796,10 +10796,8 @@ size_t VM_MallocSizeString(ValkeyModuleString *str) { * it does not include the allocation size of the keys and values. */ size_t VM_MallocSizeDict(ValkeyModuleDict *dict) { - size_t size = sizeof(ValkeyModuleDict) + sizeof(rax); - size += dict->rax->numnodes * sizeof(raxNode); - /* For more info about this weird line, see streamRadixTreeMemoryUsage */ - size += dict->rax->numnodes * sizeof(long) * 30; + size_t size = sizeof(ValkeyModuleDict); + size += raxAllocSize(dict->rax); return size; } diff --git a/src/object.c b/src/object.c index 73c3de55dd..31260625b8 100644 --- a/src/object.c +++ b/src/object.c @@ -943,29 +943,6 @@ char *strEncoding(int encoding) { /* =========================== Memory introspection ========================= */ -/* This is a helper function with the goal of estimating the memory - * size of a radix tree that is used to store Stream IDs. - * - * Note: to guess the size of the radix tree is not trivial, so we - * approximate it considering 16 bytes of data overhead for each - * key (the ID), and then adding the number of bare nodes, plus some - * overhead due by the data and child pointers. This secret recipe - * was obtained by checking the average radix tree created by real - * workloads, and then adjusting the constants to get numbers that - * more or less match the real memory usage. - * - * Actually the number of nodes and keys may be different depending - * on the insertion speed and thus the ability of the radix tree - * to compress prefixes. */ -size_t streamRadixTreeMemoryUsage(rax *rax) { - size_t size = sizeof(*rax); - size = rax->numele * sizeof(streamID); - size += rax->numnodes * sizeof(raxNode); - /* Add a fixed overhead due to the aux data pointer, children, ... */ - size += rax->numnodes * sizeof(long) * 30; - return size; -} - /* Returns the size in bytes consumed by the key's value in RAM. * Note that the returned value is just an approximation, especially in the * case of aggregated data types where only "sample_size" elements @@ -1063,7 +1040,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { } else if (o->type == OBJ_STREAM) { stream *s = o->ptr; asize = sizeof(*o) + sizeof(*s); - asize += streamRadixTreeMemoryUsage(s->rax); + asize += raxAllocSize(s->rax); /* Now we have to add the listpacks. The last listpack is often non * complete, so we estimate the size of the first N listpacks, and @@ -1103,7 +1080,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { while (raxNext(&ri)) { streamCG *cg = ri.data; asize += sizeof(*cg); - asize += streamRadixTreeMemoryUsage(cg->pel); + asize += raxAllocSize(cg->pel); asize += sizeof(streamNACK) * raxSize(cg->pel); /* For each consumer we also need to add the basic data @@ -1115,7 +1092,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { streamConsumer *consumer = cri.data; asize += sizeof(*consumer); asize += sdslen(consumer->name); - asize += streamRadixTreeMemoryUsage(consumer->pel); + asize += raxAllocSize(consumer->pel); /* Don't count NACKs again, they are shared with the * consumer group PEL. */ } From fee849dbdfeaece927a58dc168dda17c4ad4e421 Mon Sep 17 00:00:00 2001 From: Guillaume Koenig Date: Wed, 2 Oct 2024 12:09:06 -0400 Subject: [PATCH 26/26] Fix formatting Signed-off-by: Guillaume Koenig --- src/unit/test_rax.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/unit/test_rax.c b/src/unit/test_rax.c index 5dcc4aa7c3..5f346b4115 100644 --- a/src/unit/test_rax.c +++ b/src/unit/test_rax.c @@ -539,8 +539,8 @@ int test_raxRandomWalk(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - char *toadd[] = {"alligator", "alien", "byword", "chromodynamic", "romane", "romanus", "romulus", "rubens", - "ruber", "rubicon", "rubicundus", "all", "rub", "by", NULL}; + char *toadd[] = {"alligator", "alien", "byword", "chromodynamic", "romane", "romanus", "romulus", "rubens", + "ruber", "rubicon", "rubicundus", "all", "rub", "by", NULL}; long numele; for (numele = 0; toadd[numele] != NULL; numele++) { @@ -582,8 +582,8 @@ int test_raxIteratorUnitTests(int argc, char **argv, int flags) { UNUSED(flags); rax *t = raxNew(); - char *toadd[] = {"alligator", "alien", "byword", "chromodynamic", "romane", "romanus", "romulus", "rubens", - "ruber", "rubicon", "rubicundus", "all", "rub", "by", NULL}; + char *toadd[] = {"alligator", "alien", "byword", "chromodynamic", "romane", "romanus", "romulus", "rubens", + "ruber", "rubicon", "rubicundus", "all", "rub", "by", NULL}; for (int x = 0; x < 10000; x++) genrand64_int64();