From 76fc041685217ad7279c486be892c16f182a3454 Mon Sep 17 00:00:00 2001 From: Harkrishn Patro Date: Thu, 13 Jun 2024 15:58:03 -0700 Subject: [PATCH 01/11] represent cluster node flags with bitwise shift value (#642) While debugging a cluster bus issue, found the cluster node flags were represented in numbers. I generally find it easy when these are represented as bitwise shift operation. It improves readability a bit. Signed-off-by: Harkrishn Patro --- src/cluster_legacy.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/cluster_legacy.h b/src/cluster_legacy.h index 4cb715730f..fb80f45eec 100644 --- a/src/cluster_legacy.h +++ b/src/cluster_legacy.h @@ -41,17 +41,17 @@ typedef struct clusterLink { } clusterLink; /* Cluster node flags and macros. */ -#define CLUSTER_NODE_PRIMARY 1 /* The node is a primary */ -#define CLUSTER_NODE_REPLICA 2 /* The node is a replica */ -#define CLUSTER_NODE_PFAIL 4 /* Failure? Need acknowledge */ -#define CLUSTER_NODE_FAIL 8 /* The node is believed to be malfunctioning */ -#define CLUSTER_NODE_MYSELF 16 /* This node is myself */ -#define CLUSTER_NODE_HANDSHAKE 32 /* We have still to exchange the first ping */ -#define CLUSTER_NODE_NOADDR 64 /* We don't know the address of this node */ -#define CLUSTER_NODE_MEET 128 /* Send a MEET message to this node */ -#define CLUSTER_NODE_MIGRATE_TO 256 /* Primary eligible for replica migration. */ -#define CLUSTER_NODE_NOFAILOVER 512 /* replica will not try to failover. */ -#define CLUSTER_NODE_EXTENSIONS_SUPPORTED 1024 /* This node supports extensions. */ +#define CLUSTER_NODE_PRIMARY (1 << 0) /* The node is a primary */ +#define CLUSTER_NODE_REPLICA (1 << 1) /* The node is a replica */ +#define CLUSTER_NODE_PFAIL (1 << 2) /* Failure? Need acknowledge */ +#define CLUSTER_NODE_FAIL (1 << 3) /* The node is believed to be malfunctioning */ +#define CLUSTER_NODE_MYSELF (1 << 4) /* This node is myself */ +#define CLUSTER_NODE_HANDSHAKE (1 << 5) /* We have still to exchange the first ping */ +#define CLUSTER_NODE_NOADDR (1 << 6) /* We don't know the address of this node */ +#define CLUSTER_NODE_MEET (1 << 7) /* Send a MEET message to this node */ +#define CLUSTER_NODE_MIGRATE_TO (1 << 8) /* Primary eligible for replica migration. */ +#define CLUSTER_NODE_NOFAILOVER (1 << 9) /* replica will not try to failover. */ +#define CLUSTER_NODE_EXTENSIONS_SUPPORTED (1 << 10) /* This node supports extensions. */ #define CLUSTER_NODE_NULL_NAME \ "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ "\000\000\000\000\000\000\000\000\000\000\000\000" From d309b9b23550cd64621bf62f42504bf23838ab15 Mon Sep 17 00:00:00 2001 From: Binbin Date: Fri, 14 Jun 2024 07:47:20 +0800 Subject: [PATCH 02/11] Make configs dir/dbfilename/cluster-config-file reject empty string (#636) Until now, these configuration items allowed typing empty strings, but empty strings behave strangely. Empty dir will fail in chdir with No such file or directory: ``` ./src/valkey-server --dir "" *** FATAL CONFIG FILE ERROR (Version 255.255.255) *** Reading the configuration file, at line 2 >>> 'dir ""' No such file or directory ``` Empty dbfilename will cause shutdown to fail since it will always fail in rdb save: ``` ./src/valkey-server --dbfilename "" * User requested shutdown... * Saving the final RDB snapshot before exiting. # Error moving temp DB file temp-19530.rdb on the final destination (in server root dir /xxx/xxx/valkey): No such file or directory # Error trying to save the DB, can't exit. # Errors trying to shut down the server. Check the logs for more information. ``` Empty cluster-config-file will fail in clusterLockConfig: ``` ./src/valkey-server --cluster-enabled yes --cluster-config-file "" Can't open in order to acquire a lock: No such file or directory ``` With this patch, now we will just reject it in config set like: ``` *** FATAL CONFIG FILE ERROR (Version 255.255.255) *** Reading the configuration file, at line 2 >>> 'xxx ""' xxx can't be empty ``` Signed-off-by: Binbin --- src/config.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index a609a8f18d..83e2a51db1 100644 --- a/src/config.c +++ b/src/config.c @@ -2295,7 +2295,19 @@ static int isValidActiveDefrag(int val, const char **err) { return 1; } +static int isValidClusterConfigFile(char *val, const char **err) { + if (!strcmp(val, "")) { + *err = "cluster-config-file can't be empty"; + return 0; + } + return 1; +} + static int isValidDBfilename(char *val, const char **err) { + if (!strcmp(val, "")) { + *err = "dbfilename can't be empty"; + return 0; + } if (!pathIsBaseName(val)) { *err = "dbfilename can't be a path, just a filename"; return 0; @@ -2658,6 +2670,10 @@ static int setConfigDirOption(standardConfig *config, sds *argv, int argc, const *err = "wrong number of arguments"; return 0; } + if (!strcmp(argv[0], "")) { + *err = "dir can't be empty"; + return 0; + } if (chdir(argv[0]) == -1) { *err = strerror(errno); return 0; @@ -3061,7 +3077,7 @@ standardConfig static_configs[] = { createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.replica_announce_ip, NULL, NULL, NULL), createStringConfig("primaryuser", "masteruser", MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.primary_user, NULL, NULL, NULL), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, updateClusterIp), - createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", NULL, NULL), + createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", isValidClusterConfigFile, NULL), createStringConfig("cluster-announce-hostname", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_hostname, NULL, isValidAnnouncedHostname, updateClusterHostname), createStringConfig("cluster-announce-human-nodename", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_human_nodename, NULL, isValidAnnouncedNodename, updateClusterHumanNodename), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, SERVER_NAME, NULL, NULL), From 5d9d41868df1a1e21de73f69e8b36feadf271fe7 Mon Sep 17 00:00:00 2001 From: Ping Xie Date: Thu, 13 Jun 2024 17:52:50 -0700 Subject: [PATCH 03/11] Replace `DEBUG RESTART` with `pause_server` and `resume_server` (#652) --- tests/unit/cluster/slot-migration.tcl | 33 ++++++++++----------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/tests/unit/cluster/slot-migration.tcl b/tests/unit/cluster/slot-migration.tcl index 2016edcbc1..3ff669db90 100644 --- a/tests/unit/cluster/slot-migration.tcl +++ b/tests/unit/cluster/slot-migration.tcl @@ -43,20 +43,11 @@ proc check_server_response {server_id} { } # restart a server and wait for it to come back online -proc restart_server_and_wait {server_id} { +proc fail_server {server_id} { set node_timeout [lindex [R 0 CONFIG GET cluster-node-timeout] 1] - set result [catch {R $server_id DEBUG RESTART [expr 3*$node_timeout]} err] - - # Check if the error is the expected "I/O error reading reply" - if {$result != 0 && $err ne "I/O error reading reply"} { - fail "Unexpected error restarting server $server_id: $err" - } - - wait_for_condition 100 100 { - [check_server_response $server_id] eq 1 - } else { - fail "Server $server_id didn't come back online in time" - } + pause_process [srv [expr -1*$server_id] pid] + after [expr 3*$node_timeout] + resume_process [srv [expr -1*$server_id] pid] } start_cluster 3 3 {tags {external:skip cluster} overrides {cluster-allow-replica-migration no cluster-node-timeout 1000} } { @@ -91,8 +82,8 @@ start_cluster 3 3 {tags {external:skip cluster} overrides {cluster-allow-replica } test "Migration target is auto-updated after failover in target shard" { - # Restart R1 to trigger an auto-failover to R4 - restart_server_and_wait 1 + # Trigger an auto-failover from R1 to R4 + fail_server 1 # Wait for R1 to become a replica wait_for_role 1 slave # Validate final states @@ -111,8 +102,8 @@ start_cluster 3 3 {tags {external:skip cluster} overrides {cluster-allow-replica } test "Migration source is auto-updated after failover in source shard" { - # Restart R0 to trigger an auto-failover to R3 - restart_server_and_wait 0 + # Trigger an auto-failover from R0 to R3 + fail_server 0 # Wait for R0 to become a replica wait_for_role 0 slave # Validate final states @@ -261,8 +252,8 @@ start_cluster 3 5 {tags {external:skip cluster} overrides {cluster-allow-replica test "Empty-shard migration target is auto-updated after faiover in target shard" { wait_for_role 6 master - # Restart R6 to trigger an auto-failover to R7 - restart_server_and_wait 6 + # Trigger an auto-failover from R6 to R7 + fail_server 6 # Wait for R6 to become a replica wait_for_role 6 slave # Validate final states @@ -282,8 +273,8 @@ start_cluster 3 5 {tags {external:skip cluster} overrides {cluster-allow-replica test "Empty-shard migration source is auto-updated after faiover in source shard" { wait_for_role 0 master - # Restart R0 to trigger an auto-failover to R3 - restart_server_and_wait 0 + # Trigger an auto-failover from R0 to R3 + fail_server 0 # Wait for R0 to become a replica wait_for_role 0 slave # Validate final states From 8a776c35090c287a309ed49514cc3051d9f39707 Mon Sep 17 00:00:00 2001 From: Ping Xie Date: Thu, 13 Jun 2024 23:43:36 -0700 Subject: [PATCH 04/11] Fix potential infinite loop in `clusterNodeGetPrimary` (#651) --- src/cluster_legacy.c | 44 +++++++++----------------------------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index cd3786fe05..f566cf5a35 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -3128,39 +3128,7 @@ int clusterProcessPacket(clusterLink *link) { clusterUpdateSlotsConfigWith(sender_primary, senderConfigEpoch, hdr->myslots); /* Explicitly check for a replication loop before attempting the replication - * chain folding logic. - * - * In some rare case, slot config updates (via either PING/PONG or UPDATE) - * can be delivered out of order as illustrated below. - * - * 1. To keep the discussion simple, let's assume we have 2 shards, shard a - * and shard b. Let's also assume there are two slots in total with shard - * a owning slot 1 and shard b owning slot 2. - * 2. Shard a has two nodes: primary A and replica A*; shard b has primary - * B and replica B*. - * 3. A manual failover was initiated on A* and A* just wins the election. - * 4. A* announces to the world that it now owns slot 1 using PING messages. - * These PING messages are queued in the outgoing buffer to every other - * node in the cluster, namely, A, B, and B*. - * 5. Keep in mind that there is no ordering in the delivery of these PING - * messages. For the stale PING message to appear, we need the following - * events in the exact order as they are laid out. - * a. An old PING message before A* becomes the new primary is still queued - * in A*'s outgoing buffer to A. This later becomes the stale message, - * which says A* is a replica of A. It is followed by A*'s election - * winning announcement PING message. - * b. B or B* processes A's election winning announcement PING message - * and sets slots[1]=A*. - * c. A sends a PING message to B (or B*). Since A hasn't learnt that A* - * wins the election, it claims that it owns slot 1 but with a lower - * epoch than B has on slot 1. This leads to B sending an UPDATE to - * A directly saying A* is the new owner of slot 1 with a higher epoch. - * d. A receives the UPDATE from B and executes clusterUpdateSlotsConfigWith. - * A now realizes that it is a replica of A* hence setting myself->replicaof - * to A*. - * e. Finally, the pre-failover PING message queued up in A*'s outgoing - * buffer to A is delivered and processed, out of order though, to A. - * f. This stale PING message creates the replication loop */ + * chain folding logic. */ if (myself->replicaof && myself->replicaof->replicaof && myself->replicaof->replicaof != myself) { /* Safeguard against sub-replicas. A replica's primary can turn itself * into a replica if its last slot is removed. If no other node takes @@ -5853,8 +5821,14 @@ int clusterNodeIsReplica(clusterNode *node) { } clusterNode *clusterNodeGetPrimary(clusterNode *node) { - while (node->replicaof != NULL) node = node->replicaof; - return node; + clusterNode *primary = node; + while (primary->replicaof != NULL) { + primary = primary->replicaof; + if (primary == node) break; + } + /* Assert that a node's replicaof/primary chain does not form a cycle. */ + debugServerAssert(primary->replicaof == NULL); + return primary; } char *clusterNodeGetName(clusterNode *node) { From 4c6bf30f58335baa4564415a0a75011d23aa1714 Mon Sep 17 00:00:00 2001 From: Wenwen-Chen Date: Fri, 14 Jun 2024 18:56:59 +0800 Subject: [PATCH 05/11] Latency report: Rebranding and refine Dave dialog (#644) This patch try to correct the latency report. 1. Rename Redis to Valkey. 2. Remove redundant Dave dialog, and refine the output message. --------- Signed-off-by: Wenwen Chen Signed-off-by: hwware --- src/latency.c | 69 ++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/src/latency.c b/src/latency.c index 94840379bd..eacb5fbbc4 100644 --- a/src/latency.c +++ b/src/latency.c @@ -216,9 +216,8 @@ sds createLatencyReport(void) { * was never enabled so far. */ if (dictSize(server.latency_events) == 0 && server.latency_monitor_threshold == 0) { report = sdscat( - report, "I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Redis instance. You may " - "use \"CONFIG SET latency-monitor-threshold .\" in order to enable it. If we weren't " - "in a deep space mission I'd suggest to take a look at https://redis.io/topics/latency-monitor.\n"); + report, "I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Valkey instance. You may " + "use \"CONFIG SET latency-monitor-threshold .\" in order to enable it.\n"); return report; } @@ -237,8 +236,7 @@ sds createLatencyReport(void) { if (ts == NULL) continue; eventnum++; if (eventnum == 1) { - report = sdscat(report, "Dave, I have observed latency spikes in this Redis instance. You don't mind " - "talking about it, do you Dave?\n\n"); + report = sdscat(report, "Latency spikes are observed in this Valkey instance.\n\n"); } analyzeLatencyForEvent(event, &ls); @@ -358,18 +356,16 @@ sds createLatencyReport(void) { } if (eventnum == 0 && advices == 0) { - report = sdscat(report, "Dave, no latency spike was observed during the lifetime of this Redis instance, not " - "in the slightest bit. I honestly think you ought to sit down calmly, take a stress " - "pill, and think things over.\n"); + report = sdscat(report, "No latency spike was observed during the lifetime of this Valkey instance, not " + "in the slightest bit.\n"); } else if (eventnum > 0 && advices == 0) { - report = - sdscat(report, "\nWhile there are latency events logged, I'm not able to suggest any easy fix. Please use " - "the Redis community to get some help, providing this report in your help request.\n"); + report = sdscat(report, "\nThere are latency events logged that are not easy to fix. Please get some " + "help from Valkey community, providing this report in your help request.\n"); } else { /* Add all the suggestions accumulated so far. */ /* Better VM. */ - report = sdscat(report, "\nI have a few advices for you:\n\n"); + report = sdscat(report, "\nHere is some advice for you:\n\n"); if (advise_better_vm) { report = sdscat(report, "- If you are using a virtual machine, consider upgrading it with a faster one " "using a hypervisior that provides less latency during fork() calls. Xen is known " @@ -382,8 +378,8 @@ sds createLatencyReport(void) { report = sdscatprintf(report, "- There are latency issues with potentially slow commands you are using. Try to enable " - "the Slow Log Redis feature using the command 'CONFIG SET slowlog-log-slower-than %llu'. " - "If the Slow log is disabled Redis is not able to log slow commands execution for you.\n", + "the Slow Log Valkey feature using the command 'CONFIG SET slowlog-log-slower-than %llu'. " + "If the Slow log is disabled Valkey is not able to log slow commands execution for you.\n", (unsigned long long)server.latency_monitor_threshold * 1000); } @@ -398,20 +394,20 @@ sds createLatencyReport(void) { if (advise_slowlog_inspect) { report = sdscat(report, "- Check your Slow Log to understand what are the commands you are running which are too " - "slow to execute. Please check https://redis.io/commands/slowlog for more information.\n"); + "slow to execute. Please check https://valkey.io/commands/slowlog for more information.\n"); } /* Intrinsic latency. */ if (advise_scheduler) { report = sdscat( report, - "- The system is slow to execute Redis code paths not containing system calls. This usually means the " - "system does not provide Redis CPU time to run for long periods. You should try to:\n" + "- The system is slow to execute Valkey code paths not containing system calls. This usually means the " + "system does not provide Valkey CPU time to run for long periods. You should try to:\n" " 1) Lower the system load.\n" - " 2) Use a computer / VM just for Redis if you are running other software in the same system.\n" + " 2) Use a computer / VM just for Valkey if you are running other software in the same system.\n" " 3) Check if you have a \"noisy neighbour\" problem.\n" - " 4) Check with 'redis-cli --intrinsic-latency 100' what is the intrinsic latency in your system.\n" - " 5) Check if the problem is allocator-related by recompiling Redis with MALLOC=libc, if you are " + " 4) Check with 'valkey-cli --intrinsic-latency 100' what is the intrinsic latency in your system.\n" + " 5) Check if the problem is allocator-related by recompiling Valkey with MALLOC=libc, if you are " "using Jemalloc. However this may create fragmentation problems.\n"); } @@ -423,11 +419,11 @@ sds createLatencyReport(void) { } if (advise_ssd) { - report = - sdscat(report, "- SSD disks are able to reduce fsync latency, and total time needed for snapshotting " - "and AOF log rewriting (resulting in smaller memory usage). With extremely high write " - "load SSD disks can be a good option. However Redis should perform reasonably with high " - "load using normal disks. Use this advice as a last resort.\n"); + report = sdscat(report, + "- SSD disks are able to reduce fsync latency, and total time needed for snapshotting " + "and AOF log rewriting (resulting in smaller memory usage). With extremely high write " + "load SSD disks can be a good option. However Valkey should perform reasonably with high " + "load using normal disks. Use this advice as a last resort.\n"); } if (advise_data_writeback) { @@ -435,12 +431,12 @@ sds createLatencyReport(void) { sdscat(report, "- Mounting ext3/4 filesystems with data=writeback can provide a performance boost " "compared to data=ordered, however this mode of operation provides less guarantees, and " "sometimes it can happen that after a hard crash the AOF file will have a half-written " - "command at the end and will require to be repaired before Redis restarts.\n"); + "command at the end and will require to be repaired before Valkey restarts.\n"); } if (advise_disk_contention) { report = sdscat(report, "- Try to lower the disk contention. This is often caused by other disk intensive " - "processes running in the same computer (including other Redis instances).\n"); + "processes running in the same computer (including other Valkey instances).\n"); } if (advise_no_appendfsync) { @@ -464,7 +460,7 @@ sds createLatencyReport(void) { } if (advise_hz && server.hz < 100) { - report = sdscat(report, "- In order to make the Redis keys expiring process more incremental, try to set " + report = sdscat(report, "- In order to make the Valkey keys expiring process more incremental, try to set " "the 'hz' configuration parameter to 100 using 'CONFIG SET hz 100'.\n"); } @@ -477,19 +473,20 @@ sds createLatencyReport(void) { if (advise_mass_eviction) { report = sdscat(report, "- Sudden changes to the 'maxmemory' setting via 'CONFIG SET', or allocation of " - "large objects via sets or sorted sets intersections, STORE option of SORT, Redis " + "large objects via sets or sorted sets intersections, STORE option of SORT, Valkey " "Cluster large keys migrations (RESTORE command), may create sudden memory " "pressure forcing the server to block trying to evict keys. \n"); } if (advise_disable_thp) { - report = sdscat(report, "- I detected a non zero amount of anonymous huge pages used by your process. This " - "creates very serious latency events in different conditions, especially when " - "Redis is persisting on disk. To disable THP support use the command 'echo never > " - "/sys/kernel/mm/transparent_hugepage/enabled', make sure to also add it into " - "/etc/rc.local so that the command will be executed again after a reboot. Note " - "that even if you have already disabled THP, you still need to restart the Redis " - "process to get rid of the huge pages already created.\n"); + report = + sdscat(report, "- I detected a non zero amount of anonymous huge pages used by your process. This " + "creates very serious latency events in different conditions, especially when " + "Valkey is persisting on disk. To disable THP support use the command 'echo never > " + "/sys/kernel/mm/transparent_hugepage/enabled', make sure to also add it into " + "/etc/rc.local so that the command will be executed again after a reboot. Note " + "that even if you have already disabled THP, you still need to restart the Valkey " + "process to get rid of the huge pages already created.\n"); } } From d5496e42bc601abc76bcd58bd9a4e6f2271a4578 Mon Sep 17 00:00:00 2001 From: Binbin Date: Fri, 14 Jun 2024 23:32:19 +0800 Subject: [PATCH 06/11] Lua scripts promoted from eval to script load to avoid evict (#637) In ad28d222edcef9d4496fd7a94656013f07dd08e5, we added a Lua eval scripts eviction. If the script was previously added via EVAL, we promote it to SCRIPT LOAD, prevent it from being evicted later. Signed-off-by: Binbin --- src/eval.c | 7 +++++++ tests/unit/scripting.tcl | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/eval.c b/src/eval.c index 8c6db3f18b..e747c233e8 100644 --- a/src/eval.c +++ b/src/eval.c @@ -447,6 +447,13 @@ sds luaCreateFunction(client *c, robj *body, int evalsha) { sha1hex(funcname + 2, body->ptr, sdslen(body->ptr)); if ((de = dictFind(lctx.lua_scripts, funcname + 2)) != NULL) { + /* If the script was previously added via EVAL, we promote it to + * SCRIPT LOAD, prevent it from being evicted later. */ + luaScript *l = dictGetVal(de); + if (evalsha && l->node) { + listDelNode(lctx.lua_scripts_lru_list, l->node); + l->node = NULL; + } return dictGetKey(de); } diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 0b7206e2a0..3f8c761bd3 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -1646,6 +1646,19 @@ start_server {tags {"scripting external:skip"}} { # cached = num load scripts + 500 eval scripts assert_equal $cached [expr $num+500] } + + test {Lua scripts promoted from eval to script load} { + r script flush + r config resetstat + + r eval "return 'hello world'" 0 + set sha [r script load "return 'hello world'"] + for {set j 1} {$j <= 500} {incr j} { + r script load "return $j" + r eval "return 'str_$j'" 0 + } + assert_equal {hello world} [r evalsha $sha 0] + } } } ;# is_eval From 6faa48a358ead15ea4b86798aa832a4820dfbe4e Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 14 Jun 2024 08:42:00 -0700 Subject: [PATCH 07/11] Don't initialize the key buffer in getKeysResult (#631) getKeysResults is typically initialized with 2kb of zeros (16 * 256), which isn't strictly necessary since the only thing we have to initialize is some of the metadata fields. The rest of the data can remain junk as long as we don't access it. This was a bit of a regression in 7.0 with the keyspecs, since we doubled the size of the zeros, but hopefully this recovers a lot of the performance drop. I saw a modest performance bump for deep pipeline of cluster mode (~8%). I think we would see some comparable improvements in the other places where we are using it such as tracking and ACLs. --------- Signed-off-by: Madelyn Olson --- src/acl.c | 5 +++-- src/cluster.c | 3 ++- src/db.c | 2 +- src/module.c | 3 ++- src/server.c | 3 ++- src/server.h | 11 ++++++++--- src/tracking.c | 3 ++- 7 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/acl.c b/src/acl.c index 533782acad..bda449e8d2 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1713,7 +1713,7 @@ static int ACLSelectorCheckCmd(aclSelector *selector, * mentioned in the command arguments. */ if (!(selector->flags & SELECTOR_FLAG_ALLKEYS) && doesCommandHaveKeys(cmd)) { if (!(cache->keys_init)) { - cache->keys = (getKeysResult)GETKEYS_RESULT_INIT; + initGetKeysResult(&(cache->keys)); getKeysFromCommandWithSpecs(cmd, argv, argc, GET_KEYSPEC_DEFAULT, &(cache->keys)); cache->keys_init = 1; } @@ -1733,7 +1733,8 @@ static int ACLSelectorCheckCmd(aclSelector *selector, * mentioned in the command arguments */ const int channel_flags = CMD_CHANNEL_PUBLISH | CMD_CHANNEL_SUBSCRIBE; if (!(selector->flags & SELECTOR_FLAG_ALLCHANNELS) && doesCommandHaveChannelsWithFlags(cmd, channel_flags)) { - getKeysResult channels = (getKeysResult)GETKEYS_RESULT_INIT; + getKeysResult channels; + initGetKeysResult(&channels); getChannelsFromCommand(cmd, argv, argc, &channels); keyReference *channelref = channels.keys; for (int j = 0; j < channels.numkeys; j++) { diff --git a/src/cluster.c b/src/cluster.c index 3a4dccdff5..00f3c2d889 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -1017,7 +1017,8 @@ getNodeByQuery(client *c, struct serverCommand *cmd, robj **argv, int argc, int margc = ms->commands[i].argc; margv = ms->commands[i].argv; - getKeysResult result = GETKEYS_RESULT_INIT; + getKeysResult result; + initGetKeysResult(&result); numkeys = getKeysFromCommand(mcmd, margv, margc, &result); keyindex = result.keys; diff --git a/src/db.c b/src/db.c index 1843395d8c..c879b2ffb5 100644 --- a/src/db.c +++ b/src/db.c @@ -1894,7 +1894,7 @@ unsigned long long dbScan(serverDb *db, unsigned long long cursor, dictScanFunct * the result, and can be called repeatedly to enlarge the result array. */ keyReference *getKeysPrepareResult(getKeysResult *result, int numkeys) { - /* GETKEYS_RESULT_INIT initializes keys to NULL, point it to the pre-allocated stack + /* initGetKeysResult initializes keys to NULL, point it to the pre-allocated stack * buffer here. */ if (!result->keys) { serverAssert(!result->numkeys); diff --git a/src/module.c b/src/module.c index f149443c80..c98c837a5c 100644 --- a/src/module.c +++ b/src/module.c @@ -13247,7 +13247,8 @@ int *VM_GetCommandKeysWithFlags(ValkeyModuleCtx *ctx, return NULL; } - getKeysResult result = GETKEYS_RESULT_INIT; + getKeysResult result; + initGetKeysResult(&result); getKeysFromCommand(cmd, argv, argc, &result); *num_keys = result.numkeys; diff --git a/src/server.c b/src/server.c index 5065e40dd2..e5df60be65 100644 --- a/src/server.c +++ b/src/server.c @@ -4838,7 +4838,8 @@ void addReplyCommandDocs(client *c, struct serverCommand *cmd) { /* Helper for COMMAND GETKEYS and GETKEYSANDFLAGS */ void getKeysSubcommandImpl(client *c, int with_flags) { struct serverCommand *cmd = lookupCommand(c->argv + 2, c->argc - 2); - getKeysResult result = GETKEYS_RESULT_INIT; + getKeysResult result; + initGetKeysResult(&result); int j; if (!cmd) { diff --git a/src/server.h b/src/server.h index bf16a1893d..ae2d23b99f 100644 --- a/src/server.h +++ b/src/server.h @@ -2136,12 +2136,17 @@ typedef struct { * for returning channel information. */ typedef struct { - keyReference keysbuf[MAX_KEYS_BUFFER]; /* Pre-allocated buffer, to save heap allocations */ - keyReference *keys; /* Key indices array, points to keysbuf or heap */ int numkeys; /* Number of key indices return */ int size; /* Available array size */ + keyReference *keys; /* Key indices array, points to keysbuf or heap */ + keyReference keysbuf[MAX_KEYS_BUFFER]; /* Pre-allocated buffer, to save heap allocations */ } getKeysResult; -#define GETKEYS_RESULT_INIT {{{0}}, NULL, 0, MAX_KEYS_BUFFER} + +static inline void initGetKeysResult(getKeysResult *result) { + result->numkeys = 0; + result->size = MAX_KEYS_BUFFER; + result->keys = NULL; +} /* Key specs definitions. * diff --git a/src/tracking.c b/src/tracking.c index e33b9f46a4..b95ca05b3e 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -216,7 +216,8 @@ void trackingRememberKeys(client *tracking, client *executing) { uint64_t caching_given = tracking->flags & CLIENT_TRACKING_CACHING; if ((optin && !caching_given) || (optout && caching_given)) return; - getKeysResult result = GETKEYS_RESULT_INIT; + getKeysResult result; + initGetKeysResult(&result); int numkeys = getKeysFromCommand(executing->cmd, executing->argv, executing->argc, &result); if (!numkeys) { getKeysFreeResult(&result); From 93123f97a086c625171d65c1cadf4549cb7b0983 Mon Sep 17 00:00:00 2001 From: Samuel Adetunji <45036507+adetunjii@users.noreply.github.com> Date: Fri, 14 Jun 2024 21:40:06 +0100 Subject: [PATCH 08/11] Format yaml files (#615) Closes #533 --------- Signed-off-by: adetunjii --- .config/format.yml | 7 + .github/dependabot.yml | 1 - .github/workflows/build-release-packages.yml | 30 +- .../call-build-linux-arm-packages.yml | 6 +- .../call-build-linux-x86-packages.yml | 2 +- .github/workflows/ci.yml | 119 +- .github/workflows/clang-format.yml | 68 +- .github/workflows/codecov.yml | 22 +- .github/workflows/codeql-analysis.yml | 22 +- .github/workflows/coverity.yml | 22 +- .github/workflows/daily.yml | 1513 ++++++++--------- .github/workflows/external.yml | 64 +- codecov.yml | 2 +- 13 files changed, 957 insertions(+), 921 deletions(-) create mode 100644 .config/format.yml diff --git a/.config/format.yml b/.config/format.yml new file mode 100644 index 0000000000..a50e0a9971 --- /dev/null +++ b/.config/format.yml @@ -0,0 +1,7 @@ +formatter: + type: basic + indent: 2 + retain_line_breaks_single: true + +exclude: + - "deps/" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 376c4171a6..3cd8df01dd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,7 +2,6 @@ # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates ---- version: 2 updates: - package-ecosystem: github-actions diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 8c32548f9e..094d82de08 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -3,7 +3,7 @@ name: Build Release Packages on: release: types: [published] - + workflow_dispatch: inputs: version: @@ -44,20 +44,20 @@ jobs: INPUT_VERSION: ${{ inputs.version || github.ref_name }} generate-build-matrix: - name: Generating build matrix - runs-on: ubuntu-latest - outputs: - x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }} - arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Set up the list of target to build so we can pass the JSON to the reusable job - - uses: ./.github/actions/generate-package-build-matrix - id: set-matrix - with: - ref: ${{ inputs.version || github.ref_name }} - + name: Generating build matrix + runs-on: ubuntu-latest + outputs: + x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }} + arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Set up the list of target to build so we can pass the JSON to the reusable job + - uses: ./.github/actions/generate-package-build-matrix + id: set-matrix + with: + ref: ${{ inputs.version || github.ref_name }} + release-build-linux-x86-packages: needs: - release-build-get-meta diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml index db987b3816..80416806c9 100644 --- a/.github/workflows/call-build-linux-arm-packages.yml +++ b/.github/workflows/call-build-linux-arm-packages.yml @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ inputs.version }} - + - name: Make Valkey uses: uraimo/run-on-arch-action@v2 with: @@ -53,7 +53,7 @@ jobs: distro: ${{matrix.distro.target}} install: apt-get update && apt-get install -y build-essential libssl-dev run: make -C src all BUILD_TLS=yes - + - name: Create Tarball and SHA256sums run: | TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}} @@ -64,7 +64,7 @@ jobs: sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256 mkdir -p packages-files cp -rfv $TAR_FILE_NAME.tar* packages-files/ - + - name: Install AWS cli. run: | sudo apt-get install -y awscli diff --git a/.github/workflows/call-build-linux-x86-packages.yml b/.github/workflows/call-build-linux-x86-packages.yml index 59c9fac52f..b037f9c507 100644 --- a/.github/workflows/call-build-linux-x86-packages.yml +++ b/.github/workflows/call-build-linux-x86-packages.yml @@ -51,7 +51,7 @@ jobs: - name: Make Valkey run: make -C src all BUILD_TLS=yes - + - name: Create Tarball and SHA256sums run: | TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 508565d296..4ee0a59039 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,30 +6,29 @@ permissions: contents: read jobs: - test-ubuntu-latest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: make - # Fail build if there are warnings - # build with TLS just for compilation coverage - run: make all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes - - name: test - run: | - sudo apt-get install tcl8.6 tclx - ./runtest --verbose --tags -slow --dump-logs - - name: module api test - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs - - name: validate commands.def up to date - run: | - touch src/commands/ping.json - make commands.def - dirty=$(git diff) - if [[ ! -z $dirty ]]; then echo $dirty; exit 1; fi - - name: unit tests - run: | - ./src/valkey-unit-tests + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: make + # Fail build if there are warnings + # build with TLS just for compilation coverage + run: make all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes + - name: test + run: | + sudo apt-get install tcl8.6 tclx + ./runtest --verbose --tags -slow --dump-logs + - name: module api test + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs + - name: validate commands.def up to date + run: | + touch src/commands/ping.json + make commands.def + dirty=$(git diff) + if [[ ! -z $dirty ]]; then echo $dirty; exit 1; fi + - name: unit tests + run: | + ./src/valkey-unit-tests test-sanitizer-address: runs-on: ubuntu-latest @@ -49,43 +48,79 @@ jobs: runs-on: ubuntu-latest container: debian:buster steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: make - run: | - apt-get update && apt-get install -y build-essential - make SERVER_CFLAGS='-Werror' + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: make + run: | + apt-get update && apt-get install -y build-essential + make SERVER_CFLAGS='-Werror' build-macos-latest: runs-on: macos-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: make - run: make SERVER_CFLAGS='-Werror' + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: make + run: make SERVER_CFLAGS='-Werror' build-32bit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: make - run: | - sudo apt-get update && sudo apt-get install libc6-dev-i386 - make SERVER_CFLAGS='-Werror' 32bit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make SERVER_CFLAGS='-Werror' 32bit build-libc-malloc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: make - run: make SERVER_CFLAGS='-Werror' MALLOC=libc + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: make + run: make SERVER_CFLAGS='-Werror' MALLOC=libc build-almalinux8-jemalloc: runs-on: ubuntu-latest container: almalinux:8 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: make + run: | + dnf -y install epel-release gcc make procps-ng which + make -j SERVER_CFLAGS='-Werror' + + format-yaml: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Set up Go + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: "1.22.4" + + - name: Setup YAML formatter + run: | + go install github.com/google/yamlfmt/cmd/yamlfmt@latest - - name: make - run: | - dnf -y install epel-release gcc make procps-ng which - make -j SERVER_CFLAGS='-Werror' + - name: Run yamlfmt + id: yamlfmt + run: | + yamlfmt -lint -conf .config/format.yml . + # Capture the diff output + DIFF=$(git diff) + if [ ! -z "$DIFF" ]; then + # Encode the diff in Base64 to ensure it's handled as a single line + ENCODED_DIFF=$(echo "$DIFF" | base64 -w 0) + echo "diff=$ENCODED_DIFF" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Check for formatting changes + if: ${{ steps.yamlfmt.outputs.diff }} + run: | + echo "ERROR: YAML file is not formatted properly. Here is the diff: " + # Decode the Base64 diff to display it + echo "${{ steps.clang-format.outputs.diff }}" | base64 --decode + exit 1 + shell: bash diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index cb76216d4e..7d53cc28e7 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -10,39 +10,39 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up Clang - run: | - sudo apt-get update -y - sudo apt-get upgrade -y - sudo apt-get install software-properties-common -y - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor | sudo tee /usr/share/keyrings/llvm-toolchain.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/llvm-toolchain.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-18 main" | sudo tee /etc/apt/sources.list.d/llvm.list - sudo apt-get update -y - sudo apt-get install clang-format-18 -y - - name: Run clang-format - id: clang-format - run: | - # Run clang-format and capture the diff - cd src - shopt -s globstar - clang-format-18 -i **/*.c **/*.h - # Capture the diff output - DIFF=$(git diff) - if [ ! -z "$DIFF" ]; then - # Encode the diff in Base64 to ensure it's handled as a single line - ENCODED_DIFF=$(echo "$DIFF" | base64 -w 0) - echo "diff=$ENCODED_DIFF" >> $GITHUB_OUTPUT - fi - shell: bash + - name: Set up Clang + run: | + sudo apt-get update -y + sudo apt-get upgrade -y + sudo apt-get install software-properties-common -y + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor | sudo tee /usr/share/keyrings/llvm-toolchain.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/llvm-toolchain.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-18 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update -y + sudo apt-get install clang-format-18 -y + - name: Run clang-format + id: clang-format + run: | + # Run clang-format and capture the diff + cd src + shopt -s globstar + clang-format-18 -i **/*.c **/*.h + # Capture the diff output + DIFF=$(git diff) + if [ ! -z "$DIFF" ]; then + # Encode the diff in Base64 to ensure it's handled as a single line + ENCODED_DIFF=$(echo "$DIFF" | base64 -w 0) + echo "diff=$ENCODED_DIFF" >> $GITHUB_OUTPUT + fi + shell: bash - - name: Check for formatting changes - if: ${{ steps.clang-format.outputs.diff }} - run: | - echo "ERROR: Code is not formatted correctly. Here is the diff:" - # Decode the Base64 diff to display it - echo "${{ steps.clang-format.outputs.diff }}" | base64 --decode - exit 1 - shell: bash + - name: Check for formatting changes + if: ${{ steps.clang-format.outputs.diff }} + run: | + echo "ERROR: Code is not formatted correctly. Here is the diff:" + # Decode the Base64 diff to display it + echo "${{ steps.clang-format.outputs.diff }}" | base64 --decode + exit 1 + shell: bash diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index e1493db19b..0a0d4a23db 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -9,16 +9,16 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Install lcov and run test - run: | - sudo apt-get install lcov - make lcov + - name: Install lcov and run test + run: | + sudo apt-get install lcov + make lcov - - name: Upload code coverage - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./src/valkey.info \ No newline at end of file + - name: Upload code coverage + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./src/valkey.info diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d4db9bcc29..6165429c3a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -20,19 +20,19 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'cpp' ] + language: ['cpp'] steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Initialize CodeQL - uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 - with: - languages: ${{ matrix.language }} + - name: Initialize CodeQL + uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 + with: + languages: ${{ matrix.language }} - - name: Autobuild - uses: github/codeql-action/autobuild@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 + - name: Autobuild + uses: github/codeql-action/autobuild@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 47eb27276e..cfd0df1af3 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -2,8 +2,8 @@ name: Coverity Scan on: schedule: - # Run once daily, since below 500k LOC can have 21 builds per week, per https://scan.coverity.com/faq#frequency - - cron: '0 0 * * *' + # Run once daily, since below 500k LOC can have 21 builds per week, per https://scan.coverity.com/faq#frequency + - cron: '0 0 * * *' # Support manual execution workflow_dispatch: permissions: @@ -13,18 +13,18 @@ jobs: if: github.repository == 'valkey-io/valkey' runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Download and extract the Coverity Build Tool - run: | + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Download and extract the Coverity Build Tool + run: | wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=${{ secrets.COVERITY_SCAN_TOKEN }}&project=valkey-io%2Fvalkey" -O cov-analysis-linux64.tar.gz mkdir cov-analysis-linux64 tar xzf cov-analysis-linux64.tar.gz --strip 1 -C cov-analysis-linux64 - - name: Install Valkey dependencies - run: sudo apt install -y gcc procps libssl-dev - - name: Build with cov-build - run: cov-analysis-linux64/bin/cov-build --dir cov-int make - - name: Upload the result - run: | + - name: Install Valkey dependencies + run: sudo apt install -y gcc procps libssl-dev + - name: Build with cov-build + run: cov-analysis-linux64/bin/cov-build --dir cov-int make + - name: Upload the result + run: | tar czvf cov-int.tgz cov-int curl \ --form email=${{ secrets.COVERITY_SCAN_EMAIL }} \ diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index ff7a9ad67b..7679856d1d 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -4,35 +4,34 @@ on: pull_request: branches: # any PR to a release branch. - - '[0-9].[0-9]' + - "[0-9].[0-9]" schedule: - - cron: '0 0 * * *' + - cron: "0 0 * * *" workflow_dispatch: inputs: skipjobs: - description: 'jobs to skip (delete the ones you wanna keep, do not leave empty)' - default: 'valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,rpm-distros,malloc,specific,fortify,reply-schema' + description: "jobs to skip (delete the ones you wanna keep, do not leave empty)" + default: "valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,rpm-distros,malloc,specific,fortify,reply-schema" skiptests: - description: 'tests to skip (delete the ones you wanna keep, do not leave empty)' - default: 'valkey,modules,sentinel,cluster,unittest' + description: "tests to skip (delete the ones you wanna keep, do not leave empty)" + default: "valkey,modules,sentinel,cluster,unittest" test_args: - description: 'extra test arguments' - default: '' + description: "extra test arguments" + default: "" cluster_test_args: - description: 'extra cluster / sentinel test arguments' - default: '' + description: "extra cluster / sentinel test arguments" + default: "" use_repo: - description: 'repo owner and name' - default: 'valkey-io/valkey' + description: "repo owner and name" + default: "valkey-io/valkey" use_git_ref: - description: 'git branch or sha to use' - default: 'unstable' + description: "git branch or sha to use" + default: "unstable" permissions: contents: read jobs: - test-ubuntu-jemalloc: runs-on: ubuntu-latest if: | @@ -40,41 +39,41 @@ jobs: !contains(github.event.inputs.skipjobs, 'ubuntu') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make all-with-unit-tests SERVER_CFLAGS='-Werror -DSERVER_TEST' - - name: testprep - run: sudo apt-get install tcl8.6 tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - - name: legacy unit tests - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: ./src/valkey-server test all --accurate - - name: new unit tests - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: ./src/valkey-unit-tests --accurate + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make all-with-unit-tests SERVER_CFLAGS='-Werror -DSERVER_TEST' + - name: testprep + run: sudo apt-get install tcl8.6 tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: legacy unit tests + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: ./src/valkey-server test all --accurate + - name: new unit tests + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: ./src/valkey-unit-tests --accurate test-ubuntu-jemalloc-fortify: runs-on: ubuntu-latest @@ -84,44 +83,44 @@ jobs: container: ubuntu:lunar timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | - apt-get update && apt-get install -y make gcc-13 - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 - make all-with-unit-tests CC=gcc OPT=-O3 SERVER_CFLAGS='-Werror -DSERVER_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3' - - name: testprep - run: apt-get install -y tcl8.6 tclx procps - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - - name: legacy unit tests - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: ./src/valkey-server test all --accurate - - name: new unit tests - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: ./src/valkey-unit-tests --accurate + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + apt-get update && apt-get install -y make gcc-13 + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 + make all-with-unit-tests CC=gcc OPT=-O3 SERVER_CFLAGS='-Werror -DSERVER_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3' + - name: testprep + run: apt-get install -y tcl8.6 tclx procps + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: legacy unit tests + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: ./src/valkey-server test all --accurate + - name: new unit tests + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: ./src/valkey-unit-tests --accurate test-ubuntu-libc-malloc: runs-on: ubuntu-latest @@ -130,35 +129,35 @@ jobs: !contains(github.event.inputs.skipjobs, 'malloc') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make MALLOC=libc SERVER_CFLAGS='-Werror' - - name: testprep - run: sudo apt-get install tcl8.6 tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make MALLOC=libc SERVER_CFLAGS='-Werror' + - name: testprep + run: sudo apt-get install tcl8.6 tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-ubuntu-no-malloc-usable-size: runs-on: ubuntu-latest @@ -167,35 +166,35 @@ jobs: !contains(github.event.inputs.skipjobs, 'malloc') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE SERVER_CFLAGS='-Werror' - - name: testprep - run: sudo apt-get install tcl8.6 tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE SERVER_CFLAGS='-Werror' + - name: testprep + run: sudo apt-get install tcl8.6 tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-ubuntu-32bit: runs-on: ubuntu-latest @@ -204,45 +203,45 @@ jobs: !contains(github.event.inputs.skipjobs, '32bit') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | - sudo apt-get update && sudo apt-get install libc6-dev-i386 - make 32bit SERVER_CFLAGS='-Werror -DSERVER_TEST' - - name: testprep - run: sudo apt-get install tcl8.6 tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: | - make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time - CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - - name: legacy unit tests - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: ./src/valkey-server test all --accurate - - name: new unit tests - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: ./src/valkey-unit-tests --accurate + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make 32bit SERVER_CFLAGS='-Werror -DSERVER_TEST' + - name: testprep + run: sudo apt-get install tcl8.6 tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time + CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: legacy unit tests + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: ./src/valkey-server test all --accurate + - name: new unit tests + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: ./src/valkey-unit-tests --accurate test-ubuntu-tls: runs-on: ubuntu-latest @@ -251,42 +250,42 @@ jobs: !contains(github.event.inputs.skipjobs, 'tls') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | - make BUILD_TLS=yes SERVER_CFLAGS='-Werror' - - name: testprep - run: | - sudo apt-get install tcl8.6 tclx tcl-tls - ./utils/gen-test-certs.sh - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: | - ./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: | - CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: | - ./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: | - ./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + make BUILD_TLS=yes SERVER_CFLAGS='-Werror' + - name: testprep + run: | + sudo apt-get install tcl8.6 tclx tcl-tls + ./utils/gen-test-certs.sh + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: | + ./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: | + ./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: | + ./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}} test-ubuntu-tls-no-tls: runs-on: ubuntu-latest @@ -295,42 +294,42 @@ jobs: !contains(github.event.inputs.skipjobs, 'tls') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | - make BUILD_TLS=yes SERVER_CFLAGS='-Werror' - - name: testprep - run: | - sudo apt-get install tcl8.6 tclx tcl-tls - ./utils/gen-test-certs.sh - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: | - ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: | - CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: | - ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: | - ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + make BUILD_TLS=yes SERVER_CFLAGS='-Werror' + - name: testprep + run: | + sudo apt-get install tcl8.6 tclx tcl-tls + ./utils/gen-test-certs.sh + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: | + ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: | + ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: | + ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-ubuntu-io-threads: runs-on: ubuntu-latest @@ -339,30 +338,30 @@ jobs: !contains(github.event.inputs.skipjobs, 'iothreads') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | - make SERVER_CFLAGS='-Werror' - - name: testprep - run: sudo apt-get install tcl8.6 tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs ${{github.event.inputs.test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + make SERVER_CFLAGS='-Werror' + - name: testprep + run: sudo apt-get install tcl8.6 tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs ${{github.event.inputs.test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes ${{github.event.inputs.cluster_test_args}} test-ubuntu-reclaim-cache: runs-on: ubuntu-latest @@ -371,76 +370,72 @@ jobs: !contains(github.event.inputs.skipjobs, 'specific') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | - make SERVER_CFLAGS='-Werror' - - name: testprep - run: | - sudo apt-get install vmtouch - mkdir /tmp/master - mkdir /tmp/slave - - name: warm up - run: | - ./src/valkey-server --daemonize yes --logfile /dev/null - ./src/valkey-benchmark -n 1 > /dev/null - ./src/valkey-cli save | grep OK > /dev/null - vmtouch -v ./dump.rdb > /dev/null - - name: test - run: | - echo "test SAVE doesn't increase cache" - CACHE0=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') - echo "$CACHE0" - ./src/valkey-server --daemonize yes --logfile /dev/null --dir /tmp/master --port 8080 --repl-diskless-sync no --pidfile /tmp/master/valkey.pid --rdbcompression no --enable-debug-command yes - ./src/valkey-cli -p 8080 debug populate 10000 k 102400 - ./src/valkey-server --daemonize yes --logfile /dev/null --dir /tmp/slave --port 8081 --repl-diskless-load disabled --rdbcompression no - ./src/valkey-cli -p 8080 save > /dev/null - VMOUT=$(vmtouch -v /tmp/master/dump.rdb) - echo $VMOUT - grep -q " 0%" <<< $VMOUT - CACHE=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') - echo "$CACHE" - if [ "$(( $CACHE-$CACHE0 ))" -gt "8000000" ]; then exit 1; fi + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + make SERVER_CFLAGS='-Werror' + - name: testprep + run: "sudo apt-get install vmtouch\nmkdir /tmp/master \nmkdir /tmp/slave\n" + - name: warm up + run: | + ./src/valkey-server --daemonize yes --logfile /dev/null + ./src/valkey-benchmark -n 1 > /dev/null + ./src/valkey-cli save | grep OK > /dev/null + vmtouch -v ./dump.rdb > /dev/null + - name: test + run: | + echo "test SAVE doesn't increase cache" + CACHE0=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') + echo "$CACHE0" + ./src/valkey-server --daemonize yes --logfile /dev/null --dir /tmp/master --port 8080 --repl-diskless-sync no --pidfile /tmp/master/valkey.pid --rdbcompression no --enable-debug-command yes + ./src/valkey-cli -p 8080 debug populate 10000 k 102400 + ./src/valkey-server --daemonize yes --logfile /dev/null --dir /tmp/slave --port 8081 --repl-diskless-load disabled --rdbcompression no + ./src/valkey-cli -p 8080 save > /dev/null + VMOUT=$(vmtouch -v /tmp/master/dump.rdb) + echo $VMOUT + grep -q " 0%" <<< $VMOUT + CACHE=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') + echo "$CACHE" + if [ "$(( $CACHE-$CACHE0 ))" -gt "8000000" ]; then exit 1; fi + echo "test replication doesn't increase cache" + ./src/valkey-cli -p 8081 REPLICAOF 127.0.0.1 8080 > /dev/null + while [ $(./src/valkey-cli -p 8081 info replication | grep "master_link_status:down") ]; do sleep 1; done; + sleep 1 # wait for the completion of cache reclaim bio + VMOUT=$(vmtouch -v /tmp/master/dump.rdb) + echo $VMOUT + grep -q " 0%" <<< $VMOUT + VMOUT=$(vmtouch -v /tmp/slave/dump.rdb) + echo $VMOUT + grep -q " 0%" <<< $VMOUT + CACHE=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') + echo "$CACHE" + if [ "$(( $CACHE-$CACHE0 ))" -gt "8000000" ]; then exit 1; fi - echo "test replication doesn't increase cache" - ./src/valkey-cli -p 8081 REPLICAOF 127.0.0.1 8080 > /dev/null - while [ $(./src/valkey-cli -p 8081 info replication | grep "master_link_status:down") ]; do sleep 1; done; - sleep 1 # wait for the completion of cache reclaim bio - VMOUT=$(vmtouch -v /tmp/master/dump.rdb) - echo $VMOUT - grep -q " 0%" <<< $VMOUT - VMOUT=$(vmtouch -v /tmp/slave/dump.rdb) - echo $VMOUT - grep -q " 0%" <<< $VMOUT - CACHE=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') - echo "$CACHE" - if [ "$(( $CACHE-$CACHE0 ))" -gt "8000000" ]; then exit 1; fi - - echo "test reboot doesn't increase cache" - PID=$(cat /tmp/master/valkey.pid) - kill -15 $PID - while [ -x /proc/${PID} ]; do sleep 1; done - ./src/valkey-server --daemonize yes --logfile /dev/null --dir /tmp/master --port 8080 - while [ $(./src/valkey-cli -p 8080 info persistence | grep "loading:1") ]; do sleep 1; done; - sleep 1 # wait for the completion of cache reclaim bio - VMOUT=$(vmtouch -v /tmp/master/dump.rdb) - echo $VMOUT - grep -q " 0%" <<< $VMOUT - CACHE=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') - echo "$CACHE" - if [ "$(( $CACHE-$CACHE0 ))" -gt "8000000" ]; then exit 1; fi + echo "test reboot doesn't increase cache" + PID=$(cat /tmp/master/valkey.pid) + kill -15 $PID + while [ -x /proc/${PID} ]; do sleep 1; done + ./src/valkey-server --daemonize yes --logfile /dev/null --dir /tmp/master --port 8080 + while [ $(./src/valkey-cli -p 8080 info persistence | grep "loading:1") ]; do sleep 1; done; + sleep 1 # wait for the completion of cache reclaim bio + VMOUT=$(vmtouch -v /tmp/master/dump.rdb) + echo $VMOUT + grep -q " 0%" <<< $VMOUT + CACHE=$(grep -w file /sys/fs/cgroup/memory.stat | awk '{print $2}') + echo "$CACHE" + if [ "$(( $CACHE-$CACHE0 ))" -gt "8000000" ]; then exit 1; fi test-valgrind-test: runs-on: ubuntu-latest @@ -449,28 +444,28 @@ jobs: !contains(github.event.inputs.skipjobs, 'valgrind') && !contains(github.event.inputs.skiptests, 'valkey') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make valgrind SERVER_CFLAGS='-Werror -DSERVER_TEST' - - name: testprep - run: | - sudo apt-get update - sudo apt-get install tcl8.6 tclx valgrind -y - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make valgrind SERVER_CFLAGS='-Werror -DSERVER_TEST' + - name: testprep + run: | + sudo apt-get update + sudo apt-get install tcl8.6 tclx valgrind -y + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} test-valgrind-misc: runs-on: ubuntu-latest @@ -479,33 +474,33 @@ jobs: !contains(github.event.inputs.skipjobs, 'valgrind') && !(contains(github.event.inputs.skiptests, 'modules') && contains(github.event.inputs.skiptests, 'unittest')) timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make valgrind SERVER_CFLAGS='-Werror -DSERVER_TEST' - - name: testprep - run: | - sudo apt-get update - sudo apt-get install tcl8.6 tclx valgrind -y - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} - - name: unittest - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: | - valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/valkey-server test all --valgrind - if grep -q 0x err.txt; then cat err.txt; exit 1; fi + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make valgrind SERVER_CFLAGS='-Werror -DSERVER_TEST' + - name: testprep + run: | + sudo apt-get update + sudo apt-get install tcl8.6 tclx valgrind -y + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/valkey-server test all --valgrind + if grep -q 0x err.txt; then cat err.txt; exit 1; fi test-valgrind-no-malloc-usable-size-test: runs-on: ubuntu-latest @@ -514,28 +509,28 @@ jobs: !contains(github.event.inputs.skipjobs, 'valgrind') && !contains(github.event.inputs.skiptests, 'valkey') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DSERVER_TEST" SERVER_CFLAGS='-Werror' - - name: testprep - run: | - sudo apt-get update - sudo apt-get install tcl8.6 tclx valgrind -y - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DSERVER_TEST" SERVER_CFLAGS='-Werror' + - name: testprep + run: | + sudo apt-get update + sudo apt-get install tcl8.6 tclx valgrind -y + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} test-valgrind-no-malloc-usable-size-misc: runs-on: ubuntu-latest @@ -544,33 +539,33 @@ jobs: !contains(github.event.inputs.skipjobs, 'valgrind') && !(contains(github.event.inputs.skiptests, 'modules') && contains(github.event.inputs.skiptests, 'unittest')) timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DSERVER_TEST" SERVER_CFLAGS='-Werror' - - name: testprep - run: | - sudo apt-get update - sudo apt-get install tcl8.6 tclx valgrind -y - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} - - name: unittest - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: | - valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/valkey-server test all --valgrind - if grep -q 0x err.txt; then cat err.txt; exit 1; fi + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DSERVER_TEST" SERVER_CFLAGS='-Werror' + - name: testprep + run: | + sudo apt-get update + sudo apt-get install tcl8.6 tclx valgrind -y + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/valkey-server test all --valgrind + if grep -q 0x err.txt; then cat err.txt; exit 1; fi test-sanitizer-address: runs-on: ubuntu-latest @@ -580,7 +575,7 @@ jobs: timeout-minutes: 14400 strategy: matrix: - compiler: [ gcc, clang ] + compiler: [gcc, clang] env: CC: ${{ matrix.compiler }} steps: @@ -630,7 +625,7 @@ jobs: timeout-minutes: 14400 strategy: matrix: - compiler: [ gcc, clang ] + compiler: [gcc, clang] env: CC: ${{ matrix.compiler }} steps: @@ -674,8 +669,8 @@ jobs: test-rpm-distros-jemalloc: if: | - (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && - !contains(github.event.inputs.skipjobs, 'rpm-distros') + (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && + !contains(github.event.inputs.skipjobs, 'rpm-distros') strategy: fail-fast: false matrix: @@ -701,45 +696,45 @@ jobs: timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: Install EPEL - if: matrix.install_epel - run: dnf -y install epel-release - - name: make - run: | - dnf -y install gcc make procps-ng which /usr/bin/kill - make -j SERVER_CFLAGS='-Werror' - - name: testprep - run: dnf -y install tcl tcltls - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: Install EPEL + if: matrix.install_epel + run: dnf -y install epel-release + - name: make + run: | + dnf -y install gcc make procps-ng which /usr/bin/kill + make -j SERVER_CFLAGS='-Werror' + - name: testprep + run: dnf -y install tcl tcltls + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-rpm-distros-tls-module: if: | - (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && - !contains(github.event.inputs.skipjobs, 'tls') + (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && + !contains(github.event.inputs.skipjobs, 'tls') strategy: fail-fast: false matrix: @@ -765,51 +760,51 @@ jobs: timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: Install EPEL - if: matrix.install_epel - run: dnf -y install epel-release - - name: make - run: | - dnf -y install make gcc openssl-devel openssl procps-ng which /usr/bin/kill - make -j BUILD_TLS=module SERVER_CFLAGS='-Werror' - - name: testprep - run: | - dnf -y install tcl tcltls - ./utils/gen-test-certs.sh - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: | - ./runtest --accurate --verbose --dump-logs --tls-module --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: | - CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --tls-module --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: | - ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: | - ./runtest-cluster --tls-module ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: Install EPEL + if: matrix.install_epel + run: dnf -y install epel-release + - name: make + run: | + dnf -y install make gcc openssl-devel openssl procps-ng which /usr/bin/kill + make -j BUILD_TLS=module SERVER_CFLAGS='-Werror' + - name: testprep + run: | + dnf -y install tcl tcltls + ./utils/gen-test-certs.sh + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: | + ./runtest --accurate --verbose --dump-logs --tls-module --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --tls-module --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: | + ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: | + ./runtest-cluster --tls-module ${{github.event.inputs.cluster_test_args}} test-rpm-distros-tls-module-no-tls: if: | - (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && - !contains(github.event.inputs.skipjobs, 'tls') + (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && + !contains(github.event.inputs.skipjobs, 'tls') strategy: fail-fast: false matrix: @@ -835,46 +830,46 @@ jobs: timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: Install EPEL - if: matrix.install_epel - run: dnf -y install epel-release - - name: make - run: | - dnf -y install make gcc openssl-devel openssl procps-ng which /usr/bin/kill - make -j BUILD_TLS=module SERVER_CFLAGS='-Werror' - - name: testprep - run: | - dnf -y install tcl tcltls - ./utils/gen-test-certs.sh - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: | - ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: | - CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: | - ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: | - ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: Install EPEL + if: matrix.install_epel + run: dnf -y install epel-release + - name: make + run: | + dnf -y install make gcc openssl-devel openssl procps-ng which /usr/bin/kill + make -j BUILD_TLS=module SERVER_CFLAGS='-Werror' + - name: testprep + run: | + dnf -y install tcl tcltls + ./utils/gen-test-certs.sh + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: | + ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: | + ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: | + ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-macos-latest: runs-on: macos-latest @@ -883,27 +878,27 @@ jobs: !contains(github.event.inputs.skipjobs, 'macos') && !(contains(github.event.inputs.skiptests, 'valkey') && contains(github.event.inputs.skiptests, 'modules')) timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make SERVER_CFLAGS='-Werror' - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --clients 1 --no-latency --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --clients 1 --no-latency --dump-logs ${{github.event.inputs.test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make SERVER_CFLAGS='-Werror' + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --clients 1 --no-latency --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --clients 1 --no-latency --dump-logs ${{github.event.inputs.test_args}} test-macos-latest-sentinel: runs-on: macos-latest @@ -912,24 +907,24 @@ jobs: !contains(github.event.inputs.skipjobs, 'macos') && !contains(github.event.inputs.skiptests, 'sentinel') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make SERVER_CFLAGS='-Werror' - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make SERVER_CFLAGS='-Werror' + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} test-macos-latest-cluster: runs-on: macos-latest @@ -938,24 +933,24 @@ jobs: !contains(github.event.inputs.skipjobs, 'macos') && !contains(github.event.inputs.skiptests, 'cluster') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make SERVER_CFLAGS='-Werror' - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make SERVER_CFLAGS='-Werror' + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} build-macos: strategy: @@ -967,24 +962,24 @@ jobs: !contains(github.event.inputs.skipjobs, 'macos') timeout-minutes: 14400 steps: - - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0 - with: - xcode-version: latest - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make SERVER_CFLAGS='-Werror -DSERVER_TEST' + - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0 + with: + xcode-version: latest + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make SERVER_CFLAGS='-Werror -DSERVER_TEST' test-freebsd: runs-on: macos-12 @@ -993,26 +988,26 @@ jobs: !contains(github.event.inputs.skipjobs, 'freebsd') timeout-minutes: 14400 steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: test - uses: cross-platform-actions/action@5800fa0060a22edf69992a779adac3d2bb3a6f8a # v0.22.0 - with: - operating_system: freebsd - environment_variables: MAKE - version: 13.2 - shell: bash - run: | - sudo pkg install -y bash gmake lang/tcl86 lang/tclx - gmake - ./runtest --single unit/keyspace --single unit/auth --single unit/networking --single unit/protocol + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: test + uses: cross-platform-actions/action@5800fa0060a22edf69992a779adac3d2bb3a6f8a # v0.22.0 + with: + operating_system: freebsd + environment_variables: MAKE + version: 13.2 + shell: bash + run: | + sudo pkg install -y bash gmake lang/tcl86 lang/tclx + gmake + ./runtest --single unit/keyspace --single unit/auth --single unit/networking --single unit/protocol test-alpine-jemalloc: runs-on: ubuntu-latest @@ -1021,37 +1016,37 @@ jobs: !contains(github.event.inputs.skipjobs, 'alpine') container: alpine:latest steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | apk add build-base make SERVER_CFLAGS='-Werror' - - name: testprep - run: apk add tcl procps tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: testprep + run: apk add tcl procps tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-alpine-libc-malloc: runs-on: ubuntu-latest @@ -1060,37 +1055,37 @@ jobs: !contains(github.event.inputs.skipjobs, 'alpine') container: alpine:latest steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: | + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | apk add build-base make SERVER_CFLAGS='-Werror' USE_JEMALLOC=no CFLAGS=-DUSE_MALLOC_USABLE_SIZE - - name: testprep - run: apk add tcl procps tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} + - name: testprep + run: apk add tcl procps tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} reply-schemas-validator: runs-on: ubuntu-latest @@ -1099,41 +1094,41 @@ jobs: (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'valkey-io/valkey')) && !contains(github.event.inputs.skipjobs, 'reply-schema') steps: - - name: prep - if: github.event_name == 'workflow_dispatch' - run: | - echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV - echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV - echo "skipjobs: ${{github.event.inputs.skipjobs}}" - echo "skiptests: ${{github.event.inputs.skiptests}}" - echo "test_args: ${{github.event.inputs.test_args}}" - echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: ${{ env.GITHUB_REPOSITORY }} - ref: ${{ env.GITHUB_HEAD_REF }} - - name: make - run: make SERVER_CFLAGS='-Werror -DLOG_REQ_RES' - - name: testprep - run: sudo apt-get install tcl8.6 tclx - - name: test - if: true && !contains(github.event.inputs.skiptests, 'valkey') - run: ./runtest --log-req-res --no-latency --dont-clean --force-resp3 --tags -slow --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: CFLAGS='-Werror' ./runtest-moduleapi --log-req-res --no-latency --dont-clean --force-resp3 --dont-pre-clean --verbose --dump-logs ${{github.event.inputs.test_args}} - - name: sentinel tests - if: true && !contains(github.event.inputs.skiptests, 'sentinel') - run: ./runtest-sentinel --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}} - - name: cluster tests - if: true && !contains(github.event.inputs.skiptests, 'cluster') - run: ./runtest-cluster --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}} - - name: Install Python dependencies - uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 - with: - path: "./utils/req-res-validator/requirements.txt" - - name: validator - run: ./utils/req-res-log-validator.py --verbose --fail-missing-reply-schemas ${{ (!contains(github.event.inputs.skiptests, 'valkey') && !contains(github.event.inputs.skiptests, 'module') && !contains(github.event.inputs.sentinel, 'valkey') && !contains(github.event.inputs.skiptests, 'cluster')) && github.event.inputs.test_args == '' && github.event.inputs.cluster_test_args == '' && '--fail-commands-not-all-hit' || '' }} + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + echo "skipjobs: ${{github.event.inputs.skipjobs}}" + echo "skiptests: ${{github.event.inputs.skiptests}}" + echo "test_args: ${{github.event.inputs.test_args}}" + echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make SERVER_CFLAGS='-Werror -DLOG_REQ_RES' + - name: testprep + run: sudo apt-get install tcl8.6 tclx + - name: test + if: true && !contains(github.event.inputs.skiptests, 'valkey') + run: ./runtest --log-req-res --no-latency --dont-clean --force-resp3 --tags -slow --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: CFLAGS='-Werror' ./runtest-moduleapi --log-req-res --no-latency --dont-clean --force-resp3 --dont-pre-clean --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: ./runtest-sentinel --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: ./runtest-cluster --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}} + - name: Install Python dependencies + uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 + with: + path: "./utils/req-res-validator/requirements.txt" + - name: validator + run: ./utils/req-res-log-validator.py --verbose --fail-missing-reply-schemas ${{ (!contains(github.event.inputs.skiptests, 'valkey') && !contains(github.event.inputs.skiptests, 'module') && !contains(github.event.inputs.sentinel, 'valkey') && !contains(github.event.inputs.skiptests, 'cluster')) && github.event.inputs.test_args == '' && github.event.inputs.cluster_test_args == '' && '--fail-commands-not-all-hit' || '' }} notify-about-job-results: runs-on: ubuntu-latest @@ -1151,7 +1146,7 @@ jobs: FAILED_JOBS+=($JOB) fi done - + if [[ ${#FAILED_JOBS[@]} -ne 0 ]]; then echo "FAILED_JOBS=${FAILED_JOBS[@]}" >> $GITHUB_ENV echo "STATUS=failure" >> $GITHUB_ENV diff --git a/.github/workflows/external.yml b/.github/workflows/external.yml index 8946526c8d..e021a964d1 100644 --- a/.github/workflows/external.yml +++ b/.github/workflows/external.yml @@ -15,53 +15,53 @@ jobs: if: github.event_name != 'schedule' || github.repository == 'valkey-io/valkey' timeout-minutes: 14400 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Build - run: make SERVER_CFLAGS=-Werror - - name: Start valkey-server - run: | - ./src/valkey-server --daemonize yes --save "" --logfile external-server.log \ - --enable-protected-configs yes --enable-debug-command yes --enable-module-command yes - - name: Run external test - run: | + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Build + run: make SERVER_CFLAGS=-Werror + - name: Start valkey-server + run: | + ./src/valkey-server --daemonize yes --save "" --logfile external-server.log \ + --enable-protected-configs yes --enable-debug-command yes --enable-module-command yes + - name: Run external test + run: | ./runtest \ --host 127.0.0.1 --port 6379 \ --verbose \ --tags -slow - - name: Archive server log - if: ${{ failure() }} - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: test-external-standalone-log - path: external-server.log + - name: Archive server log + if: ${{ failure() }} + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: test-external-standalone-log + path: external-server.log test-external-cluster: runs-on: ubuntu-latest if: github.event_name != 'schedule' || github.repository == 'valkey-io/valkey' timeout-minutes: 14400 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Build - run: make SERVER_CFLAGS=-Werror - - name: Start valkey-server - run: | - ./src/valkey-server --cluster-enabled yes --daemonize yes --save "" --logfile external-server.log \ - --enable-protected-configs yes --enable-debug-command yes --enable-module-command yes - - name: Create a single node cluster - run: ./src/valkey-cli cluster addslots $(for slot in {0..16383}; do echo $slot; done); sleep 5 - - name: Run external test - run: | + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Build + run: make SERVER_CFLAGS=-Werror + - name: Start valkey-server + run: | + ./src/valkey-server --cluster-enabled yes --daemonize yes --save "" --logfile external-server.log \ + --enable-protected-configs yes --enable-debug-command yes --enable-module-command yes + - name: Create a single node cluster + run: ./src/valkey-cli cluster addslots $(for slot in {0..16383}; do echo $slot; done); sleep 5 + - name: Run external test + run: | ./runtest \ --host 127.0.0.1 --port 6379 \ --verbose \ --cluster-mode \ --tags -slow - - name: Archive server log - if: ${{ failure() }} - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: test-external-cluster-log - path: external-server.log + - name: Archive server log + if: ${{ failure() }} + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: test-external-cluster-log + path: external-server.log test-external-nodebug: runs-on: ubuntu-latest diff --git a/codecov.yml b/codecov.yml index c8d096e44f..05e315dcc3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -16,4 +16,4 @@ comment: behavior: default github_checks: - annotations: false \ No newline at end of file + annotations: false From a81c32079cfd63ae022a3d1a330904f8d9fd1858 Mon Sep 17 00:00:00 2001 From: Sankar <1890648+srgsanky@users.noreply.github.com> Date: Sun, 16 Jun 2024 20:37:09 -0700 Subject: [PATCH 09/11] Make cluster meet reliable under link failures (#461) When there is a link failure while an ongoing MEET request is sent the sending node stops sending anymore MEET and starts sending PINGs. Since every node responds to PINGs from unknown nodes with a PONG, the receiving node never adds the sending node. But the sending node adds the receiving node when it sees a PONG. This can lead to asymmetry in cluster membership. This changes makes the sender keep sending MEET until it sees a PONG, avoiding the asymmetry. --------- Signed-off-by: Sankar <1890648+srgsanky@users.noreply.github.com> --- src/cluster_legacy.c | 35 ++++++-- src/debug.c | 6 ++ src/server.h | 2 + tests/unit/cluster/cluster-multiple-meets.tcl | 83 +++++++++++++++++++ tests/unit/cluster/cluster-reliable-meet.tcl | 71 ++++++++++++++++ 5 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 tests/unit/cluster/cluster-multiple-meets.tcl create mode 100644 tests/unit/cluster/cluster-reliable-meet.tcl diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index f566cf5a35..9104a76d87 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -2844,7 +2844,16 @@ int clusterIsValidPacket(clusterLink *link) { * received from the wrong sender ID). */ int clusterProcessPacket(clusterLink *link) { /* Validate that the packet is well-formed */ - if (!clusterIsValidPacket(link)) return 1; + if (!clusterIsValidPacket(link)) { + clusterMsg *hdr = (clusterMsg *)link->rcvbuf; + uint16_t type = ntohs(hdr->type); + if (server.debug_cluster_close_link_on_packet_drop && type == server.cluster_drop_packet_filter) { + freeClusterLink(link); + serverLog(LL_WARNING, "Closing link for matching packet type %hu", type); + return 0; + } + return 1; + } clusterMsg *hdr = (clusterMsg *)link->rcvbuf; uint16_t type = ntohs(hdr->type); @@ -2942,6 +2951,13 @@ int clusterProcessPacket(clusterLink *link) { if (type == CLUSTERMSG_TYPE_PING || type == CLUSTERMSG_TYPE_PONG || type == CLUSTERMSG_TYPE_MEET) { serverLog(LL_DEBUG, "%s packet received: %.40s", clusterGetMessageTypeString(type), link->node ? link->node->name : "NULL"); + + if (sender && (sender->flags & CLUSTER_NODE_MEET)) { + /* Once we get a response for MEET from the sender, we can stop sending more MEET. */ + sender->flags &= ~CLUSTER_NODE_MEET; + serverLog(LL_NOTICE, "Successfully completed handshake with %.40s (%s)", sender->name, + sender->human_nodename); + } if (!link->inbound) { if (nodeInHandshake(link->node)) { /* If we already have this node, try to change the @@ -3376,12 +3392,17 @@ void clusterLinkConnectHandler(connection *conn) { * replaced by the clusterSendPing() call. */ node->ping_sent = old_ping_sent; } - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~CLUSTER_NODE_MEET; + /* NOTE: Assume the current node is A and is asked to MEET another node B. + * Once A sends MEET to B, it cannot clear the MEET flag for B until it + * gets a response from B. If the MEET packet is not accepted by B due to + * link failure, A must continue sending MEET. If A doesn't continue sending + * MEET, A will know about B, but B will never add A. Every node always + * responds to PINGs from unknown nodes with a PONG, so A will know about B + * and continue sending PINGs. But B won't add A until it sees a MEET (or it + * gets to know about A from a trusted third node C). In this case, clearing + * the MEET flag here leads to asymmetry in the cluster membership. So, we + * clear the MEET flag in clusterProcessPacket. + */ serverLog(LL_DEBUG, "Connecting with Node %.40s at %s:%d", node->name, node->ip, node->cport); } diff --git a/src/debug.c b/src/debug.c index 6394e3f0f4..d9fe93c7d4 100644 --- a/src/debug.c +++ b/src/debug.c @@ -429,6 +429,9 @@ void debugCommand(client *c) { " Show low level info about `key` and associated value.", "DROP-CLUSTER-PACKET-FILTER ", " Drop all packets that match the filtered type. Set to -1 allow all packets.", + "CLOSE-CLUSTER-LINK-ON-PACKET-DROP <0|1>", + " This is valid only when DROP-CLUSTER-PACKET-FILTER is set to a valid packet type." + " When set to 1, the cluster link is closed after dropping a packet based on the filter." "OOM", " Crash the server simulating an out-of-memory error.", "PANIC", @@ -593,6 +596,9 @@ void debugCommand(client *c) { if (getLongFromObjectOrReply(c, c->argv[2], &packet_type, NULL) != C_OK) return; server.cluster_drop_packet_filter = packet_type; addReply(c, shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr, "close-cluster-link-on-packet-drop") && c->argc == 3) { + server.debug_cluster_close_link_on_packet_drop = atoi(c->argv[2]->ptr); + addReply(c, shared.ok); } else if (!strcasecmp(c->argv[1]->ptr, "object") && c->argc == 3) { dictEntry *de; robj *val; diff --git a/src/server.h b/src/server.h index ae2d23b99f..c4ce6f655e 100644 --- a/src/server.h +++ b/src/server.h @@ -2069,6 +2069,8 @@ struct valkeyServer { unsigned long long cluster_link_msg_queue_limit_bytes; /* Memory usage limit on individual link msg queue */ int cluster_drop_packet_filter; /* Debug config that allows tactically * dropping packets of a specific type */ + /* Debug config that goes along with cluster_drop_packet_filter. When set, the link is closed on packet drop. */ + uint32_t debug_cluster_close_link_on_packet_drop : 1; sds cached_cluster_slot_info[CACHE_CONN_TYPE_MAX]; /* Scripting */ mstime_t busy_reply_threshold; /* Script / module timeout in milliseconds */ diff --git a/tests/unit/cluster/cluster-multiple-meets.tcl b/tests/unit/cluster/cluster-multiple-meets.tcl new file mode 100644 index 0000000000..07a2582133 --- /dev/null +++ b/tests/unit/cluster/cluster-multiple-meets.tcl @@ -0,0 +1,83 @@ +# make sure the test infra won't use SELECT +set old_singledb $::singledb +set ::singledb 1 + +tags {tls:skip external:skip cluster} { + set base_conf [list cluster-enabled yes] + start_multiple_servers 2 [list overrides $base_conf] { + test "Cluster nodes are reachable" { + for {set id 0} {$id < [llength $::servers]} {incr id} { + # Every node should be reachable. + wait_for_condition 1000 50 { + ([catch {R $id ping} ping_reply] == 0) && + ($ping_reply eq {PONG}) + } else { + catch {R $id ping} err + fail "Node #$id keeps replying '$err' to PING." + } + } + } + + test "Before slots allocation, all nodes report cluster failure" { + wait_for_cluster_state fail + } + + set CLUSTER_PACKET_TYPE_PONG 1 + set CLUSTER_PACKET_TYPE_NONE -1 + + test "Cluster nodes haven't met each other" { + assert {[llength [get_cluster_nodes 1]] == 1} + assert {[llength [get_cluster_nodes 0]] == 1} + } + + test "Allocate slots" { + cluster_allocate_slots 2 0;# primaries replicas + } + + test "Multiple MEETs from Node 1 to Node 0 should work" { + # Make 1 drop the PONG responses to MEET + R 1 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_PONG + # It is important to close the connection on drop, otherwise a subsequent MEET won't be sent + R 1 DEBUG CLOSE-CLUSTER-LINK-ON-PACKET-DROP 1 + + R 1 CLUSTER MEET 127.0.0.1 [srv 0 port] + + # Wait for at least a few MEETs to be sent so that we are sure that 1 is dropping the response to MEET. + wait_for_condition 1000 50 { + [CI 0 cluster_stats_messages_meet_received] > 1 && + [CI 1 cluster_state] eq {fail} && [CI 0 cluster_state] eq {ok} + } else { + fail "Cluster node 1 never sent multiple MEETs to 0" + } + + # 0 will be connected to 1, but 1 won't see that 0 is connected + assert {[llength [get_cluster_nodes 1 connected]] == 1} + assert {[llength [get_cluster_nodes 0 connected]] == 2} + + # Drop incoming and outgoing links from/to 1 + R 0 DEBUG CLUSTERLINK KILL ALL [R 1 CLUSTER MYID] + + # Wait for 0 to know about 1 again after 1 sends a MEET + wait_for_condition 1000 50 { + [llength [get_cluster_nodes 0 connected]] == 2 + } else { + fail "Cluster node 1 never sent multiple MEETs to 0" + } + + # Undo packet drop + R 1 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_NONE + R 1 DEBUG CLOSE-CLUSTER-LINK-ON-PACKET-DROP 0 + + # Both a and b will turn to cluster state ok + wait_for_condition 1000 50 { + [CI 1 cluster_state] eq {ok} && [CI 0 cluster_state] eq {ok} && + [CI 1 cluster_stats_messages_meet_sent] == [CI 0 cluster_stats_messages_meet_received] + } else { + fail "1 cluster_state:[CI 1 cluster_state], 0 cluster_state: [CI 0 cluster_state]" + } + } + } ;# stop servers +} ;# tags + +set ::singledb $old_singledb + diff --git a/tests/unit/cluster/cluster-reliable-meet.tcl b/tests/unit/cluster/cluster-reliable-meet.tcl new file mode 100644 index 0000000000..41da97ab9b --- /dev/null +++ b/tests/unit/cluster/cluster-reliable-meet.tcl @@ -0,0 +1,71 @@ +# make sure the test infra won't use SELECT +set old_singledb $::singledb +set ::singledb 1 + +tags {tls:skip external:skip cluster} { + set base_conf [list cluster-enabled yes] + start_multiple_servers 2 [list overrides $base_conf] { + test "Cluster nodes are reachable" { + for {set id 0} {$id < [llength $::servers]} {incr id} { + # Every node should be reachable. + wait_for_condition 1000 50 { + ([catch {R $id ping} ping_reply] == 0) && + ($ping_reply eq {PONG}) + } else { + catch {R $id ping} err + fail "Node #$id keeps replying '$err' to PING." + } + } + } + + test "Before slots allocation, all nodes report cluster failure" { + wait_for_cluster_state fail + } + + set CLUSTER_PACKET_TYPE_MEET 2 + set CLUSTER_PACKET_TYPE_NONE -1 + + test "Cluster nodes haven't met each other" { + assert {[llength [get_cluster_nodes 1]] == 1} + assert {[llength [get_cluster_nodes 0]] == 1} + } + + test "Allocate slots" { + cluster_allocate_slots 2 0 + } + + test "MEET is reliable when target drops the initial MEETs" { + # Make 0 drop the initial MEET messages due to link failure + R 0 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_MEET + R 0 DEBUG CLOSE-CLUSTER-LINK-ON-PACKET-DROP 1 + + R 1 CLUSTER MEET 127.0.0.1 [srv 0 port] + + # Wait for at least a few MEETs to be sent so that we are sure that 0 is + # dropping them. + wait_for_condition 1000 50 { + [CI 0 cluster_stats_messages_meet_received] >= 3 + } else { + fail "Cluster node 1 never sent multiple MEETs to 0" + } + + # Make sure the nodes still don't know about each other + assert {[llength [get_cluster_nodes 1 connected]] == 1} + assert {[llength [get_cluster_nodes 0 connected]] == 1} + + R 0 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_NONE + + # If the MEET is reliable, both a and b will turn to cluster state ok + wait_for_condition 1000 50 { + [CI 1 cluster_state] eq {ok} && [CI 0 cluster_state] eq {ok} && + [CI 0 cluster_stats_messages_meet_received] >= 4 && + [CI 1 cluster_stats_messages_meet_sent] == [CI 0 cluster_stats_messages_meet_received] + } else { + fail "1 cluster_state:[CI 1 cluster_state], 0 cluster_state: [CI 0 cluster_state]" + } + } + } ;# stop servers +} ;# tags + +set ::singledb $old_singledb + From db6d3c1138695947412a745146ca29dbdf2e91c6 Mon Sep 17 00:00:00 2001 From: Binbin Date: Mon, 17 Jun 2024 11:46:08 +0800 Subject: [PATCH 10/11] Only primary with slots has the right to mark a node as failed (#634) In markNodeAsFailingIfNeeded we will count needed_quorum and failures, needed_quorum is the half the cluster->size and plus one, and cluster-size is the size of primary node which contain slots, but when counting failures, we dit not check if primary has slots. Only the primary has slots that has the rights to vote, adding a new clusterNodeIsVotingPrimary to formalize this concept. Release notes: bugfix where nodes not in the quorum group might spuriously mark nodes as failed --------- Signed-off-by: Binbin Co-authored-by: Ping Xie --- src/cluster_legacy.c | 23 +++++---- tests/unit/cluster/failure-marking.tcl | 68 ++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index 9104a76d87..21aa620dd9 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -116,6 +116,12 @@ int verifyClusterNodeId(const char *name, int length); sds clusterEncodeOpenSlotsAuxField(int rdbflags); int clusterDecodeOpenSlotsAuxField(int rdbflags, sds s); +/* Only primaries that own slots have voting rights. + * Returns 1 if the node has voting rights, otherwise returns 0. */ +static inline int clusterNodeIsVotingPrimary(clusterNode *n) { + return (n->flags & CLUSTER_NODE_PRIMARY) && n->numslots; +} + int getNodeDefaultClientPort(clusterNode *n) { return server.tls_cluster ? n->tls_port : n->tcp_port; } @@ -1867,8 +1873,8 @@ void markNodeAsFailingIfNeeded(clusterNode *node) { if (nodeFailed(node)) return; /* Already FAILing. */ failures = clusterNodeFailureReportsCount(node); - /* Also count myself as a voter if I'm a primary. */ - if (clusterNodeIsPrimary(myself)) failures++; + /* Also count myself as a voter if I'm a voting primary. */ + if (clusterNodeIsVotingPrimary(myself)) failures++; if (failures < needed_quorum) return; /* No weak agreement from primaries. */ serverLog(LL_NOTICE, "Marking node %.40s (%s) as failing (quorum reached).", node->name, node->human_nodename); @@ -1908,7 +1914,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) { * 1) The FAIL state is old enough. * 2) It is yet serving slots from our point of view (not failed over). * Apparently no one is going to fix these slots, clear the FAIL flag. */ - if (clusterNodeIsPrimary(node) && node->numslots > 0 && + if (clusterNodeIsVotingPrimary(node) && (now - node->fail_time) > (server.cluster_node_timeout * CLUSTER_FAIL_UNDO_TIME_MULT)) { serverLog( LL_NOTICE, @@ -2090,8 +2096,8 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { /* Ignore gossips about self. */ if (node && node != myself) { /* We already know this node. - Handle failure reports, only when the sender is a primary. */ - if (sender && clusterNodeIsPrimary(sender)) { + Handle failure reports, only when the sender is a voting primary. */ + if (sender && clusterNodeIsVotingPrimary(sender)) { if (flags & (CLUSTER_NODE_FAIL | CLUSTER_NODE_PFAIL)) { if (clusterNodeAddFailureReport(node, sender)) { serverLog(LL_VERBOSE, "Node %.40s (%s) reported node %.40s (%s) as not reachable.", @@ -3250,8 +3256,7 @@ int clusterProcessPacket(clusterLink *link) { /* We consider this vote only if the sender is a primary serving * a non zero number of slots, and its currentEpoch is greater or * equal to epoch where this node started the election. */ - if (clusterNodeIsPrimary(sender) && sender->numslots > 0 && - senderCurrentEpoch >= server.cluster->failover_auth_epoch) { + if (clusterNodeIsVotingPrimary(sender) && senderCurrentEpoch >= server.cluster->failover_auth_epoch) { server.cluster->failover_auth_count++; /* Maybe we reached a quorum here, set a flag to make sure * we check ASAP. */ @@ -4768,7 +4773,7 @@ void clusterCron(void) { if (!(node->flags & (CLUSTER_NODE_PFAIL | CLUSTER_NODE_FAIL))) { node->flags |= CLUSTER_NODE_PFAIL; update_state = 1; - if (clusterNodeIsPrimary(myself) && server.cluster->size == 1) { + if (server.cluster->size == 1 && clusterNodeIsVotingPrimary(myself)) { markNodeAsFailingIfNeeded(node); } else { serverLog(LL_DEBUG, "*** NODE %.40s possibly failing", node->name); @@ -5038,7 +5043,7 @@ void clusterUpdateState(void) { while ((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); - if (clusterNodeIsPrimary(node) && node->numslots) { + if (clusterNodeIsVotingPrimary(node)) { server.cluster->size++; if ((node->flags & (CLUSTER_NODE_FAIL | CLUSTER_NODE_PFAIL)) == 0) reachable_primaries++; } diff --git a/tests/unit/cluster/failure-marking.tcl b/tests/unit/cluster/failure-marking.tcl index c4746c8264..cfed7fff0f 100644 --- a/tests/unit/cluster/failure-marking.tcl +++ b/tests/unit/cluster/failure-marking.tcl @@ -16,6 +16,8 @@ start_cluster 1 1 {tags {external:skip cluster}} { pause_process $replica1_pid wait_node_marked_fail 0 $replica1_instance_id + + resume_process $replica1_pid } } @@ -49,5 +51,71 @@ start_cluster 2 1 {tags {external:skip cluster}} { resume_process $primary2_pid wait_node_marked_fail 0 $replica1_instance_id + + resume_process $replica1_pid + } +} + +set old_singledb $::singledb +set ::singledb 1 + +tags {external:skip tls:skip cluster} { + set base_conf [list cluster-enabled yes cluster-ping-interval 100 cluster-node-timeout 3000 save ""] + start_multiple_servers 5 [list overrides $base_conf] { + test "Only primary with slots has the right to mark a node as failed" { + set primary_host [srv 0 host] + set primary_port [srv 0 port] + set primary_pid [srv 0 pid] + set primary_id [R 0 CLUSTER MYID] + set replica_id [R 1 CLUSTER MYID] + set replica_pid [srv -1 pid] + + # Meet others nodes. + R 1 CLUSTER MEET $primary_host $primary_port + R 2 CLUSTER MEET $primary_host $primary_port + R 3 CLUSTER MEET $primary_host $primary_port + R 4 CLUSTER MEET $primary_host $primary_port + + # Build a single primary cluster. + cluster_allocate_slots 1 1 + wait_for_cluster_propagation + R 1 CLUSTER REPLICATE $primary_id + wait_for_cluster_propagation + wait_for_cluster_state "ok" + + # Pause the primary, marking the primary as pfail. + pause_process $primary_pid + wait_node_marked_pfail 1 $primary_id + wait_node_marked_pfail 2 $primary_id + wait_node_marked_pfail 3 $primary_id + wait_node_marked_pfail 4 $primary_id + + # Pause the replica, marking the replica as pfail. + pause_process $replica_pid + wait_node_marked_pfail 2 $replica_id + wait_node_marked_pfail 3 $replica_id + wait_node_marked_pfail 4 $replica_id + + # Resume the primary, marking the replica as fail. + resume_process $primary_pid + wait_node_marked_fail 0 $replica_id + wait_node_marked_fail 2 $replica_id + wait_node_marked_fail 3 $replica_id + wait_node_marked_fail 4 $replica_id + + # Check if we got the right failure reports. + wait_for_condition 1000 50 { + [R 0 CLUSTER COUNT-FAILURE-REPORTS $replica_id] == 0 && + [R 2 CLUSTER COUNT-FAILURE-REPORTS $replica_id] == 1 && + [R 3 CLUSTER COUNT-FAILURE-REPORTS $replica_id] == 1 && + [R 4 CLUSTER COUNT-FAILURE-REPORTS $replica_id] == 1 + } else { + fail "Cluster COUNT-FAILURE-REPORTS is not right." + } + + resume_process $replica_pid + } } } + +set ::singledb $old_singledb From 5a51bf5045d1713f5ebf58011dcb8900805be164 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Mon, 17 Jun 2024 12:18:20 +0800 Subject: [PATCH 11/11] Combine events to eliminate redundant kevent(2) calls (#638) Combine events to eliminate redundant kevent(2) calls to improve performance. --------- Signed-off-by: Andy Pan --- src/ae_kqueue.c | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 3cb6fbae4a..4159f25744 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -101,31 +101,24 @@ static void aeApiFree(aeEventLoop *eventLoop) { static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { aeApiState *state = eventLoop->apidata; - struct kevent ke; + struct kevent evs[2]; + int nch = 0; - if (mask & AE_READABLE) { - EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); - if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; - } - if (mask & AE_WRITABLE) { - EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL); - if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; - } - return 0; + if (mask & AE_READABLE) EV_SET(evs + nch++, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); + if (mask & AE_WRITABLE) EV_SET(evs + nch++, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL); + + return kevent(state->kqfd, evs, nch, NULL, 0, NULL); } static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { aeApiState *state = eventLoop->apidata; - struct kevent ke; + struct kevent evs[2]; + int nch = 0; - if (mask & AE_READABLE) { - EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); - kevent(state->kqfd, &ke, 1, NULL, 0, NULL); - } - if (mask & AE_WRITABLE) { - EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); - kevent(state->kqfd, &ke, 1, NULL, 0, NULL); - } + if (mask & AE_READABLE) EV_SET(evs + nch++, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + if (mask & AE_WRITABLE) EV_SET(evs + nch++, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); + + kevent(state->kqfd, evs, nch, NULL, 0, NULL); } static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {