From 844b1c3906ddde6663c5787951209c066ef79b2d Mon Sep 17 00:00:00 2001 From: Sajjad Rahnama Date: Wed, 29 Sep 2021 14:26:16 -0700 Subject: [PATCH 1/3] Major Changes, Refactoring --- .gitignore | 3 +- Makefile | 4 +- benchmarks/ycsb.h | 1 - benchmarks/ycsb_query.cpp | 1 - benchmarks/ycsb_query.h | 1 - benchmarks/ycsb_txn.cpp | 1 - benchmarks/ycsb_wl.cpp | 1 - client/client_query.h | 1 - config.h | 433 ++++++++++++++------------- data_structures/ring_all_comb.h | 48 +++ scripts/cross_shard.sh | 16 + scripts/io_idleness.sh | 70 +++++ scripts/result.sh | 11 +- scripts/result_colorized.sh | 92 +++++- scripts/result_v2.sh | 298 ++++++++++++++++++ scripts/scp_binaries.sh | 12 +- scripts/scp_results.sh | 8 +- scripts/set_ulimit.sh | 2 +- scripts/simRun.py | 3 +- scripts/startResilientDB.sh | 54 +++- scripts/vcloud_cmd.sh | 2 +- scripts/vcloud_deploy.sh | 6 +- scripts/vcloud_monitor.sh | 2 +- smart_contracts/banking_sc.cpp | 1 - smart_contracts/smart_contract.h | 1 - smart_contracts/smart_contract_txn.h | 1 - statistics/stats.cpp | 319 ++++++++++---------- statistics/stats.h | 39 ++- statistics/stats_array.cpp | 1 - system/array.h | 1 - system/client_thread.cpp | 54 +--- system/global.cpp | 63 ++-- system/global.h | 29 +- system/helper.cpp | 69 +++-- system/helper.h | 40 +-- system/io_thread.cpp | 64 ++-- system/main.cpp | 9 +- system/mem_alloc.cpp | 1 - system/msg_queue.cpp | 14 +- system/msg_queue.h | 3 +- system/parser.cpp | 1 - system/pool.cpp | 3 +- system/pool.h | 1 - system/query.h | 1 - system/sim_manager.cpp | 1 - system/thread.cpp | 30 +- system/thread.h | 3 + system/timer.cpp | 3 + system/txn.cpp | 27 +- system/txn.h | 3 - system/txn_table.h | 1 - system/txn_test.cpp | 1 - system/wl.cpp | 1 - system/work_queue.cpp | 134 +++------ system/work_queue.h | 60 +--- system/work_queue_nopipeline.cpp | 4 +- system/worker_thread.cpp | 323 +++++++------------- system/worker_thread.h | 8 +- system/worker_thread_pbft.cpp | 30 +- transport/message.cpp | 68 +++-- transport/message.h | 2 - transport/msg_thread.cpp | 12 +- transport/msg_thread.h | 6 +- transport/transport.cpp | 69 ++--- transport/transport.h | 3 +- 65 files changed, 1445 insertions(+), 1129 deletions(-) create mode 100644 data_structures/ring_all_comb.h create mode 100755 scripts/cross_shard.sh create mode 100755 scripts/io_idleness.sh create mode 100755 scripts/result_v2.sh diff --git a/.gitignore b/.gitignore index cb44dfbf8..01f99b367 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ scripts/rundb hostnames.py docker-compose.yml res.out -.DS_Store \ No newline at end of file +.DS_Store +exp/ \ No newline at end of file diff --git a/Makefile b/Makefile index 3bc6f66b1..5c3993621 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ CC=g++ -CFLAGS=-Wall -g -gdwarf-3 -std=c++0x -rdynamic # +CFLAGS=-Wall -g -gdwarf-3 -std=c++11 -rdynamic # JEMALLOC=deps/jemalloc-5.1.0 NNMSG=deps/nng-1.3.2 BOOST=deps/boost_1_67_0 @@ -9,7 +9,7 @@ SQLITE=deps/sqlite-autoconf-3290000/build .SUFFIXES: .o .cpp .h SRC_DIRS = ./ ./benchmarks/ ./client/ ./transport/ ./system/ ./statistics/ ./blockchain/ ./db/ ./smart_contracts/ ./data_structures -DEPS = -I. -I./benchmarks -I./client/ -I./transport -I./system -I./statistics -I./blockchain -I./smart_contracts -I./data_structures -I$(JEMALLOC)/include -I$(NNMSG)/include -I$(BOOST) -I$(CRYPTOPP) -I./db -I$(SQLITE)/include +DEPS = -I. -I./benchmarks -I./client/ -I./transport -I./system -I./statistics -I./blockchain -I./smart_contracts -I./data_structures -I$(JEMALLOC)/include -I$(NNMSG)/include -I$(BOOST) -I$(CRYPTOPP) -I./db -I$(SQLITE)/include CFLAGS += $(DEPS) -D NOGRAPHITE=1 -Werror -Wno-sizeof-pointer-memaccess LDFLAGS = -Wall -L. -L$(NNMSG)/lib -L$(JEMALLOC)/lib -Wl,-rpath,$(JEMALLOC)/lib -pthread -gdwarf-3 -lrt -std=c++0x -L$(CRYPTOPP) -L$(SQLITE)/lib diff --git a/benchmarks/ycsb.h b/benchmarks/ycsb.h index cffd6a6ed..16dc57888 100644 --- a/benchmarks/ycsb.h +++ b/benchmarks/ycsb.h @@ -4,7 +4,6 @@ #include "wl.h" #include "txn.h" #include "global.h" -#include "helper.h" #if !BANKING_SMART_CONTRACT class YCSBQuery; diff --git a/benchmarks/ycsb_query.cpp b/benchmarks/ycsb_query.cpp index 98a31fd6c..9193c96de 100644 --- a/benchmarks/ycsb_query.cpp +++ b/benchmarks/ycsb_query.cpp @@ -3,7 +3,6 @@ #include "mem_alloc.h" //#include "wl.h" #include "ycsb.h" -#include "helper.h" #include "message.h" uint64_t YCSBQueryGenerator::the_n = 0; diff --git a/benchmarks/ycsb_query.h b/benchmarks/ycsb_query.h index 52e2220c4..5692dd861 100644 --- a/benchmarks/ycsb_query.h +++ b/benchmarks/ycsb_query.h @@ -2,7 +2,6 @@ #define _YCSBQuery_H_ #include "global.h" -#include "helper.h" #include "query.h" #include "array.h" diff --git a/benchmarks/ycsb_txn.cpp b/benchmarks/ycsb_txn.cpp index 8e63cced5..42c6f7df7 100644 --- a/benchmarks/ycsb_txn.cpp +++ b/benchmarks/ycsb_txn.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "ycsb.h" #include "ycsb_query.h" #include "wl.h" diff --git a/benchmarks/ycsb_wl.cpp b/benchmarks/ycsb_wl.cpp index 632da1886..ca094cbcb 100644 --- a/benchmarks/ycsb_wl.cpp +++ b/benchmarks/ycsb_wl.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "ycsb.h" #include "wl.h" #include "thread.h" diff --git a/client/client_query.h b/client/client_query.h index f03827b63..e36ba3b99 100644 --- a/client/client_query.h +++ b/client/client_query.h @@ -2,7 +2,6 @@ #define _CLIENT_QUERY_H_ #include "global.h" -#include "helper.h" #include "query.h" //class Workload; diff --git a/config.h b/config.h index baf67032b..5ac0395a9 100644 --- a/config.h +++ b/config.h @@ -1,227 +1,232 @@ -#ifndef _CONFIG_H_ -#define _CONFIG_H_ -// Specify the number of servers or replicas -#define NODE_CNT 4 -// Number of worker threads at primary. For RBFT (6) and other algorithms (5). -#define THREAD_CNT 5 -#define REM_THREAD_CNT 3 -#define SEND_THREAD_CNT 1 -#define CORE_CNT 8 -#define PART_CNT 1 -// Specify the number of clients. -#define CLIENT_NODE_CNT 1 -#define CLIENT_THREAD_CNT 2 -#define CLIENT_REM_THREAD_CNT 1 -#define CLIENT_SEND_THREAD_CNT 1 -#define CLIENT_RUNTIME false -#define LOAD_PER_SERVER 1 -#define REPLICA_CNT 0 -#define REPL_TYPE AP -#define VIRTUAL_PART_CNT PART_CNT -#define PAGE_SIZE 4096 -#define CL_SIZE 64 -#define CPU_FREQ 2.6 -#define HW_MIGRATE false -#define WARMUP 0 -#define WORKLOAD YCSB -#define PRT_LAT_DISTR false -#define STATS_ENABLE true -#define TIME_ENABLE true -#define TIME_PROF_ENABLE false -#define FIN_BY_TIME true -// Number of transactions each client should send without waiting. -#define MAX_TXN_IN_FLIGHT 20000 -#define MESSAGE_PER_BUFFER 1 -#define SERVER_GENERATE_QUERIES false -#define MEM_ALLIGN 8 -#define THREAD_ALLOC false -#define THREAD_ARENA_SIZE (1UL << 22) -#define MEM_PAD true -#define PART_ALLOC false -#define MEM_SIZE (1UL << 30) -#define NO_FREE false -#define TPORT_TYPE TCP -#define TPORT_PORT 17000 -#define SET_AFFINITY false -#define MAX_TPORT_NAME 128 -#define MSG_SIZE 128 -#define HEADER_SIZE sizeof(uint32_t)*2 -#define MSG_TIMEOUT 5000000000UL // in ns -#define NETWORK_TEST false -#define NETWORK_DELAY_TEST false -#define NETWORK_DELAY 0UL -#define MAX_QUEUE_LEN NODE_CNT * 2 -#define PRIORITY_WORK_QUEUE false -#define PRIORITY PRIORITY_ACTIVE -#define MSG_SIZE_MAX 1048576 -#define MSG_TIME_LIMIT 0 -#define KEY_ORDER false -#define ENABLE_LATCH false -#define CENTRAL_INDEX false -#define CENTRAL_MANAGER false -#define INDEX_STRUCT IDX_HASH -#define BTREE_ORDER 16 -#define TS_TWR false -#define TS_ALLOC TS_CLOCK -#define TS_BATCH_ALLOC false -#define TS_BATCH_NUM 1 -#define HIS_RECYCLE_LEN 10 -#define MAX_PRE_REQ MAX_TXN_IN_FLIGHT * NODE_CNT -#define MAX_READ_REQ MAX_TXN_IN_FLIGHT * NODE_CNT -#define MIN_TS_INTVL 10 * 1000000UL -#define MAX_WRITE_SET 10 -#define PER_ROW_VALID false -#define TXN_QUEUE_SIZE_LIMIT THREAD_CNT -#define SEQ_THREAD_CNT 4 -#define MAX_ROW_PER_TXN 64 -#define QUERY_INTVL 1UL -#define MAX_TXN_PER_PART 4000 -#define FIRST_PART_LOCAL true -#define MAX_TUPLE_SIZE 1024 -#define GEN_BY_MPR false -#define SKEW_METHOD ZIPF -#define DATA_PERC 100 -#define ACCESS_PERC 0.03 -#define INIT_PARALLELISM 8 +#ifndef _CONFIG_H_ +#define _CONFIG_H_ +// Specify the number of servers or replicas +#define NODE_CNT 12 +// Number of worker threads at primary. +#define THREAD_CNT 5 // This Should be the sum of following thread count + protocol specifig threads +#define WORKER_THREAD_CNT 1 +#define BATCH_THREAD_CNT 2 +#define CHECKPOINT_THREAD_CNT 1 +#define EXECUTE_THREAD_CNT 1 +// IO THREADS +#define REM_THREAD_CNT 2 +#define SEND_THREAD_CNT 1 +#define CORE_CNT 8 +#define PART_CNT 1 +// Specify the number of clients. +#define CLIENT_NODE_CNT 3 +#define CLIENT_THREAD_CNT 2 +#define CLIENT_REM_THREAD_CNT 1 +#define CLIENT_SEND_THREAD_CNT 1 +#define CLIENT_RUNTIME false + +#define MESSAGE_PER_BUFFER 24 + +#define LOAD_PER_SERVER 1 +#define REPLICA_CNT 0 +#define REPL_TYPE AP +#define VIRTUAL_PART_CNT PART_CNT +#define PAGE_SIZE 4096 +#define CL_SIZE 64 +#define CPU_FREQ 2.2 +#define HW_MIGRATE false +#define WARMUP 0 +#define WORKLOAD YCSB +#define PRT_LAT_DISTR false +#define STATS_ENABLE true +#define STATS_DETAILED false +#define STAT_BAND_WIDTH_ENABLE false +#define TIME_ENABLE true +#define TIME_PROF_ENABLE false +#define FIN_BY_TIME true +// Number of transactions each client should send without waiting. +#define MAX_TXN_IN_FLIGHT 20000 +#define SERVER_GENERATE_QUERIES false +#define MEM_ALLIGN 8 +#define THREAD_ALLOC false +#define THREAD_ARENA_SIZE (1UL << 22) +#define MEM_PAD true +#define PART_ALLOC false +#define MEM_SIZE (1UL << 30) +#define NO_FREE false +#define TPORT_TYPE TCP +#define TPORT_PORT 10000 +#define TPORT_WINDOW 20000 +#define SET_AFFINITY false +#define MAX_TPORT_NAME 128 +#define MSG_SIZE 128 +#define HEADER_SIZE sizeof(uint32_t) * 2 +#define MSG_TIMEOUT 5000000000UL // in ns +#define NETWORK_TEST false +#define NETWORK_DELAY_TEST false +#define NETWORK_DELAY 0UL +#define MAX_QUEUE_LEN NODE_CNT * 2 +#define PRIORITY_WORK_QUEUE false +#define PRIORITY PRIORITY_ACTIVE +#define MSG_SIZE_MAX 1048576 +#define MSG_TIME_LIMIT 0 +#define KEY_ORDER false +#define ENABLE_LATCH false +#define CENTRAL_INDEX false +#define CENTRAL_MANAGER false +#define INDEX_STRUCT IDX_HASH +#define BTREE_ORDER 16 +#define TS_TWR false +#define TS_ALLOC TS_CLOCK +#define TS_BATCH_ALLOC false +#define TS_BATCH_NUM 1 +#define HIS_RECYCLE_LEN 10 +#define MAX_PRE_REQ MAX_TXN_IN_FLIGHT *NODE_CNT +#define MAX_READ_REQ MAX_TXN_IN_FLIGHT *NODE_CNT +#define MIN_TS_INTVL 10 * 1000000UL +#define MAX_WRITE_SET 10 +#define PER_ROW_VALID false +#define TXN_QUEUE_SIZE_LIMIT THREAD_CNT +#define SEQ_THREAD_CNT 4 +#define MAX_ROW_PER_TXN 64 +#define QUERY_INTVL 1UL +#define MAX_TXN_PER_PART 4000 +#define FIRST_PART_LOCAL true +#define MAX_TUPLE_SIZE 1024 +#define GEN_BY_MPR false +#define SKEW_METHOD ZIPF +#define DATA_PERC 100 +#define ACCESS_PERC 0.03 +#define INIT_PARALLELISM 8 #define SYNTH_TABLE_SIZE 524288 -#define ZIPF_THETA 0.5 -#define WRITE_PERC 0.9 -#define TXN_WRITE_PERC 0.5 -#define TUP_WRITE_PERC 0.5 -#define SCAN_PERC 0 -#define SCAN_LEN 20 -#define PART_PER_TXN PART_CNT -#define PERC_MULTI_PART MPR -#define REQ_PER_QUERY 1 -#define FIELD_PER_TUPLE 10 -#define CREATE_TXN_FILE false -#define SINGLE_THREAD_WL_GEN true -#define STRICT_PPT 1 -#define MPR 1.0 -#define MPIR 0.01 -#define WL_VERB true -#define IDX_VERB false -#define VERB_ALLOC true -#define DEBUG_LOCK false -#define DEBUG_TIMESTAMP false -#define DEBUG_SYNTH false -#define DEBUG_ASSERT false -#define DEBUG_DISTR false -#define DEBUG_ALLOC false -#define DEBUG_RACE false -#define DEBUG_TIMELINE false -#define DEBUG_BREAKDOWN false -#define DEBUG_LATENCY false -#define DEBUG_QUECC false -#define DEBUG_WLOAD false -#define MODE NORMAL_MODE -#define DBTYPE REPLICATED -#define IDX_HASH 1 -#define IDX_BTREE 2 -#define YCSB 1 -#define TEST 4 -#define TS_MUTEX 1 -#define TS_CAS 2 -#define TS_HW 3 -#define TS_CLOCK 4 -#define ZIPF 1 -#define HOT 2 -#define PRIORITY_FCFS 1 -#define PRIORITY_ACTIVE 2 -#define PRIORITY_HOME 3 -#define AA1 1 -#define AP 2 -#define LOAD_MAX 1 -#define LOAD_RATE 2 -#define TCP 1 -#define IPC 2 -#define BILLION 1000000000UL -#define MILLION 1000000UL -#define STAT_ARR_SIZE 1024 -#define PROG_TIMER 10 * BILLION -#define SEQ_BATCH_TIMER 5 * 1 * MILLION -#define SEED 0 -#define SHMEM_ENV false -#define ENVIRONMENT_EC2 false -#define PARTITIONED 0 -#define REPLICATED 1 -// To select the amount of time to warmup and run. -#define DONE_TIMER 2 * 60 * BILLION -#define WARMUP_TIMER 1 * 60 * BILLION -// Select the consensus algorithm to run. +#define ZIPF_THETA 0.5 +#define WRITE_PERC 0.9 +#define TXN_WRITE_PERC 0.5 +#define TUP_WRITE_PERC 0.5 +#define SCAN_PERC 0 +#define SCAN_LEN 20 +#define PART_PER_TXN PART_CNT +#define PERC_MULTI_PART MPR +#define REQ_PER_QUERY 1 +#define FIELD_PER_TUPLE 10 +#define CREATE_TXN_FILE false +#define SINGLE_THREAD_WL_GEN true +#define STRICT_PPT 1 +#define MPR 1.0 +#define MPIR 0.01 +#define WL_VERB true +#define IDX_VERB false +#define VERB_ALLOC true +#define DEBUG_LOCK false +#define DEBUG_TIMESTAMP false +#define DEBUG_SYNTH false +#define DEBUG_ASSERT false +#define DEBUG_DISTR false +#define DEBUG_ALLOC false +#define DEBUG_RACE false +#define DEBUG_TIMELINE false +#define DEBUG_BREAKDOWN false +#define DEBUG_LATENCY false +#define DEBUG_QUECC false +#define DEBUG_WLOAD false +#define MODE NORMAL_MODE +#define DBTYPE REPLICATED +#define IDX_HASH 1 +#define IDX_BTREE 2 +#define YCSB 1 +#define TEST 4 +#define TS_MUTEX 1 +#define TS_CAS 2 +#define TS_HW 3 +#define TS_CLOCK 4 +#define ZIPF 1 +#define HOT 2 +#define PRIORITY_FCFS 1 +#define PRIORITY_ACTIVE 2 +#define PRIORITY_HOME 3 +#define AA1 1 +#define AP 2 +#define LOAD_MAX 1 +#define LOAD_RATE 2 +#define TCP 1 +#define IPC 2 +#define BILLION 1000000000UL +#define MILLION 1000000UL +#define STAT_ARR_SIZE 1024 +#define PROG_TIMER 5 * BILLION +#define SEQ_BATCH_TIMER 5 * 1 * MILLION +#define SEED 0 +#define SHMEM_ENV false +#define ENVIRONMENT_EC2 false +#define PARTITIONED 0 +#define REPLICATED 1 +// To select the amount of time to warmup and run. +#define DONE_TIMER 1 * 60 * BILLION +#define WARMUP_TIMER 5 * BILLION +// Select the consensus algorithm to run. #define CONSENSUS PBFT -#define DBFT 1 -#define PBFT 2 -#define ZYZZYVA 3 -#define HOTSTUFF 4 -// Switching on RBFT consensus. -// Status: Partial implementation, only for PBFT. -#define RBFT_ON false -// Select the type of RBFT, (1) RBFT+PBFT, and (2) RBFT+DBFT -#define RBFT_ALG RPBFT -#define RPBFT 1 -#define RDBFT 2 -// Enable or Disable pipeline at primary replica. -#define ENABLE_PIPELINE true -// Number of threads to create batches at primary replica. -#define BATCH_THREADS 2 -// Size of each batch. -#define BATCH_SIZE 100 -#define BATCH_ENABLE BSET -#define BSET 1 -#define BUNSET 0 -// Number of transactions to wait for period checkpointing. -#define TXN_PER_CHKPT 600 -#define EXECUTION_THREAD true -#define EXECUTE_THD_CNT 1 -#define SIGN_THREADS false -#define SIGN_THD_CNT 1 -#define CLIENT_BATCH true -#define CLIENT_RESPONSE_BATCH true +#define DBFT 1 +#define PBFT 2 +#define ZYZZYVA 3 +#define HOTSTUFF 4 +// Switching on RBFT consensus. +// Status: Partial implementation, only for PBFT. +#define RBFT_ON false +// Select the type of RBFT, (1) RBFT+PBFT, and (2) RBFT+DBFT +#define RBFT_ALG RPBFT +#define RPBFT 1 +#define RDBFT 2 +// Enable or Disable pipeline at primary replica. +#define ENABLE_PIPELINE true +// Size of each batch. +#define BATCH_SIZE 100 +#define BATCH_ENABLE BSET +#define BSET 1 +#define BUNSET 0 +// Number of transactions to wait for period checkpointing. +#define TXN_PER_CHKPT 6 * BATCH_SIZE +#define SIGN_THREADS false +#define SIGN_THD_CNT 1 +#define CLIENT_BATCH true +#define CLIENT_RESPONSE_BATCH true // To Enable or disable the blockchain implementation. -#define ENABLE_CHAIN true -// To fail non-primary replicas. -#define LOCAL_FAULT false -#define NODE_FAIL_CNT 1 -// To allow view changes. -#define VIEW_CHANGES false -// The amount of timeout value. -#define EXE_TIMEOUT 10000000000 -#define CEXE_TIMEOUT 12000000000 -// To turn the timer on. -#define TIMER_ON false -//Global variables to choose the encryptation algorithm -#define USE_CRYPTO true -#define CRYPTO_METHOD_RSA false //Options RSA, -#define CRYPTO_METHOD_ED25519 true // Option ED25519 -#define CRYPTO_METHOD_CMAC_AES true // CMAC -// Test cases to check basic functioning. -// Status: Implementation only for PBFT. -#define TESTING_ON false -#define TEST_CASE ONLY_PRIMARY_BATCH_EXECUTE -#define ONLY_PRIMARY_NO_EXECUTE 1 -#define ONLY_PRIMARY_EXECUTE 2 -#define ONLY_PRIMARY_BATCH_EXECUTE 3 -// Message Payload. -// We allow creation of two different message payloads, -// to see affects on latency and throughput. -// These payloads are added to each message. -#define PAYLOAD_ENABLE false -#define PAYLOAD M100 -#define M100 1 // 100KB. -#define M200 2 // 200KB. -#define M400 3 // 400KB. +#define ENABLE_CHAIN false +// To fail non-primary replicas. +#define LOCAL_FAULT false +#define NODE_FAIL_CNT 1 +// To allow view changes. +#define VIEW_CHANGES false +// The amount of timeout value. +#define EXE_TIMEOUT 10000000000 +#define CEXE_TIMEOUT 12000000000 +// To turn the timer on. +#define TIMER_ON false +//Global variables to choose the encryptation algorithm +#define USE_CRYPTO true +#define CRYPTO_METHOD_RSA false //Options RSA, +#define CRYPTO_METHOD_ED25519 true // Option ED25519 +#define CRYPTO_METHOD_CMAC_AES true // CMAC +// Test cases to check basic functioning. +// Status: Implementation only for PBFT. +#define TESTING_ON false +#define TEST_CASE ONLY_PRIMARY_BATCH_EXECUTE +#define ONLY_PRIMARY_NO_EXECUTE 1 +#define ONLY_PRIMARY_EXECUTE 2 +#define ONLY_PRIMARY_BATCH_EXECUTE 3 +// Message Payload. +// We allow creation of two different message payloads, +// to see affects on latency and throughput. +// These payloads are added to each message. +#define PAYLOAD_ENABLE false +#define PAYLOAD M100 +#define M100 1 // 100KB. +#define M200 2 // 200KB. +#define M400 3 // 400KB. -// To allow testing in-memory database or SQLite. -// Further, using SQLite a user can also choose to persist the data. +// To allow testing in-memory database or SQLite. +// Further, using SQLite a user can also choose to persist the data. #define EXT_DB MEMORY #define MEMORY 1 #define SQL 2 #define SQL_PERSISTENT 3 -// To allow testing of a Banking Smart Contracts. +// To allow testing of a Banking Smart Contracts. #define BANKING_SMART_CONTRACT false #endif - diff --git a/data_structures/ring_all_comb.h b/data_structures/ring_all_comb.h new file mode 100644 index 000000000..54960bb03 --- /dev/null +++ b/data_structures/ring_all_comb.h @@ -0,0 +1,48 @@ +#ifndef _RING_ALL_COMB_H_ +#define _RING_ALL_COMB_H_ + + +#include /* printf, scanf, puts, NULL */ +#include /* srand, rand */ +#include /* time */ +#include +#include + +class AllComb +{ +private: + uint64_t number_of_invloved_shards; + uint64_t gnode_id; + uint64_t shards_count; + +public: + std::vector> output; + AllComb(uint64_t inv, uint64_t gnode, uint64_t sh_cnt) : number_of_invloved_shards(inv), gnode_id(gnode), shards_count(sh_cnt) {} + + std::vector> combine() + { + std::vector curr; + back_track(0, curr); + return output; + } + + void back_track(uint64_t start, std::vector &curr) + { + if (curr.size() == number_of_invloved_shards) + { + std::vector temp = curr; + output.push_back(temp); + return; + } + for (uint64_t i = start; i < shards_count; i++) + { + if (i == gnode_id) + continue; + curr.push_back(i); + back_track(start + 1, curr); + curr.pop_back(); + } + } +}; + +#endif \ No newline at end of file diff --git a/scripts/cross_shard.sh b/scripts/cross_shard.sh new file mode 100755 index 000000000..596282f2f --- /dev/null +++ b/scripts/cross_shard.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +precentages=(0 5 10 20 40 80 100) + +name="S" +runs=2 +nodes=420 +clients=75 + +for i in "${precentages[@]}" +do + sed -i "22s/.*/#define CROSS_SHARD_PRECENTAGE ${i}/" config.h + make clean; + make -j8; + ./scripts/startResilientDB.sh ${nodes} ${clients} ${name}_${i}p $runs +done diff --git a/scripts/io_idleness.sh b/scripts/io_idleness.sh new file mode 100755 index 000000000..330d68a13 --- /dev/null +++ b/scripts/io_idleness.sh @@ -0,0 +1,70 @@ +#!/bin/bash +unset GREP_OPTIONS +# Black='\033[1;90m' # Black +Nc='\033[0m' +Red='\033[1;91m' # Red + +# Green='\033[1;92m' # Green +# Yellow='\033[1;93m' # Yellow +# Blue='\033[1;94m' # Blue +# Purple='\033[1;95m' # Purple +# Cyan='\033[1;96m' # Cyan +# White='\033[1;97m' # White + +snodes=$1 +cnodes=$2 +protocol=$3 +shard_size=$4 +bsize=$5 +run=$6 +folder=$7 +total_nodes=$((cnodes + snodes - 1)) +avg_thp=0 +thp_cnt=0 +client_thp=0 +avg_lt=0.0 +lt_cnt=0 +avg_msg=0 +msg_cnt=0 +shard_counter=0 +shard_tp=0 +shard_ctp=0 +shards_tp=() +shards_ctp=() +if [ "$(uname)" == "Darwin" ]; then + flags="-E" +else + flags="-P" +fi + +cd results/ +#if [ ! -z "$folder" ];then +# cd $folder +#fi + +echo "Outputs - Inputs" +for i in $(seq 0 $(($snodes - 1))); do + echo "I Node Output: $i" + times=($(tail -70 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} "Output .{1,10}: .{1,6}" | grep -o ${flags} '\d+\.\d+')) + for ((j = 0; j < ${#times[@]}; j++)); do + # echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" + echo -en "${Red}${times[$j]}${Nc} " + done + echo -en " ---- " + times=($(tail -70 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} "Input .{1,10}: .{1,6}" | grep -o ${flags} '\d+\.\d+')) + for ((j = 0; j < ${#times[@]}; j++)); do + # echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" + echo -en "${Red}${times[$j]}${Nc} " + done + echo +done +# echo "Inputs:" +# for i in $(seq 0 $(($snodes - 1))); do +# echo "I Node Input: $i" +# times=($(tail -70 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} "Input .{1,10}: .{1,6}" | grep -o ${flags} '\d+\.\d+')) +# for ((j = 0; j < ${#times[@]}; j++)); do +# # echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" +# echo -en "${Red}${times[$j]}${Nc} " +# done +# echo +# done \ No newline at end of file diff --git a/scripts/result.sh b/scripts/result.sh index 8e293c2a6..bcf4f4bd1 100755 --- a/scripts/result.sh +++ b/scripts/result.sh @@ -1,6 +1,9 @@ #!/bin/bash unset GREP_OPTIONS # Black='\033[1;90m' # Black +Nc='\033[0m' +Red='\033[1;91m' # Red + # Green='\033[1;92m' # Green # Yellow='\033[1;93m' # Yellow # Blue='\033[1;94m' # Blue @@ -35,7 +38,7 @@ cd results/ echo "Throughputs:" for i in $(seq 0 $(($total_nodes))); do if [ "$i" -lt "$snodes" ]; then - temp=$(tail -74 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput[\s]+=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') temp2=$(tail -74 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'msg_send_cnt=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') if [ ! -z "$temp" ]; then avg_thp=$(($avg_thp + $temp)) @@ -44,7 +47,7 @@ for i in $(seq 0 $(($total_nodes))); do #msg_cnt=$(($msg_cnt + 1)) fi else - temp=$(tail -4 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') client_thp=$(($client_thp + $temp)) fi echo -e "$i: ${Red}${temp}${Nc}" @@ -52,7 +55,7 @@ done echo "Latencies:" for i in $(seq $snodes $(($total_nodes))); do - temp=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'AVG: .{1,13}' | grep -o ${flags} '\d+\.\d+') + temp=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^Latency=.{1,13}' | grep -o ${flags} '\d+\.\d+') avg_lt=$(echo "$avg_lt + $temp" | bc) lt_cnt=$(($lt_cnt + 1)) echo -e "latency $i: ${Red}${temp}${Nc}" @@ -71,7 +74,7 @@ done echo "Memory:" for i in $(seq 0 $total_nodes); do - mem=$(tail -5 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + mem=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') [ ! -z "$mem" ] || mem=0 echo "$i: $(($mem / 1000)) MB" done diff --git a/scripts/result_colorized.sh b/scripts/result_colorized.sh index 40126d2cc..66df0096b 100755 --- a/scripts/result_colorized.sh +++ b/scripts/result_colorized.sh @@ -14,9 +14,10 @@ Red='\033[1;91m' # Red snodes=$1 cnodes=$2 protocol=$3 -bsize=$4 -run=$5 -folder=$6 +shard_size=$4 +bsize=$5 +run=$6 +folder=$7 total_nodes=$((cnodes + snodes - 1)) avg_thp=0 thp_cnt=0 @@ -25,6 +26,11 @@ avg_lt=0.0 lt_cnt=0 avg_msg=0 msg_cnt=0 +shard_counter=0 +shard_tp=0 +shard_ctp=0 +shards_tp=() +shards_ctp=() if [ "$(uname)" == "Darwin" ]; then flags="-E" else @@ -38,49 +44,109 @@ cd results/ echo "Throughputs:" for i in $(seq 0 $(($total_nodes))); do if [ "$i" -lt "$snodes" ]; then - temp=$(tail -74 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp=$(tail -20 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp3=$(tail -20 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'cput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') temp2=$(tail -74 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'msg_send_cnt=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') if [ ! -z "$temp" ]; then avg_thp=$(($avg_thp + $temp)) thp_cnt=$(($thp_cnt + 1)) + shard_counter=$(($shard_counter + 1)) + shard_tp=$(($shard_tp + $temp)) + shard_ctp=$(($shard_ctp + $temp3)) #avg_msg=$(($avg_msg + $temp2)) #msg_cnt=$(($msg_cnt + 1)) fi + if [[ $shard_counter -eq $shard_size ]]; then + shards_tp=("${shards_tp[@]}" "$shard_tp") + shards_ctp=("${shards_ctp[@]}" "$shard_ctp") + shard_counter=0 + shard_tp=0 + shard_ctp=0 + fi + j=$i + if [ $j -lt 10 ]; then + j="0$j" + fi + echo -e "$j: ${Red}${temp}${Nc}\t${Red}${temp3}${Nc}" else temp=$(tail -4 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') client_thp=$(($client_thp + $temp)) + j=$i + if [ $j -lt 10 ]; then + j="0$j" + fi + echo -e "$j: ${Red}${temp}${Nc}" fi - echo -e "$i: ${Red}${temp}${Nc}" + done echo "Latencies:" for i in $(seq $snodes $(($total_nodes))); do temp=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'AVG: .{1,13}' | grep -o ${flags} '\d+\.\d+') - avg_lt=$(echo "$avg_lt + $temp" | bc) - lt_cnt=$(($lt_cnt + 1)) - echo -e "latency $i: ${Red}${temp}${Nc}" + if [ ! -z "$temp" ]; then + avg_lt=$(echo "$avg_lt + $temp" | bc) + lt_cnt=$(($lt_cnt + 1)) + echo -e "latency $i: ${Red}${temp}${Nc}" + else + echo -e "latency $i: ${Red}NAAAAAAN${Nc}" + fi + done echo echo "idle times:" for i in $(seq 0 $(($snodes - 1))); do - echo "Idleness of node: $i" + echo "I Node: $i" times=($(tail -50 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} "idle_time_worker.{1,10}" | grep -o ${flags} '\d+\.\d+')) for ((j = 0; j < ${#times[@]}; j++)); do - echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" + # echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" + echo -en "${Red}${times[$j]}${Nc} " done + echo done echo "Memory:" for i in $(seq 0 $total_nodes); do - mem=$(tail -5 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + lines=10 + if [ $i -lt $snodes ]; then + lines=20 + fi + mem=$(tail -${lines} s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') [ ! -z "$mem" ] || mem=0 echo "$i: $(($mem / 1000)) MB" done echo -echo "avg thp: ${thp_cnt}: $(echo -e ${Red}$(expr $avg_thp / $thp_cnt)${Nc})" -echo "avg lt : ${lt_cnt}: $(echo -e ${Red}$(echo "scale=3;$avg_lt / $lt_cnt" | bc)${Nc})" +total_tp=0 +avg_ctp=0 +counter=0 +echo "Shards TP:" +for t in ${shards_tp[@]}; do + shard_avg_tp=$(echo "$t / $shard_size" | bc) + total_tp=$(($total_tp + $shard_avg_tp)) + echo "Shards ${counter} TP: $(echo -e ${Red}${shard_avg_tp}${Nc})" + counter=$(($counter + 1)) +done +counter=0 +echo "Shards CTP:" +for t in ${shards_ctp[@]}; do + shard_avg_tp=$(echo "$t / $shard_size" | bc) + avg_ctp=$(($avg_ctp + $shard_avg_tp)) + echo "Shards ${counter} CTP: $(echo -e ${Red}${shard_avg_tp}${Nc})" + if [ $shard_avg_tp -gt 0 ]; then + counter=$(($counter + 1)) + fi + +done +echo +if [ $counter -eq 0 ]; then + counter=1 +fi +echo "total avg TP: $(echo -e ${Red}${total_tp}${Nc})" +echo "total avg CTP: $(echo -e ${Red}$(echo "scale=0;$avg_ctp / $counter" | bc)${Nc})" +echo "" +echo "Final avg TP: $(echo -e ${Red}$(echo "scale=0;$avg_ctp / $counter + $total_tp" | bc)${Nc})" +echo "avg lt ${lt_cnt}: $(echo -e ${Red}$(echo "scale=3;$avg_lt / $lt_cnt" | bc)${Nc})" #echo "avg msg: ${msg_cnt}: $(echo -e ${Red}$(expr $avg_msg / $msg_cnt)${Nc})" # echo -e "cli thp sum: ${RED}$client_thp${NC}" diff --git a/scripts/result_v2.sh b/scripts/result_v2.sh new file mode 100755 index 000000000..f70254034 --- /dev/null +++ b/scripts/result_v2.sh @@ -0,0 +1,298 @@ +#!/bin/bash +unset GREP_OPTIONS +# Black='\033[1;90m' # Black +Nc='\033[0m' +Red='\033[1;91m' # Red + +# Green='\033[1;92m' # Green +# Yellow='\033[1;93m' # Yellow +# Blue='\033[1;94m' # Blue +# Purple='\033[1;95m' # Purple +# Cyan='\033[1;96m' # Cyan +# White='\033[1;97m' # White + +snodes=$1 +cnodes=$2 +protocol=$3 +shard_size=$4 +c_shard_size=$(echo "$snodes / $shard_size" | bc) +c_shard_size=$(echo "$cnodes / $c_shard_size" | bc) +bsize=$5 +run=$6 +folder=$7 +total_nodes=$((cnodes + snodes - 1)) +avg_lt=0.0 +lt_cnt=0 +avg_clt=0 +clt_cnt=0 +avg_lt_cnt=0 +avg_msg=0 +msg_cnt=0 + +shard_server_counter=0 +shard_server_tp=0 +shard_server_ctp=0 +shards_server_tp=() +shards_server_ctp=() + +shard_cl_counter=0 +shard_cl_tp=0 +shard_cl_ctp=0 +shards_cl_tp=() +shards_cl_ctp=() + +shard_lt=0 +shard_clt=0 +avg_shard_lt=0 +shards_lt=() +shards_clt=() +avg_shards_lt=() +if [ "$(uname)" == "Darwin" ]; then + flags="-E" +else + flags="-P" +fi + +if [ ! -z "$folder" ]; then + cd $folder +else + cd results/ +fi +echo "Throughputs:" +for i in $(seq 0 $(($total_nodes))); do + if [ "$i" -lt "$snodes" ]; then + temp=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp3=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^cput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + # temp2=$(tail -74 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'msg_send_cnt=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + if [ -z "$temp" ]; then + temp=0 + fi + if [ -z "$temp3" ]; then + temp3=0 + fi + shard_server_counter=$(($shard_server_counter + 1)) + shard_server_tp=$(($shard_server_tp + $temp)) + shard_server_ctp=$(($shard_server_ctp + $temp3)) + #avg_msg=$(($avg_msg + $temp2)) + #msg_cnt=$(($msg_cnt + 1)) + if [[ $shard_server_counter -eq $shard_size ]]; then + shards_server_tp=("${shards_server_tp[@]}" "$shard_server_tp") + shards_server_ctp=("${shards_server_ctp[@]}" "$shard_server_ctp") + shard_server_counter=0 + shard_server_tp=0 + shard_server_ctp=0 + fi + j=$i + if [ $j -lt 10 ]; then + j="0$j" + fi + echo -e "$j: ${Red}${temp}${Nc}\t${Red}${temp3}${Nc}" + else + temp=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp2=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^cput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + if [ -z "$temp" ]; then + temp=0 + fi + if [ -z "$temp2" ]; then + temp2=0 + fi + + shard_cl_counter=$(($shard_cl_counter + 1)) + shard_cl_tp=$(($shard_cl_tp + $temp)) + shard_cl_ctp=$(($shard_cl_ctp + $temp2)) + #avg_msg=$(($avg_msg + $temp2)) + #msg_cnt=$(($msg_cnt + 1)) + if [[ $shard_cl_counter -eq $c_shard_size ]]; then + shards_cl_tp=("${shards_cl_tp[@]}" "$shard_cl_tp") + shards_cl_ctp=("${shards_cl_ctp[@]}" "$shard_cl_ctp") + shard_cl_counter=0 + shard_cl_tp=0 + shard_cl_ctp=0 + fi + + j=$i + if [ $j -lt 10 ]; then + j="0$j" + fi + echo -e "$j: ${Red}${temp}${Nc}\t${Red}${temp2}${Nc}" + fi + +done + +echo "Latencies:" +for i in $(seq $snodes $(($total_nodes))); do + temp=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^Latency=.{1,13}' | grep -o ${flags} '\d+\.\d+') + temp2=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^CLatency=.{1,13}' | grep -o ${flags} '\d+\.\d+') + temp3=$(tail -50 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^AVG Latency:.{1,13}' | grep -o ${flags} '\d+\.\d+') + if [ ! -z "$temp" ]; then + shard_lt=$(echo "scale=3;$shard_lt + $temp" | bc) + lt_cnt=$(($lt_cnt + 1)) + else + temp="NAN" + fi + if [ ! -z "$temp2" ]; then + shard_clt=$(echo "scale=3;$shard_clt + $temp2" | bc) + clt_cnt=$(($clt_cnt + 1)) + else + temp2="NAN" + fi + if [ ! -z "$temp3" ]; then + avg_shard_lt=$(echo "scale=3;$avg_shard_lt + $temp3" | bc) + avg_lt_cnt=$(($avg_lt_cnt + 1)) + else + temp3="NAN" + fi + shard_cl_counter=$(($shard_cl_counter + 1)) + echo -e "$i: ${Red}${temp}${Nc}\t${Red}${temp2}${Nc}" + if [[ $shard_cl_counter -eq $c_shard_size ]]; then + if [ $lt_cnt -ne 0 ]; then + t=$(echo "scale=3;$shard_lt / $lt_cnt" | bc) + else + t=0 + fi + if [ $clt_cnt -ne 0 ]; then + t2=$(echo "scale=3;$shard_clt / $clt_cnt" | bc) + else + t2=0 + fi + if [ $avg_lt_cnt -ne 0 ]; then + t3=$(echo "scale=3;$avg_shard_lt / $avg_lt_cnt" | bc) + else + t3=0 + fi + shards_lt=("${shards_lt[@]}" "$t") + shards_clt=("${shards_clt[@]}" "$t2") + avg_shards_lt=("${avg_shards_lt[@]}" "$t3") + shard_cl_counter=0 + shard_lt=0 + shard_clt=0 + avg_shard_lt=0 + lt_cnt=0 + clt_cnt=0 + avg_lt_cnt=0 + fi + +done + +echo + +echo "idle times:" +for i in $(seq 0 $(($snodes - 1))); do + echo "I Node: $i" + times=($(tail -50 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} "idle_time_worker.{1,10}" | grep -o ${flags} '\d+\.\d+')) + for ((j = 0; j < ${#times[@]}; j++)); do + # echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" + echo -en "${Red}${times[$j]}${Nc} " + done + echo +done + +echo "Memory:" +for i in $(seq 0 $total_nodes); do + lines=10 + if [ $i -lt $snodes ]; then + lines=20 + fi + mem=$(tail -${lines} s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + [ ! -z "$mem" ] || mem=0 + echo "$i: $(echo "$mem / 1000" | bc)" +done +echo + +# ----------------------------------- Servers +total_server_tp=0 +avg_server_ctp=0 +counter=0 +echo "Shards Servers TP:" +for t in ${shards_server_tp[@]}; do + shard_avg_tp=$(echo "$t / $shard_size" | bc) + total_server_tp=$(($total_server_tp + $shard_avg_tp)) + echo "${counter}: $(echo -e ${Red}${shard_avg_tp}${Nc})" + counter=$(($counter + 1)) +done +counter=0 +echo "Shards Servers CTP:" +for t in ${shards_server_ctp[@]}; do + shard_avg_tp=$(echo "$t / $shard_size" | bc) + avg_server_ctp=$(($avg_server_ctp + $shard_avg_tp)) + echo "${counter}: $(echo -e ${Red}${shard_avg_tp}${Nc})" + if [ $shard_avg_tp -gt 0 ]; then + counter=$(($counter + 1)) + fi + +done +echo +if [ $counter -eq 0 ]; then + counter=1 +fi +echo "total avg servers TP: $(echo -e ${Red}${total_server_tp}${Nc})" +echo "total avg servers CTP: $(echo -e ${Red}$(echo "scale=0;$avg_server_ctp / $counter" | bc)${Nc})" +echo "Final avg servers TP: $(echo -e ${Red}$(echo "scale=0;$avg_server_ctp / $counter + $total_server_tp" | bc)${Nc})" +echo "--------------------------------------------" +# ----------------------------------- Clients +total_cl_tp=0 +total_cl_ctp=0 +counter=0 +echo -e "\nShards Clients TP:" +for t in ${shards_cl_tp[@]}; do + total_cl_tp=$(($total_cl_tp + $t)) + echo -e "${counter}: $(echo -e ${Red}${t}${Nc})" + counter=$(($counter + 1)) +done +counter=0 +echo "Shards Clients CTP:" +for t in ${shards_cl_ctp[@]}; do + total_cl_ctp=$(($total_cl_ctp + ${t})) + echo -e "${counter}: $(echo -e ${Red}${t}${Nc})" + counter=$(($counter + 1)) + +done + +echo "total avg clients TP: $(echo -e ${Red}${total_cl_tp}${Nc})" +echo "total avg clients CTP: $(echo -e ${Red}${total_cl_ctp}${Nc})" +echo "Final avg clients TP: $(echo -e ${Red}$(echo "scale=0;$total_cl_tp + $total_cl_ctp" | bc)${Nc})" +echo "--------------------------------------------" +# ----------------------------------- Latency +avg_latency=0 +avg_latency_cnt=0 +avg_clatency=0 +avg_clatency_cnt=0 +avg_avg_latency=0 +avg_avg_latency_cnt=0 +echo -e "\nShards Latency:" +for t in ${shards_lt[@]}; do + if [ ! "$t" = "0" ]; then + avg_latency=$(echo "$avg_latency + $t" | bc) + echo -e "${avg_latency_cnt}: $(echo -e ${Red}${t}${Nc})" + avg_latency_cnt=$(($avg_latency_cnt + 1)) + else + echo -e "${avg_latency_cnt}: $(echo -e ${Red}NAN${Nc})" + fi +done +echo "Shards Cross Latency:" +for t in ${shards_clt[@]}; do + if [ ! "$t" = "0" ]; then + avg_clatency=$(echo "$avg_clatency + ${t}" | bc) + echo -e "${avg_clatency_cnt}: $(echo -e ${Red}${t}${Nc})" + avg_clatency_cnt=$(($avg_clatency_cnt + 1)) + else + echo -e "${avg_clatency_cnt}: $(echo -e ${Red}NAN${Nc})" + fi +done + +echo "Shards AVG Latency:" +for t in ${avg_shards_lt[@]}; do + if [ ! "$t" = "0" ]; then + avg_avg_latency=$(echo "$avg_avg_latency + ${t}" | bc) + echo -e "${avg_avg_latency_cnt}: $(echo -e ${Red}${t}${Nc})" + avg_avg_latency_cnt=$(($avg_avg_latency_cnt + 1)) + else + echo -e "${avg_avg_latency_cnt}: $(echo -e ${Red}NAN${Nc})" + fi +done + +echo "total avg Intra Latency: $(echo -e ${Red}$(echo "scale=3;$avg_latency / $avg_latency_cnt" | bc)${Nc})" +if [ $avg_clatency_cnt -ne 0 ]; then + echo "total avg Cross Latency: $(echo -e ${Red}$(echo "scale=3;$avg_clatency / $avg_clatency_cnt" | bc)${Nc})" +fi +echo "Final avg Latency: $(echo -e ${Red}$(echo "scale=3;($avg_latency + $avg_clatency) / ($avg_clatency_cnt + $avg_latency_cnt)" | bc)${Nc})" diff --git a/scripts/scp_binaries.sh b/scripts/scp_binaries.sh index f05b080ff..04d4c9c2f 100755 --- a/scripts/scp_binaries.sh +++ b/scripts/scp_binaries.sh @@ -1,6 +1,6 @@ #!/bin/bash - -home_directory="/home/expo" +USERNAME=ubuntu +home_directory="/home/ubuntu" nodes=$1 ifconfig=$2 input="./ifconfig.txt" @@ -8,14 +8,14 @@ i=0 while IFS= read -r line do - if_cmd="scp ifconfig.txt expo@${line}:${home_directory}/resilientdb/" + if_cmd="scp ifconfig.txt ${USERNAME}@${line}:${home_directory}/resilientdb/" if [ "$i" -lt "$nodes" ];then - cmd="scp rundb expo@${line}:${home_directory}/resilientdb/" + cmd="scp rundb ${USERNAME}@${line}:${home_directory}/resilientdb/" else - cmd="scp runcl expo@${line}:${home_directory}/resilientdb/" + cmd="scp runcl ${USERNAME}@${line}:${home_directory}/resilientdb/" fi - monitor="scp monitorResults.sh expo@${line}:${home_directory}/resilientdb/" + monitor="scp monitorResults.sh ${USERNAME}@${line}:${home_directory}/resilientdb/" if [ "$ifconfig" -eq 1 ];then # echo "$if_cmd" diff --git a/scripts/scp_results.sh b/scripts/scp_results.sh index 8a018ca55..bb55ea84a 100755 --- a/scripts/scp_results.sh +++ b/scripts/scp_results.sh @@ -1,6 +1,6 @@ #!/bin/bash - -home_directory="/home/expo" +USERNAME=ubuntu +home_directory="/home/ubuntu" nodes=$1 name=$2 result_dir=$3 @@ -8,7 +8,7 @@ input="./ifconfig.txt" i=0 while IFS= read -r line do - cmd="scp expo@${line}:${home_directory}/resilientdb/${name}*.out ${result_dir}" + cmd="scp ${USERNAME}@${line}:${home_directory}/resilientdb/${name}*.out ${result_dir}" echo "$cmd" $($cmd) & i=$(($i+1)) @@ -18,7 +18,7 @@ wait i=0 while IFS= read -r line do - cmd="ssh expo@${line} rm ${home_directory}/resilientdb/*.out & rm ${home_directory}/resilientdb/monitor/*.out" + cmd="ssh ${USERNAME}@${line} rm -f ${home_directory}/resilientdb/*;" $($cmd) & i=$(($i+1)) done < "$input" diff --git a/scripts/set_ulimit.sh b/scripts/set_ulimit.sh index ccf7ac10b..ee15182b6 100755 --- a/scripts/set_ulimit.sh +++ b/scripts/set_ulimit.sh @@ -1,5 +1,5 @@ #!/bin/bash -USERNAME=expo +USERNAME=ubuntu HOSTS="$1" count=0 diff --git a/scripts/simRun.py b/scripts/simRun.py index 4f4dbff20..6bba8e0f2 100644 --- a/scripts/simRun.py +++ b/scripts/simRun.py @@ -16,7 +16,7 @@ import socket dashboard = None -home_directory = "/home/expo" +home_directory = "/home/ubuntu" PATH = os.getcwd() #result_dir = PATH + "/results/" result_dir = home_directory+"/resilientdb/results/" @@ -76,3 +76,4 @@ # collecting the output os.system("./scp_results.sh {} {} {}".format(nds, resfile, result_dir)) +os.system("cp config.h {}".format(result_dir + "config_" + resfile)) diff --git a/scripts/startResilientDB.sh b/scripts/startResilientDB.sh index 98ab61ea4..924351b5d 100755 --- a/scripts/startResilientDB.sh +++ b/scripts/startResilientDB.sh @@ -4,18 +4,51 @@ # This script allows to compile and run the code. You need to specify IP addresses of your servers and clients. The scripts expect three arguments and the result is stored in a folder named "results". Do create a folder with the name "results" before running this script. ###### -i=$1 # Argument 1 to script --> Number of replicas -cli=$2 # Argument 2 to script --> Number of clients -bsize=$3 # Argumnet 3 to script --> Batch Size +i=$1 # Argument 1 to script --> Number of replicas +cli=$2 # Argument 2 to script --> Number of clients +name=$3 +runs=$4 +bsize=$5 # Argumnet 3 to script --> Batch Size +if [ -z $bsize ]; then + bsize=100 +fi SNODES=( - # Specify private IP addresses of your replicas (as below). - #"10.128.0.246" + "10.0.0.96" + "10.0.0.46" + "10.0.0.193" + "10.0.0.100" + "10.0.0.169" + "10.0.0.223" + "10.0.0.87" + "10.0.0.90" + "10.0.0.20" + "10.0.0.192" + "10.0.0.19" + "10.0.0.226" + # "10.0.0.218" + # "10.0.0.140" + # "10.0.0.178" + # "10.0.0.209" ) CNODES=( - # Specify Private IPs of your clients. - #"10.128.1.6" + # "10.0.0.96" + # "10.0.0.46" + # "10.0.0.193" + # "10.0.0.100" + # "10.0.0.169" + # "10.0.0.223" + # "10.0.0.87" + # "10.0.0.90" + # "10.0.0.20" + # "10.0.0.192" + # "10.0.0.19" + # "10.0.0.226" + "10.0.0.218" + "10.0.0.140" + "10.0.0.178" + # "10.0.0.209" ) rm ifconfig.txt hostnames.py @@ -65,8 +98,7 @@ done echo "]" >>hostnames.py # Compiling the Code -make clean -make -j8 +# make clean; make -j8 tm=0 @@ -78,8 +110,8 @@ cp hostnames.py scripts/ cd scripts # Number of times you want to run the code (default 1) -while [ $tm -lt 1 ]; do - python3 simRun.py $i s${i}_c${cli}_results_PBFT_b${bsize}_run${tm}_node $tm +while [ $tm -lt $runs ]; do + python3 simRun.py $i s${i}_c${cli}_results_${name}_b${bsize}_run${tm}_node $tm tm=$((tm + 1)) done diff --git a/scripts/vcloud_cmd.sh b/scripts/vcloud_cmd.sh index f22938504..b66ceb527 100755 --- a/scripts/vcloud_cmd.sh +++ b/scripts/vcloud_cmd.sh @@ -1,5 +1,5 @@ #!/bin/bash -USERNAME=expo +USERNAME=ubuntu HOSTS="$1" SCRIPT="$2" count=0 diff --git a/scripts/vcloud_deploy.sh b/scripts/vcloud_deploy.sh index cd200e69e..472ed1712 100755 --- a/scripts/vcloud_deploy.sh +++ b/scripts/vcloud_deploy.sh @@ -1,7 +1,7 @@ #!/bin/bash # RES_FILE --> Name of the result file. # -USERNAME=expo +USERNAME=ubuntu HOSTS="$1" NODE_CNT="$2" RES_FILE="$3" @@ -10,10 +10,10 @@ for HOSTNAME in ${HOSTS}; do i=1 if [ $count -ge $NODE_CNT ]; then i=0 - SCRIPT="./runcl -nid${count} > ${RES_FILE}${count}.out 2>&1" + SCRIPT="ulimit -n 4096;./runcl -nid${count} > ${RES_FILE}${count}.out 2>&1" echo "${HOSTNAME}: runcl ${count}" else - SCRIPT="./rundb -nid${count} > ${RES_FILE}${count}.out 2>&1" + SCRIPT="ulimit -n 4096;./rundb -nid${count} > ${RES_FILE}${count}.out 2>&1" echo "${HOSTNAME}: rundb ${count}" fi # if [ "$i" -eq 0 ];then diff --git a/scripts/vcloud_monitor.sh b/scripts/vcloud_monitor.sh index 9e2c37872..cc6aa1982 100755 --- a/scripts/vcloud_monitor.sh +++ b/scripts/vcloud_monitor.sh @@ -1,7 +1,7 @@ #!/bin/bash # RES_FILE --> Name of the result file. # -USERNAME=expo +USERNAME=ubuntu HOSTS="$1" IPAddr="$2" diff --git a/smart_contracts/banking_sc.cpp b/smart_contracts/banking_sc.cpp index a8d9fa4fb..901809b8f 100644 --- a/smart_contracts/banking_sc.cpp +++ b/smart_contracts/banking_sc.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "smart_contract.h" #include "smart_contract_txn.h" diff --git a/smart_contracts/smart_contract.h b/smart_contracts/smart_contract.h index 91b1ac3bb..de48568c4 100644 --- a/smart_contracts/smart_contract.h +++ b/smart_contracts/smart_contract.h @@ -1,7 +1,6 @@ #ifndef _SC_H_ #define _SC_H_ #include "global.h" -#include "helper.h" #include "wl.h" #if BANKING_SMART_CONTRACT diff --git a/smart_contracts/smart_contract_txn.h b/smart_contracts/smart_contract_txn.h index 31ce0f26a..c7f55155b 100644 --- a/smart_contracts/smart_contract_txn.h +++ b/smart_contracts/smart_contract_txn.h @@ -1,7 +1,6 @@ #ifndef _SC_TXN_H_ #define _SC_TXN_H_ #include "global.h" -#include "helper.h" #include "txn.h" #include "wl.h" diff --git a/statistics/stats.cpp b/statistics/stats.cpp index cdf02880e..a3d97319d 100644 --- a/statistics/stats.cpp +++ b/statistics/stats.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "stats.h" #include "mem_alloc.h" #include "client_txn.h" @@ -51,6 +50,9 @@ void Stats_thd::clear() // Execution txn_cnt = 0; + cross_shard_txn_cnt = 0; + previous_interval_txn_cnt = 0; + previous_interval_cross_shard_txn_cnt = 0; remote_txn_cnt = 0; local_txn_cnt = 0; local_txn_start_cnt = 0; @@ -62,6 +64,7 @@ void Stats_thd::clear() local_txn_abort_cnt = 0; remote_txn_abort_cnt = 0; txn_run_time = 0; + cross_txn_run_time = 0; multi_part_txn_cnt = 0; multi_part_txn_run_time = 0; single_part_txn_cnt = 0; @@ -217,26 +220,26 @@ void Stats_thd::clear() void Stats_thd::print_client(FILE *outf, bool prog) { - string node_id = to_string(g_node_cnt - g_node_id + 1); + string node_id = to_string(g_node_cnt - g_node_id + 1); string filename = "toInflux_C"; filename.append(node_id); filename.append("_.out"); - filename="./monitor/"+filename; - + filename = "./monitor/" + filename; + ofstream file; - file.open(filename.c_str(), ios_base::out | ios_base::in); // will not create file - fprintf(outf, "filename: %s", filename.c_str()); + file.open(filename.c_str(), ios_base::out | ios_base::in); // will not create file + fprintf(outf, "filename: %s\n", filename.c_str()); if (!file.is_open()) - { + { file.clear(); - file.open(filename.c_str(), std::ofstream::app); // will create if necessary + file.open(filename.c_str(), std::ofstream::app); // will create if necessary //file << ip_add; file << "tput\n"; - } + } else { file.close(); - file.open (filename.c_str(),std::ofstream::app); + file.open(filename.c_str(), std::ofstream::app); } double txn_run_avg_time = 0; @@ -251,14 +254,15 @@ void Stats_thd::print_client(FILE *outf, bool prog) file.close(); fprintf(outf, - "total_runtime=%f" - ",tput=%f" - ",txn_cnt=%ld" - ",txn_sent_cnt=%ld" - ",txn_run_time=%f" - ",txn_run_avg_time=%f" - ",cl_send_intv=%f", - total_runtime / BILLION, tput, txn_cnt, txn_sent_cnt, txn_run_time / BILLION, txn_run_avg_time / BILLION, cl_send_intv / BILLION); + "\ntotal_runtime=%f" + "\ntput=%f" + "\ntxn_cnt=%ld" + "\ntxn_sent_cnt=%ld" + "\ntxn_run_time=%f" + "\ncross_txn_run_time=%f" + "\ntxn_run_avg_time=%f" + "\ncl_send_intv=%f", + total_runtime / BILLION, tput, txn_cnt, txn_sent_cnt, txn_run_time / BILLION, cross_txn_run_time / BILLION, txn_run_avg_time / BILLION, cl_send_intv / BILLION); // IO double mbuf_send_intv_time_avg = 0; double msg_unpack_time_avg = 0; @@ -285,29 +289,29 @@ void Stats_thd::print_client(FILE *outf, bool prog) msg_send_time_avg = msg_send_time / msg_send_cnt; } fprintf(outf, - ",msg_queue_delay_time=%f" - ",msg_queue_cnt=%ld" - ",msg_queue_enq_cnt=%ld" - ",msg_queue_delay_time_avg=%f" - ",msg_send_time=%f" - ",msg_send_time_avg=%f" - ",msg_recv_time=%f" - ",msg_recv_time_avg=%f" - ",msg_recv_idle_time=%f" - ",msg_batch_cnt=%ld" - ",msg_batch_size_msgs=%ld" - ",msg_batch_size_msgs_avg=%f" - ",msg_batch_size_bytes=%ld" - ",msg_batch_size_bytes_avg=%f" - ",msg_batch_size_bytes_to_server=%ld" - ",msg_batch_size_bytes_to_client=%ld" - ",msg_send_cnt=%ld" - ",msg_recv_cnt=%ld" - ",msg_unpack_time=%f" - ",msg_unpack_time_avg=%f" - ",mbuf_send_intv_time=%f" - ",mbuf_send_intv_time_avg=%f" - ",msg_copy_output_time=%f", + "\nmsg_queue_delay_time=%f" + "\nmsg_queue_cnt=%ld" + "\nmsg_queue_enq_cnt=%ld" + "\nmsg_queue_delay_time_avg=%f" + "\nmsg_send_time=%f" + "\nmsg_send_time_avg=%f" + "\nmsg_recv_time=%f" + "\nmsg_recv_time_avg=%f" + "\nmsg_recv_idle_time=%f" + "\nmsg_batch_cnt=%ld" + "\nmsg_batch_size_msgs=%ld" + "\nmsg_batch_size_msgs_avg=%f" + "\nmsg_batch_size_bytes=%ld" + "\nmsg_batch_size_bytes_avg=%f" + "\nmsg_batch_size_bytes_to_server=%ld" + "\nmsg_batch_size_bytes_to_client=%ld" + "\nmsg_send_cnt=%ld" + "\nmsg_recv_cnt=%ld" + "\nmsg_unpack_time=%f" + "\nmsg_unpack_time_avg=%f" + "\nmbuf_send_intv_time=%f" + "\nmbuf_send_intv_time_avg=%f" + "\nmsg_copy_output_time=%f", msg_queue_delay_time / BILLION, msg_queue_cnt, msg_queue_enq_cnt, msg_queue_delay_time_avg / BILLION, msg_send_time / BILLION, msg_send_time_avg / BILLION, msg_recv_time / BILLION, msg_recv_time_avg / BILLION, msg_recv_idle_time / BILLION, msg_batch_cnt, msg_batch_size_msgs, msg_batch_size_msgs_avg, msg_batch_size_bytes, msg_batch_size_bytes_avg, msg_batch_size_bytes_to_server, msg_batch_size_bytes_to_client, msg_send_cnt, msg_recv_cnt, msg_unpack_time / BILLION, msg_unpack_time_avg / BILLION, mbuf_send_intv_time / BILLION, mbuf_send_intv_time_avg / BILLION, msg_copy_output_time / BILLION); #if TIME_PROF_ENABLE @@ -319,22 +323,22 @@ void Stats_thd::print_client(FILE *outf, bool prog) if (!prog) { - client_client_latency.quicksort(0, client_client_latency.cnt - 1); - fprintf(outf, - ",ccl0=%f" - ",ccl1=%f" - ",ccl10=%f" - ",ccl25=%f" - ",ccl50=%f" - ",ccl75=%f" - ",ccl90=%f" - ",ccl95=%f" - ",ccl96=%f" - ",ccl97=%f" - ",ccl98=%f" - ",ccl99=%f" - ",ccl100=%f", - (double)client_client_latency.get_idx(0) / BILLION, (double)client_client_latency.get_percentile(1) / BILLION, (double)client_client_latency.get_percentile(10) / BILLION, (double)client_client_latency.get_percentile(25) / BILLION, (double)client_client_latency.get_percentile(50) / BILLION, (double)client_client_latency.get_percentile(75) / BILLION, (double)client_client_latency.get_percentile(90) / BILLION, (double)client_client_latency.get_percentile(95) / BILLION, (double)client_client_latency.get_percentile(96) / BILLION, (double)client_client_latency.get_percentile(97) / BILLION, (double)client_client_latency.get_percentile(98) / BILLION, (double)client_client_latency.get_percentile(99) / BILLION, (double)client_client_latency.get_idx(client_client_latency.cnt - 1) / BILLION); + // client_client_latency.quicksort(0, client_client_latency.cnt - 1); + // fprintf(outf, + // ",ccl0=%f" + // ",ccl1=%f" + // ",ccl10=%f" + // ",ccl25=%f" + // ",ccl50=%f" + // ",ccl75=%f" + // ",ccl90=%f" + // ",ccl95=%f" + // ",ccl96=%f" + // ",ccl97=%f" + // ",ccl98=%f" + // ",ccl99=%f" + // ",ccl100=%f", + // (double)client_client_latency.get_idx(0) / BILLION, (double)client_client_latency.get_percentile(1) / BILLION, (double)client_client_latency.get_percentile(10) / BILLION, (double)client_client_latency.get_percentile(25) / BILLION, (double)client_client_latency.get_percentile(50) / BILLION, (double)client_client_latency.get_percentile(75) / BILLION, (double)client_client_latency.get_percentile(90) / BILLION, (double)client_client_latency.get_percentile(95) / BILLION, (double)client_client_latency.get_percentile(96) / BILLION, (double)client_client_latency.get_percentile(97) / BILLION, (double)client_client_latency.get_percentile(98) / BILLION, (double)client_client_latency.get_percentile(99) / BILLION, (double)client_client_latency.get_idx(client_client_latency.cnt - 1) / BILLION); } //client_client_latency.print(outf); @@ -351,6 +355,9 @@ void Stats_thd::combine(Stats_thd *stats) client_client_latency.append(stats->client_client_latency); // Execution txn_cnt += stats->txn_cnt; + cross_shard_txn_cnt += stats->cross_shard_txn_cnt; + previous_interval_txn_cnt += stats->previous_interval_txn_cnt; + previous_interval_cross_shard_txn_cnt += stats->previous_interval_cross_shard_txn_cnt; remote_txn_cnt += stats->remote_txn_cnt; local_txn_cnt += stats->local_txn_cnt; local_txn_start_cnt += stats->local_txn_start_cnt; @@ -362,6 +369,7 @@ void Stats_thd::combine(Stats_thd *stats) local_txn_abort_cnt += stats->local_txn_abort_cnt; remote_txn_abort_cnt += stats->remote_txn_abort_cnt; txn_run_time += stats->txn_run_time; + cross_txn_run_time += stats->cross_txn_run_time; multi_part_txn_cnt += stats->multi_part_txn_cnt; multi_part_txn_run_time += stats->multi_part_txn_run_time; single_part_txn_cnt += stats->single_part_txn_cnt; @@ -582,20 +590,30 @@ void Stats::print_client(bool prog) totals->print_client(outf, prog); mem_util(outf); cpu_util(outf); - - if (prog) - { - fprintf(outf, "\n"); - //for (uint32_t k = 0; k < g_node_id; ++k) { - for (uint32_t k = 0; k < g_servers_per_client; ++k) - { - printf("tif_node%u=%d, ", k, client_man.get_inflight(k)); - } - printf("\n"); - } - else + double tput = 0; + if (totals->total_runtime > 0) { + tput = totals->txn_cnt / (totals->total_runtime / BILLION); } + fprintf(outf, "\ntotal_runtime=%f\n", totals->total_runtime / BILLION); + fprintf(outf, "tput=%f\ttxn_cnt=%ld\n", tput, totals->txn_cnt); + fprintf(outf, "Latency=%f\n", (totals->txn_run_time / BILLION) / totals->txn_cnt); + fprintf(outf, "CLatency=%f\n", (totals->cross_txn_run_time / BILLION) / totals->cross_shard_txn_cnt); + fflush(outf); + + // if (prog) + // { + // fprintf(outf, "\n"); + // //for (uint32_t k = 0; k < g_node_id; ++k) { + // for (uint32_t k = 0; k < g_servers_per_client; ++k) + // { + // printf("tif_node%u=%d, ", k, client_man.get_inflight(k)); + // } + // printf("\n"); + // } + // else + // { + // } fflush(stdout); } @@ -623,14 +641,48 @@ void Stats::print(bool prog) //else outf = stdout; if (prog) - fprintf(outf, "[prog] "); + fprintf(outf, "[prog] \n"); else fprintf(outf, "[summary] "); - totals->print(outf, prog); - mem_util(outf); - cpu_util(outf); + if (!prog || STATS_DETAILED) + { + totals->print(outf, prog); + mem_util(outf); + cpu_util(outf); + print_msg_sizes(outf); + } - fprintf(outf, "\n"); + double tput = 0, interval_tput = 0; + if (STAT_BAND_WIDTH_ENABLE) + { + fprintf(outf, "\nnodes: "); + for (uint64_t i = 0; i < g_total_node_cnt; i++) + { + fprintf(outf, "%ld \t", i); + } + fprintf(outf, "\nmb_received: "); + for (uint64_t i = 0; i < g_total_node_cnt; i++) + { + fprintf(outf, "%ld \t", (bytes_received[i] / 1024 / 1024)); + } + fprintf(outf, "\nmb_sent: "); + for (uint64_t i = 0; i < g_total_node_cnt; i++) + { + fprintf(outf, "%ld \t", (bytes_sent[i] / 1024 / 1024)); + } + } + if (totals->total_runtime > 0) + { + tput = totals->txn_cnt / (totals->total_runtime / BILLION); + interval_tput = (totals->txn_cnt - totals->previous_interval_txn_cnt) / (PROG_TIMER / BILLION); + } + fprintf(outf, "total_runtime=%f\n", totals->total_runtime / BILLION); + fprintf(outf, "--------------------------------\n"); + fprintf(outf, "interval_tput=%f\ttxn_cnt=%ld\n", interval_tput, totals->txn_cnt - totals->previous_interval_txn_cnt); + // fprintf(outf, "previous txn_count %ld\n", totals->previous_interval_txn_cnt); + fprintf(outf, "--------------------------------\n"); + fprintf(outf, "tput =%f\ttxn_cnt=%ld\n", tput, totals->txn_cnt); + fprintf(outf, "=======================================================\n"); fflush(outf); if (!prog) { @@ -765,25 +817,25 @@ void Stats::cpu_util(FILE *outf) void Stats_thd::print(FILE *outf, bool prog) { - string node_id = to_string(g_node_id +1); + string node_id = to_string(g_node_id + 1); string filename = "toInflux_R"; filename.append(node_id); filename.append("_.out"); - filename="./monitor/"+filename; + filename = "./monitor/" + filename; ofstream file; - file.open(filename.c_str(), ios_base::out | ios_base::in); // will not create file + file.open(filename.c_str(), ios_base::out | ios_base::in); // will not create file if (!file.is_open()) - { + { file.clear(); - file.open(filename.c_str(), std::ofstream::app); // will create if necessary + file.open(filename.c_str(), std::ofstream::app); // will create if necessary //file << ip_add; file << "tput\n"; - } + } else { file.close(); - file.open (filename.c_str(),std::ofstream::app); + file.open(filename.c_str(), std::ofstream::app); } fprintf(outf, "filename: %s", filename.c_str()); @@ -805,8 +857,8 @@ void Stats_thd::print(FILE *outf, bool prog) file << tput; file << "\n"; - file.close(); - + file.close(); + fprintf(outf, "\ntput=%f\ntxn_cnt=%ld\n", tput, txn_cnt); double work_queue_wait_avg_time = 0; @@ -881,51 +933,6 @@ void Stats_thd::print(FILE *outf, bool prog) , msg_send_time / BILLION, msg_send_time_avg / BILLION, msg_recv_time / BILLION, msg_recv_time_avg / BILLION, msg_recv_idle_time / BILLION, msg_send_cnt, msg_recv_cnt); - fprintf(outf, - "bytes_received=%ld\n" - "bytes_sent=%ld\n", - bytes_received, bytes_sent); - - fprintf(outf, - "msg_size_client_batch=%ld\n" - "msg_size_batch_req=%ld\n" -#if CONSENSUS == PBFT - "msg_size_commit=%ld\n" - "msg_size_prepare=%ld\n" -#endif - "msg_size_checkpoint=%ld\n" - "msg_size_client_response=%ld\n" -#if GBFT - "msg_size_ccm=%ld\n" -#endif -#if CONSENSUS == HOTSTUFF - "msg_size_vote=%ld\n" -#endif -#if STEWARD - "acc_cert_msg_size=%ld\n" -#endif - , - client_batch_msg_size, batch_req_msg_size -#if CONSENSUS == PBFT - , - commit_msg_size, prepare_msg_size -#endif - , - checkpoint_msg_size, client_response_msg_size -#if GBFT - , - ccm_msg_size -#endif -#if CONSENSUS == HOTSTUFF - , - vote_msg_size -#endif -#if STEWARD - , - acc_cert_msg_size -#endif - ); - #if TIME_PROF_ENABLE for (uint64_t i = 0; i < (g_rem_thread_cnt + g_send_thread_cnt); ++i) { @@ -956,25 +963,17 @@ void Stats_thd::print(FILE *outf, bool prog) txn_table_new_cnt, txn_table_get_cnt, txn_table_release_cnt, txn_table_cflt_cnt, txn_table_cflt_size, txn_table_get_time / BILLION, txn_table_release_time / BILLION, txn_table_min_ts_time / BILLION, txn_table_get_avg_time / BILLION, txn_table_release_avg_time / BILLION); } -void Stats_thd::set_message_size(uint64_t rtype, uint64_t size) +void Stats::set_message_size(uint64_t rtype, uint64_t size) { switch (rtype) { case CL_RSP: this->client_response_msg_size = size; break; -#if CLIENT_BATCH case CL_BATCH: -#else - case CL_QRY: -#endif this->client_batch_msg_size = size; break; -#if BATCH_ENABLE == BSET case BATCH_REQ: -#else - case PBFT_PRE: -#endif this->batch_req_msg_size = size; break; case PBFT_CHKPT_MSG: @@ -988,29 +987,31 @@ void Stats_thd::set_message_size(uint64_t rtype, uint64_t size) this->prepare_msg_size = size; break; #endif -#if GBFT - case COMMIT_CERTIFICATE_MSG: - this->ccm_msg_size = size; - break; -#endif -#if STEWARD - case STW_ACC_CERT: - this->acc_cert_msg_size = size; - break; -#endif -#if CONSENSUS == HOTSTUFF - case PREP_VOTE: // 27 - case PRE_COMMIT: - case COMMIT_VOTE: - case COMMIT: - case DECIDE_VOTE: - case DECIDE: - case SEND_NEXT_VIEW: - case NEXT_VIEW: - this->vote_msg_size = size; - break; -#endif + default: break; } } +void Stats::print_msg_sizes(FILE *outf) +{ + fprintf(outf, + "\nmsg_size_client_batch=%ld\n" + "msg_size_batch_req=%ld\n" +#if CONSENSUS == PBFT + "msg_size_commit=%ld\n" + "msg_size_prepare=%ld\n" +#endif + "msg_size_checkpoint=%ld\n" + "msg_size_client_response=%ld\n" + + , + client_batch_msg_size, batch_req_msg_size +#if CONSENSUS == PBFT + , + commit_msg_size, prepare_msg_size +#endif + , + checkpoint_msg_size, client_response_msg_size + + ); +} diff --git a/statistics/stats.h b/statistics/stats.h index 217952f03..20819a22d 100644 --- a/statistics/stats.h +++ b/statistics/stats.h @@ -1,6 +1,7 @@ #ifndef _STATS_H_ #define _STATS_H_ #include "stats_array.h" +#include "../config.h" class StatValue { @@ -41,6 +42,9 @@ class Stats_thd // Execution uint64_t txn_cnt; + uint64_t cross_shard_txn_cnt; + uint64_t previous_interval_txn_cnt; + uint64_t previous_interval_cross_shard_txn_cnt; uint64_t remote_txn_cnt; uint64_t local_txn_cnt; uint64_t local_txn_start_cnt; @@ -52,6 +56,7 @@ class Stats_thd uint64_t local_txn_abort_cnt; uint64_t remote_txn_abort_cnt; double txn_run_time; + double cross_txn_run_time; uint64_t multi_part_txn_cnt; double multi_part_txn_run_time; uint64_t single_part_txn_cnt; @@ -122,26 +127,6 @@ class Stats_thd double mbuf_send_intv_time; double msg_copy_output_time; - uint64_t bytes_received; - uint64_t bytes_sent; - uint64_t client_batch_msg_size; - uint64_t batch_req_msg_size; -#if CONSENSUS == PBFT - uint64_t commit_msg_size; - uint64_t prepare_msg_size; -#endif - uint64_t checkpoint_msg_size; - uint64_t client_response_msg_size; -#if GBFT - uint64_t ccm_msg_size; -#endif -#if CONSENSUS == HOTSTUFF - uint64_t vote_msg_size; -#endif -#if STEWARD - uint64_t acc_cert_msg_size; -#endif - #if TIME_PROF_ENABLE double *io_thread_idle_time; #endif @@ -256,8 +241,22 @@ class Stats void mem_util(FILE *outf); void cpu_util(FILE *outf); void print_prof(FILE *outf); + void print_msg_sizes(FILE *outf); + void set_message_size(uint64_t rtype, uint64_t size); clock_t lastCPU, lastSysCPU, lastUserCPU; + uint64_t bytes_sent[NODE_CNT + CLIENT_NODE_CNT]; + uint64_t bytes_received[NODE_CNT + CLIENT_NODE_CNT]; + + uint64_t client_batch_msg_size; + uint64_t batch_req_msg_size; +#if CONSENSUS == PBFT + uint64_t commit_msg_size; + uint64_t prepare_msg_size; +#endif + uint64_t checkpoint_msg_size; + uint64_t client_response_msg_size; + private: uint64_t thd_cnt; diff --git a/statistics/stats_array.cpp b/statistics/stats_array.cpp index 8735592d1..fab49ebc3 100644 --- a/statistics/stats_array.cpp +++ b/statistics/stats_array.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "stats.h" #include "mem_alloc.h" #include "client_txn.h" diff --git a/system/array.h b/system/array.h index 12abedf84..2a65d2b8d 100644 --- a/system/array.h +++ b/system/array.h @@ -2,7 +2,6 @@ #define _ARR_H_ #include "global.h" -#include "helper.h" #include "mem_alloc.h" template diff --git a/system/client_thread.cpp b/system/client_thread.cpp index 56195bdda..2a76f0bbb 100644 --- a/system/client_thread.cpp +++ b/system/client_thread.cpp @@ -11,6 +11,7 @@ #include "wl.h" #include "message.h" #include "timer.h" +#include "ring_all_comb.h" void ClientThread::send_key() { @@ -37,11 +38,10 @@ void ClientThread::send_key() keyex->pkeySz = keyex->pkey.size(); keyex->return_node = g_node_id; - vector emptyvec; vector dest; dest.push_back(i); - msg_queue.enqueue(get_thd_id(), keyex, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), keyex, dest); #endif #if CRYPTO_METHOD_CMAC_AES @@ -53,7 +53,7 @@ void ClientThread::send_key() keyexCMAC->pkeySz = keyexCMAC->pkey.size(); keyexCMAC->return_node = g_node_id; //msg_queue.enqueue(get_thd_id(), keyexCMAC, i); - msg_queue.enqueue(get_thd_id(), keyexCMAC, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), keyexCMAC, dest); dest.clear(); #endif } @@ -67,6 +67,7 @@ void ClientThread::setup() commonVar++; batchMTX.unlock(); + if (_thd_id == 0) { while (commonVar < g_client_thread_cnt + g_client_rem_thread_cnt + g_client_send_thread_cnt) @@ -79,7 +80,6 @@ void ClientThread::setup() RC ClientThread::run() { - tsetup(); printf("Running ClientThread %ld\n", _thd_id); @@ -112,14 +112,16 @@ RC ClientThread::run() ClientQueryBatch *bmsg = (ClientQueryBatch *)mssg; bmsg->init(); #endif - uint32_t next_node_id = get_view(); + + uint32_t next_node_id = get_client_view(); while (!simulation->is_done()) { heartbeat(); progress_stats(); int32_t inf_cnt; - uint32_t next_node = get_view(); - next_node_id = get_view(); + uint32_t next_node = get_client_view(); + + next_node_id = get_client_view(); #if VIEW_CHANGES //if a request by this client hasnt been completed in time @@ -127,7 +129,10 @@ RC ClientThread::run() if (client_timer->checkTimer(cbatch)) { cout << "TIMEOUT!!!!!!\n"; - resend_msg(cbatch); + + //TODO for experimental purpose: force one view change + if (get_client_view() == 0) + resend_msg(cbatch); } #endif @@ -219,17 +224,15 @@ RC ClientThread::run() char *buf = create_msg_buffer(bmsg); Message *deepCMsg = deep_copy_msg(buf, bmsg); ClientQueryBatch *deepCqry = (ClientQueryBatch *)deepCMsg; - + uint64_t c_txn_id = get_sys_clock(); + deepCqry->txn_id = c_txn_id; client_timer->startTimer(deepCqry->cqrySet[get_batch_size() - 1]->client_startts, deepCqry); delete_msg_buffer(buf); #endif // TIMER_ON - vector emptyvec; - emptyvec.push_back(bmsg->signature); - vector dest; dest.push_back(next_node_id); - msg_queue.enqueue(get_thd_id(), bmsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), bmsg, dest); dest.clear(); num_txns_sent += g_batch_size; @@ -250,28 +253,3 @@ RC ClientThread::run() return FINISH; } -#if VIEW_CHANGES -// Resend message to all the servers. -void ClientThread::resend_msg(ClientQueryBatch *symsg) -{ - //cout << "Resend: " << symsg->cqrySet[get_batch_size()-1]->client_startts << "\n"; - //fflush(stdout); - - vector emptyvec; - emptyvec.push_back(symsg->signature); // Sign the message. - - char *buf = create_msg_buffer(symsg); - for (uint64_t j = 0; j < g_node_cnt; j++) - { - vector dest; - dest.push_back(j); - - Message *deepCMsg = deep_copy_msg(buf, symsg); - msg_queue.enqueue(get_thd_id(), deepCMsg, emptyvec, dest); - dest.clear(); - } - emptyvec.clear(); - delete_msg_buffer(buf); - Message::release_message(symsg); -} -#endif // VIEW_CHANGES diff --git a/system/global.cpp b/system/global.cpp index 9843db26d..c1bb5a3be 100644 --- a/system/global.cpp +++ b/system/global.cpp @@ -11,6 +11,7 @@ #include "pool.h" #include "txn_table.h" #include "client_txn.h" +#include "txn.h" #include "../config.h" mem_alloc mem_allocator; @@ -63,12 +64,11 @@ UInt32 g_part_cnt = PART_CNT; UInt32 g_virtual_part_cnt = VIRTUAL_PART_CNT; UInt32 g_core_cnt = CORE_CNT; UInt32 g_thread_cnt = THREAD_CNT; +UInt32 g_worker_thread_cnt = WORKER_THREAD_CNT; +UInt32 g_batching_thread_cnt = BATCH_THREAD_CNT; +UInt32 g_checkpointing_thread_cnt = CHECKPOINT_THREAD_CNT; +UInt32 g_execution_thread_cnt = EXECUTE_THREAD_CNT; -#if EXECUTION_THREAD -UInt32 g_execute_thd = EXECUTE_THD_CNT; -#else -UInt32 g_execute_thd = 0; -#endif #if SIGN_THREADS UInt32 g_sign_thd = SIGN_THD_CNT; @@ -122,7 +122,6 @@ UInt64 g_msg_time_limit = MSG_TIME_LIMIT; double g_mpr = MPR; double g_mpitem = MPIR; - UInt32 g_repl_type = REPL_TYPE; UInt32 g_repl_cnt = REPLICA_CNT; @@ -193,7 +192,8 @@ uint64_t txn_per_chkpt() void set_curr_chkpt(uint64_t txn_id) { - g_last_stable_chkpt = txn_id; + if(txn_id > g_last_stable_chkpt) + g_last_stable_chkpt = txn_id; } uint64_t get_curr_chkpt() @@ -212,8 +212,6 @@ void inc_last_deleted_txn() } // Information about batching threads. -uint g_batch_threads = BATCH_THREADS; -uint g_btorder_thd = 1; uint64_t expectedExecuteCount = g_batch_size - 2; uint64_t expectedCheckpoint = TXN_PER_CHKPT - 5; uint64_t get_expectedExecuteCount() @@ -228,19 +226,24 @@ void set_expectedExecuteCount(uint64_t val) // Variable used by all threads during setup, to mark they are ready std::mutex batchMTX; +std::mutex next_idx_lock; uint commonVar = 0; // Variable used by Input thread at the primary to linearize batches. uint64_t next_idx = 0; uint64_t get_and_inc_next_idx() { + next_idx_lock.lock(); uint64_t val = next_idx++; + next_idx_lock.unlock(); return val; } void set_next_idx(uint64_t val) { + next_idx_lock.lock(); next_idx = val; + next_idx_lock.unlock(); } // Counters for input threads to access next socket (only used by replicas). @@ -273,37 +276,36 @@ vector nodes_to_send(uint64_t beg, uint64_t end) // STORAGE OF CLIENT DATA uint64_t ClientDataStore[SYNTH_TABLE_SIZE] = {0}; -// Entities pertaining to the current view. -uint32_t local_view[THREAD_CNT + REM_THREAD_CNT] = {0}; +// returns the current view. uint64_t get_current_view(uint64_t thd_id) { - return local_view[thd_id]; -} - -void set_current_view(uint64_t thd_id, uint64_t view) -{ - local_view[thd_id] = view; + return get_view(thd_id); } #if VIEW_CHANGES == true // For updating view of different threads. -std::mutex newViewMTX[BATCH_THREADS + REM_THREAD_CNT + 2]; -bool newView[BATCH_THREADS + REM_THREAD_CNT + 2] = {false}; -bool get_newView(uint64_t thd_id) +std::mutex newViewMTX[THREAD_CNT + REM_THREAD_CNT + SEND_THREAD_CNT]; +uint64_t newView[THREAD_CNT + REM_THREAD_CNT + SEND_THREAD_CNT] = {0}; +uint64_t get_view(uint64_t thd_id) { - bool nchange = false; + uint64_t nchange = 0; newViewMTX[thd_id].lock(); nchange = newView[thd_id]; newViewMTX[thd_id].unlock(); return nchange; } -void set_newView(uint64_t thd_id, bool val) +void set_view(uint64_t thd_id, uint64_t val) { newViewMTX[thd_id].lock(); newView[thd_id] = val; newViewMTX[thd_id].unlock(); } +#else +uint64_t get_view(uint64_t thd_id) +{ + return 0; +} #endif // Size of the batch @@ -314,17 +316,23 @@ uint64_t get_batch_size() return g_batch_size; } + +uint64_t view_to_primary(uint64_t view, uint64_t node) +{ + return view; +} + // This variable is mainly used by the client to know its current primary. uint32_t g_view = 0; std::mutex viewMTX; -void set_view(uint64_t nview) +void set_client_view(uint64_t nview) { viewMTX.lock(); g_view = nview; viewMTX.unlock(); } -uint64_t get_view() +uint64_t get_client_view() { uint64_t val; viewMTX.lock(); @@ -352,6 +360,7 @@ double idle_worker_times[THREAD_CNT] = {0}; // Statistics to print output_thread_idle_times. double output_thd_idle_time[SEND_THREAD_CNT] = {0}; +double input_thd_idle_time[REM_THREAD_CNT] = {0}; // Maps for client response couting SpinLockMap client_responses_count; @@ -369,9 +378,7 @@ uint64_t payload_size = 51200; #endif #if EXT_DB == SQL || EXT_DB == SQL_PERSISTENT - DataBase* db = new SQLite(); +DataBase *db = new SQLite(); #elif EXT_DB == MEMORY - DataBase* db = new InMemoryDB(); +DataBase *db = new InMemoryDB(); #endif - - diff --git a/system/global.h b/system/global.h index 01ab7dee7..cc8f680fb 100644 --- a/system/global.h +++ b/system/global.h @@ -23,8 +23,9 @@ #include #include +#include "./helper.h" #include "pthread.h" -#include "config.h" +#include "../config.h" #include "stats.h" #include "pool.h" #include "txn_table.h" @@ -36,6 +37,7 @@ #include "sha.h" #include "database.h" #include "hash_map.h" +#include "hash_set.h" using namespace std; @@ -54,6 +56,7 @@ class QWorkQueue; class MessageQueue; class Client_query_queue; class Client_txn; +class CommitCertificateMessage; class ClientResponseMessage; typedef uint32_t UInt32; @@ -119,7 +122,11 @@ extern UInt32 g_this_rem_thread_cnt; extern UInt32 g_this_send_thread_cnt; extern UInt32 g_this_total_thread_cnt; extern UInt32 g_thread_cnt; -extern UInt32 g_execute_thd; +extern UInt32 g_worker_thread_cnt; +extern UInt32 g_batching_thread_cnt; +extern UInt32 g_checkpointing_thread_cnt; +extern UInt32 g_execution_thread_cnt; + extern UInt32 g_sign_thd; extern UInt32 g_send_thread_cnt; extern UInt32 g_rem_thread_cnt; @@ -326,8 +333,6 @@ extern uint64_t lastDeletedTxnMan; // index of last deleted txn manager. void inc_last_deleted_txn(); uint64_t get_last_deleted_txn(); -extern uint g_batch_threads; -extern uint g_btorder_thd; extern uint64_t expectedExecuteCount; extern uint64_t expectedCheckpoint; uint64_t get_expectedExecuteCount(); @@ -355,29 +360,28 @@ extern uint64_t ClientDataStore[SYNTH_TABLE_SIZE]; // Entities related to RBFT protocol. // Entities pertaining to the current view. -extern uint32_t local_view[THREAD_CNT + REM_THREAD_CNT]; uint64_t get_current_view(uint64_t thd_id); -void set_current_view(uint64_t thd_id, uint64_t view); +uint64_t get_view(uint64_t thd_id); #if VIEW_CHANGES // For updating view for input threads, batching threads, execute thread // and checkpointing thread. -extern std::mutex newViewMTX[BATCH_THREADS + REM_THREAD_CNT + 2]; -extern bool newView[BATCH_THREADS + REM_THREAD_CNT + 2]; -bool get_newView(uint64_t thd_id); -void set_newView(uint64_t thd_id, bool val); +extern std::mutex newViewMTX[THREAD_CNT + REM_THREAD_CNT + SEND_THREAD_CNT]; +extern uint64_t view[THREAD_CNT + REM_THREAD_CNT + SEND_THREAD_CNT]; +void set_view(uint64_t thd_id, uint64_t val); #endif // Size of the batch. extern uint64_t g_batch_size; uint64_t get_batch_size(); extern uint64_t batchSet[2 * CLIENT_NODE_CNT * MAX_TXN_IN_FLIGHT]; +uint64_t view_to_primary(uint64_t view, uint64_t node = g_node_id); // This variable is mainly used by the client to know its current primary. extern uint32_t g_view; extern std::mutex viewMTX; -void set_view(uint64_t nview); -uint64_t get_view(); +void set_client_view(uint64_t nview); +uint64_t get_client_view(); #if LOCAL_FAULT || VIEW_CHANGES // Server parameters for tracking failed replicas @@ -398,6 +402,7 @@ extern double idle_worker_times[THREAD_CNT]; // Statistics to print output_thread_idle_times. extern double output_thd_idle_time[SEND_THREAD_CNT]; +extern double input_thd_idle_time[REM_THREAD_CNT]; // Maps for client response couting extern SpinLockMap client_responses_count; diff --git a/system/helper.cpp b/system/helper.cpp index 0b1c954bc..6d620682f 100644 --- a/system/helper.cpp +++ b/system/helper.cpp @@ -1,39 +1,7 @@ #include "global.h" -#include "helper.h" #include "mem_alloc.h" #include "time.h" -bool itemid_t::operator==(const itemid_t &other) const -{ - return (type == other.type && location == other.location); -} - -bool itemid_t::operator!=(const itemid_t &other) const -{ - return !(*this == other); -} - -void itemid_t::operator=(const itemid_t &other) -{ - this->valid = other.valid; - this->type = other.type; - this->location = other.location; - assert(*this == other); - assert(this->valid); -} - -void itemid_t::init() -{ - valid = false; - location = 0; - next = NULL; -} - -int get_thdid_from_txnid(uint64_t txnid) -{ - return txnid % g_thread_cnt; -} - uint64_t get_part_id(void *addr) { return ((uint64_t)addr / PAGE_SIZE) % g_part_cnt; @@ -71,7 +39,6 @@ uint64_t merge_idx_key(uint64_t key1, uint64_t key2, uint64_t key3) return key1 << 42 | key2 << 21 | key3; } - void init_client_globals() { g_servers_per_client = g_node_cnt; @@ -128,6 +95,17 @@ uint64_t get_sys_clock() return 0; } +std::string hexStr(const char *data, int len) +{ + std::string s(len * 2, ' '); + for (int i = 0; i < len; ++i) + { + s[2 * i] = hexmap[(data[i] & 0xF0) >> 4]; + s[2 * i + 1] = hexmap[data[i] & 0x0F]; + } + return s; +} + void myrand::init(uint64_t seed) { this->seed = seed; @@ -138,3 +116,28 @@ uint64_t myrand::next() seed = (seed * 1103515247UL + 12345UL) % (1UL << 63); return (seed / 65537) % RAND_MAX; } + +uint64_t get_commit_message_txn_id(uint64_t txn_id) +{ + uint64_t result = txn_id / get_batch_size(); + result = (result + 1) * get_batch_size(); + return result - 1; +} +uint64_t get_prep_message_txn_id(uint64_t txn_id) +{ + uint64_t result = txn_id / get_batch_size(); + result = (result + 1) * get_batch_size(); + return result - 1; +} +uint64_t get_checkpoint_message_txn_id(uint64_t txn_id) +{ + uint64_t result = txn_id / get_batch_size(); + result = (result + 1) * get_batch_size(); + return result - 6; +} +uint64_t get_execute_message_txn_id(uint64_t txn_id) +{ + uint64_t result = txn_id / get_batch_size(); + result = (result + 1) * get_batch_size(); + return result - 2; +} \ No newline at end of file diff --git a/system/helper.h b/system/helper.h index 605a9c18b..2b070ffbd 100644 --- a/system/helper.h +++ b/system/helper.h @@ -4,7 +4,6 @@ #include #include #include -#include "global.h" /************************************************/ // Debugging @@ -197,6 +196,9 @@ #define INC_GLOB_STATS(name, value) \ if (STATS_ENABLE && simulation->is_warmup_done()) \ stats.name += value; +#define INC_GLOB_STATS_ARR(name, index, value) \ + if (STATS_ENABLE && simulation->is_warmup_done()) \ + stats.name[index] += value; /************************************************/ // mem copy helper @@ -286,29 +288,6 @@ enum Data_type DT_row }; -// TODO currently, only DR_row supported -// data item type. -class itemid_t -{ -public: - itemid_t(){}; - itemid_t(Data_type type, void *loc) - { - this->type = type; - this->location = loc; - }; - Data_type type; - void *location; // points to the table | page | row - itemid_t *next; - bool valid; - void init(); - bool operator==(const itemid_t &other) const; - bool operator!=(const itemid_t &other) const; - void operator=(const itemid_t &other); -}; - -int get_thdid_from_txnid(uint64_t txnid); - // key_to_part() is only for ycsb uint64_t key_to_part(uint64_t key); uint64_t get_part_id(void *addr); @@ -325,6 +304,19 @@ uint64_t get_wall_clock(); uint64_t get_server_clock(); uint64_t get_sys_clock(); // return: in ns +constexpr char hexmap[] = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; +std::string hexStr(const char *data, int len); + + +// In resilientDB each message for an instance of consensus +// may have specific txn_id in range of its batch size +uint64_t get_commit_message_txn_id(uint64_t txn_id); +uint64_t get_prep_message_txn_id(uint64_t txn_id); +uint64_t get_checkpoint_message_txn_id(uint64_t txn_id); +uint64_t get_execute_message_txn_id(uint64_t txn_id); + + class myrand { public: diff --git a/system/io_thread.cpp b/system/io_thread.cpp index 4e429771b..329394f0e 100644 --- a/system/io_thread.cpp +++ b/system/io_thread.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "thread.h" #include "io_thread.h" #include "query.h" @@ -138,9 +137,6 @@ RC InputThread::run() RC InputThread::client_recv_loop() { - int rsp_cnts[g_node_cnt]; - memset(rsp_cnts, 0, g_node_cnt * sizeof(int)); - run_starttime = get_sys_clock(); uint64_t return_node_offset; uint64_t inf; @@ -205,7 +201,8 @@ RC InputThread::client_recv_loop() //cout<<"Node: "<return_node_id <<" :: Txn: "<< msg->txn_id <<"\n"; //fflush(stdout); - return_node_offset = get_view(); + return_node_offset = get_client_view(); + if (msg->rtype != CL_RSP) { cout << "Mtype: " << msg->rtype << " :: Nd: " << msg->return_node_id << "\n"; @@ -248,47 +245,47 @@ RC InputThread::client_recv_loop() // End the timer. client_timer->endTimer(clrsp->client_ts[get_batch_size() - 1]); #endif - //cout << "validated: " << clrsp->txn_id << "\n"; - //fflush(stdout); + // cout << "validated: " << clrsp->txn_id << " " << clrsp->return_node_id << "\n"; + // fflush(stdout); #if VIEW_CHANGES //cout << "View: " << clrsp->view << "\n"; //fflush(stdout); // This should happen only once after the view change. - if (get_view() != clrsp->view) + if (get_client_view() != clrsp->view) { // Extract the number of pending requests. - uint64_t pending = client_man.get_inflight(get_view()); + uint64_t pending = client_man.get_inflight(get_client_view()); for (uint64_t j = 0; j < pending; j++) { client_man.inc_inflight(clrsp->view); } // Move to new view. - set_view(clrsp->view); - return_node_offset = get_view(); + set_client_view(clrsp->view); + return_node_offset = get_client_view(); } #endif #if CLIENT_RESPONSE_BATCH == true for (uint64_t k = 0; k < g_batch_size; k++) { - rsp_cnts[return_node_offset]++; - INC_STATS(get_thd_id(), txn_cnt, 1); - uint64_t timespan = get_sys_clock() - clrsp->client_ts[k]; - INC_STATS(get_thd_id(), txn_run_time, timespan); - - sumlat = sumlat + timespan; - txncmplt++; + if (simulation->is_warmup_done()) + { + INC_STATS(get_thd_id(), txn_cnt, 1); + uint64_t timespan = get_sys_clock() - clrsp->client_ts[k]; + INC_STATS(get_thd_id(), txn_run_time, timespan); + sumlat = sumlat + timespan; + txncmplt++; + } inf = client_man.dec_inflight(return_node_offset); } Message::release_message(client_responses_directory.get(msg->txn_id)); client_responses_directory.remove(msg->txn_id); #else // !CLIENT_RESPONSE_BATCH - rsp_cnts[return_node_offset]++; INC_STATS(get_thd_id(), txn_cnt, 1); uint64_t timespan = get_sys_clock() - clrsp->client_startts; INC_STATS(get_thd_id(), txn_run_time, timespan); @@ -312,13 +309,8 @@ RC InputThread::client_recv_loop() delete msgs; } - printf("AVG: %f\n", (sumlat / (txncmplt * BILLION))); - fflush(stdout); - printf("TXNCOMP: %ld\n", txncmplt); - fflush(stdout); - - printf("FINISH %ld:%ld\n", _node_id, _thd_id); - fflush(stdout); + printf("AVG Latency: %f\n", (sumlat / (txncmplt * BILLION))); + printf("TXN_CNT: %ld\n", txncmplt); return FINISH; } @@ -336,19 +328,6 @@ RC InputThread::server_recv_loop() { heartbeat(); -#if VIEW_CHANGES - if (g_node_id != get_current_view(get_thd_id())) - { - uint64_t tid = get_thd_id() - 1; - uint32_t nchange = get_newView(tid); - - if (nchange) - { - set_current_view(get_thd_id(), get_current_view(get_thd_id()) + 1); - set_newView(tid, false); - } - } -#endif msgs = tport_man.recv_msg(get_thd_id()); @@ -360,7 +339,7 @@ RC InputThread::server_recv_loop() } continue; } - if (idle_starttime > 0) + if (idle_starttime > 0 && simulation->is_warmup_done()) { starttime += get_sys_clock() - idle_starttime; idle_starttime = 0; @@ -386,7 +365,8 @@ RC InputThread::server_recv_loop() } delete msgs; } - cout << "Input: " << _thd_id << " :: " << (starttime * 1.0) / BILLION << "\n"; + // cout << "Input: " << _thd_id << " :: " << (starttime * 1.0) / BILLION << "\n"; + input_thd_idle_time[_thd_id - g_thread_cnt] = starttime; fflush(stdout); return FINISH; } @@ -433,7 +413,7 @@ RC OutputThread::run() messager->run(); } - printf("Output %ld: %f\n", _thd_id, output_thd_idle_time[_thd_id % g_send_thread_cnt] / BILLION); + // printf("Output %ld: %f\n", _thd_id, output_thd_idle_time[_thd_id % g_send_thread_cnt] / BILLION); fflush(stdout); return FINISH; } diff --git a/system/main.cpp b/system/main.cpp index 7a61762db..82e2002c0 100644 --- a/system/main.cpp +++ b/system/main.cpp @@ -76,7 +76,7 @@ int main(int argc, char *argv[]) #else Workload *m_wl = new YCSBWorkload; #endif - + m_wl->init(); printf("Workload initialized!\n"); fflush(stdout); @@ -123,7 +123,7 @@ int main(int argc, char *argv[]) printf("Initializing Chain... "); fflush(stdout); - BlockChain = new BChain(); + BlockChain = new BChain(); printf("Done\n"); #if TIMER_ON @@ -296,6 +296,11 @@ int main(int argc, char *argv[]) fflush(stdout); printf("PASS! SimTime = %f\n", (float)(endtime - starttime) / BILLION); + for (uint64_t i = 0; i < g_send_thread_cnt; i++) + printf("Output %ld: %f\n", i + g_thread_cnt + g_rem_thread_cnt, output_thd_idle_time[i] / BILLION); + for (uint64_t i = 0; i < g_rem_thread_cnt; i++) + printf("Input %ld: %f\n", i + g_thread_cnt, input_thd_idle_time[i] / BILLION); + if (STATS_ENABLE) stats.print(false); diff --git a/system/mem_alloc.cpp b/system/mem_alloc.cpp index d0d447247..2b54519e6 100644 --- a/system/mem_alloc.cpp +++ b/system/mem_alloc.cpp @@ -1,6 +1,5 @@ #include "mem_alloc.h" -#include "helper.h" #include "global.h" #include "jemalloc/jemalloc.h" diff --git a/system/msg_queue.cpp b/system/msg_queue.cpp index 776af0501..4884152b2 100644 --- a/system/msg_queue.cpp +++ b/system/msg_queue.cpp @@ -29,7 +29,7 @@ void MessageQueue::init() sthd_m_cache.push_back(NULL); } -void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector &ndsign, const vector &dest) +void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector &dest) { msg_entry *entry = (msg_entry *)mem_allocator.alloc(sizeof(struct msg_entry)); @@ -38,6 +38,7 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector & entry->msg = msg; if (msg == NULL) { + assert(0); return; } @@ -57,12 +58,13 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector & case CL_BATCH: ((ClientQueryBatch *)msg)->sign(dest[0]); - entry->allsign.push_back(ndsign[0]); + entry->allsign.push_back(msg->signature); break; case BATCH_REQ: - for (uint64_t i = 0; i < ndsign.size(); i++) + for (uint64_t i = 0; i < dest.size(); i++) { - entry->allsign.push_back(ndsign[i]); + ((BatchRequests *)msg)->sign(dest[i]); + entry->allsign.push_back(msg->signature); } break; case PBFT_CHKPT_MSG: @@ -80,6 +82,7 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector & } break; +#if CONSENSUS == PBFT case PBFT_COMMIT_MSG: for (uint64_t i = 0; i < dest.size(); i++) { @@ -87,6 +90,7 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector & entry->allsign.push_back(((PBFTCommitMessage *)msg)->signature); } break; +#endif #if VIEW_CHANGES case VIEW_CHANGE: @@ -130,6 +134,7 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector & INC_STATS(thd_id, msg_queue_enq_cnt, 1); break; } + case BATCH_REQ: case PBFT_CHKPT_MSG: case PBFT_PREP_MSG: @@ -248,6 +253,7 @@ vector MessageQueue::dequeue(uint64_t thd_id, vector &allsign, INC_STATS(thd_id, msg_queue_cnt, 1); msg->mq_time = curr_time - entry->starttime; DEBUG_M("MessageQueue::enqueue msg_entry free\n"); + entry->allsign.clear(); mem_allocator.free(entry, sizeof(struct msg_entry)); } else diff --git a/system/msg_queue.h b/system/msg_queue.h index 04efaf59d..780354dfb 100644 --- a/system/msg_queue.h +++ b/system/msg_queue.h @@ -2,7 +2,6 @@ #define _MSG_QUEUE_H_ #include "global.h" -#include "helper.h" #include "lock_free_queue.h" #include @@ -24,7 +23,7 @@ class MessageQueue public: void init(); - void enqueue(uint64_t thd_id, Message *msg, const vector &ndsign, const vector &dest); + void enqueue(uint64_t thd_id, Message *msg, const vector &dest); vector dequeue(uint64_t thd_id, vector &allsign, Message *&msg); private: diff --git a/system/parser.cpp b/system/parser.cpp index e0bc2b0c9..f797b2589 100644 --- a/system/parser.cpp +++ b/system/parser.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" void print_usage() { diff --git a/system/pool.cpp b/system/pool.cpp index 9eb71b633..931d26d60 100644 --- a/system/pool.cpp +++ b/system/pool.cpp @@ -1,6 +1,5 @@ #include "pool.h" #include "global.h" -#include "helper.h" #include "txn.h" #include "mem_alloc.h" #include "wl.h" @@ -176,7 +175,7 @@ void QryPool::get(uint64_t pool_id, BaseQuery *&item) void QryPool::put(uint64_t pool_id, BaseQuery *item) { assert(item); - ((YCSBQuery *)item)->reset(); + ((YCSBQuery *)item)->release(); DEBUG_R("put 0x%lx\n", (uint64_t)item); int tries = 0; diff --git a/system/pool.h b/system/pool.h index e7c91ec4a..e9558e07f 100644 --- a/system/pool.h +++ b/system/pool.h @@ -2,7 +2,6 @@ #define _TXN_POOL_H_ #include "global.h" -#include "helper.h" #include #include diff --git a/system/query.h b/system/query.h index 8ef77779b..860427f7f 100644 --- a/system/query.h +++ b/system/query.h @@ -2,7 +2,6 @@ #define _QUERY_H_ #include "global.h" -#include "helper.h" #include "array.h" class Workload; diff --git a/system/sim_manager.cpp b/system/sim_manager.cpp index 3b04d2bbb..9d57e5ca0 100644 --- a/system/sim_manager.cpp +++ b/system/sim_manager.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "sim_manager.h" void SimManager::init() diff --git a/system/thread.cpp b/system/thread.cpp index d008e77a3..bdfdd9627 100644 --- a/system/thread.cpp +++ b/system/thread.cpp @@ -4,7 +4,6 @@ #include "wl.h" #include "query.h" #include "math.h" -#include "helper.h" #include "msg_queue.h" #include "message.h" @@ -20,10 +19,9 @@ void Thread::send_init_done_to_all_nodes() { printf("Send INIT_DONE to %ld\n", i); - vector emptyvec; vector dest; dest.push_back(i); - msg_queue.enqueue(get_thd_id(), Message::create_message(INIT_DONE), emptyvec, dest); + msg_queue.enqueue(get_thd_id(), Message::create_message(INIT_DONE), dest); dest.clear(); } } @@ -63,19 +61,27 @@ void Thread::tsetup() pthread_barrier_wait(&warmup_bar); } +bool Thread::has_view_changed() +{ + if (get_current_view(get_thd_id())) + return true; + return false; +} + void Thread::progress_stats() { - if (get_thd_id() == 0) - { #if TIME_ENABLE - uint64_t now_time = get_sys_clock(); + uint64_t now_time = get_sys_clock(); #else - uint64_t now_time = get_wall_clock(); + uint64_t now_time = get_wall_clock(); #endif - if (now_time - prog_time >= g_prog_timer) + + if (now_time - prog_time >= g_prog_timer) + { + prog_time = now_time; + if (get_thd_id() == 0) { - prog_time = now_time; - SET_STATS(get_thd_id(), total_runtime, prog_time - simulation->run_starttime); + SET_STATS(get_thd_id(), total_runtime, prog_time - simulation->warmup_end_time); if (ISCLIENT) { @@ -85,6 +91,10 @@ void Thread::progress_stats() { stats.print(true); } + // uint64_t exec_thread = g_thread_cnt - g_checkpointing_thread_cnt - g_execution_thread_cnt; + uint64_t exec_thread = g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt + g_execution_thread_cnt - 1; + SET_STATS(exec_thread, previous_interval_cross_shard_txn_cnt, stats._stats[exec_thread]->cross_shard_txn_cnt); + SET_STATS(exec_thread, previous_interval_txn_cnt, stats._stats[exec_thread]->txn_cnt); } } } diff --git a/system/thread.h b/system/thread.h index 0b75c6474..39138bad6 100644 --- a/system/thread.h +++ b/system/thread.h @@ -21,6 +21,9 @@ class Thread uint64_t get_thd_id(); uint64_t get_node_id(); void tsetup(); + + // TODO for experimental purpose: force one view change + bool has_view_changed(); void init(uint64_t thd_id, uint64_t node_id, Workload *workload); // the following function must be in the form void* (*)(void*) diff --git a/system/timer.cpp b/system/timer.cpp index 77e98a789..93498d119 100644 --- a/system/timer.cpp +++ b/system/timer.cpp @@ -154,6 +154,7 @@ void ClientTimer::startTimer(uint64_t timestp, ClientQueryBatch *cqry) tlock.lock(); txn_map.push_back(tmap); + // printf("added %ld %ld\n",cqry->txn_id, txn_map.size()); tlock.unlock(); } @@ -170,6 +171,7 @@ void ClientTimer::endTimer(uint64_t timestp) txn_map.erase(txn_map.begin() + i); Message::release_message(tmap->get_msg()); mem_allocator.free(tmap, sizeof(Timer)); + // printf("remvd %ld %ld\n",tmap->get_msg()->txn_id, i); break; } } @@ -208,6 +210,7 @@ bool ClientTimer::checkTimer(ClientQueryBatch *&cbatch) // Found so return true. flag = true; + printf("resending to primary %ld\n",tmap->get_msg()->txn_id); } } diff --git a/system/txn.cpp b/system/txn.cpp index 5318f986e..57af5598c 100644 --- a/system/txn.cpp +++ b/system/txn.cpp @@ -1,4 +1,3 @@ -#include "helper.h" #include "txn.h" #include "wl.h" #include "query.h" @@ -137,6 +136,7 @@ void TxnManager::init(uint64_t pool_id, Workload *h_wl) { DEBUG_M("TxnManager::init Query alloc\n"); qry_pool.get(pool_id, query); + this->query->init(); } #endif sem_init(&rsp_mutex, 0, 1); @@ -151,6 +151,7 @@ void TxnManager::init(uint64_t pool_id, Workload *h_wl) prep_rsp_cnt = 2 * g_min_invalid_nodes; commit_rsp_cnt = prep_rsp_cnt + 1; chkpt_cnt = 2 * g_min_invalid_nodes; + chkpt_flag = false; batchreq = NULL; @@ -197,7 +198,8 @@ void TxnManager::release(uint64_t pool_id) prep_rsp_cnt = 2 * g_min_invalid_nodes; commit_rsp_cnt = prep_rsp_cnt + 1; - chkpt_cnt = 2 * g_min_invalid_nodes + 1; + chkpt_cnt = 2 * g_min_invalid_nodes; + chkpt_flag = false; release_all_messages(tid); txn_stats.init(); @@ -253,11 +255,9 @@ void TxnManager::commit_stats() txn_stats.commit_stats(get_thd_id(), get_txn_id(), get_batch_id(), timespan_long, timespan_short); return; } - - INC_STATS(get_thd_id(), txn_cnt, 1); + INC_STATS(get_thd_id(), txn_run_time, timespan_long); INC_STATS(get_thd_id(), single_part_txn_cnt, 1); - txn_stats.commit_stats(get_thd_id(), get_txn_id(), get_batch_id(), timespan_long, timespan_short); } @@ -448,7 +448,6 @@ void TxnManager::send_pbft_prep_msgs() } #endif - vector emptyvec; vector dest; for (uint64_t i = 0; i < g_node_cnt; i++) { @@ -459,7 +458,7 @@ void TxnManager::send_pbft_prep_msgs() dest.push_back(i); } - msg_queue.enqueue(get_thd_id(), pmsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), pmsg, dest); dest.clear(); } @@ -479,7 +478,8 @@ void TxnManager::send_pbft_commit_msgs() } #endif - vector emptyvec; + + vector dest; for (uint64_t i = 0; i < g_node_cnt; i++) { @@ -490,7 +490,7 @@ void TxnManager::send_pbft_commit_msgs() dest.push_back(i); } - msg_queue.enqueue(get_thd_id(), cmsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), cmsg, dest); dest.clear(); } @@ -498,11 +498,7 @@ void TxnManager::send_pbft_commit_msgs() void TxnManager::release_all_messages(uint64_t txn_id) { - if ((txn_id + 3) % get_batch_size() == 0) - { - allsign.clear(); - } - else if ((txn_id + 1) % get_batch_size() == 0) + if ((txn_id + 1) % get_batch_size() == 0) { info_prepare.clear(); info_commit.clear(); @@ -529,7 +525,6 @@ void TxnManager::send_checkpoint_msgs() Message *msg = Message::create_message(this, PBFT_CHKPT_MSG); CheckpointMessage *ckmsg = (CheckpointMessage *)msg; - vector emptyvec; vector dest; for (uint64_t i = 0; i < g_node_cnt; i++) { @@ -540,6 +535,6 @@ void TxnManager::send_checkpoint_msgs() dest.push_back(i); } - msg_queue.enqueue(get_thd_id(), ckmsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), ckmsg, dest); dest.clear(); } diff --git a/system/txn.h b/system/txn.h index a65b2766f..900d30c36 100644 --- a/system/txn.h +++ b/system/txn.h @@ -2,7 +2,6 @@ #define _TXN_H_ #include "global.h" -#include "helper.h" #include "semaphore.h" #include "array.h" #include "message.h" @@ -135,8 +134,6 @@ class TxnManager BatchRequests *batchreq; void set_primarybatch(BatchRequests *breq); - vector allsign; - uint64_t get_abort_cnt() { return abort_cnt; } uint64_t abort_cnt; int received_response(RC rc); diff --git a/system/txn_table.h b/system/txn_table.h index 1dd725280..e8eb5a7f7 100644 --- a/system/txn_table.h +++ b/system/txn_table.h @@ -2,7 +2,6 @@ #define _TXN_TABLE_H_ #include "global.h" -#include "helper.h" #include #include diff --git a/system/txn_test.cpp b/system/txn_test.cpp index 96302688d..a857ad705 100644 --- a/system/txn_test.cpp +++ b/system/txn_test.cpp @@ -1,4 +1,3 @@ -#include "helper.h" #include "txn.h" #include "wl.h" #include "query.h" diff --git a/system/wl.cpp b/system/wl.cpp index 2260ac878..60a89dd4e 100644 --- a/system/wl.cpp +++ b/system/wl.cpp @@ -1,5 +1,4 @@ #include "global.h" -#include "helper.h" #include "wl.h" #include "mem_alloc.h" diff --git a/system/work_queue.cpp b/system/work_queue.cpp index 744a3d59c..36730aeb5 100644 --- a/system/work_queue.cpp +++ b/system/work_queue.cpp @@ -5,38 +5,23 @@ #include "client_query.h" #include -void QWorkQueue::init() +void QWorkQueue::push_to_queue(work_queue_entry *entry, boost::lockfree::queue *queue) { - - last_sched_dq = NULL; - sched_ptr = 0; - seq_queue = new boost::lockfree::queue(0); - - // Queue for worker thread 0. - uint64_t effective_queue_cnt = 1; - -#if EXECUTION_THREAD - effective_queue_cnt += indexSize; -#endif - - // A queue for checkpoint messages. - effective_queue_cnt++; - - cout << "Total queues: " << effective_queue_cnt << "\n"; - fflush(stdout); - - work_queue = new boost::lockfree::queue *[effective_queue_cnt]; - for (uint64_t i = 0; i < effective_queue_cnt; i++) + assert(entry->rtype < 100); + while (!queue->push(entry) && !simulation->is_done()) { - work_queue[i] = new boost::lockfree::queue(0); } - - new_txn_queue = new boost::lockfree::queue(0); - sched_queue = new boost::lockfree::queue *[g_node_cnt]; - for (uint64_t i = 0; i < g_node_cnt; i++) +} +void QWorkQueue::init() +{ + work_queue = new boost::lockfree::queue(0); + execution_queues = new boost::lockfree::queue *[indexSize]; + for (uint64_t i = 0; i < indexSize; i++) { - sched_queue[i] = new boost::lockfree::queue(0); + execution_queues[i] = new boost::lockfree::queue(0); } + new_txn_queue = new boost::lockfree::queue(0); + checkpoint_queue = new boost::lockfree::queue(0); } #if ENABLE_PIPELINE @@ -55,54 +40,37 @@ void QWorkQueue::enqueue(uint64_t thd_id, Message *msg, bool busy) assert(ISSERVER || ISREPLICA); DEBUG("Work Enqueue (%ld,%ld) %d\n", entry->txn_id, entry->batch_id, entry->rtype); + if (msg->rtype == CL_QRY || msg->rtype == CL_BATCH) { - if (g_node_id == get_current_view(thd_id)) - { - //cout << "Placing \n"; - while (!new_txn_queue->push(entry) && !simulation->is_done()) - { - } - } - else - { - assert(entry->rtype < 100); - while (!work_queue[0]->push(entry) && !simulation->is_done()) - { - } - } + // Client Requests to batching queues + push_to_queue(entry, new_txn_queue); } else if (msg->rtype == BATCH_REQ) { - // Queue for Thread for ordered sending of batches. - assert(entry->rtype < 100); - if (g_node_id != get_current_view(thd_id)) + // The Primary should not receive pre-prepare message + if (g_node_id == view_to_primary(get_current_view(thd_id))) { - while (!work_queue[0]->push(entry) && !simulation->is_done()) - { - } + assert(0); + } + else + { + push_to_queue(entry, work_queue); } } else if (msg->rtype == EXECUTE_MSG) { - uint64_t bid = ((msg->txn_id + 2) - get_batch_size()) / get_batch_size(); - uint64_t qid = bid % indexSize; - while (!work_queue[qid + 1]->push(entry) && !simulation->is_done()) - { - } + // Execution Map using [indexSize] queues + uint64_t queue_id = (msg->txn_id / get_batch_size()) % indexSize; + push_to_queue(entry, execution_queues[queue_id]); } else if (msg->rtype == PBFT_CHKPT_MSG) { - while (!work_queue[indexSize + 1]->push(entry) && !simulation->is_done()) - { - } + push_to_queue(entry, checkpoint_queue); } else { - assert(entry->rtype < 100); - while (!work_queue[0]->push(entry) && !simulation->is_done()) - { - } + push_to_queue(entry, work_queue); } INC_STATS(thd_id, work_queue_enqueue_time, get_sys_clock() - starttime); @@ -112,57 +80,51 @@ void QWorkQueue::enqueue(uint64_t thd_id, Message *msg, bool busy) Message *QWorkQueue::dequeue(uint64_t thd_id) { uint64_t starttime = get_sys_clock(); + assert(ISSERVER || ISREPLICA); - Message *msg = NULL; + bool valid = false; + Message *message = NULL; work_queue_entry *entry = NULL; - bool valid = false; + assert(g_thread_cnt == g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt + g_execution_thread_cnt); - // Thread 0 only looks at work queue - if (thd_id == 0) + // Worker Threads + if (thd_id < g_worker_thread_cnt) { - valid = work_queue[0]->pop(entry); + valid = work_queue->pop(entry); } - - UInt32 tcount = g_thread_cnt - g_execute_thd - g_btorder_thd; - - if (thd_id >= tcount && thd_id < (tcount + g_execute_thd)) + // Batching Threads + else if (thd_id < g_worker_thread_cnt + g_batching_thread_cnt) { - // Thread for handling execute messages. - uint64_t bid = ((get_expectedExecuteCount() + 2) - get_batch_size()) / get_batch_size(); - uint64_t qid = bid % indexSize; - valid = work_queue[qid + 1]->pop(entry); + valid = new_txn_queue->pop(entry); } - else if (thd_id >= tcount + g_execute_thd) + // Checkpointing Threads + else if (thd_id < g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt) { - // Thread for handling checkpoint messages. - valid = work_queue[indexSize + 1]->pop(entry); + valid = checkpoint_queue->pop(entry); } - - if (!valid) + // Execution Threads + else if (thd_id < g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt + g_execution_thread_cnt) { - // Allowing new transactions to be accessed by batching threads. - if (thd_id > 0 && thd_id <= tcount - 1) - { - valid = new_txn_queue->pop(entry); - } + uint64_t queue_id = (get_expectedExecuteCount() / get_batch_size()) % indexSize; + valid = execution_queues[queue_id]->pop(entry); } if (valid) { - msg = entry->msg; - assert(msg); + message = entry->msg; + assert(message); uint64_t queue_time = get_sys_clock() - entry->starttime; INC_STATS(thd_id, work_queue_wait_time, queue_time); INC_STATS(thd_id, work_queue_cnt, 1); - msg->wq_time = queue_time; + message->wq_time = queue_time; DEBUG("Work Dequeue (%ld,%ld)\n", entry->txn_id, entry->batch_id); mem_allocator.free(entry, sizeof(work_queue_entry)); INC_STATS(thd_id, work_queue_dequeue_time, get_sys_clock() - starttime); } - return msg; + return message; } #endif // ENABLE_PIPELINE == true diff --git a/system/work_queue.h b/system/work_queue.h index cea753f99..37f3018ce 100644 --- a/system/work_queue.h +++ b/system/work_queue.h @@ -2,7 +2,6 @@ #define _WORK_QUEUE_H_ #include "global.h" -#include "helper.h" #include #include @@ -19,42 +18,7 @@ struct work_queue_entry uint64_t starttime; }; -struct CompareSchedEntry -{ - bool operator()(const work_queue_entry *lhs, const work_queue_entry *rhs) - { - if (lhs->batch_id == rhs->batch_id) - return lhs->starttime > rhs->starttime; - return lhs->batch_id < rhs->batch_id; - } -}; -struct CompareWQEntry -{ -#if PRIORITY == PRIORITY_FCFS - bool operator()(const work_queue_entry *lhs, const work_queue_entry *rhs) - { - return lhs->starttime < rhs->starttime; - } -#elif PRIORITY == PRIORITY_ACTIVE - bool operator()(const work_queue_entry *lhs, const work_queue_entry *rhs) - { - if (lhs->rtype == CL_QRY && rhs->rtype != CL_QRY) - return true; - if (rhs->rtype == CL_QRY && lhs->rtype != CL_QRY) - return false; - return lhs->starttime < rhs->starttime; - } -#elif PRIORITY == PRIORITY_HOME - bool operator()(const work_queue_entry *lhs, const work_queue_entry *rhs) - { - if (ISLOCAL(lhs->txn_id) && !ISLOCAL(rhs->txn_id)) - return true; - if (ISLOCAL(rhs->txn_id) && !ISLOCAL(lhs->txn_id)) - return false; - return lhs->starttime < rhs->starttime; - } -#endif -}; + class QWorkQueue { @@ -62,26 +26,14 @@ class QWorkQueue void init(); void enqueue(uint64_t thd_id, Message *msg, bool busy); Message *dequeue(uint64_t thd_id); - void sched_enqueue(uint64_t thd_id, Message *msg); - Message *sched_dequeue(uint64_t thd_id); - void sequencer_enqueue(uint64_t thd_id, Message *msg); - Message *sequencer_dequeue(uint64_t thd_id); - - uint64_t get_cnt() { return get_wq_cnt() + get_rem_wq_cnt() + get_new_wq_cnt(); } - uint64_t get_wq_cnt() { return 0; } - uint64_t get_sched_wq_cnt() { return 0; } - uint64_t get_rem_wq_cnt() { return 0; } - uint64_t get_new_wq_cnt() { return 0; } + void push_to_queue(work_queue_entry* entry, boost::lockfree::queue *queue); private: - boost::lockfree::queue **work_queue; - + boost::lockfree::queue **execution_queues; + boost::lockfree::queue *work_queue; boost::lockfree::queue *new_txn_queue; - boost::lockfree::queue *seq_queue; - boost::lockfree::queue **sched_queue; - uint64_t sched_ptr; - BaseQuery *last_sched_dq; - uint64_t curr_epoch; + boost::lockfree::queue *checkpoint_queue; + }; #endif diff --git a/system/work_queue_nopipeline.cpp b/system/work_queue_nopipeline.cpp index df0d0ee38..4bcccd3f1 100644 --- a/system/work_queue_nopipeline.cpp +++ b/system/work_queue_nopipeline.cpp @@ -75,8 +75,8 @@ Message *QWorkQueue::dequeue(uint64_t thd_id) } #if EXECUTION_THREAD - UInt32 tcount = g_thread_cnt - g_btorder_thd - g_execute_thd; - if (thd_id >= tcount + g_btorder_thd) + UInt32 tcount = g_thread_cnt - g_checkpointing_thread_cnt - g_execution_thread_cnt; + if (thd_id >= tcount + g_checkpointing_thread_cnt) { valid = work_queue[indexSize + 1]->pop(entry); } diff --git a/system/worker_thread.cpp b/system/worker_thread.cpp index dd02e3f1f..a72bcd98e 100644 --- a/system/worker_thread.cpp +++ b/system/worker_thread.cpp @@ -7,7 +7,6 @@ #include "query.h" #include "ycsb_query.h" #include "math.h" -#include "helper.h" #include "msg_thread.h" #include "msg_queue.h" #include "work_queue.h" @@ -41,10 +40,10 @@ void WorkerThread::send_key() keyex->return_node = g_node_id; //msg_queue.enqueue(get_thd_id(), keyex, i); - vector emptyvec; vector dest; dest.push_back(i); - msg_queue.enqueue(get_thd_id(), keyex, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), keyex, dest); + dest.clear(); #endif @@ -56,7 +55,7 @@ void WorkerThread::send_key() keyexCMAC->return_node = g_node_id; //msg_queue.enqueue(get_thd_id(), keyexCMAC, i); - msg_queue.enqueue(get_thd_id(), keyexCMAC, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), keyexCMAC, dest); dest.clear(); cout << "Sending CMAC " << cmacPrivateKeys[i] << endl; @@ -64,7 +63,26 @@ void WorkerThread::send_key() #endif } } - +void WorkerThread::unset_ready_txn(TxnManager *tman) +{ + // uint64_t spin_wait_starttime = 0; + while (true) + { + bool ready = tman->unset_ready(); + if (!ready) + { + // if (spin_wait_starttime == 0) + // spin_wait_starttime = get_sys_clock(); + continue; + } + else + { + // if (spin_wait_starttime > 0) + // INC_STATS(_thd_id, worker_spin_wait_time, get_sys_clock() - spin_wait_starttime); + break; + } + } +} void WorkerThread::setup() { // Increment commonVar. @@ -120,7 +138,7 @@ void WorkerThread::process(Message *msg) rc = process_pbft_commit_msg(msg); break; default: - printf("Msg: %d\n", msg->get_rtype()); + printf("rtype: %d from %ld\n", msg->get_rtype(), msg->return_node_id); fflush(stdout); assert(false); break; @@ -185,10 +203,9 @@ RC WorkerThread::process_key_exchange(Message *msg) Message *rdy = Message::create_message(READY); //msg_queue.enqueue(get_thd_id(), rdy, i); - vector emptyvec; vector dest; dest.push_back(i); - msg_queue.enqueue(get_thd_id(), rdy, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), rdy, dest); dest.clear(); } @@ -258,11 +275,25 @@ void WorkerThread::fail_nonprimary() #if TIMER_ON void WorkerThread::add_timer(Message *msg, string qryhash) { + // TODO if condition is for experimental purpose: force one view change + if (this->has_view_changed()) + return; + char *tbuf = create_msg_buffer(msg); Message *deepMsg = deep_copy_msg(tbuf, msg); + deepMsg->return_node_id = msg->return_node_id; server_timer->startTimer(qryhash, deepMsg); delete_msg_buffer(tbuf); } + +void WorkerThread::remove_timer(string qryhash) +{ + // TODO if condition is for experimental purpose: force one view change + if (this->has_view_changed()) + return; + + server_timer->endTimer(qryhash); +} #endif #if VIEW_CHANGES @@ -273,27 +304,29 @@ This requires sending a view change message to each replica. */ void WorkerThread::check_for_timeout() { - if (g_node_id != get_current_view(get_thd_id()) && - server_timer->checkTimer()) + // TODO if condition is for experimental purpose: force one view change + if (this->has_view_changed()) + return; + + if (g_node_id != get_current_view(get_thd_id()) && server_timer->checkTimer()) { // Pause the timer to avoid causing further view changes. server_timer->pauseTimer(); - cout << "Begin Changing View" << endl; + // cout << "Begin Changing View" << endl; fflush(stdout); Message *msg = Message::create_message(VIEW_CHANGE); TxnManager *local_tman = get_transaction_manager(get_curr_chkpt(), 0); - cout << "Initializing" << endl; + // cout << "Initializing" << endl; fflush(stdout); ((ViewChangeMsg *)msg)->init(get_thd_id(), local_tman); - cout << "Going to send" << endl; + // cout << "Going to send" << endl; fflush(stdout); //send view change messages - vector emptyvec; vector dest; for (uint64_t i = 0; i < g_node_cnt; i++) { @@ -312,7 +345,7 @@ void WorkerThread::check_for_timeout() char *buf = create_msg_buffer(msg); Message *deepCMsg = deep_copy_msg(buf, msg); // Send to other replicas. - msg_queue.enqueue(get_thd_id(), deepCMsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), deepCMsg, dest); dest.clear(); // process a message for itself @@ -322,43 +355,20 @@ void WorkerThread::check_for_timeout() delete_msg_buffer(buf); Message::release_message(msg); // Releasing the message. - cout << "Sent from here" << endl; fflush(stdout); } } -/* -In case there is a view change, all the threads would need a change of views. -*/ -void WorkerThread::check_switch_view() -{ - uint64_t thd_id = get_thd_id(); - if (thd_id == 0) - { - return; - } - else - { - thd_id--; - } - - uint32_t nchange = get_newView(thd_id); - - if (nchange) - { - cout << "Change: " << thd_id + 1 << "\n"; - fflush(stdout); - - set_current_view(thd_id + 1, get_current_view(thd_id + 1) + 1); - set_newView(thd_id, false); - } -} /* This function causes the forced failure of the primary replica at a -desired batch. */ -void WorkerThread::fail_primary(Message *msg, uint64_t batch_to_fail) +desired time. */ +void WorkerThread::fail_primary(Message *msg, uint64_t time) { - if (g_node_id == 0 && msg->txn_id > batch_to_fail) + if (!simulation->is_warmup_done()) + return; + uint64_t elapsesd_time = get_sys_clock() - simulation->warmup_end_time; + if (g_node_id == 0 && elapsesd_time > time) + // if (g_node_id == 0 && msg->txn_id > 9) { uint64_t count = 0; while (true) @@ -387,8 +397,12 @@ Hence store the request, start timer and forward it to the primary replica. */ void WorkerThread::client_query_check(ClientQueryBatch *clbtch) { - //cout << "REQUEST: " << clbtch->return_node_id << "\n"; - //fflush(stdout); + // TODO if condition is for experimental purpose: force one view change + if (this->has_view_changed()) + return; + + // cout << "REQUEST: " << clbtch->return_node_id << "\n"; + // fflush(stdout); //start timer when client broadcasts an unexecuted message // Last request of the batch. @@ -396,14 +410,13 @@ void WorkerThread::client_query_check(ClientQueryBatch *clbtch) add_timer(clbtch, calculateHash(qry->getString())); // Forward to the primary. - vector emptyvec; - emptyvec.push_back(clbtch->signature); // Sign the message. vector dest; dest.push_back(get_current_view(get_thd_id())); char *tbuf = create_msg_buffer(clbtch); Message *deepCMsg = deep_copy_msg(tbuf, clbtch); - msg_queue.enqueue(get_thd_id(), deepCMsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), deepCMsg, dest); + dest.clear(); delete_msg_buffer(tbuf); } @@ -413,8 +426,7 @@ void WorkerThread::client_query_check(ClientQueryBatch *clbtch) RC WorkerThread::process_view_change_msg(Message *msg) { - cout << "PROCESS VIEW CHANGE " - << "\n"; + cout << "PROCESS VIEW CHANGE from" << msg->return_node_id << "\n"; fflush(stdout); ViewChangeMsg *vmsg = (ViewChangeMsg *)msg; @@ -446,34 +458,11 @@ RC WorkerThread::process_view_change_msg(Message *msg) //cout << "New primary rules!" << endl; //fflush(stdout); - set_current_view(get_thd_id(), g_node_id); - - // Reset the views for different threads. - uint64_t total_thds = g_batch_threads + g_rem_thread_cnt + 2; + // Move to next view + uint64_t total_thds = g_thread_cnt + g_rem_thread_cnt + g_send_thread_cnt; for (uint64_t i = 0; i < total_thds; i++) { - set_newView(i, true); - } - - // Need to ensure that all the threads of the new primary are ready. - uint64_t count = 0; - bool lflag[total_thds] = {false}; - while (true) - { - for (uint64_t i = 0; i < total_thds; i++) - { - bool nchange = get_newView(i); - if (!nchange && !lflag[i]) - { - count++; - lflag[i] = true; - } - } - - if (count == total_thds) - { - break; - } + set_view(i, vmsg->view); } Message *newViewMsg = Message::create_message(NEW_VIEW); @@ -492,7 +481,6 @@ RC WorkerThread::process_view_change_msg(Message *msg) } //send new view messages - vector emptyvec; vector dest; for (uint64_t i = 0; i < g_node_cnt; i++) { @@ -505,7 +493,7 @@ RC WorkerThread::process_view_change_msg(Message *msg) char *buf = create_msg_buffer(nvmsg); Message *deepCMsg = deep_copy_msg(buf, nvmsg); - msg_queue.enqueue(get_thd_id(), deepCMsg, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), deepCMsg, dest); dest.clear(); delete_msg_buffer(buf); @@ -523,19 +511,19 @@ RC WorkerThread::process_view_change_msg(Message *msg) // Start the re-directed requests. Timer *tmap; - Message *msg; + Message *retrieved_msg; for (uint64_t i = 0; i < server_timer->timerSize(); i++) { tmap = server_timer->fetchPendingRequests(i); - msg = tmap->get_msg(); - + retrieved_msg = tmap->get_msg(); //YCSBClientQueryMessage *yc = ((ClientQueryBatch *)msg)->cqrySet[0]; //cout << "MSG: " << yc->return_node << " :: Key: " << yc->requests[0]->key << "\n"; //fflush(stdout); - char *buf = create_msg_buffer(msg); - Message *deepCMsg = deep_copy_msg(buf, msg); + char *buf = create_msg_buffer(retrieved_msg); + Message *deepCMsg = deep_copy_msg(buf, retrieved_msg); + deepCMsg->return_node_id = retrieved_msg->return_node_id; // Assigning an identifier to the batch. deepCMsg->txn_id = get_and_inc_next_idx(); @@ -571,35 +559,12 @@ RC WorkerThread::process_new_view_msg(Message *msg) } // Move to the next view. - set_current_view(get_thd_id(), nvmsg->view); - - // Reset the views for different threads. - uint64_t total_thds = g_batch_threads + g_rem_thread_cnt + 2; + uint64_t total_thds = g_thread_cnt + g_rem_thread_cnt + g_send_thread_cnt; for (uint64_t i = 0; i < total_thds; i++) { - set_newView(i, true); + set_view(i, nvmsg->view); } - // Need to ensure that all the threads of the new primary are ready. - uint64_t count = 0; - bool lflag[total_thds] = {false}; - while (true) - { - for (uint64_t i = 0; i < total_thds; i++) - { - bool nchange = get_newView(i); - if (!nchange && !lflag[i]) - { - count++; - lflag[i] = true; - } - } - - if (count == total_thds) - { - break; - } - } //cout << "new primary changed view" << endl; //fflush(stdout); @@ -654,10 +619,6 @@ RC WorkerThread::run() check_for_timeout(); } - if (g_node_id != get_current_view(get_thd_id())) - { - check_switch_view(); - } #endif // Dequeue a message from its work_queue. @@ -674,21 +635,6 @@ RC WorkerThread::run() idle_starttime = 0; } -#if VIEW_CHANGES - // Ensure that thread 0 of the primary never processes ClientQueryBatch. - if (g_node_id == get_current_view(get_thd_id())) - { - if (msg->rtype == CL_BATCH) - { - if (get_thd_id() == 0) - { - work_queue.enqueue(get_thd_id(), msg, false); - continue; - } - } - } -#endif - // Remove redundant messages. if (exception_msg_handling(msg)) { @@ -725,8 +671,11 @@ RC WorkerThread::run() } } + if (!simulation->is_warmup_done()) + { + stats.set_message_size(msg->rtype, msg->get_size()); + } process(msg); - ready_starttime = get_sys_clock(); if (txn_man) { @@ -768,7 +717,8 @@ bool WorkerThread::is_cc_new_timestamp() * * @param clqry One Client Transaction (or Query). */ -void WorkerThread::init_txn_man(BankingSmartContractMessage *bsc){ +void WorkerThread::init_txn_man(BankingSmartContractMessage *bsc) +{ txn_man->client_id = bsc->return_node_id; txn_man->client_startts = bsc->client_startts; SmartContract *smart_contract; @@ -855,8 +805,11 @@ void WorkerThread::send_execute_msg() */ RC WorkerThread::process_execute_msg(Message *msg) { - //cout << "EXECUTE " << msg->txn_id << " :: " << get_thd_id() <<"\n"; - //fflush(stdout); + // if (msg->txn_id / get_batch_size() % 100 == 0) + // { + // cout << "EXECUTE " << msg->txn_id / get_batch_size() << " THREAD: " << get_thd_id() << "\n"; + // fflush(stdout); + // } uint64_t ctime = get_sys_clock(); @@ -884,6 +837,8 @@ RC WorkerThread::process_execute_msg(Message *msg) // Commit the results. tman->commit(); + INC_STATS(get_thd_id(), txn_cnt, 1); + crsp->copy_from_txn(tman); } @@ -893,18 +848,7 @@ RC WorkerThread::process_execute_msg(Message *msg) for (; i < emsg->end_index; i++) { TxnManager *tman = get_transaction_manager(i, 0); - while (true) - { - bool ready = tman->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(tman); inc_next_index(); @@ -916,6 +860,7 @@ RC WorkerThread::process_execute_msg(Message *msg) crsp->copy_from_txn(tman); + INC_STATS(get_thd_id(), txn_cnt, 1); // Making this txn man available. bool ready = tman->set_ready(); assert(ready); @@ -923,18 +868,7 @@ RC WorkerThread::process_execute_msg(Message *msg) // Last Transaction of the batch. txn_man = get_transaction_manager(i, 0); - while (true) - { - bool ready = txn_man->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(txn_man); inc_next_index(); @@ -951,12 +885,13 @@ RC WorkerThread::process_execute_msg(Message *msg) crsp->copy_from_txn(txn_man); - vector emptyvec; vector dest; dest.push_back(txn_man->client_id); - msg_queue.enqueue(get_thd_id(), crsp, emptyvec, dest); + msg_queue.enqueue(get_thd_id(), crsp, dest); dest.clear(); + INC_STATS(get_thd_id(), txn_cnt, 1); + INC_STATS(_thd_id, tput_msg, 1); INC_STATS(_thd_id, msg_cl_out, 1); @@ -1005,7 +940,7 @@ void WorkerThread::send_checkpoints(uint64_t txn_id) RC WorkerThread::process_pbft_chkpt_msg(Message *msg) { CheckpointMessage *ckmsg = (CheckpointMessage *)msg; - + // printf("CHKPOINT from %ld: %ld\n", msg->return_node_id, msg->txn_id); // Check if message is valid. validate_msg(ckmsg); @@ -1030,6 +965,7 @@ RC WorkerThread::process_pbft_chkpt_msg(Message *msg) // Also update the next checkpoint to the identifier for this message, set_curr_chkpt(msg->txn_id); + // printf("CHKPOINT Done %ld: %ld %ld\n", msg->return_node_id, msg->txn_id,get_curr_chkpt()); // Now we determine what all transaction managers can we release. uint64_t del_range = 0; @@ -1045,8 +981,8 @@ RC WorkerThread::process_pbft_chkpt_msg(Message *msg) } } - //printf("Chkpt: %ld :: LD: %ld :: Del: %ld\n",msg->get_txn_id(), get_last_deleted_txn(), del_range); - //fflush(stdout); + // printf("Chkpt: %ld :: LD: %ld :: Del: %ld Thread:%ld\n",msg->get_txn_id(), get_last_deleted_txn(), del_range, get_thd_id()); + // fflush(stdout); // Release Txn Managers. for (uint64_t i = get_last_deleted_txn(); i < del_range; i++) @@ -1082,6 +1018,7 @@ bool WorkerThread::exception_msg_handling(Message *msg) // Release Messages that arrive after txn completion, except obviously // CL_BATCH as it is never late. + if (msg->rtype != CL_BATCH) { if (msg->rtype != PBFT_CHKPT_MSG) @@ -1127,18 +1064,7 @@ void WorkerThread::set_txn_man_fields(BatchRequests *breq, uint64_t bid) { txn_man = get_transaction_manager(breq->index[i], bid); - while (true) - { - bool ready = txn_man->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(txn_man); txn_man->register_thread(this); txn_man->return_id = breq->return_node_id; @@ -1152,18 +1078,7 @@ void WorkerThread::set_txn_man_fields(BatchRequests *breq, uint64_t bid) } // We need to unset txn_man again for last txn in the batch. - while (true) - { - bool ready = txn_man->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(txn_man); txn_man->set_hash(breq->hash); } @@ -1200,18 +1115,7 @@ void WorkerThread::create_and_send_batchreq(ClientQueryBatch *msg, uint64_t tid) txn_man = get_transaction_manager(txn_id, 0); // Unset this txn man so that no other thread can concurrently use. - while (true) - { - bool ready = txn_man->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(txn_man); txn_man->register_thread(this); txn_man->return_id = msg->return_node; @@ -1233,18 +1137,7 @@ void WorkerThread::create_and_send_batchreq(ClientQueryBatch *msg, uint64_t tid) } // Now we need to unset the txn_man again for the last txn of batch. - while (true) - { - bool ready = txn_man->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(txn_man); // Generating the hash representing the whole batch in last txn man. txn_man->set_hash(calculateHash(batchStr)); @@ -1256,23 +1149,19 @@ void WorkerThread::create_and_send_batchreq(ClientQueryBatch *msg, uint64_t tid) txn_man->set_primarybatch(breq); // Storing all the signatures. - vector emptyvec; - TxnManager *tman = get_transaction_manager(txn_man->get_txn_id() - 2, 0); for (uint64_t i = 0; i < g_node_cnt; i++) { + if (i == g_node_id) { continue; } - breq->sign(i); - tman->allsign.push_back(breq->signature); // Redundant - emptyvec.push_back(breq->signature); } // Send the BatchRequests message to all the other replicas. vector dest = nodes_to_send(0, g_node_cnt); - msg_queue.enqueue(get_thd_id(), breq, emptyvec, dest); - emptyvec.clear(); + msg_queue.enqueue(get_thd_id(), breq, dest); + dest.clear(); } /** Validates the contents of a message. */ diff --git a/system/worker_thread.h b/system/worker_thread.h index f38ab577c..6f25c5c38 100644 --- a/system/worker_thread.h +++ b/system/worker_thread.h @@ -41,24 +41,22 @@ class WorkerThread : public Thread #else void init_txn_man(YCSBClientQueryMessage *msg); #endif -#if EXECUTION_THREAD void send_execute_msg(); RC process_execute_msg(Message *msg); -#endif #if TIMER_ON void add_timer(Message *msg, string qryhash); + void remove_timer(string qryhash); #endif #if VIEW_CHANGES void client_query_check(ClientQueryBatch *clbtch); void check_for_timeout(); - void check_switch_view(); void store_batch_msg(BatchRequests *breq); RC process_view_change_msg(Message *msg); RC process_new_view_msg(Message *msg); void reset(); - void fail_primary(Message *msg, uint64_t batch_to_fail); + void fail_primary(Message *msg, uint64_t time); #endif #if LOCAL_FAULT @@ -70,7 +68,7 @@ class WorkerThread : public Thread bool committed_local(PBFTCommitMessage *msg); RC process_pbft_commit_msg(Message *msg); - + void unset_ready_txn(TxnManager * tman); #if TESTING_ON void testcases(Message *msg); #if TEST_CASE == ONLY_PRIMARY_NO_EXECUTE diff --git a/system/worker_thread_pbft.cpp b/system/worker_thread_pbft.cpp index 67f67f72c..7d1eba2bf 100644 --- a/system/worker_thread_pbft.cpp +++ b/system/worker_thread_pbft.cpp @@ -7,7 +7,6 @@ #include "query.h" #include "ycsb_query.h" #include "math.h" -#include "helper.h" #include "msg_thread.h" #include "msg_queue.h" #include "work_queue.h" @@ -28,11 +27,10 @@ */ RC WorkerThread::process_client_batch(Message *msg) { - //printf("ClientQueryBatch: %ld, THD: %ld :: CL: %ld :: RQ: %ld\n",msg->txn_id, get_thd_id(), msg->return_node_id, clbtch->cqrySet[0]->requests[0]->key); - //fflush(stdout); - ClientQueryBatch *clbtch = (ClientQueryBatch *)msg; + // printf("ClientQueryBatch: %ld, THD: %ld :: CL: %ld :: RQ: %ld\n", msg->txn_id, get_thd_id(), msg->return_node_id, clbtch->cqrySet[0]->requests[0]->key); + // fflush(stdout); // Authenticate the client signature. validate_msg(clbtch); @@ -41,11 +39,12 @@ RC WorkerThread::process_client_batch(Message *msg) if (g_node_id != get_current_view(get_thd_id())) { client_query_check(clbtch); + cout << "returning... " << get_current_view(get_thd_id()) << endl; return RCOK; } // Partial failure of Primary 0. - fail_primary(msg, 9); + fail_primary(msg, 10 * BILLION); #endif // Initialize all transaction mangers and Send BatchRequests message. @@ -73,8 +72,8 @@ RC WorkerThread::process_batch(Message *msg) BatchRequests *breq = (BatchRequests *)msg; - //printf("BatchRequests: TID:%ld : VIEW: %ld : THD: %ld\n",breq->txn_id, breq->view, get_thd_id()); - //fflush(stdout); + // printf("BatchRequests: TID:%ld : VIEW: %ld : THD: %ld\n",breq->txn_id, breq->view, get_thd_id()); + // fflush(stdout); // Assert that only a non-primary replica has received this message. assert(g_node_id != get_current_view(get_thd_id())); @@ -143,7 +142,7 @@ RC WorkerThread::process_batch(Message *msg) { #if TIMER_ON // End the timer for this client batch. - server_timer->endTimer(txn_man->hash); + remove_timer(txn_man->hash); #endif // Proceed to executing this batch of transactions. send_execute_msg(); @@ -167,18 +166,7 @@ RC WorkerThread::process_batch(Message *msg) // UnSetting the ready for the txn id representing this batch. txn_man = get_transaction_manager(msg->txn_id, 0); - while (true) - { - bool ready = txn_man->unset_ready(); - if (!ready) - { - continue; - } - else - { - break; - } - } + unset_ready_txn(txn_man); return RCOK; } @@ -303,7 +291,7 @@ RC WorkerThread::process_pbft_commit_msg(Message *msg) { #if TIMER_ON // End the timer for this client batch. - server_timer->endTimer(txn_man->hash); + remove_timer(txn_man->hash); #endif // Add this message to execute thread's queue. diff --git a/transport/message.cpp b/transport/message.cpp index ba29d27b8..6a0b0299b 100644 --- a/transport/message.cpp +++ b/transport/message.cpp @@ -101,7 +101,7 @@ Message *Message::create_message(RemReqType rtype) case READY: msg = new ReadyServer; break; - #if BANKING_SMART_CONTRACT +#if BANKING_SMART_CONTRACT case BSC_MSG: msg = new BankingSmartContractMessage; break; @@ -960,7 +960,6 @@ bool ClientResponseMessage::validate() //cout << "IN: " << this->txn_id << " :: " << this->return_node_id << "\n"; //fflush(stdout); - return true; } @@ -1252,16 +1251,26 @@ bool ClientQueryBatch::validate() #if USE_CRYPTO string message = this->getString(); - //cout << "Recv Message: " << message << " :: " << this->return_node_id << endl; - //cout << "Recv Signature: " << this->signature << " :: " << signature.size() << endl; - //fflush(stdout); - +#if VIEW_CHANGES + // ===================== + // Sign Bug for forwarded messages + uint64_t source_node_id = this->return_node; + if (this->return_node_id < g_node_cnt) + source_node_id = this->return_node_id; + // ===================== + if (!validateClientNode(message, this->pubKey, this->signature, source_node_id)) + { + assert(0); + return false; + } +#else // make sure signature is valid if (!validateClientNode(message, this->pubKey, this->signature, this->return_node)) { assert(0); return false; } +#endif #endif return true; } @@ -1308,7 +1317,7 @@ void BatchRequests::copy_from_txn(TxnManager *txn, BankingSmartContractMessage * uint64_t idx = txnid % get_batch_size(); // TODO: Some memory is getting consumed while storing client query. - char *bfr = (char *)malloc(clqry->get_size() + 1); + char *bfr = (char *)malloc(clqry->get_size()); clqry->copy_to_buf(bfr); Message *tmsg = Message::create_message(bfr); BankingSmartContractMessage *yqry = (BankingSmartContractMessage *)tmsg; @@ -1325,7 +1334,7 @@ void BatchRequests::copy_from_txn(TxnManager *txn, YCSBClientQueryMessage *clqry uint64_t idx = txnid % get_batch_size(); // TODO: Some memory is getting consumed while storing client query. - char *bfr = (char *)malloc(clqry->get_size() + 1); + char *bfr = (char *)malloc(clqry->get_size()); clqry->copy_to_buf(bfr); Message *tmsg = Message::create_message(bfr); YCSBClientQueryMessage *yqry = (YCSBClientQueryMessage *)tmsg; @@ -1726,7 +1735,7 @@ uint64_t Message::buf_to_string(char *buf, uint64_t ptr, string &str, uint64_t s // Message Creation methods. char *create_msg_buffer(Message *msg) { - char *buf = (char *)malloc(msg->get_size() + 1); + char *buf = (char *)malloc(msg->get_size()); return buf; } @@ -1763,7 +1772,7 @@ void PBFTPrepMessage::copy_from_txn(TxnManager *txn) { Message::mcopy_from_txn(txn); - this->view = get_current_view(local_view[txn->get_thd_id()]); + this->view = get_current_view(txn->get_thd_id()); this->end_index = txn->get_txn_id(); this->index = this->end_index + 1 - get_batch_size(); this->hash = txn->get_hash(); @@ -1880,7 +1889,7 @@ void PBFTCommitMessage::copy_from_txn(TxnManager *txn) { Message::mcopy_from_txn(txn); - this->view = get_current_view(local_view[txn->get_thd_id()]); + this->view = get_current_view(txn->get_thd_id()); this->end_index = txn->get_txn_id(); this->index = this->end_index + 1 - get_batch_size(); this->hash = txn->get_hash(); @@ -1968,6 +1977,7 @@ bool PBFTCommitMessage::validate() string message = this->toString(); #if USE_CRYPTO + //verify signature of message if (!validateNodeNode(message, this->pubKey, this->signature, this->return_node_id)) { @@ -1975,7 +1985,6 @@ bool PBFTCommitMessage::validate() return false; } #endif - return true; } @@ -2017,12 +2026,12 @@ void ViewChangeMsg::init(uint64_t thd_id, TxnManager *txn) this->index = txn->get_txn_id(); // Last checkpoint idx. this->return_node = g_node_id; - cout << "Checkpoint: " << this->index << "\n"; + // cout << "Checkpoint: " << this->index << "\n"; fflush(stdout); // Start collecting from the next batch, but as curr_next_index would // only point to the last request in the batch, so we add required amt - uint64_t st = this->index + 4 + get_batch_size(); + uint64_t st = get_commit_message_txn_id(this->index + get_batch_size()); bool found; uint64_t j = 0; @@ -2059,10 +2068,25 @@ void ViewChangeMsg::init(uint64_t thd_id, TxnManager *txn) for (uint64_t i = st; i <= curr_next_index(); i += get_batch_size()) { TxnManager *tman = txn_table.get_transaction_manager(thd_id, i, 0); + while (true) + { + bool ready = tman->unset_ready(); + if (!ready) + { + printf("trying to get txn_man %ld\n", tman->get_txn_id()); + continue; + } + else + { + break; + } + } prepView.push_back(get_current_view(thd_id)); prepIdx.push_back(i); prepHash.push_back(tman->get_hash()); prepHsize.push_back(tman->get_hashSize()); + bool ready = tman->set_ready(); + assert(ready); } this->numPreMsgs = this->preMsg.size(); @@ -2087,10 +2111,10 @@ void ViewChangeMsg::copy_from_buf(char *buf) preMsg.push_back((BatchRequests *)msg); } - uint64_t tval; - string hsh; for (uint64_t i = 0; i < numPrepMsgs; i++) { + uint64_t tval; + string hsh; COPY_VAL(tval, buf, ptr); prepView.push_back(tval); @@ -2242,11 +2266,13 @@ bool ViewChangeMsg::validate(uint64_t thd_id) for (uint64_t i = 0; i < preMsg.size(); i++) { bmsg = preMsg[i]; + /* if (!bmsg->validate(thd_id)) { assert(0); return false; } + */ count = 0; for (uint64_t j = 0; j < prepIdx.size(); j++) @@ -2277,7 +2303,7 @@ string ViewChangeMsg::toString() for (uint i = 0; i < preMsg.size(); i++) { - message += preMsg[i]->getString(g_node_id) + '_'; + message += preMsg[i]->getString(this->return_node) + '_'; } for (uint64_t i = 0; i < prepIdx.size(); i++) @@ -2304,11 +2330,12 @@ void NewViewMsg::init(uint64_t thd_id) { vmsg = view_change_msgs[i]; - cout << "View MSG: " << vmsg->index << "\n"; - fflush(stdout); + // cout << "View MSG: " << vmsg->index << "\n"; + // fflush(stdout); char *buf = create_msg_buffer(vmsg); Message *deepCMsg = deep_copy_msg(buf, vmsg); + deepCMsg->return_node_id = vmsg->return_node_id; this->viewMsg.push_back((ViewChangeMsg *)deepCMsg); delete_msg_buffer(buf); } @@ -2325,6 +2352,7 @@ void NewViewMsg::init(uint64_t thd_id) pmsg = vmsg->preMsg[i]; char *buf = create_msg_buffer(pmsg); Message *deepCMsg = deep_copy_msg(buf, pmsg); + deepCMsg->return_node_id = pmsg->return_node_id; this->preMsg.push_back((BatchRequests *)deepCMsg); delete_msg_buffer(buf); } @@ -2439,7 +2467,7 @@ string NewViewMsg::toString() for (uint64_t i = 0; i < preMsg.size(); i++) { - message += preMsg[i]->getString(g_node_id); + message += preMsg[i]->getString(this->return_node_id); message += '_'; } diff --git a/transport/message.h b/transport/message.h index 3b1179b8e..0d8883074 100644 --- a/transport/message.h +++ b/transport/message.h @@ -2,7 +2,6 @@ #define _MESSAGE_H_ #include "global.h" -#include "helper.h" #include "array.h" #include @@ -365,7 +364,6 @@ char *create_msg_buffer(Message *msg); Message *deep_copy_msg(char *buf, Message *msg); void delete_msg_buffer(char *buf); - class PBFTPrepMessage : public Message { public: diff --git a/transport/msg_thread.cpp b/transport/msg_thread.cpp index c8f7d3fe3..323617699 100644 --- a/transport/msg_thread.cpp +++ b/transport/msg_thread.cpp @@ -68,7 +68,7 @@ void MessageThread::run() } return; } - if (idle_starttime > 0) + if (idle_starttime > 0 && simulation->is_warmup_done()) { output_thd_idle_time[td_id] += get_sys_clock() - idle_starttime; idle_starttime = 0; @@ -107,9 +107,17 @@ void MessageThread::run() if (!sbuf->fits(msg->get_size())) { assert(sbuf->cnt > 0); + cout << "not fitting " << sbuf->cnt << endl; send_batch(dest_node_id); } - +#if VIEW_CHANGES + if (msg->rtype == VIEW_CHANGE) + sbuf->force = true; + if (msg->rtype == NEW_VIEW) + sbuf->force = true; +#endif + if (msg->rtype == PBFT_CHKPT_MSG) + sbuf->force = true; msg->copy_to_buf(&(sbuf->buffer[sbuf->ptr])); sbuf->cnt += 1; diff --git a/transport/msg_thread.h b/transport/msg_thread.h index df3a8be65..8cf510dd8 100644 --- a/transport/msg_thread.h +++ b/transport/msg_thread.h @@ -2,7 +2,6 @@ #define _MSG_THREAD_H_ #include "global.h" -#include "helper.h" #include "nn.hpp" struct mbuf @@ -28,6 +27,7 @@ struct mbuf ((uint32_t *)buffer)[0] = dest_id; ((uint32_t *)buffer)[1] = g_node_id; ptr = sizeof(uint32_t) * 3; + dest_node_id = dest_id; } void copy(char *p, uint64_t s) { @@ -44,10 +44,10 @@ struct mbuf { if (simulation->is_warmup_done() && ISSERVER) { - if (cnt == MESSAGE_PER_BUFFER || (force && cnt)) - { + if (cnt == MESSAGE_PER_BUFFER || (force && cnt)){ force = false; return true; + } return false; } diff --git a/transport/transport.cpp b/transport/transport.cpp index fa6653af8..92454297e 100644 --- a/transport/transport.cpp +++ b/transport/transport.cpp @@ -88,6 +88,7 @@ uint64_t Transport::get_port_id(uint64_t src_node_id, uint64_t dest_node_id, uin // port_id *= max_send_thread_cnt; port_id += send_thread_id * g_total_node_cnt * g_total_node_cnt; DEBUG("%ld\n", port_id); + port_id = port_id % TPORT_WINDOW; port_id += TPORT_PORT; DEBUG("%ld\n", port_id); printf("Port ID: %ld, %ld -> %ld : %ld\n", send_thread_id, src_node_id, dest_node_id, port_id); @@ -160,6 +161,12 @@ Socket *Transport::connect(uint64_t dest_id, uint64_t port_id) void Transport::init() { + + /* + * S, R number of seding and receving threads + * Create S listening sockets for each node in the system, put them in queues for receiveing threads + * Crate S sending sockets pair with dest_id and put them in hash map that keys are (node_id, thd_id) and value is socket + */ _sock_cnt = get_socket_count(); rr = 0; @@ -193,9 +200,10 @@ void Transport::init() } else { - for (uint64_t server_thread_id = g_thread_cnt + g_rem_thread_cnt; server_thread_id < g_thread_cnt + g_rem_thread_cnt + g_send_thread_cnt; server_thread_id++) + // for on SEND_THREAD_CNT + for (uint64_t server_thread_id = 0; server_thread_id < g_send_thread_cnt; server_thread_id++) { - uint64_t port_id = get_port_id(node_id, g_node_id, server_thread_id % g_send_thread_cnt); + uint64_t port_id = get_port_id(node_id, g_node_id, server_thread_id); Socket *sock = bind(port_id); // Sockets for clients and servers in different sets. if (!ISSERVER) @@ -204,14 +212,7 @@ void Transport::init() } else { - if (node_id % (g_this_rem_thread_cnt - 1) == 0) - { - recv_sockets_servers1.push_back(sock); - } - else - { - recv_sockets_servers2.push_back(sock); - } + recv_sockets_servers[node_id % (g_rem_thread_cnt - 1)].push_back(sock); } DEBUG("Socket insert: {%ld}: %ld\n", node_id, (uint64_t)sock); } @@ -251,8 +252,8 @@ void Transport::send_msg(uint64_t send_thread_id, uint64_t dest_node_id, void *s Socket *socket = send_sockets.find(std::make_pair(dest_node_id, send_thread_id))->second; // Copy messages to nanomsg buffer - DEBUG("%ld Sending batch of %d bytes to node %ld on socket %ld\n", send_thread_id, size, dest_node_id, (uint64_t)socket); + INC_GLOB_STATS_ARR(bytes_sent, dest_node_id, size); #if VIEW_CHANGES || LOCAL_FAULT bool failednode = false; @@ -288,18 +289,17 @@ void Transport::send_msg(uint64_t send_thread_id, uint64_t dest_node_id, void *s if (!failednode) { - uint64_t ptr = 0; - RemReqType rtype; - //COPY_VAL(rtype,(char*)buf,ptr); - ptr += sizeof(rtype); - ptr += sizeof(rtype); - ptr += sizeof(rtype); - memcpy(&rtype, &((char *)buf)[ptr], sizeof(rtype)); - //cout << rtype << endl; + // uint64_t ptr = 0; + // RemReqType rtype; + // ptr += sizeof(rtype); + // ptr += sizeof(rtype); + // ptr += sizeof(rtype); + // memcpy(&rtype, &((char *)buf)[ptr], sizeof(rtype)); + // cout << rtype << endl; int rc = -1; uint64_t time = get_sys_clock(); - while ((rc < 0 && (get_sys_clock() - time < MSG_TIMEOUT || !simulation->is_setup_done())) && (!simulation->is_setup_done() || (simulation->is_setup_done() && !simulation->is_done()))) + while ((rc < 0 && (get_sys_clock() - time < MSG_TIMEOUT || !simulation->is_setup_done())) && (!simulation->is_setup_done() || !simulation->is_done())) { rc = socket->sock.send(sbuf, size, NNG_FLAG_NONBLOCK); } @@ -323,7 +323,7 @@ void Transport::send_msg(uint64_t send_thread_id, uint64_t dest_node_id, void *s } #else int rc = -1; - while ((rc < 0 && (!simulation->is_setup_done() || (simulation->is_setup_done() && !simulation->is_done())))) + while (rc < 0 && (!simulation->is_setup_done() || !simulation->is_done())) { rc = socket->sock.send(sbuf, size, NNG_FLAG_NONBLOCK); } @@ -367,15 +367,12 @@ std::vector *Transport::recv_msg(uint64_t thd_id) // One thread manages client sockets, while others handles server sockets. if (thd_id % g_this_rem_thread_cnt == 0) { - ctr = get_next_socket(thd_id, recv_sockets_servers1.size()); - } - else if (thd_id % g_this_rem_thread_cnt == 1) - { - ctr = get_next_socket(thd_id, recv_sockets_servers2.size()); + ctr = get_next_socket(thd_id, recv_sockets_clients.size()); } else { - ctr = get_next_socket(thd_id, recv_sockets_clients.size()); + uint64_t abs_tid = thd_id % (g_rem_thread_cnt - 1); + ctr = get_next_socket(thd_id, recv_sockets_servers[abs_tid].size()); } } start_ctr = ctr; @@ -399,15 +396,12 @@ std::vector *Transport::recv_msg(uint64_t thd_id) { // Only servers. if (thd_id % g_this_rem_thread_cnt == 0) { - socket = recv_sockets_servers1[ctr]; - } - else if (thd_id % g_this_rem_thread_cnt == 1) - { - socket = recv_sockets_servers2[ctr]; + socket = recv_sockets_clients[ctr]; } else { - socket = recv_sockets_clients[ctr]; + uint64_t abs_tid = thd_id % (g_rem_thread_cnt - 1); + socket = recv_sockets_servers[abs_tid][ctr]; } bytes = socket->sock.recv(&buf, NNG_FLAG_ALLOC | NNG_FLAG_NONBLOCK); } @@ -422,15 +416,12 @@ std::vector *Transport::recv_msg(uint64_t thd_id) { if (thd_id % g_this_rem_thread_cnt == 0) { - ctr = get_next_socket(thd_id, recv_sockets_servers1.size()); - } - else if (thd_id % g_this_rem_thread_cnt == 1) - { - ctr = get_next_socket(thd_id, recv_sockets_servers2.size()); + ctr = get_next_socket(thd_id, recv_sockets_clients.size()); } else { - ctr = get_next_socket(thd_id, recv_sockets_clients.size()); + uint64_t abs_tid = thd_id % (g_rem_thread_cnt - 1); + ctr = get_next_socket(thd_id, recv_sockets_servers[abs_tid].size()); } if (ctr == start_ctr) break; diff --git a/transport/transport.h b/transport/transport.h index f0c3fe91f..75578fc65 100644 --- a/transport/transport.h +++ b/transport/transport.h @@ -47,8 +47,7 @@ class Transport // To be used replicas. std::vector recv_sockets_clients; - std::vector recv_sockets_servers1; - std::vector recv_sockets_servers2; + std::vector recv_sockets_servers[REM_THREAD_CNT - 1]; uint64_t _node_cnt; uint64_t _sock_cnt; From cf35913ee89fa87f463dec36443694072c49aba7 Mon Sep 17 00:00:00 2001 From: Sajjad Rahnama Date: Wed, 29 Sep 2021 23:38:18 -0700 Subject: [PATCH 2/3] GeoBFT --- config.h | 9 +- data_structures/ring_all_comb.h | 48 ---- scripts/startResilientDB.sh | 18 +- statistics/stats.cpp | 20 +- statistics/stats.h | 3 + system/client_thread.cpp | 50 +++- system/global.cpp | 88 ++++++- system/global.h | 27 ++- system/main.cpp | 5 + system/msg_queue.cpp | 24 +- system/txn.cpp | 41 +++- system/work_queue.cpp | 23 +- system/work_queue.h | 7 +- system/worker_thread.cpp | 61 ++++- system/worker_thread.h | 5 + system/worker_thread_gbft.cpp | 395 ++++++++++++++++++++++++++++++++ system/worker_thread_pbft.cpp | 2 + transport/message.cpp | 39 +++- transport/message.h | 36 ++- transport/message_gbft.cpp | 264 +++++++++++++++++++++ transport/msg_thread.cpp | 6 + 21 files changed, 1077 insertions(+), 94 deletions(-) delete mode 100644 data_structures/ring_all_comb.h create mode 100644 system/worker_thread_gbft.cpp create mode 100644 transport/message_gbft.cpp diff --git a/config.h b/config.h index 5ac0395a9..572cb9ea8 100644 --- a/config.h +++ b/config.h @@ -1,7 +1,7 @@ #ifndef _CONFIG_H_ #define _CONFIG_H_ // Specify the number of servers or replicas -#define NODE_CNT 12 +#define NODE_CNT 4 // Number of worker threads at primary. #define THREAD_CNT 5 // This Should be the sum of following thread count + protocol specifig threads #define WORKER_THREAD_CNT 1 @@ -14,7 +14,7 @@ #define CORE_CNT 8 #define PART_CNT 1 // Specify the number of clients. -#define CLIENT_NODE_CNT 3 +#define CLIENT_NODE_CNT 1 #define CLIENT_THREAD_CNT 2 #define CLIENT_REM_THREAD_CNT 1 #define CLIENT_SEND_THREAD_CNT 1 @@ -22,6 +22,11 @@ #define MESSAGE_PER_BUFFER 24 +// GeoBFT Setting +#define GBFT false +#define GBFT_CLUSTER_SIZE 4 +#define GBFT_CCM_THREAD_CNT 1 + #define LOAD_PER_SERVER 1 #define REPLICA_CNT 0 #define REPL_TYPE AP diff --git a/data_structures/ring_all_comb.h b/data_structures/ring_all_comb.h deleted file mode 100644 index 54960bb03..000000000 --- a/data_structures/ring_all_comb.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef _RING_ALL_COMB_H_ -#define _RING_ALL_COMB_H_ - - -#include /* printf, scanf, puts, NULL */ -#include /* srand, rand */ -#include /* time */ -#include -#include - -class AllComb -{ -private: - uint64_t number_of_invloved_shards; - uint64_t gnode_id; - uint64_t shards_count; - -public: - std::vector> output; - AllComb(uint64_t inv, uint64_t gnode, uint64_t sh_cnt) : number_of_invloved_shards(inv), gnode_id(gnode), shards_count(sh_cnt) {} - - std::vector> combine() - { - std::vector curr; - back_track(0, curr); - return output; - } - - void back_track(uint64_t start, std::vector &curr) - { - if (curr.size() == number_of_invloved_shards) - { - std::vector temp = curr; - output.push_back(temp); - return; - } - for (uint64_t i = start; i < shards_count; i++) - { - if (i == gnode_id) - continue; - curr.push_back(i); - back_track(start + 1, curr); - curr.pop_back(); - } - } -}; - -#endif \ No newline at end of file diff --git a/scripts/startResilientDB.sh b/scripts/startResilientDB.sh index 924351b5d..d21be12e1 100755 --- a/scripts/startResilientDB.sh +++ b/scripts/startResilientDB.sh @@ -22,10 +22,10 @@ SNODES=( "10.0.0.223" "10.0.0.87" "10.0.0.90" - "10.0.0.20" - "10.0.0.192" - "10.0.0.19" - "10.0.0.226" + # "10.0.0.20" + # "10.0.0.192" + # "10.0.0.19" + # "10.0.0.226" # "10.0.0.218" # "10.0.0.140" # "10.0.0.178" @@ -41,13 +41,13 @@ CNODES=( # "10.0.0.223" # "10.0.0.87" # "10.0.0.90" - # "10.0.0.20" - # "10.0.0.192" + "10.0.0.20" + "10.0.0.192" # "10.0.0.19" # "10.0.0.226" - "10.0.0.218" - "10.0.0.140" - "10.0.0.178" + # "10.0.0.218" + # "10.0.0.140" + # "10.0.0.178" # "10.0.0.209" ) diff --git a/statistics/stats.cpp b/statistics/stats.cpp index a3d97319d..f7f264571 100644 --- a/statistics/stats.cpp +++ b/statistics/stats.cpp @@ -987,6 +987,11 @@ void Stats::set_message_size(uint64_t rtype, uint64_t size) this->prepare_msg_size = size; break; #endif +#if GBFT + case GBFT_COMMIT_CERTIFICATE_MSG: + this->gbft_ccm_msg_size = size; + break; +#endif default: break; @@ -1000,17 +1005,20 @@ void Stats::print_msg_sizes(FILE *outf) #if CONSENSUS == PBFT "msg_size_commit=%ld\n" "msg_size_prepare=%ld\n" +#endif +#if GBFT + "msg_size_gbft_ccm=%ld\n" #endif "msg_size_checkpoint=%ld\n" - "msg_size_client_response=%ld\n" + "msg_size_client_response=%ld\n", - , - client_batch_msg_size, batch_req_msg_size + client_batch_msg_size, batch_req_msg_size, #if CONSENSUS == PBFT - , - commit_msg_size, prepare_msg_size + commit_msg_size, prepare_msg_size, +#endif +#if GBFT + gbft_ccm_msg_size, #endif - , checkpoint_msg_size, client_response_msg_size ); diff --git a/statistics/stats.h b/statistics/stats.h index 20819a22d..27b83d228 100644 --- a/statistics/stats.h +++ b/statistics/stats.h @@ -253,6 +253,9 @@ class Stats #if CONSENSUS == PBFT uint64_t commit_msg_size; uint64_t prepare_msg_size; +#endif +#if GBFT + uint64_t gbft_ccm_msg_size; #endif uint64_t checkpoint_msg_size; uint64_t client_response_msg_size; diff --git a/system/client_thread.cpp b/system/client_thread.cpp index 2a76f0bbb..269770f64 100644 --- a/system/client_thread.cpp +++ b/system/client_thread.cpp @@ -11,7 +11,6 @@ #include "wl.h" #include "message.h" #include "timer.h" -#include "ring_all_comb.h" void ClientThread::send_key() { @@ -67,7 +66,6 @@ void ClientThread::setup() commonVar++; batchMTX.unlock(); - if (_thd_id == 0) { while (commonVar < g_client_thread_cnt + g_client_rem_thread_cnt + g_client_send_thread_cnt) @@ -121,7 +119,11 @@ RC ClientThread::run() int32_t inf_cnt; uint32_t next_node = get_client_view(); +#if GBFT + next_node_id = view_to_primary(get_client_view(get_cluster_number())); +#else next_node_id = get_client_view(); +#endif #if VIEW_CHANGES //if a request by this client hasnt been completed in time @@ -253,3 +255,47 @@ RC ClientThread::run() return FINISH; } +#if VIEW_CHANGES +#if GBFT +// Resend message to all the servers. +void ClientThread::resend_msg(ClientQueryBatch *symsg) +{ + //cout << "Resend: " << symsg->cqrySet[get_batch_size()-1]->client_startts << "\n"; + //fflush(stdout); + + char *buf = create_msg_buffer(symsg); + uint64_t first_node_in_cluster = get_cluster_number(g_node_id) * gbft_cluster_size; + for (uint64_t j = first_node_in_cluster; j < first_node_in_cluster + gbft_cluster_size; j++) + { + vector dest; + dest.push_back(j); + + Message *deepCMsg = deep_copy_msg(buf, symsg); + msg_queue.enqueue(get_thd_id(), deepCMsg, dest); + dest.clear(); + } + delete_msg_buffer(buf); + Message::release_message(symsg); +} +#else +// Resend message to all the servers. +void ClientThread::resend_msg(ClientQueryBatch *symsg) +{ + //cout << "Resend: " << symsg->cqrySet[get_batch_size()-1]->client_startts << "\n"; + //fflush(stdout); + + char *buf = create_msg_buffer(symsg); + for (uint64_t j = 0; j < g_node_cnt; j++) + { + vector dest; + dest.push_back(j); + + Message *deepCMsg = deep_copy_msg(buf, symsg); + msg_queue.enqueue(get_thd_id(), deepCMsg, dest); + dest.clear(); + } + delete_msg_buffer(buf); + Message::release_message(symsg); +} +#endif +#endif // VIEW_CHANGES \ No newline at end of file diff --git a/system/global.cpp b/system/global.cpp index c1bb5a3be..d4c5bcfc4 100644 --- a/system/global.cpp +++ b/system/global.cpp @@ -69,7 +69,6 @@ UInt32 g_batching_thread_cnt = BATCH_THREAD_CNT; UInt32 g_checkpointing_thread_cnt = CHECKPOINT_THREAD_CNT; UInt32 g_execution_thread_cnt = EXECUTE_THREAD_CNT; - #if SIGN_THREADS UInt32 g_sign_thd = SIGN_THD_CNT; #else @@ -148,7 +147,11 @@ bool keyAvail = false; uint64_t totKey = 0; uint64_t indexSize = 2 * g_client_node_cnt * g_inflight_max; +#if GBFT +uint64_t g_min_invalid_nodes = (gbft_cluster_size - 1) / 3; //min number of valid nodes +#else uint64_t g_min_invalid_nodes = (g_node_cnt - 1) / 3; //min number of valid nodes +#endif // Funtion to calculate hash of a string. string calculateHash(string str) @@ -192,8 +195,8 @@ uint64_t txn_per_chkpt() void set_curr_chkpt(uint64_t txn_id) { - if(txn_id > g_last_stable_chkpt) - g_last_stable_chkpt = txn_id; + if (txn_id > g_last_stable_chkpt) + g_last_stable_chkpt = txn_id; } uint64_t get_curr_chkpt() @@ -231,6 +234,16 @@ uint commonVar = 0; // Variable used by Input thread at the primary to linearize batches. uint64_t next_idx = 0; +#if GBFT +uint64_t get_and_inc_next_idx() +{ + next_idx_lock.lock(); + uint64_t val = next_idx; + next_idx += gbft_cluster_cnt; + next_idx_lock.unlock(); + return val; +} +#else uint64_t get_and_inc_next_idx() { next_idx_lock.lock(); @@ -238,6 +251,7 @@ uint64_t get_and_inc_next_idx() next_idx_lock.unlock(); return val; } +#endif void set_next_idx(uint64_t val) { @@ -315,13 +329,78 @@ uint64_t get_batch_size() { return g_batch_size; } +#if !GBFT +uint64_t view_to_primary(uint64_t view, uint64_t node) +{ + return view; +} +#endif + +#if GBFT +SpinLockSet gbft_ccm_checklist; +uint32_t last_commited_txn = 0; +UInt32 gbft_cluster_size = GBFT_CLUSTER_SIZE; +UInt32 gbft_cluster_cnt = NODE_CNT / GBFT_CLUSTER_SIZE; +UInt32 gbft_commit_certificate_thread_cnt = GBFT_CCM_THREAD_CNT; + +// This variable is mainly used by the client to know its current primary. +uint32_t g_view[NODE_CNT / GBFT_CLUSTER_SIZE] = {0}; +std::mutex viewMTX[NODE_CNT / GBFT_CLUSTER_SIZE]; +void set_client_view(uint64_t nview, int cluster) +{ + viewMTX[cluster].lock(); + g_view[cluster] = nview; + viewMTX[cluster].unlock(); +} + +uint64_t get_client_view(int cluster) +{ + uint64_t val; + viewMTX[cluster].lock(); + val = g_view[cluster]; + viewMTX[cluster].unlock(); + return val; +} +bool is_primary_node(uint64_t thd_id, uint64_t node) +{ + return view_to_primary(get_current_view(thd_id), node) == node; +} + +uint64_t get_cluster_number(uint64_t i) +{ + if (i >= g_node_cnt && i < g_node_cnt + g_client_node_cnt) + { + int client_number = i - g_node_cnt; + return (client_number / (g_client_node_cnt / gbft_cluster_cnt)); + } + else + return (i / gbft_cluster_size); +} uint64_t view_to_primary(uint64_t view, uint64_t node) { - return view; + return get_cluster_number(node) * gbft_cluster_size + view; +} + +int is_in_same_cluster(uint64_t first_id, uint64_t second_id) +{ + return (int)(first_id / gbft_cluster_size) == (int)(second_id / gbft_cluster_size); +} +bool is_local_request(TxnManager *tman) +{ + uint64_t node_id = tman->client_id; + assert(node_id < g_total_node_cnt); + int client_number = node_id - g_node_cnt; + int primary = client_number / (g_client_node_cnt / gbft_cluster_cnt); + if (is_in_same_cluster(primary * gbft_cluster_size, g_node_id)) + { + return true; + } + return false; } +#else // This variable is mainly used by the client to know its current primary. uint32_t g_view = 0; std::mutex viewMTX; @@ -340,6 +419,7 @@ uint64_t get_client_view() viewMTX.unlock(); return val; } +#endif #if LOCAL_FAULT || VIEW_CHANGES // Server parameters for tracking failed replicas diff --git a/system/global.h b/system/global.h index cc8f680fb..fa429fdf0 100644 --- a/system/global.h +++ b/system/global.h @@ -56,7 +56,7 @@ class QWorkQueue; class MessageQueue; class Client_query_queue; class Client_txn; -class CommitCertificateMessage; +class GeoBFTCommitCertificateMessage; class ClientResponseMessage; typedef uint32_t UInt32; @@ -211,6 +211,9 @@ enum RemReqType BSC_MSG, #endif +#if GBFT + GBFT_COMMIT_CERTIFICATE_MSG, +#endif PBFT_PREP_MSG, // Prepare PBFT_COMMIT_MSG, // Commit PBFT_CHKPT_MSG // Checkpoint and Garbage Collection @@ -377,11 +380,33 @@ uint64_t get_batch_size(); extern uint64_t batchSet[2 * CLIENT_NODE_CNT * MAX_TXN_IN_FLIGHT]; uint64_t view_to_primary(uint64_t view, uint64_t node = g_node_id); +#if GBFT +extern uint32_t g_view[]; +extern std::mutex viewMTX[]; +uint64_t get_cluster_number(uint64_t i = g_node_id); +void set_client_view(uint64_t nview, int shard = 0); +uint64_t get_client_view(int cluster = 0); +int is_in_same_cluster(uint64_t first_id, uint64_t second_id); +bool is_local_request(TxnManager *tman); +bool is_primary_node(uint64_t thd_id, uint64_t node = g_node_id); + + +// void set_view(uint64_t nview, int cluster = 0); +// uint64_t get_view(int cluster = 0); + +extern uint32_t gbft_last_commited_txn; +extern UInt32 gbft_cluster_size; +extern UInt32 gbft_cluster_cnt; +extern UInt32 gbft_commit_certificate_thread_cnt; +extern SpinLockSet gbft_ccm_checklist; + +#else // This variable is mainly used by the client to know its current primary. extern uint32_t g_view; extern std::mutex viewMTX; void set_client_view(uint64_t nview); uint64_t get_client_view(); +#endif #if LOCAL_FAULT || VIEW_CHANGES // Server parameters for tracking failed replicas diff --git a/system/main.cpp b/system/main.cpp index 82e2002c0..b206bce12 100644 --- a/system/main.cpp +++ b/system/main.cpp @@ -37,6 +37,11 @@ int main(int argc, char *argv[]) #else uint64_t seed = get_sys_clock(); #endif + +#if GBFT + next_idx = g_node_id / gbft_cluster_size; +#endif + srand(seed); printf("Random seed: %ld\n", seed); diff --git a/system/msg_queue.cpp b/system/msg_queue.cpp index 4884152b2..fcb2faba7 100644 --- a/system/msg_queue.cpp +++ b/system/msg_queue.cpp @@ -82,7 +82,7 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector } break; -#if CONSENSUS == PBFT +#if CONSENSUS == PBFT && !GBFT case PBFT_COMMIT_MSG: for (uint64_t i = 0; i < dest.size(); i++) { @@ -91,6 +91,24 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector } break; #endif +#if GBFT + case PBFT_COMMIT_MSG: + ((PBFTCommitMessage *)msg)->sign(dest[0]); + for (uint64_t i = 0; i < dest.size(); i++) + { + entry->allsign.push_back(((PBFTCommitMessage *)msg)->signature); + } + + break; + case GBFT_COMMIT_CERTIFICATE_MSG: + if (((GeoBFTCommitCertificateMessage *)msg)->forwarding_from == (uint64_t)-1) + ((GeoBFTCommitCertificateMessage *)msg)->sign(dest[0]); + for (uint64_t i = 0; i < dest.size(); i++) + { + entry->allsign.push_back(((GeoBFTCommitCertificateMessage *)msg)->signature); + } + break; +#endif #if VIEW_CHANGES case VIEW_CHANGE: @@ -140,6 +158,10 @@ void MessageQueue::enqueue(uint64_t thd_id, Message *msg, const vector case PBFT_PREP_MSG: case PBFT_COMMIT_MSG: +#if GBFT + case GBFT_COMMIT_CERTIFICATE_MSG: +#endif + #if VIEW_CHANGES case VIEW_CHANGE: case NEW_VIEW: diff --git a/system/txn.cpp b/system/txn.cpp index 57af5598c..ede85cce4 100644 --- a/system/txn.cpp +++ b/system/txn.cpp @@ -255,7 +255,7 @@ void TxnManager::commit_stats() txn_stats.commit_stats(get_thd_id(), get_txn_id(), get_batch_id(), timespan_long, timespan_short); return; } - + INC_STATS(get_thd_id(), txn_run_time, timespan_long); INC_STATS(get_thd_id(), single_part_txn_cnt, 1); txn_stats.commit_stats(get_thd_id(), get_txn_id(), get_batch_id(), timespan_long, timespan_short); @@ -415,6 +415,7 @@ void TxnManager::add_commit_msg(PBFTCommitMessage *pcmsg) { char *buf = create_msg_buffer(pcmsg); Message *deepMsg = deep_copy_msg(buf, pcmsg); + deepMsg->return_node_id = pcmsg->return_node_id; commit_msgs.push_back((PBFTCommitMessage *)deepMsg); delete_msg_buffer(buf); } @@ -455,6 +456,12 @@ void TxnManager::send_pbft_prep_msgs() { continue; } +#if GBFT + if (!is_in_same_cluster(i, g_node_id)) + { + continue; + } +#endif dest.push_back(i); } @@ -478,8 +485,6 @@ void TxnManager::send_pbft_commit_msgs() } #endif - - vector dest; for (uint64_t i = 0; i < g_node_cnt; i++) { @@ -487,6 +492,12 @@ void TxnManager::send_pbft_commit_msgs() { continue; } +#if GBFT + if (!is_in_same_cluster(i, g_node_id)) + { + continue; + } +#endif dest.push_back(i); } @@ -498,6 +509,23 @@ void TxnManager::send_pbft_commit_msgs() void TxnManager::release_all_messages(uint64_t txn_id) { +#if GBFT + if ((txn_id + 1) % get_batch_size() == 0 && is_local_request(this)) + { + info_prepare.clear(); + info_commit.clear(); + + Message::release_message(batchreq); + + PBFTCommitMessage *cmsg; + while (commit_msgs.size() > 0) + { + cmsg = (PBFTCommitMessage *)this->commit_msgs[0]; + commit_msgs.erase(commit_msgs.begin()); + Message::release_message(cmsg); + } + } +#else if ((txn_id + 1) % get_batch_size() == 0) { info_prepare.clear(); @@ -513,6 +541,7 @@ void TxnManager::release_all_messages(uint64_t txn_id) Message::release_message(cmsg); } } +#endif } #endif // !TESTING @@ -532,6 +561,12 @@ void TxnManager::send_checkpoint_msgs() { continue; } +#if GBFT + if (!is_in_same_cluster(i, g_node_id)) + { + continue; + } +#endif dest.push_back(i); } diff --git a/system/work_queue.cpp b/system/work_queue.cpp index 36730aeb5..563b72fbe 100644 --- a/system/work_queue.cpp +++ b/system/work_queue.cpp @@ -22,6 +22,10 @@ void QWorkQueue::init() } new_txn_queue = new boost::lockfree::queue(0); checkpoint_queue = new boost::lockfree::queue(0); + +#if GBFT + gbft_ccm_queue = new boost::lockfree::queue(0); +#endif } #if ENABLE_PIPELINE @@ -40,7 +44,6 @@ void QWorkQueue::enqueue(uint64_t thd_id, Message *msg, bool busy) assert(ISSERVER || ISREPLICA); DEBUG("Work Enqueue (%ld,%ld) %d\n", entry->txn_id, entry->batch_id, entry->rtype); - if (msg->rtype == CL_QRY || msg->rtype == CL_BATCH) { // Client Requests to batching queues @@ -68,6 +71,12 @@ void QWorkQueue::enqueue(uint64_t thd_id, Message *msg, bool busy) { push_to_queue(entry, checkpoint_queue); } +#if GBFT + else if (msg->rtype == GBFT_COMMIT_CERTIFICATE_MSG) + { + push_to_queue(entry, gbft_ccm_queue); + } +#endif else { push_to_queue(entry, work_queue); @@ -85,9 +94,11 @@ Message *QWorkQueue::dequeue(uint64_t thd_id) bool valid = false; Message *message = NULL; work_queue_entry *entry = NULL; - +#if GBFT + assert(g_thread_cnt == g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt + g_execution_thread_cnt + gbft_commit_certificate_thread_cnt); +#else assert(g_thread_cnt == g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt + g_execution_thread_cnt); - +#endif // Worker Threads if (thd_id < g_worker_thread_cnt) { @@ -109,6 +120,12 @@ Message *QWorkQueue::dequeue(uint64_t thd_id) uint64_t queue_id = (get_expectedExecuteCount() / get_batch_size()) % indexSize; valid = execution_queues[queue_id]->pop(entry); } +#if GBFT + else if (thd_id < g_worker_thread_cnt + g_batching_thread_cnt + g_checkpointing_thread_cnt + g_execution_thread_cnt + gbft_commit_certificate_thread_cnt) + { + valid = gbft_ccm_queue->pop(entry); + } +#endif if (valid) { diff --git a/system/work_queue.h b/system/work_queue.h index 37f3018ce..fde17ff59 100644 --- a/system/work_queue.h +++ b/system/work_queue.h @@ -18,15 +18,13 @@ struct work_queue_entry uint64_t starttime; }; - - class QWorkQueue { public: void init(); void enqueue(uint64_t thd_id, Message *msg, bool busy); Message *dequeue(uint64_t thd_id); - void push_to_queue(work_queue_entry* entry, boost::lockfree::queue *queue); + void push_to_queue(work_queue_entry *entry, boost::lockfree::queue *queue); private: boost::lockfree::queue **execution_queues; @@ -34,6 +32,9 @@ class QWorkQueue boost::lockfree::queue *new_txn_queue; boost::lockfree::queue *checkpoint_queue; +#if GBFT + boost::lockfree::queue *gbft_ccm_queue; +#endif }; #endif diff --git a/system/worker_thread.cpp b/system/worker_thread.cpp index a72bcd98e..2ceefb0da 100644 --- a/system/worker_thread.cpp +++ b/system/worker_thread.cpp @@ -137,6 +137,11 @@ void WorkerThread::process(Message *msg) case PBFT_COMMIT_MSG: rc = process_pbft_commit_msg(msg); break; +#if GBFT + case GBFT_COMMIT_CERTIFICATE_MSG: + rc = process_gbft_commit_certificate_msg(msg); + break; +#endif default: printf("rtype: %d from %ld\n", msg->get_rtype(), msg->return_node_id); fflush(stdout); @@ -359,7 +364,6 @@ void WorkerThread::check_for_timeout() } } - /* This function causes the forced failure of the primary replica at a desired time. */ void WorkerThread::fail_primary(Message *msg, uint64_t time) @@ -400,7 +404,7 @@ void WorkerThread::client_query_check(ClientQueryBatch *clbtch) // TODO if condition is for experimental purpose: force one view change if (this->has_view_changed()) return; - + // cout << "REQUEST: " << clbtch->return_node_id << "\n"; // fflush(stdout); @@ -565,7 +569,6 @@ RC WorkerThread::process_new_view_msg(Message *msg) set_view(i, nvmsg->view); } - //cout << "new primary changed view" << endl; //fflush(stdout); @@ -814,9 +817,22 @@ RC WorkerThread::process_execute_msg(Message *msg) uint64_t ctime = get_sys_clock(); // This message uses txn man of index calling process_execute. +#if GBFT + + TxnManager *test = get_transaction_manager(msg->txn_id + 1, 0); + bool local_request = is_local_request(test); + ClientResponseMessage *crsp = 0; + if (local_request) + { + Message *rsp = Message::create_message(CL_RSP); + crsp = (ClientResponseMessage *)rsp; + crsp->init(); + } +#else Message *rsp = Message::create_message(CL_RSP); ClientResponseMessage *crsp = (ClientResponseMessage *)rsp; crsp->init(); +#endif ExecuteMessage *emsg = (ExecuteMessage *)msg; @@ -839,7 +855,14 @@ RC WorkerThread::process_execute_msg(Message *msg) INC_STATS(get_thd_id(), txn_cnt, 1); +#if GBFT + if (local_request) + { + crsp->copy_from_txn(tman); + } +#else crsp->copy_from_txn(tman); +#endif } // Transactions (**95 - **98) of the batch. @@ -858,7 +881,14 @@ RC WorkerThread::process_execute_msg(Message *msg) // Commit the results. tman->commit(); +#if GBFT + if (local_request) + { + crsp->copy_from_txn(tman); + } +#else crsp->copy_from_txn(tman); +#endif INC_STATS(get_thd_id(), txn_cnt, 1); // Making this txn man available. @@ -882,13 +912,24 @@ RC WorkerThread::process_execute_msg(Message *msg) // Commit the results. txn_man->commit(); - +#if GBFT + if (local_request) + { + crsp->copy_from_txn(txn_man); + vector dest; + assert(is_local_request(txn_man)); + dest.push_back(txn_man->client_id); + msg_queue.enqueue(get_thd_id(), crsp, dest); + dest.clear(); + } +#else crsp->copy_from_txn(txn_man); vector dest; dest.push_back(txn_man->client_id); msg_queue.enqueue(get_thd_id(), crsp, dest); dest.clear(); +#endif INC_STATS(get_thd_id(), txn_cnt, 1); @@ -1148,18 +1189,26 @@ void WorkerThread::create_and_send_batchreq(ClientQueryBatch *msg, uint64_t tid) // Storing the BatchRequests message. txn_man->set_primarybatch(breq); + vector dest; + // Storing all the signatures. for (uint64_t i = 0; i < g_node_cnt; i++) { +#if GBFT + + if (!is_in_same_cluster(i, g_node_id)) + { + continue; + } +#endif if (i == g_node_id) { continue; } + dest.push_back(i); } - // Send the BatchRequests message to all the other replicas. - vector dest = nodes_to_send(0, g_node_cnt); msg_queue.enqueue(get_thd_id(), breq, dest); dest.clear(); } diff --git a/system/worker_thread.h b/system/worker_thread.h index 6f25c5c38..512856d20 100644 --- a/system/worker_thread.h +++ b/system/worker_thread.h @@ -69,6 +69,11 @@ class WorkerThread : public Thread bool committed_local(PBFTCommitMessage *msg); RC process_pbft_commit_msg(Message *msg); void unset_ready_txn(TxnManager * tman); + +#if GBFT + RC process_gbft_commit_certificate_msg(Message * msg); +#endif + #if TESTING_ON void testcases(Message *msg); #if TEST_CASE == ONLY_PRIMARY_NO_EXECUTE diff --git a/system/worker_thread_gbft.cpp b/system/worker_thread_gbft.cpp new file mode 100644 index 000000000..577f4fd90 --- /dev/null +++ b/system/worker_thread_gbft.cpp @@ -0,0 +1,395 @@ +#include "global.h" +#include "message.h" +#include "thread.h" +#include "worker_thread.h" +#include "txn.h" +#include "wl.h" +#include "query.h" +#include "ycsb_query.h" +#include "math.h" +#include "msg_thread.h" +#include "msg_queue.h" +#include "work_queue.h" +#include "message.h" +#include "timer.h" +#include "chain.h" + +#if CONSENSUS == PBFT && GBFT +/** + * Processes an incoming client batch and sends a Pre-prepare message to al replicas. + * + * This function assumes that a client sends a batch of transactions and + * for each transaction in the batch, a separate transaction manager is created. + * Next, this batch is forwarded to all the replicas as a BatchRequests Message, + * which corresponds to the Pre-Prepare stage in the PBFT protocol. + * + * @param msg Batch of Transactions of type CientQueryBatch from the client. + * @return RC + */ +RC WorkerThread::process_client_batch(Message *msg) +{ + ClientQueryBatch *clbtch = (ClientQueryBatch *)msg; + + // printf("ClientQueryBatch: %ld, THD: %ld :: CL: %ld :: RQ: %ld\n", msg->txn_id, get_thd_id(), msg->return_node_id, clbtch->cqrySet[0]->requests[0]->key); + // fflush(stdout); + + // Authenticate the client signature. + validate_msg(clbtch); + +#if VIEW_CHANGES + // If message forwarded to the non-primary. + if (g_node_id != get_current_view(get_thd_id())) + { + client_query_check(clbtch); + cout << "returning... " << get_current_view(get_thd_id()) << endl; + return RCOK; + } + + // Partial failure of Primary 0. + fail_primary(msg, 10 * BILLION); +#endif + + // Initialize all transaction mangers and Send BatchRequests message. + create_and_send_batchreq(clbtch, clbtch->txn_id); + + return RCOK; +} + +/** + * Process incoming BatchRequests message from the Primary. + * + * This function is used by the non-primary or backup replicas to process an incoming + * BatchRequests message sent by the primary replica. This processing would require + * sending messages of type PBFTPrepMessage, which correspond to the Prepare phase of + * the PBFT protocol. Due to network delays, it is possible that a repica may have + * received some messages of type PBFTPrepMessage and PBFTCommitMessage, prior to + * receiving this BatchRequests message. + * + * @param msg Batch of Transactions of type BatchRequests from the primary. + * @return RC + */ +RC WorkerThread::process_batch(Message *msg) +{ + uint64_t cntime = get_sys_clock(); + + BatchRequests *breq = (BatchRequests *)msg; + + // printf("BatchRequests: TID:%ld : VIEW: %ld : THD: %ld\n",breq->txn_id, breq->view, get_thd_id()); + // fflush(stdout); + + // Assert that only a non-primary replica has received this message. + assert(g_node_id != get_current_view(get_thd_id())); + + // Check if the message is valid. + validate_msg(breq); + +#if VIEW_CHANGES + // Store the batch as it could be needed during view changes. + store_batch_msg(breq); +#endif + + // Allocate transaction managers for all the transactions in the batch. + set_txn_man_fields(breq, 0); + +#if TIMER_ON + // The timer for this client batch stores the hash of last request. + add_timer(breq, txn_man->get_hash()); +#endif + + // Storing the BatchRequests message. + txn_man->set_primarybatch(breq); + + // Send Prepare messages. + txn_man->send_pbft_prep_msgs(); + + // End the counter for pre-prepare phase as prepare phase starts next. + double timepre = get_sys_clock() - cntime; + INC_STATS(get_thd_id(), time_pre_prepare, timepre); + + // Only when BatchRequests message comes after some Prepare message. + for (uint64_t i = 0; i < txn_man->info_prepare.size(); i++) + { + // Decrement. + uint64_t num_prep = txn_man->decr_prep_rsp_cnt(); + if (num_prep == 0) + { + txn_man->set_prepared(); + break; + } + } + + // If enough Prepare messages have already arrived. + if (txn_man->is_prepared()) + { + // Send Commit messages. + txn_man->send_pbft_commit_msgs(); + + double timeprep = get_sys_clock() - txn_man->txn_stats.time_start_prepare - timepre; + INC_STATS(get_thd_id(), time_prepare, timeprep); + double timediff = get_sys_clock() - cntime; + + // Check if any Commit messages arrived before this BatchRequests message. + for (uint64_t i = 0; i < txn_man->info_commit.size(); i++) + { + uint64_t num_comm = txn_man->decr_commit_rsp_cnt(); + if (num_comm == 0) + { + txn_man->set_committed(); + break; + } + } + + // If enough Commit messages have already arrived. + if (txn_man->is_committed()) + { +#if TIMER_ON + // End the timer for this client batch. + remove_timer(txn_man->hash); +#endif + // Proceed to executing this batch of transactions. + send_execute_msg(); + + // End the commit counter. + INC_STATS(get_thd_id(), time_commit, get_sys_clock() - txn_man->txn_stats.time_start_commit - timediff); + } + } + else + { + // Although batch has not prepared, still some commit messages could have arrived. + for (uint64_t i = 0; i < txn_man->info_commit.size(); i++) + { + txn_man->decr_commit_rsp_cnt(); + } + } + + // Release this txn_man for other threads to use. + bool ready = txn_man->set_ready(); + assert(ready); + + // UnSetting the ready for the txn id representing this batch. + txn_man = get_transaction_manager(msg->txn_id, 0); + unset_ready_txn(txn_man); + + return RCOK; +} + +/** + * Processes incoming Prepare message. + * + * This functions precessing incoming messages of type PBFTPrepMessage. If a replica + * received 2f identical Prepare messages from distinct replicas, then it creates + * and sends a PBFTCommitMessage to all the other replicas. + * + * @param msg Prepare message of type PBFTPrepMessage from a replica. + * @return RC + */ +RC WorkerThread::process_pbft_prep_msg(Message *msg) +{ + // cout << "PBFTPrepMessage: TID: " << msg->txn_id << " FROM: " << msg->return_node_id << endl; + // fflush(stdout); + + // Start the counter for prepare phase. + if (txn_man->prep_rsp_cnt == 2 * g_min_invalid_nodes) + { + txn_man->txn_stats.time_start_prepare = get_sys_clock(); + } + + // Check if the incoming message is valid. + PBFTPrepMessage *pmsg = (PBFTPrepMessage *)msg; + validate_msg(pmsg); + + // Check if sufficient number of Prepare messages have arrived. + if (prepared(pmsg)) + { + // Send Commit messages. + txn_man->send_pbft_commit_msgs(); + + // End the prepare counter. + INC_STATS(get_thd_id(), time_prepare, get_sys_clock() - txn_man->txn_stats.time_start_prepare); + } + + return RCOK; +} + +/** + * Checks if the incoming PBFTCommitMessage can be accepted. + * + * This functions checks if the hash and view of the commit message matches that of + * the Pre-Prepare message. Once 2f+1 messages are received it returns a true and + * sets the `is_committed` flag for furtue identification. + * + * @param msg PBFTCommitMessage. + * @return bool True if the transactions of this batch can be executed. + */ +bool WorkerThread::committed_local(PBFTCommitMessage *msg) +{ + // cout << "Check Commit: TID: " << txn_man->get_txn_id() << "\n"; + // fflush(stdout); + + // Once committed is set for this transaction, no further processing. + if (txn_man->is_committed()) + { + return false; + } + + // If BatchRequests messages has not arrived, then hash is empty; return false. + if (txn_man->get_hash().empty()) + { + //cout << "hash empty: " << txn_man->get_txn_id() << "\n"; + //fflush(stdout); + txn_man->info_commit.push_back(msg->return_node); + return false; + } + else + { + if (!checkMsg(msg)) + { + // If message did not match. + //cout << txn_man->get_hash() << " :: " << msg->hash << "\n"; + //cout << get_current_view(get_thd_id()) << " :: " << msg->view << "\n"; + //fflush(stdout); + return false; + } + } + + uint64_t comm_cnt = txn_man->decr_commit_rsp_cnt(); + if (comm_cnt == 0 && txn_man->is_prepared()) + { + txn_man->set_committed(); + return true; + } + + return false; +} + +/** + * Processes incoming Commit message. + * + * This functions precessing incoming messages of type PBFTCommitMessage. If a replica + * received 2f+1 identical Commit messages from distinct replicas, then it asks the + * execute-thread to execute all the transactions in this batch. + * + * @param msg Commit message of type PBFTCommitMessage from a replica. + * @return RC + */ +RC WorkerThread::process_pbft_commit_msg(Message *msg) +{ + // cout << "PBFTCommitMessage: TID " << msg->txn_id / get_batch_size() << " FROM: " << msg->return_node_id << "\n"; + // fflush(stdout); + + if (txn_man->commit_rsp_cnt == 2 * g_min_invalid_nodes + 1) + { + txn_man->txn_stats.time_start_commit = get_sys_clock(); + } + + // Check if message is valid. + PBFTCommitMessage *pcmsg = (PBFTCommitMessage *)msg; + validate_msg(pcmsg); + txn_man->add_commit_msg(pcmsg); + + // Check if sufficient number of Commit messages have arrived. + if (committed_local(pcmsg)) + { +#if TIMER_ON + // End the timer for this client batch. + remove_timer(txn_man->hash); +#endif + +#if GBFT + if (is_primary_node(get_thd_id(), g_node_id) && g_node_cnt > gbft_cluster_size) + { + GeoBFTCommitCertificateMessage *ccm = (GeoBFTCommitCertificateMessage *)Message::create_message(txn_man, GBFT_COMMIT_CERTIFICATE_MSG); + ccm->txn_id = txn_man->get_txn_id(); + + vector dest; + for (uint64_t i = 0; i < g_node_cnt; i++) + { + if (is_in_same_cluster(g_node_id, i) || i % gbft_cluster_size > g_min_invalid_nodes) + continue; + dest.push_back(i); + } + + msg_queue.enqueue(get_thd_id(), ccm, dest); + dest.clear(); + } +#endif + + // Add this message to execute thread's queue. + send_execute_msg(); + + INC_STATS(get_thd_id(), time_commit, get_sys_clock() - txn_man->txn_stats.time_start_commit); + } + + return RCOK; +} + +#if GBFT +RC WorkerThread::process_gbft_commit_certificate_msg(Message *msg) +{ + + GeoBFTCommitCertificateMessage *ccm = (GeoBFTCommitCertificateMessage *)msg; + if (curr_next_index() > ccm->index[0]) + { + return RCOK; + } + + // TxnManager *tman = txn_table.get_transaction_manager(get_thd_id(), ccm->index[g_batch_size - 1], 0); + if (gbft_ccm_checklist.exists(ccm->hash)) + { + return RCOK; + } + + // printf("PROCESS CCM %ld\tFROM: %ld THREAD: %ld\n", msg->txn_id / 100, msg->return_node_id, get_thd_id()); + // fflush(stdout); + + + // ONLY NON PRIMARY NODES. + assert(ccm->validate()); + + if (!is_in_same_cluster(msg->return_node_id, g_node_id)) + { + + char *buf = create_msg_buffer(ccm); + Message *deepCopyMsg = deep_copy_msg(buf, ccm); + ((GeoBFTCommitCertificateMessage *)deepCopyMsg)->forwarding_from = msg->return_node_id; + delete_msg_buffer(buf); + + vector dest; + for (uint64_t i = 0; i < g_node_cnt; i++) + { + if (i == g_node_id || !is_in_same_cluster(g_node_id, i)) + continue; + dest.push_back(i); + } + msg_queue.enqueue(get_thd_id(), deepCopyMsg, dest); + dest.clear(); + } + + // Allocate transaction manager for all the requests in batch. + uint64_t i; + for (i = 0; i < get_batch_size() - 1; i++) + { + uint64_t txn_id = ccm->index[i]; + + txn_man = get_transaction_manager(txn_id, 0); + txn_man->register_thread(this); + txn_man->return_id = msg->return_node_id; + + init_txn_man(ccm->requestMsg[i]); + } + + uint64_t txn_id = ccm->index[i]; + txn_man = get_transaction_manager(txn_id, 0); + txn_man->return_id = msg->return_node_id; + init_txn_man(ccm->requestMsg[i]); + gbft_ccm_checklist.add(ccm->hash); + + Message *emsg = Message::create_message(txn_man, EXECUTE_MSG); + emsg->return_node_id = msg->return_node_id; + work_queue.enqueue(get_thd_id(), emsg, false); + + return RCOK; +} +#endif // GBFT + +#endif // Main GBFT diff --git a/system/worker_thread_pbft.cpp b/system/worker_thread_pbft.cpp index 7d1eba2bf..c0f36e4b5 100644 --- a/system/worker_thread_pbft.cpp +++ b/system/worker_thread_pbft.cpp @@ -14,6 +14,7 @@ #include "timer.h" #include "chain.h" +#if CONSENSUS == PBFT && !GBFT /** * Processes an incoming client batch and sends a Pre-prepare message to al replicas. * @@ -302,3 +303,4 @@ RC WorkerThread::process_pbft_commit_msg(Message *msg) return RCOK; } +#endif diff --git a/transport/message.cpp b/transport/message.cpp index 6a0b0299b..6865fa0e3 100644 --- a/transport/message.cpp +++ b/transport/message.cpp @@ -138,6 +138,12 @@ Message *Message::create_message(RemReqType rtype) break; #endif +#if GBFT + case GBFT_COMMIT_CERTIFICATE_MSG: + msg = new GeoBFTCommitCertificateMessage; + break; +#endif + case PBFT_CHKPT_MSG: msg = new CheckpointMessage; break; @@ -393,6 +399,15 @@ void Message::release_message(Message *msg) delete m_msg; break; } +#if GBFT + case GBFT_COMMIT_CERTIFICATE_MSG: + { + GeoBFTCommitCertificateMessage *m_msg = (GeoBFTCommitCertificateMessage *)msg; + m_msg->release(); + delete m_msg; + break; + } +#endif default: { @@ -1302,8 +1317,12 @@ uint64_t BatchRequests::get_size() // Initialization void BatchRequests::init(uint64_t thd_id) { - // Only primary should create this message +// Only primary should create this message +#if GBFT + assert(is_primary_node(g_node_id)); +#else assert(get_current_view(thd_id) == g_node_id); +#endif this->view = get_current_view(thd_id); this->index.init(get_batch_size()); @@ -1962,8 +1981,12 @@ void PBFTCommitMessage::sign(uint64_t dest_node) #if USE_CRYPTO string message = this->toString(); - //cout << "Signing Commit msg: " << message << endl; +#if GBFT + signingClientNode(message, this->signature, this->pubKey, dest_node); +#else signingNodeNode(message, this->signature, this->pubKey, dest_node); +#endif + #else this->signature = "0"; #endif @@ -1978,13 +2001,21 @@ bool PBFTCommitMessage::validate() #if USE_CRYPTO - //verify signature of message +#if GBFT + if (!validateClientNode(message, this->pubKey, this->signature, this->return_node_id)) + { + assert(0); + return false; + } +#else if (!validateNodeNode(message, this->pubKey, this->signature, this->return_node_id)) { assert(0); return false; } -#endif +#endif // GBFT + +#endif // USE_CRYPTO return true; } diff --git a/transport/message.h b/transport/message.h index 0d8883074..9d9b44d66 100644 --- a/transport/message.h +++ b/transport/message.h @@ -403,14 +403,46 @@ class PBFTCommitMessage : public Message bool validate(); uint64_t view; // primary node id - uint64_t index; // position in sequence of requests + uint64_t index; // position in sequence of requests x00 string hash; //request message digest uint64_t hashSize; //size of hash (for removing from buf) uint64_t return_node; //id of node that sent this message - uint64_t end_index; + uint64_t end_index; // x99 uint64_t batch_size; }; +/****************************************/ +/* GEO BFT SPECIFIC */ +/****************************************/ + +#if GBFT +class GeoBFTCommitCertificateMessage : public Message +{ +public: + void copy_from_buf(char *buf); + void copy_to_buf(char *buf); + void copy_from_txn(TxnManager *txn); + void copy_to_txn(TxnManager *txn); + uint64_t get_size(); + void init() {} + void release(); + + void sign(uint64_t dest_node_id); + bool validate(); + string toString(); + + uint64_t forwarding_from = (uint64_t)-1; + uint64_t view; + Array index; + uint64_t hashSize; + string hash; + + Array signSize; + Array signOwner; + vector requestMsg; + vector signatures; +}; +#endif // GBFT /****************************************/ /* VIEW CHANGE SPECIFIC */ diff --git a/transport/message_gbft.cpp b/transport/message_gbft.cpp new file mode 100644 index 000000000..feddbee9e --- /dev/null +++ b/transport/message_gbft.cpp @@ -0,0 +1,264 @@ +#include "mem_alloc.h" +#include "query.h" +#include "ycsb_query.h" +#include "ycsb.h" +#include "global.h" +#include "message.h" +#include "crypto.h" +#include +#include +#include + +#if GBFT + +uint64_t GeoBFTCommitCertificateMessage::get_size() +{ + uint64_t size = Message::mget_size(); + size += sizeof(view); + size += sizeof(forwarding_from); + size += sizeof(uint64_t) * index.size(); + size += sizeof(hashSize); + size += hash.length(); + size += sizeof(uint64_t) * signOwner.size(); + size += sizeof(uint64_t) * signSize.size(); + for (uint i = 0; i < g_batch_size; i++) + { + size += requestMsg[i]->get_size(); + } + for (uint i = 0; i < (g_min_invalid_nodes * 2); i++) + { + size += signatures[i].length(); + } + return size; +} + +void GeoBFTCommitCertificateMessage::copy_from_txn(TxnManager *txn) +{ + + // assert(is_primary_node(g_node_id)); // only primary creates this message + + // Setting txn_id 2 less than the actual value. + + this->txn_id = txn->get_txn_id() - 2; + + // Initialization + this->index.init(get_batch_size()); + this->requestMsg.resize(get_batch_size()); + this->signatures.resize(g_min_invalid_nodes * 2); + this->signOwner.init(g_min_invalid_nodes * 2); + this->signSize.init(g_min_invalid_nodes * 2); + this->hash = txn->hash; + this->hashSize = txn->hashSize; + + + for (uint64_t txnid = txn->get_txn_id() - g_batch_size + 1; txnid < txn->get_txn_id() + 1; txnid++) + { + //sajjad + TxnManager *t_man = txn_table.get_transaction_manager(0, txnid, 0); + uint64_t idx = txnid % get_batch_size(); + + YCSBQuery *yquery = ((YCSBQuery *)t_man->query); + + YCSBClientQueryMessage *yqry = (YCSBClientQueryMessage *)Message::create_message(CL_QRY); + ((ClientQueryMessage *)yqry)->client_startts = t_man->client_startts; + yqry->return_node = t_man->client_id; + yqry->requests.init(g_req_per_query); + for (uint64_t i = 0; i < g_req_per_query; i ++) { + ycsb_request * req = (ycsb_request*) mem_allocator.alloc(sizeof(ycsb_request)); + req->key = yquery->requests[i]->key; + req->value = yquery->requests[i]->value; + yqry->requests.add(req); + } + this->requestMsg[idx] = yqry; + + this->index.add(txnid); + + } + PBFTCommitMessage *cmsg; + uint64_t i = 0; + for (i = 0; i < txn->commit_msgs.size(); i++) + { + if (i >= (uint64_t)(g_min_invalid_nodes * 2)) + break; + cmsg = txn->commit_msgs[i]; + // One time to fill ccm itself + this->signOwner.add(cmsg->return_node_id); + this->signatures[i] = cmsg->signature; + this->signSize.add(cmsg->signature.length()); + } + this->view = txn->commit_msgs[i]->view; +} + +void GeoBFTCommitCertificateMessage::release() +{ + index.release(); + signOwner.release(); + signSize.release(); + for (size_t i = 0; i < requestMsg.size(); i++) + { + requestMsg[i]->release(); + mem_allocator.free(requestMsg[i], sizeof(YCSBClientQueryMessage)); + } + vector().swap(requestMsg); + vector().swap(signatures); +} + +void GeoBFTCommitCertificateMessage::copy_to_txn(TxnManager *txn) +{ + Message::mcopy_to_txn(txn); +} + +void GeoBFTCommitCertificateMessage::copy_from_buf(char *buf) +{ + Message::mcopy_from_buf(buf); + + uint64_t ptr = Message::mget_size(); + + uint64_t elem; + // Initialization + index.init(g_batch_size); + requestMsg.resize(g_batch_size); + signatures.resize(g_min_invalid_nodes * 2); + signSize.init(g_min_invalid_nodes * 2); + signOwner.init(g_min_invalid_nodes * 2); + + COPY_VAL(elem, buf, ptr); + this->view = elem; + + COPY_VAL(elem, buf, ptr); + this->forwarding_from = elem; + + COPY_VAL(elem, buf, ptr); + this->hashSize = elem; + + string temp_hash; + ptr = buf_to_string(buf, ptr, temp_hash, this->hashSize); + this->hash = temp_hash; + + for (uint i = 0; i < g_batch_size; i++) + { + COPY_VAL(elem, buf, ptr); + index.add(elem); + + Message *msg = create_message(&buf[ptr]); + ptr += msg->get_size(); + requestMsg[i] = ((YCSBClientQueryMessage *)msg); + } + + for (uint i = 0; i < (g_min_invalid_nodes * 2); i++) + { + COPY_VAL(elem, buf, ptr); + signOwner.add(elem); + + COPY_VAL(elem, buf, ptr); + signSize.add(elem); + + string tsign; + ptr = buf_to_string(buf, ptr, tsign, signSize[i]); + signatures[i] = tsign; + } + + assert(ptr == get_size()); +} + +void GeoBFTCommitCertificateMessage::copy_to_buf(char *buf) +{ + Message::mcopy_to_buf(buf); + uint64_t ptr = Message::mget_size(); + + COPY_BUF(buf,view,ptr); + COPY_BUF(buf,forwarding_from,ptr); + COPY_BUF(buf, this->hashSize, ptr); + + string hstr = this->hash; + char v; + for (uint j = 0; j < hstr.size(); j++) + { + v = hstr[j]; + COPY_BUF(buf, v, ptr); + } + + uint64_t elem; + for (uint i = 0; i < g_batch_size; i++) + { + elem = index[i]; + COPY_BUF(buf, elem, ptr); + + //copy client request stored in message to buf + requestMsg[i]->copy_to_buf(&buf[ptr]); + ptr += requestMsg[i]->get_size(); + } + for (uint i = 0; i < (g_min_invalid_nodes * 2); i++) + { + + elem = signOwner[i]; + COPY_BUF(buf, elem, ptr); + elem = signSize[i]; + COPY_BUF(buf, elem, ptr); + char v; + string sstr = signatures[i]; + for (uint j = 0; j < sstr.size(); j++) + { + v = sstr[j]; + COPY_BUF(buf, v, ptr); + } + } + + assert(ptr == get_size()); +} + +void GeoBFTCommitCertificateMessage::sign(uint64_t dest_node_id){ +#if USE_CRYPTO + string message = toString(); + signingClientNode(message, this->signature, this->pubKey, dest_node_id); +#else + this->signature = "0"; +#endif + this->sigSize = this->signature.size(); + this->keySize = this->pubKey.size(); +} + +string GeoBFTCommitCertificateMessage::toString() +{ + string message = ""; + message += this->hash; + message += this->view; + message += this->txn_id; + + return message; +} + +//makes sure message is valid, returns true for false +bool GeoBFTCommitCertificateMessage::validate() +{ +#if USE_CRYPTO + string message = this->toString(); + uint64_t return_node = this->forwarding_from != (uint64_t)-1 ? this->forwarding_from : this->return_node_id; + if (!validateClientNode(message, g_pub_keys[return_node], this->signature, return_node)) + { + assert(0); + } + for (uint64_t i = 0; i < this->signatures.size(); i++) + { + + + string signString = std::to_string(this->view); + signString += '_' + std::to_string(this->txn_id - get_batch_size() + 1); // index + signString += '_' + this->hash; + signString += '_' + std::to_string(this->signOwner[i]); + signString += '_' + to_string(this->txn_id); // end_index + assert(validateClientNode(signString, g_pub_keys[this->signOwner[i]], this->signatures[i], this->signOwner[i])); + } + +#endif + string batchStr = ""; + for(uint64_t i=0; irequestMsg[i]->getString(); + } + if (this->hash != calculateHash(batchStr)){ + assert(0); + } + return true; +} + +#endif diff --git a/transport/msg_thread.cpp b/transport/msg_thread.cpp index 323617699..e0865fe2d 100644 --- a/transport/msg_thread.cpp +++ b/transport/msg_thread.cpp @@ -96,6 +96,12 @@ void MessageThread::run() case CL_BATCH: msg->pubKey = getOtherRequiredKey(dest_node_id); break; +#if GBFT + case GBFT_COMMIT_CERTIFICATE_MSG: + if (((GeoBFTCommitCertificateMessage *)msg)->forwarding_from == (uint64_t)-1) + msg->pubKey = getOtherRequiredKey(dest_node_id); + break; +#endif default: msg->pubKey = getCmacRequiredKey(dest_node_id); } From e2efff960140a2d02a78c9b792af67ff298eeda5 Mon Sep 17 00:00:00 2001 From: Sajjad Rahnama Date: Thu, 30 Sep 2021 11:44:16 -0700 Subject: [PATCH 3/3] Make Config Script --- config.h | 42 +--- scripts/make_config.sh | 416 +++++++++++++++++------------------- scripts/result.sh | 4 +- scripts/result_colorized.sh | 96 ++------- system/global.cpp | 7 +- system/global.h | 1 - transport/message.cpp | 2 - 7 files changed, 218 insertions(+), 350 deletions(-) diff --git a/config.h b/config.h index 572cb9ea8..eae9a3cec 100644 --- a/config.h +++ b/config.h @@ -66,8 +66,6 @@ #define NETWORK_DELAY_TEST false #define NETWORK_DELAY 0UL #define MAX_QUEUE_LEN NODE_CNT * 2 -#define PRIORITY_WORK_QUEUE false -#define PRIORITY PRIORITY_ACTIVE #define MSG_SIZE_MAX 1048576 #define MSG_TIME_LIMIT 0 #define KEY_ORDER false @@ -77,7 +75,6 @@ #define INDEX_STRUCT IDX_HASH #define BTREE_ORDER 16 #define TS_TWR false -#define TS_ALLOC TS_CLOCK #define TS_BATCH_ALLOC false #define TS_BATCH_NUM 1 #define HIS_RECYCLE_LEN 10 @@ -94,7 +91,6 @@ #define FIRST_PART_LOCAL true #define MAX_TUPLE_SIZE 1024 #define GEN_BY_MPR false -#define SKEW_METHOD ZIPF #define DATA_PERC 100 #define ACCESS_PERC 0.03 #define INIT_PARALLELISM 8 @@ -104,51 +100,26 @@ #define WRITE_PERC 0.9 #define TXN_WRITE_PERC 0.5 #define TUP_WRITE_PERC 0.5 -#define SCAN_PERC 0 -#define SCAN_LEN 20 #define PART_PER_TXN PART_CNT #define PERC_MULTI_PART MPR #define REQ_PER_QUERY 1 #define FIELD_PER_TUPLE 10 -#define CREATE_TXN_FILE false -#define SINGLE_THREAD_WL_GEN true #define STRICT_PPT 1 #define MPR 1.0 #define MPIR 0.01 -#define WL_VERB true -#define IDX_VERB false -#define VERB_ALLOC true -#define DEBUG_LOCK false -#define DEBUG_TIMESTAMP false -#define DEBUG_SYNTH false -#define DEBUG_ASSERT false #define DEBUG_DISTR false #define DEBUG_ALLOC false #define DEBUG_RACE false -#define DEBUG_TIMELINE false -#define DEBUG_BREAKDOWN false #define DEBUG_LATENCY false #define DEBUG_QUECC false #define DEBUG_WLOAD false -#define MODE NORMAL_MODE #define DBTYPE REPLICATED #define IDX_HASH 1 #define IDX_BTREE 2 #define YCSB 1 #define TEST 4 -#define TS_MUTEX 1 -#define TS_CAS 2 -#define TS_HW 3 -#define TS_CLOCK 4 #define ZIPF 1 -#define HOT 2 -#define PRIORITY_FCFS 1 -#define PRIORITY_ACTIVE 2 -#define PRIORITY_HOME 3 -#define AA1 1 #define AP 2 -#define LOAD_MAX 1 -#define LOAD_RATE 2 #define TCP 1 #define IPC 2 #define BILLION 1000000000UL @@ -166,17 +137,10 @@ #define WARMUP_TIMER 5 * BILLION // Select the consensus algorithm to run. #define CONSENSUS PBFT -#define DBFT 1 #define PBFT 2 #define ZYZZYVA 3 -#define HOTSTUFF 4 -// Switching on RBFT consensus. -// Status: Partial implementation, only for PBFT. -#define RBFT_ON false -// Select the type of RBFT, (1) RBFT+PBFT, and (2) RBFT+DBFT -#define RBFT_ALG RPBFT -#define RPBFT 1 -#define RDBFT 2 + + // Enable or Disable pipeline at primary replica. #define ENABLE_PIPELINE true // Size of each batch. @@ -187,7 +151,6 @@ // Number of transactions to wait for period checkpointing. #define TXN_PER_CHKPT 6 * BATCH_SIZE #define SIGN_THREADS false -#define SIGN_THD_CNT 1 #define CLIENT_BATCH true #define CLIENT_RESPONSE_BATCH true // To Enable or disable the blockchain implementation. @@ -235,3 +198,4 @@ #define BANKING_SMART_CONTRACT false #endif + diff --git a/scripts/make_config.sh b/scripts/make_config.sh index f3c1e6824..f5088940a 100755 --- a/scripts/make_config.sh +++ b/scripts/make_config.sh @@ -5,226 +5,204 @@ cnodes=$2 [ ! -z "$3" ] && max_inf=$3 || max_inf="20000" consensus="PBFT" - -echo -e "#ifndef _CONFIG_H_ " >config.h -echo -e "#define _CONFIG_H_ " >>config.h -echo -e "// Specify the number of servers or replicas " >>config.h -echo -e "#define NODE_CNT ${nodes} " >>config.h -echo -e "// Number of worker threads at primary. For RBFT (6) and other algorithms (5). " >>config.h -echo -e "#define THREAD_CNT 5 " >>config.h -echo -e "#define REM_THREAD_CNT 3" >>config.h -echo -e "#define SEND_THREAD_CNT 1 " >>config.h -echo -e "#define CORE_CNT 8 " >>config.h -echo -e "#define PART_CNT 1 " >>config.h -echo -e "// Specify the number of clients. " >>config.h -echo -e "#define CLIENT_NODE_CNT ${cnodes}" >>config.h -echo -e "#define CLIENT_THREAD_CNT 2 " >>config.h -echo -e "#define CLIENT_REM_THREAD_CNT 1 " >>config.h -echo -e "#define CLIENT_SEND_THREAD_CNT 1 " >>config.h -echo -e "#define CLIENT_RUNTIME false " >>config.h -echo -e "#define LOAD_PER_SERVER 1 " >>config.h -echo -e "#define REPLICA_CNT 0 " >>config.h -echo -e "#define REPL_TYPE AP " >>config.h -echo -e "#define VIRTUAL_PART_CNT PART_CNT " >>config.h -echo -e "#define PAGE_SIZE 4096 " >>config.h -echo -e "#define CL_SIZE 64 " >>config.h -echo -e "#define CPU_FREQ 2.6 " >>config.h -echo -e "#define HW_MIGRATE false " >>config.h -echo -e "#define WARMUP 0 " >>config.h -echo -e "#define WORKLOAD YCSB " >>config.h -echo -e "#define PRT_LAT_DISTR false " >>config.h -echo -e "#define STATS_ENABLE true " >>config.h -echo -e "#define TIME_ENABLE true " >>config.h -echo -e "#define TIME_PROF_ENABLE false " >>config.h -echo -e "#define FIN_BY_TIME true " >>config.h -echo -e "// Number of transactions each client should send without waiting. " >>config.h -echo -e "#define MAX_TXN_IN_FLIGHT $max_inf " >>config.h -echo -e "#define MESSAGE_PER_BUFFER 1 " >>config.h -echo -e "#define SERVER_GENERATE_QUERIES false " >>config.h -echo -e "#define MEM_ALLIGN 8 " >>config.h -echo -e "#define THREAD_ALLOC false " >>config.h -echo -e "#define THREAD_ARENA_SIZE (1UL << 22) " >>config.h -echo -e "#define MEM_PAD true " >>config.h -echo -e "#define PART_ALLOC false " >>config.h -echo -e "#define MEM_SIZE (1UL << 30) " >>config.h -echo -e "#define NO_FREE false " >>config.h -echo -e "#define TPORT_TYPE TCP " >>config.h -echo -e "#define TPORT_PORT 17000 " >>config.h -echo -e "#define SET_AFFINITY false " >>config.h -echo -e "#define MAX_TPORT_NAME 128 " >>config.h -echo -e "#define MSG_SIZE 128 " >>config.h -echo -e "#define HEADER_SIZE sizeof(uint32_t)*2 " >>config.h -echo -e "#define MSG_TIMEOUT 5000000000UL // in ns " >>config.h -echo -e "#define NETWORK_TEST false " >>config.h -echo -e "#define NETWORK_DELAY_TEST false " >>config.h -echo -e "#define NETWORK_DELAY 0UL " >>config.h -echo -e "#define MAX_QUEUE_LEN NODE_CNT * 2 " >>config.h -echo -e "#define PRIORITY_WORK_QUEUE false " >>config.h -echo -e "#define PRIORITY PRIORITY_ACTIVE " >>config.h -echo -e "#define MSG_SIZE_MAX 1048576 " >>config.h -echo -e "#define MSG_TIME_LIMIT 0 " >>config.h -echo -e "#define KEY_ORDER false " >>config.h -echo -e "#define ENABLE_LATCH false " >>config.h -echo -e "#define CENTRAL_INDEX false " >>config.h -echo -e "#define CENTRAL_MANAGER false " >>config.h -echo -e "#define INDEX_STRUCT IDX_HASH " >>config.h -echo -e "#define BTREE_ORDER 16 " >>config.h -echo -e "#define TS_TWR false " >>config.h -echo -e "#define TS_ALLOC TS_CLOCK " >>config.h -echo -e "#define TS_BATCH_ALLOC false " >>config.h -echo -e "#define TS_BATCH_NUM 1 " >>config.h -echo -e "#define HIS_RECYCLE_LEN 10 " >>config.h -echo -e "#define MAX_PRE_REQ MAX_TXN_IN_FLIGHT * NODE_CNT " >>config.h -echo -e "#define MAX_READ_REQ MAX_TXN_IN_FLIGHT * NODE_CNT " >>config.h -echo -e "#define MIN_TS_INTVL 10 * 1000000UL " >>config.h -echo -e "#define MAX_WRITE_SET 10 " >>config.h -echo -e "#define PER_ROW_VALID false " >>config.h -echo -e "#define TXN_QUEUE_SIZE_LIMIT THREAD_CNT " >>config.h -echo -e "#define SEQ_THREAD_CNT 4 " >>config.h -echo -e "#define MAX_ROW_PER_TXN 64 " >>config.h -echo -e "#define QUERY_INTVL 1UL " >>config.h -echo -e "#define MAX_TXN_PER_PART 4000 " >>config.h -echo -e "#define FIRST_PART_LOCAL true " >>config.h -echo -e "#define MAX_TUPLE_SIZE 1024 " >>config.h -echo -e "#define GEN_BY_MPR false " >>config.h -echo -e "#define SKEW_METHOD ZIPF " >>config.h -echo -e "#define DATA_PERC 100 " >>config.h -echo -e "#define ACCESS_PERC 0.03 " >>config.h -echo -e "#define INIT_PARALLELISM 8 " >>config.h -echo -e "#define SYNTH_TABLE_SIZE 524288" >>config.h -echo -e "" >>config.h -echo -e "#define ZIPF_THETA 0.5 " >>config.h -echo -e "#define WRITE_PERC 0.9 " >>config.h -echo -e "#define TXN_WRITE_PERC 0.5 " >>config.h -echo -e "#define TUP_WRITE_PERC 0.5 " >>config.h -echo -e "#define SCAN_PERC 0 " >>config.h -echo -e "#define SCAN_LEN 20 " >>config.h -echo -e "#define PART_PER_TXN PART_CNT " >>config.h -echo -e "#define PERC_MULTI_PART MPR " >>config.h -echo -e "#define REQ_PER_QUERY 1 " >>config.h -echo -e "#define FIELD_PER_TUPLE 10 " >>config.h -echo -e "#define CREATE_TXN_FILE false " >>config.h -echo -e "#define SINGLE_THREAD_WL_GEN true " >>config.h -echo -e "#define STRICT_PPT 1 " >>config.h -echo -e "#define MPR 1.0 " >>config.h -echo -e "#define MPIR 0.01 " >>config.h -echo -e "#define WL_VERB true " >>config.h -echo -e "#define IDX_VERB false " >>config.h -echo -e "#define VERB_ALLOC true " >>config.h -echo -e "#define DEBUG_LOCK false " >>config.h -echo -e "#define DEBUG_TIMESTAMP false " >>config.h -echo -e "#define DEBUG_SYNTH false " >>config.h -echo -e "#define DEBUG_ASSERT false " >>config.h -echo -e "#define DEBUG_DISTR false " >>config.h -echo -e "#define DEBUG_ALLOC false " >>config.h -echo -e "#define DEBUG_RACE false " >>config.h -echo -e "#define DEBUG_TIMELINE false " >>config.h -echo -e "#define DEBUG_BREAKDOWN false " >>config.h -echo -e "#define DEBUG_LATENCY false " >>config.h -echo -e "#define DEBUG_QUECC false " >>config.h -echo -e "#define DEBUG_WLOAD false " >>config.h -echo -e "#define MODE NORMAL_MODE " >>config.h -echo -e "#define DBTYPE REPLICATED " >>config.h -echo -e "#define IDX_HASH 1 " >>config.h -echo -e "#define IDX_BTREE 2 " >>config.h -echo -e "#define YCSB 1 " >>config.h -echo -e "#define TEST 4 " >>config.h -echo -e "#define TS_MUTEX 1 " >>config.h -echo -e "#define TS_CAS 2 " >>config.h -echo -e "#define TS_HW 3 " >>config.h -echo -e "#define TS_CLOCK 4 " >>config.h -echo -e "#define ZIPF 1 " >>config.h -echo -e "#define HOT 2 " >>config.h -echo -e "#define PRIORITY_FCFS 1 " >>config.h -echo -e "#define PRIORITY_ACTIVE 2 " >>config.h -echo -e "#define PRIORITY_HOME 3 " >>config.h -echo -e "#define AA1 1 " >>config.h -echo -e "#define AP 2 " >>config.h -echo -e "#define LOAD_MAX 1 " >>config.h -echo -e "#define LOAD_RATE 2 " >>config.h -echo -e "#define TCP 1 " >>config.h -echo -e "#define IPC 2 " >>config.h -echo -e "#define BILLION 1000000000UL " >>config.h -echo -e "#define MILLION 1000000UL " >>config.h -echo -e "#define STAT_ARR_SIZE 1024 " >>config.h -echo -e "#define PROG_TIMER 10 * BILLION " >>config.h -echo -e "#define SEQ_BATCH_TIMER 5 * 1 * MILLION " >>config.h -echo -e "#define SEED 0 " >>config.h -echo -e "#define SHMEM_ENV false " >>config.h -echo -e "#define ENVIRONMENT_EC2 false " >>config.h -echo -e "#define PARTITIONED 0 " >>config.h -echo -e "#define REPLICATED 1 " >>config.h -echo -e "// To select the amount of time to warmup and run. " >>config.h -echo -e "#define DONE_TIMER 2 * 60 * BILLION " >>config.h -echo -e "#define WARMUP_TIMER 1 * 60 * BILLION " >>config.h -echo -e "// Select the consensus algorithm to run. " >>config.h -echo -e "#define CONSENSUS ${consensus}" >>config.h -echo -e "#define DBFT 1 " >>config.h -echo -e "#define PBFT 2 " >>config.h -echo -e "#define ZYZZYVA 3 " >>config.h -echo -e "#define HOTSTUFF 4 " >>config.h -echo -e "// Switching on RBFT consensus. " >>config.h -echo -e "// Status: Partial implementation, only for PBFT. " >>config.h -echo -e "#define RBFT_ON false " >>config.h -echo -e "// Select the type of RBFT, (1) RBFT+PBFT, and (2) RBFT+DBFT " >>config.h -echo -e "#define RBFT_ALG RPBFT " >>config.h -echo -e "#define RPBFT 1 " >>config.h -echo -e "#define RDBFT 2 " >>config.h -echo -e "// Enable or Disable pipeline at primary replica. " >>config.h -echo -e "#define ENABLE_PIPELINE true " >>config.h -echo -e "// Number of threads to create batches at primary replica. " >>config.h -echo -e "#define BATCH_THREADS 2 " >>config.h -echo -e "// Size of each batch. " >>config.h -echo -e "#define BATCH_SIZE 100 " >>config.h -echo -e "#define BATCH_ENABLE BSET " >>config.h -echo -e "#define BSET 1 " >>config.h -echo -e "#define BUNSET 0 " >>config.h -echo -e "// Number of transactions to wait for period checkpointing. " >>config.h -echo -e "#define TXN_PER_CHKPT 600 " >>config.h -echo -e "#define EXECUTION_THREAD true " >>config.h -echo -e "#define EXECUTE_THD_CNT 1 " >>config.h -echo -e "#define SIGN_THREADS false " >>config.h -echo -e "#define SIGN_THD_CNT 1 " >>config.h -echo -e "#define CLIENT_BATCH true " >>config.h -echo -e "#define CLIENT_RESPONSE_BATCH true " >>config.h -echo -e "// To Enable or disable the blockchain implementation." >>config.h -echo -e "#define ENABLE_CHAIN true" >>config.h -echo -e "// To fail non-primary replicas. " >>config.h -echo -e "#define LOCAL_FAULT false " >>config.h -echo -e "#define NODE_FAIL_CNT 1 " >>config.h -echo -e "// To allow view changes. " >>config.h -echo -e "#define VIEW_CHANGES false " >>config.h -echo -e "// The amount of timeout value. " >>config.h -echo -e "#define EXE_TIMEOUT 10000000000 " >>config.h -echo -e "#define CEXE_TIMEOUT 12000000000 " >>config.h -echo -e "// To turn the timer on. " >>config.h -echo -e "#define TIMER_ON false " >>config.h -echo -e "//Global variables to choose the encryptation algorithm " >>config.h -echo -e "#define USE_CRYPTO true " >>config.h -echo -e "#define CRYPTO_METHOD_RSA false //Options RSA, " >>config.h -echo -e "#define CRYPTO_METHOD_ED25519 true // Option ED25519 " >>config.h -echo -e "#define CRYPTO_METHOD_CMAC_AES true // CMAC " >>config.h -echo -e "// Test cases to check basic functioning. " >>config.h -echo -e "// Status: Implementation only for PBFT. " >>config.h -echo -e "#define TESTING_ON false " >>config.h -echo -e "#define TEST_CASE ONLY_PRIMARY_BATCH_EXECUTE " >>config.h -echo -e "#define ONLY_PRIMARY_NO_EXECUTE 1 " >>config.h -echo -e "#define ONLY_PRIMARY_EXECUTE 2 " >>config.h -echo -e "#define ONLY_PRIMARY_BATCH_EXECUTE 3 " >>config.h -echo -e "// Message Payload. " >>config.h -echo -e "// We allow creation of two different message payloads, " >>config.h -echo -e "// to see affects on latency and throughput. " >>config.h -echo -e "// These payloads are added to each message. " >>config.h -echo -e "#define PAYLOAD_ENABLE false " >>config.h -echo -e "#define PAYLOAD M100 " >>config.h -echo -e "#define M100 1 // 100KB. " >>config.h -echo -e "#define M200 2 // 200KB. " >>config.h -echo -e "#define M400 3 // 400KB. " >>config.h -echo -e "" >>config.h +echo -e "#ifndef _CONFIG_H_" > config.h +echo -e "#define _CONFIG_H_" >> config.h +echo -e "// Specify the number of servers or replicas" >> config.h +echo -e "#define NODE_CNT ${nodes}" >> config.h +echo -e "// Number of worker threads at primary. " >> config.h +echo -e "#define THREAD_CNT 5 // This Should be the sum of following thread count + protocol specifig threads" >> config.h +echo -e "#define WORKER_THREAD_CNT 1" >> config.h +echo -e "#define BATCH_THREAD_CNT 2" >> config.h +echo -e "#define CHECKPOINT_THREAD_CNT 1" >> config.h +echo -e "#define EXECUTE_THREAD_CNT 1" >> config.h +echo -e "// IO THREADS" >> config.h +echo -e "#define REM_THREAD_CNT 2" >> config.h +echo -e "#define SEND_THREAD_CNT 1" >> config.h +echo -e "#define CORE_CNT 8" >> config.h +echo -e "#define PART_CNT 1" >> config.h +echo -e "// Specify the number of clients." >> config.h +echo -e "#define CLIENT_NODE_CNT ${cnodes}" >> config.h +echo -e "#define CLIENT_THREAD_CNT 2" >> config.h +echo -e "#define CLIENT_REM_THREAD_CNT 1" >> config.h +echo -e "#define CLIENT_SEND_THREAD_CNT 1" >> config.h +echo -e "#define CLIENT_RUNTIME false" >> config.h +echo -e "" >> config.h +echo -e "#define MESSAGE_PER_BUFFER 24" >> config.h +echo -e "" >> config.h +echo -e "// GeoBFT Setting " >> config.h +echo -e "#define GBFT false" >> config.h +echo -e "#define GBFT_CLUSTER_SIZE 4" >> config.h +echo -e "#define GBFT_CCM_THREAD_CNT 1" >> config.h +echo -e "" >> config.h +echo -e "#define LOAD_PER_SERVER 1" >> config.h +echo -e "#define REPLICA_CNT 0" >> config.h +echo -e "#define REPL_TYPE AP" >> config.h +echo -e "#define VIRTUAL_PART_CNT PART_CNT" >> config.h +echo -e "#define PAGE_SIZE 4096" >> config.h +echo -e "#define CL_SIZE 64" >> config.h +echo -e "#define CPU_FREQ 2.2" >> config.h +echo -e "#define HW_MIGRATE false" >> config.h +echo -e "#define WARMUP 0" >> config.h +echo -e "#define WORKLOAD YCSB" >> config.h +echo -e "#define PRT_LAT_DISTR false" >> config.h +echo -e "#define STATS_ENABLE true" >> config.h +echo -e "#define STATS_DETAILED false" >> config.h +echo -e "#define STAT_BAND_WIDTH_ENABLE false" >> config.h +echo -e "#define TIME_ENABLE true" >> config.h +echo -e "#define TIME_PROF_ENABLE false" >> config.h +echo -e "#define FIN_BY_TIME true" >> config.h +echo -e "// Number of transactions each client should send without waiting." >> config.h +echo -e "#define MAX_TXN_IN_FLIGHT ${max_inf}" >> config.h +echo -e "#define SERVER_GENERATE_QUERIES false" >> config.h +echo -e "#define MEM_ALLIGN 8" >> config.h +echo -e "#define THREAD_ALLOC false" >> config.h +echo -e "#define THREAD_ARENA_SIZE (1UL << 22)" >> config.h +echo -e "#define MEM_PAD true" >> config.h +echo -e "#define PART_ALLOC false" >> config.h +echo -e "#define MEM_SIZE (1UL << 30)" >> config.h +echo -e "#define NO_FREE false" >> config.h +echo -e "#define TPORT_TYPE TCP" >> config.h +echo -e "#define TPORT_PORT 10000" >> config.h +echo -e "#define TPORT_WINDOW 20000" >> config.h +echo -e "#define SET_AFFINITY false" >> config.h +echo -e "#define MAX_TPORT_NAME 128" >> config.h +echo -e "#define MSG_SIZE 128" >> config.h +echo -e "#define HEADER_SIZE sizeof(uint32_t) * 2" >> config.h +echo -e "#define MSG_TIMEOUT 5000000000UL // in ns" >> config.h +echo -e "#define NETWORK_TEST false" >> config.h +echo -e "#define NETWORK_DELAY_TEST false" >> config.h +echo -e "#define NETWORK_DELAY 0UL" >> config.h +echo -e "#define MAX_QUEUE_LEN NODE_CNT * 2" >> config.h +echo -e "#define MSG_SIZE_MAX 1048576" >> config.h +echo -e "#define MSG_TIME_LIMIT 0" >> config.h +echo -e "#define KEY_ORDER false" >> config.h +echo -e "#define ENABLE_LATCH false" >> config.h +echo -e "#define CENTRAL_INDEX false" >> config.h +echo -e "#define CENTRAL_MANAGER false" >> config.h +echo -e "#define INDEX_STRUCT IDX_HASH" >> config.h +echo -e "#define BTREE_ORDER 16" >> config.h +echo -e "#define TS_TWR false" >> config.h +echo -e "#define TS_BATCH_ALLOC false" >> config.h +echo -e "#define TS_BATCH_NUM 1" >> config.h +echo -e "#define HIS_RECYCLE_LEN 10" >> config.h +echo -e "#define MAX_PRE_REQ MAX_TXN_IN_FLIGHT *NODE_CNT" >> config.h +echo -e "#define MAX_READ_REQ MAX_TXN_IN_FLIGHT *NODE_CNT" >> config.h +echo -e "#define MIN_TS_INTVL 10 * 1000000UL" >> config.h +echo -e "#define MAX_WRITE_SET 10" >> config.h +echo -e "#define PER_ROW_VALID false" >> config.h +echo -e "#define TXN_QUEUE_SIZE_LIMIT THREAD_CNT" >> config.h +echo -e "#define SEQ_THREAD_CNT 4" >> config.h +echo -e "#define MAX_ROW_PER_TXN 64" >> config.h +echo -e "#define QUERY_INTVL 1UL" >> config.h +echo -e "#define MAX_TXN_PER_PART 4000" >> config.h +echo -e "#define FIRST_PART_LOCAL true" >> config.h +echo -e "#define MAX_TUPLE_SIZE 1024" >> config.h +echo -e "#define GEN_BY_MPR false" >> config.h +echo -e "#define DATA_PERC 100" >> config.h +echo -e "#define ACCESS_PERC 0.03" >> config.h +echo -e "#define INIT_PARALLELISM 8" >> config.h +echo -e "#define SYNTH_TABLE_SIZE 524288" >> config.h +echo -e "" >> config.h +echo -e "#define ZIPF_THETA 0.5" >> config.h +echo -e "#define WRITE_PERC 0.9" >> config.h +echo -e "#define TXN_WRITE_PERC 0.5" >> config.h +echo -e "#define TUP_WRITE_PERC 0.5" >> config.h +echo -e "#define PART_PER_TXN PART_CNT" >> config.h +echo -e "#define PERC_MULTI_PART MPR" >> config.h +echo -e "#define REQ_PER_QUERY 1" >> config.h +echo -e "#define FIELD_PER_TUPLE 10" >> config.h +echo -e "#define STRICT_PPT 1" >> config.h +echo -e "#define MPR 1.0" >> config.h +echo -e "#define MPIR 0.01" >> config.h +echo -e "#define DEBUG_DISTR false" >> config.h +echo -e "#define DEBUG_ALLOC false" >> config.h +echo -e "#define DEBUG_RACE false" >> config.h +echo -e "#define DEBUG_LATENCY false" >> config.h +echo -e "#define DEBUG_QUECC false" >> config.h +echo -e "#define DEBUG_WLOAD false" >> config.h +echo -e "#define DBTYPE REPLICATED" >> config.h +echo -e "#define IDX_HASH 1" >> config.h +echo -e "#define IDX_BTREE 2" >> config.h +echo -e "#define YCSB 1" >> config.h +echo -e "#define TEST 4" >> config.h +echo -e "#define ZIPF 1" >> config.h +echo -e "#define AP 2" >> config.h +echo -e "#define TCP 1" >> config.h +echo -e "#define IPC 2" >> config.h +echo -e "#define BILLION 1000000000UL" >> config.h +echo -e "#define MILLION 1000000UL" >> config.h +echo -e "#define STAT_ARR_SIZE 1024" >> config.h +echo -e "#define PROG_TIMER 5 * BILLION" >> config.h +echo -e "#define SEQ_BATCH_TIMER 5 * 1 * MILLION" >> config.h +echo -e "#define SEED 0" >> config.h +echo -e "#define SHMEM_ENV false" >> config.h +echo -e "#define ENVIRONMENT_EC2 false" >> config.h +echo -e "#define PARTITIONED 0" >> config.h +echo -e "#define REPLICATED 1" >> config.h +echo -e "// To select the amount of time to warmup and run." >> config.h +echo -e "#define DONE_TIMER 1 * 60 * BILLION" >> config.h +echo -e "#define WARMUP_TIMER 5 * BILLION" >> config.h +echo -e "// Select the consensus algorithm to run." >> config.h +echo -e "#define CONSENSUS PBFT" >> config.h +echo -e "#define PBFT 2" >> config.h +echo -e "#define ZYZZYVA 3" >> config.h +echo -e "" >> config.h +echo -e "" >> config.h +echo -e "// Enable or Disable pipeline at primary replica." >> config.h +echo -e "#define ENABLE_PIPELINE true" >> config.h +echo -e "// Size of each batch." >> config.h +echo -e "#define BATCH_SIZE 100" >> config.h +echo -e "#define BATCH_ENABLE BSET" >> config.h +echo -e "#define BSET 1" >> config.h +echo -e "#define BUNSET 0" >> config.h +echo -e "// Number of transactions to wait for period checkpointing." >> config.h +echo -e "#define TXN_PER_CHKPT 6 * BATCH_SIZE" >> config.h +echo -e "#define SIGN_THREADS false" >> config.h +echo -e "#define CLIENT_BATCH true" >> config.h +echo -e "#define CLIENT_RESPONSE_BATCH true" >> config.h +echo -e "// To Enable or disable the blockchain implementation." >> config.h +echo -e "#define ENABLE_CHAIN false" >> config.h +echo -e "// To fail non-primary replicas." >> config.h +echo -e "#define LOCAL_FAULT false" >> config.h +echo -e "#define NODE_FAIL_CNT 1" >> config.h +echo -e "// To allow view changes." >> config.h +echo -e "#define VIEW_CHANGES false" >> config.h +echo -e "// The amount of timeout value." >> config.h +echo -e "#define EXE_TIMEOUT 10000000000" >> config.h +echo -e "#define CEXE_TIMEOUT 12000000000" >> config.h +echo -e "// To turn the timer on." >> config.h +echo -e "#define TIMER_ON false" >> config.h +echo -e "//Global variables to choose the encryptation algorithm" >> config.h +echo -e "#define USE_CRYPTO true" >> config.h +echo -e "#define CRYPTO_METHOD_RSA false //Options RSA," >> config.h +echo -e "#define CRYPTO_METHOD_ED25519 true // Option ED25519" >> config.h +echo -e "#define CRYPTO_METHOD_CMAC_AES true // CMAC" >> config.h +echo -e "// Test cases to check basic functioning." >> config.h +echo -e "// Status: Implementation only for PBFT." >> config.h +echo -e "#define TESTING_ON false" >> config.h +echo -e "#define TEST_CASE ONLY_PRIMARY_BATCH_EXECUTE" >> config.h +echo -e "#define ONLY_PRIMARY_NO_EXECUTE 1" >> config.h +echo -e "#define ONLY_PRIMARY_EXECUTE 2" >> config.h +echo -e "#define ONLY_PRIMARY_BATCH_EXECUTE 3" >> config.h +echo -e "// Message Payload." >> config.h +echo -e "// We allow creation of two different message payloads," >> config.h +echo -e "// to see affects on latency and throughput." >> config.h +echo -e "// These payloads are added to each message." >> config.h +echo -e "#define PAYLOAD_ENABLE false" >> config.h +echo -e "#define PAYLOAD M100" >> config.h +echo -e "#define M100 1 // 100KB." >> config.h +echo -e "#define M200 2 // 200KB." >> config.h +echo -e "#define M400 3 // 400KB." >> config.h +echo -e "" >> config.h +echo -e "// To allow testing in-memory database or SQLite." >> config.h +echo -e "// Further, using SQLite a user can also choose to persist the data." >> config.h echo -e "#define EXT_DB MEMORY" >> config.h echo -e "#define MEMORY 1" >> config.h echo -e "#define SQL 2" >> config.h echo -e "#define SQL_PERSISTENT 3" >> config.h -echo -e "" >>config.h -echo -e "#endif" >>config.h -echo -e "" >>config.h +echo -e "" >> config.h +echo -e "// To allow testing of a Banking Smart Contracts." >> config.h +echo -e "#define BANKING_SMART_CONTRACT false" >> config.h +echo -e "" >> config.h +echo -e "#endif" >> config.h +echo -e "" >> config.h \ No newline at end of file diff --git a/scripts/result.sh b/scripts/result.sh index bcf4f4bd1..c07034565 100755 --- a/scripts/result.sh +++ b/scripts/result.sh @@ -1,8 +1,8 @@ #!/bin/bash unset GREP_OPTIONS # Black='\033[1;90m' # Black -Nc='\033[0m' -Red='\033[1;91m' # Red +# Nc='\033[0m' +# Red='\033[1;91m' # Red # Green='\033[1;92m' # Green # Yellow='\033[1;93m' # Yellow diff --git a/scripts/result_colorized.sh b/scripts/result_colorized.sh index 66df0096b..bcf4f4bd1 100755 --- a/scripts/result_colorized.sh +++ b/scripts/result_colorized.sh @@ -14,10 +14,9 @@ Red='\033[1;91m' # Red snodes=$1 cnodes=$2 protocol=$3 -shard_size=$4 -bsize=$5 -run=$6 -folder=$7 +bsize=$4 +run=$5 +folder=$6 total_nodes=$((cnodes + snodes - 1)) avg_thp=0 thp_cnt=0 @@ -26,11 +25,6 @@ avg_lt=0.0 lt_cnt=0 avg_msg=0 msg_cnt=0 -shard_counter=0 -shard_tp=0 -shard_ctp=0 -shards_tp=() -shards_ctp=() if [ "$(uname)" == "Darwin" ]; then flags="-E" else @@ -44,109 +38,49 @@ cd results/ echo "Throughputs:" for i in $(seq 0 $(($total_nodes))); do if [ "$i" -lt "$snodes" ]; then - temp=$(tail -20 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') - temp3=$(tail -20 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'cput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput[\s]+=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') temp2=$(tail -74 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'msg_send_cnt=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') if [ ! -z "$temp" ]; then avg_thp=$(($avg_thp + $temp)) thp_cnt=$(($thp_cnt + 1)) - shard_counter=$(($shard_counter + 1)) - shard_tp=$(($shard_tp + $temp)) - shard_ctp=$(($shard_ctp + $temp3)) #avg_msg=$(($avg_msg + $temp2)) #msg_cnt=$(($msg_cnt + 1)) fi - if [[ $shard_counter -eq $shard_size ]]; then - shards_tp=("${shards_tp[@]}" "$shard_tp") - shards_ctp=("${shards_ctp[@]}" "$shard_ctp") - shard_counter=0 - shard_tp=0 - shard_ctp=0 - fi - j=$i - if [ $j -lt 10 ]; then - j="0$j" - fi - echo -e "$j: ${Red}${temp}${Nc}\t${Red}${temp3}${Nc}" else - temp=$(tail -4 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + temp=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'tput=.{1,13}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') client_thp=$(($client_thp + $temp)) - j=$i - if [ $j -lt 10 ]; then - j="0$j" - fi - echo -e "$j: ${Red}${temp}${Nc}" fi - + echo -e "$i: ${Red}${temp}${Nc}" done echo "Latencies:" for i in $(seq $snodes $(($total_nodes))); do - temp=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 'AVG: .{1,13}' | grep -o ${flags} '\d+\.\d+') - if [ ! -z "$temp" ]; then - avg_lt=$(echo "$avg_lt + $temp" | bc) - lt_cnt=$(($lt_cnt + 1)) - echo -e "latency $i: ${Red}${temp}${Nc}" - else - echo -e "latency $i: ${Red}NAAAAAAN${Nc}" - fi - + temp=$(tail -11 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} '^Latency=.{1,13}' | grep -o ${flags} '\d+\.\d+') + avg_lt=$(echo "$avg_lt + $temp" | bc) + lt_cnt=$(($lt_cnt + 1)) + echo -e "latency $i: ${Red}${temp}${Nc}" done echo echo "idle times:" for i in $(seq 0 $(($snodes - 1))); do - echo "I Node: $i" + echo "Idleness of node: $i" times=($(tail -50 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} "idle_time_worker.{1,10}" | grep -o ${flags} '\d+\.\d+')) for ((j = 0; j < ${#times[@]}; j++)); do - # echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" - echo -en "${Red}${times[$j]}${Nc} " + echo -e "Worker THD ${j}: ${Red}${times[$j]}${Nc}" done - echo done echo "Memory:" for i in $(seq 0 $total_nodes); do - lines=10 - if [ $i -lt $snodes ]; then - lines=20 - fi - mem=$(tail -${lines} s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') + mem=$(tail -10 s${snodes}_c${cnodes}_results_${protocol}_b${bsize}_run${run}_node${i}.out | grep -o ${flags} 's_mem_usage=.{1,7}' | grep -o ${flags} '=\d+' | grep -o ${flags} '\d+') [ ! -z "$mem" ] || mem=0 echo "$i: $(($mem / 1000)) MB" done echo -total_tp=0 -avg_ctp=0 -counter=0 -echo "Shards TP:" -for t in ${shards_tp[@]}; do - shard_avg_tp=$(echo "$t / $shard_size" | bc) - total_tp=$(($total_tp + $shard_avg_tp)) - echo "Shards ${counter} TP: $(echo -e ${Red}${shard_avg_tp}${Nc})" - counter=$(($counter + 1)) -done -counter=0 -echo "Shards CTP:" -for t in ${shards_ctp[@]}; do - shard_avg_tp=$(echo "$t / $shard_size" | bc) - avg_ctp=$(($avg_ctp + $shard_avg_tp)) - echo "Shards ${counter} CTP: $(echo -e ${Red}${shard_avg_tp}${Nc})" - if [ $shard_avg_tp -gt 0 ]; then - counter=$(($counter + 1)) - fi - -done -echo -if [ $counter -eq 0 ]; then - counter=1 -fi -echo "total avg TP: $(echo -e ${Red}${total_tp}${Nc})" -echo "total avg CTP: $(echo -e ${Red}$(echo "scale=0;$avg_ctp / $counter" | bc)${Nc})" -echo "" -echo "Final avg TP: $(echo -e ${Red}$(echo "scale=0;$avg_ctp / $counter + $total_tp" | bc)${Nc})" -echo "avg lt ${lt_cnt}: $(echo -e ${Red}$(echo "scale=3;$avg_lt / $lt_cnt" | bc)${Nc})" +echo "avg thp: ${thp_cnt}: $(echo -e ${Red}$(expr $avg_thp / $thp_cnt)${Nc})" +echo "avg lt : ${lt_cnt}: $(echo -e ${Red}$(echo "scale=3;$avg_lt / $lt_cnt" | bc)${Nc})" #echo "avg msg: ${msg_cnt}: $(echo -e ${Red}$(expr $avg_msg / $msg_cnt)${Nc})" # echo -e "cli thp sum: ${RED}$client_thp${NC}" diff --git a/system/global.cpp b/system/global.cpp index d4c5bcfc4..16deb2301 100644 --- a/system/global.cpp +++ b/system/global.cpp @@ -34,7 +34,6 @@ bool volatile warmup_done = false; bool volatile enable_thread_mem_pool = false; pthread_barrier_t warmup_bar; -UInt32 g_ts_alloc = TS_ALLOC; bool g_key_order = KEY_ORDER; bool g_ts_batch_alloc = TS_BATCH_ALLOC; UInt32 g_ts_batch_num = TS_BATCH_NUM; @@ -69,11 +68,7 @@ UInt32 g_batching_thread_cnt = BATCH_THREAD_CNT; UInt32 g_checkpointing_thread_cnt = CHECKPOINT_THREAD_CNT; UInt32 g_execution_thread_cnt = EXECUTE_THREAD_CNT; -#if SIGN_THREADS -UInt32 g_sign_thd = SIGN_THD_CNT; -#else -UInt32 g_sign_thd = 0; -#endif + UInt32 g_rem_thread_cnt = REM_THREAD_CNT; UInt32 g_send_thread_cnt = SEND_THREAD_CNT; diff --git a/system/global.h b/system/global.h index fa429fdf0..620001252 100644 --- a/system/global.h +++ b/system/global.h @@ -131,7 +131,6 @@ extern UInt32 g_sign_thd; extern UInt32 g_send_thread_cnt; extern UInt32 g_rem_thread_cnt; -extern UInt32 g_ts_alloc; extern bool g_key_order; extern bool g_ts_batch_alloc; extern UInt32 g_ts_batch_num; diff --git a/transport/message.cpp b/transport/message.cpp index 6865fa0e3..afc9352cf 100644 --- a/transport/message.cpp +++ b/transport/message.cpp @@ -1525,13 +1525,11 @@ bool BatchRequests::validate(uint64_t thd_id) //fflush(stdout); //is the view the same as the view observed by this message -#if !RBFT_ON if (this->view != get_current_view(thd_id)) { assert(0); return false; } -#endif return true; }