Issue 2261/json config overridable cli

charts/taraxa-node/templates/initconfig-consensus-node.yaml

@@ -155,7 +155,7 @@ data:
       --config $CONFIG_PATH \
       --genesis $GENESIS_PATH \
       --wallet $WALLET_PATH \
-      --data-dir $DATA_PATH
+      --data_path $DATA_PATH
 
     {{- if and (ne .Values.config.network "841") (ne .Values.config.network "842") (ne .Values.config.network "843") }}
     python3 /bin/genconfig.py $GENESIS_PATH > $GENESIS_PATH.tmp && mv $GENESIS_PATH.tmp $GENESIS_PATH

charts/taraxa-node/templates/initconfig-node.yaml

@@ -141,7 +141,7 @@ data:
       --config $CONFIG_PATH \
       --genesis $GENESIS_PATH \
       --wallet $WALLET_PATH \
-      --data-dir $DATA_PATH
+      --data_path $DATA_PATH
 
     {{- if and (ne .Values.config.network "841") (ne .Values.config.network "842") (ne .Values.config.network "843") }}
     python3 /bin/genconfig.py $GENESIS_PATH > $GENESIS_PATH.tmp && mv $GENESIS_PATH.tmp $GENESIS_PATH

doc/quickstart_guide.md

@@ -23,7 +23,7 @@ Quickstart Guide
   #### OPTIONAL PORTS   
   TODO: add
 
-### Config
+### ConfigParser
 You can see example config [here](doc/example_config.json)
 
 #### Param1

for_devs/local-net

@@ -47,7 +47,7 @@ def cli():
 @click.option('--boot-nodes', default=3, help='Number of boot nodes')
 @click.option('--rpc-nodes', default=3, help='Number of RPC nodes')
 @click.option('--tps', default=0, help='Number of transactions per second (if zero the faucet will not start)')
-@click.option('--enable-test-rpc', is_flag=True, default=False, help='Enables Test JsonRPC')
+@click.option('--enable_test_rpc', is_flag=True, default=False, help='Enables Test JsonRPC')
 @click.argument('binary')
 def start(boot_nodes, rpc_nodes, tps, enable_test_rpc, binary):
     """Start a local testnet"""
@@ -121,15 +121,14 @@ def config(binary, boot_nodes, rpc_nodes):
             if type == "boot" and existing_genesis:
                 sys.exit("Cannot increase number of boot nodes without resetting the network")
 
-            if not os.path.isdir(f'{local_net}/db-{worker_name}'):
-                os.mkdir(f'{local_net}/db-{worker_name}')
+            if not os.path.isdir(f'{local_net}/data-{worker_name}'):
+                os.mkdir(f'{local_net}/data-{worker_name}')
 
             cmd = ''
             cmd += f'{binary} --command config --chain-id {chain_id} '
             cmd += f'--boot-nodes {boot_nodes_flags} '
 
             if type == "boot":
-                # cmd += '--boot-node ' // TODO: we should replace this with bootnode binary?
                 cmd += f'--node-secret {private_keys[worker_name]} '
                 cmd += f'--vrf-secret {vrf_secrets[worker_name]} '
 
@@ -141,7 +140,7 @@ def config(binary, boot_nodes, rpc_nodes):
             cmd += f'--config {config_path} '
             cmd += f'--genesis {genesis_path} '
             cmd += f'--wallet {wallet_path} '
-            cmd += f'--data-dir {local_net}/db-{worker_name} '
+            cmd += f'--data_path {local_net}/data-{worker_name} '
             os.system(cmd)
 
             config = json.loads('{}')
@@ -285,7 +284,7 @@ def start_workers(binary, boot_nodes, rpc_nodes, tps, enable_test_rpc):
 def node_worker(binary, worker_name, enable_test_rpc):
     cmd = f'{binary} --config ./{data_dir}/conf-{worker_name}.json --wallet ./{data_dir}/wallet-{worker_name}.json --genesis ./{data_dir}/genesis-{worker_name}.json'
     if enable_test_rpc:
-        cmd += ' --enable-test-rpc '
+        cmd += ' --enable_test_rpc '
     process = subprocess.Popen(cmd,
                                shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     logs(worker_name, process)

libraries/cli/CMakeLists.txt

@@ -1,11 +1,11 @@
 set(HEADERS
-    include/cli/config.hpp
+    include/cli/config_parser.hpp
     include/cli/config_updater.hpp
     include/cli/tools.hpp
 )
 
 set(SOURCES
-    src/config.cpp
+    src/config_parser.cpp
     src/config_updater.cpp
     src/tools.cpp
 )

libraries/cli/include/cli/config_jsons/default/default_config.json

@@ -4,14 +4,17 @@
   "final_chain_cache_in_blocks": 5,
   "network": {
     "rpc": {
+      "enabled": true,
       "http_port": 7777,
       "ws_port": 8777
     },
     "graphql": {
+      "enabled": false,
       "http_port": 9777,
       "ws_port": 6777
     },
     "prometheus": {
+      "enabled": false,
       "listen_port": 8888,
       "polling_interval_ms": 1000
     },

libraries/cli/include/cli/config_jsons/devnet/devnet_config.json

@@ -4,14 +4,17 @@
   "final_chain_cache_in_blocks": 5,
   "network": {
     "rpc": {
+      "enabled": true,
       "http_port": 7777,
       "ws_port": 8777
     },
     "graphql": {
+      "enabled": false,
       "http_port": 9777,
       "ws_port": 6777
     },
     "prometheus": {
+      "enabled": false,
       "listen_port": 8888,
       "polling_interval_ms": 1000
     },

libraries/cli/include/cli/config_jsons/mainnet/mainnet_config.json

@@ -4,14 +4,17 @@
   "final_chain_cache_in_blocks": 5,
   "network": {
     "rpc": {
+      "enabled": true,
       "http_port": 7777,
       "ws_port": 8777
     },
     "graphql": {
+      "enabled": false,
       "http_port": 9777,
       "ws_port": 6777
     },
     "prometheus": {
+      "enabled": false,
       "listen_port": 8888,
       "polling_interval_ms": 1000
     },

libraries/cli/include/cli/config_jsons/testnet/testnet_config.json

@@ -4,14 +4,17 @@
   "final_chain_cache_in_blocks": 5,
   "network": {
     "rpc": {
+      "enabled": true,
       "http_port": 7777,
       "ws_port": 8777
     },
     "graphql": {
+      "enabled": false,
       "http_port": 9777,
       "ws_port": 6777
     },
     "prometheus": {
+      "enabled": false,
       "listen_port": 8888,
       "polling_interval_ms": 1000
     },

libraries/cli/include/cli/config_parser.hpp

@@ -0,0 +1,117 @@
+#pragma once
+
+#include <boost/program_options.hpp>
+#include <string>
+
+#include "cli/configs.hpp"
+#include "config/config.hpp"
+
+namespace taraxa::cli {
+
+class ConfigParser {
+ public:
+  enum class ChainIdType { Mainnet = 841, Testnet, Devnet, LastNetworkId };
+  static constexpr ChainIdType kDefaultChainId = ChainIdType::Mainnet;
+
+ public:
+  /**
+   * @brief Creates FullNodeConfig based on provided command line arguments
+   *
+   * @return empty optional in case created config is invalid or specified command did not require config creation,
+   *         otherwise newly created and parsed config
+   */
+  std::optional<FullNodeConfig> createConfig(int argc, const char* argv[]);
+
+  bool updateConfigFromJson(FullNodeConfig& config, const std::string& json_config_path);
+
+ private:
+  /**
+   * @brief Registeres options that can be specified only through command line
+   *
+   * @return options_description
+   */
+  boost::program_options::options_description registerCliOnlyOptions();
+
+  /**
+   * @brief Registeres options that can be specified in config as well as command line arguments
+   *
+   * @param config object into which are options directly parsed later on
+   * @return options_description
+   */
+  boost::program_options::options_description registerCliConfigOptions(FullNodeConfig& config);
+
+  /**
+   * @brief Updates config based on json config values stored in file (json_config_path)
+   *
+   * @param config
+   * @param json_config_path
+   * @param genesis_path
+   * @param wallet_path
+   * @param cli_options
+   */
+  bool parseJsonConfigFiles(const boost::program_options::options_description& allowed_options,
+                            boost::program_options::variables_map& option_vars, FullNodeConfig& config,
+                            const std::string& json_config_path, std::optional<std::string> genesis_path = {},
+                            std::optional<std::string> wallet_path = {});
+
+  /**
+   * @brief Adds command line arguments values(those that are not also part of config arguments) to the config stucture
+   *
+   * @param config
+   * @param override_boot_nodes
+   * @param append_boot_nodes
+   * @param enable_log_configurations
+   * @param override_log_channels
+   * @param log_channels_append
+   * @param node_secret
+   * @param vrf_secret
+   */
+  static void updateConfigFromCliSpecialOptions(FullNodeConfig& config,
+                                                const std::vector<std::string>& override_boot_nodes,
+                                                const std::vector<std::string>& append_boot_nodes,
+                                                const std::vector<std::string>& enable_log_configurations,
+                                                const std::vector<std::string>& override_log_channels,
+                                                const std::vector<std::string>& append_log_channels,
+                                                const std::string& node_secret, const std::string& vrf_secret);
+
+  /**
+   * @brief Adds config values that are not registered as config cmd options
+   *
+   * @param config
+   * @param config_path
+   * @param genesis_path
+   * @param wallet_path
+   */
+  static void parseUnregisteredConfigOptionsValues(FullNodeConfig& config, const std::string& config_path,
+                                                   const std::optional<std::string>& genesis_path,
+                                                   const std::optional<std::string>& wallet_path);
+
+  /**
+   * @brief Generates default paths and configs in case provided paths are either empty or they dont exist
+   *
+   * @param chain_id
+   * @param config_path
+   * @param genesis_path
+   * @param wallet_path
+   */
+  void generateDefaultPathsAndConfigs(ConfigParser::ChainIdType chain_id, std::string& config_path,
+                                      std::string& genesis_path, std::string& wallet_path);
+
+ private:
+  // Cli or config options that cannot be parsed directly to the FullNodeConfig object due to some required additional
+  // processing or incompatible types
+  std::string genesis_path_, config_path_, wallet_path_, chain_str_, node_secret_, vrf_secret_;
+  std::vector<std::string> command_, boot_nodes_, log_channels_, log_configurations_, boot_nodes_append_,
+      log_channels_append_;
+
+  int chain_id_{static_cast<int>(kDefaultChainId)};
+  bool overwrite_config_{false};
+  bool destroy_db_{false};
+  bool rebuild_network_{false};
+
+  std::string data_path_str_;
+  uint64_t packets_stats_time_period_ms_{0};
+  uint64_t peer_max_packets_processing_time_us_{0};
+};
+
+}  // namespace taraxa::cli

libraries/cli/include/cli/tools.hpp

@@ -5,7 +5,7 @@
 
 #include <string>
 
-#include "cli/config.hpp"
+#include "cli/config_parser.hpp"
 #include "common/vrf_wrapper.hpp"
 
 namespace taraxa::cli::tools {
@@ -29,18 +29,11 @@ void generateVrfFromKey(const std::string& key);
 
 // Generate default config and wallet files
 int getChainIdFromString(std::string& chain_str);
-void getConfig(const std::string& config, cli::Config::ChainIdType chain_id);
-Json::Value getConfig(Config::ChainIdType chain_id);
-Json::Value getGenesis(Config::ChainIdType chain_id);
+void getConfig(const std::string& config, cli::ConfigParser::ChainIdType chain_id);
+Json::Value getConfig(ConfigParser::ChainIdType chain_id);
+Json::Value getGenesis(ConfigParser::ChainIdType chain_id);
 void generateWallet(const std::string& wallet);
 
-// Override existing config and wallet files
-Json::Value overrideConfig(Json::Value& config, std::string& data_dir, std::vector<std::string> boot_nodes,
-                           std::vector<std::string> log_channels, std::vector<std::string> log_configurations,
-                           const std::vector<std::string>& boot_nodes_append,
-                           const std::vector<std::string>& log_channels_append);
-Json::Value overrideWallet(Json::Value& wallet, const std::string& node_key, const std::string& vrf_key);
-
 std::string getHomeDir();
 std::string getTaraxaDefaultDir();
 std::string getTaraxaDataDefaultDir();