-
Notifications
You must be signed in to change notification settings - Fork 199
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Have basics of a configuration generation tool for expanded testing. …
…I foresee much more expanded functionality in the future and a lot of potential. This will require some possible rework of some existing code, however, as well as integration tests to make use of this proposed functionality. Also reorganized config files into their own directory which seems neater. Also needed to fix docker stff. Might need to revert docker stuff. Not sure how that will affect others. Signed-off-by: Yiannis Karavas <[email protected]>
- Loading branch information
Showing
15 changed files
with
1,134 additions
and
500 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| // Copyright (c) 2022 MIT Digital Currency Initiative, | ||
| // Federal Reserve Bank of Boston | ||
| // MITRE Corporation | ||
| // Distributed under the MIT software license, see the accompanying | ||
| // file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||
|
|
||
| #ifndef OPENCBDC_TX_CONFIG_TOOLS_CONFIG_GENERATOR_H_ | ||
| #define OPENCBDC_TX_CONFIG_TOOLS_CONFIG_GENERATOR_H_ | ||
|
|
||
| #include "util/common/config.hpp" | ||
| #include "util/common/random_source.hpp" | ||
|
|
||
| #include <random> | ||
| #include <secp256k1.h> | ||
|
|
||
| #define MAX_PORT_NUM 65535 | ||
|
|
||
| // Structure to assist in creating shard id coverage | ||
| using ShardInfo = struct shard_info_struct { | ||
| std::vector<size_t> coverage; | ||
| size_t shard_id; | ||
| size_t numbers_covered; | ||
| double overlap_percentage_allowed; | ||
| bool still_expanding; | ||
| bool allow_overlap; | ||
| std::pair<size_t, size_t> current_coverage_expansion_limits; | ||
| }; | ||
|
|
||
| using value_t = std::variant<std::string, size_t, double>; | ||
|
|
||
| namespace cbdc::generate_config { | ||
| /// \brief Config_generator implementation. | ||
| /// | ||
| /// This class takes in a template configuration file, denoted by the | ||
| /// ending "*.tmpl" and produces are usable (*.cfg) configuration file. The | ||
| /// purpose of this class is to allow the user to create more complicated | ||
| /// testing scenarios by removing some amount of manual effort when | ||
| /// creating configurations. | ||
| class config_generator { | ||
| public: | ||
| /// Constructor. | ||
| /// | ||
| /// \param _template_config_file The template configuration file from which | ||
| /// the larger more intricate configuration file will be generated. | ||
| /// \param _start_port Port to begin using and incrementing from for generated | ||
| /// configuration file's endpoints | ||
| config_generator(std::string& _template_config_file, | ||
| size_t _start_port); | ||
|
|
||
| ~config_generator() = default; | ||
|
|
||
| /// \brief generate_configuration_file | ||
| /// Main workhorse method of this class. This method will generate a | ||
| /// usable configuration file | ||
| /// | ||
| /// \return a string with all the error/warning/success messages, if any, | ||
| /// that were produced while executing | ||
| [[maybe_unused]] auto generate_configuration_file() -> std::string; | ||
|
|
||
| private: | ||
| // Boolean that tells us if file is valid or not | ||
| bool template_file_is_valid; | ||
| // Template file loaded to create configuration file from | ||
| std::string& m_template_config_file; | ||
| // Incrementing port to use in config file for all ports | ||
| unsigned short m_current_port; | ||
| // Map with shard ranges (shard_id, (start range, end_range) | ||
| std::vector<ShardInfo> shard_info; | ||
| std::default_random_engine generator; | ||
| static const inline auto m_random_source | ||
| = std::make_unique<cbdc::random_source>( | ||
| cbdc::config::random_source); | ||
| static const inline auto m_secp | ||
| = std::unique_ptr<secp256k1_context, | ||
| decltype(&secp256k1_context_destroy)>( | ||
| secp256k1_context_create(SECP256K1_CONTEXT_SIGN), | ||
| &secp256k1_context_destroy); | ||
|
|
||
| // Where the newly created configuration parameters will be stored as | ||
| // we go along generating them | ||
| std::stringstream m_new_config; | ||
|
|
||
| std::map<std::string, value_t> template_options; | ||
|
|
||
| // Calculate shard coverage | ||
| void calculate_shard_coverage(size_t num_shards, | ||
| bool randomize, | ||
| size_t shard_size); | ||
| // Helper for calculating shard id coverage | ||
| void shard_bookkeeping(const std::vector<size_t>& array_total, | ||
| size_t shard_id); | ||
| // Get random value based on mean and standard deviation | ||
| [[nodiscard]] auto calculate_normal_distribution_point(size_t mean, | ||
| double std_dev, | ||
| bool randomize) | ||
| -> double; | ||
| // This is not a failsafe because we are simply generating a | ||
| // configuration file here, however, it is still good to check that the | ||
| // port is available | ||
| [[nodiscard]] auto get_open_port() -> unsigned short; | ||
| // Create Private/Public key pair repeatably (repeatable from run to | ||
| // run) | ||
| [[nodiscard]] auto create_repeatable_key_pair() | ||
| -> std::pair<std::string, std::string>; | ||
| // Create Private/Public key pair randomly (NOT repeatable from run to | ||
| // run) | ||
| [[nodiscard]] auto create_random_key_pair() | ||
| -> std::pair<std::string, std::string>; | ||
| // Create Private/Public key pair | ||
| [[nodiscard]] auto create_key_pair(bool randomize) | ||
| -> std::pair<std::string, std::string>; | ||
| // Parse value as int, double or string from config file | ||
| [[nodiscard]] auto parse_value(const std::string& value, | ||
| bool keep_quotes) -> value_t; | ||
| [[nodiscard]] auto get_param_from_template_file( | ||
| std::string option, | ||
| std::map<std::string, std::string>& config_map) | ||
| -> std::variant<size_t, double, std::string>; | ||
| void set_param_to_config_file(std::string key, std::string value); | ||
| void set_param_to_config_file(std::string key, size_t value); | ||
| void set_param_to_config_file(std::string key, double value); | ||
| // Helper to set proper log level for either Two-Phase Commit or | ||
| // Atomizer components | ||
| void set_log_level(std::string key, std::string& log_level); | ||
| // Method to create all the Two-Phase Commit related components for | ||
| // generated config file | ||
| void create_2pc_component(const char* type, size_t number); | ||
| // Method to create all the Atomizer related components for generated | ||
| // config file | ||
| void create_atomizer_component(const char* type, size_t number); | ||
| // Load template file from which we will generate the configuration | ||
| // file. | ||
| void load_template(std::string filename, | ||
| std::map<std::string, std::string>& config_map); | ||
| // Write out the generated configuration file out to | ||
| // <project_root>/build/config/tools | ||
| void write_generated_config_to_file(const std::string& _config_file); | ||
| // Copies the generated *.cfg file to the <project_root>/build | ||
| // directory | ||
| [[nodiscard]] auto copy_to_build_dir(std::string filename) -> bool; | ||
| // Copies the *.tmpl file to the <project_root>/build/config/tools | ||
| // directory | ||
| void copy_templates_to_build_dir(); | ||
| template<typename T> | ||
| [[maybe_unused]] auto find_value(std::string key, T& output) -> bool; | ||
| }; | ||
| } | ||
|
|
||
| #endif // OPENCBDC_TX_CONFIG_TOOLS_CONFIG_GENERATOR_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| #include "config_generator.hpp" | ||
|
|
||
| #include <iostream> | ||
| #include <random> | ||
|
|
||
| auto main(int argc, char** argv) -> int { | ||
| auto args = cbdc::config::get_args(argc, argv); | ||
|
|
||
| std::mt19937 rng; | ||
| static const auto m_random_source | ||
| = std::make_unique<cbdc::random_source>(cbdc::config::random_source); | ||
|
|
||
| if(args.size() < 3) { | ||
| std::cerr << "Usage: " << args[0] | ||
| << " <config template file> <starting port number to " | ||
| "increment from>" | ||
| << std::endl; | ||
| return 0; | ||
| } | ||
|
|
||
| auto port_is_valid = std::isdigit(*args[2].c_str()); | ||
| if(!port_is_valid) { | ||
| std::cerr << "Port number provided, " << args[2] | ||
| << ", is not a valid number. Exiting..." << std::endl; | ||
| return 0; | ||
| } | ||
| size_t port_num = static_cast<size_t>(std::stoull(args[2])); | ||
| if(port_num > MAX_PORT_NUM) { | ||
| std::cerr << "Port number provided, " << args[2] | ||
| << ", is too large. Exiting..." << std::endl; | ||
| return 0; | ||
| } | ||
| cbdc::generate_config::config_generator new_config_gen(args[1], port_num); | ||
| auto cfg_or_err = new_config_gen.generate_configuration_file(); | ||
| std::cout << cfg_or_err << std::endl; | ||
| return 0; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| 2pc=1 | ||
| sentinel_count=1 | ||
| shard_count=1 | ||
| coordinator_count=1 | ||
| coordinator_max_threads=1 | ||
| election_timeout_upper=4000 | ||
| election_timeout_lower=3000 | ||
| heartbeat=1000 | ||
| raft_max_batch=100000 | ||
| snapshot_distance=1000000000 | ||
| batch_size=1 | ||
| wait_for_followers=0 | ||
| loadgen_invalid_tx_rate=0.00 | ||
| loadgen_fixed_tx_rate=0.01 | ||
| loadgen_sendtx_output_count=1 | ||
| loadgen_sendtx_input_count=1 | ||
| initial_mint_count=20000 | ||
| initial_mint_value=100 | ||
| tmpl_randomize_values=1 | ||
| tmpl_shard_start=0 | ||
| tmpl_shard_size=255 | ||
| tmpl_avg_shard_start_end_overlap_percent=0.15 | ||
| tmpl_default_log_level="INFO" | ||
| tmpl_universal_override_log_level="WARN" | ||
| tmpl_sentinel_log_level="WARN" | ||
| tmpl_coordinator_log_level="DEBUG" | ||
| tmpl_shard_log_level="DEBUG" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| 2pc=0 | ||
| archiver_count=1 | ||
| atomizer_count=1 | ||
| shard_count=1 | ||
| sentinel_count=1 | ||
| watchtower_count=1 | ||
| target_block_interval=3000 | ||
| stxo_cache_depth=2 | ||
| target_block_interval=250 | ||
| election_timeout_upper=4000 | ||
| election_timeout_lower=3000 | ||
| heartbeat=1000 | ||
| raft_max_batch=100000 | ||
| snapshot_distance=1000000000 | ||
| batch_size=1 | ||
| wait_for_followers=0 | ||
| loadgen_invalid_tx_rate=0.00 | ||
| loadgen_fixed_tx_rate=0.01 | ||
| loadgen_sendtx_output_count=1 | ||
| loadgen_sendtx_input_count=1 | ||
| initial_mint_count=20000 | ||
| initial_mint_value=100 | ||
| tmpl_randomize_values=1 | ||
| tmpl_shard_start=0 | ||
| tmpl_shard_size=255 | ||
| tmpl_avg_shard_start_end_overlap_percent=0.15 | ||
| tmpl_default_log_level="INFO" | ||
| tmpl_sentinel_log_level="DEBUG" | ||
| tmpl_shard_log_level="INFO" | ||
| tmpl_watchtower_log_level="DEBUG" | ||
| tmpl_archiver_log_level="DEBUG" | ||
| tmpl_atomizer_log_level="DEBUG" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.