diff --git a/addons/apecloud-mysql/Chart.yaml b/addons/apecloud-mysql/Chart.yaml index 491f39c75..e6634cf5c 100644 --- a/addons/apecloud-mysql/Chart.yaml +++ b/addons/apecloud-mysql/Chart.yaml @@ -5,7 +5,7 @@ description: ApeCloud MySQL is a database that is compatible with MySQL syntax a type: application -version: 0.8.0-beta.5 +version: 0.8.0-beta.6 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/addons/apecloud-mysql/config/mysql-scale-vtgate-config-constraint.cue b/addons/apecloud-mysql/config/mysql-scale-vtgate-config-constraint.cue index b27a6d5e5..1dade22f6 100644 --- a/addons/apecloud-mysql/config/mysql-scale-vtgate-config-constraint.cue +++ b/addons/apecloud-mysql/config/mysql-scale-vtgate-config-constraint.cue @@ -77,12 +77,27 @@ //Path to ssl key for mysql server plugin SSL mysql_server_ssl_key: string + //Path to the ssl ca for mysql server plugin SSL + mysql_server_ssl_ca: string + //Path to the ssl cert for mysql server plugin SSL mysql_server_ssl_cert: string //Reject insecure connections but only if mysql_server_ssl_cert and mysql_server_ssl_key are provided.(default "false") mysql_server_require_secure_transport: bool + // Set default strategy for DDL statements. Override with @@ddl_strategy session variable + ddl_strategy: string & "direct" | "online" | "mysql" + + // Enable or disable the feature of showing information about the vttablet node which executing the SQL. (default false) + enable_display_sql_execution_vttablets: bool + + // Enable or disable the feature of read write splitting for read only txn (default false) + enable_read_write_split_for_read_only_txn: bool + + // Enable or disable the feature of interception for DML without where clause (default true) + enable_interception_for_dml_without_where: bool + ... } diff --git a/addons/apecloud-mysql/config/mysql-scale-vtgate-config-effect-scope.yaml b/addons/apecloud-mysql/config/mysql-scale-vtgate-config-effect-scope.yaml index 4296cf1a3..13ce9648d 100644 --- a/addons/apecloud-mysql/config/mysql-scale-vtgate-config-effect-scope.yaml +++ b/addons/apecloud-mysql/config/mysql-scale-vtgate-config-effect-scope.yaml @@ -17,10 +17,7 @@ staticParameters: - buffer_max_failover_duration - buffer_min_time_between_failovers - mysql_auth_server_impl - - mysql_server_require_secure_transport - mysql_auth_server_static_file - - mysql_server_ssl_key - - mysql_server_ssl_cert - enable_logs - enable_query_log @@ -28,4 +25,12 @@ dynamicParameters: - read_write_splitting_policy - read_write_splitting_ratio - read_after_write_consistency - - read_after_write_timeout \ No newline at end of file + - read_after_write_timeout + - ddl_strategy + - enable_display_sql_execution_vttablets + - enable_read_write_split_for_read_only_txn + - mysql_server_ssl_key + - mysql_server_ssl_cert + - mysql_server_ssl_ca + - mysql_server_require_secure_transport + - enable_interception_for_dml_without_where \ No newline at end of file diff --git a/addons/apecloud-mysql/config/mysql-scale-vtgate-config.tpl b/addons/apecloud-mysql/config/mysql-scale-vtgate-config.tpl index aa9618a75..d0a30f65e 100644 --- a/addons/apecloud-mysql/config/mysql-scale-vtgate-config.tpl +++ b/addons/apecloud-mysql/config/mysql-scale-vtgate-config.tpl @@ -14,11 +14,9 @@ buffer_size=10000 buffer_window=30s buffer_max_failover_duration=60s buffer_min_time_between_failovers=60s -mysql_auth_server_impl=none -mysql_server_require_secure_transport=false +mysql_auth_server_impl=mysqlbased mysql_auth_server_static_file= -mysql_server_ssl_key= -mysql_server_ssl_cert= +mysql_server_require_secure_transport=false {{ block "logsBlock" . }} {{- if hasKey $.component "enabledLogs" }} enable_logs=true @@ -27,3 +25,21 @@ enable_query_log=true {{- end }} {{- end }} {{ end }} +ddl_strategy=direct +enable_display_sql_execution_vttablets=false +enable_read_write_split_for_read_only_txn=false +enable_interception_for_dml_without_where=true + +{{- if $.component.tlsConfig }} +{{- $ca_file := getCAFile }} +{{- $cert_file := getCertFile }} +{{- $key_file := getKeyFile }} +mysql_server_ssl_ca={{ $ca_file }} +mysql_server_ssl_cert={{ $cert_file }} +mysql_server_ssl_key={{ $key_file }} +# tls +{{- else }} +mysql_server_ssl_ca= +mysql_server_ssl_cert= +mysql_server_ssl_key= +{{- end }} diff --git a/addons/apecloud-mysql/config/mysql-scale-vttablet-config-constraint.cue b/addons/apecloud-mysql/config/mysql-scale-vttablet-config-constraint.cue index 2bc970e1c..c149cbb29 100644 --- a/addons/apecloud-mysql/config/mysql-scale-vttablet-config-constraint.cue +++ b/addons/apecloud-mysql/config/mysql-scale-vttablet-config-constraint.cue @@ -59,6 +59,30 @@ // query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout) queryserver_config_transaction_cap: int & >=0 + // the size of database connection pool in non transaction dml + non_transactional_dml_database_pool_size: int & >=1 + + // the number of rows to be processed in one batch by default + non_transactional_dml_default_batch_size: int & >=1 + + // the interval of batch processing in milliseconds by default + non_transactional_dml_default_batch_interval: int & >=1 + + // the interval of table GC in hours + non_transactional_dml_table_gc_interval: int & >=1 + + // the interval of job scheduler running in seconds + non_transactional_dml_job_manager_running_interval: int & >=1 + + // the interval of throttle check in milliseconds + non_transactional_dml_throttle_check_interval: int & >=1 + + // the threshold of batch size + non_transactional_dml_batch_size_threshold: int & >=1 & <=1000000 + + // final threshold = ratio * non_transactional_dml_batch_size_threshold / table index numbers + non_transactional_dml_batch_size_threshold_ratio: float & >=0 & <=1 + ... } diff --git a/addons/apecloud-mysql/config/mysql-scale-vttablet-config-effect-scope.yaml b/addons/apecloud-mysql/config/mysql-scale-vttablet-config-effect-scope.yaml index 1ccd4e0c2..4da340d46 100644 --- a/addons/apecloud-mysql/config/mysql-scale-vttablet-config-effect-scope.yaml +++ b/addons/apecloud-mysql/config/mysql-scale-vttablet-config-effect-scope.yaml @@ -20,4 +20,12 @@ staticParameters: dynamicParameters: - queryserver_config_pool_size - queryserver_config_stream_pool_size - - queryserver_config_transaction_cap \ No newline at end of file + - queryserver_config_transaction_cap + - non_transactional_dml_database_pool_size + - non_transactional_dml_default_batch_size + - non_transactional_dml_default_batch_interval + - non_transactional_dml_table_gc_interval + - non_transactional_dml_job_manager_running_interval + - non_transactional_dml_throttle_check_interval + - non_transactional_dml_batch_size_threshold + - non_transactional_dml_batch_size_threshold_ratio \ No newline at end of file diff --git a/addons/apecloud-mysql/config/mysql-scale-vttablet-config.tpl b/addons/apecloud-mysql/config/mysql-scale-vttablet-config.tpl index 3bea5d9d4..0e71ad355 100644 --- a/addons/apecloud-mysql/config/mysql-scale-vttablet-config.tpl +++ b/addons/apecloud-mysql/config/mysql-scale-vttablet-config.tpl @@ -3,19 +3,60 @@ health_check_interval=1s shard_sync_retry_delay=1s remote_operation_timeout=1s db_connect_timeout_ms=500 -table_acl_config_mode=simple +table_acl_config_mode=mysqlbased enable_logs=true enable_query_log=true table_acl_config= -queryserver_config_strict_table_acl=false -table_acl_config_reload_interval=30s +queryserver_config_strict_table_acl=true +table_acl_config_reload_interval=5s enforce_tableacl_config=false +{{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} +{{- $thread_stack := 262144 }} +{{- $binlog_cache_size := 32768 }} +{{- $join_buffer_size := 262144 }} +{{- $sort_buffer_size := 262144 }} +{{- $read_buffer_size := 262144 }} +{{- $read_rnd_buffer_size := 524288 }} +{{- $single_thread_memory := add $thread_stack $binlog_cache_size $join_buffer_size $sort_buffer_size $read_buffer_size $read_rnd_buffer_size }} +{{- if gt $phy_memory 0 }} +# max_connections={{ div ( div $phy_memory 4 ) $single_thread_memory }} +{{- end}} + +{{- $max_connections := div ( div $phy_memory 4 ) $single_thread_memory }} +# 10 percentage +{{- $pool_k := max 1 ( div (sub $max_connections 35) 10 ) }} + +# TxPool +queryserver_config_transaction_cap={{ mul 5 $pool_k }} + # OltpReadPool -queryserver_config_pool_size=30 +queryserver_config_pool_size={{ mul 4 $pool_k }} # OlapReadPool -queryserver_config_stream_pool_size=30 +queryserver_config_stream_pool_size={{ mul $pool_k }} -# TxPool -queryserver_config_transaction_cap=50 \ No newline at end of file + +# the size of database connection pool in non transaction dml +non_transactional_dml_database_pool_size=3 + +# the number of rows to be processed in one batch by default +non_transactional_dml_default_batch_size=2000 + +# the interval of batch processing in milliseconds by default +non_transactional_dml_default_batch_interval=1 + +# the interval of table GC in hours +non_transactional_dml_table_gc_interval=24 + +# the interval of job scheduler running in seconds +non_transactional_dml_job_manager_running_interval=24 + +# the interval of throttle check in milliseconds +non_transactional_dml_throttle_check_interval=250 + +# the threshold of batch size +non_transactional_dml_batch_size_threshold=10000 + +# final threshold = ratio * non_transactional_dml_batch_size_threshold / table index numbers +non_transactional_dml_batch_size_threshold_ratio=0.5 \ No newline at end of file diff --git a/addons/apecloud-mysql/scripts/set_config_variables.sh b/addons/apecloud-mysql/scripts/set_config_variables.sh index 6d1e401be..f9ab67302 100755 --- a/addons/apecloud-mysql/scripts/set_config_variables.sh +++ b/addons/apecloud-mysql/scripts/set_config_variables.sh @@ -5,7 +5,7 @@ function set_config_variables(){ config_content=$(sed -n '/\['$1'\]/,/\[/ { /\['$1'\]/d; /\[/q; p; }' $config_file) while read line do - if [[ $line =~ ^[a-zA-Z_][a-zA-Z0-9_]*=[a-zA-Z0-9_.]*$ ]]; then + if [[ $line =~ ^[a-zA-Z_][a-zA-Z0-9_]*=[a-zA-Z0-9_./-]*$ ]]; then echo $line eval "export $line" elif ! [[ -z $line || $line =~ ^[[:space:]]*# ]]; then diff --git a/addons/apecloud-mysql/scripts/vtgate.sh b/addons/apecloud-mysql/scripts/vtgate.sh index 02a3dab86..bcb312843 100755 --- a/addons/apecloud-mysql/scripts/vtgate.sh +++ b/addons/apecloud-mysql/scripts/vtgate.sh @@ -13,11 +13,6 @@ su vitess <