diff --git a/docs/presto-admin-commands.rst b/docs/presto-admin-commands.rst index ff7ebfb6..236e498f 100644 --- a/docs/presto-admin-commands.rst +++ b/docs/presto-admin-commands.rst @@ -132,6 +132,7 @@ For workers: :: http-server.http.port=8080 query.max-memory-per-node=1GB query.max-memory=50GB + task.max-memory=1GB For coordinator: :: @@ -142,6 +143,7 @@ For coordinator: :: node.scheduler.include-coordinator=false query.max-memory-per-node=1GB query.max-memory=50GB + task.max-memory=1GB # if the coordinator is also a worker, it will have the following property instead node-scheduler.include-coordinator=true diff --git a/prestoadmin/coordinator.py b/prestoadmin/coordinator.py index eaa4989d..192a11d4 100644 --- a/prestoadmin/coordinator.py +++ b/prestoadmin/coordinator.py @@ -46,6 +46,7 @@ 'coordinator': 'true', 'discovery-server.enabled': 'true', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'node-scheduler.include-coordinator': 'false', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} diff --git a/prestoadmin/workers.py b/prestoadmin/workers.py index 00662013..a5b7d5a0 100644 --- a/prestoadmin/workers.py +++ b/prestoadmin/workers.py @@ -47,6 +47,7 @@ '-DHADOOP_USER_NAME=hive'], 'config.properties': {'coordinator': 'false', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} } diff --git a/tests/product/base_product_case.py b/tests/product/base_product_case.py index aa112d56..60260203 100644 --- a/tests/product/base_product_case.py +++ b/tests/product/base_product_case.py @@ -47,13 +47,15 @@ class BaseProductTestCase(BaseTestCase): discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=1GB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" default_workers_test_config_ = """coordinator=false discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" default_node_properties_ = """node.data-dir=/var/lib/presto/data node.environment=presto @@ -76,7 +78,8 @@ class BaseProductTestCase(BaseTestCase): http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=1GB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" default_coordinator_test_config_ = """coordinator=true discovery-server.enabled=true @@ -84,7 +87,8 @@ class BaseProductTestCase(BaseTestCase): http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" down_node_connection_string = r'(\nWarning: (\[%(host)s\] )?Low level socket ' \ r'error connecting to host %(host)s on ' \ diff --git a/tests/product/resources/configuration_show_config.txt b/tests/product/resources/configuration_show_config.txt index dc4f98bc..83673709 100644 --- a/tests/product/resources/configuration_show_config.txt +++ b/tests/product/resources/configuration_show_config.txt @@ -7,6 +7,7 @@ http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave1: Configuration file at /etc/presto/config.properties: @@ -15,6 +16,7 @@ discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave2: Configuration file at /etc/presto/config.properties: @@ -23,6 +25,7 @@ discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave3: Configuration file at /etc/presto/config.properties: @@ -31,4 +34,5 @@ discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB diff --git a/tests/product/resources/configuration_show_default.txt b/tests/product/resources/configuration_show_default.txt index ffcf9b88..30d75add 100644 --- a/tests/product/resources/configuration_show_default.txt +++ b/tests/product/resources/configuration_show_default.txt @@ -26,6 +26,7 @@ http-server.http.port=8080 node.scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave1: Configuration file at /etc/presto/node.properties: @@ -54,6 +55,7 @@ discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave2: Configuration file at /etc/presto/node.properties: @@ -82,6 +84,7 @@ discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave3: Configuration file at /etc/presto/node.properties: @@ -110,3 +113,4 @@ discovery.uri=http://master:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB diff --git a/tests/product/resources/configuration_show_down_node.txt b/tests/product/resources/configuration_show_down_node.txt index 519608af..101e3fcc 100644 --- a/tests/product/resources/configuration_show_down_node.txt +++ b/tests/product/resources/configuration_show_down_node.txt @@ -5,6 +5,7 @@ discovery.uri=http://.*:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave2: Configuration file at /etc/presto/config.properties: @@ -13,6 +14,7 @@ discovery.uri=http://.*:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB slave3: Configuration file at /etc/presto/config.properties: @@ -21,3 +23,4 @@ discovery.uri=http://.*:8080 http-server.http.port=8080 query.max-memory-per-node=512MB query.max-memory=50GB +task.max-memory=1GB diff --git a/tests/product/test_server_install.py b/tests/product/test_server_install.py index a20d62e1..c4016c9e 100644 --- a/tests/product/test_server_install.py +++ b/tests/product/test_server_install.py @@ -110,7 +110,8 @@ class TestServerInstall(BaseProductTestCase): discovery.uri=http://slave1:8080 http-server.http.port=8080 query.max-memory-per-node=512MB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" default_coord_config_with_slave1_ = """coordinator=true discovery-server.enabled=true @@ -118,13 +119,15 @@ class TestServerInstall(BaseProductTestCase): http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" default_workers_config_regex_ = """coordinator=false discovery.uri=http:.*:8080 http-server.http.port=8080 query.max-memory-per-node=512MB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" default_coord_config_regex_ = """coordinator=true discovery-server.enabled=true @@ -132,7 +135,8 @@ class TestServerInstall(BaseProductTestCase): http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB -query.max-memory=50GB\n""" +query.max-memory=50GB +task.max-memory=1GB\n""" def setUp(self): super(TestServerInstall, self).setUp() diff --git a/tests/unit/test_coordinator.py b/tests/unit/test_coordinator.py index e461e8b5..a9fe38a1 100644 --- a/tests/unit/test_coordinator.py +++ b/tests/unit/test_coordinator.py @@ -47,6 +47,7 @@ def test_build_defaults(self): 'discovery-server.enabled': 'true', 'discovery.uri': 'http://a:8080', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'node-scheduler.include-coordinator': 'false', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} @@ -77,6 +78,7 @@ def test_defaults_coord_is_worker(self): 'discovery-server.enabled': 'true', 'discovery.uri': 'http://a:8080', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'node-scheduler.include-coordinator': 'true', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} @@ -165,6 +167,7 @@ def test_get_conf(self, get_conf_from_file_mock): 'discovery-server.enabled': 'true', 'discovery.uri': 'http://j:8080', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'node-scheduler.include-coordinator': 'false', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} diff --git a/tests/unit/test_workers.py b/tests/unit/test_workers.py index 4bce0fe4..bab5cf81 100644 --- a/tests/unit/test_workers.py +++ b/tests/unit/test_workers.py @@ -46,6 +46,7 @@ def test_build_defaults(self): 'config.properties': {'coordinator': 'false', 'discovery.uri': 'http://a:8080', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} } @@ -113,6 +114,7 @@ def test_get_conf(self, get_presto_conf_mock): 'config.properties': {'coordinator': 'false', 'discovery.uri': 'http://j:8080', 'http-server.http.port': '8080', + 'task.max-memory': '1GB', 'query.max-memory': '50GB', 'query.max-memory-per-node': '1GB'} }