From 8d0682cd4a24d71e8763e5e340dcf1e8192fa5c2 Mon Sep 17 00:00:00 2001 From: Konjac Huang Date: Wed, 16 Oct 2024 03:26:23 -0700 Subject: [PATCH] Add session property for jdbc metadata, pool size, caching enabled --- .../presto/plugin/jdbc/JdbcMetadataCache.java | 14 ++++- .../plugin/jdbc/JdbcMetadataConfig.java | 27 +++++++++ ...JdbcMetadataSessionPropertiesProvider.java | 59 +++++++++++++++++++ .../plugin/jdbc/TestJdbcMetadataConfig.java | 10 +++- 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java index 8ef7d92a0a1e..ec10b3646060 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java @@ -30,11 +30,12 @@ import java.util.concurrent.ExecutorService; import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.plugin.jdbc.JdbcMetadataSessionPropertiesProvider.isJdbcMetadataCacheEnabled; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.cache.CacheLoader.asyncReloading; import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.concurrent.TimeUnit.MILLISECONDS; public class JdbcMetadataCache @@ -48,7 +49,7 @@ public class JdbcMetadataCache public JdbcMetadataCache(JdbcClient jdbcClient, JdbcMetadataConfig config, JdbcMetadataCacheStats stats) { this( - newCachedThreadPool(daemonThreadsNamed("jdbc-metadata-cache" + "-%s")), + newFixedThreadPool(config.getMetadataCacheThreadPoolSize(), daemonThreadsNamed("jdbc-metadata-cache" + "-%s")), jdbcClient, stats, OptionalLong.of(config.getMetadataCacheTtl().toMillis()), @@ -65,7 +66,6 @@ public JdbcMetadataCache( long cacheMaximumSize) { this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); - this.tableHandleCache = newCacheBuilder(cacheTtl, refreshInterval, cacheMaximumSize) .build(asyncReloading(CacheLoader.from(this::loadTableHandle), executor)); stats.setTableHandleCache(tableHandleCache); @@ -77,11 +77,19 @@ public JdbcMetadataCache( public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { + if (!isJdbcMetadataCacheEnabled(session)) { + return jdbcClient.getTableHandle(session, JdbcIdentity.from(session), tableName); + } + return get(tableHandleCache, new KeyAndSession<>(session, tableName)).orElse(null); } public List getColumns(ConnectorSession session, JdbcTableHandle jdbcTableHandle) { + if (!isJdbcMetadataCacheEnabled(session)) { + return jdbcClient.getColumns(session, jdbcTableHandle); + } + return get(columnHandlesCache, new KeyAndSession<>(session, jdbcTableHandle)); } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java index d2bd555f2c53..d8d8b5000169 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java @@ -29,6 +29,8 @@ public class JdbcMetadataConfig private Duration metadataCacheTtl = new Duration(0, TimeUnit.SECONDS); private Duration metadataCacheRefreshInterval = new Duration(0, TimeUnit.SECONDS); private long metadataCacheMaximumSize = 10000; + private int metadataCacheThreadPoolSize = 10; + private boolean jdbcMetadataCacheEnabled; public boolean isAllowDropTable() { @@ -83,4 +85,29 @@ public JdbcMetadataConfig setMetadataCacheMaximumSize(long metadataCacheMaximumS this.metadataCacheMaximumSize = metadataCacheMaximumSize; return this; } + + public int getMetadataCacheThreadPoolSize() + { + return metadataCacheThreadPoolSize; + } + + @Min(1) + @Config("metadata-cache-thread-pool-size") + public JdbcMetadataConfig setMetadataCacheThreadPoolSize(int threadPoolSize) + { + this.metadataCacheThreadPoolSize = threadPoolSize; + return this; + } + + public boolean getJdbcMetadataCacheEnabled() + { + return jdbcMetadataCacheEnabled; + } + + @Config("metadata-jdbc-cache-enabled") + public JdbcMetadataConfig setJdbcMetadataCacheEnabled(boolean jdbcMetadataCacheEnabled) + { + this.jdbcMetadataCacheEnabled = jdbcMetadataCacheEnabled; + return this; + } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java new file mode 100644 index 000000000000..83bd200cc16d --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.session.PropertyMetadata; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.spi.session.PropertyMetadata.booleanProperty; + +public class JdbcMetadataSessionPropertiesProvider + implements JdbcSessionPropertiesProvider +{ + private final List> sessionProperties; + private static final String JDBC_METADATA_CACHE_ENABLED = "jdbc_metadata_cache_enabled"; + + @Inject + public JdbcMetadataSessionPropertiesProvider(JdbcMetadataConfig jdbcMetadataConfig) + { + this.sessionProperties = ImmutableList.of( + booleanProperty( + JDBC_METADATA_CACHE_ENABLED, + "Whether Jdbc Metadata In Memory Cache is Enabled", + jdbcMetadataConfig.getJdbcMetadataCacheEnabled(), + false)); + } + + @Override + public List> getSessionProperties() + { + return sessionProperties; + } + + public static boolean isJdbcMetadataCacheEnabled(ConnectorSession session) + { + try { + return session.getProperty(JDBC_METADATA_CACHE_ENABLED, Boolean.class); + } + catch (Exception e) { + return false; + } + } +} + diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java index 6477239814c8..73f49fe9179d 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java @@ -34,7 +34,9 @@ public void testDefaults() .setAllowDropTable(false) .setMetadataCacheTtl(new Duration(0, SECONDS)) .setMetadataCacheRefreshInterval(new Duration(0, SECONDS)) - .setMetadataCacheMaximumSize(10000)); + .setMetadataCacheMaximumSize(10000) + .setMetadataCacheThreadPoolSize(10) + .setJdbcMetadataCacheEnabled(false)); } @Test @@ -45,13 +47,17 @@ public void testExplicitPropertyMappings() .put("metadata-cache-ttl", "1h") .put("metadata-cache-refresh-interval", "10s") .put("metadata-cache-maximum-size", "100") + .put("metadata-cache-thread-pool-size", "50") + .put("metadata-jdbc-cache-enabled", "true") .build(); JdbcMetadataConfig expected = new JdbcMetadataConfig() .setAllowDropTable(true) .setMetadataCacheTtl(new Duration(1, HOURS)) .setMetadataCacheRefreshInterval(new Duration(10, SECONDS)) - .setMetadataCacheMaximumSize(100); + .setMetadataCacheMaximumSize(100) + .setMetadataCacheThreadPoolSize(50) + .setJdbcMetadataCacheEnabled(true); assertFullMapping(properties, expected); }