From f1280d8f00815cd1acd35c345bb0a8cf568505d9 Mon Sep 17 00:00:00 2001 From: linghengqian Date: Fri, 31 May 2024 13:31:32 +0800 Subject: [PATCH] Add GraalVM Reachability Metadata and corresponding nativeTest for Iceberg table in HiveServer2 --- .../graalvm-native-image/_index.cn.md | 158 ++++++++++++++++- .../graalvm-native-image/_index.en.md | 160 ++++++++++++++++++ .../proxy-config.json | 4 + .../reflect-config.json | 60 +++++-- .../resource-config.json | 3 + .../reflect-config.json | 25 +++ .../native-image-filter/extra-filter.json | 54 ++---- test/native/pom.xml | 23 +++ .../jdbc/commons/TestShardingService.java | 43 +++++ .../commons/repository/AddressRepository.java | 35 ++++ .../repository/OrderItemRepository.java | 23 +++ .../commons/repository/OrderRepository.java | 23 +++ .../test/natived/jdbc/databases/HiveTest.java | 134 +++++++++++++++ .../grpc-netty-shaded/native-image.properties | 46 +++++ .../sql/test-native-databases-hive.sql | 53 ++++++ .../test-native/yaml/databases/hive.yaml | 72 ++++++++ 16 files changed, 854 insertions(+), 62 deletions(-) create mode 100644 test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/HiveTest.java create mode 100644 test/native/src/test/resources/META-INF/native-image/io.grpc/grpc-netty-shaded/native-image.properties create mode 100644 test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql create mode 100644 test/native/src/test/resources/test-native/yaml/databases/hive.yaml diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md index 4879622e9d6aa..00e3b8d6c8dbc 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md @@ -327,6 +327,162 @@ Caused by: java.io.UnsupportedEncodingException: Codepage Cp1252 is not supporte ClickHouse 不支持 ShardingSphere 集成级别的本地事务，XA 事务和 Seata AT 模式事务，更多讨论位于 https://github.com/ClickHouse/clickhouse-docs/issues/2300 。 +7. 当需要通过 ShardingSphere JDBC 使用 Hive 方言时，受 https://issues.apache.org/jira/browse/HIVE-28308 影响， +用户不应该使用 `classifier` 为 `standalone` 的 `org.apache.hive:hive-jdbc:4.0.0`，以避免依赖冲突。 +可能的配置例子如下， + +```xml + + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-infra-database-hive + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-hive + ${shardingsphere.version} + + + org.apache.hive + hive-jdbc + 4.0.0 + + + org.apache.hive + hive-service + 4.0.0 + + + org.apache.hadoop + hadoop-client-api + 3.3.6 + + + +``` + +这会导致大量的依赖冲突。 +如果用户不希望手动解决潜在的数千行的依赖冲突，可以使用 HiveServer2 JDBC Driver 的 `Thin JAR` 的第三方构建。 +可能的配置例子如下， + +```xml + + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-infra-database-hive + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-hive + ${shardingsphere.version} + + + io.github.linghengqian + hive-server2-jdbc-driver-thin + 1.0.0 + + + com.fasterxml.woodstox + woodstox-core + + + + + +``` + +受 https://github.com/grpc/grpc-java/issues/10601 影响，用户如果在项目中引入了 `org.apache.hive:hive-jdbc`， +则需要在项目的 classpath 的 `META-INF/native-image/io.grpc/grpc-netty-shaded` 文件夹下创建包含如下内容的文件 `native-image.properties`， +```properties +Args=--initialize-at-run-time=\ + io.grpc.netty.shaded.io.netty.channel.ChannelHandlerMask,\ + io.grpc.netty.shaded.io.netty.channel.nio.AbstractNioChannel,\ + io.grpc.netty.shaded.io.netty.channel.socket.nio.SelectorProviderUtil,\ + io.grpc.netty.shaded.io.netty.util.concurrent.DefaultPromise,\ + io.grpc.netty.shaded.io.netty.util.internal.MacAddressUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.SystemPropertyUtil,\ + io.grpc.netty.shaded.io.netty.util.NetUtilInitializations,\ + io.grpc.netty.shaded.io.netty.channel.AbstractChannel,\ + io.grpc.netty.shaded.io.netty.util.NetUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent,\ + io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0,\ + io.grpc.netty.shaded.io.netty.channel.DefaultChannelPipeline,\ + io.grpc.netty.shaded.io.netty.channel.DefaultChannelId,\ + io.grpc.netty.shaded.io.netty.util.ResourceLeakDetector,\ + io.grpc.netty.shaded.io.netty.channel.AbstractChannelHandlerContext,\ + io.grpc.netty.shaded.io.netty.channel.ChannelOutboundBuffer,\ + io.grpc.netty.shaded.io.netty.util.internal.InternalThreadLocalMap,\ + io.grpc.netty.shaded.io.netty.util.internal.CleanerJava9,\ + io.grpc.netty.shaded.io.netty.util.internal.StringUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.CleanerJava6,\ + io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil$HexUtil,\ + io.grpc.netty.shaded.io.netty.buffer.AbstractByteBufAllocator,\ + io.grpc.netty.shaded.io.netty.util.concurrent.FastThreadLocalThread,\ + io.grpc.netty.shaded.io.netty.buffer.PoolArena,\ + io.grpc.netty.shaded.io.netty.buffer.EmptyByteBuf,\ + io.grpc.netty.shaded.io.netty.buffer.PoolThreadCache,\ + io.grpc.netty.shaded.io.netty.util.AttributeKey +``` + +为了能够使用 `delete` 等 DML SQL 语句，当连接到 HiveServer2 时， +用户应当考虑在 ShardingSphere JDBC 中仅使用支持 ACID 的表。`apache/hive` 提供了多种事务解决方案。 + +第1种选择是使用 ACID 表，可能的建表流程如下。 +如果用户在 ACID 表上频繁更新和删除数据，用户可能不得不在 DML 语句执行前后进行等待， +以让 HiveServer2 完成低效的 DML 操作，因为 ACID 表中数据是在文件夹级别跟踪。 + +```sql +set metastore.compactor.initiator.on=true; +set metastore.compactor.cleaner.on=true; +set metastore.compactor.worker.threads=5; + +set hive.support.concurrency=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE IF NOT EXISTS t_order +( + order_id BIGINT, + order_type INT, + user_id INT NOT NULL, + address_id BIGINT NOT NULL, + status VARCHAR(50), + PRIMARY KEY (order_id) disable novalidate +) CLUSTERED BY (order_id) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional' = 'true'); +``` + +第2种选择是使用 Iceberg 表，可能的建表流程如下。`apache/iceberg` 表格式有望在未来几年取代传统的 Hive 表格式。 + +```sql +set iceberg.mr.schema.auto.conversion=true; + +CREATE TABLE IF NOT EXISTS t_order +( + order_id BIGINT, + order_type INT, + user_id INT NOT NULL, + address_id BIGINT NOT NULL, + status VARCHAR(50), + PRIMARY KEY (order_id) disable novalidate +) STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2'); +``` + +HiveServer2 不支持 ShardingSphere 集成级别的本地事务，XA 事务和 Seata AT 模式事务，更多讨论位于 https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions 。 + ## 贡献 GraalVM Reachability Metadata ShardingSphere 对在 GraalVM Native Image 下的可用性的验证，是通过 GraalVM Native Build Tools 的 Maven Plugin 子项目来完成的。 @@ -362,7 +518,7 @@ https://github.com/oracle/graalvm-reachability-metadata 打开新的 issue， Metadata 的 PR。ShardingSphere 在 `shardingsphere-infra-reachability-metadata` 子模块主动托管了部分第三方库的 GraalVM Reachability Metadata。 如果 nativeTest 执行失败， 应为单元测试生成初步的 GraalVM Reachability Metadata， -并手动调整 `shardingsphere-infra-reachability-metadata` 子模块的 classpath 的 `META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata` 文件夹下的内容以修复 nativeTest。 +并手动调整 `shardingsphere-infra-reachability-metadata` 子模块的 classpath 的 `META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/` 文件夹下的内容以修复 nativeTest。 如有需要，请使用 `org.junit.jupiter.api.condition.DisabledInNativeImage` 注解或 `org.graalvm.nativeimage.imagecode` 的 System Property 屏蔽部分单元测试在 GraalVM Native Image 下运行。 diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md index 18547ec9a0c53..8b21583c66a00 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md @@ -342,6 +342,166 @@ Possible configuration examples are as follows, ClickHouse does not support local transactions, XA transactions, and Seata AT mode transactions at the ShardingSphere integration level. More discussion is at https://github.com/ClickHouse/clickhouse-docs/issues/2300 . +7. When using the Hive dialect through ShardingSphere JDBC, affected by https://issues.apache.org/jira/browse/HIVE-28308 , + users should not use `org.apache.hive:hive-jdbc:4.0.0` with `classifier` as `standalone` to avoid dependency conflicts. + Possible configuration examples are as follows, + +```xml + + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-infra-database-hive + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-hive + ${shardingsphere.version} + + + org.apache.hive + hive-jdbc + 4.0.0 + + + org.apache.hive + hive-service + 4.0.0 + + + org.apache.hadoop + hadoop-client-api + 3.3.6 + + + +``` + +This can lead to a large number of dependency conflicts. +If the user does not want to manually resolve potentially thousands of lines of dependency conflicts, +a third-party build of the HiveServer2 JDBC Driver `Thin JAR` can be used. +An example of a possible configuration is as follows, + +```xml + + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-infra-database-hive + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-hive + ${shardingsphere.version} + + + io.github.linghengqian + hive-server2-jdbc-driver-thin + 1.0.0 + + + com.fasterxml.woodstox + woodstox-core + + + + + +``` + +Affected by https://github.com/grpc-java/issues/10601, should users incorporate `org.apache.hive:hive-service` into their project, +it is imperative to create a file named `native-image.properties` within the directory `META-INF/native-image/io.grpc/grpc-netty-shaded` of the classpath, +containing the following content, +```properties +Args=--initialize-at-run-time=\ + io.grpc.netty.shaded.io.netty.channel.ChannelHandlerMask,\ + io.grpc.netty.shaded.io.netty.channel.nio.AbstractNioChannel,\ + io.grpc.netty.shaded.io.netty.channel.socket.nio.SelectorProviderUtil,\ + io.grpc.netty.shaded.io.netty.util.concurrent.DefaultPromise,\ + io.grpc.netty.shaded.io.netty.util.internal.MacAddressUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.SystemPropertyUtil,\ + io.grpc.netty.shaded.io.netty.util.NetUtilInitializations,\ + io.grpc.netty.shaded.io.netty.channel.AbstractChannel,\ + io.grpc.netty.shaded.io.netty.util.NetUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent,\ + io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0,\ + io.grpc.netty.shaded.io.netty.channel.DefaultChannelPipeline,\ + io.grpc.netty.shaded.io.netty.channel.DefaultChannelId,\ + io.grpc.netty.shaded.io.netty.util.ResourceLeakDetector,\ + io.grpc.netty.shaded.io.netty.channel.AbstractChannelHandlerContext,\ + io.grpc.netty.shaded.io.netty.channel.ChannelOutboundBuffer,\ + io.grpc.netty.shaded.io.netty.util.internal.InternalThreadLocalMap,\ + io.grpc.netty.shaded.io.netty.util.internal.CleanerJava9,\ + io.grpc.netty.shaded.io.netty.util.internal.StringUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.CleanerJava6,\ + io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil$HexUtil,\ + io.grpc.netty.shaded.io.netty.buffer.AbstractByteBufAllocator,\ + io.grpc.netty.shaded.io.netty.util.concurrent.FastThreadLocalThread,\ + io.grpc.netty.shaded.io.netty.buffer.PoolArena,\ + io.grpc.netty.shaded.io.netty.buffer.EmptyByteBuf,\ + io.grpc.netty.shaded.io.netty.buffer.PoolThreadCache,\ + io.grpc.netty.shaded.io.netty.util.AttributeKey +``` + +In order to be able to use DML SQL statements such as `delete`, when connecting to HiveServer2, +users should consider using only ACID-supported tables in ShardingSphere JDBC. `apache/hive` provides a variety of transaction solutions. + +The first option is to use ACID tables, and the possible table creation process is as follows. +If users frequently update and delete data on ACID tables, users may have to wait before and after the execution of DML statements, +to allow HiveServer2 to complete inefficient DML operations, because data in ACID tables is tracked at the folder level. + +```sql +set metastore.compactor.initiator.on=true; +set metastore.compactor.cleaner.on=true; +set metastore.compactor.worker.threads=5; + +set hive.support.concurrency=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE IF NOT EXISTS t_order +( + order_id BIGINT, + order_type INT, + user_id INT NOT NULL, + address_id BIGINT NOT NULL, + status VARCHAR(50), + PRIMARY KEY (order_id) disable novalidate +) CLUSTERED BY (order_id) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional' = 'true'); +``` + +The second option is to use Iceberg table. The possible table creation process is as follows. +The `apache/iceberg` table format is expected to replace the traditional Hive table format in the next few years. + +```sql +set iceberg.mr.schema.auto.conversion=true; + +CREATE TABLE IF NOT EXISTS t_order +( + order_id BIGINT, + order_type INT, + user_id INT NOT NULL, + address_id BIGINT NOT NULL, + status VARCHAR(50), + PRIMARY KEY (order_id) disable novalidate +) STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2'); +``` + +HiveServer2 does not support local transactions, XA transactions, and Seata AT mode transactions at the ShardingSphere integration level. +More discussion is available at https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions . + ## Contribute GraalVM Reachability Metadata The verification of ShardingSphere's availability under GraalVM Native Image is completed through the Maven Plugin subproject diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json index 5ee27623d6668..7a6a8cf430f64 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json @@ -6,5 +6,9 @@ { "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.ClusterContextManagerBuilder"}, "interfaces":["java.sql.Connection"] + }, + { + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.standalone.StandaloneContextManagerBuilder"}, + "interfaces":["org.apache.hive.service.rpc.thrift.TCLIService$Iface"] } ] \ No newline at end of file diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json index fd7dc365d0f80..aea90f251fd4d 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json @@ -20,13 +20,17 @@ "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.metadata.data.loader.type.TableMetaDataLoader"}, + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.metadata.data.loader.MetaDataLoader"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.h2.metadata.data.loader.H2MetaDataLoader"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"}, + "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.opengauss.metadata.data.loader.OpenGaussMetaDataLoader"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" @@ -399,6 +403,18 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.type.DatabaseTypeFactory"}, "name":"org.apache.shardingsphere.infra.database.h2.type.H2DatabaseType" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.metadata.data.loader.MetaDataLoader"}, + "name":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader" +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.type.DatabaseTypeRegistry"}, + "name":"org.apache.shardingsphere.infra.database.hive.metadata.database.HiveDatabaseMetaData" +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.type.DatabaseTypeFactory"}, + "name":"org.apache.shardingsphere.infra.database.hive.type.HiveDatabaseType" +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.type.DatabaseTypeFactory"}, "name":"org.apache.shardingsphere.infra.database.mariadb.type.MariaDBDatabaseType" @@ -578,12 +594,12 @@ "name":"org.apache.shardingsphere.infra.util.eventbus.EventSubscriber" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.event.subsciber.DeliverEventSubscriberRegistry$$Lambda/0x00007f35bb470bf8"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.event.subsciber.DeliverEventSubscriberRegistry$$Lambda/0x00007fce7f479688"}, "name":"org.apache.shardingsphere.infra.util.eventbus.EventSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.infra.util.eventbus.EventSubscriber", "queryAllDeclaredMethods":true }, @@ -897,7 +913,7 @@ "name":"org.apache.shardingsphere.mask.yaml.config.rule.YamlMaskTableRuleConfigurationCustomizer" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.event.subsciber.DeliverEventSubscriberRegistry$$Lambda/0x00007f35bb470bf8"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.event.subsciber.DeliverEventSubscriberRegistry$$Lambda/0x00007fce7f479688"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.deliver.DeliverQualifiedDataSourceSubscriber", "queryAllDeclaredMethods":true }, @@ -907,42 +923,42 @@ "methods":[{"name":"cleanCache","parameterTypes":["org.apache.shardingsphere.mode.event.dispatch.DispatchEvent"] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.CacheEvictedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.ComputeNodeOnlineSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.DatabaseChangedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.GlobalRuleConfigurationEventSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.ListenerAssistedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.ProcessListChangedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.PropertiesEventSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.QualifiedDataSourceSubscriber", "queryAllDeclaredMethods":true }, @@ -951,26 +967,26 @@ "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.ResourceMetaDataChangedSubscriber" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.ResourceMetaDataChangedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.MetaDataChangedListener$$Lambda/0x00007f35bb5d31e8"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.MetaDataChangedListener$$Lambda/0x00007fce7f5d5000"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.ResourceMetaDataChangedSubscriber" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.RuleItemChangedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.StateChangedSubscriber", "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007f35bb48eee0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.registry.ClusterDispatchEventSubscriberRegistry$$Lambda/0x00007fce7f490a30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.StorageUnitEventSubscriber", "queryAllDeclaredMethods":true }, @@ -1438,6 +1454,11 @@ "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.database.cache.ParseTreeCacheBuilder"}, "name":"org.apache.shardingsphere.sql.parser.core.database.cache.ParseTreeCacheLoader" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.database.visitor.SQLStatementVisitorFactory"}, + "name":"org.apache.shardingsphere.sql.parser.hive.visitor.statement.HiveStatementVisitorFacade", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.SQLParserFactory"}, "name":"org.apache.shardingsphere.sql.parser.mysql.parser.MySQLLexer", @@ -1558,6 +1579,11 @@ "name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseInsertStatement", "methods":[{"name":"","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.binder.engine.statement.dml.DeleteStatementBinder"}, + "name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveDeleteStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.binder.engine.statement.dml.DeleteStatementBinder"}, "name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLDeleteStatement", diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json index cb3d89b3b8454..988ed917e9a52 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json @@ -2730,6 +2730,9 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, "pattern":"\\Qtest-native/yaml/databases/clickhouse.yaml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, + "pattern":"\\Qtest-native/yaml/databases/hive.yaml\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, "pattern":"\\Qtest-native/yaml/databases/mysql.yaml\\E" diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json index e557ffccedae9..d06109806703f 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json @@ -64,5 +64,30 @@ "name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfiguration", "allDeclaredFields":true, "methods":[{"name":"getDefaultType","parameterTypes":[] }, {"name":"getProps","parameterTypes":[] }, {"name":"getProviderType","parameterTypes":[] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.hive.parser.HiveLexer"}, + "name":"org.apache.shardingsphere.sql.parser.hive.parser.HiveLexer", + "methods":[{"name":"","parameterTypes":["org.antlr.v4.runtime.CharStream"] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.hive.parser.HiveParser"}, + "name":"org.apache.shardingsphere.sql.parser.hive.parser.HiveParser", + "methods":[{"name":"","parameterTypes":["org.antlr.v4.runtime.TokenStream"] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.hive.visitor.statement.type.HiveDMLStatementVisitor"}, + "name":"org.apache.shardingsphere.sql.parser.hive.visitor.statement.type.HiveDMLStatementVisitor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveInsertStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveInsertStatement", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"}, + "name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveSelectStatement", + "methods":[{"name":"","parameterTypes":[] }] } ] diff --git a/test/native/native-image-filter/extra-filter.json b/test/native/native-image-filter/extra-filter.json index ca2b00516a04f..dcb58d0608e95 100644 --- a/test/native/native-image-filter/extra-filter.json +++ b/test/native/native-image-filter/extra-filter.json @@ -2,55 +2,21 @@ "rules": [ {"includeClasses": "**"}, - {"excludeClasses": "com.sun.crypto.provider.**"}, - {"excludeClasses": "com.sun.org.apache.xerces.internal.jaxp.**"}, - {"excludeClasses": "java.**"}, - {"includeClasses": "java.util.Properties"}, - {"excludeClasses": "javax.security.auth.x500.**"}, - {"excludeClasses": "javax.smartcardio.**"}, - {"excludeClasses": "javax.xml.stream.**"}, - {"excludeClasses": "jdk.internal.misc.**"}, - {"excludeClasses": "sun.misc.**"}, - {"excludeClasses": "sun.nio.ch.**"}, - {"excludeClasses": "sun.security.**"}, - {"excludeClasses": "android.app.**"}, - {"excludeClasses": "com.arjuna.**"}, - {"excludeClasses": "com.atomikos.**"}, - {"excludeClasses": "com.alibaba.druid.**"}, - {"excludeClasses": "com.clickhouse.**"}, - {"excludeClasses": "com.ctc.wstx.stax.**"}, - {"excludeClasses": "com.fasterxml.jackson.databind.**"}, - {"excludeClasses": "com.github.benmanes.caffeine.cache.**"}, - {"excludeClasses": "com.github.dockerjava.api.**"}, - {"excludeClasses": "com.google.common.util.concurrent.**"}, - {"excludeClasses": "com.ibm.icu.text.**"}, - {"excludeClasses": "com.microsoft.sqlserver.jdbc.**"}, - {"excludeClasses": "com.mysql.cj.**"}, - {"excludeClasses": "com.zaxxer.hikari.**"}, + {"excludeClasses": "com.**"}, {"excludeClasses": "ch.qos.logback.classic.**"}, - {"excludeClasses": "io.etcd.**"}, - {"excludeClasses": "io.grpc.**"}, - {"excludeClasses": "io.netty.**"}, - {"excludeClasses": "io.seata.**"}, - {"excludeClasses": "io.vertx.core.**"}, {"excludeClasses": "groovy.**"}, + {"excludeClasses": "io.**"}, + {"excludeClasses": "java.**"}, + {"includeClasses": "java.util.Properties"}, + {"includeClasses": "java.util.ServiceLoader"}, + {"excludeClasses": "javax.**"}, + {"excludeClasses": "jdk.internal.misc.**"}, {"excludeClasses": "libcore.io.**"}, {"excludeClasses": "net.bytebuddy.**"}, - {"excludeClasses": "org.apache.calcite.**"}, - {"excludeClasses": "org.apache.commons.logging.**"}, - {"excludeClasses": "org.apache.logging.log4j.**"}, - {"excludeClasses": "org.apache.log4j.**"}, - {"excludeClasses": "org.apache.zookeeper.**"}, - {"excludeClasses": "org.codehaus.groovy.**"}, - {"excludeClasses": "org.h2.**"}, - {"excludeClasses": "org.jboss.logmanager.**"}, - {"excludeClasses": "org.locationtech.jts.geom.**"}, - {"excludeClasses": "org.opengauss.**"}, - {"excludeClasses": "org.postgresql.**"}, - {"excludeClasses": "org.robolectric.**"}, - {"excludeClasses": "org.slf4j.**"}, - {"excludeClasses": "org.testcontainers.**"}, + {"excludeClasses": "org.**"}, + {"includeClasses": "org.apache.shardingsphere.**"}, + {"excludeClasses": "sun.**"}, {"excludeClasses": "org.apache.shardingsphere.test.natived.**"} ], diff --git a/test/native/pom.xml b/test/native/pom.xml index 872f6b9741fd0..d2e327c31734c 100644 --- a/test/native/pom.xml +++ b/test/native/pom.xml @@ -131,6 +131,17 @@ using Seata Client under GraalVM Native Image requires using the version release ${project.version} test + + org.apache.shardingsphere + shardingsphere-infra-database-hive + ${project.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-hive + ${project.version} + test + org.awaitility @@ -163,6 +174,18 @@ using Seata Client under GraalVM Native Image requires using the version release http test + + io.github.linghengqian + hive-server2-jdbc-driver-thin + ${hive-server2-jdbc-driver-thin.version} + test + + + com.fasterxml.woodstox + woodstox-core + + + org.testcontainers junit-jupiter diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/TestShardingService.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/TestShardingService.java index 762ab420aa3ff..72b78f24e46c7 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/TestShardingService.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/TestShardingService.java @@ -119,6 +119,23 @@ public void processSuccessInClickHouse() throws SQLException { assertThat(addressRepository.selectAll(), equalTo(Collections.emptyList())); } + /** + * Process success in Hive. + * Hive has not fully supported BEGIN, COMMIT, and ROLLBACK. Refer to Hive Transactions. + * So ShardingSphere should not use {@link OrderItemRepository#assertRollbackWithTransactions()} + * TODO It looks like HiveServer2 insert statements are inserted out of order. Waiting for further investigation. + * The result of the insert is not currently asserted. + * TODO It is currently not convenient to operate on the `t_order` and `t_order_item` tables because + * {@link org.apache.hive.jdbc.HiveStatement} does not implement {@link org.apache.hive.jdbc.HiveStatement#getGeneratedKeys()} + * + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + public void processSuccessInHive() throws SQLException { + insertDataInHive(); + deleteDataInHive(); + assertThat(addressRepository.selectAll(), equalTo(Collections.emptyList())); + } + /** * Insert data. * @@ -151,6 +168,20 @@ public Collection insertData(final int autoGeneratedKeys) throws SQLExcept return result; } + /** + * Insert data in Hive. + */ + public void insertDataInHive() { + LongStream.range(1L, 11L).forEach(action -> { + Address address = new Address(action, "address_test_" + action); + try { + addressRepository.insert(address); + } catch (final SQLException ex) { + throw new RuntimeException(ex); + } + }); + } + /** * Delete data. * @@ -181,6 +212,18 @@ public void deleteDataInClickHouse(final Collection orderIds) throws SQLEx } } + /** + * Delete data in Hive. + * + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + public void deleteDataInHive() throws SQLException { + long count = 1L; + for (int i = 1; i <= 10; i++) { + addressRepository.deleteInHive(count++); + } + } + /** * Clean environment. * diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/AddressRepository.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/AddressRepository.java index 182913898cd47..48dba1585cc43 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/AddressRepository.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/AddressRepository.java @@ -70,6 +70,25 @@ public void createTableInSQLServer() throws SQLException { } } + /** + * create table t_address if not exists in Hive. + * + * @throws SQLException SQL exception + */ + public void createTableIfNotExistsInHive() throws SQLException { + String sql = "CREATE TABLE IF NOT EXISTS t_address\n" + + "(\n" + + " address_id BIGINT NOT NULL,\n" + + " address_name VARCHAR(100) NOT NULL,\n" + + " PRIMARY KEY (address_id) disable novalidate\n" + + ") STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2')"; + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate(sql); + } + } + /** * drop table t_address. * @@ -133,6 +152,22 @@ public void delete(final Long id) throws SQLException { } } + /** + * delete by id. + * + * @param id id + * @throws SQLException SQL exception + */ + public void deleteInHive(final Long id) throws SQLException { + String sql = "DELETE FROM t_address WHERE address_id=?"; + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + preparedStatement.setLong(1, id); + preparedStatement.executeUpdate(); + } + } + /** * delete by id in ClickHouse. * diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderItemRepository.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderItemRepository.java index b53071fb09087..fd43e4660c949 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderItemRepository.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderItemRepository.java @@ -125,6 +125,29 @@ public void createTableIfNotExistsInClickHouse() throws SQLException { } } + /** + * create table if not exists in Hive. + * Hive does not support `AUTO_INCREMENT`, refer to HIVE-6905 . + * + * @throws SQLException SQL exception + */ + public void createTableIfNotExistsInHive() throws SQLException { + String sql = "CREATE TABLE IF NOT EXISTS t_order_item\n" + + "(\n" + + " order_item_id BIGINT,\n" + + " order_id BIGINT NOT NULL,\n" + + " user_id INT NOT NULL,\n" + + " phone VARCHAR(50),\n" + + " status VARCHAR(50),\n" + + " PRIMARY KEY (order_item_id) disable novalidate\n" + + ") STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2')"; + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate(sql); + } + } + /** * drop table. * diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderRepository.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderRepository.java index 7cded2baf10f5..c8a1957118208 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderRepository.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/commons/repository/OrderRepository.java @@ -122,6 +122,29 @@ public void createTableIfNotExistsInClickHouse() throws SQLException { } } + /** + * create table in Hive. + * Hive does not support `AUTO_INCREMENT`, refer to HIVE-6905 . + * + * @throws SQLException SQL exception + */ + public void createTableIfNotExistsInHive() throws SQLException { + String sql = "CREATE TABLE IF NOT EXISTS t_order\n" + + "(\n" + + " order_id BIGINT,\n" + + " order_type INT,\n" + + " user_id INT NOT NULL,\n" + + " address_id BIGINT NOT NULL,\n" + + " status VARCHAR(50),\n" + + " PRIMARY KEY (order_id) disable novalidate\n" + + ") STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2')"; + try ( + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate(sql); + } + } + /** * drop table. * TODO There is a bug in this function in shadow's unit test and requires additional fixes. diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/HiveTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/HiveTest.java new file mode 100644 index 0000000000000..03be9d1cfaef9 --- /dev/null +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/HiveTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.test.natived.jdbc.databases; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.apache.shardingsphere.test.natived.jdbc.commons.TestShardingService; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledInNativeImage; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import javax.sql.DataSource; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Properties; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +@SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection"}) +@EnabledInNativeImage +@Testcontainers +class HiveTest { + + @SuppressWarnings("resource") + @Container + public static final GenericContainer CONTAINER = new GenericContainer<>(DockerImageName.parse("apache/hive:4.0.0")) + .withEnv("SERVICE_NAME", "hiveserver2") + .withExposedPorts(10000, 10002); + + private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.hive."; + + // Due to https://issues.apache.org/jira/browse/HIVE-28317 , the `initFile` parameter of HiveServer2 JDBC Driver must be an absolute path. + private static final String ABSOLUTE_PATH = Paths.get("src/test/resources/test-native/sql/test-native-databases-hive.sql").toAbsolutePath().normalize().toString(); + + private String jdbcUrlPrefix; + + private TestShardingService testShardingService; + + @BeforeAll + static void beforeAll() { + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"), is(nullValue())); + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url"), is(nullValue())); + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"), is(nullValue())); + } + + @AfterAll + static void afterAll() { + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url"); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"); + } + + /** + * TODO Need to fix `shardingsphere-parser-sql-hive` module to use {@link TestShardingService#cleanEnvironment()} + * after {@link TestShardingService#processSuccessInHive()}. + * + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + @Test + void assertShardingInLocalTransactions() throws SQLException { + jdbcUrlPrefix = "jdbc:hive2://localhost:" + CONTAINER.getMappedPort(10000) + "/"; + DataSource dataSource = createDataSource(); + testShardingService = new TestShardingService(dataSource); + testShardingService.processSuccessInHive(); + } + + /** + * TODO Need to fix `shardingsphere-parser-sql-hive` module to use `initEnvironment()` before {@link TestShardingService#processSuccessInHive()}. + * + * @throws SQLException An exception that provides information on a database access error or other errors. + */ + @SuppressWarnings("unused") + private void initEnvironment() throws SQLException { + testShardingService.getOrderRepository().createTableIfNotExistsInHive(); + testShardingService.getOrderItemRepository().createTableIfNotExistsInHive(); + testShardingService.getAddressRepository().createTableIfNotExistsInHive(); + testShardingService.getOrderRepository().truncateTable(); + testShardingService.getOrderItemRepository().truncateTable(); + testShardingService.getAddressRepository().truncateTable(); + } + + private Connection openConnection() throws SQLException { + Properties props = new Properties(); + return DriverManager.getConnection(jdbcUrlPrefix, props); + } + + private DataSource createDataSource() throws SQLException { + Awaitility.await().atMost(Duration.ofMinutes(1L)).ignoreExceptions().until(() -> { + openConnection().close(); + return true; + }); + try ( + Connection connection = openConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate("CREATE DATABASE demo_ds_0"); + statement.executeUpdate("CREATE DATABASE demo_ds_1"); + statement.executeUpdate("CREATE DATABASE demo_ds_2"); + } + HikariConfig config = new HikariConfig(); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/databases/hive.yaml?placeholder-type=system_props"); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url", jdbcUrlPrefix + "demo_ds_0" + ";initFile=" + ABSOLUTE_PATH); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url", jdbcUrlPrefix + "demo_ds_1" + ";initFile=" + ABSOLUTE_PATH); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url", jdbcUrlPrefix + "demo_ds_2" + ";initFile=" + ABSOLUTE_PATH); + return new HikariDataSource(config); + } +} diff --git a/test/native/src/test/resources/META-INF/native-image/io.grpc/grpc-netty-shaded/native-image.properties b/test/native/src/test/resources/META-INF/native-image/io.grpc/grpc-netty-shaded/native-image.properties new file mode 100644 index 0000000000000..eb515b70d31bb --- /dev/null +++ b/test/native/src/test/resources/META-INF/native-image/io.grpc/grpc-netty-shaded/native-image.properties @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO This file exists to address https://github.com/grpc/grpc-java/issues/10601 . +Args=--initialize-at-run-time=\ + io.grpc.netty.shaded.io.netty.channel.ChannelHandlerMask,\ + io.grpc.netty.shaded.io.netty.channel.nio.AbstractNioChannel,\ + io.grpc.netty.shaded.io.netty.channel.socket.nio.SelectorProviderUtil,\ + io.grpc.netty.shaded.io.netty.util.concurrent.DefaultPromise,\ + io.grpc.netty.shaded.io.netty.util.internal.MacAddressUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.SystemPropertyUtil,\ + io.grpc.netty.shaded.io.netty.util.NetUtilInitializations,\ + io.grpc.netty.shaded.io.netty.channel.AbstractChannel,\ + io.grpc.netty.shaded.io.netty.util.NetUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent,\ + io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0,\ + io.grpc.netty.shaded.io.netty.channel.DefaultChannelPipeline,\ + io.grpc.netty.shaded.io.netty.channel.DefaultChannelId,\ + io.grpc.netty.shaded.io.netty.util.ResourceLeakDetector,\ + io.grpc.netty.shaded.io.netty.channel.AbstractChannelHandlerContext,\ + io.grpc.netty.shaded.io.netty.channel.ChannelOutboundBuffer,\ + io.grpc.netty.shaded.io.netty.util.internal.InternalThreadLocalMap,\ + io.grpc.netty.shaded.io.netty.util.internal.CleanerJava9,\ + io.grpc.netty.shaded.io.netty.util.internal.StringUtil,\ + io.grpc.netty.shaded.io.netty.util.internal.CleanerJava6,\ + io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil$HexUtil,\ + io.grpc.netty.shaded.io.netty.buffer.AbstractByteBufAllocator,\ + io.grpc.netty.shaded.io.netty.util.concurrent.FastThreadLocalThread,\ + io.grpc.netty.shaded.io.netty.buffer.PoolArena,\ + io.grpc.netty.shaded.io.netty.buffer.EmptyByteBuf,\ + io.grpc.netty.shaded.io.netty.buffer.PoolThreadCache,\ + io.grpc.netty.shaded.io.netty.util.AttributeKey diff --git a/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql b/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql new file mode 100644 index 0000000000000..43bc516e458e2 --- /dev/null +++ b/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql @@ -0,0 +1,53 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- TODO `shardingsphere-parser-sql-hive` module does not support `CREATE`, `SET`, `TRUNCATE` statements yet, +-- we always need to execute the following Hive Session-level SQL in the current `javax.sql.DataSource`. + +-- Fixes `Unsupported Hive type VARCHAR, use string instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true` +set iceberg.mr.schema.auto.conversion=true; + +CREATE TABLE IF NOT EXISTS t_order +( + order_id BIGINT, + order_type INT, + user_id INT NOT NULL, + address_id BIGINT NOT NULL, + status VARCHAR(50), + PRIMARY KEY (order_id) disable novalidate +) STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2'); + +CREATE TABLE IF NOT EXISTS t_order_item +( + order_item_id BIGINT, + order_id BIGINT NOT NULL, + user_id INT NOT NULL, + phone VARCHAR(50), + status VARCHAR(50), + PRIMARY KEY (order_item_id) disable novalidate +) STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2'); + +CREATE TABLE IF NOT EXISTS t_address +( + address_id BIGINT NOT NULL, + address_name VARCHAR(100) NOT NULL, + PRIMARY KEY (address_id) disable novalidate +) STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2'); + +TRUNCATE TABLE t_order; +TRUNCATE TABLE t_order_item; +TRUNCATE TABLE t_address; diff --git a/test/native/src/test/resources/test-native/yaml/databases/hive.yaml b/test/native/src/test/resources/test-native/yaml/databases/hive.yaml new file mode 100644 index 0000000000000..c44cd303bb3bc --- /dev/null +++ b/test/native/src/test/resources/test-native/yaml/databases/hive.yaml @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +mode: + type: Standalone + repository: + type: JDBC + +dataSources: + ds_0: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: org.apache.hive.jdbc.HiveDriver + jdbcUrl: $${fixture.test-native.yaml.database.hive.ds0.jdbc-url::} + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: org.apache.hive.jdbc.HiveDriver + jdbcUrl: $${fixture.test-native.yaml.database.hive.ds1.jdbc-url::} + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: org.apache.hive.jdbc.HiveDriver + jdbcUrl: $${fixture.test-native.yaml.database.hive.ds2.jdbc-url::} + +rules: +- !SHARDING + tables: + t_order: + actualDataNodes: + keyGenerateStrategy: + column: order_id + keyGeneratorName: snowflake + t_order_item: + actualDataNodes: + keyGenerateStrategy: + column: order_item_id + keyGeneratorName: snowflake + defaultDatabaseStrategy: + standard: + shardingColumn: user_id + shardingAlgorithmName: inline + shardingAlgorithms: + inline: + type: CLASS_BASED + props: + strategy: STANDARD + algorithmClassName: org.apache.shardingsphere.test.natived.jdbc.commons.algorithm.ClassBasedInlineShardingAlgorithmFixture + keyGenerators: + snowflake: + type: SNOWFLAKE + auditors: + sharding_key_required_auditor: + type: DML_SHARDING_CONDITIONS + +- !BROADCAST + tables: + - t_address + +props: + sql-show: false