Add GraalVM Reachability Metadata and corresponding nativeTest for MS…

… SQL Server integration (#29404)
apache · Dec 14, 2023 · 6fa3b70 · 6fa3b70
1 parent d69b3f1
commit 6fa3b70
Show file tree

Hide file tree

Showing 16 changed files with 321 additions and 29 deletions.
diff --git a/distribution/proxy-native/pom.xml b/distribution/proxy-native/pom.xml
@@ -86,6 +86,7 @@
                             <mainClass>org.apache.shardingsphere.proxy.Bootstrap</mainClass>
                             <buildArgs>
                                 <arg>-J-Xmx7g</arg>
+                                <arg>-H:+AddAllCharsets</arg>
                             </buildArgs>
                             <metadataRepository>
                                 <enabled>true</enabled>

diff --git a/...ument/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md b/...ument/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md
@@ -41,6 +41,9 @@ ShardingSphere JDBC 要求在如下或更高版本的 `GraalVM CE` 完成构建
                 <version>0.9.28</version>
                 <extensions>true</extensions>
                 <configuration>
+                    <buildArgs>
+                        <buildArg>-H:+AddAllCharsets</buildArg>
+                    </buildArgs>
                     <metadataRepository>
                         <enabled>true</enabled>
                     </metadataRepository>
@@ -74,17 +77,25 @@ ShardingSphere JDBC 要求在如下或更高版本的 `GraalVM CE` 完成构建
 
 ```groovy
 plugins {
-    id 'org.graalvm.buildtools.native' version '0.9.28'
+   id 'org.graalvm.buildtools.native' version '0.9.28'
 }
 
 dependencies {
-    implementation 'org.apache.shardingsphere:shardingsphere-jdbc-core:${shardingsphere.version}'
+   implementation 'org.apache.shardingsphere:shardingsphere-jdbc-core:${shardingsphere.version}'
 }
 
 graalvmNative {
-    metadataRepository {
-        enabled = true
-    }
+   binaries {
+      main {
+         buildArgs.add('-H:+AddAllCharsets')
+      }
+      test {
+         buildArgs.add('-H:+AddAllCharsets')
+      }
+   }
+   metadataRepository {
+      enabled = true
+   }
 }
 ```
 
@@ -193,7 +204,21 @@ rules:
 文件的 GraalVM Reachability Metadata。使用者可通过 GraalVM Native Build Tools 的 GraalVM Tracing Agent 来快速采集 GraalVM 
 Reachability Metadata。
 
-4. 尚未验证 DistSQL 的可用性。使用者需自行添加额外的 GraalVM Reachability Metadata。
+4. 以 MS SQL Server 的 JDBC Driver 为代表的 `com.microsoft.sqlserver:mssql-jdbc` 等 Maven 模块会根据数据库中使用的编码动态加载不同的字符集，这是不可预测的行为。
+当遇到如下 Error，使用者需要添加 `-H:+AddAllCharsets` 的 `buildArg` 到 GraalVM Native Build Tools 的配置中。
+
+```shell
+Caused by: java.io.UnsupportedEncodingException: SQL Server collation SQL_Latin1_General_CP1_CI_AS is not supported by this driver.
+ com.microsoft.sqlserver.jdbc.SQLCollation.encodingFromSortId(SQLCollation.java:506)
+ com.microsoft.sqlserver.jdbc.SQLCollation.<init>(SQLCollation.java:63)
+ com.microsoft.sqlserver.jdbc.SQLServerConnection.processEnvChange(SQLServerConnection.java:3174)
+ [...]
+Caused by: java.io.UnsupportedEncodingException: Codepage Cp1252 is not supported by the Java environment.
+ com.microsoft.sqlserver.jdbc.Encoding.checkSupported(SQLCollation.java:572)
+ com.microsoft.sqlserver.jdbc.SQLCollation$SortOrder.getEncoding(SQLCollation.java:473)
+ com.microsoft.sqlserver.jdbc.SQLCollation.encodingFromSortId(SQLCollation.java:501)
+ [...]
+```
 
 ## 贡献 GraalVM Reachability Metadata
 

diff --git a/...ument/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md b/...ument/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md
@@ -43,9 +43,12 @@ and the documentation of GraalVM Native Build Tools shall prevail.
                  <version>0.9.28</version>
                  <extensions>true</extensions>
                  <configuration>
-                     <metadataRepository>
-                         <enabled>true</enabled>
-                     </metadataRepository>
+                    <buildArgs>
+                       <buildArg>-H:+AddAllCharsets</buildArg>
+                    </buildArgs>
+                    <metadataRepository>
+                       <enabled>true</enabled>
+                    </metadataRepository>
                  </configuration>
                  <executions>
                      <execution>
@@ -85,9 +88,17 @@ dependencies {
 }
 
 graalvmNative {
-     metadataRepository {
-         enabled = true
-     }
+   binaries {
+      main {
+         buildArgs.add('-H:+AddAllCharsets')
+      }
+      test {
+         buildArgs.add('-H:+AddAllCharsets')
+      }
+   }
+   metadataRepository {
+      enabled = true
+   }
 }
 ```
 
@@ -201,7 +212,22 @@ rules:
 folder or `src/test/resources/META-INF/native-image` folder. Users can quickly collect GraalVM Reachability Metadata through 
 the GraalVM Tracing Agent of GraalVM Native Build Tools. 
 
-4. DistSQL availability has not been verified. Users need to add additional GraalVM Reachability Metadata by themselves.
+4. Maven modules such as `com.microsoft.sqlserver:mssql-jdbc`, represented by the JDBC Driver of MS SQL Server,
+will dynamically load different character sets based on the encoding used in the database, which is unpredictable behavior. 
+When encountering the following Error, users need to add the `buildArg` of `-H:+AddAllCharsets` to the configuration of GraalVM Native Build Tools.
+
+```shell
+Caused by: java.io.UnsupportedEncodingException: SQL Server collation SQL_Latin1_General_CP1_CI_AS is not supported by this driver.
+ com.microsoft.sqlserver.jdbc.SQLCollation.encodingFromSortId(SQLCollation.java:506)
+ com.microsoft.sqlserver.jdbc.SQLCollation.<init>(SQLCollation.java:63)
+ com.microsoft.sqlserver.jdbc.SQLServerConnection.processEnvChange(SQLServerConnection.java:3174)
+ [...]
+Caused by: java.io.UnsupportedEncodingException: Codepage Cp1252 is not supported by the Java environment.
+ com.microsoft.sqlserver.jdbc.Encoding.checkSupported(SQLCollation.java:572)
+ com.microsoft.sqlserver.jdbc.SQLCollation$SortOrder.getEncoding(SQLCollation.java:473)
+ com.microsoft.sqlserver.jdbc.SQLCollation.encodingFromSortId(SQLCollation.java:501)
+ [...]
+```
 
 ## Contribute GraalVM Reachability Metadata
 

diff --git a/...hardingsphere/infra/database/sqlserver/connector/SQLServerConnectionPropertiesParser.java b/...hardingsphere/infra/database/sqlserver/connector/SQLServerConnectionPropertiesParser.java
@@ -34,7 +34,7 @@ public final class SQLServerConnectionPropertiesParser implements ConnectionProp
 
     private static final int DEFAULT_PORT = 1433;
 
-    private static final Pattern URL_PATTERN = Pattern.compile("jdbc:(microsoft:)?sqlserver://([\\w\\-\\.]+):?(\\d*);\\S*(DatabaseName|database)=([\\w\\-\\.]+);?", Pattern.CASE_INSENSITIVE);
+    private static final Pattern URL_PATTERN = Pattern.compile("jdbc:(microsoft:|tc:)?sqlserver:.*//([\\w\\-\\.]+):?(\\d*);\\S*(databaseName|database)=([\\w\\-\\.]+);?", Pattern.CASE_INSENSITIVE);
 
     @Override
     public ConnectionProperties parse(final String url, final String username, final String catalog) {

diff --git a/...ingsphere/infra/database/sqlserver/connector/SQLServerConnectionPropertiesParserTest.java b/...ingsphere/infra/database/sqlserver/connector/SQLServerConnectionPropertiesParserTest.java
@@ -67,7 +67,8 @@ public Stream<? extends Arguments> provideArguments(final ExtensionContext exten
                     Arguments.of("defaultPortAndMicrosoft", "jdbc:microsoft:sqlserver://127.0.0.1;DatabaseName=foo_ds", "127.0.0.1", 1433, "foo_ds", null),
                     Arguments.of("defaultPortWithoutMicrosoft", "jdbc:sqlserver://127.0.0.1;DatabaseName=foo_ds", "127.0.0.1", 1433, "foo_ds", null),
                     Arguments.of("databaseNameContainDotAndMicrosoft", "jdbc:microsoft:sqlserver://127.0.0.1:9999;DatabaseName=foo_0.0.0", "127.0.0.1", 9999, "foo_0.0.0", null),
-                    Arguments.of("databaseNameContainDotAndWithoutMicrosoft", "jdbc:sqlserver://127.0.0.1:9999;DatabaseName=foo_0.0.0", "127.0.0.1", 9999, "foo_0.0.0", null));
+                    Arguments.of("databaseNameContainDotAndWithoutMicrosoft", "jdbc:sqlserver://127.0.0.1:9999;DatabaseName=foo_0.0.0", "127.0.0.1", 9999, "foo_0.0.0", null),
+                    Arguments.of("testcontainers", "jdbc:tc:sqlserver:2022-CU10-ubuntu-22.04://test-hostname;databaseName=test_database", "test-hostname", 1433, "test_database", null));
         }
     }
 }
diff --git a/.../org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json b/.../org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json
@@ -19,7 +19,7 @@
   "name":"org.apache.shardingsphere.authority.yaml.config.YamlAuthorityRuleConfiguration"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.broadcast.yaml.config.YamlBroadcastRuleConfiguration"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.broadcast.yaml.swapper.NewYamlBroadcastRuleConfigurationSwapper"},
   "name":"org.apache.shardingsphere.broadcast.yaml.config.YamlBroadcastRuleConfiguration",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -34,7 +34,7 @@
   "name":"org.apache.shardingsphere.broadcast.yaml.config.YamlBroadcastRuleConfigurationCustomizer"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.DatabaseTypeEngine"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.infra.state.datasource.DataSourceStateManager"},
   "name":"org.apache.shardingsphere.driver.ShardingSphereDriver"
 },
 {
@@ -67,7 +67,7 @@
   "name":"org.apache.shardingsphere.encrypt.yaml.config.YamlCompatibleEncryptRuleConfiguration"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.encrypt.yaml.config.YamlEncryptRuleConfiguration"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.config.shortcut.YamlRuleConfigurationShortcuts"},
   "name":"org.apache.shardingsphere.encrypt.yaml.config.YamlEncryptRuleConfiguration",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -206,7 +206,7 @@
   "methods":[{"name":"getRows","parameterTypes":[] }, {"name":"getUniqueKey","parameterTypes":[] }]
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.infra.metadata.database.schema.builder.SystemSchemaBuilder"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase"},
   "name":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereColumn",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -221,7 +221,7 @@
   "name":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereColumnCustomizer"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereIndex"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.infra.metadata.database.schema.builder.SystemSchemaBuilder"},
   "name":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereIndex",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -236,7 +236,7 @@
   "name":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereIndexCustomizer"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereTable"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.metadata.persist.service.schema.NewTableMetaDataPersistService"},
   "name":"org.apache.shardingsphere.infra.yaml.schema.pojo.YamlShardingSphereTable",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -406,7 +406,7 @@
   "name":"org.apache.shardingsphere.parser.yaml.config.YamlSQLParserCacheOptionRuleConfigurationCustomizer"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.parser.yaml.config.YamlSQLParserRuleConfiguration"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.config.shortcut.YamlRuleConfigurationShortcuts"},
   "name":"org.apache.shardingsphere.parser.yaml.config.YamlSQLParserRuleConfiguration",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -613,7 +613,7 @@
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfiguration"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.config.shortcut.YamlRuleConfigurationShortcuts"},
   "name":"org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfiguration",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -628,7 +628,7 @@
   "name":"org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfigurationCustomizer"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.sharding.yaml.config.rule.YamlTableRuleConfiguration"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.sharding.yaml.swapper.NewYamlShardingRuleConfigurationSwapper"},
   "name":"org.apache.shardingsphere.sharding.yaml.config.rule.YamlTableRuleConfiguration",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -658,7 +658,7 @@
   "name":"org.apache.shardingsphere.sharding.yaml.config.strategy.keygen.YamlKeyGenerateStrategyConfigurationCustomizer"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.sharding.yaml.config.strategy.sharding.YamlShardingStrategyConfiguration"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.sharding.yaml.swapper.NewYamlShardingRuleConfigurationSwapper"},
   "name":"org.apache.shardingsphere.sharding.yaml.config.strategy.sharding.YamlShardingStrategyConfiguration",
   "allDeclaredFields":true,
   "queryAllPublicMethods":true,
@@ -755,6 +755,11 @@
   "name":"org.apache.shardingsphere.sql.parser.opengauss.visitor.statement.type.OpenGaussDDLStatementVisitor",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.opengauss.visitor.statement.type.OpenGaussDMLStatementVisitor"},
+  "name":"org.apache.shardingsphere.sql.parser.opengauss.visitor.statement.type.OpenGaussDMLStatementVisitor",
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+},
 {
   "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.database.visitor.SQLStatementVisitorFactory"},
   "name":"org.apache.shardingsphere.sql.parser.oracle.visitor.statement.OracleStatementVisitorFacade",
@@ -830,18 +835,48 @@
   "name":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.postgresql.dml.PostgreSQLSelectStatement",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.sqlserver.dml.SQLServerDeleteStatement"},
+  "name":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.sqlserver.dml.SQLServerDeleteStatement",
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.sqlserver.dml.SQLServerInsertStatement"},
+  "name":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.sqlserver.dml.SQLServerInsertStatement",
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.sqlserver.dml.SQLServerSelectStatement"},
+  "name":"org.apache.shardingsphere.sql.parser.sql.dialect.statement.sqlserver.dml.SQLServerSelectStatement",
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+},
 {
   "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.database.visitor.SQLStatementVisitorFactory"},
   "name":"org.apache.shardingsphere.sql.parser.sql92.visitor.statement.SQL92StatementVisitorFacade",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sqlserver.parser.SQLServerLexer"},
+  "name":"org.apache.shardingsphere.sql.parser.sqlserver.parser.SQLServerLexer",
+  "methods":[{"name":"<init>","parameterTypes":["org.antlr.v4.runtime.CharStream"] }]
+},
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sqlserver.parser.SQLServerParser"},
+  "name":"org.apache.shardingsphere.sql.parser.sqlserver.parser.SQLServerParser",
+  "methods":[{"name":"<init>","parameterTypes":["org.antlr.v4.runtime.TokenStream"] }]
+},
 {
   "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.database.visitor.SQLStatementVisitorFactory"},
   "name":"org.apache.shardingsphere.sql.parser.sqlserver.visitor.statement.SQLServerStatementVisitorFacade",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.optimizer.planner.util.SQLFederationFunctionUtils"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sqlserver.visitor.statement.type.SQLServerDDLStatementVisitor"},
+  "name":"org.apache.shardingsphere.sql.parser.sqlserver.visitor.statement.type.SQLServerDDLStatementVisitor",
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.rule.SQLFederationRule"},
   "name":"org.apache.shardingsphere.sqlfederation.optimizer.planner.util.SQLFederationFunctionUtils",
   "queryAllPublicMethods":true
 },
@@ -867,8 +902,8 @@
   "name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfiguration"
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.opengauss.visitor.statement.type.OpenGaussDMLStatementVisitor"},
-  "name":"org.apache.shardingsphere.sql.parser.opengauss.visitor.statement.type.OpenGaussDMLStatementVisitor",
+  "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.sqlserver.visitor.statement.type.SQLServerDMLStatementVisitor"},
+  "name":"org.apache.shardingsphere.sql.parser.sqlserver.visitor.statement.type.SQLServerDMLStatementVisitor",
   "methods":[{"name":"<init>","parameterTypes":[] }]
 }
 ]
diff --git a/pom.xml b/pom.xml
@@ -1078,6 +1078,9 @@
                             <version>${native-maven-plugin.version}</version>
                             <extensions>true</extensions>
                             <configuration>
+                                <buildArgs>
+                                    <buildArg>-H:+AddAllCharsets</buildArg>
+                                </buildArgs>
                                 <classesDirectory>${project.build.outputDirectory}</classesDirectory>
                                 <metadataRepository>
                                     <enabled>true</enabled>

diff --git a/test/native/native-image-filter/extra-filter.json b/test/native/native-image-filter/extra-filter.json
@@ -20,7 +20,7 @@
     {"excludeClasses": "com.ibm.icu.text.**"},
     {"excludeClasses": "com.mysql.cj.**"},
     {"excludeClasses": "com.zaxxer.hikari.**"},
-    {"excludeClasses": "javax.xml.bind.annotation.**"},
+    {"excludeClasses": "javax.xml.**"},
     {"excludeClasses": "groovy.**"},
     {"excludeClasses": "org.apache.calcite.**"},
     {"excludeClasses": "org.codehaus.groovy.**"},

diff --git a/test/native/pom.xml b/test/native/pom.xml
@@ -65,11 +65,21 @@
             <artifactId>opengauss-jdbc</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.microsoft.sqlserver</groupId>
+            <artifactId>mssql-jdbc</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>postgresql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mssqlserver</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...ava/org/apache/shardingsphere/test/natived/jdbc/commons/repository/AddressRepository.java b/...ava/org/apache/shardingsphere/test/natived/jdbc/commons/repository/AddressRepository.java
@@ -50,6 +50,24 @@ public void createTableIfNotExists() throws SQLException {
         }
     }
 
+    /**
+     * create table t_address in MS SQL Server.
+     * This also ignored the default schema of the `dbo`.
+     * @throws SQLException SQL exception
+     */
+    public void createTableInSQLServer() throws SQLException {
+        String sql = "CREATE TABLE [t_address] (\n"
+                + "    address_id bigint NOT NULL,\n"
+                + "    address_name varchar(100) NOT NULL,\n"
+                + "    PRIMARY KEY (address_id)\n"
+                + ");";
+        try (
+                Connection connection = dataSource.getConnection();
+                Statement statement = connection.createStatement()) {
+            statement.executeUpdate(sql);
+        }
+    }
+
     /**
      * drop table t_address.
      * @throws SQLException SQL exception