diff --git a/.github/workflows/nightly-build-artifact.yml b/.github/workflows/nightly-build-artifact.yml index d09166afca5f2..0b9a23d5ae8f4 100644 --- a/.github/workflows/nightly-build-artifact.yml +++ b/.github/workflows/nightly-build-artifact.yml @@ -116,7 +116,7 @@ jobs: with: version: '22.3.1' java-version: '17' - components: 'native-image' + components: 'espresso,native-image' github-token: ${{ secrets.GITHUB_TOKEN }} cache: 'maven' - uses: docker/login-action@v2 diff --git a/distribution/proxy-native/Dockerfile b/distribution/proxy-native/Dockerfile index 4d10a573be0f5..4452ec0d486c4 100644 --- a/distribution/proxy-native/Dockerfile +++ b/distribution/proxy-native/Dockerfile @@ -23,10 +23,14 @@ FROM oraclelinux:9-slim MAINTAINER ShardingSphere "dev@shardingsphere.apache.org" -COPY --from=prepare /conf/ /conf +ENV LOCAL_PATH /opt/shardingsphere-proxy-native + +bash <(curl -sL https://get.graalvm.org/jdk) -c espresso graalvm-ce-java17-22.3.1 + +COPY --from=prepare /conf/ ${LOCAL_PATH}/conf ARG APP_NAME -ADD target/${APP_NAME} ./ +ADD target/${APP_NAME} ${LOCAL_PATH}/ -ENTRYPOINT ./${APP_NAME} 3307 /conf +ENTRYPOINT ${LOCAL_PATH}/${APP_NAME} 3307 ${LOCAL_PATH}/conf diff --git a/distribution/proxy-native/pom.xml b/distribution/proxy-native/pom.xml index 1e42518e118ae..8470a33acdf68 100644 --- a/distribution/proxy-native/pom.xml +++ b/distribution/proxy-native/pom.xml @@ -100,6 +100,7 @@ native 5.0.1 + 22.3.1 @@ -124,6 +125,7 @@ false true + --language:java --report-unsupported-elements-at-runtime @@ -131,6 +133,7 @@ true + 0.2.6 diff --git a/distribution/proxy-native/src/main/release-docs/LICENSE b/distribution/proxy-native/src/main/release-docs/LICENSE index 9ed1877c9fc1d..8ddb2088c2219 100644 --- a/distribution/proxy-native/src/main/release-docs/LICENSE +++ b/distribution/proxy-native/src/main/release-docs/LICENSE @@ -357,4 +357,13 @@ The text of each license is also included at licenses/LICENSE-[project].txt. checker-qual 3.5.0: https://github.com/typetools/checker-framework/blob/master/checker-qual, MIT jul-to-slf4j 1.7.36: https://www.slf4j.org, MIT slf4j-api 1.7.36: https://www.slf4j.org, MIT - jnanoid 2.0.0: https://github.com/aventrix/jnanoid, MIT + jnanoid 2.0.0: https://github.com/aventrix/jnanoid, MIT + +======================================================================== +UPL licenses +======================================================================== + +The following components are provided under the UPL License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + + truffle-api 22.3.1: http://www.graalvm.org/, UPL diff --git a/distribution/proxy-native/src/main/release-docs/licenses/LICENSE-truffle-api.txt b/distribution/proxy-native/src/main/release-docs/licenses/LICENSE-truffle-api.txt new file mode 100644 index 0000000000000..d656a31cb15d7 --- /dev/null +++ b/distribution/proxy-native/src/main/release-docs/licenses/LICENSE-truffle-api.txt @@ -0,0 +1,35 @@ +The Universal Permissive License (UPL), Version 1.0 + +Subject to the condition set forth below, permission is hereby granted to any +person obtaining a copy of this software, associated documentation and/or +data (collectively the "Software"), free of charge and under any and all +copyright rights in the Software, and any and all patent rights owned or +freely licensable by each licensor hereunder covering either (i) the +unmodified Software as contributed to or provided by such licensor, or (ii) +the Larger Works (as defined below), to deal in both + +(a) the Software, and + +(b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +one is included with the Software each a "Larger Work" to which the Software +is contributed by such licensors), + +without restriction, including without limitation the rights to copy, create +derivative works of, display, perform, and distribute the Software and make, +use, sell, offer for sale, import, export, have made, and have sold the +Software and the Larger Work(s), and to sublicense the foregoing rights on +either these or other terms. + +This license is subject to the following condition: + +The above copyright notice and either this complete permission notice or at a +minimum a reference to the UPL must be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/distribution/proxy/src/main/release-docs/LICENSE b/distribution/proxy/src/main/release-docs/LICENSE index 37e07a7e84a09..a099c99cec930 100644 --- a/distribution/proxy/src/main/release-docs/LICENSE +++ b/distribution/proxy/src/main/release-docs/LICENSE @@ -360,3 +360,12 @@ The text of each license is also included at licenses/LICENSE-[project].txt. jul-to-slf4j 1.7.36: https://www.slf4j.org, MIT slf4j-api 1.7.36: https://www.slf4j.org, MIT jnanoid 2.0.0: https://github.com/aventrix/jnanoid, MIT + +======================================================================== +UPL licenses +======================================================================== + +The following components are provided under the UPL License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + + truffle-api 21.2.0: http://www.graalvm.org/, UPL diff --git a/distribution/proxy/src/main/release-docs/licenses/LICENSE-truffle-api.txt b/distribution/proxy/src/main/release-docs/licenses/LICENSE-truffle-api.txt new file mode 100644 index 0000000000000..d656a31cb15d7 --- /dev/null +++ b/distribution/proxy/src/main/release-docs/licenses/LICENSE-truffle-api.txt @@ -0,0 +1,35 @@ +The Universal Permissive License (UPL), Version 1.0 + +Subject to the condition set forth below, permission is hereby granted to any +person obtaining a copy of this software, associated documentation and/or +data (collectively the "Software"), free of charge and under any and all +copyright rights in the Software, and any and all patent rights owned or +freely licensable by each licensor hereunder covering either (i) the +unmodified Software as contributed to or provided by such licensor, or (ii) +the Larger Works (as defined below), to deal in both + +(a) the Software, and + +(b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +one is included with the Software each a "Larger Work" to which the Software +is contributed by such licensors), + +without restriction, including without limitation the rights to copy, create +derivative works of, display, perform, and distribute the Software and make, +use, sell, offer for sale, import, export, have made, and have sold the +Software and the Larger Work(s), and to sublicense the foregoing rights on +either these or other terms. + +This license is subject to the following condition: + +The above copyright notice and either this complete permission notice or at a +minimum a reference to the UPL must be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md index 8e52315838341..90005df99b593 100644 --- a/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md +++ b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md @@ -21,7 +21,7 @@ services: apache-shardingsphere-proxy-native: image: ghcr.io/apache/shardingsphere-proxy-native:latest volumes: - - ./custom/conf:/conf + - ./custom/conf:/opt/shardingsphere-proxy-native/conf ports: - "3307:3307" ``` @@ -34,17 +34,29 @@ services: 需要等待 Junit 5 Platform 的集成，你总是需要在构建 GraalVM Native Image 的过程中， 加上特定于 `GraalVM Native Build Tools` 的 `-DskipNativeTests` 或 `-DskipTests` 参数跳过 Native Image 中的单元测试。 +- 如下 3 个算法类由于涉及到 GraalVM Truffle Espresso 不方便在 host JVM 和 guest JVM 之间交互的 `groovy.lang.Closure` + 类，暂未可在 GraalVM Native Image 下使用。 + - `org.apache.shardingsphere.sharding.algorithm.sharding.complex.ComplexInlineShardingAlgorithm` + - `org.apache.shardingsphere.sharding.algorithm.sharding.hint.HintInlineShardingAlgorithm` + - `org.apache.shardingsphere.sharding.algorithm.sharding.inline.InlineShardingAlgorithm` + +- 当前阶段，GraalVM Native Image 形态的 ShardingSphere Proxy 处于混合 AOT ( GraalVM Native Image ) 和 JIT ( GraalVM + Truffle Espresso ) 运行的阶段。由于 https://github.com/oracle/graal/issues/4555 尚未关闭，GraalVM Truffle Espresso + 运行需要的 `.so` 文件并不会进入 GraalVM Native Image 内。因此如果你需要在 Docker Image 外运行 ShardingSphere Proxy + Native 的二进制文件，你需要确保系统环境变量 `GRAALVM_HOME` 或 `JAVA_HOME` 指向 GraalVM 的 `bin` 目录，并且此 GraalVM + 实例已经通过 `GraalVM Updater` 安装了 `espresso` 组件。目前，`GRAALVM_HOME` 优先级比 `JAVA_HOME` 高。 + - 本节假定处于 Linux（amd64，aarch64）， MacOS（amd64）或 Windows（amd64）环境。 - 如果你位于 MacOS（aarch64/M1） 环境，你需要关注尚未关闭的 https://github.com/oracle/graal/issues/2666。 + 如果你位于 MacOS（aarch64/M1） 环境，你需要关注尚未关闭的 https://github.com/oracle/graal/issues/2666 。 ## 前提条件 1. 根据 https://www.graalvm.org/downloads/ 要求安装和配置 JDK 17 对应的 `GraalVM CE` 或 `GraalVM EE`。 同时可以通过 `SDKMAN!` 安装 JDK 17 对应的 `GraalVM CE`。 -2. 通过 `GraalVM Updater` 工具安装 `native-image` 组件。 +2. 通过 `GraalVM Updater` 工具安装 `native-image` 和 `espresso` 组件。 -3. 根据 https://www.graalvm.org/22.2/reference-manual/native-image/#prerequisites 的要求安装本地工具链。 +3. 根据 https://www.graalvm.org/22.3/reference-manual/native-image/#prerequisites 的要求安装本地工具链。 4. 如果需要构建 Docker Image， 确保 `docker-cli` 在系统环境变量内。 @@ -73,16 +85,16 @@ services: ```xml - - com.mysql - mysql-connector-j - 8.0.31 - - - org.apache.shardingsphere - shardingsphere-sql-translator-jooq-provider - 5.2.0 - + + com.mysql + mysql-connector-j + 8.0.32 + + + org.apache.shardingsphere + shardingsphere-sql-translator-jooq-provider + 5.3.1 + ``` @@ -116,14 +128,14 @@ services: apache-shardingsphere-proxy-native: image: apache/shardingsphere-proxy-native:latest volumes: - - ./custom/conf:/conf + - ./custom/conf:/opt/shardingsphere-proxy-native/conf ports: - "3307:3307" ``` - 如果你不对 Git Source 做任何更改， 上文提及的命令将使用 `oraclelinux:9-slim` 作为 Base Docker Image。 但如果你希望使用 `busybox:glic`，`gcr.io/distroless/base` 或 `scratch` 等更小体积的 Docker Image 作为 Base Docker - Image，你需要根据 https://www.graalvm.org/22.2/reference-manual/native-image/guides/build-static-executables/ 的要求， + Image，你需要根据 https://www.graalvm.org/22.3/reference-manual/native-image/guides/build-static-executables/ 的要求， 做为 `pom.xml`的 `native profile` 添加 `-H:+StaticExecutableWithDynamicLibC` 的 `jvmArgs` 等操作。 另请注意，某些第三方依赖将需要在 `Dockerfile` 安装更多系统库，例如 `libdl`。 因此请确保根据你的使用情况调整 `distribution/proxy-native` @@ -135,7 +147,7 @@ services: Proxy，其提供的可观察性的能力与 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-proxy/observability/ 并不一致。 -- 你可以使用 https://www.graalvm.org/22.2/tools/ 提供的一系列命令行工具或可视化工具观察 GraalVM Native Image +- 你可以使用 https://www.graalvm.org/22.3/tools/ 提供的一系列命令行工具或可视化工具观察 GraalVM Native Image 的内部行为，并根据其要求使用 VSCode 完成调试工作。 如果你正在使用 IntelliJ IDEA 并且希望调试生成的 GraalVM Native Image， 你可以关注 https://blog.jetbrains.com/idea/2022/06/intellij-idea-2022-2-eap-5/#Experimental_GraalVM_Native_Debugger_for_Java diff --git a/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md index bf482eeb6f439..0f03b9d88342a 100644 --- a/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md +++ b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md @@ -23,7 +23,7 @@ services: apache-shardingsphere-proxy-native: image: ghcr.io/apache/shardingsphere-proxy-native:latest volumes: - - ./custom/conf:/conf + - ./custom/conf:/opt/shardingsphere-proxy-native/conf ports: - "3307:3307" ```` @@ -38,8 +38,23 @@ services: Plus `-DskipNativeTests` or `-DskipTests` parameter specific to `GraalVM Native Build Tools` to skip unit tests in Native Image. +- The following three algorithm classes are not available under GraalVM Native Image because they involve + the `groovy.lang.Closure` class that is inconvenient for GraalVM Truffle Espresso to interact between the host JVM and + the guest JVM. + - `org.apache.shardingsphere.sharding.algorithm.sharding.complex.ComplexInlineShardingAlgorithm` + - `org.apache.shardingsphere.sharding.algorithm.sharding.hint.HintInlineShardingAlgorithm` + - `org.apache.shardingsphere.sharding.algorithm.sharding.inline.InlineShardingAlgorithm` + +- At the current stage, ShardingSphere Proxy in GraalVM Native Image is in the stage of mixed AOT ( GraalVM + Native Image ) and JIT ( GraalVM Truffle Espresso ) operation. Since https://github.com/oracle/graal/issues/4555 has + not been closed, the `.so` file required for GraalVM Truffle Espresso to run does not enter the GraalVM Native Image. + So if you need to run the binary files of ShardingSphere Proxy Native outside the Docker Image, you need to ensure + that the system environment variable `GRAALVM_HOME` or `JAVA_HOME` points to the `bin` directory of GraalVM, and this + GraalVM instance has been installed `espresso` component by `GraalVM Updater`. Currently, `GRAALVM_HOME` has higher + priority than `JAVA_HOME`. + - This section assumes a Linux (amd64, aarch64), MacOS (amd64) or Windows (amd64) environment. - If you are on MacOS(aarch64/M1) environment, you need to follow https://github.com/oracle/graal/issues/2666 which is + If you are on MacOS (aarch64/M1) environment, you need to follow https://github.com/oracle/graal/issues/2666 which is not closed yet. ## Premise @@ -47,9 +62,9 @@ services: 1. Install and configure `GraalVM CE` or `GraalVM EE` for JDK 17 according to https://www.graalvm.org/downloads/. `GraalVM CE` for JDK 17 can also be installed via `SDKMAN!`. -2. Install the `native-image` component via the `GraalVM Updater` tool. +2. Install the `native-image` and `espresso` component via the `GraalVM Updater` tool. -3. Install the local toolchain as required by https://www.graalvm.org/22.2/reference-manual/native-image/#prerequisites. +3. Install the local toolchain as required by https://www.graalvm.org/22.3/reference-manual/native-image/#prerequisites. 4. If you need to build a Docker Image, make sure `docker-cli` is in the system environment variables. @@ -83,12 +98,12 @@ services: com.mysql mysql-connector-j - 8.0.31 + 8.0.32 org.apache.shardingsphere shardingsphere-sql-translator-jooq-provider - 5.2.0 + 5.3.1 ``` @@ -125,7 +140,7 @@ services: apache-shardingsphere-proxy-native: image: apache/shardingsphere-proxy-native:latest volumes: - - ./custom/conf:/conf + - ./custom/conf:/opt/shardingsphere-proxy-native/conf ports: - "3307:3307" ``` @@ -134,7 +149,7 @@ services: Base Docker Image. But if you want to use a smaller Docker Image like `busybox:glic`, `gcr.io/distroless/base` or `scratch` as the Base Docker Image, you need according - to https://www.graalvm.org/22.2/reference-manual/native-image/guides/build-static-executables/, + to https://www.graalvm.org/22.3/reference-manual/native-image/guides/build-static-executables/, Add operations such as `-H:+StaticExecutableWithDynamicLibC` to `jvmArgs` as the `native profile` of `pom.xml`. Also note that some 3rd party dependencies will require more system libraries such as `libdl` to be installed in the `Dockerfile`. @@ -148,7 +163,7 @@ services: Not consistent. - You can observe GraalVM Native Image using a series of command line tools or visualization tools available - at https://www.graalvm.org/22.2/tools/, and use VSCode to debug it according to its requirements. + at https://www.graalvm.org/22.3/tools/, and use VSCode to debug it according to its requirements. If you are using IntelliJ IDEA and want to debug the generated GraalVM Native Image, You can follow https://blog.jetbrains.com/idea/2022/06/intellij-idea-2022-2-eap-5/#Experimental_GraalVM_Native_Debugger_for_Java diff --git a/infra/pom.xml b/infra/pom.xml index 3ae2d171c02b1..bc49370479aa9 100644 --- a/infra/pom.xml +++ b/infra/pom.xml @@ -38,5 +38,6 @@ executor merge context + util-groovy diff --git a/infra/util-groovy/pom.xml b/infra/util-groovy/pom.xml new file mode 100644 index 0000000000000..6ddcfb9df74ee --- /dev/null +++ b/infra/util-groovy/pom.xml @@ -0,0 +1,40 @@ + + + + + 4.0.0 + + org.apache.shardingsphere + shardingsphere-infra + 5.3.2-SNAPSHOT + + + shardingsphere-infra-util-groovy + + + + org.apache.groovy + groovy + + + com.google.guava + guava + + + diff --git a/infra/util-groovy/src/main/java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParser.java b/infra/util-groovy/src/main/java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParser.java new file mode 100644 index 0000000000000..caacaf5c6917e --- /dev/null +++ b/infra/util-groovy/src/main/java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParser.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.infra.util.groovy.expr; + +import com.google.common.base.Strings; +import com.google.common.collect.Sets; +import groovy.lang.Closure; +import groovy.lang.GString; +import groovy.lang.GroovyShell; +import groovy.lang.Script; +import lombok.RequiredArgsConstructor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Inline expression parser. + */ +@RequiredArgsConstructor +public final class HotspotInlineExpressionParser { + + private static final char SPLITTER = ','; + + private static final Map SCRIPTS = new ConcurrentHashMap<>(); + + private static final GroovyShell SHELL = new GroovyShell(); + + private final String inlineExpression; + + /** + * Replace all inline expression placeholders. + * + * @param inlineExpression inline expression with {@code $->} + * @return result inline expression with {@code $} + */ + public static String handlePlaceHolder(final String inlineExpression) { + return inlineExpression.contains("$->{") ? inlineExpression.replaceAll("\\$->\\{", "\\$\\{") : inlineExpression; + } + + /** + * Split and evaluate inline expression. + * + * @return result list + */ + public List splitAndEvaluate() { + return Strings.isNullOrEmpty(inlineExpression) ? Collections.emptyList() : flatten(evaluate(split())); + } + + /** + * Evaluate closure. + * + * @return closure + */ + public Closure evaluateClosure() { + return (Closure) evaluate("{it -> \"" + inlineExpression + "\"}"); + } + + private List evaluate(final List inlineExpressions) { + List result = new ArrayList<>(inlineExpressions.size()); + for (String each : inlineExpressions) { + StringBuilder expression = new StringBuilder(handlePlaceHolder(each)); + if (!each.startsWith("\"")) { + expression.insert(0, "\""); + } + if (!each.endsWith("\"")) { + expression.append("\""); + } + result.add(evaluate(expression.toString())); + } + return result; + } + + private Object evaluate(final String expression) { + Script script; + if (SCRIPTS.containsKey(expression)) { + script = SCRIPTS.get(expression); + } else { + script = SHELL.parse(expression); + SCRIPTS.put(expression, script); + } + return script.run(); + } + + private List split() { + List result = new ArrayList<>(); + StringBuilder segment = new StringBuilder(); + int bracketsDepth = 0; + for (int i = 0; i < inlineExpression.length(); i++) { + char each = inlineExpression.charAt(i); + switch (each) { + case SPLITTER: + if (bracketsDepth > 0) { + segment.append(each); + } else { + result.add(segment.toString().trim()); + segment.setLength(0); + } + break; + case '$': + if ('{' == inlineExpression.charAt(i + 1)) { + bracketsDepth++; + } + if ("->{".equals(inlineExpression.substring(i + 1, i + 4))) { + bracketsDepth++; + } + segment.append(each); + break; + case '}': + if (bracketsDepth > 0) { + bracketsDepth--; + } + segment.append(each); + break; + default: + segment.append(each); + break; + } + } + if (segment.length() > 0) { + result.add(segment.toString().trim()); + } + return result; + } + + private List flatten(final List segments) { + List result = new ArrayList<>(); + for (Object each : segments) { + if (each instanceof GString) { + result.addAll(assemblyCartesianSegments((GString) each)); + } else { + result.add(each.toString()); + } + } + return result; + } + + private List assemblyCartesianSegments(final GString segment) { + Set> cartesianValues = getCartesianValues(segment); + List result = new ArrayList<>(cartesianValues.size()); + for (List each : cartesianValues) { + result.add(assemblySegment(each, segment)); + } + return result; + } + + @SuppressWarnings("unchecked") + private Set> getCartesianValues(final GString segment) { + List> result = new ArrayList<>(segment.getValues().length); + for (Object each : segment.getValues()) { + if (null == each) { + continue; + } + if (each instanceof Collection) { + result.add(((Collection) each).stream().map(Object::toString).collect(Collectors.toCollection(LinkedHashSet::new))); + } else { + result.add(Sets.newHashSet(each.toString())); + } + } + return Sets.cartesianProduct(result); + } + + private String assemblySegment(final List cartesianValue, final GString segment) { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < segment.getStrings().length; i++) { + result.append(segment.getStrings()[i]); + if (i < cartesianValue.size()) { + result.append(cartesianValue.get(i)); + } + } + return result.toString(); + } +} diff --git a/infra/util-groovy/src/test/java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParserTest.java b/infra/util-groovy/src/test/java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParserTest.java new file mode 100644 index 0000000000000..3b60a857b7fd2 --- /dev/null +++ b/infra/util-groovy/src/test/java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParserTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.infra.util.groovy.expr; + +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public final class HotspotInlineExpressionParserTest { + + @Test + public void assertEvaluateForExpressionIsNull() { + List expected = new HotspotInlineExpressionParser(null).splitAndEvaluate(); + assertThat(expected, is(Collections.emptyList())); + } + + @Test + public void assertEvaluateForSimpleString() { + List expected = new HotspotInlineExpressionParser(" t_order_0, t_order_1 ").splitAndEvaluate(); + assertThat(expected.size(), is(2)); + assertThat(expected, hasItems("t_order_0", "t_order_1")); + } + + @Test + public void assertEvaluateForNull() { + List expected = new HotspotInlineExpressionParser("t_order_${null}").splitAndEvaluate(); + assertThat(expected.size(), is(1)); + assertThat(expected, hasItems("t_order_")); + } + + @Test + public void assertEvaluateForLiteral() { + List expected = new HotspotInlineExpressionParser("t_order_${'xx'}").splitAndEvaluate(); + assertThat(expected.size(), is(1)); + assertThat(expected, hasItems("t_order_xx")); + } + + @Test + public void assertEvaluateForArray() { + List expected = new HotspotInlineExpressionParser("t_order_${[0, 1, 2]},t_order_item_${[0, 2]}").splitAndEvaluate(); + assertThat(expected.size(), is(5)); + assertThat(expected, hasItems("t_order_0", "t_order_1", "t_order_2", "t_order_item_0", "t_order_item_2")); + } + + @Test + public void assertEvaluateForRange() { + List expected = new HotspotInlineExpressionParser("t_order_${0..2},t_order_item_${0..1}").splitAndEvaluate(); + assertThat(expected.size(), is(5)); + assertThat(expected, hasItems("t_order_0", "t_order_1", "t_order_2", "t_order_item_0", "t_order_item_1")); + } + + @Test + public void assertEvaluateForComplex() { + List expected = new HotspotInlineExpressionParser("t_${['new','old']}_order_${1..2}, t_config").splitAndEvaluate(); + assertThat(expected.size(), is(5)); + assertThat(expected, hasItems("t_new_order_1", "t_new_order_2", "t_old_order_1", "t_old_order_2", "t_config")); + } + + @Test + public void assertEvaluateForCalculate() { + List expected = new HotspotInlineExpressionParser("t_${[\"new${1+2}\",'old']}_order_${1..2}").splitAndEvaluate(); + assertThat(expected.size(), is(4)); + assertThat(expected, hasItems("t_new3_order_1", "t_new3_order_2", "t_old_order_1", "t_old_order_2")); + } + + @Test + public void assertEvaluateForExpressionPlaceHolder() { + List expected = new HotspotInlineExpressionParser("t_$->{[\"new$->{1+2}\",'old']}_order_$->{1..2}").splitAndEvaluate(); + assertThat(expected.size(), is(4)); + assertThat(expected, hasItems("t_new3_order_1", "t_new3_order_2", "t_old_order_1", "t_old_order_2")); + } + + @Test + public void assertEvaluateForLong() { + StringBuilder expression = new StringBuilder(); + for (int i = 0; i < 1024; i++) { + expression.append("ds_"); + expression.append(i / 64); + expression.append(".t_user_"); + expression.append(i); + if (i != 1023) { + expression.append(","); + } + } + List expected = new HotspotInlineExpressionParser(expression.toString()).splitAndEvaluate(); + assertThat(expected.size(), is(1024)); + assertThat(expected, hasItems("ds_0.t_user_0", "ds_15.t_user_1023")); + } + + @Test + public void assertHandlePlaceHolder() { + assertThat(HotspotInlineExpressionParser.handlePlaceHolder("t_$->{[\"new$->{1+2}\"]}"), is("t_${[\"new${1+2}\"]}")); + assertThat(HotspotInlineExpressionParser.handlePlaceHolder("t_${[\"new$->{1+2}\"]}"), is("t_${[\"new${1+2}\"]}")); + } + + @Test + public void assertEvaluateClosure() { + assertThat(new HotspotInlineExpressionParser("${1+2}").evaluateClosure().call().toString(), is("3")); + } +} diff --git a/infra/util/pom.xml b/infra/util/pom.xml index aec174fd5967d..48d5927f65b21 100644 --- a/infra/util/pom.xml +++ b/infra/util/pom.xml @@ -33,14 +33,67 @@ ${project.version} test + + org.apache.shardingsphere + shardingsphere-infra-util-groovy + ${project.version} + org.yaml snakeyaml - org.apache.groovy - groovy + org.graalvm.truffle + truffle-api + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + + copy + + process-test-classes + + + + + org.apache.shardingsphere + shardingsphere-infra-util + 5.3.1 + jar + true + shardingsphere-infra-util.jar + + + org.apache.groovy + groovy + ${groovy.version} + jar + true + groovy.jar + + + com.google.guava + guava + ${guava.version} + jar + true + guava.jar + + + ${project.build.outputDirectory}/espresso-need-libs + + + + + + diff --git a/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/EspressoInlineExpressionParser.java b/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/EspressoInlineExpressionParser.java new file mode 100644 index 0000000000000..bd1114329698d --- /dev/null +++ b/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/EspressoInlineExpressionParser.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shardingsphere.infra.util.expr; + +import groovy.lang.Closure; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Value; + +import java.net.URL; +import java.util.Collections; +import java.util.List; + +/** + * Espresso Inline expression parser. + */ +public class EspressoInlineExpressionParser { + + private static final Context POLYGLOT; + + private final Value espressoInlineExpressionParser; + + static { + // https://github.com/oracle/graal/issues/4555 not yet closed + String javaHome = System.getenv("GRAALVM_HOME"); + if (javaHome == null) { + javaHome = System.getenv("JAVA_HOME"); + } + if (javaHome == null) { + throw new RuntimeException("Failed to determine the system's environment variable GRAALVM_HOME or JAVA_HOME!"); + } + System.setProperty("org.graalvm.home", javaHome); + URL resource = Thread.currentThread().getContextClassLoader().getResource("espresso-need-libs"); + assert null != resource; + String dir = resource.getPath(); + String javaClasspath = String.join(":", dir + "/groovy.jar", dir + "/guava.jar", dir + "/shardingsphere-infra-util.jar"); + POLYGLOT = Context.newBuilder().allowAllAccess(true) + .option("java.MultiThreaded", "true") + .option("java.Classpath", javaClasspath) + .build(); + } + + public EspressoInlineExpressionParser(final String inlineExpression) { + espressoInlineExpressionParser = POLYGLOT.getBindings("java") + .getMember("org.apache.shardingsphere.infra.util.expr.InlineExpressionParser") + .newInstance(inlineExpression); + } + + /** + * Replace all inline expression placeholders. + * + * @param inlineExpression inline expression with {@code $->} + * @return result inline expression with {@code $} + */ + public static String handlePlaceHolder(final String inlineExpression) { + return POLYGLOT.getBindings("java") + .getMember("org.apache.shardingsphere.infra.util.expr.InlineExpressionParser") + .invokeMember("handlePlaceHolder", inlineExpression) + .as(String.class); + } + + /** + * Split and evaluate inline expression. + * + * @return result list + */ + @SuppressWarnings("unchecked") + public List splitAndEvaluate() { + List splitAndEvaluate = espressoInlineExpressionParser.invokeMember("splitAndEvaluate").as(List.class); + // GraalVM Truffle Espresso CE 22.3.1 has a different behavior for generic List than Hotspot. + return splitAndEvaluate.size() == 0 ? Collections.emptyList() : splitAndEvaluate; + } + + /** + * Evaluate closure. + * + * @return closure + */ + public Closure evaluateClosure() { + return espressoInlineExpressionParser.invokeMember("evaluateClosure").as(Closure.class); + } +} diff --git a/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParser.java b/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParser.java index f57969908abf7..516d61eb53e90 100644 --- a/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParser.java +++ b/infra/util/src/main/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParser.java @@ -17,23 +17,11 @@ package org.apache.shardingsphere.infra.util.expr; -import com.google.common.base.Strings; -import com.google.common.collect.Sets; import groovy.lang.Closure; -import groovy.lang.GString; -import groovy.lang.GroovyShell; -import groovy.lang.Script; import lombok.RequiredArgsConstructor; +import org.apache.shardingsphere.infra.util.groovy.expr.HotspotInlineExpressionParser; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** * Inline expression parser. @@ -41,22 +29,39 @@ @RequiredArgsConstructor public final class InlineExpressionParser { - private static final char SPLITTER = ','; + private static final boolean IS_SUBSTRATE_VM; - private static final Map SCRIPTS = new ConcurrentHashMap<>(); + private final EspressoInlineExpressionParser espressoInlineExpressionParser; - private static final GroovyShell SHELL = new GroovyShell(); + private final HotspotInlineExpressionParser hotspotInlineExpressionParser; - private final String inlineExpression; + static { + // workaround for https://github.com/helidon-io/helidon-build-tools/issues/858 + IS_SUBSTRATE_VM = System.getProperty("java.vm.name").equals("Substrate VM"); + } + + public InlineExpressionParser(final String inlineExpression) { + if (IS_SUBSTRATE_VM) { + this.hotspotInlineExpressionParser = null; + this.espressoInlineExpressionParser = new EspressoInlineExpressionParser(inlineExpression); + } else { + this.hotspotInlineExpressionParser = new HotspotInlineExpressionParser(inlineExpression); + this.espressoInlineExpressionParser = null; + } + } /** * Replace all inline expression placeholders. - * + * * @param inlineExpression inline expression with {@code $->} * @return result inline expression with {@code $} */ public static String handlePlaceHolder(final String inlineExpression) { - return inlineExpression.contains("$->{") ? inlineExpression.replaceAll("\\$->\\{", "\\$\\{") : inlineExpression; + if (IS_SUBSTRATE_VM) { + return EspressoInlineExpressionParser.handlePlaceHolder(inlineExpression); + } else { + return HotspotInlineExpressionParser.handlePlaceHolder(inlineExpression); + } } /** @@ -65,7 +70,13 @@ public static String handlePlaceHolder(final String inlineExpression) { * @return result list */ public List splitAndEvaluate() { - return Strings.isNullOrEmpty(inlineExpression) ? Collections.emptyList() : flatten(evaluate(split())); + if (IS_SUBSTRATE_VM) { + assert null != espressoInlineExpressionParser; + return espressoInlineExpressionParser.splitAndEvaluate(); + } else { + assert null != hotspotInlineExpressionParser; + return hotspotInlineExpressionParser.splitAndEvaluate(); + } } /** @@ -74,121 +85,12 @@ public List splitAndEvaluate() { * @return closure */ public Closure evaluateClosure() { - return (Closure) evaluate("{it -> \"" + inlineExpression + "\"}"); - } - - private List evaluate(final List inlineExpressions) { - List result = new ArrayList<>(inlineExpressions.size()); - for (String each : inlineExpressions) { - StringBuilder expression = new StringBuilder(handlePlaceHolder(each)); - if (!each.startsWith("\"")) { - expression.insert(0, "\""); - } - if (!each.endsWith("\"")) { - expression.append("\""); - } - result.add(evaluate(expression.toString())); - } - return result; - } - - private Object evaluate(final String expression) { - Script script; - if (SCRIPTS.containsKey(expression)) { - script = SCRIPTS.get(expression); + if (IS_SUBSTRATE_VM) { + assert null != espressoInlineExpressionParser; + return espressoInlineExpressionParser.evaluateClosure(); } else { - script = SHELL.parse(expression); - SCRIPTS.put(expression, script); - } - return script.run(); - } - - private List split() { - List result = new ArrayList<>(); - StringBuilder segment = new StringBuilder(); - int bracketsDepth = 0; - for (int i = 0; i < inlineExpression.length(); i++) { - char each = inlineExpression.charAt(i); - switch (each) { - case SPLITTER: - if (bracketsDepth > 0) { - segment.append(each); - } else { - result.add(segment.toString().trim()); - segment.setLength(0); - } - break; - case '$': - if ('{' == inlineExpression.charAt(i + 1)) { - bracketsDepth++; - } - if ("->{".equals(inlineExpression.substring(i + 1, i + 4))) { - bracketsDepth++; - } - segment.append(each); - break; - case '}': - if (bracketsDepth > 0) { - bracketsDepth--; - } - segment.append(each); - break; - default: - segment.append(each); - break; - } - } - if (segment.length() > 0) { - result.add(segment.toString().trim()); - } - return result; - } - - private List flatten(final List segments) { - List result = new ArrayList<>(); - for (Object each : segments) { - if (each instanceof GString) { - result.addAll(assemblyCartesianSegments((GString) each)); - } else { - result.add(each.toString()); - } - } - return result; - } - - private List assemblyCartesianSegments(final GString segment) { - Set> cartesianValues = getCartesianValues(segment); - List result = new ArrayList<>(cartesianValues.size()); - for (List each : cartesianValues) { - result.add(assemblySegment(each, segment)); - } - return result; - } - - @SuppressWarnings("unchecked") - private Set> getCartesianValues(final GString segment) { - List> result = new ArrayList<>(segment.getValues().length); - for (Object each : segment.getValues()) { - if (null == each) { - continue; - } - if (each instanceof Collection) { - result.add(((Collection) each).stream().map(Object::toString).collect(Collectors.toCollection(LinkedHashSet::new))); - } else { - result.add(Sets.newHashSet(each.toString())); - } - } - return Sets.cartesianProduct(result); - } - - private String assemblySegment(final List cartesianValue, final GString segment) { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < segment.getStrings().length; i++) { - result.append(segment.getStrings()[i]); - if (i < cartesianValue.size()) { - result.append(cartesianValue.get(i)); - } + assert null != hotspotInlineExpressionParser; + return hotspotInlineExpressionParser.evaluateClosure(); } - return result.toString(); } } diff --git a/infra/util/src/test/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParserTest.java b/infra/util/src/test/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParserTest.java index 8b3de8a53780c..57d13f1d38eb6 100644 --- a/infra/util/src/test/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParserTest.java +++ b/infra/util/src/test/java/org/apache/shardingsphere/infra/util/expr/InlineExpressionParserTest.java @@ -113,8 +113,16 @@ public void assertHandlePlaceHolder() { assertThat(InlineExpressionParser.handlePlaceHolder("t_${[\"new$->{1+2}\"]}"), is("t_${[\"new${1+2}\"]}")); } + /** + * TODO + * This method needs to avoid returning a groovy.lang.Closure class instance, + * and instead return the result of `Closure#call`. + * Because `org.graalvm.polyglot.Value#as` does not allow this type to be returned from the guest JVM. + */ @Test public void assertEvaluateClosure() { - assertThat(new InlineExpressionParser("${1+2}").evaluateClosure().call().toString(), is("3")); + if (!System.getProperty("java.vm.name").equals("Substrate VM")) { + assertThat(new InlineExpressionParser("${1+2}").evaluateClosure().call().toString(), is("3")); + } } } diff --git a/pom.xml b/pom.xml index 10e846322e4eb..d42764f9e68ca 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,8 @@ 2.3.0 1.3.2 1.2.0 - + 21.2.0 + 1.32.0 4.1.86.Final @@ -128,7 +129,8 @@ 2.8 3.4 2.5 - + 2.10 + 1.0.0 4.3.0 2.7 @@ -580,6 +582,12 @@ ${awaitility.version} test + + + org.graalvm.truffle + truffle-api + ${truffle-api.version} + @@ -850,6 +858,11 @@ + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + @@ -1071,6 +1084,9 @@ org.jacoco jacoco-maven-plugin ${jacoco-maven-plugin.version} + + **/*.jar +