From 1a6ae6e0d74ade0064d65eba761261acb0dca54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9rique=20Mittelstaedt?= Date: Thu, 31 Aug 2023 10:22:53 +0000 Subject: [PATCH 1/6] support flink 1.17.1 --- pom.xml | 98 +++++++++++++++++-- statefun-flink/statefun-flink-core/pom.xml | 3 +- .../flink/core/StatefulFunctionsConfig.java | 2 +- .../flink/core/StatefulFunctionsJob.java | 2 +- .../flink/core/functions/ReductionsTest.java | 5 +- .../statefun-flink-datastream/pom.xml | 16 +-- .../statefun-flink-distribution/pom.xml | 4 +- .../statefun-flink-io-bundle/pom.xml | 9 +- statefun-flink/statefun-flink-io/pom.xml | 2 +- .../statefun-protobuf-shaded/pom.xml | 3 +- .../statefun-protocol-shaded/pom.xml | 9 +- tools/docker/Dockerfile | 4 +- 12 files changed, 122 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 6bf1c7933..99806bb49 100644 --- a/pom.xml +++ b/pom.xml @@ -76,15 +76,28 @@ under the License. 1.8 1.20.0 1.0-rc6 - 3.7.1 - 2.3.2 - 3.11.1 - 1.15.2 + 3.23.2 + 2.6.2 + 3.11.4 + 1.17.1 2.12 2.12.7 1.8.0 - 2.12.4-15.0 + 2.14.2-17.0 1.7.32 + 4.1.0-1.17 + 4.1.0-1.17 + 3.14.6 + 4.1.82.Final-16.1 + 4.12 + 1.3 + 2.24.0 + 2.13.2.2 + 4.1.82.Final-16.1 + 16.1 + 1.15 + 1.2 + 1.7.36 **/*Test.* @@ -101,15 +114,20 @@ under the License. junit junit - 4.12 + ${junit.version} test org.hamcrest hamcrest-all - 1.3 + ${hamcrest-all.version} test + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + org.apache.flink + flink-shaded-netty + ${flink-shaded-netty.version} + + + + org.apache.flink + flink-core + ${flink.version} + + + + org.apache.flink + flink-table-common + ${flink.version} + + + + org.apache.flink + flink-connector-base + ${flink.version} + + + + org.apache.flink + flink-shaded-force-shading + ${flink-shaded-force-shading.version} + + + + org.apache.flink + flink-shaded-jackson + ${flink-shaded-jackson.version} + + + + commons-codec + commons-codec + ${commons-codec.version} + + + + commons-logging + commons-logging + ${commons-logging.version} + + + + org.slf4j + slf4j-api + ${slf4j-api.version} @@ -234,10 +311,12 @@ under the License. descriptor ${basedir}/target/test-classes + main java ${basedir}/target/generated-test-sources/protoc-jar + test @@ -412,4 +491,3 @@ under the License. - diff --git a/statefun-flink/statefun-flink-core/pom.xml b/statefun-flink/statefun-flink-core/pom.xml index 55e8c0061..bfe4d832b 100644 --- a/statefun-flink/statefun-flink-core/pom.xml +++ b/statefun-flink/statefun-flink-core/pom.xml @@ -30,9 +30,7 @@ under the License. statefun-flink-core - 3.14.6 target/additional-sources - 4.1.70.Final-15.0 @@ -90,6 +88,7 @@ under the License. com.kohlschutter.junixsocket junixsocket-core ${unixsocket.version} + pom diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java index 36a14198b..a59d45a3c 100644 --- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java +++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java @@ -265,7 +265,7 @@ public void setEmbedded(boolean embedded) { */ public StatefulFunctionsUniverseProvider getProvider(ClassLoader cl) { try { - return InstantiationUtil.deserializeObject(universeInitializerClassBytes, cl, false); + return InstantiationUtil.deserializeObject(universeInitializerClassBytes, cl); } catch (IOException | ClassNotFoundException e) { throw new IllegalStateException("Unable to initialize.", e); } diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java index c86ef5814..6803ce65c 100644 --- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java +++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java @@ -23,12 +23,12 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; import org.apache.flink.statefun.flink.core.feedback.FeedbackKey; import org.apache.flink.statefun.flink.core.message.Message; import org.apache.flink.statefun.flink.core.translation.FlinkUniverse; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.FlinkUserCodeClassLoader; +import org.apache.flink.util.FlinkUserCodeClassLoaders; public class StatefulFunctionsJob { diff --git a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java index ad65827e2..64f33642e 100644 --- a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java +++ b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java @@ -352,10 +352,11 @@ public boolean deregisterKeySelectionListener(KeySelectionListener liste @Nonnull @Override - public IS createInternalState( + public IS createOrUpdateInternalState( @Nonnull TypeSerializer namespaceSerializer, @Nonnull StateDescriptor stateDesc, - @Nonnull StateSnapshotTransformFactory snapshotTransformFactory) { + @Nonnull StateSnapshotTransformFactory snapshotTransformFactory) + throws Exception { throw new UnsupportedOperationException(); } diff --git a/statefun-flink/statefun-flink-datastream/pom.xml b/statefun-flink/statefun-flink-datastream/pom.xml index ccb8aac8c..5afb088ce 100644 --- a/statefun-flink/statefun-flink-datastream/pom.xml +++ b/statefun-flink/statefun-flink-datastream/pom.xml @@ -51,13 +51,13 @@ under the License. statefun-flink-io ${project.version} - - org.apache.flink @@ -87,7 +87,7 @@ under the License. org.hamcrest hamcrest-all - + @@ -152,4 +152,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-flink/statefun-flink-distribution/pom.xml b/statefun-flink/statefun-flink-distribution/pom.xml index 5ea7b2eb1..6d0b22b40 100644 --- a/statefun-flink/statefun-flink-distribution/pom.xml +++ b/statefun-flink/statefun-flink-distribution/pom.xml @@ -74,7 +74,7 @@ under the License. statefun-flink-launcher ${project.version} - + org.apache.flink @@ -199,4 +199,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-flink/statefun-flink-io-bundle/pom.xml b/statefun-flink/statefun-flink-io-bundle/pom.xml index db79e96ad..490e08e03 100644 --- a/statefun-flink/statefun-flink-io-bundle/pom.xml +++ b/statefun-flink/statefun-flink-io-bundle/pom.xml @@ -90,7 +90,12 @@ under the License. org.apache.flink flink-connector-kinesis - ${flink.version} + ${flink-connector-kinesis.version} + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${flink-connector-aws-kinesis-streams.version} @@ -110,4 +115,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-flink/statefun-flink-io/pom.xml b/statefun-flink/statefun-flink-io/pom.xml index b26b912ff..280f83a51 100644 --- a/statefun-flink/statefun-flink-io/pom.xml +++ b/statefun-flink/statefun-flink-io/pom.xml @@ -124,4 +124,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-shaded/statefun-protobuf-shaded/pom.xml b/statefun-shaded/statefun-protobuf-shaded/pom.xml index d97dae2a7..ec67f7e15 100644 --- a/statefun-shaded/statefun-protobuf-shaded/pom.xml +++ b/statefun-shaded/statefun-protobuf-shaded/pom.xml @@ -32,7 +32,6 @@ under the License. ${generated-sources.basedir}/shaded-protobuf-java/ - 3.7.1 @@ -107,4 +106,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-shaded/statefun-protocol-shaded/pom.xml b/statefun-shaded/statefun-protocol-shaded/pom.xml index 9ad5c9bc0..d31938eb4 100644 --- a/statefun-shaded/statefun-protocol-shaded/pom.xml +++ b/statefun-shaded/statefun-protocol-shaded/pom.xml @@ -33,7 +33,7 @@ under the License. ${generated-sources.basedir}/protocol-messages/ target/proto-sources - 3.11.1 + 3.11.4 @@ -47,6 +47,11 @@ under the License. statefun-protobuf-shaded ${project.version} + + com.google.protobuf + protobuf-java + ${protobuf.version} + @@ -130,4 +135,4 @@ under the License. - \ No newline at end of file + diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 662d034c0..62e579ad5 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM apache/flink:1.15.2-scala_2.12-java8 +FROM flink:1.17.1-scala_2.12-java11 ENV ROLE worker ENV MASTER_HOST localhost @@ -45,7 +45,7 @@ RUN mkdir -p $FLINK_HOME/plugins/azure-fs-hadoop && \ # add tcnative RUN mv $FLINK_HOME/opt/flink-shaded-netty-tcnative-dynamic-*.jar $FLINK_HOME/lib/ -# entry point +# entry point ADD docker-entry-point.sh /docker-entry-point.sh ENTRYPOINT ["/docker-entry-point.sh"] From 44c638fad4373c01c5a0c583ed04938886d4ec4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9rique=20Mittelstaedt?= Date: Thu, 31 Aug 2023 10:53:24 +0000 Subject: [PATCH 2/6] add vscode and devcontainer config --- .devcontainer/Dockerfile | 45 ++++++++++++++++++++++ .devcontainer/devcontainer.json | 64 ++++++++++++++++++++++++++++++++ .devcontainer/docker-compose.yml | 22 +++++++++++ .vscode/extensions.json | 46 +++++++++++++++++++++++ .vscode/settings.json | 32 ++++++++++++++++ .vscode/tasks.json | 56 ++++++++++++++++++++++++++++ 6 files changed, 265 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..38b889b5f --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,45 @@ +ARG tag="11" +ARG workdir="/src" +ARG user="vscode" + +FROM openjdk:${tag} + +ARG workdir +ARG user + +ENV DEBIAN_FRONTEND=noninteractive +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +WORKDIR /tmp + +# Install common packages + +RUN apt-get update --allow-releaseinfo-change && \ + apt-get -y install --no-install-recommends zsh curl git sudo vim ca-certificates + +# Install Taskfile + +RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin ${task_version} + +# Set the root password + +RUN echo "root:root" | chpasswd + +# Add the non-root user + +RUN adduser ${user} --shell /bin/zsh +RUN adduser ${user} sudo +RUN echo "${user}:${user}" | chpasswd + +# Install Oh My Zsh for the user + +RUN su ${user} -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" + +# Install maven + +RUN apt-get install maven + +WORKDIR ${workdir} + +USER ${user} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..99c770300 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,64 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.212.0/containers/go +{ + "name": "flink-statefun", + "dockerComposeFile": [ + "./docker-compose.yml" + ], + "service": "flink-statefun", + "shutdownAction": "stopCompose", + "workspaceFolder": "/src", + "customizations": { + "vscode": { + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + // + // Extensions recommended for all projects + // + // Intellicode + "visualstudioexptteam.vscodeintellicode", + // Editorconfig + "editorconfig.editorconfig", + // Markdownlint + "davidanson.vscode-markdownlint", + // Terraform + "hashicorp.terraform", + "hashicorp.hcl", + // Docker + "ms-azuretools.vscode-docker", + // Bazel (primarily for Starlark support) + "stackbuild.bazel-stack-vscode", + // Tilt + "tchoupinax.tilt", + // Gitlens + "eamodio.gitlens", + // Protobuf support + "zxh404.vscode-proto3", + // Thrift support + "cduruk.thrift", + // YAML + "redhat.vscode-yaml", + // + // Java + // + "vscjava.vscode-java-pack", + "scala-lang.scala", + "scalameta.metals", + "fwcd.kotlin" + ], + // Set *default* container specific settings.json values on container create. + "settings": {}, + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Use 'postCreateCommand' to run commands after the container is created. + "onCreateCommand": "task devcontainer:onCreate", + "postCreateCommand": "task devcontainer:postCreate", + "postAttachCommand": "task devcontainer:postAttach", + "postStartCommand": "task devcontainer:postStart", + "initializeCommand": "", + "updateContentCommand": "task devcontainer:updateContent", + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..e852aa84f --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,22 @@ +version: "3.8" +services: + flink-statefun: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + tag: 11 + volumes: + - ..:/src:cached + - /var/run/docker.sock:/var/run/docker.sock + user: vscode + # Ensure the container runs forever + command: "-f /dev/null" + entrypoint: /usr/bin/tail + tty: true + expose: + - "9090" + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..27467328f --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,46 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + // List of extensions which should be recommended for users of this workspace. + "recommendations": [ + // + // VS Code Remote Containers + // + "ms-vscode-remote.remote-containers", + // + // Extensions recommended for all projects + // + // Intellicode + "visualstudioexptteam.vscodeintellicode", + // Editorconfig + "editorconfig.editorconfig", + // Markdownlint + "davidanson.vscode-markdownlint", + // Terraform + "hashicorp.terraform", + "hashicorp.hcl", + // Docker + "ms-azuretools.vscode-docker", + // Bazel (primarily for Starlark support) + "stackbuild.bazel-stack-vscode", + // Tilt + "tchoupinax.tilt", + // Gitlens + "eamodio.gitlens", + // Protobuf support + "zxh404.vscode-proto3", + // Thrift support + "cduruk.thrift", + // YAML + "redhat.vscode-yaml", + // + // Java + // + "vscjava.vscode-java-pack", + "scala-lang.scala", + "scalameta.metals", + "fwcd.kotlin" + ], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..d07164d5a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,32 @@ +{ + "editor.codeActionsOnSave": { + "source.fixAll": true, + "source.organizeImports": false + }, + "editor.formatOnSave": true, + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/__pycache__": true, + "**/*.pyc": true + }, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "files.trimTrailingWhitespace": true, + "editor.trimAutoWhitespace": true, + "java.compile.nullAnalysis.mode": "automatic", + "java.configuration.updateBuildConfiguration": "interactive", + "java.project.sourcePaths": [ + "src/main/java" + ], + "terminal.integrated.defaultProfile.linux": "zsh", + "terminal.integrated.defaultProfile.osx": "zsh", + "bsv.bzl.remoteCache.enabled": false, + "redhat.telemetry.enabled": false, + "[yaml]": { + "editor.defaultFormatter": "redhat.vscode-yaml" + } +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..5582fd177 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,56 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "install", + "type": "shell", + "command": "task install" + }, + { + "label": "setup", + "type": "shell", + "command": "task setup" + }, + { + "label": "build", + "type": "shell", + "command": "task build", + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "start", + "type": "shell", + "command": "task start" + }, + { + "label": "test", + "type": "shell", + "command": "task test", + "problemMatcher": [], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "lint", + "type": "shell", + "command": "task lint" + }, + { + "label": "fix", + "type": "shell", + "command": "task fix" + }, + { + "label": "fmt", + "type": "shell", + "command": "task fmt" + } + ] +} From a09af3859f97e9aba765cef3fcb84f1b1cc94146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9rique=20Mittelstaedt?= Date: Thu, 31 Aug 2023 10:53:29 +0000 Subject: [PATCH 3/6] add taskfile --- Taskfile.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ pom.xml | 1 + 2 files changed, 47 insertions(+) create mode 100644 Taskfile.yml diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 000000000..4b17b4aff --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,46 @@ +version: "3" + +vars: {} + +tasks: + setup: + cmds: + - task: install + + install: + cmds: [] + + build: + cmds: [] + + start: + cmds: [] + + test: + cmds: [] + + lint: + cmds: [] + + fmt: + cmds: [] + + fix: + cmds: + - task: fmt + + devcontainer:onCreate: + cmds: [] + + devcontainer:postAttach: + cmds: [] + + devcontainer:postCreate: + cmds: + - task: install + + devcontainer:postStart: + cmds: [] + + devcontainer:updateContent: + cmds: [] diff --git a/pom.xml b/pom.xml index 6bf1c7933..20099b8af 100644 --- a/pom.xml +++ b/pom.xml @@ -286,6 +286,7 @@ under the License. **/.*/** **/*.prefs **/*.log + Taskfile.yml **/README.md **/README.zh.md From 450842babfe8a572fcad8439610cf6ea30392ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9rique=20Mittelstaedt?= Date: Thu, 31 Aug 2023 10:53:41 +0000 Subject: [PATCH 4/6] ignore bin --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c3c9fedf4..8bab5c774 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ statefun-e2e-tests/*/statefun-app-checkpoints-* statefun-sdk-js/package-lock.json statefun-sdk-js/node_modules/* statefun-sdk-js/dist/* + +/*/bin +/*/*/bin From 6478acdad1c480c55184151c9288d480a2e22720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9rique=20Mittelstaedt?= Date: Thu, 31 Aug 2023 10:22:53 +0000 Subject: [PATCH 5/6] support flink 1.17.1 --- pom.xml | 98 +++++++++++++++++-- statefun-flink/statefun-flink-core/pom.xml | 3 +- .../flink/core/StatefulFunctionsConfig.java | 2 +- .../flink/core/StatefulFunctionsJob.java | 2 +- .../flink/core/functions/ReductionsTest.java | 5 +- .../statefun-flink-datastream/pom.xml | 16 +-- .../statefun-flink-distribution/pom.xml | 4 +- .../statefun-flink-io-bundle/pom.xml | 9 +- statefun-flink/statefun-flink-io/pom.xml | 2 +- .../statefun-protobuf-shaded/pom.xml | 3 +- .../statefun-protocol-shaded/pom.xml | 9 +- tools/docker/Dockerfile | 4 +- 12 files changed, 122 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 20099b8af..919f73f89 100644 --- a/pom.xml +++ b/pom.xml @@ -76,15 +76,28 @@ under the License. 1.8 1.20.0 1.0-rc6 - 3.7.1 - 2.3.2 - 3.11.1 - 1.15.2 + 3.23.2 + 2.6.2 + 3.11.4 + 1.17.1 2.12 2.12.7 1.8.0 - 2.12.4-15.0 + 2.14.2-17.0 1.7.32 + 4.1.0-1.17 + 4.1.0-1.17 + 3.14.6 + 4.1.82.Final-16.1 + 4.12 + 1.3 + 2.24.0 + 2.13.2.2 + 4.1.82.Final-16.1 + 16.1 + 1.15 + 1.2 + 1.7.36 **/*Test.* @@ -101,15 +114,20 @@ under the License. junit junit - 4.12 + ${junit.version} test org.hamcrest hamcrest-all - 1.3 + ${hamcrest-all.version} test + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + org.apache.flink + flink-shaded-netty + ${flink-shaded-netty.version} + + + + org.apache.flink + flink-core + ${flink.version} + + + + org.apache.flink + flink-table-common + ${flink.version} + + + + org.apache.flink + flink-connector-base + ${flink.version} + + + + org.apache.flink + flink-shaded-force-shading + ${flink-shaded-force-shading.version} + + + + org.apache.flink + flink-shaded-jackson + ${flink-shaded-jackson.version} + + + + commons-codec + commons-codec + ${commons-codec.version} + + + + commons-logging + commons-logging + ${commons-logging.version} + + + + org.slf4j + slf4j-api + ${slf4j-api.version} @@ -234,10 +311,12 @@ under the License. descriptor ${basedir}/target/test-classes + main java ${basedir}/target/generated-test-sources/protoc-jar + test @@ -413,4 +492,3 @@ under the License. - diff --git a/statefun-flink/statefun-flink-core/pom.xml b/statefun-flink/statefun-flink-core/pom.xml index 55e8c0061..bfe4d832b 100644 --- a/statefun-flink/statefun-flink-core/pom.xml +++ b/statefun-flink/statefun-flink-core/pom.xml @@ -30,9 +30,7 @@ under the License. statefun-flink-core - 3.14.6 target/additional-sources - 4.1.70.Final-15.0 @@ -90,6 +88,7 @@ under the License. com.kohlschutter.junixsocket junixsocket-core ${unixsocket.version} + pom diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java index 36a14198b..a59d45a3c 100644 --- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java +++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java @@ -265,7 +265,7 @@ public void setEmbedded(boolean embedded) { */ public StatefulFunctionsUniverseProvider getProvider(ClassLoader cl) { try { - return InstantiationUtil.deserializeObject(universeInitializerClassBytes, cl, false); + return InstantiationUtil.deserializeObject(universeInitializerClassBytes, cl); } catch (IOException | ClassNotFoundException e) { throw new IllegalStateException("Unable to initialize.", e); } diff --git a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java index c86ef5814..6803ce65c 100644 --- a/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java +++ b/statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java @@ -23,12 +23,12 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; import org.apache.flink.statefun.flink.core.feedback.FeedbackKey; import org.apache.flink.statefun.flink.core.message.Message; import org.apache.flink.statefun.flink.core.translation.FlinkUniverse; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.FlinkUserCodeClassLoader; +import org.apache.flink.util.FlinkUserCodeClassLoaders; public class StatefulFunctionsJob { diff --git a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java index ad65827e2..64f33642e 100644 --- a/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java +++ b/statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java @@ -352,10 +352,11 @@ public boolean deregisterKeySelectionListener(KeySelectionListener liste @Nonnull @Override - public IS createInternalState( + public IS createOrUpdateInternalState( @Nonnull TypeSerializer namespaceSerializer, @Nonnull StateDescriptor stateDesc, - @Nonnull StateSnapshotTransformFactory snapshotTransformFactory) { + @Nonnull StateSnapshotTransformFactory snapshotTransformFactory) + throws Exception { throw new UnsupportedOperationException(); } diff --git a/statefun-flink/statefun-flink-datastream/pom.xml b/statefun-flink/statefun-flink-datastream/pom.xml index ccb8aac8c..5afb088ce 100644 --- a/statefun-flink/statefun-flink-datastream/pom.xml +++ b/statefun-flink/statefun-flink-datastream/pom.xml @@ -51,13 +51,13 @@ under the License. statefun-flink-io ${project.version} - - org.apache.flink @@ -87,7 +87,7 @@ under the License. org.hamcrest hamcrest-all - + @@ -152,4 +152,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-flink/statefun-flink-distribution/pom.xml b/statefun-flink/statefun-flink-distribution/pom.xml index 5ea7b2eb1..6d0b22b40 100644 --- a/statefun-flink/statefun-flink-distribution/pom.xml +++ b/statefun-flink/statefun-flink-distribution/pom.xml @@ -74,7 +74,7 @@ under the License. statefun-flink-launcher ${project.version} - + org.apache.flink @@ -199,4 +199,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-flink/statefun-flink-io-bundle/pom.xml b/statefun-flink/statefun-flink-io-bundle/pom.xml index db79e96ad..490e08e03 100644 --- a/statefun-flink/statefun-flink-io-bundle/pom.xml +++ b/statefun-flink/statefun-flink-io-bundle/pom.xml @@ -90,7 +90,12 @@ under the License. org.apache.flink flink-connector-kinesis - ${flink.version} + ${flink-connector-kinesis.version} + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${flink-connector-aws-kinesis-streams.version} @@ -110,4 +115,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-flink/statefun-flink-io/pom.xml b/statefun-flink/statefun-flink-io/pom.xml index b26b912ff..280f83a51 100644 --- a/statefun-flink/statefun-flink-io/pom.xml +++ b/statefun-flink/statefun-flink-io/pom.xml @@ -124,4 +124,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-shaded/statefun-protobuf-shaded/pom.xml b/statefun-shaded/statefun-protobuf-shaded/pom.xml index d97dae2a7..ec67f7e15 100644 --- a/statefun-shaded/statefun-protobuf-shaded/pom.xml +++ b/statefun-shaded/statefun-protobuf-shaded/pom.xml @@ -32,7 +32,6 @@ under the License. ${generated-sources.basedir}/shaded-protobuf-java/ - 3.7.1 @@ -107,4 +106,4 @@ under the License. - \ No newline at end of file + diff --git a/statefun-shaded/statefun-protocol-shaded/pom.xml b/statefun-shaded/statefun-protocol-shaded/pom.xml index 9ad5c9bc0..d31938eb4 100644 --- a/statefun-shaded/statefun-protocol-shaded/pom.xml +++ b/statefun-shaded/statefun-protocol-shaded/pom.xml @@ -33,7 +33,7 @@ under the License. ${generated-sources.basedir}/protocol-messages/ target/proto-sources - 3.11.1 + 3.11.4 @@ -47,6 +47,11 @@ under the License. statefun-protobuf-shaded ${project.version} + + com.google.protobuf + protobuf-java + ${protobuf.version} + @@ -130,4 +135,4 @@ under the License. - \ No newline at end of file + diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 662d034c0..62e579ad5 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM apache/flink:1.15.2-scala_2.12-java8 +FROM flink:1.17.1-scala_2.12-java11 ENV ROLE worker ENV MASTER_HOST localhost @@ -45,7 +45,7 @@ RUN mkdir -p $FLINK_HOME/plugins/azure-fs-hadoop && \ # add tcnative RUN mv $FLINK_HOME/opt/flink-shaded-netty-tcnative-dynamic-*.jar $FLINK_HOME/lib/ -# entry point +# entry point ADD docker-entry-point.sh /docker-entry-point.sh ENTRYPOINT ["/docker-entry-point.sh"] From 2d699917515aea42bf413ea4673f2969d6a34cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9rique=20Mittelstaedt?= Date: Thu, 31 Aug 2023 11:22:49 +0000 Subject: [PATCH 6/6] remove accidentally configured files --- .devcontainer/Dockerfile | 45 ---------------------- .devcontainer/devcontainer.json | 64 -------------------------------- .devcontainer/docker-compose.yml | 22 ----------- .gitignore | 3 -- .vscode/extensions.json | 46 ----------------------- .vscode/settings.json | 32 ---------------- .vscode/tasks.json | 56 ---------------------------- Taskfile.yml | 46 ----------------------- pom.xml | 1 - 9 files changed, 315 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/devcontainer.json delete mode 100644 .devcontainer/docker-compose.yml delete mode 100644 .vscode/extensions.json delete mode 100644 .vscode/settings.json delete mode 100644 .vscode/tasks.json delete mode 100644 Taskfile.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 38b889b5f..000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -ARG tag="11" -ARG workdir="/src" -ARG user="vscode" - -FROM openjdk:${tag} - -ARG workdir -ARG user - -ENV DEBIAN_FRONTEND=noninteractive -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 - -WORKDIR /tmp - -# Install common packages - -RUN apt-get update --allow-releaseinfo-change && \ - apt-get -y install --no-install-recommends zsh curl git sudo vim ca-certificates - -# Install Taskfile - -RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin ${task_version} - -# Set the root password - -RUN echo "root:root" | chpasswd - -# Add the non-root user - -RUN adduser ${user} --shell /bin/zsh -RUN adduser ${user} sudo -RUN echo "${user}:${user}" | chpasswd - -# Install Oh My Zsh for the user - -RUN su ${user} -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" - -# Install maven - -RUN apt-get install maven - -WORKDIR ${workdir} - -USER ${user} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 99c770300..000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,64 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.212.0/containers/go -{ - "name": "flink-statefun", - "dockerComposeFile": [ - "./docker-compose.yml" - ], - "service": "flink-statefun", - "shutdownAction": "stopCompose", - "workspaceFolder": "/src", - "customizations": { - "vscode": { - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - // - // Extensions recommended for all projects - // - // Intellicode - "visualstudioexptteam.vscodeintellicode", - // Editorconfig - "editorconfig.editorconfig", - // Markdownlint - "davidanson.vscode-markdownlint", - // Terraform - "hashicorp.terraform", - "hashicorp.hcl", - // Docker - "ms-azuretools.vscode-docker", - // Bazel (primarily for Starlark support) - "stackbuild.bazel-stack-vscode", - // Tilt - "tchoupinax.tilt", - // Gitlens - "eamodio.gitlens", - // Protobuf support - "zxh404.vscode-proto3", - // Thrift support - "cduruk.thrift", - // YAML - "redhat.vscode-yaml", - // - // Java - // - "vscjava.vscode-java-pack", - "scala-lang.scala", - "scalameta.metals", - "fwcd.kotlin" - ], - // Set *default* container specific settings.json values on container create. - "settings": {}, - } - }, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - // Use 'postCreateCommand' to run commands after the container is created. - "onCreateCommand": "task devcontainer:onCreate", - "postCreateCommand": "task devcontainer:postCreate", - "postAttachCommand": "task devcontainer:postAttach", - "postStartCommand": "task devcontainer:postStart", - "initializeCommand": "", - "updateContentCommand": "task devcontainer:updateContent", - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" -} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml deleted file mode 100644 index e852aa84f..000000000 --- a/.devcontainer/docker-compose.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: "3.8" -services: - flink-statefun: - build: - context: .. - dockerfile: .devcontainer/Dockerfile - args: - tag: 11 - volumes: - - ..:/src:cached - - /var/run/docker.sock:/var/run/docker.sock - user: vscode - # Ensure the container runs forever - command: "-f /dev/null" - entrypoint: /usr/bin/tail - tty: true - expose: - - "9090" - cap_add: - - SYS_PTRACE - security_opt: - - seccomp:unconfined diff --git a/.gitignore b/.gitignore index 8bab5c774..c3c9fedf4 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,3 @@ statefun-e2e-tests/*/statefun-app-checkpoints-* statefun-sdk-js/package-lock.json statefun-sdk-js/node_modules/* statefun-sdk-js/dist/* - -/*/bin -/*/*/bin diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index 27467328f..000000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. - // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp - // List of extensions which should be recommended for users of this workspace. - "recommendations": [ - // - // VS Code Remote Containers - // - "ms-vscode-remote.remote-containers", - // - // Extensions recommended for all projects - // - // Intellicode - "visualstudioexptteam.vscodeintellicode", - // Editorconfig - "editorconfig.editorconfig", - // Markdownlint - "davidanson.vscode-markdownlint", - // Terraform - "hashicorp.terraform", - "hashicorp.hcl", - // Docker - "ms-azuretools.vscode-docker", - // Bazel (primarily for Starlark support) - "stackbuild.bazel-stack-vscode", - // Tilt - "tchoupinax.tilt", - // Gitlens - "eamodio.gitlens", - // Protobuf support - "zxh404.vscode-proto3", - // Thrift support - "cduruk.thrift", - // YAML - "redhat.vscode-yaml", - // - // Java - // - "vscjava.vscode-java-pack", - "scala-lang.scala", - "scalameta.metals", - "fwcd.kotlin" - ], - // List of extensions recommended by VS Code that should not be recommended for users of this workspace. - "unwantedRecommendations": [] -} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index d07164d5a..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "editor.codeActionsOnSave": { - "source.fixAll": true, - "source.organizeImports": false - }, - "editor.formatOnSave": true, - "files.exclude": { - "**/.git": true, - "**/.svn": true, - "**/.hg": true, - "**/CVS": true, - "**/.DS_Store": true, - "**/__pycache__": true, - "**/*.pyc": true - }, - "files.insertFinalNewline": true, - "files.trimFinalNewlines": true, - "files.trimTrailingWhitespace": true, - "editor.trimAutoWhitespace": true, - "java.compile.nullAnalysis.mode": "automatic", - "java.configuration.updateBuildConfiguration": "interactive", - "java.project.sourcePaths": [ - "src/main/java" - ], - "terminal.integrated.defaultProfile.linux": "zsh", - "terminal.integrated.defaultProfile.osx": "zsh", - "bsv.bzl.remoteCache.enabled": false, - "redhat.telemetry.enabled": false, - "[yaml]": { - "editor.defaultFormatter": "redhat.vscode-yaml" - } -} diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index 5582fd177..000000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "2.0.0", - "tasks": [ - { - "label": "install", - "type": "shell", - "command": "task install" - }, - { - "label": "setup", - "type": "shell", - "command": "task setup" - }, - { - "label": "build", - "type": "shell", - "command": "task build", - "group": { - "kind": "build", - "isDefault": true - } - }, - { - "label": "start", - "type": "shell", - "command": "task start" - }, - { - "label": "test", - "type": "shell", - "command": "task test", - "problemMatcher": [], - "group": { - "kind": "test", - "isDefault": true - } - }, - { - "label": "lint", - "type": "shell", - "command": "task lint" - }, - { - "label": "fix", - "type": "shell", - "command": "task fix" - }, - { - "label": "fmt", - "type": "shell", - "command": "task fmt" - } - ] -} diff --git a/Taskfile.yml b/Taskfile.yml deleted file mode 100644 index 4b17b4aff..000000000 --- a/Taskfile.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: "3" - -vars: {} - -tasks: - setup: - cmds: - - task: install - - install: - cmds: [] - - build: - cmds: [] - - start: - cmds: [] - - test: - cmds: [] - - lint: - cmds: [] - - fmt: - cmds: [] - - fix: - cmds: - - task: fmt - - devcontainer:onCreate: - cmds: [] - - devcontainer:postAttach: - cmds: [] - - devcontainer:postCreate: - cmds: - - task: install - - devcontainer:postStart: - cmds: [] - - devcontainer:updateContent: - cmds: [] diff --git a/pom.xml b/pom.xml index 919f73f89..99806bb49 100644 --- a/pom.xml +++ b/pom.xml @@ -365,7 +365,6 @@ under the License. **/.*/** **/*.prefs **/*.log - Taskfile.yml **/README.md **/README.zh.md