From 27643f3a739aec1a4795794628e6043068cc0c07 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 4 Oct 2020 15:04:19 +0200 Subject: [PATCH 1/8] add debug/run config --- .run/Application - 1.run.xml | 22 ++++++++++++++++++++++ .run/Application - 2.run.xml | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 .run/Application - 1.run.xml create mode 100644 .run/Application - 2.run.xml diff --git a/.run/Application - 1.run.xml b/.run/Application - 1.run.xml new file mode 100644 index 000000000..24482ec71 --- /dev/null +++ b/.run/Application - 1.run.xml @@ -0,0 +1,22 @@ + + + + + \ No newline at end of file diff --git a/.run/Application - 2.run.xml b/.run/Application - 2.run.xml new file mode 100644 index 000000000..d2d0c8b5a --- /dev/null +++ b/.run/Application - 2.run.xml @@ -0,0 +1,23 @@ + + + + + \ No newline at end of file From 2bb382394a59aad098f8450d7cc36183ab3a0406 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001> Date: Sun, 4 Oct 2020 22:05:10 +0200 Subject: [PATCH 2/8] fix default workspace property --- core/src/main/resources/flow.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/flow.properties b/core/src/main/resources/flow.properties index c63ded185..fa68dda05 100644 --- a/core/src/main/resources/flow.properties +++ b/core/src/main/resources/flow.properties @@ -1,4 +1,4 @@ -app.workspace=${FLOWCI_WORKSPACE:${HOME}/.flow.ci} +app.workspace=${FLOWCI_WORKSPACE:${user.home}/.flow.ci} app.flow-dir=${app.workspace}/flows app.site-dir=${app.workspace}/sites-static-resources app.secret=${FLOWCI_SECRET:qazWSXedcRFV12#$} From fe6567f7aeb70f850332c11988e2b055c3ad77a2 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001> Date: Wed, 7 Oct 2020 15:24:33 +0200 Subject: [PATCH 3/8] fix session is missing if token invalid --- .../flowci/core/agent/manager/AgentEventManager.java | 11 ++++++++++- .../flowci/core/agent/service/AgentServiceImpl.java | 9 ++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java index a61af9f17..b28975ca8 100644 --- a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java +++ b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java @@ -120,6 +120,15 @@ public void afterConnectionClosed(WebSocketSession session, CloseStatus status) agentSessionStore.remove(token, session); } + private void writeMessage(WebSocketSession session, ResponseMessage msg) { + try { + byte[] bytes = objectMapper.writeValueAsBytes(msg); + session.sendMessage(new BinaryMessage(bytes)); + } catch (IOException e) { + log.warn(e); + } + } + private void onConnected(WebSocketSession session, String token, byte[] body) { try { AgentInit init = objectMapper.readValue(body, AgentInit.class); @@ -134,7 +143,7 @@ private void onConnected(WebSocketSession session, String token, byte[] body) { log.debug("Agent {} is connected with status {}", token, init.getStatus()); } catch (Exception e) { log.warn(e); - writeMessage(token, new ResponseMessage(StatusCode.FATAL, e.getMessage(), null)); + writeMessage(session, new ResponseMessage(StatusCode.FATAL, e.getMessage(), null)); } } diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 3f7aae20b..553e97635 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -354,10 +354,13 @@ public void onConnected(OnConnectedEvent event) { @EventListener public void onDisconnected(OnDisconnectedEvent event) { - Agent target = getByToken(event.getToken()); + try { + Agent target = getByToken(event.getToken()); + update(target, OFFLINE); + syncLockNode(target, false); + } catch (NotFoundException ignore) { - update(target, OFFLINE); - syncLockNode(target, false); + } } @EventListener From 8ed5abd5d0051cea09150f614c56f7a58e588a4b Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 8 Oct 2020 22:10:21 +0200 Subject: [PATCH 4/8] rm script instead of bash and pwsh --- tree/src/main/java/com/flowci/tree/StepNode.java | 11 ++++++++--- tree/src/main/java/com/flowci/tree/yml/StepYml.java | 10 +++++++--- .../test/java/com/flowci/tree/test/YmlParserTest.java | 3 ++- tree/src/test/resources/flow-with-exports.yml | 2 +- tree/src/test/resources/flow-with-invalid-name.yml | 2 +- tree/src/test/resources/flow.yml | 3 ++- tree/src/test/resources/parent-step-with-plugin.yml | 6 +++--- tree/src/test/resources/runtime-with-command.yml | 2 +- tree/src/test/resources/step-in-step.yml | 8 ++++---- tree/src/test/resources/step-with-dockers.yml | 2 +- 10 files changed, 30 insertions(+), 19 deletions(-) diff --git a/tree/src/main/java/com/flowci/tree/StepNode.java b/tree/src/main/java/com/flowci/tree/StepNode.java index 3709a5547..e14084ebf 100644 --- a/tree/src/main/java/com/flowci/tree/StepNode.java +++ b/tree/src/main/java/com/flowci/tree/StepNode.java @@ -22,9 +22,14 @@ public class StepNode extends Node { private String condition; /** - * Node execute script, can be null + * bash script */ - private String script; + private String bash; + + /** + * powershell script + */ + private String pwsh; /** * Plugin name @@ -56,7 +61,7 @@ public boolean hasCondition() { @JsonIgnore public boolean hasScript() { - return !Strings.isNullOrEmpty(script); + return !Strings.isNullOrEmpty(bash) || !Strings.isNullOrEmpty(pwsh); } @JsonIgnore diff --git a/tree/src/main/java/com/flowci/tree/yml/StepYml.java b/tree/src/main/java/com/flowci/tree/yml/StepYml.java index 58334abc2..960fd40bd 100644 --- a/tree/src/main/java/com/flowci/tree/yml/StepYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/StepYml.java @@ -45,7 +45,9 @@ public class StepYml extends YmlBase { */ private String condition; - private String script; + private String bash; // bash script + + private String pwsh; // powershell script private String plugin; @@ -56,7 +58,8 @@ public class StepYml extends YmlBase { StepYml(StepNode node) { setName(node.getName()); setEnvs(node.getEnvironments()); - setScript(node.getScript()); + setBash(node.getBash()); + setPwsh(node.getPwsh()); setPlugin(node.getPlugin()); setAllow_failure(node.isAllowFailure()); } @@ -64,7 +67,8 @@ public class StepYml extends YmlBase { public StepNode toNode(Node parent, int index) { StepNode node = new StepNode(buildName(index), parent); node.setCondition(condition); - node.setScript(script); + node.setBash(bash); + node.setPwsh(pwsh); node.setPlugin(plugin); node.setExports(Sets.newHashSet(exports)); node.setAllowFailure(allow_failure); diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index a125f23cf..da228b58b 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -86,7 +86,8 @@ public void should_get_node_from_yml() { StepNode step2 = steps.get(1); Assert.assertEquals("step2", step2.getName()); - Assert.assertEquals("echo 2", step2.getScript()); + Assert.assertEquals("echo 2", step2.getBash()); + Assert.assertEquals("echo powershell", step2.getPwsh()); DockerOption dockerOption = step2.getDockers().get(0); Assert.assertNotNull(dockerOption); diff --git a/tree/src/test/resources/flow-with-exports.yml b/tree/src/test/resources/flow-with-exports.yml index 7666ed0db..b0de5499d 100644 --- a/tree/src/test/resources/flow-with-exports.yml +++ b/tree/src/test/resources/flow-with-exports.yml @@ -27,4 +27,4 @@ steps: - name: step2 allow_failure: false - script: "echo 2" + bash: "echo 2" diff --git a/tree/src/test/resources/flow-with-invalid-name.yml b/tree/src/test/resources/flow-with-invalid-name.yml index 68e5eebc7..283020113 100644 --- a/tree/src/test/resources/flow-with-invalid-name.yml +++ b/tree/src/test/resources/flow-with-invalid-name.yml @@ -27,4 +27,4 @@ steps: - name: step;'2 allow_failure: false - script: "echo 2" + bash: "echo 2" diff --git a/tree/src/test/resources/flow.yml b/tree/src/test/resources/flow.yml index 8ed709efb..1bf2ae606 100644 --- a/tree/src/test/resources/flow.yml +++ b/tree/src/test/resources/flow.yml @@ -42,4 +42,5 @@ steps: - "2700:2700" entrypoint: ["/bin/sh"] network: host - script: "echo 2" + bash: "echo 2" + pwsh: "echo powershell" diff --git a/tree/src/test/resources/parent-step-with-plugin.yml b/tree/src/test/resources/parent-step-with-plugin.yml index dda617ddd..d77765b99 100644 --- a/tree/src/test/resources/parent-step-with-plugin.yml +++ b/tree/src/test/resources/parent-step-with-plugin.yml @@ -16,14 +16,14 @@ steps: - image: "ubuntu:18.04" is_runtime: true - image: "mysql" - script: "echo 2" + bash: "echo 2" plugin: "invalid" steps: - name: step-2-1 - script: | + bash: | echo "step-1 in step2" - name: step-2-2 - script: | + bash: | echo "step-2 in step2" diff --git a/tree/src/test/resources/runtime-with-command.yml b/tree/src/test/resources/runtime-with-command.yml index db4c5f05c..4e2a57430 100644 --- a/tree/src/test/resources/runtime-with-command.yml +++ b/tree/src/test/resources/runtime-with-command.yml @@ -17,4 +17,4 @@ steps: MY_PW: "12345" network_mode: none - script: "echo 2" \ No newline at end of file + bash: "echo 2" \ No newline at end of file diff --git a/tree/src/test/resources/step-in-step.yml b/tree/src/test/resources/step-in-step.yml index 5f6ee65b2..06c9dea86 100644 --- a/tree/src/test/resources/step-in-step.yml +++ b/tree/src/test/resources/step-in-step.yml @@ -16,16 +16,16 @@ steps: - image: "ubuntu:18.04" is_runtime: true - image: "mysql" - script: "echo 2" + bash: "echo 2" steps: - name: step-2-1 - script: | + bash: | echo "step-1 in step2" - name: step-2-2 - script: | + bash: | echo "step-2 in step2" - name: create test - script: "echo end" + bash: "echo end" diff --git a/tree/src/test/resources/step-with-dockers.yml b/tree/src/test/resources/step-with-dockers.yml index b472c2815..df5771f97 100644 --- a/tree/src/test/resources/step-with-dockers.yml +++ b/tree/src/test/resources/step-with-dockers.yml @@ -30,4 +30,4 @@ steps: environment: MY_PW: "12345" network: none - script: "echo 2" \ No newline at end of file + bash: "echo 2" \ No newline at end of file From 3cfe9defefeddc32e5936a7609c9db893e25c815 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 8 Oct 2020 22:35:23 +0200 Subject: [PATCH 5/8] add pwsh in shell in --- .../com/flowci/core/agent/domain/ShellIn.java | 10 ++++---- .../core/job/manager/CmdManagerImpl.java | 24 +++++++++++++++---- .../flowci/core/test/job/CmdManagerTest.java | 10 ++++---- .../flowci/core/test/job/JobServiceTest.java | 4 ++-- core/src/test/resources/flow-all-failure.yml | 4 ++-- .../test/resources/flow-with-condition.yml | 4 ++-- core/src/test/resources/flow-with-failure.yml | 4 ++-- core/src/test/resources/flow-with-filter.yml | 4 ++-- core/src/test/resources/flow-with-notify.yml | 4 ++-- .../resources/flow-with-plugin-not-found.yml | 4 ++-- core/src/test/resources/flow-with-plugin.yml | 4 ++-- .../test/resources/flow-with-root-docker.yml | 4 ++-- core/src/test/resources/flow.yml | 4 ++-- core/src/test/resources/step-in-step.yml | 8 +++---- 14 files changed, 53 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java index bdbf842fe..12fd83698 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java +++ b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java @@ -10,8 +10,6 @@ import lombok.Setter; import lombok.experimental.Accessors; -import java.util.LinkedHashSet; -import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -36,7 +34,9 @@ public final class ShellIn extends CmdIn { @JsonIgnore private String condition; - private List scripts; + private List bash; + + private List pwsh; private int timeout = 1800; @@ -48,11 +48,11 @@ public ShellIn() { super(Type.SHELL); } - public void addScript(String script) { + public void addBash(String script) { if (Strings.isNullOrEmpty(script)) { return; } - scripts.add(script); + bash.add(script); } public void addEnvFilters(Set exports) { diff --git a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java index 19621ac64..a9868df1b 100644 --- a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java @@ -46,6 +46,12 @@ @Component public class CmdManagerImpl implements CmdManager { + private enum ShellType { + Bash, + + PowerShell; + } + @Autowired private SpringEventManager eventManager; @@ -60,7 +66,8 @@ public CmdIn createShellCmd(Job job, Step step, NodeTree tree) { .setCondition(node.getCondition()) .setAllowFailure(node.isAllowFailure()) .setDockers(findDockerOptions(node)) - .setScripts(linkScript(node)) + .setBash(linkScript(node, ShellType.Bash)) + .setPwsh(linkScript(node, ShellType.PowerShell)) .setEnvFilters(linkFilters(node)) .setInputs(linkInputs(node).merge(job.getContext(), false)) .setTimeout(job.getTimeout()); @@ -125,17 +132,24 @@ private Set linkFilters(StepNode current) { return output; } - private List linkScript(StepNode current) { + private List linkScript(StepNode current, ShellType shellType) { List output = new LinkedList<>(); if (current.hasParent()) { Node parent = current.getParent(); if (parent instanceof StepNode) { - output.addAll(linkScript((StepNode) parent)); + output.addAll(linkScript((StepNode) parent, shellType)); } } - output.add(current.getScript()); + if (shellType == ShellType.Bash) { + output.add(current.getBash()); + } + + if (shellType == ShellType.PowerShell) { + output.add(current.getPwsh()); + } + return output; } @@ -161,7 +175,7 @@ private void setPlugin(String name, ShellIn cmd) { cmd.setPlugin(name); cmd.setAllowFailure(plugin.isAllowFailure()); cmd.addEnvFilters(plugin.getExports()); - cmd.addScript(plugin.getScript()); + cmd.addBash(plugin.getScript()); // apply docker from plugin as run time if it's specified ObjectsHelper.ifNotNull(plugin.getDocker(), (docker) -> { diff --git a/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java b/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java index e35183365..df87537b5 100644 --- a/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java +++ b/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java @@ -127,7 +127,7 @@ public void should_create_cmd_in_with_default_plugin_value() throws IOException // then: Vars inputs = cmdIn.getInputs(); - List scripts = cmdIn.getScripts(); + List scripts = cmdIn.getBash(); Assert.assertEquals(2, scripts.size()); Assert.assertEquals("gittest", cmdIn.getPlugin()); @@ -168,8 +168,8 @@ public void should_handle_step_in_step() throws IOException { Assert.assertEquals("overwrite-parent", cmdStep2_1.getInputs().get("STEP_2")); // scripts should be linked - Assert.assertEquals("echo 2", cmdStep2_1.getScripts().get(0)); - Assert.assertEquals("echo \"step-2-1\"\n", cmdStep2_1.getScripts().get(1)); + Assert.assertEquals("echo 2", cmdStep2_1.getBash().get(0)); + Assert.assertEquals("echo \"step-2-1\"\n", cmdStep2_1.getBash().get(1)); // docker should from parent step Assert.assertEquals("ubuntu:18.04", cmdStep2_1.getDockers().get(0).getImage()); @@ -180,8 +180,8 @@ public void should_handle_step_in_step() throws IOException { Assert.assertEquals("parent", cmdStep2_2.getInputs().get("STEP_2")); // scripts should be linked - Assert.assertEquals("echo 2", cmdStep2_2.getScripts().get(0)); - Assert.assertEquals("echo \"step-2-2\"\n", cmdStep2_2.getScripts().get(1)); + Assert.assertEquals("echo 2", cmdStep2_2.getBash().get(0)); + Assert.assertEquals("echo \"step-2-2\"\n", cmdStep2_2.getBash().get(1)); // docker should be applied from step2-2 Assert.assertEquals("redis", cmdStep2_2.getDockers().get(0).getImage()); diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index 41efd3218..95f3779e0 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -295,7 +295,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { Assert.assertTrue(cmd.isAllowFailure()); Assert.assertEquals("echo step version", cmd.getInputs().get("FLOW_VERSION")); Assert.assertEquals("echo step", cmd.getInputs().get("FLOW_WORKSPACE")); - Assert.assertEquals("echo hello\n", cmd.getScripts().get(0)); + Assert.assertEquals("echo hello\n", cmd.getBash().get(0)); // when: make dummy response from agent for step 1 firstStep.setStatus(Step.Status.SUCCESS); @@ -318,7 +318,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { cmd = (ShellIn) cmdForStep2.getValue(); Assert.assertEquals(secondStep.getId(), cmd.getId()); Assert.assertFalse(cmd.isAllowFailure()); - Assert.assertEquals("echo 2", cmd.getScripts().get(0)); + Assert.assertEquals("echo 2", cmd.getBash().get(0)); // when: make dummy response from agent for step 2 secondStep.setStatus(Step.Status.SUCCESS); diff --git a/core/src/test/resources/flow-all-failure.yml b/core/src/test/resources/flow-all-failure.yml index 814587dd0..2ea1ed2a2 100644 --- a/core/src/test/resources/flow-all-failure.yml +++ b/core/src/test/resources/flow-all-failure.yml @@ -7,9 +7,9 @@ steps: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" allow_failure: true - script: | + bash: | echo hello - name: step2 allow_failure: true - script: "echo 2" + bash: "echo 2" diff --git a/core/src/test/resources/flow-with-condition.yml b/core/src/test/resources/flow-with-condition.yml index a6aa722d4..787cc35ea 100644 --- a/core/src/test/resources/flow-with-condition.yml +++ b/core/src/test/resources/flow-with-condition.yml @@ -12,9 +12,9 @@ steps: FLOW_VERSION: "echo step version" condition: | return false - script: | + bash: | echo hello - name: step2 allow_failure: false - script: "echo 2" \ No newline at end of file + bash: "echo 2" \ No newline at end of file diff --git a/core/src/test/resources/flow-with-failure.yml b/core/src/test/resources/flow-with-failure.yml index 7f2c545ff..95845236a 100644 --- a/core/src/test/resources/flow-with-failure.yml +++ b/core/src/test/resources/flow-with-failure.yml @@ -6,8 +6,8 @@ steps: - envs: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" - script: | + bash: | echo shoulde failure -- script: | +- bash: | echo cannot run diff --git a/core/src/test/resources/flow-with-filter.yml b/core/src/test/resources/flow-with-filter.yml index c6b08ea1c..cfb79d26f 100644 --- a/core/src/test/resources/flow-with-filter.yml +++ b/core/src/test/resources/flow-with-filter.yml @@ -11,9 +11,9 @@ steps: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" allow_failure: true - script: | + bash: | echo hello - name: step2 allow_failure: false - script: "echo 2" \ No newline at end of file + bash: "echo 2" \ No newline at end of file diff --git a/core/src/test/resources/flow-with-notify.yml b/core/src/test/resources/flow-with-notify.yml index c9214d408..b6f6daa9e 100644 --- a/core/src/test/resources/flow-with-notify.yml +++ b/core/src/test/resources/flow-with-notify.yml @@ -12,9 +12,9 @@ steps: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" allow_failure: true - script: | + bash: | echo hello - name: step2 allow_failure: false - script: "echo 2" + bash: "echo 2" diff --git a/core/src/test/resources/flow-with-plugin-not-found.yml b/core/src/test/resources/flow-with-plugin-not-found.yml index 5abbd5072..558949f13 100644 --- a/core/src/test/resources/flow-with-plugin-not-found.yml +++ b/core/src/test/resources/flow-with-plugin-not-found.yml @@ -7,8 +7,8 @@ steps: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" allow_failure: true - plugin: "not found.." + bash: "not found.." - name: step2 allow_failure: false - script: "echo 2" + bash: "echo 2" diff --git a/core/src/test/resources/flow-with-plugin.yml b/core/src/test/resources/flow-with-plugin.yml index dfbe646f3..d1e137bbc 100644 --- a/core/src/test/resources/flow-with-plugin.yml +++ b/core/src/test/resources/flow-with-plugin.yml @@ -7,7 +7,7 @@ steps: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" allow_failure: true - script: | + bash: | echo hello - name: plugin-test @@ -15,5 +15,5 @@ steps: GIT_STR_VAL: "test" GIT_DEFAULT_VAL: "60" plugin: gittest - script: | + bash: | echo "plugin-test" \ No newline at end of file diff --git a/core/src/test/resources/flow-with-root-docker.yml b/core/src/test/resources/flow-with-root-docker.yml index ec99440c2..1e9d1f0f0 100644 --- a/core/src/test/resources/flow-with-root-docker.yml +++ b/core/src/test/resources/flow-with-root-docker.yml @@ -9,9 +9,9 @@ steps: - name: step-docker docker: image: "step:0.1" - script: | + bash: | echo hello - name: flow-docker - script: | + bash: | echo "test" \ No newline at end of file diff --git a/core/src/test/resources/flow.yml b/core/src/test/resources/flow.yml index 57c26b25f..370c54382 100644 --- a/core/src/test/resources/flow.yml +++ b/core/src/test/resources/flow.yml @@ -7,9 +7,9 @@ steps: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" allow_failure: true - script: | + bash: | echo hello - name: step2 allow_failure: false - script: "echo 2" + bash: "echo 2" diff --git a/core/src/test/resources/step-in-step.yml b/core/src/test/resources/step-in-step.yml index 93c87f5a8..32f34cf94 100644 --- a/core/src/test/resources/step-in-step.yml +++ b/core/src/test/resources/step-in-step.yml @@ -18,20 +18,20 @@ steps: - image: "ubuntu:18.04" is_runtime: true - image: "mysql" - script: "echo 2" + bash: "echo 2" steps: - name: step-2-1 envs: STEP_2: "overwrite-parent" - script: | + bash: | echo "step-2-1" - name: step-2-2 docker: image: "redis" - script: | + bash: | echo "step-2-2" - name: create test - script: "echo end" + bash: "echo end" From 6a5cee57f8233c23d51e278ff0e833dc189c2460 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 9 Oct 2020 10:49:43 +0200 Subject: [PATCH 6/8] compatible with script section --- tree/src/main/java/com/flowci/tree/yml/StepYml.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tree/src/main/java/com/flowci/tree/yml/StepYml.java b/tree/src/main/java/com/flowci/tree/yml/StepYml.java index 960fd40bd..4c14b1421 100644 --- a/tree/src/main/java/com/flowci/tree/yml/StepYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/StepYml.java @@ -49,6 +49,8 @@ public class StepYml extends YmlBase { private String pwsh; // powershell script + private String script; // keep it to compatible old yaml + private String plugin; private List exports = new LinkedList<>(); @@ -87,6 +89,11 @@ public StepNode toNode(Node parent, int index) { setSteps(node); } + // backward compatible, set script to bash + if (StringHelper.hasValue(script) && !StringHelper.hasValue(bash)) { + node.setBash(script); + } + return node; } From 119063d56208c7d65a47bb020d69df8151a8a49d Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 9 Oct 2020 11:23:33 +0200 Subject: [PATCH 7/8] support bash and pwsh in plugin --- .../com/flowci/core/agent/domain/ShellIn.java | 17 +++++++++++++++-- .../core/job/manager/CmdManagerImpl.java | 19 +++++++------------ .../job/service/LocalTaskServiceImpl.java | 2 +- .../com/flowci/core/plugin/domain/Plugin.java | 7 +++++-- .../core/plugin/domain/PluginParser.java | 16 +++++++++++++++- .../flowci/core/test/job/CmdManagerTest.java | 2 +- .../core/test/job/LocalTaskManagerTest.java | 2 +- 7 files changed, 45 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java index 12fd83698..9cbf5eb89 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java +++ b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java @@ -18,6 +18,12 @@ @Accessors(chain = true) public final class ShellIn extends CmdIn { + public enum ShellType { + Bash, + + PowerShell; + } + // from ExecutedCmd id private String id; @@ -48,11 +54,18 @@ public ShellIn() { super(Type.SHELL); } - public void addBash(String script) { + public void addScript(String script, ShellType type) { if (Strings.isNullOrEmpty(script)) { return; } - bash.add(script); + + if (type == ShellType.Bash) { + bash.add(script); + } + + if (type == ShellType.PowerShell) { + pwsh.add(script); + } } public void addEnvFilters(Set exports) { diff --git a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java index a9868df1b..cfd0bfc75 100644 --- a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java @@ -46,12 +46,6 @@ @Component public class CmdManagerImpl implements CmdManager { - private enum ShellType { - Bash, - - PowerShell; - } - @Autowired private SpringEventManager eventManager; @@ -66,8 +60,8 @@ public CmdIn createShellCmd(Job job, Step step, NodeTree tree) { .setCondition(node.getCondition()) .setAllowFailure(node.isAllowFailure()) .setDockers(findDockerOptions(node)) - .setBash(linkScript(node, ShellType.Bash)) - .setPwsh(linkScript(node, ShellType.PowerShell)) + .setBash(linkScript(node, ShellIn.ShellType.Bash)) + .setPwsh(linkScript(node, ShellIn.ShellType.PowerShell)) .setEnvFilters(linkFilters(node)) .setInputs(linkInputs(node).merge(job.getContext(), false)) .setTimeout(job.getTimeout()); @@ -132,7 +126,7 @@ private Set linkFilters(StepNode current) { return output; } - private List linkScript(StepNode current, ShellType shellType) { + private List linkScript(StepNode current, ShellIn.ShellType shellType) { List output = new LinkedList<>(); if (current.hasParent()) { @@ -142,11 +136,11 @@ private List linkScript(StepNode current, ShellType shellType) { } } - if (shellType == ShellType.Bash) { + if (shellType == ShellIn.ShellType.Bash) { output.add(current.getBash()); } - if (shellType == ShellType.PowerShell) { + if (shellType == ShellIn.ShellType.PowerShell) { output.add(current.getPwsh()); } @@ -175,7 +169,8 @@ private void setPlugin(String name, ShellIn cmd) { cmd.setPlugin(name); cmd.setAllowFailure(plugin.isAllowFailure()); cmd.addEnvFilters(plugin.getExports()); - cmd.addBash(plugin.getScript()); + cmd.addScript(plugin.getBash(), ShellIn.ShellType.Bash); + cmd.addScript(plugin.getPwsh(), ShellIn.ShellType.PowerShell); // apply docker from plugin as run time if it's specified ObjectsHelper.ifNotNull(plugin.getDocker(), (docker) -> { diff --git a/core/src/main/java/com/flowci/core/job/service/LocalTaskServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/LocalTaskServiceImpl.java index 2e9e1fbdd..eb848f477 100644 --- a/core/src/main/java/com/flowci/core/job/service/LocalTaskServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/LocalTaskServiceImpl.java @@ -140,7 +140,7 @@ public ExecutedLocalTask execute(Job job, LocalTask task) { } Plugin plugin = event.getFetched(); - option.addEntryPoint(plugin.getScript()); + option.addEntryPoint(plugin.getBash()); option.addBind(event.getDir().toString(), "/ws/.plugins/" + plugin.getName()); ObjectsHelper.ifNotNull(plugin.getDocker(), (docker) -> option.setImage(docker.getImage())); } diff --git a/core/src/main/java/com/flowci/core/plugin/domain/Plugin.java b/core/src/main/java/com/flowci/core/plugin/domain/Plugin.java index 7967f6b95..d091ad0c0 100644 --- a/core/src/main/java/com/flowci/core/plugin/domain/Plugin.java +++ b/core/src/main/java/com/flowci/core/plugin/domain/Plugin.java @@ -57,7 +57,9 @@ public class Plugin extends PluginRepoInfo { private DockerOption docker; - private String script; + private String bash; + + private String pwsh; private String icon; @@ -72,7 +74,8 @@ public void update(Plugin src) { this.setInputs(src.inputs); this.setStatsTypes(src.statsTypes); this.setAllowFailure(src.allowFailure); - this.setScript(src.script); + this.setBash(src.bash); + this.setPwsh(src.pwsh); this.setDocker(src.docker); } } diff --git a/core/src/main/java/com/flowci/core/plugin/domain/PluginParser.java b/core/src/main/java/com/flowci/core/plugin/domain/PluginParser.java index 46accf37c..d873b0447 100644 --- a/core/src/main/java/com/flowci/core/plugin/domain/PluginParser.java +++ b/core/src/main/java/com/flowci/core/plugin/domain/PluginParser.java @@ -22,10 +22,12 @@ import com.flowci.domain.Version; import com.flowci.tree.yml.DockerYml; import com.flowci.util.ObjectsHelper; +import com.flowci.util.StringHelper; import com.flowci.util.YamlHelper; import lombok.NoArgsConstructor; import lombok.NonNull; import org.yaml.snakeyaml.Yaml; +import sun.jvm.hotspot.oops.ObjectHeap; import java.io.InputStream; import java.util.LinkedList; @@ -62,6 +64,10 @@ private static class PluginWrapper { public Boolean allow_failure; + public String bash; + + public String pwsh; + public String script; public DockerYml docker; @@ -69,7 +75,8 @@ private static class PluginWrapper { public Plugin toPlugin() { Plugin plugin = new Plugin(name, Version.parse(version)); plugin.setIcon(icon); - plugin.setScript(script); + plugin.setBash(bash); + plugin.setPwsh(pwsh); ObjectsHelper.ifNotNull(docker, val -> plugin.setDocker(val.toDockerOption())); ObjectsHelper.ifNotNull(exports, plugin::setExports); @@ -85,6 +92,13 @@ public Plugin toPlugin() { } }); + // backward compatible, set script to bash + if (StringHelper.hasValue(script)) { + if (!StringHelper.hasValue(bash)) { + plugin.setBash(script); + } + } + return plugin; } } diff --git a/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java b/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java index df87537b5..86ba5210e 100644 --- a/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java +++ b/core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java @@ -207,7 +207,7 @@ private Plugin createDummyPlugin() { plugin.setName("gittest"); plugin.setInputs(Lists.newArrayList(intInput, strInput)); plugin.setDocker(option); - plugin.setScript("echo ${GIT_DEFAULT_VAL} ${GIT_STR_VAL}"); + plugin.setBash("echo ${GIT_DEFAULT_VAL} ${GIT_STR_VAL}"); return plugin; } diff --git a/core/src/test/java/com/flowci/core/test/job/LocalTaskManagerTest.java b/core/src/test/java/com/flowci/core/test/job/LocalTaskManagerTest.java index 84c084157..78283a2ca 100644 --- a/core/src/test/java/com/flowci/core/test/job/LocalTaskManagerTest.java +++ b/core/src/test/java/com/flowci/core/test/job/LocalTaskManagerTest.java @@ -37,7 +37,7 @@ public void should_execute_local_task() { Plugin p = new Plugin(); p.setName("test-plugin"); p.setVersion(Version.parse("0.1.0")); - p.setScript("echo aaa \n echo bbb"); + p.setBash("echo aaa \n echo bbb"); p.setDocker(new DockerOption().setImage("sonarqube:latest")); pluginDao.save(p); From 3fc72efed7ba285d93735ac829c22ba73a1097ee Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 9 Oct 2020 11:28:57 +0200 Subject: [PATCH 8/8] fix ut of bash and pwsh in plugin --- .../com/flowci/core/test/plugin/PluginParserTest.java | 8 ++++++++ core/src/test/resources/plugin.yml | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/com/flowci/core/test/plugin/PluginParserTest.java b/core/src/test/java/com/flowci/core/test/plugin/PluginParserTest.java index 980ada074..aacaf80e1 100644 --- a/core/src/test/java/com/flowci/core/test/plugin/PluginParserTest.java +++ b/core/src/test/java/com/flowci/core/test/plugin/PluginParserTest.java @@ -56,5 +56,13 @@ public void should_parse_yml_to_plugin() { Assert.assertEquals(2, exports.size()); Assert.assertTrue(exports.contains("VAR_EXPORT_1")); Assert.assertTrue(exports.contains("VAR_EXPORT_2")); + + String pwsh = plugin.getPwsh(); + Assert.assertNotNull(pwsh); + Assert.assertEquals("$Env.PK_FILE=keyfile", pwsh.trim()); + + String bash = plugin.getBash(); + Assert.assertNotNull(bash); + Assert.assertEquals("chmod 400 ${PK_FILE}", bash.trim()); } } diff --git a/core/src/test/resources/plugin.yml b/core/src/test/resources/plugin.yml index 5b8538382..b1ad8ca29 100644 --- a/core/src/test/resources/plugin.yml +++ b/core/src/test/resources/plugin.yml @@ -33,6 +33,7 @@ stats: allow_failure: true script: | - PK_FILE=${FLOWCI_PLUGIN_WORKSPACE}/FLOWCI_RSA_PRIVATE_KEY chmod 400 ${PK_FILE} - ssh-agent bash -c 'ssh-add ${PK_FILE}; git clone -b ${GIT_BRANCH} ${GIT_URL}' \ No newline at end of file + +pwsh: | + $Env.PK_FILE=keyfile \ No newline at end of file