Skip to content

Commit

Permalink
Merge pull request #371 from FlowCI/feature/1501
Browse files Browse the repository at this point in the history
Feature/1501
  • Loading branch information
gy2006 authored Oct 9, 2020
2 parents e5ac5d3 + 3fc72ef commit 08929a3
Show file tree
Hide file tree
Showing 35 changed files with 194 additions and 71 deletions.
22 changes: 22 additions & 0 deletions .run/Application - 1.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Application - 1" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<module name="core" />
<option name="SPRING_BOOT_MAIN_CLASS" value="com.flowci.core.Application" />
<option name="VM_PARAMETERS" value="-Xms1g -Xmx1g -Xmn600m -XX:-UseAdaptiveSizePolicy -XX:SurvivorRatio=4 -verbose:gc -XX:+PrintGCDateStamps -XX:+PrintGCDetails" />
<option name="ALTERNATIVE_JRE_PATH" />
<envs>
<env name="FLOWCI_LOG_LEVEL" value="DEBUG" />
<env name="FLOWCI_PLUGIN_URL" value="https://raw.githubusercontent.com/FlowCI/plugins/develop/repository.json" />
<env name="FLOWCI_RABBITMQ_URI" value="amqp://guest:[email protected]:5672" />
<env name="FLOWCI_ZK_HOST" value="127.0.0.1" />
<env name="FLOWCI_AUTO_AGENT" value="false" />
<env name="FLOWCI_AGENT_VOLUMES" value="name=pyenv,dest=/ci/python,script=init.sh,image=flowci/pyenv:1.3,init=init-pyenv-volume.sh" />
<env name="FLOWCI_SERVER_PORT" value="8080" />
<env name="FLOWCI_TEMPLATES" value="https://raw.githubusercontent.com/FlowCI/templates/master/templates.json" />
<env name="FLOWCI_AGENT_IMAGE" value="flowci/agent:dev" />
</envs>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>
23 changes: 23 additions & 0 deletions .run/Application - 2.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Application - 2" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<module name="core" />
<option name="SPRING_BOOT_MAIN_CLASS" value="com.flowci.core.Application" />
<option name="VM_PARAMETERS" value="-Xms1g -Xmx1g -Xmn600m -XX:-UseAdaptiveSizePolicy -XX:SurvivorRatio=4 -verbose:gc -XX:+PrintGCDateStamps -XX:+PrintGCDetails" />
<option name="ALTERNATIVE_JRE_PATH" />
<envs>
<env name="FLOWCI_LOG_LEVEL" value="DEBUG" />
<env name="FLOWCI_SERVER_URL" value="http://192.168.0.104:8081" />
<env name="FLOWCI_DEFAULT_ADMIN_EMAIL" value="[email protected]" />
<env name="FLOWCI_DEFAULT_ADMIN_PASSWORD" value="12345" />
<env name="FLOWCI_PLUGIN_URL" value="https://raw.githubusercontent.com/FlowCI/plugins/develop/repository.json" />
<env name="FLOWCI_RABBITMQ_URI" value="amqp://guest:[email protected]:5672" />
<env name="FLOWCI_ZK_HOST" value="192.168.0.104" />
<env name="FLOWCI_AUTO_AGENT" value="false" />
<env name="FLOWCI_AGENT_VOLUMES" value="name=pyenv,dest=/ci/python,script=init.sh" />
<env name="FLOWCI_SERVER_PORT" value="8081" />
</envs>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>
23 changes: 18 additions & 5 deletions core/src/main/java/com/flowci/core/agent/domain/ShellIn.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
import lombok.Setter;
import lombok.experimental.Accessors;

import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

Expand All @@ -20,6 +18,12 @@
@Accessors(chain = true)
public final class ShellIn extends CmdIn {

public enum ShellType {
Bash,

PowerShell;
}

// from ExecutedCmd id
private String id;

Expand All @@ -36,7 +40,9 @@ public final class ShellIn extends CmdIn {
@JsonIgnore
private String condition;

private List<String> scripts;
private List<String> bash;

private List<String> pwsh;

private int timeout = 1800;

Expand All @@ -48,11 +54,18 @@ public ShellIn() {
super(Type.SHELL);
}

public void addScript(String script) {
public void addScript(String script, ShellType type) {
if (Strings.isNullOrEmpty(script)) {
return;
}
scripts.add(script);

if (type == ShellType.Bash) {
bash.add(script);
}

if (type == ShellType.PowerShell) {
pwsh.add(script);
}
}

public void addEnvFilters(Set<String> exports) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,15 @@ public void afterConnectionClosed(WebSocketSession session, CloseStatus status)
agentSessionStore.remove(token, session);
}

private <T> void writeMessage(WebSocketSession session, ResponseMessage<T> msg) {
try {
byte[] bytes = objectMapper.writeValueAsBytes(msg);
session.sendMessage(new BinaryMessage(bytes));
} catch (IOException e) {
log.warn(e);
}
}

private void onConnected(WebSocketSession session, String token, byte[] body) {
try {
AgentInit init = objectMapper.readValue(body, AgentInit.class);
Expand All @@ -134,7 +143,7 @@ private void onConnected(WebSocketSession session, String token, byte[] body) {
log.debug("Agent {} is connected with status {}", token, init.getStatus());
} catch (Exception e) {
log.warn(e);
writeMessage(token, new ResponseMessage<Void>(StatusCode.FATAL, e.getMessage(), null));
writeMessage(session, new ResponseMessage<Void>(StatusCode.FATAL, e.getMessage(), null));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -354,10 +354,13 @@ public void onConnected(OnConnectedEvent event) {

@EventListener
public void onDisconnected(OnDisconnectedEvent event) {
Agent target = getByToken(event.getToken());
try {
Agent target = getByToken(event.getToken());
update(target, OFFLINE);
syncLockNode(target, false);
} catch (NotFoundException ignore) {

update(target, OFFLINE);
syncLockNode(target, false);
}
}

@EventListener
Expand Down
19 changes: 14 additions & 5 deletions core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ public CmdIn createShellCmd(Job job, Step step, NodeTree tree) {
.setCondition(node.getCondition())
.setAllowFailure(node.isAllowFailure())
.setDockers(findDockerOptions(node))
.setScripts(linkScript(node))
.setBash(linkScript(node, ShellIn.ShellType.Bash))
.setPwsh(linkScript(node, ShellIn.ShellType.PowerShell))
.setEnvFilters(linkFilters(node))
.setInputs(linkInputs(node).merge(job.getContext(), false))
.setTimeout(job.getTimeout());
Expand Down Expand Up @@ -125,17 +126,24 @@ private Set<String> linkFilters(StepNode current) {
return output;
}

private List<String> linkScript(StepNode current) {
private List<String> linkScript(StepNode current, ShellIn.ShellType shellType) {
List<String> output = new LinkedList<>();

if (current.hasParent()) {
Node parent = current.getParent();
if (parent instanceof StepNode) {
output.addAll(linkScript((StepNode) parent));
output.addAll(linkScript((StepNode) parent, shellType));
}
}

output.add(current.getScript());
if (shellType == ShellIn.ShellType.Bash) {
output.add(current.getBash());
}

if (shellType == ShellIn.ShellType.PowerShell) {
output.add(current.getPwsh());
}

return output;
}

Expand All @@ -161,7 +169,8 @@ private void setPlugin(String name, ShellIn cmd) {
cmd.setPlugin(name);
cmd.setAllowFailure(plugin.isAllowFailure());
cmd.addEnvFilters(plugin.getExports());
cmd.addScript(plugin.getScript());
cmd.addScript(plugin.getBash(), ShellIn.ShellType.Bash);
cmd.addScript(plugin.getPwsh(), ShellIn.ShellType.PowerShell);

// apply docker from plugin as run time if it's specified
ObjectsHelper.ifNotNull(plugin.getDocker(), (docker) -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ public ExecutedLocalTask execute(Job job, LocalTask task) {
}

Plugin plugin = event.getFetched();
option.addEntryPoint(plugin.getScript());
option.addEntryPoint(plugin.getBash());
option.addBind(event.getDir().toString(), "/ws/.plugins/" + plugin.getName());
ObjectsHelper.ifNotNull(plugin.getDocker(), (docker) -> option.setImage(docker.getImage()));
}
Expand Down
7 changes: 5 additions & 2 deletions core/src/main/java/com/flowci/core/plugin/domain/Plugin.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ public class Plugin extends PluginRepoInfo {

private DockerOption docker;

private String script;
private String bash;

private String pwsh;

private String icon;

Expand All @@ -72,7 +74,8 @@ public void update(Plugin src) {
this.setInputs(src.inputs);
this.setStatsTypes(src.statsTypes);
this.setAllowFailure(src.allowFailure);
this.setScript(src.script);
this.setBash(src.bash);
this.setPwsh(src.pwsh);
this.setDocker(src.docker);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
import com.flowci.domain.Version;
import com.flowci.tree.yml.DockerYml;
import com.flowci.util.ObjectsHelper;
import com.flowci.util.StringHelper;
import com.flowci.util.YamlHelper;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import org.yaml.snakeyaml.Yaml;
import sun.jvm.hotspot.oops.ObjectHeap;

import java.io.InputStream;
import java.util.LinkedList;
Expand Down Expand Up @@ -62,14 +64,19 @@ private static class PluginWrapper {

public Boolean allow_failure;

public String bash;

public String pwsh;

public String script;

public DockerYml docker;

public Plugin toPlugin() {
Plugin plugin = new Plugin(name, Version.parse(version));
plugin.setIcon(icon);
plugin.setScript(script);
plugin.setBash(bash);
plugin.setPwsh(pwsh);

ObjectsHelper.ifNotNull(docker, val -> plugin.setDocker(val.toDockerOption()));
ObjectsHelper.ifNotNull(exports, plugin::setExports);
Expand All @@ -85,6 +92,13 @@ public Plugin toPlugin() {
}
});

// backward compatible, set script to bash
if (StringHelper.hasValue(script)) {
if (!StringHelper.hasValue(bash)) {
plugin.setBash(script);
}
}

return plugin;
}
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/resources/flow.properties
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
app.workspace=${FLOWCI_WORKSPACE:${HOME}/.flow.ci}
app.workspace=${FLOWCI_WORKSPACE:${user.home}/.flow.ci}
app.flow-dir=${app.workspace}/flows
app.site-dir=${app.workspace}/sites-static-resources
app.secret=${FLOWCI_SECRET:qazWSXedcRFV12#$}
Expand Down
12 changes: 6 additions & 6 deletions core/src/test/java/com/flowci/core/test/job/CmdManagerTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public void should_create_cmd_in_with_default_plugin_value() throws IOException

// then:
Vars<String> inputs = cmdIn.getInputs();
List<String> scripts = cmdIn.getScripts();
List<String> scripts = cmdIn.getBash();
Assert.assertEquals(2, scripts.size());

Assert.assertEquals("gittest", cmdIn.getPlugin());
Expand Down Expand Up @@ -168,8 +168,8 @@ public void should_handle_step_in_step() throws IOException {
Assert.assertEquals("overwrite-parent", cmdStep2_1.getInputs().get("STEP_2"));

// scripts should be linked
Assert.assertEquals("echo 2", cmdStep2_1.getScripts().get(0));
Assert.assertEquals("echo \"step-2-1\"\n", cmdStep2_1.getScripts().get(1));
Assert.assertEquals("echo 2", cmdStep2_1.getBash().get(0));
Assert.assertEquals("echo \"step-2-1\"\n", cmdStep2_1.getBash().get(1));

// docker should from parent step
Assert.assertEquals("ubuntu:18.04", cmdStep2_1.getDockers().get(0).getImage());
Expand All @@ -180,8 +180,8 @@ public void should_handle_step_in_step() throws IOException {
Assert.assertEquals("parent", cmdStep2_2.getInputs().get("STEP_2"));

// scripts should be linked
Assert.assertEquals("echo 2", cmdStep2_2.getScripts().get(0));
Assert.assertEquals("echo \"step-2-2\"\n", cmdStep2_2.getScripts().get(1));
Assert.assertEquals("echo 2", cmdStep2_2.getBash().get(0));
Assert.assertEquals("echo \"step-2-2\"\n", cmdStep2_2.getBash().get(1));

// docker should be applied from step2-2
Assert.assertEquals("redis", cmdStep2_2.getDockers().get(0).getImage());
Expand All @@ -207,7 +207,7 @@ private Plugin createDummyPlugin() {
plugin.setName("gittest");
plugin.setInputs(Lists.newArrayList(intInput, strInput));
plugin.setDocker(option);
plugin.setScript("echo ${GIT_DEFAULT_VAL} ${GIT_STR_VAL}");
plugin.setBash("echo ${GIT_DEFAULT_VAL} ${GIT_STR_VAL}");

return plugin;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException {
Assert.assertTrue(cmd.isAllowFailure());
Assert.assertEquals("echo step version", cmd.getInputs().get("FLOW_VERSION"));
Assert.assertEquals("echo step", cmd.getInputs().get("FLOW_WORKSPACE"));
Assert.assertEquals("echo hello\n", cmd.getScripts().get(0));
Assert.assertEquals("echo hello\n", cmd.getBash().get(0));

// when: make dummy response from agent for step 1
firstStep.setStatus(Step.Status.SUCCESS);
Expand All @@ -318,7 +318,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException {
cmd = (ShellIn) cmdForStep2.getValue();
Assert.assertEquals(secondStep.getId(), cmd.getId());
Assert.assertFalse(cmd.isAllowFailure());
Assert.assertEquals("echo 2", cmd.getScripts().get(0));
Assert.assertEquals("echo 2", cmd.getBash().get(0));

// when: make dummy response from agent for step 2
secondStep.setStatus(Step.Status.SUCCESS);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public void should_execute_local_task() {
Plugin p = new Plugin();
p.setName("test-plugin");
p.setVersion(Version.parse("0.1.0"));
p.setScript("echo aaa \n echo bbb");
p.setBash("echo aaa \n echo bbb");
p.setDocker(new DockerOption().setImage("sonarqube:latest"));
pluginDao.save(p);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,5 +56,13 @@ public void should_parse_yml_to_plugin() {
Assert.assertEquals(2, exports.size());
Assert.assertTrue(exports.contains("VAR_EXPORT_1"));
Assert.assertTrue(exports.contains("VAR_EXPORT_2"));

String pwsh = plugin.getPwsh();
Assert.assertNotNull(pwsh);
Assert.assertEquals("$Env.PK_FILE=keyfile", pwsh.trim());

String bash = plugin.getBash();
Assert.assertNotNull(bash);
Assert.assertEquals("chmod 400 ${PK_FILE}", bash.trim());
}
}
4 changes: 2 additions & 2 deletions core/src/test/resources/flow-all-failure.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ steps:
FLOW_WORKSPACE: "echo step"
FLOW_VERSION: "echo step version"
allow_failure: true
script: |
bash: |
echo hello
- name: step2
allow_failure: true
script: "echo 2"
bash: "echo 2"
4 changes: 2 additions & 2 deletions core/src/test/resources/flow-with-condition.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ steps:
FLOW_VERSION: "echo step version"
condition: |
return false
script: |
bash: |
echo hello
- name: step2
allow_failure: false
script: "echo 2"
bash: "echo 2"
4 changes: 2 additions & 2 deletions core/src/test/resources/flow-with-failure.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ steps:
- envs:
FLOW_WORKSPACE: "echo step"
FLOW_VERSION: "echo step version"
script: |
bash: |
echo shoulde failure
- script: |
- bash: |
echo cannot run
Loading

0 comments on commit 08929a3

Please sign in to comment.