From 8b0adb51f714c735717d759018a52854539e6292 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 16 May 2021 16:05:16 +0200 Subject: [PATCH 01/10] support config from yml --- .../com/flowci/core/agent/domain/ShellIn.java | 2 ++ .../flowci/core/flow/service/YmlServiceImpl.java | 16 ++++++++++++++++ .../flowci/core/job/manager/CmdManagerImpl.java | 1 + tree/src/main/java/com/flowci/tree/NodeTree.java | 8 +++++++- .../java/com/flowci/tree/RegularStepNode.java | 10 ++++++++++ .../main/java/com/flowci/tree/yml/StepYml.java | 3 +++ 6 files changed, 39 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java index d1b8bcfed..3be9a1f13 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java +++ b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java @@ -52,6 +52,8 @@ public enum ShellType { private Set secrets; + private Set configs; + public ShellIn() { super(Type.SHELL); } diff --git a/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java b/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java index 040415ea1..15df8c3a7 100644 --- a/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java +++ b/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java @@ -18,6 +18,7 @@ import com.flowci.core.common.manager.ConditionManager; import com.flowci.core.common.manager.SpringEventManager; +import com.flowci.core.config.event.GetConfigEvent; import com.flowci.core.flow.dao.FlowDao; import com.flowci.core.flow.dao.YmlDao; import com.flowci.core.flow.domain.Flow; @@ -51,6 +52,7 @@ public class YmlServiceImpl implements YmlService { private final List elementCheckers = ImmutableList.of( new ConditionChecker(), + new ConfigChecker(), new PluginChecker(), new SecretChecker() ); @@ -206,4 +208,18 @@ public Optional apply(NodeTree tree) { return Optional.empty(); } } + + private class ConfigChecker implements NodeElementChecker { + + @Override + public Optional apply(NodeTree tree) { + for (String c : tree.getConfigs()) { + GetConfigEvent event = eventManager.publish(new GetConfigEvent(this, c)); + if (event.hasError()) { + return Optional.of(event.getError()); + } + } + return Optional.empty(); + } + } } diff --git a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java index 781f1f169..9bf1de671 100644 --- a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java @@ -65,6 +65,7 @@ public ShellIn createShellCmd(Job job, Step step, Node node) { .setTimeout(r.fetchTimeout(job.getTimeout())) .setRetry(r.fetchRetry(0)) .setSecrets(r.getSecrets()) + .setConfigs(r.getConfigs()) .setCache(r.getCache()); if (r.hasPlugin()) { diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index 45f9056cc..161938be5 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -29,7 +29,7 @@ @Getter public final class NodeTree { - private static final int DefaultSize = 20; + private static final int DefaultSize = 10; private static final int DefaultSizeForPrev = 5; @@ -57,6 +57,8 @@ public static NodeTree create(FlowNode root) { private final Set secrets = new HashSet<>(DefaultSize); + private final Set configs = new HashSet<>(DefaultSize); + private int maxHeight = 1; public NodeTree(FlowNode root) { @@ -207,6 +209,10 @@ private void buildMetaData() { if (r.hasSecrets()) { secrets.addAll(r.getSecrets()); } + + if (r.hasConfigs()) { + configs.addAll(r.getConfigs()); + } } if (node instanceof FlowNode) { diff --git a/tree/src/main/java/com/flowci/tree/RegularStepNode.java b/tree/src/main/java/com/flowci/tree/RegularStepNode.java index bf8300003..78b0a56d3 100644 --- a/tree/src/main/java/com/flowci/tree/RegularStepNode.java +++ b/tree/src/main/java/com/flowci/tree/RegularStepNode.java @@ -57,6 +57,11 @@ public final class RegularStepNode extends Node { */ private Set secrets = new HashSet<>(0); + /** + * Included config name in the step + */ + private Set configs = new HashSet<>(0); + /** * Is allow failure */ @@ -91,6 +96,11 @@ public boolean hasSecrets() { return !secrets.isEmpty(); } + @JsonIgnore + public boolean hasConfigs() { + return !configs.isEmpty(); + } + @JsonIgnore public boolean hasTimeout() { return timeout != null; diff --git a/tree/src/main/java/com/flowci/tree/yml/StepYml.java b/tree/src/main/java/com/flowci/tree/yml/StepYml.java index 34855189a..53c791448 100644 --- a/tree/src/main/java/com/flowci/tree/yml/StepYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/StepYml.java @@ -66,6 +66,8 @@ public class StepYml extends YmlBase { private List secrets = new LinkedList<>(); + private List configs = new LinkedList<>(); + /** * Only for parallel step, other fields will not valid */ @@ -114,6 +116,7 @@ public Node toNode(Node parent, int index) { step.setAllowFailure(allow_failure != null && allow_failure); step.setEnvironments(getVariableMap()); step.setSecrets(Sets.newHashSet(secrets)); + step.setConfigs(Sets.newHashSet(configs)); setCacheToNode(step); setDockerToNode(step); From b113c126726fda2642a8bfc0ae1b4e2817127419 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 8 Jun 2021 15:44:39 +0200 Subject: [PATCH 02/10] rm private key from rest api --- core/src/main/java/com/flowci/core/api/OpenRestController.java | 2 +- core/src/main/resources/default/smtp-demo-config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/flowci/core/api/OpenRestController.java b/core/src/main/java/com/flowci/core/api/OpenRestController.java index 0ebddbc7e..d3cea5d80 100644 --- a/core/src/main/java/com/flowci/core/api/OpenRestController.java +++ b/core/src/main/java/com/flowci/core/api/OpenRestController.java @@ -73,7 +73,7 @@ public Secret getSecret(@PathVariable String name) { if (secret instanceof RSASecret) { RSASecret rsa = (RSASecret) secret; - rsa.setPublicKey(null); + rsa.setPrivateKey(null); } return secret; diff --git a/core/src/main/resources/default/smtp-demo-config.yml b/core/src/main/resources/default/smtp-demo-config.yml index 438634ad1..8e437eea8 100644 --- a/core/src/main/resources/default/smtp-demo-config.yml +++ b/core/src/main/resources/default/smtp-demo-config.yml @@ -1,5 +1,5 @@ kind: smtp -name: sendgrid-demo +name: sendgrid_demo smtp: server: smtp.sendgrid.net port: 587 From 09975cc1d0f191eb2050635ab78191661beb11ae Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 22 Jul 2021 23:19:38 +0200 Subject: [PATCH 03/10] add cn source link --- .../java/com/flowci/core/flow/domain/Template.java | 4 ++++ .../com/flowci/core/plugin/domain/PluginRepoInfo.java | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/flowci/core/flow/domain/Template.java b/core/src/main/java/com/flowci/core/flow/domain/Template.java index b9670e9fc..f3aa768ea 100644 --- a/core/src/main/java/com/flowci/core/flow/domain/Template.java +++ b/core/src/main/java/com/flowci/core/flow/domain/Template.java @@ -1,5 +1,6 @@ package com.flowci.core.flow.domain; +import com.fasterxml.jackson.annotation.JsonAlias; import lombok.Getter; import lombok.Setter; @@ -9,6 +10,9 @@ public final class Template { private String url; + @JsonAlias("url_cn") + private String urlCn; + private String desc; private boolean isDefault; diff --git a/core/src/main/java/com/flowci/core/plugin/domain/PluginRepoInfo.java b/core/src/main/java/com/flowci/core/plugin/domain/PluginRepoInfo.java index 8d21e5bac..b47d9cd2b 100644 --- a/core/src/main/java/com/flowci/core/plugin/domain/PluginRepoInfo.java +++ b/core/src/main/java/com/flowci/core/plugin/domain/PluginRepoInfo.java @@ -16,16 +16,18 @@ package com.flowci.core.plugin.domain; +import com.fasterxml.jackson.annotation.JsonAlias; import com.flowci.domain.Version; -import java.io.Serializable; -import java.util.HashSet; -import java.util.Set; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; import org.springframework.data.mongodb.core.index.Indexed; +import java.io.Serializable; +import java.util.HashSet; +import java.util.Set; + /** * @author yang */ @@ -40,6 +42,9 @@ public class PluginRepoInfo implements Serializable { private String source; // git repo url + @JsonAlias("source_cn") + private String sourceCn; // git repo url for cn + private String branch = "master"; private String description; From 95170ac202761fb16c4e571bfb6945c2f4bdccd5 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 25 Jul 2021 23:40:26 +0200 Subject: [PATCH 04/10] optimize template and repo data repersent class --- .../core/common/config/AppProperties.java | 5 ++- .../common/manager/HttpRequestManager.java | 40 ------------------- .../flowci/core/flow/config/FlowConfig.java | 8 ++-- .../core/plugin/service/PluginService.java | 3 +- .../plugin/service/PluginServiceImpl.java | 19 +++------ .../core/test/plugin/PluginServiceTest.java | 26 +++++------- 6 files changed, 25 insertions(+), 76 deletions(-) delete mode 100644 core/src/main/java/com/flowci/core/common/manager/HttpRequestManager.java diff --git a/core/src/main/java/com/flowci/core/common/config/AppProperties.java b/core/src/main/java/com/flowci/core/common/config/AppProperties.java index 49a2b3da5..3e1196efb 100644 --- a/core/src/main/java/com/flowci/core/common/config/AppProperties.java +++ b/core/src/main/java/com/flowci/core/common/config/AppProperties.java @@ -23,6 +23,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import org.springframework.core.io.Resource; import org.springframework.validation.annotation.Validated; import javax.validation.constraints.NotBlank; @@ -106,7 +107,7 @@ public Minio minio() { @Data public static class Flow { - private String templatesUrl; + private Resource templatesUrl; } @Data @@ -118,7 +119,7 @@ public static class Job { @Data public static class Plugin { - private String defaultRepo; + private Resource defaultRepo; private Boolean autoUpdate; } diff --git a/core/src/main/java/com/flowci/core/common/manager/HttpRequestManager.java b/core/src/main/java/com/flowci/core/common/manager/HttpRequestManager.java deleted file mode 100644 index ff49ede3f..000000000 --- a/core/src/main/java/com/flowci/core/common/manager/HttpRequestManager.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.flowci.core.common.manager; - -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.io.IOException; - -@Component("httpManager") -public class HttpRequestManager { - - private final static int DefaultTimeout = 30 * 1000; - - private final static String DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36"; - - private HttpClient client; - - @PostConstruct - public void init() { - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(DefaultTimeout) - .setConnectionRequestTimeout(DefaultTimeout) - .setSocketTimeout(DefaultTimeout) - .build(); - client = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); - } - - public String get(String url) throws IOException { - HttpGet request = new HttpGet(url); - request.setHeader("User-Agent", DefaultUserAgent); - - HttpResponse execute = client.execute(request); - return EntityUtils.toString(execute.getEntity()); - } -} diff --git a/core/src/main/java/com/flowci/core/flow/config/FlowConfig.java b/core/src/main/java/com/flowci/core/flow/config/FlowConfig.java index cf7fac04c..0d44c88d5 100644 --- a/core/src/main/java/com/flowci/core/flow/config/FlowConfig.java +++ b/core/src/main/java/com/flowci/core/flow/config/FlowConfig.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flowci.core.common.config.AppProperties; import com.flowci.core.common.helper.CacheHelper; -import com.flowci.core.common.manager.HttpRequestManager; import com.flowci.core.flow.domain.Template; import com.flowci.tree.NodeTree; import com.github.benmanes.caffeine.cache.Cache; @@ -28,6 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler; @@ -58,13 +58,13 @@ public Cache> gitBranchCache() { } @Bean("templates") - public List