From 1020db3294a50a89aa5b77851c7c2033c1e07e1b Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Mon, 2 Nov 2020 09:06:54 +0100 Subject: [PATCH 001/128] set default upload dir to tmp --- .../java/com/flowci/core/job/controller/JobController.java | 2 -- core/src/main/resources/application.properties | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/controller/JobController.java b/core/src/main/java/com/flowci/core/job/controller/JobController.java index d1a9dce46..ebd0b89fc 100644 --- a/core/src/main/java/com/flowci/core/job/controller/JobController.java +++ b/core/src/main/java/com/flowci/core/job/controller/JobController.java @@ -26,9 +26,7 @@ import com.flowci.core.job.service.*; import com.flowci.core.user.domain.User; import com.flowci.exception.ArgumentException; -import com.flowci.exception.StatusException; import com.flowci.tree.NodePath; -import com.flowci.util.StringHelper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; diff --git a/core/src/main/resources/application.properties b/core/src/main/resources/application.properties index 09c8cb40c..754aaefb7 100644 --- a/core/src/main/resources/application.properties +++ b/core/src/main/resources/application.properties @@ -14,7 +14,8 @@ management.endpoint.health.show-details=always management.endpoint.shutdown.enabled=true management.endpoints.web.base-path=/ -spring.servlet.multipart.location=${app.workspace}/tmp +spring.servlet.multipart.enabled=true +spring.servlet.multipart.location=${java.io.tmpdir} spring.servlet.multipart.max-file-size=100MB spring.servlet.multipart.max-request-size=100MB From c049b1d91de0e02b6dec3d9648d67cc6f5d745eb Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Mon, 2 Nov 2020 10:07:09 +0100 Subject: [PATCH 002/128] refactor split into diff controllers --- .../job/controller/ArtifactController.java | 48 ++++++++++ .../core/job/controller/BaseController.java | 42 +++++++++ .../core/job/controller/JobController.java | 89 +------------------ .../job/controller/LoggingController.java | 49 ++++++++++ 4 files changed, 143 insertions(+), 85 deletions(-) create mode 100644 core/src/main/java/com/flowci/core/job/controller/ArtifactController.java create mode 100644 core/src/main/java/com/flowci/core/job/controller/BaseController.java create mode 100644 core/src/main/java/com/flowci/core/job/controller/LoggingController.java diff --git a/core/src/main/java/com/flowci/core/job/controller/ArtifactController.java b/core/src/main/java/com/flowci/core/job/controller/ArtifactController.java new file mode 100644 index 000000000..6718354d1 --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/controller/ArtifactController.java @@ -0,0 +1,48 @@ +package com.flowci.core.job.controller; + +import com.flowci.core.auth.annotation.Action; +import com.flowci.core.job.domain.Job; +import com.flowci.core.job.domain.JobAction; +import com.flowci.core.job.domain.JobArtifact; +import com.flowci.core.job.service.ArtifactService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/jobs") +public class ArtifactController extends BaseController { + + @Autowired + private ArtifactService artifactService; + + @GetMapping("/{flow}/{buildNumber}/artifacts") + @Action(JobAction.LIST_ARTIFACTS) + public List listArtifact(@PathVariable String flow, @PathVariable String buildNumber) { + Job job = getJob(flow, buildNumber); + return artifactService.list(job); + } + + @GetMapping(value = "/{flow}/{buildNumber}/artifacts/{artifactId}") + @Action(JobAction.DOWNLOAD_ARTIFACT) + public ResponseEntity downloadArtifact(@PathVariable String flow, + @PathVariable String buildNumber, + @PathVariable String artifactId) { + Job job = getJob(flow, buildNumber); + JobArtifact artifact = artifactService.fetch(job, artifactId); + + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType(MediaType.APPLICATION_OCTET_STREAM_VALUE)) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + artifact.getFileName() + "\"") + .body(new InputStreamResource(artifact.getSrc())); + } +} diff --git a/core/src/main/java/com/flowci/core/job/controller/BaseController.java b/core/src/main/java/com/flowci/core/job/controller/BaseController.java new file mode 100644 index 000000000..50305ca30 --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/controller/BaseController.java @@ -0,0 +1,42 @@ +package com.flowci.core.job.controller; + +import com.flowci.core.flow.domain.Flow; +import com.flowci.core.flow.service.FlowService; +import com.flowci.core.job.domain.Job; +import com.flowci.core.job.service.JobService; +import com.flowci.core.job.service.StepService; +import com.flowci.exception.ArgumentException; +import org.springframework.beans.factory.annotation.Autowired; + +public abstract class BaseController { + + protected static final String DefaultPage = "0"; + + protected static final String DefaultSize = "20"; + + protected static final String ParameterLatest = "latest"; + + @Autowired + protected FlowService flowService; + + @Autowired + protected JobService jobService; + + @Autowired + protected StepService stepService; + + protected Job getJob(String name, String buildNumberOrLatest) { + Flow flow = flowService.get(name); + + if (ParameterLatest.equals(buildNumberOrLatest)) { + return jobService.getLatest(flow.getId()); + } + + try { + long buildNumber = Long.parseLong(buildNumberOrLatest); + return jobService.get(flow.getId(), buildNumber); + } catch (NumberFormatException e) { + throw new ArgumentException("Build number must be a integer"); + } + } +} diff --git a/core/src/main/java/com/flowci/core/job/controller/JobController.java b/core/src/main/java/com/flowci/core/job/controller/JobController.java index ebd0b89fc..5c565a91c 100644 --- a/core/src/main/java/com/flowci/core/job/controller/JobController.java +++ b/core/src/main/java/com/flowci/core/job/controller/JobController.java @@ -19,27 +19,21 @@ import com.flowci.core.auth.annotation.Action; import com.flowci.core.common.manager.SessionManager; import com.flowci.core.flow.domain.Flow; -import com.flowci.core.flow.service.FlowService; import com.flowci.core.flow.service.YmlService; import com.flowci.core.job.domain.*; import com.flowci.core.job.domain.Job.Trigger; -import com.flowci.core.job.service.*; +import com.flowci.core.job.service.LocalTaskService; +import com.flowci.core.job.service.ReportService; import com.flowci.core.user.domain.User; import com.flowci.exception.ArgumentException; -import com.flowci.tree.NodePath; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.InputStreamResource; -import org.springframework.core.io.Resource; import org.springframework.core.task.TaskExecutor; import org.springframework.data.domain.Page; -import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import java.util.Base64; -import java.util.Collection; import java.util.List; import java.util.Objects; @@ -48,41 +42,20 @@ */ @RestController @RequestMapping("/jobs") -public class JobController { - - private static final String DefaultPage = "0"; - - private static final String DefaultSize = "20"; - - private static final String ParameterLatest = "latest"; +public class JobController extends BaseController { @Autowired private SessionManager sessionManager; - @Autowired - private FlowService flowService; - @Autowired private YmlService ymlService; - @Autowired - private JobService jobService; - - @Autowired - private StepService stepService; - @Autowired private LocalTaskService localTaskService; - @Autowired - private LoggingService loggingService; - @Autowired private ReportService reportService; - @Autowired - private ArtifactService artifactService; - @Autowired private TaskExecutor appTaskExecutor; @@ -99,18 +72,7 @@ public Page list(@PathVariable("flow") String name, @GetMapping("/{flow}/{buildNumberOrLatest}") @Action(JobAction.GET) public Job get(@PathVariable("flow") String name, @PathVariable String buildNumberOrLatest) { - Flow flow = flowService.get(name); - - if (ParameterLatest.equals(buildNumberOrLatest)) { - return jobService.getLatest(flow.getId()); - } - - try { - long buildNumber = Long.parseLong(buildNumberOrLatest); - return jobService.get(flow.getId(), buildNumber); - } catch (NumberFormatException e) { - throw new ArgumentException("Build number must be a integer"); - } + return super.getJob(name, buildNumberOrLatest); } @GetMapping("/{jobId}/desc") @@ -143,28 +105,6 @@ public List listTasks(@PathVariable String flow, return localTaskService.list(job); } - @GetMapping("/logs/{stepId}/read") - @Action(JobAction.DOWNLOAD_STEP_LOG) - public Collection readStepLog(@PathVariable String stepId) { - return loggingService.read(stepId); - } - - @GetMapping("/logs/{stepId}/download") - @Action(JobAction.DOWNLOAD_STEP_LOG) - public ResponseEntity downloadStepLog(@PathVariable String stepId) { - Step step = stepService.get(stepId); - Resource resource = loggingService.get(stepId); - Flow flow = flowService.getById(step.getFlowId()); - - NodePath path = NodePath.create(step.getNodePath()); - String fileName = String.format("%s-#%s-%s.log", flow.getName(), step.getBuildNumber(), path.name()); - - return ResponseEntity.ok() - .contentType(MediaType.parseMediaType(MediaType.APPLICATION_OCTET_STREAM_VALUE)) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"") - .body(resource); - } - @PostMapping @Action(JobAction.CREATE) public Job create(@Validated @RequestBody CreateJob data) { @@ -223,25 +163,4 @@ public String fetchReport(@PathVariable String flow, Job job = get(flow, buildNumber); return reportService.fetch(job, reportId); } - - @GetMapping("/{flow}/{buildNumber}/artifacts") - @Action(JobAction.LIST_ARTIFACTS) - public List listArtifact(@PathVariable String flow, @PathVariable String buildNumber) { - Job job = get(flow, buildNumber); - return artifactService.list(job); - } - - @GetMapping(value = "/{flow}/{buildNumber}/artifacts/{artifactId}") - @Action(JobAction.DOWNLOAD_ARTIFACT) - public ResponseEntity downloadArtifact(@PathVariable String flow, - @PathVariable String buildNumber, - @PathVariable String artifactId) { - Job job = get(flow, buildNumber); - JobArtifact artifact = artifactService.fetch(job, artifactId); - - return ResponseEntity.ok() - .contentType(MediaType.parseMediaType(MediaType.APPLICATION_OCTET_STREAM_VALUE)) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + artifact.getFileName() + "\"") - .body(new InputStreamResource(artifact.getSrc())); - } } diff --git a/core/src/main/java/com/flowci/core/job/controller/LoggingController.java b/core/src/main/java/com/flowci/core/job/controller/LoggingController.java new file mode 100644 index 000000000..5fa26f68d --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/controller/LoggingController.java @@ -0,0 +1,49 @@ +package com.flowci.core.job.controller; + +import com.flowci.core.auth.annotation.Action; +import com.flowci.core.flow.domain.Flow; +import com.flowci.core.job.domain.JobAction; +import com.flowci.core.job.domain.Step; +import com.flowci.core.job.service.LoggingService; +import com.flowci.tree.NodePath; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Collection; + +@RestController +@RequestMapping("/jobs/logs") +public class LoggingController extends BaseController { + + @Autowired + private LoggingService loggingService; + + @GetMapping("/{stepId}/read") + @Action(JobAction.DOWNLOAD_STEP_LOG) + public Collection readStepLog(@PathVariable String stepId) { + return loggingService.read(stepId); + } + + @GetMapping("/{stepId}/download") + @Action(JobAction.DOWNLOAD_STEP_LOG) + public ResponseEntity downloadStepLog(@PathVariable String stepId) { + Step step = stepService.get(stepId); + Resource resource = loggingService.get(stepId); + Flow flow = flowService.getById(step.getFlowId()); + + NodePath path = NodePath.create(step.getNodePath()); + String fileName = String.format("%s-#%s-%s.log", flow.getName(), step.getBuildNumber(), path.name()); + + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType(MediaType.APPLICATION_OCTET_STREAM_VALUE)) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"") + .body(resource); + } +} From bbbc6a9c8a0d9546b323df7aeee2ad76b28b758b Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Mon, 2 Nov 2020 15:28:44 +0100 Subject: [PATCH 003/128] support cache both on flow and step level --- tree/src/main/java/com/flowci/tree/Cache.java | 15 +++++++++++++++ tree/src/main/java/com/flowci/tree/Node.java | 2 ++ .../java/com/flowci/tree/yml/CacheYml.java | 15 +++++++++++++++ .../java/com/flowci/tree/yml/FlowYml.java | 3 ++- .../java/com/flowci/tree/yml/StepYml.java | 2 ++ .../java/com/flowci/tree/yml/YmlBase.java | 19 +++++++++++++++++++ .../com/flowci/tree/test/YmlParserTest.java | 6 ++++++ tree/src/test/resources/flow.yml | 5 +++++ 8 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tree/src/main/java/com/flowci/tree/Cache.java create mode 100644 tree/src/main/java/com/flowci/tree/yml/CacheYml.java diff --git a/tree/src/main/java/com/flowci/tree/Cache.java b/tree/src/main/java/com/flowci/tree/Cache.java new file mode 100644 index 000000000..50e870037 --- /dev/null +++ b/tree/src/main/java/com/flowci/tree/Cache.java @@ -0,0 +1,15 @@ +package com.flowci.tree; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +public class Cache { + + private String key; + + private List paths; +} diff --git a/tree/src/main/java/com/flowci/tree/Node.java b/tree/src/main/java/com/flowci/tree/Node.java index c989457ba..50f34f0f7 100644 --- a/tree/src/main/java/com/flowci/tree/Node.java +++ b/tree/src/main/java/com/flowci/tree/Node.java @@ -54,6 +54,8 @@ public abstract class Node implements Serializable { @JsonIgnore protected Node parent; + protected Cache cache; + /** * Inner option has higher priority * Ex: Plugin > Step > Flow diff --git a/tree/src/main/java/com/flowci/tree/yml/CacheYml.java b/tree/src/main/java/com/flowci/tree/yml/CacheYml.java new file mode 100644 index 000000000..1b64c59e7 --- /dev/null +++ b/tree/src/main/java/com/flowci/tree/yml/CacheYml.java @@ -0,0 +1,15 @@ +package com.flowci.tree.yml; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +public class CacheYml { + + private String key; + + private List paths; +} diff --git a/tree/src/main/java/com/flowci/tree/yml/FlowYml.java b/tree/src/main/java/com/flowci/tree/yml/FlowYml.java index 7639179c5..48e8d151c 100644 --- a/tree/src/main/java/com/flowci/tree/yml/FlowYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/FlowYml.java @@ -54,8 +54,9 @@ public FlowNode toNode() { FlowNode node = new FlowNode(name); node.setSelector(selector); node.setCondition(condition); - node.setEnvironments(getVariableMap()); + node.setEnvironments(getVariableMap());; + setCache(node); setDocker(node); setupNotifications(node); diff --git a/tree/src/main/java/com/flowci/tree/yml/StepYml.java b/tree/src/main/java/com/flowci/tree/yml/StepYml.java index 78849569b..edf75abc1 100644 --- a/tree/src/main/java/com/flowci/tree/yml/StepYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/StepYml.java @@ -78,6 +78,8 @@ public StepNode toNode(Node parent, int index) { node.setExports(Sets.newHashSet(exports)); node.setAllowFailure(allow_failure); node.setEnvironments(getVariableMap()); + + setCache(node); setDocker(node); if (StringHelper.hasValue(node.getName()) && !NodePath.validate(node.getName())) { diff --git a/tree/src/main/java/com/flowci/tree/yml/YmlBase.java b/tree/src/main/java/com/flowci/tree/yml/YmlBase.java index 1565613eb..ebc2d50f0 100644 --- a/tree/src/main/java/com/flowci/tree/yml/YmlBase.java +++ b/tree/src/main/java/com/flowci/tree/yml/YmlBase.java @@ -19,6 +19,7 @@ import com.flowci.domain.DockerOption; import com.flowci.domain.StringVars; import com.flowci.exception.YmlException; +import com.flowci.tree.Cache; import com.flowci.tree.Node; import com.flowci.tree.StepNode; import com.flowci.util.ObjectsHelper; @@ -51,6 +52,8 @@ public abstract class YmlBase implements Serializable { // or dockers public List dockers; + public CacheYml cache; + @NonNull public List steps = new LinkedList<>(); @@ -62,6 +65,22 @@ StringVars getVariableMap() { return variables; } + void setCache(T node) { + if (Objects.isNull(cache)) { + return; + } + + if (!ObjectsHelper.hasCollection(cache.getPaths())) { + return; + } + + Cache c = new Cache(); + c.setKey(cache.getKey()); + c.setPaths(cache.getPaths()); + + node.setCache(c); + } + void setEnvs(StringVars variables) { for (Map.Entry entry : variables.entrySet()) { this.envs.put(entry.getKey(), entry.getValue()); diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index 7eb5985f3..63f9fc06f 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -62,6 +62,12 @@ public void should_get_node_from_yml() { Assert.assertNotNull(root.getCondition()); + // verify cache + Assert.assertNotNull(root.getCache()); + Assert.assertEquals(2, root.getCache().getPaths().size()); + Assert.assertEquals("./", root.getCache().getPaths().get(0)); + Assert.assertEquals("vendor", root.getCache().getPaths().get(1)); + // verify docker Assert.assertTrue(root.getDockers().size() > 0); Assert.assertEquals("helloworld:0.1", root.getDockers().get(0).getImage()); diff --git a/tree/src/test/resources/flow.yml b/tree/src/test/resources/flow.yml index 3d0730027..6d2f85e48 100644 --- a/tree/src/test/resources/flow.yml +++ b/tree/src/test/resources/flow.yml @@ -14,6 +14,11 @@ selector: - ios - local +cache: + paths: + - "./" + - "vendor" + notifications: - plugin: 'email-notify' envs: From d210852f93bc0155201fd7ec5cb3134933e4f16a Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 4 Nov 2020 14:51:52 +0100 Subject: [PATCH 004/128] enable to put cache --- .../com/flowci/core/agent/domain/ShellIn.java | 3 ++ .../core/job/controller/CacheController.java | 29 +++++++++++++ .../core/job/manager/CmdManagerImpl.java | 21 +++++++--- .../flowci/core/job/service/CacheService.java | 13 ++++++ .../core/job/service/CacheServiceImpl.java | 42 +++++++++++++++++++ tree/src/main/java/com/flowci/tree/Node.java | 5 +++ 6 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/com/flowci/core/job/controller/CacheController.java create mode 100644 core/src/main/java/com/flowci/core/job/service/CacheService.java create mode 100644 core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java index d0eb89fa3..8ad851a73 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java +++ b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java @@ -3,6 +3,7 @@ import com.flowci.domain.DockerOption; import com.flowci.domain.StringVars; import com.flowci.domain.Vars; +import com.flowci.tree.Cache; import com.google.common.base.Strings; import lombok.Getter; import lombok.Setter; @@ -33,6 +34,8 @@ public enum ShellType { private String plugin; + private Cache cache; + private List dockers; private List bash; diff --git a/core/src/main/java/com/flowci/core/job/controller/CacheController.java b/core/src/main/java/com/flowci/core/job/controller/CacheController.java new file mode 100644 index 000000000..25ccb4cea --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/controller/CacheController.java @@ -0,0 +1,29 @@ +package com.flowci.core.job.controller; + +import com.flowci.core.job.domain.Job; +import com.flowci.core.job.service.CacheService; +import com.flowci.exception.ArgumentException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequestMapping("/jobs") +public class CacheController extends BaseController { + + @Autowired + private CacheService cacheService; + + @PostMapping("/{flow}/{buildNumber}/cache/{key}") + public void put(@PathVariable String flow, + @PathVariable String buildNumber, + @PathVariable String key, + @RequestParam("files") MultipartFile[] files) { + if (files.length == 0) { + throw new ArgumentException("the cached files are empty"); + } + + Job job = getJob(flow, buildNumber); + cacheService.put(job, key, files); + } +} diff --git a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java index 87a96a66e..de2aa3eda 100644 --- a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java @@ -28,10 +28,7 @@ import com.flowci.domain.DockerOption; import com.flowci.domain.StringVars; import com.flowci.domain.Vars; -import com.flowci.tree.Node; -import com.flowci.tree.NodePath; -import com.flowci.tree.NodeTree; -import com.flowci.tree.StepNode; +import com.flowci.tree.*; import com.flowci.util.ObjectsHelper; import com.flowci.util.StringHelper; import org.springframework.beans.factory.annotation.Autowired; @@ -63,7 +60,8 @@ public ShellIn createShellCmd(Job job, Step step, NodeTree tree) { .setEnvFilters(linkFilters(node)) .setInputs(linkInputs(node).merge(job.getContext(), false)) .setTimeout(linkTimeout(node, job.getTimeout())) - .setRetry(linkRetry(node, 0)); + .setRetry(linkRetry(node, 0)) + .setCache(linkCache(node)); if (node.hasPlugin()) { setPlugin(node.getPlugin(), in); @@ -126,6 +124,19 @@ private Integer linkRetry(StepNode current, Integer defaultRetry) { return defaultRetry; } + private Cache linkCache(Node current) { + if (current.hasCache()) { + return current.getCache(); + } + + if (current.hasParent()) { + Node parent = current.getParent(); + return linkCache(parent); + } + + return null; + } + private Integer linkTimeout(StepNode current, Integer defaultTimeout) { if (current.hasTimeout()) { return current.getTimeout(); diff --git a/core/src/main/java/com/flowci/core/job/service/CacheService.java b/core/src/main/java/com/flowci/core/job/service/CacheService.java new file mode 100644 index 000000000..39514333e --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/service/CacheService.java @@ -0,0 +1,13 @@ +package com.flowci.core.job.service; + +import com.flowci.core.job.domain.Job; +import org.springframework.web.multipart.MultipartFile; + +/** + * Job Cache to save cached file/dir into file store + */ +public interface CacheService { + + void put(Job job, String key, MultipartFile[] files); + +} diff --git a/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java new file mode 100644 index 000000000..5c8eb1082 --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java @@ -0,0 +1,42 @@ +package com.flowci.core.job.service; + +import com.flowci.core.job.domain.Job; +import com.flowci.store.FileManager; +import com.flowci.store.Pathable; +import com.flowci.store.StringPath; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +@Log4j2 +@Service +public class CacheServiceImpl implements CacheService { + + private final static Pathable CacheRoot = new StringPath("_cache_"); + + @Autowired + private FileManager fileManager; + + @Override + public void put(Job job, String key, MultipartFile[] files) { + for (MultipartFile file : files) { + try { + Pathable[] cachePath = getCachePath(job, key); + fileManager.save(file.getOriginalFilename(), file.getInputStream(), cachePath); + } catch (IOException e) { + log.warn("failed to save file {} for cache {}", file.getName(), key); + } + } + } + + private Pathable[] getCachePath(Job job, String key) { + return new Pathable[]{ + new StringPath(job.getFlowId()), + CacheRoot, + new StringPath(key) + }; + } +} diff --git a/tree/src/main/java/com/flowci/tree/Node.java b/tree/src/main/java/com/flowci/tree/Node.java index 50f34f0f7..8a1684555 100644 --- a/tree/src/main/java/com/flowci/tree/Node.java +++ b/tree/src/main/java/com/flowci/tree/Node.java @@ -105,4 +105,9 @@ public boolean hasChildren() { public boolean hasParent() { return parent != null; } + + @JsonIgnore + public boolean hasCache() { + return cache != null; + } } From e48fabe59d2b353f00c330edb9fa7ad88a16631f Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 5 Nov 2020 15:18:40 +0100 Subject: [PATCH 005/128] support cache api for agent --- .../agent/controller/AgentAPIController.java | 78 +++++++++++++++++++ .../agent/controller/AgentController.java | 28 ++----- .../core/job/controller/BaseController.java | 4 + .../core/job/controller/CacheController.java | 29 ------- .../com/flowci/core/job/dao/JobCacheDao.java | 13 ++++ .../com/flowci/core/job/domain/JobCache.java | 35 +++++++++ .../flowci/core/job/service/CacheService.java | 9 ++- .../core/job/service/CacheServiceImpl.java | 73 ++++++++++++++++- 8 files changed, 211 insertions(+), 58 deletions(-) create mode 100644 core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java delete mode 100644 core/src/main/java/com/flowci/core/job/controller/CacheController.java create mode 100644 core/src/main/java/com/flowci/core/job/dao/JobCacheDao.java create mode 100644 core/src/main/java/com/flowci/core/job/domain/JobCache.java diff --git a/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java b/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java new file mode 100644 index 000000000..d48efe86a --- /dev/null +++ b/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java @@ -0,0 +1,78 @@ +package com.flowci.core.agent.controller; + +import com.flowci.core.agent.domain.Agent; +import com.flowci.core.agent.service.AgentService; +import com.flowci.core.job.domain.JobCache; +import com.flowci.core.job.service.CacheService; +import com.flowci.core.job.service.LoggingService; +import com.flowci.exception.ArgumentException; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Functions require agent token header + */ +@Log4j2 +@RestController +@RequestMapping("/agents/api") +public class AgentAPIController { + + @Autowired + private AgentService agentService; + + @Autowired + private LoggingService loggingService; + + @Autowired + private CacheService cacheService; + + @PostMapping("/profile") + public void profile(@RequestHeader(AgentAuth.HeaderAgentToken) String token, + @RequestBody Agent.Resource resource) { + agentService.update(token, resource); + } + + @PostMapping("/logs/upload") + public void upload(@RequestPart("file") MultipartFile file) { + try (InputStream stream = file.getInputStream()) { + loggingService.save(file.getOriginalFilename(), stream); + } catch (IOException e) { + log.warn("Unable to save log, cause {}", e.getMessage()); + } + } + + @PostMapping("/cache/{jobId}/{key}") + public void putCache(@PathVariable String jobId, + @PathVariable String key, + @RequestParam("files") MultipartFile[] files) { + if (files.length == 0) { + throw new ArgumentException("the cached files are empty"); + } + + cacheService.put(jobId, key, files); + } + + @GetMapping("/cache/{jobId}/{key}") + public JobCache getCache(@PathVariable String jobId, @PathVariable String key) { + return cacheService.get(jobId, key); + } + + @GetMapping("/cache/{cacheId}") + public ResponseEntity downloadCache(@PathVariable String cacheId, @RequestParam String file) { + InputStream stream = cacheService.fetch(cacheId, file); + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType(MediaType.APPLICATION_OCTET_STREAM_VALUE)) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file + "\"") + .body(new InputStreamResource(stream)); + } +} diff --git a/core/src/main/java/com/flowci/core/agent/controller/AgentController.java b/core/src/main/java/com/flowci/core/agent/controller/AgentController.java index 4116478be..d6ed2b8c3 100644 --- a/core/src/main/java/com/flowci/core/agent/controller/AgentController.java +++ b/core/src/main/java/com/flowci/core/agent/controller/AgentController.java @@ -16,21 +16,19 @@ package com.flowci.core.agent.controller; +import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.AgentAction; import com.flowci.core.agent.domain.CreateOrUpdateAgent; import com.flowci.core.agent.domain.DeleteAgent; import com.flowci.core.agent.service.AgentService; import com.flowci.core.auth.annotation.Action; +import com.flowci.core.job.service.CacheService; import com.flowci.core.job.service.LoggingService; -import com.flowci.core.agent.domain.Agent; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; -import java.io.InputStream; import java.util.List; import java.util.Optional; @@ -48,6 +46,9 @@ public class AgentController { @Autowired private LoggingService loggingService; + @Autowired + private CacheService cacheService; + @GetMapping("/{name}") @Action(AgentAction.GET) public Agent getByName(@PathVariable String name) { @@ -75,23 +76,4 @@ public Agent createOrUpdate(@Validated @RequestBody CreateOrUpdateAgent body) { public Agent delete(@Validated @RequestBody DeleteAgent body) { return agentService.delete(body.getToken()); } - - // -------------------------------------------------------- - // Functions require agent token header - // -------------------------------------------------------- - - @PostMapping("/api/profile") - public void profile(@RequestHeader(AgentAuth.HeaderAgentToken) String token, - @RequestBody Agent.Resource resource) { - agentService.update(token, resource); - } - - @PostMapping("/api/logs/upload") - public void upload(@RequestPart("file") MultipartFile file) { - try(InputStream stream = file.getInputStream()) { - loggingService.save(file.getOriginalFilename(), stream); - } catch (IOException e) { - log.warn("Unable to save log, cause {}", e.getMessage()); - } - } } diff --git a/core/src/main/java/com/flowci/core/job/controller/BaseController.java b/core/src/main/java/com/flowci/core/job/controller/BaseController.java index 50305ca30..08ddd81b8 100644 --- a/core/src/main/java/com/flowci/core/job/controller/BaseController.java +++ b/core/src/main/java/com/flowci/core/job/controller/BaseController.java @@ -25,6 +25,10 @@ public abstract class BaseController { @Autowired protected StepService stepService; + protected Job getJob(String id) { + return jobService.get(id); + } + protected Job getJob(String name, String buildNumberOrLatest) { Flow flow = flowService.get(name); diff --git a/core/src/main/java/com/flowci/core/job/controller/CacheController.java b/core/src/main/java/com/flowci/core/job/controller/CacheController.java deleted file mode 100644 index 25ccb4cea..000000000 --- a/core/src/main/java/com/flowci/core/job/controller/CacheController.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.flowci.core.job.controller; - -import com.flowci.core.job.domain.Job; -import com.flowci.core.job.service.CacheService; -import com.flowci.exception.ArgumentException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; - -@RestController -@RequestMapping("/jobs") -public class CacheController extends BaseController { - - @Autowired - private CacheService cacheService; - - @PostMapping("/{flow}/{buildNumber}/cache/{key}") - public void put(@PathVariable String flow, - @PathVariable String buildNumber, - @PathVariable String key, - @RequestParam("files") MultipartFile[] files) { - if (files.length == 0) { - throw new ArgumentException("the cached files are empty"); - } - - Job job = getJob(flow, buildNumber); - cacheService.put(job, key, files); - } -} diff --git a/core/src/main/java/com/flowci/core/job/dao/JobCacheDao.java b/core/src/main/java/com/flowci/core/job/dao/JobCacheDao.java new file mode 100644 index 000000000..cd14ac4b2 --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/dao/JobCacheDao.java @@ -0,0 +1,13 @@ +package com.flowci.core.job.dao; + +import com.flowci.core.job.domain.JobCache; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface JobCacheDao extends MongoRepository { + + Optional findByFlowIdAndKey(String flowId, String key); +} diff --git a/core/src/main/java/com/flowci/core/job/domain/JobCache.java b/core/src/main/java/com/flowci/core/job/domain/JobCache.java new file mode 100644 index 000000000..48de7396a --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/domain/JobCache.java @@ -0,0 +1,35 @@ +package com.flowci.core.job.domain; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.flowci.core.common.domain.Mongoable; +import lombok.Getter; +import lombok.Setter; +import org.springframework.data.annotation.Transient; +import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.io.InputStream; +import java.util.List; + +@Getter +@Setter +@Document("job_cache") +@CompoundIndex(name = "index_flow_key", def = "{'flowId' : 1, 'key': 1}", unique = true) +public class JobCache extends Mongoable { + + private String flowId; + + private String jobId; + + private String key; + + /** + * Full path with b64 encoded based on workspace + * ex: a/b/c.zip --> b64xxx.zip + */ + private List files; + + @JsonIgnore + @Transient + private InputStream src; +} diff --git a/core/src/main/java/com/flowci/core/job/service/CacheService.java b/core/src/main/java/com/flowci/core/job/service/CacheService.java index 39514333e..b3b7853ab 100644 --- a/core/src/main/java/com/flowci/core/job/service/CacheService.java +++ b/core/src/main/java/com/flowci/core/job/service/CacheService.java @@ -1,13 +1,18 @@ package com.flowci.core.job.service; -import com.flowci.core.job.domain.Job; +import com.flowci.core.job.domain.JobCache; import org.springframework.web.multipart.MultipartFile; +import java.io.InputStream; + /** * Job Cache to save cached file/dir into file store */ public interface CacheService { - void put(Job job, String key, MultipartFile[] files); + JobCache put(String jobId, String key, MultipartFile[] files); + + JobCache get(String jobId, String key); + InputStream fetch(String cacheId, String file); } diff --git a/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java index 5c8eb1082..0089e584e 100644 --- a/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java @@ -1,6 +1,10 @@ package com.flowci.core.job.service; +import com.flowci.core.job.dao.JobCacheDao; import com.flowci.core.job.domain.Job; +import com.flowci.core.job.domain.JobCache; +import com.flowci.exception.NotFoundException; +import com.flowci.exception.StatusException; import com.flowci.store.FileManager; import com.flowci.store.Pathable; import com.flowci.store.StringPath; @@ -10,6 +14,9 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Optional; @Log4j2 @Service @@ -17,24 +24,82 @@ public class CacheServiceImpl implements CacheService { private final static Pathable CacheRoot = new StringPath("_cache_"); + @Autowired + private JobCacheDao jobCacheDao; + @Autowired private FileManager fileManager; + @Autowired + private JobService jobService; + @Override - public void put(Job job, String key, MultipartFile[] files) { + public JobCache put(String jobId, String key, MultipartFile[] files) { + Job job = jobService.get(jobId); + + JobCache entity = new JobCache(); + Optional optional = jobCacheDao.findByFlowIdAndKey(job.getFlowId(), key); + if (optional.isPresent()) { + entity = optional.get(); + } + + entity.setJobId(job.getId()); + entity.setFlowId(job.getFlowId()); + entity.setKey(key); + entity.setFiles(new ArrayList<>(files.length)); + for (MultipartFile file : files) { try { - Pathable[] cachePath = getCachePath(job, key); + Pathable[] cachePath = getCachePath(job.getFlowId(), key); fileManager.save(file.getOriginalFilename(), file.getInputStream(), cachePath); + entity.getFiles().add(file.getOriginalFilename()); } catch (IOException e) { log.warn("failed to save file {} for cache {}", file.getName(), key); } } + + return jobCacheDao.save(entity); + } + + @Override + public JobCache get(String jobId, String key) { + Job job = jobService.get(jobId); + + Optional optional = jobCacheDao.findByFlowIdAndKey(job.getFlowId(), key); + if (!optional.isPresent()) { + throw new NotFoundException("Cache not found"); + } + + return optional.get(); + } + + @Override + public InputStream fetch(String cacheId, String file) { + Optional optional = jobCacheDao.findById(cacheId); + if (!optional.isPresent()) { + throw new NotFoundException("Cache not found"); + } + + JobCache cache = optional.get(); + if (!cache.getFiles().contains(file)) { + throw new NotFoundException("file not found"); + } + + Pathable[] cachePath = getCachePath(cache.getFlowId(), cache.getKey()); + if (!fileManager.exist(file, cachePath)) { + throw new NotFoundException("file not found in file store"); + } + + try { + return fileManager.read(file, cachePath); + } catch (IOException e) { + throw new StatusException("unable to read cache file"); + } } - private Pathable[] getCachePath(Job job, String key) { + private Pathable[] getCachePath(String flowId, String key) { return new Pathable[]{ - new StringPath(job.getFlowId()), + new StringPath(flowId), CacheRoot, new StringPath(key) }; From 90ff35b5bd051b018e13edc7f665705a8b758ed8 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Mon, 9 Nov 2020 22:06:27 +0100 Subject: [PATCH 006/128] add os field on job cache --- .../com/flowci/core/agent/controller/AgentAPIController.java | 5 +++-- core/src/main/java/com/flowci/core/job/domain/JobCache.java | 2 ++ .../main/java/com/flowci/core/job/service/CacheService.java | 2 +- .../java/com/flowci/core/job/service/CacheServiceImpl.java | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java b/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java index d48efe86a..193f2190e 100644 --- a/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java +++ b/core/src/main/java/com/flowci/core/agent/controller/AgentAPIController.java @@ -51,15 +51,16 @@ public void upload(@RequestPart("file") MultipartFile file) { } } - @PostMapping("/cache/{jobId}/{key}") + @PostMapping("/cache/{jobId}/{key}/{os}") public void putCache(@PathVariable String jobId, @PathVariable String key, + @PathVariable String os, @RequestParam("files") MultipartFile[] files) { if (files.length == 0) { throw new ArgumentException("the cached files are empty"); } - cacheService.put(jobId, key, files); + cacheService.put(jobId, key, os, files); } @GetMapping("/cache/{jobId}/{key}") diff --git a/core/src/main/java/com/flowci/core/job/domain/JobCache.java b/core/src/main/java/com/flowci/core/job/domain/JobCache.java index 48de7396a..252302557 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobCache.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobCache.java @@ -23,6 +23,8 @@ public class JobCache extends Mongoable { private String key; + private String os; + /** * Full path with b64 encoded based on workspace * ex: a/b/c.zip --> b64xxx.zip diff --git a/core/src/main/java/com/flowci/core/job/service/CacheService.java b/core/src/main/java/com/flowci/core/job/service/CacheService.java index b3b7853ab..74993b173 100644 --- a/core/src/main/java/com/flowci/core/job/service/CacheService.java +++ b/core/src/main/java/com/flowci/core/job/service/CacheService.java @@ -10,7 +10,7 @@ */ public interface CacheService { - JobCache put(String jobId, String key, MultipartFile[] files); + JobCache put(String jobId, String key, String os, MultipartFile[] files); JobCache get(String jobId, String key); diff --git a/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java index 0089e584e..cf9da846a 100644 --- a/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/CacheServiceImpl.java @@ -34,7 +34,7 @@ public class CacheServiceImpl implements CacheService { private JobService jobService; @Override - public JobCache put(String jobId, String key, MultipartFile[] files) { + public JobCache put(String jobId, String key, String os, MultipartFile[] files) { Job job = jobService.get(jobId); JobCache entity = new JobCache(); @@ -46,6 +46,7 @@ public JobCache put(String jobId, String key, MultipartFile[] files) { entity.setJobId(job.getId()); entity.setFlowId(job.getFlowId()); entity.setKey(key); + entity.setOs(os); entity.setFiles(new ArrayList<>(files.length)); for (MultipartFile file : files) { From 1e4a6433dc835c5dcc4426cf01741110281cb66c Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 10 Nov 2020 22:26:20 +0100 Subject: [PATCH 007/128] validation on cache paths --- .../java/com/flowci/tree/yml/YmlBase.java | 11 ++++++ .../main/java/com/flowci/util/FileHelper.java | 38 +++++++++++++++++++ .../com/flowci/util/test/FileHelperTest.java | 29 ++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/tree/src/main/java/com/flowci/tree/yml/YmlBase.java b/tree/src/main/java/com/flowci/tree/yml/YmlBase.java index ebc2d50f0..f27fc49fe 100644 --- a/tree/src/main/java/com/flowci/tree/yml/YmlBase.java +++ b/tree/src/main/java/com/flowci/tree/yml/YmlBase.java @@ -22,6 +22,7 @@ import com.flowci.tree.Cache; import com.flowci.tree.Node; import com.flowci.tree.StepNode; +import com.flowci.util.FileHelper; import com.flowci.util.ObjectsHelper; import lombok.Getter; import lombok.NonNull; @@ -74,6 +75,16 @@ void setCache(T node) { return; } + for (String path : cache.getPaths()) { + if (FileHelper.isStartWithRoot(path)) { + throw new YmlException("Cache path cannot be defined as absolute path"); + } + } + + if (FileHelper.hasOverlapOrDuplicatePath(cache.getPaths())) { + throw new YmlException("Cache paths are overlap or duplicate"); + } + Cache c = new Cache(); c.setKey(cache.getKey()); c.setPaths(cache.getPaths()); diff --git a/util/src/main/java/com/flowci/util/FileHelper.java b/util/src/main/java/com/flowci/util/FileHelper.java index 535a58c83..dee9154fb 100644 --- a/util/src/main/java/com/flowci/util/FileHelper.java +++ b/util/src/main/java/com/flowci/util/FileHelper.java @@ -21,6 +21,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -33,6 +36,41 @@ public abstract class FileHelper { private static final int BufferSize = 4096; + private static final String UnixSeparator = "/"; + + public static boolean isStartWithRoot(String path) { + if (path.startsWith(UnixSeparator)) { + return true; + } + + return path.matches("^[a-zA-Z]:\\\\.*"); + } + + public static boolean hasOverlapOrDuplicatePath(List paths) { + Set set = new HashSet<>(); + + for (String path : paths) { + Path p = Paths.get(path).normalize(); + + // duplication found + if (!set.add(p)) { + return true; + } + + for (Path exist : set) { + if (exist.equals(p)) { + continue; + } + + if (p.startsWith(exist) || exist.startsWith(p)) { + return true; + } + } + } + + return false; + } + public static Path createDirectory(Path dir) throws IOException { try { return Files.createDirectories(dir); diff --git a/util/src/test/java/com/flowci/util/test/FileHelperTest.java b/util/src/test/java/com/flowci/util/test/FileHelperTest.java index 783be7f4f..c9f763284 100644 --- a/util/src/test/java/com/flowci/util/test/FileHelperTest.java +++ b/util/src/test/java/com/flowci/util/test/FileHelperTest.java @@ -18,6 +18,7 @@ package com.flowci.util.test; import com.flowci.util.FileHelper; +import com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -49,4 +50,32 @@ public void should_unzip_file() throws IOException { Assert.assertTrue(Files.exists(destDir)); } + + @Test + public void should_verify_path_is_start_from_root() { + Assert.assertTrue(FileHelper.isStartWithRoot("/ws/abc")); + Assert.assertTrue(FileHelper.isStartWithRoot("C:\\ws\\abc")); + + Assert.assertFalse(FileHelper.isStartWithRoot("./ws/abc")); + Assert.assertFalse(FileHelper.isStartWithRoot(".\\ws\\abc")); + } + + @Test + public void should_verify_path_has_overlap_or_duplication() { + Assert.assertTrue(FileHelper.hasOverlapOrDuplicatePath(Lists.newArrayList( + "./ws/abc", + "./ws" + ))); + + Assert.assertTrue(FileHelper.hasOverlapOrDuplicatePath(Lists.newArrayList( + "./ws/abc", + "./ws/abc/" + ))); + + Assert.assertFalse(FileHelper.hasOverlapOrDuplicatePath(Lists.newArrayList( + "test", + "ws/abc", + "abc/efg" + ))); + } } From 5107d234c54ca6a80754bad88e9869ac725dad2d Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 15 Nov 2020 17:08:32 +0100 Subject: [PATCH 008/128] fix set docker name into cache --- .run/Application - 1.run.xml | 2 +- .../com/flowci/core/job/manager/CmdManagerImpl.java | 2 +- .../main/java/com/flowci/domain/DockerOption.java | 3 ++- tree/src/main/java/com/flowci/tree/Cache.java | 5 ++++- tree/src/main/java/com/flowci/tree/yml/YmlBase.java | 6 +++++- .../src/main/java/com/flowci/util/ObjectsHelper.java | 12 ++++++++++++ 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.run/Application - 1.run.xml b/.run/Application - 1.run.xml index fa31f637e..132e7cb5f 100644 --- a/.run/Application - 1.run.xml +++ b/.run/Application - 1.run.xml @@ -14,7 +14,7 @@ - +