From 189477d3bd3380c2fb5b7ab96978c758b8547b68 Mon Sep 17 00:00:00 2001 From: songtingyu Date: Tue, 11 Apr 2023 16:47:21 +0800 Subject: [PATCH 01/14] init highflip editor module --- highflip-editor/pom.xml | 88 +++++++++++++++++++ .../editor/model/HighFlipEditorException.java | 34 +++++++ .../baidu/highflip/editor/model/Result.java | 66 ++++++++++++++ .../services/HighFlipEditorService.java | 20 +++++ .../highflip/editor/utils/JacksonUtils.java | 50 +++++++++++ .../highflip/editor/vo/LoginRequest.java | 19 ++++ .../RestResponseEntityExceptionHandler.java | 39 ++++++++ .../highflip/editor/web/WebApplication.java | 11 +++ .../editor/web/WebSecurityConfig.java | 5 ++ .../web/controller/ComponentController.java | 28 ++++++ .../web/filter/GlobalExceptionFilter.java | 47 ++++++++++ .../src/main/resources/application.properties | 6 ++ .../src/main/resources/log4j.properties | 14 +++ pom.xml | 1 + 14 files changed, 428 insertions(+) create mode 100644 highflip-editor/pom.xml create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java create mode 100644 highflip-editor/src/main/resources/application.properties create mode 100644 highflip-editor/src/main/resources/log4j.properties diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml new file mode 100644 index 0000000..a765a8a --- /dev/null +++ b/highflip-editor/pom.xml @@ -0,0 +1,88 @@ + + + + highflip + com.baidu + 1.0.0-SNAPSHOT + + 4.0.0 + + highflip-editor + + + 11 + 11 + 2.7.10 + 1.18.22 + 6.11 + 1.43 + + + + + + + + + + + org.springframework.boot + spring-boot-starter-web + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-autoconfigure + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + + org.springframework.security + spring-security-config + 6.0.2 + + + + + org.springframework.boot + spring-boot-starter-log4j + 1.3.8.RELEASE + + + + org.projectlombok + lombok + ${lombok.version} + + + org.testng + testng + ${testng.version} + test + + + org.jmockit + jmockit + ${jmockit.version} + test + + + + + + + \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java new file mode 100644 index 0000000..007dc29 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java @@ -0,0 +1,34 @@ +package com.baidu.highflip.editor.model; + +public class HighFlipEditorException extends RuntimeException { + + private int code; + private String message; + + public HighFlipEditorException(int code, String message) { + super(message); + this.code = code; + this.message = message; + } + + public HighFlipEditorException(String message) { + super(message); + this.code = 400; + this.message = message; + } + + public HighFlipEditorException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + @Override + public String getMessage() { + return message; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java new file mode 100644 index 0000000..aa25331 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java @@ -0,0 +1,66 @@ +package com.baidu.highflip.editor.model; + +/** + * customized response of controller + */ +public class Result { + /** + * similar to http status code + */ + private int code; + private String message; + private Object data; + + public Result() { + } + + public Result(int code, String message, Object data) { + this.code = code; + this.message = message; + this.data = data; + } + + public static Result success() { + Result result = new Result(); + result.code = 200; + return result; + } + + public static Result success(Object data) { + Result result = new Result(); + result.code = 200; + result.data = data; + return result; + } + + public static Result failure(int errorCode, String message) { + Result result = new Result(); + result.code = errorCode; + result.message = message; + return result; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public Object getData() { + return data; + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public void setData(Object data) { + this.data = data; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java new file mode 100644 index 0000000..15fd9ef --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java @@ -0,0 +1,20 @@ +package com.baidu.highflip.editor.services; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HighFlipEditorService { + + private static final Logger LOG = + LoggerFactory.getLogger(HighFlipEditorService.class); + + public HighFlipEditorService() { + + } + + public void login(String username, String password, + String serverIp, int serverPort) { + // TODO create grpc client to login + } + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java new file mode 100644 index 0000000..4e0fbc5 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java @@ -0,0 +1,50 @@ +package com.baidu.highflip.editor.utils; + + +import java.io.IOException; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * utils for serialization and deserialization + * + * @author songtingyu + */ +public class JacksonUtils { + + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + mapper.configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false); + } + + private JacksonUtils() { + + } + + public static String toJson(T obj) { + try { + return mapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new HighFlipEditorException(400, "failed to serialize obj.", e); + } + } + + public static T fromJson(String json, Class valueType) { + if (json == null || json.isEmpty()) { + throw new HighFlipEditorException(400, "json is empty!"); + } + try { + return mapper.readValue(json, valueType); + } catch (IOException e) { + throw new HighFlipEditorException(400, e.getMessage(), e); + } + } +} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java new file mode 100644 index 0000000..c0cf92b --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java @@ -0,0 +1,19 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class LoginRequest { + + private String username; + + private String password; + + private String serverIp; + + private int serverPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java new file mode 100644 index 0000000..f53568e --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java @@ -0,0 +1,39 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; + +/** + * global exception handler + */ +@ControllerAdvice +public class RestResponseEntityExceptionHandler + extends ResponseEntityExceptionHandler { + + @ExceptionHandler(value = {RuntimeException.class}) + protected ResponseEntity handleRuntimeException( + RuntimeException e, WebRequest request) { + Result result = Result.failure(400, e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.INTERNAL_SERVER_ERROR, + request); + } + + @ExceptionHandler(value = {HighFlipEditorException.class}) + protected ResponseEntity handleDsException( + HighFlipEditorException e, WebRequest request) { + Result result = Result.failure(e.getCode(), e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.OK, request); + } +} + + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java new file mode 100644 index 0000000..6d903d2 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java @@ -0,0 +1,11 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class WebApplication { + public static void main(String[] args) { + SpringApplication.run(WebApplication.class, args); + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java new file mode 100644 index 0000000..cc8f409 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.editor.web; + +public class WebSecurityConfig { + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java new file mode 100644 index 0000000..4d60b9f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java @@ -0,0 +1,28 @@ +package com.baidu.highflip.editor.web.controller; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.services.HighFlipEditorService; +import com.baidu.highflip.editor.vo.LoginRequest; + + +@RestController +public class ComponentController { + + @RequestMapping(value = "/login", method = RequestMethod.POST) + public Result login(@RequestBody LoginRequest request) { + HighFlipEditorService highFlipEditorService = + new HighFlipEditorService(); + highFlipEditorService.login(request.getUsername(), + request.getPassword(), + request.getServerIp(), + request.getServerPort()); + return Result.success(); + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java new file mode 100644 index 0000000..57e0e7f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java @@ -0,0 +1,47 @@ +package com.baidu.highflip.editor.web.filter; + +import java.io.IOException; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class GlobalExceptionFilter extends OncePerRequestFilter { + + @Autowired + protected ObjectMapper objectMapper; + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain) + throws ServletException, IOException { + try { + filterChain.doFilter(request, response); + } catch (Exception e) { + response.setContentType("application/json;charset=UTF-8"); + // handle TicketException + if (e instanceof HighFlipEditorException) { + HighFlipEditorException ticketException = (HighFlipEditorException) e; + objectMapper.writeValue(response.getWriter(), + Result.failure( + ticketException.getCode(), + ticketException.getMessage())); + } else { + objectMapper.writeValue(response.getWriter(), + Result.failure(400, e.getMessage())); + } + } + } +} + diff --git a/highflip-editor/src/main/resources/application.properties b/highflip-editor/src/main/resources/application.properties new file mode 100644 index 0000000..2d3e216 --- /dev/null +++ b/highflip-editor/src/main/resources/application.properties @@ -0,0 +1,6 @@ +server.port=8090 + +server.servlet.session.timeout=30m +server.tomcat.connection-timeout=600000 +# httpOnly +server.servlet.session.cookie.http-only=true diff --git a/highflip-editor/src/main/resources/log4j.properties b/highflip-editor/src/main/resources/log4j.properties new file mode 100644 index 0000000..63e913e --- /dev/null +++ b/highflip-editor/src/main/resources/log4j.properties @@ -0,0 +1,14 @@ +# LOG4J configuration +log4j.rootLogger=info, console, file + +# Console Logger +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d [%p] %l %x - %m%n + +# File Logger +log4j.appender.file=org.apache.log4j.DailyRollingFileAppender +log4j.appender.file.File=./logs/ds-agent.log +log4j.appender.file.DatePattern='-'yyyy-MM-dd'.log' +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d [%p] %l %x - %m%n \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2d5fe7b..eb41ecc 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ highflip-server highflip-doc highflip-clients + highflip-editor From 7190b02dd718c5289a6e9542a0bc4a74e1608af0 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Sun, 23 Apr 2023 16:17:41 +0800 Subject: [PATCH 02/14] add access log aop of web api --- highflip-editor/pom.xml | 17 ++++ .../web/annotation/IgnoreRequestBody.java | 17 ++++ .../highflip/editor/web/aop/AccessLogAop.java | 91 +++++++++++++++++++ .../editor/web/aop/RequestAspect.java | 13 +++ .../web/controller/ComponentController.java | 7 +- .../src/main/resources/application.properties | 2 + 6 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index a765a8a..b81796e 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -18,6 +18,7 @@ 1.18.22 6.11 1.43 + 31.1-jre @@ -38,6 +39,17 @@ + + org.springframework.boot + spring-boot-starter-aop + ${spring.boot.version} + + + ch.qos.logback + logback-classic + + + org.springframework.boot spring-boot-autoconfigure @@ -80,6 +92,11 @@ ${jmockit.version} test + + com.google.guava + guava + ${guava.version} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java new file mode 100644 index 0000000..880ddeb --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.web.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * ignore request body when print access log. + * use in {@link com.baidu.dragonshare.web.aop.AccessLogAop} + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface IgnoreRequestBody { +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java new file mode 100644 index 0000000..8179e89 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java @@ -0,0 +1,91 @@ +package com.baidu.highflip.editor.web.aop; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import com.baidu.highflip.editor.utils.JacksonUtils; +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +@Aspect +@Component +@Order(0) +public class AccessLogAop extends RequestAspect { + + private static final String LOG_CONTENT = + "RequestMethod = {}, RequestUri = {}, RemoteAddress = {}, " + + "RequestParams = `{}`, RequestBody = {}"; + // when request method is post or put, log request body + private static final Set REQUEST_BODY_METHOD = + ImmutableSet.of(RequestMethod.POST.name(), + RequestMethod.PUT.name()); + + private Map loggerHolder = Maps.newConcurrentMap(); + + @Before(CONTROLLER_METHOD_NAME) + public void before(JoinPoint joinPoint) { + Class targetClass = joinPoint.getTarget().getClass(); + + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + + Object[] args = joinPoint.getArgs(); + + log(targetClass, method, args); + } + + private void log(Class targetClass, Method method, Object[] args) { + // 1. get logger + Logger logger = loggerHolder.putIfAbsent( + targetClass, LoggerFactory.getLogger(targetClass)); + logger = logger == null ? loggerHolder.get(targetClass) : logger; + + // 2. get request + ServletRequestAttributes requestAttributes = + (ServletRequestAttributes) RequestContextHolder + .currentRequestAttributes(); + HttpServletRequest request = requestAttributes.getRequest(); + + // 3. get log content + // request body is a method arg, that has @RequestBody annotation. + String remoteAddress = request.getRemoteAddr(); + String url = request.getRequestURI(); + String requestParams = request.getQueryString(); + Object body = null; + if (REQUEST_BODY_METHOD.stream() + .anyMatch(m -> m.equals(request.getMethod()))) { + if (method.getAnnotation(IgnoreRequestBody.class) == null) { + Annotation[][] annotations = method.getParameterAnnotations(); + for (int i = 0; i < annotations.length; i++) { + for (int j = 0; j < annotations[i].length; j++) { + if (annotations[i][j] instanceof RequestBody) { + body = args[i]; + } + } + } + } + } + + logger.info(LOG_CONTENT, + targetClass.getSimpleName() + "." + method.getName(), + url, remoteAddress, requestParams, JacksonUtils.toJson(body)); + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java new file mode 100644 index 0000000..85ab08f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.web.aop; + +import org.aspectj.lang.annotation.Pointcut; + +public class RequestAspect { + + protected static final String CONTROLLER_METHOD_NAME = "controllerMethod()"; + + @Pointcut("@annotation(org.springframework.web.bind.annotation.RequestMapping)") + protected void controllerMethod() { + + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java index 4d60b9f..ac1921a 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java @@ -1,7 +1,5 @@ package com.baidu.highflip.editor.web.controller; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -10,11 +8,12 @@ import com.baidu.highflip.editor.model.Result; import com.baidu.highflip.editor.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; - +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; @RestController public class ComponentController { + @IgnoreRequestBody @RequestMapping(value = "/login", method = RequestMethod.POST) public Result login(@RequestBody LoginRequest request) { HighFlipEditorService highFlipEditorService = @@ -23,6 +22,6 @@ public Result login(@RequestBody LoginRequest request) { request.getPassword(), request.getServerIp(), request.getServerPort()); - return Result.success(); + return Result.success("login successfully"); } } diff --git a/highflip-editor/src/main/resources/application.properties b/highflip-editor/src/main/resources/application.properties index 2d3e216..b9cb39f 100644 --- a/highflip-editor/src/main/resources/application.properties +++ b/highflip-editor/src/main/resources/application.properties @@ -4,3 +4,5 @@ server.servlet.session.timeout=30m server.tomcat.connection-timeout=600000 # httpOnly server.servlet.session.cookie.http-only=true + +spring.web.resources.static-locations=classpath:/ui/ \ No newline at end of file From 04e2c9f78479415dae3da3a0dfdb2d1eda40ae14 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 24 Apr 2023 16:26:18 +0800 Subject: [PATCH 03/14] list operators --- highflip-clients/highflip-sdk/pom.xml | 2 +- .../baidu/highflip/client/HighFlipClient.java | 6 +++ highflip-editor/pom.xml | 19 ++++--- .../baidu/highflip/editor/model/Result.java | 14 +++--- .../services/HighFlipEditorService.java | 20 -------- .../editor/vo/ListOperatorsRequest.java | 16 ++++++ .../web/annotation/IgnoreRequestBody.java | 2 +- ...ontroller.java => OperatorController.java} | 30 +++++++++-- .../web/services/HighFlipEditorService.java | 50 +++++++++++++++++++ .../src/main/resources/log4j.properties | 2 +- 10 files changed, 120 insertions(+), 41 deletions(-) delete mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java rename highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/{ComponentController.java => OperatorController.java} (50%) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index d982e95..b44148a 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -20,7 +20,7 @@ com.baidu highflip-proto - 1.0.0-SNAPSHOT + ${project.version} org.projectlombok diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 9a57d8c..8a68fd9 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -42,6 +42,12 @@ public HighFlipClient() { this.blockingStub = null; } + public static HighFlipClient newHighFlipClient(String target) { + HighFlipClient highFlipClient = new HighFlipClient(); + highFlipClient.connect(target); + return highFlipClient; + } + public void connect(String target) { close(); diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index b81796e..7f2e216 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -19,14 +19,20 @@ 6.11 1.43 31.1-jre + 6.0.2 - - - - - + + com.baidu + highflip-sdk + ${project.version} + + + com.baidu + highflip-proto + ${project.version} + org.springframework.boot @@ -65,7 +71,7 @@ org.springframework.security spring-security-config - 6.0.2 + ${spring.security.version} @@ -97,7 +103,6 @@ guava ${guava.version} - diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java index aa25331..dba016a 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java @@ -3,18 +3,18 @@ /** * customized response of controller */ -public class Result { +public class Result { /** * similar to http status code */ private int code; private String message; - private Object data; + private T data; public Result() { } - public Result(int code, String message, Object data) { + public Result(int code, String message, T data) { this.code = code; this.message = message; this.data = data; @@ -26,8 +26,8 @@ public static Result success() { return result; } - public static Result success(Object data) { - Result result = new Result(); + public static Result success(T data) { + Result result = new Result(); result.code = 200; result.data = data; return result; @@ -48,7 +48,7 @@ public String getMessage() { return message; } - public Object getData() { + public T getData() { return data; } @@ -60,7 +60,7 @@ public void setMessage(String message) { this.message = message; } - public void setData(Object data) { + public void setData(T data) { this.data = data; } } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java deleted file mode 100644 index 15fd9ef..0000000 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.baidu.highflip.editor.services; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class HighFlipEditorService { - - private static final Logger LOG = - LoggerFactory.getLogger(HighFlipEditorService.class); - - public HighFlipEditorService() { - - } - - public void login(String username, String password, - String serverIp, int serverPort) { - // TODO create grpc client to login - } - -} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java new file mode 100644 index 0000000..e813154 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java @@ -0,0 +1,16 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListOperatorsRequest { + + String highFlipServerIp; + + int highFlipServerPort; + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java index 880ddeb..77c9a77 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java @@ -8,7 +8,7 @@ /** * ignore request body when print access log. - * use in {@link com.baidu.dragonshare.web.aop.AccessLogAop} + * use in {@link com.baidu.highflip.editor.web.aop.AccessLogAop} */ @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java similarity index 50% rename from highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java rename to highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java index ac1921a..3744f2e 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java @@ -1,27 +1,49 @@ package com.baidu.highflip.editor.web.controller; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import com.baidu.highflip.editor.model.Result; -import com.baidu.highflip.editor.services.HighFlipEditorService; +import com.baidu.highflip.editor.vo.ListOperatorsRequest; +import com.baidu.highflip.editor.web.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; +import highflip.v1.Highflip; + @RestController -public class ComponentController { +public class OperatorController { + + @Autowired + HighFlipEditorService highFlipEditorService; @IgnoreRequestBody @RequestMapping(value = "/login", method = RequestMethod.POST) public Result login(@RequestBody LoginRequest request) { - HighFlipEditorService highFlipEditorService = - new HighFlipEditorService(); highFlipEditorService.login(request.getUsername(), request.getPassword(), request.getServerIp(), request.getServerPort()); return Result.success("login successfully"); } + + @RequestMapping(method = RequestMethod.GET) + public Result listOperators(ListOperatorsRequest request) { + + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List operators = + highFlipEditorService.getOperators( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + + return Result.success(operators); + } } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java new file mode 100644 index 0000000..3ac9764 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -0,0 +1,50 @@ +package com.baidu.highflip.editor.web.services; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.baidu.highflip.client.HighFlipClient; + +import highflip.v1.Highflip; + +@Service +public class HighFlipEditorService { + + private static final Logger LOG = + LoggerFactory.getLogger(HighFlipEditorService.class); + + public HighFlipEditorService() { + + } + + public void login(String username, String password, + String serverIp, int serverPort) { + // TODO create grpc client to login + } + + + public List getOperators(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo ="grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + // rpc call + Iterable operatorIds = highFlipClient.listOperators(0, 1000); + List operators = new ArrayList<>(); + for(String operatorId : operatorIds) { + Highflip.OperatorGetResponse operator = + highFlipClient.getOperator(operatorId); + operators.add(operator); + } + return operators; + } + +} diff --git a/highflip-editor/src/main/resources/log4j.properties b/highflip-editor/src/main/resources/log4j.properties index 63e913e..ebec8d1 100644 --- a/highflip-editor/src/main/resources/log4j.properties +++ b/highflip-editor/src/main/resources/log4j.properties @@ -8,7 +8,7 @@ log4j.appender.console.layout.ConversionPattern=%d [%p] %l %x - %m%n # File Logger log4j.appender.file=org.apache.log4j.DailyRollingFileAppender -log4j.appender.file.File=./logs/ds-agent.log +log4j.appender.file.File=./logs/highflip-editor.log log4j.appender.file.DatePattern='-'yyyy-MM-dd'.log' log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d [%p] %l %x - %m%n \ No newline at end of file From c166d89fa952ee072f381508808039cb9cc0226a Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Tue, 25 Apr 2023 11:24:33 +0800 Subject: [PATCH 04/14] add web api of list partners,data --- .../highflip/editor/utils/StringUtils.java | 21 +++++++++ .../highflip/editor/vo/ListDataRequest.java | 17 +++++++ .../editor/vo/ListPartnersRequest.java | 15 ++++++ ...ontroller.java => HighFlipController.java} | 33 +++++++++++-- .../web/services/HighFlipEditorService.java | 46 ++++++++++++++++++- .../server/rpc/v1/HighFlipRpcService.java | 27 +++++++++++ 6 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java rename highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/{OperatorController.java => HighFlipController.java} (55%) diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java new file mode 100644 index 0000000..e1ea1b1 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java @@ -0,0 +1,21 @@ +package com.baidu.highflip.editor.utils; + +public class StringUtils { + + public static boolean isNotBlank(final CharSequence cs) { + return !isBlank(cs); + } + + public static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (Character.isWhitespace(cs.charAt(i)) == false) { + return false; + } + } + return true; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java new file mode 100644 index 0000000..40a49a9 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListDataRequest { + + String partnerId; + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java new file mode 100644 index 0000000..a240873 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java @@ -0,0 +1,15 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListPartnersRequest { + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java similarity index 55% rename from highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java rename to highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java index 3744f2e..7a7b0c3 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -9,7 +9,9 @@ import org.springframework.web.bind.annotation.RestController; import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.vo.ListDataRequest; import com.baidu.highflip.editor.vo.ListOperatorsRequest; +import com.baidu.highflip.editor.vo.ListPartnersRequest; import com.baidu.highflip.editor.web.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; @@ -17,7 +19,7 @@ import highflip.v1.Highflip; @RestController -public class OperatorController { +public class HighFlipController { @Autowired HighFlipEditorService highFlipEditorService; @@ -32,9 +34,8 @@ public Result login(@RequestBody LoginRequest request) { return Result.success("login successfully"); } - @RequestMapping(method = RequestMethod.GET) + @RequestMapping(value = "operators", method = RequestMethod.GET) public Result listOperators(ListOperatorsRequest request) { - // TODO: how to transfer username and password String username = "username"; String password = "password"; @@ -46,4 +47,30 @@ public Result listOperators(ListOperatorsRequest request) { return Result.success(operators); } + + @RequestMapping(value = "partners", method = RequestMethod.GET) + public Result listPartners(ListPartnersRequest request){ + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List partners = + highFlipEditorService.getPartners( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + return Result.success(partners); + } + + @RequestMapping(value = "data", method = RequestMethod.GET) + public Result listData(ListDataRequest request) { + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List dataGetResponseList = + highFlipEditorService.getDataList( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password, request.getPartnerId()); + return Result.success(dataGetResponseList); + } } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java index 3ac9764..5a9e4fd 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -8,6 +8,7 @@ import org.springframework.stereotype.Service; import com.baidu.highflip.client.HighFlipClient; +import com.baidu.highflip.editor.utils.StringUtils; import highflip.v1.Highflip; @@ -26,7 +27,6 @@ public void login(String username, String password, // TODO create grpc client to login } - public List getOperators(String ip, int port, String username, @@ -47,4 +47,48 @@ public List getOperators(String ip, return operators; } + public List getPartners(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable partnerIds = highFlipClient.listPartners(0, 1000); + List partners = new ArrayList<>(); + for (String partnerId : partnerIds) { + Highflip.PartnerGetResponse partner = + highFlipClient.getPartner(partnerId); + partners.add(partner); + } + return partners; + } + + public List getDataList(String ip, + int port, + String username, + String password, + String targetPartnerId) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable dataIds = highFlipClient.listData(0, 1000); + List dataList = new ArrayList<>(); + for (String dataId : dataIds) { + Highflip.DataGetResponse data = highFlipClient.getData(dataId); + if (StringUtils.isBlank(targetPartnerId) || + (StringUtils.isNotBlank(data.getPartyId()) && + data.getPartyId().equals(targetPartnerId))) { + dataList.add(data); + } + } + return dataList; + } + } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index aaa80e2..e6de087 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -84,6 +84,7 @@ public void listConfig(Highflip.ConfigListRequest request, * @param request * @param responseObserver */ + @Override public void getConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -105,6 +106,7 @@ public void getConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void setConfig(Highflip.ConfigSetRequest request, StreamObserver responseObserver) { @@ -113,6 +115,7 @@ public void setConfig(Highflip.ConfigSetRequest request, returnVoid(responseObserver); } + @Override public void deleteConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -129,6 +132,7 @@ public void deleteConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void getPlatform(Highflip.Void request, StreamObserver responseObserver) { @@ -147,6 +151,7 @@ public void getPlatform(Highflip.Void request, * @param request * @param responseObserver */ + @Override public void matchPlatform(Highflip.PlatformMatchRequest request, StreamObserver responseObserver) { @@ -189,6 +194,7 @@ public void createJob(Highflip.JobCreateRequest request, * @param request * @param responseObserver */ + @Override public void getJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); @@ -210,6 +216,7 @@ public void getJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void checkJob(Highflip.JobId request, StreamObserver responseObserver) { @@ -227,6 +234,7 @@ public void checkJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void deleteJob(Highflip.JobId request, StreamObserver responseObserver) { @@ -239,6 +247,7 @@ public void deleteJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void listJob(Highflip.JobListRequest request, StreamObserver responseObserver) { @@ -259,6 +268,7 @@ public void listJob(Highflip.JobListRequest request, * @param request * @param responseObserver */ + @Override public void controlJob(Highflip.JobControlRequest request, StreamObserver responseObserver) { @@ -270,6 +280,7 @@ public void controlJob(Highflip.JobControlRequest request, returnVoid(responseObserver); } + @Override public void getJobLog(Highflip.JobLogRequest request, StreamObserver responseObserver) { @@ -282,6 +293,7 @@ public void getJobLog(Highflip.JobLogRequest request, * @param request * @param responseObserver */ + @Override public void listTask(Highflip.TaskListRequest request, StreamObserver responseObserver) { @@ -300,6 +312,7 @@ public void listTask(Highflip.TaskListRequest request, * @param request * @param responseObserver */ + @Override public void getTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -320,6 +333,7 @@ public void getTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void checkTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -340,6 +354,7 @@ public void checkTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void controlTask(Highflip.TaskControlRequest request, StreamObserver responseObserver) { @@ -355,6 +370,7 @@ public void controlTask(Highflip.TaskControlRequest request, * @param request * @param responseObserver */ + @Override public void getTaskLog(Highflip.TaskLogRequest request, StreamObserver responseObserver) { @@ -368,6 +384,7 @@ public void getTaskLog(Highflip.TaskLogRequest request, * @param request * @param responseObserver */ + @Override public void listData(Highflip.DataListRequest request, StreamObserver responseObserver) { @@ -383,6 +400,7 @@ public void listData(Highflip.DataListRequest request, * @param request * @param responseObserver */ + @Override public void getData(Highflip.DataId request, StreamObserver responseObserver) { @@ -413,6 +431,7 @@ public void getData(Highflip.DataId request, * @param responseObserver * @return */ + @Override public StreamObserver pushData( StreamObserver responseObserver) { @@ -496,6 +515,7 @@ public void onCompleted() { * @param request * @param responseObserver */ + @Override public void pullData(Highflip.DataPullRequest request, StreamObserver responseObserver) { @@ -537,6 +557,7 @@ public void pullData(Highflip.DataPullRequest request, * @param request * @param responseObserver */ + @Override public void deleteData(Highflip.DataId request, StreamObserver responseObserver) { @@ -552,6 +573,7 @@ public void deleteData(Highflip.DataId request, * @param request * @param responseObserver */ + @Override public void listOperator(Highflip.OperatorListRequest request, StreamObserver responseObserver) { Iterator response = Streams @@ -569,6 +591,7 @@ public void listOperator(Highflip.OperatorListRequest request, * @param request * @param responseObserver */ + @Override public void getOperator(Highflip.OperatorId request, StreamObserver responseObserver) { @@ -594,6 +617,7 @@ public void getOperator(Highflip.OperatorId request, * @param request * @param responseObserver */ + @Override public void createPartner(Highflip.PartnerCreateRequest request, StreamObserver responseObserver) { @@ -614,6 +638,7 @@ public void createPartner(Highflip.PartnerCreateRequest request, * @param request * @param responseObserver */ + @Override public void getPartner(Highflip.PartnerId request, StreamObserver responseObserver) { @@ -635,6 +660,7 @@ public void getPartner(Highflip.PartnerId request, * @param request * @param responseObserver */ + @Override public void listPartner(Highflip.PartnerListRequest request, StreamObserver responseObserver) { @@ -653,6 +679,7 @@ public void listPartner(Highflip.PartnerListRequest request, * @param request * @param responseObserver */ + @Override public void controlPartner(Highflip.PartnerControlRequest request, StreamObserver responseObserver) { From 595e7e33ec94fd18ee036c55cccf9ea76d711c29 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 27 Apr 2023 11:27:24 +0800 Subject: [PATCH 05/14] enable job DAG save and retrieve --- .../highflip/editor/vo/SaveJobRequest.java | 13 +++++++++ .../web/controller/HighFlipController.java | 29 ++++++++++++++++++- .../web/services/HighFlipEditorService.java | 21 +++++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java new file mode 100644 index 0000000..e25ec31 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SaveJobRequest { + + String json; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java index 7a7b0c3..72169cc 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -2,6 +2,8 @@ import java.util.List; +import javax.servlet.http.HttpServletRequest; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -9,9 +11,11 @@ import org.springframework.web.bind.annotation.RestController; import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.utils.StringUtils; import com.baidu.highflip.editor.vo.ListDataRequest; import com.baidu.highflip.editor.vo.ListOperatorsRequest; import com.baidu.highflip.editor.vo.ListPartnersRequest; +import com.baidu.highflip.editor.vo.SaveJobRequest; import com.baidu.highflip.editor.web.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; @@ -19,13 +23,14 @@ import highflip.v1.Highflip; @RestController +@RequestMapping(value = "highflip") public class HighFlipController { @Autowired HighFlipEditorService highFlipEditorService; @IgnoreRequestBody - @RequestMapping(value = "/login", method = RequestMethod.POST) + @RequestMapping(value = "login", method = RequestMethod.POST) public Result login(@RequestBody LoginRequest request) { highFlipEditorService.login(request.getUsername(), request.getPassword(), @@ -73,4 +78,26 @@ public Result listData(ListDataRequest request) { username, password, request.getPartnerId()); return Result.success(dataGetResponseList); } + + @RequestMapping(value = "job", method = RequestMethod.POST) + public Result saveJob(HttpServletRequest request, + @RequestBody SaveJobRequest saveJobRequest) { + String token = request.getHeader("TOKEN"); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + highFlipEditorService.saveJob(token, saveJobRequest.getJson()); + return Result.success(); + } + + @RequestMapping(value = "job", method = RequestMethod.GET) + public Result getJob(HttpServletRequest request) { + String token = request.getHeader("TOKEN"); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + String jobDag = highFlipEditorService.getJob(token); + return Result.success(jobDag); + } + } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java index 5a9e4fd..9e274dc 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -2,12 +2,14 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.baidu.highflip.client.HighFlipClient; +import com.baidu.highflip.editor.model.HighFlipEditorException; import com.baidu.highflip.editor.utils.StringUtils; import highflip.v1.Highflip; @@ -18,8 +20,10 @@ public class HighFlipEditorService { private static final Logger LOG = LoggerFactory.getLogger(HighFlipEditorService.class); - public HighFlipEditorService() { + private ConcurrentHashMap jobs; + public HighFlipEditorService() { + jobs = new ConcurrentHashMap<>(); } public void login(String username, String password, @@ -91,4 +95,19 @@ public List getDataList(String ip, return dataList; } + // save job + public void saveJob(String key, String jsonOfJob) { + if (jobs == null) { + jobs = new ConcurrentHashMap<>(); + } + jobs.put(key, jsonOfJob); + } + + public String getJob(String key){ + if(StringUtils.isBlank(key)){ + throw new HighFlipEditorException("Key is blank"); + } + return jobs.get(key); + } + } From 4d5697de9d08030d1d8fe8f6d1ae4a2ef5c61c0c Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 27 Apr 2023 14:43:11 +0800 Subject: [PATCH 06/14] modify the header field --- .../highflip/editor/web/controller/HighFlipController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java index 72169cc..3300b95 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -26,6 +26,8 @@ @RequestMapping(value = "highflip") public class HighFlipController { + private static String TOKEN = "Token"; + @Autowired HighFlipEditorService highFlipEditorService; @@ -82,7 +84,7 @@ public Result listData(ListDataRequest request) { @RequestMapping(value = "job", method = RequestMethod.POST) public Result saveJob(HttpServletRequest request, @RequestBody SaveJobRequest saveJobRequest) { - String token = request.getHeader("TOKEN"); + String token = request.getHeader(TOKEN); if (StringUtils.isBlank(token)) { return Result.failure(400, "The required header TOKEN has not found"); } @@ -92,7 +94,7 @@ public Result saveJob(HttpServletRequest request, @RequestMapping(value = "job", method = RequestMethod.GET) public Result getJob(HttpServletRequest request) { - String token = request.getHeader("TOKEN"); + String token = request.getHeader(TOKEN); if (StringUtils.isBlank(token)) { return Result.failure(400, "The required header TOKEN has not found"); } From c93a4246b1606fbb4762f7481acc59e48e453f50 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Fri, 12 May 2023 17:42:10 +0800 Subject: [PATCH 07/14] 1.test highflip dag and construct dag 2.add the convert of ListProto to java List and MapProto to java Map 3.modify the uuid strategy 4.register highflip runtime oject 5.complete HighFlipService 6.add highflip.adaptor.service.properties --- .../baidu/highflip/client/HighFlipClient.java | 7 + .../highflip/core/adaptor/ServiceAdaptor.java | 10 + .../core/common/AdaptorPropsList.java | 12 + .../core/common/InstanceNameList.java | 3 + .../highflip/core/engine/HighFlipRuntime.java | 2 + .../baidu/highflip/core/entity/dag/Graph.java | 4 +- .../baidu/highflip/core/entity/dag/Node.java | 8 +- .../baidu/highflip/core/entity/dag/Party.java | 1 + .../core/entity/dag/codec/TypeValue.java | 93 ++++- .../dag/common/NamedAttributeObject.java | 7 +- .../core/entity/dag/common/NodeInputRef.java | 6 +- .../core/entity/dag/common/NodeOutputRef.java | 6 +- .../highflip/core/entity/runtime/Data.java | 4 +- .../highflip/core/entity/runtime/Job.java | 4 +- .../core/entity/runtime/Operator.java | 2 +- .../highflip/core/entity/runtime/Partner.java | 2 +- .../core/entity/runtime/Platform.java | 2 +- .../highflip/core/entity/runtime/Task.java | 2 +- .../core/utils/CustomUuidGenerator.java | 24 ++ .../highflip/core/utils/SerializerUtils.java | 64 ++++ .../baidu/highflip/core/entity/TestDag.java | 360 +++++++++++++++++- .../src/main/proto/highflip-meta.proto | 4 + highflip-proto/src/main/proto/highflip.proto | 12 + .../server/adaptor/DefaultAdaptor.java | 9 + .../impl/ConfigurableServiceAdaptor.java | 42 ++ .../highflip/server/config/AdaptorConfig.java | 7 + .../server/engine/HighFlipEngine.java | 2 - .../engine/component/HighFlipContext.java | 6 + .../engine/component/HighFlipRuntime.java | 6 + .../server/rpc/v1/HighFlipRpcService.java | 21 +- .../adaptor/highflip.service.properties | 3 + .../src/main/resources/application.properties | 6 +- .../server/engine/HighFlipEngineTest.java | 5 + 33 files changed, 720 insertions(+), 26 deletions(-) create mode 100644 highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java create mode 100644 highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java create mode 100644 highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java create mode 100644 highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java create mode 100644 highflip-server/src/main/resources/adaptor/highflip.service.properties create mode 100644 highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 8a68fd9..8a4694e 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -584,4 +584,11 @@ public String createPartner(String name, String description){ return response.getPartnerId(); } + + public Highflip.GetServiceConfigResponse getServiceConfig() { + final Highflip.GetServiceConfigResponse serviceConfig = + getBlockingStub().getServiceConfig( + Highflip.Void.getDefaultInstance()); + return serviceConfig; + } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java new file mode 100644 index 0000000..104cfc1 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java @@ -0,0 +1,10 @@ +package com.baidu.highflip.core.adaptor; + +public interface ServiceAdaptor { + + String getUrl(); + + String getPartyId(); + + String getRole(); +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java index 70522ff..f156486 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java @@ -25,4 +25,16 @@ public class AdaptorPropsList { public static final String PROPS_HIGHFLIP_ADAPTOR_PLATFORM_VERSION_DEFAULT = "0.0.0"; + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL = "highflip.adaptor.service.url"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT = "http://127.0.0.1:9380"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID = "highflip.adaptor.service.party.id"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT = "9999"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE = "highflip.adaptor.service.role"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT = "guest"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java index ef166ec..1a34c7f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java @@ -25,4 +25,7 @@ public class InstanceNameList { public static final String HIGHFLIP_ADAPTOR_PARTNER = "highflip.adaptor.partner"; public static final String HIGHFLIP_ADAPTOR_USER = "highflip.adaptor.user"; + + public static final String HIGHFLIP_ADAPTOR_SERVICE = "highflip.adaptor.service"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java index 26f069a..fc4ef77 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java @@ -26,4 +26,6 @@ public interface HighFlipRuntime { User getUser(String userId); Operator getOperator(String operatorId); + + Data registerData(Data data); } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java index 2224ee4..73eacfe 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java @@ -3,6 +3,8 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; import lombok.NoArgsConstructor; @@ -96,7 +98,7 @@ public Iterable listParties() { return getParties().keySet(); } - + @JsonIgnore protected void setNodeCategory() { calcMiddleNodes(); calcOutputNodes(); diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java index 2089128..78d73c0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java @@ -6,16 +6,19 @@ import com.baidu.highflip.core.entity.dag.common.NodeInputRef; import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; -import javax.persistence.Transient; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.springframework.data.annotation.Transient; + @Data public class Node extends NamedAttributeObject implements Serializable { @@ -25,7 +28,7 @@ public class Node extends NamedAttributeObject implements Serializable { String description; - @Transient + @JsonIgnore Graph graph; Category category; @@ -108,6 +111,7 @@ public Node getInputNode(String name) { getInputs().get(name).getFromNode()); } + @JsonIgnore public List getInputNodes() { return getInputs() .values() diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java index c96e552..5d096f0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java @@ -3,6 +3,7 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; + import highflip.HighflipMeta; import lombok.Data; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java index b5227f4..1a43401 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java @@ -1,5 +1,12 @@ package com.baidu.highflip.core.entity.dag.codec; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.protobuf.ByteString; + import highflip.HighflipMeta; public class TypeValue { @@ -17,11 +24,76 @@ public static Object fromProto(HighflipMeta.TypedValueProto value) { return value.getValue().getFloat(); } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.STRING.getNumber()) { return value.getValue().getString(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.BYTES.getNumber()) { + return value.getValue().getBytes(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.LIST.getNumber()) { + return listProtoConvert2List(value.getValue().getList()); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.MAP.getNumber()) { + return mapProtoConvert2Map(value.getValue().getMap()); } else { throw new IllegalArgumentException(); } } + private static List listProtoConvert2List( + HighflipMeta.ListProto listProto) { + if (listProto == null) { + return new ArrayList<>(); + } + List result = new ArrayList<>(); + List listProtoList = listProto.getListList(); + for (HighflipMeta.ValueProto valueProto : listProtoList) { + result.add(convertValueProto(valueProto)); + } + return result; + } + + private static Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto) { + if (mapProto == null) { + return new HashMap<>(); + } + Map map = mapProto.getMapMap(); + Map result = new HashMap<>(); + for (Map.Entry entry : + map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private static Object convertValueProto(HighflipMeta.ValueProto value) { + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + public static HighflipMeta.TypedValueProto toProto(Object object) { HighflipMeta.TypedValueProto.Builder builder = HighflipMeta.TypedValueProto .newBuilder(); @@ -56,14 +128,31 @@ public static HighflipMeta.TypedValueProto toProto(Object object) { .newBuilder() .setDouble((Double) object) .build()); - } else { + } else if (object instanceof String) { builder.setType(HighflipMeta.TypedValueProto.TypeProto.STRING); builder.setValue(HighflipMeta.ValueProto .newBuilder() .setString(object.toString()) .build()); + } else if (object instanceof ByteString) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.BYTES); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setBytes((ByteString) object) + .build()); + } else if (object instanceof HighflipMeta.ListProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.LIST); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setList((HighflipMeta.ListProto) object) + .build()); + } else if (object instanceof HighflipMeta.MapProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.MAP); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setMap((HighflipMeta.MapProto) object) + .build()); } - return builder.build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java index e0f1377..0b9f14a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java @@ -2,15 +2,18 @@ import lombok.Data; -import javax.persistence.Transient; import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.springframework.data.annotation.Transient; + +import com.fasterxml.jackson.annotation.JsonIgnore; + @Data public class NamedAttributeObject implements Comparable { - @Transient + @JsonIgnore List parents = List.of(); String name; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java index a6d9a63..d6b2eef 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java @@ -19,12 +19,15 @@ public class NodeInputRef { String fromOutput; + String value; + public static HighflipMeta.NodeInputProto toProto(com.baidu.highflip.core.entity.dag.common.NodeInputRef ref) { HighflipMeta.NodeInputProto.Builder builder = HighflipMeta.NodeInputProto .newBuilder() .setName(ref.getName()) .setFromNode(ref.getFromNode()) - .setFromOutput(ref.getFromOutput()); + .setFromOutput(ref.getFromOutput()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -37,6 +40,7 @@ public static com.baidu.highflip.core.entity.dag.common.NodeInputRef fromProto(H .setDescription(proto.getDescription()) .setFromNode(proto.getFromNode()) .setFromOutput(proto.getFromOutput()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java index e9fd012..270288a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java @@ -15,10 +15,13 @@ public class NodeOutputRef { String description; + String value; + public static HighflipMeta.NodeOutputProto toProto(NodeOutputRef ref) { HighflipMeta.NodeOutputProto.Builder builder = HighflipMeta.NodeOutputProto .newBuilder() - .setName(ref.getName()); + .setName(ref.getName()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -29,6 +32,7 @@ public static NodeOutputRef fromProto(HighflipMeta.NodeOutputProto proto) { .builder() .setName(proto.getName()) .setDescription(proto.getDescription()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index 8086c0a..9ce80a0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -32,8 +32,8 @@ public class Data { @Id - @Column(name = "data_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @Column(name = "data_id", length = 72) + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String dataId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java index 9d506fc..aaf9d3f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java @@ -37,7 +37,7 @@ public class Job { @Id @Column(name = "job_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String jobId; @@ -59,7 +59,7 @@ public class Job { DateTime finishTime; @Type(type = "json") - @Column(name = "graph") + @Column(name = "graph", length = 10240) Graph graph; @Column(name = "status") diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java index d8e9054..1af34ec 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java @@ -28,7 +28,7 @@ public class Operator { @Id @Column(name = "operator_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String operatorId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java index 41f25f2..6ab0f5a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java @@ -27,7 +27,7 @@ public class Partner { @Id @Column(name = "partner_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String partnerId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java index db5eaf6..8d7936a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java @@ -33,7 +33,7 @@ public class Platform { @Id @Column(name = "platform_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String platformId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java index 07c3e35..5feae39 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java @@ -31,7 +31,7 @@ public class Task { @Id @Column(name = "task_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String taskid; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java new file mode 100644 index 0000000..4e9ea66 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java @@ -0,0 +1,24 @@ +package com.baidu.highflip.core.utils; + +import java.io.Serializable; + +import org.hibernate.HibernateException; +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.id.UUIDGenerator; + +public class CustomUuidGenerator extends UUIDGenerator { + + @Override + public Serializable generate(SharedSessionContractImplementor s, Object obj) + throws HibernateException { + Serializable id = s.getEntityPersister(null, obj).getClassMetadata() + .getIdentifier(obj, s); + + if (id != null && String.valueOf(id).length() > 0) { + return id; + } else { + return super.generate(s, obj); + } + } + +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java new file mode 100644 index 0000000..d252f4a --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java @@ -0,0 +1,64 @@ +package com.baidu.highflip.core.utils; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.lang.Nullable; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, + Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, + new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) + throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) + throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, + TypeReference typeReference) + throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } +} diff --git a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java index 7584d1a..b7f2135 100644 --- a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java +++ b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java @@ -6,12 +6,18 @@ import com.baidu.highflip.core.entity.dag.PartyNode; import com.baidu.highflip.core.entity.dag.codec.TypeValue; import com.baidu.highflip.core.entity.dag.common.MappedNode; +import com.baidu.highflip.core.utils.SerializerUtils; +import com.fasterxml.jackson.core.JsonProcessingException; + import highflip.HighflipMeta; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; @Slf4j @@ -19,10 +25,7 @@ public class TestDag { @Test public void testGraph() { - HighflipMeta.GraphProto define = HighflipMeta.GraphProto - .newBuilder() - .setName("test_graph") - .build(); + HighflipMeta.GraphProto define = constructHighflipGraph(); var graph = Graph.fromProto(define); log.info("graph = {}", graph); @@ -229,4 +232,353 @@ public void testDag() { var proto = Graph.toProto(graph); log.info("proto = {}", proto); } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + readerNodeOutputProtos.add(outputProto); + + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model").setValue(getJsonStringOfModelOutput()).build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + + + dataTransformNodeInputProtos.add(dataTransformInputProto); + + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs( + dataTransformNodeInputProtos) + .addAllOutputs( + dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + intersectNodeOutputProtos.add(intersectOutputProto); + + List intersectNodeInputProtos = + new ArrayList<>(); + + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + + intersectNodeInputProtos.add(intersectInputProto); + + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + + // Map tableMap = new HashMap<>(); + // tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + // tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto mapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + // HighflipMeta.ValueProto valueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + // highflip.HighflipMeta.TypedValueProto typedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + // + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + // Map commonTableMap = + // new HashMap<>(); + + // tableMap.put("hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("split_calculation", HighflipMeta.ValueProto.newBuilder().setBool(false).build()); + // tableMap.put("key_length", HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + + + + + // Map commonRsaParamConfMap = + // new HashMap<>(); + // commonRsaParamConfMap.put("hash_method", + // HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("split_calculation", + // HighflipMeta.ValueProto.newBuilder().setBool( + // false).build()); + // commonRsaParamConfMap.put("key_length", + // HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + // HighflipMeta.MapProto commonValueMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + // HighflipMeta.ValueProto commonValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + // highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.GUEST).addAllNodes(guestPartyNodes).setName("9999").build(); + // guest 已经组装好 + + // 开始组装host + + // Map hostTableMap = + // new HashMap<>(); + // hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + // hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto hostMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + // HighflipMeta.ValueProto hostValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + // highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + // + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.HOST).addAllNodes(hostPartyNodes).setName("10000").build(); + + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + } diff --git a/highflip-proto/src/main/proto/highflip-meta.proto b/highflip-proto/src/main/proto/highflip-meta.proto index a093630..a11552c 100644 --- a/highflip-proto/src/main/proto/highflip-meta.proto +++ b/highflip-proto/src/main/proto/highflip-meta.proto @@ -65,6 +65,8 @@ message NodeInputProto{ string from_node = 4; string from_output = 5; + + string value = 6; } message NodeOutputProto{ @@ -74,6 +76,8 @@ message NodeOutputProto{ string name = 2; optional string description = 3; + + string value = 4; } message NodeProto{ diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index 880a5b6..bdaabba 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -559,6 +559,15 @@ message ConfigListResponse{ string key = 1; } +message GetServiceConfigResponse{ + + string url = 1; + + string party_id = 2; + + string role = 3; +} + //////////////////////////////////////////////////////////////////////////////// // SERVICE @@ -608,4 +617,7 @@ service HighFlip{ rpc getPartner(PartnerId) returns(PartnerGetResponse); rpc listPartner(PartnerListRequest) returns(stream PartnerListResponse); rpc controlPartner(PartnerControlRequest) returns(Void); + + // adaptor service config + rpc getServiceConfig(Void) returns(GetServiceConfigResponse); } \ No newline at end of file diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java index a9fdf38..aa6dde4 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java @@ -6,6 +6,7 @@ import com.baidu.highflip.server.adaptor.impl.ConfigurableOperatorAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePartnerAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePlatformAdaptor; +import com.baidu.highflip.server.adaptor.impl.ConfigurableServiceAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbJobAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbTaskAdaptor; import com.baidu.highflip.server.adaptor.impl.FixedSingleDataAdaptor; @@ -22,6 +23,11 @@ public class DefaultAdaptor implements HighFlipAdaptor { public static final String PLATFORM_PROPERTIES = "/adaptor/highflip.platform.properties"; + /** + * 用于配置本侧服务的配置信息 + */ + public static final String SERVICE_PROPERTIES = "/adaptor/highflip.service.properties"; + @Override public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_JOB, new DumbJobAdaptor()); @@ -37,6 +43,9 @@ public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_PLATFORM, new ConfigurablePlatformAdaptor(HighFlipUtils.getProperty(PLATFORM_PROPERTIES))); + + register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, + new ConfigurableServiceAdaptor(HighFlipUtils.getProperty(SERVICE_PROPERTIES))); } @Override diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java new file mode 100644 index 0000000..800bbc8 --- /dev/null +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java @@ -0,0 +1,42 @@ +package com.baidu.highflip.server.adaptor.impl; + +import java.util.Properties; + +import com.baidu.highflip.core.adaptor.ServiceAdaptor; +import com.baidu.highflip.core.common.AdaptorPropsList; + +public class ConfigurableServiceAdaptor implements ServiceAdaptor { + + String url; + + String partyId; + + String role; + + public ConfigurableServiceAdaptor(Properties props) { + + this.url = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT); + + this.partyId = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT); + + this.role = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT); + } + + @Override + public String getUrl() { + return this.url; + } + + @Override + public String getPartyId() { + return this.partyId; + } + + @Override + public String getRole() { + return this.role; + } +} diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java index 0c99b7e..af0ea2e 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java @@ -1,8 +1,11 @@ package com.baidu.highflip.server.config; +import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.DefaultAdaptor; import com.baidu.highflip.server.adaptor.loader.AdaptorLoader; +import com.baidu.highflip.server.engine.component.HighFlipRuntime; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -23,6 +26,9 @@ public class AdaptorConfig { @Autowired InstanceRegister register; + @Autowired + HighFlipRuntime highFlipRuntime; + AdaptorLoader loader = null; @PostConstruct @@ -73,5 +79,6 @@ void initialAdaptor() { void initialDefaultAdaptor() { DefaultAdaptor adaptor = new DefaultAdaptor(); adaptor.setup(register); + register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java index 525db92..2085b0a 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java @@ -290,7 +290,6 @@ public Job createJob(String name, String description, Graph graph) { job = getContext().getJobAdaptor() .createJob(job); - job = getContext().getJobRepository() .save(job); @@ -317,7 +316,6 @@ public Job createJob(String name, String description, Graph graph) { return job; } - // @Scheduled protected void updateJob() { diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java index d204511..3d2a07b 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java @@ -5,6 +5,7 @@ import com.baidu.highflip.core.adaptor.OperatorAdaptor; import com.baidu.highflip.core.adaptor.PartnerAdaptor; import com.baidu.highflip.core.adaptor.PlatformAdaptor; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.adaptor.TaskAdaptor; import com.baidu.highflip.core.adaptor.UserAdaptor; import com.baidu.highflip.core.common.InstanceNameList; @@ -125,4 +126,9 @@ public UserAdaptor getUserAdaptor(){ public UserRepository getUserRepository() { return userReps; } + + // SERVICE + public ServiceAdaptor getServiceAdaptor() { + return getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java index 55c0412..934c929 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java @@ -69,4 +69,10 @@ public User getUser(String userId) { public Operator getOperator(String operatorId) { return null; } + + @Override + public Data registerData(Data data) { + Data dataSaved = getContext().getDataRepository().save(data); + return dataSaved; + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index e6de087..e16cd85 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -1,5 +1,6 @@ package com.baidu.highflip.server.rpc.v1; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.entity.dag.Graph; import com.baidu.highflip.core.entity.runtime.Config; import com.baidu.highflip.core.entity.runtime.Data; @@ -221,12 +222,17 @@ public void checkJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); + com.baidu.highflip.core.entity.runtime.basic.Status jobStatus = + getEngine().getContext() + .getJobAdaptor() + .getJobStatus(job); Highflip.JobCheckResponse response = Highflip.JobCheckResponse .newBuilder() .setJobId(job.getJobId()) + .setStatus(Highflip.JobCheckResponse.JobStatus.valueOf( + jobStatus.toString().toUpperCase())) .build(); - returnOne(responseObserver, response); } @@ -685,4 +691,17 @@ public void controlPartner(Highflip.PartnerControlRequest request, } + @Override + public void getServiceConfig(Highflip.Void request, + StreamObserver responseObserver) { + final ServiceAdaptor serviceAdaptor = + getEngine().getContext().getServiceAdaptor(); + Highflip.GetServiceConfigResponse response = + Highflip.GetServiceConfigResponse.newBuilder() + .setUrl(serviceAdaptor.getUrl()) + .setPartyId(serviceAdaptor.getPartyId()) + .setRole(serviceAdaptor.getRole()) + .build(); + returnOne(responseObserver, response); + } } diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties new file mode 100644 index 0000000..b36b940 --- /dev/null +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -0,0 +1,3 @@ +highflip.adaptor.service.url=http://10.27.130.41:8380 +highflip.adaptor.service.party.id=9999 +highflip.adaptor.service.role=guest \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index e7c6ac5..975ae95 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path= +highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar ######################################################### # spring grpc ######################################################### @@ -30,7 +30,9 @@ spring.jpa.properties.jadira.usertype.autoRegisterUserTypes=true # h2 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect spring.datasource.driver-class-name=org.h2.Driver -spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip.db +spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip +spring.datasource.username=highflip +spring.datasource.password=HighFlip@123 ######################################################### # logging ######################################################### diff --git a/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java new file mode 100644 index 0000000..d4651d8 --- /dev/null +++ b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.server.engine; + +public class HighFlipEngineTest { + +} From 729f551efc53eae348a26ff1613ec3e818192140 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 31 Jul 2023 14:24:54 +0800 Subject: [PATCH 08/14] config github actions and deploy to maven central repository --- .github/workflows/maven.yml | 24 +- .../adaptor/highflip.service.properties | 6 +- .../src/main/resources/application.properties | 2 +- pom.xml | 208 ++++++++++++++++-- 4 files changed, 217 insertions(+), 23 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1d25004..20f2f7f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,8 +27,30 @@ jobs: java-version: '11' distribution: 'temurin' cache: maven + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build with Maven - run: mvn -B package --file pom.xml + run: mvn -B -Prelease -DskipTests package --file pom.xml + - name: Create release + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + artifacts: "${{ github.workspace }}/target/*.jar" + token: ${{ secrets.GITHUB_TOKEN }} + - name: Publish to the Maven Central Repository + run: | + mvn clean \ + --no-transfer-progress \ + --batch-mode \ + -Prelease \ + -DskipTests \ + deploy + env: + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties index b36b940..07cc227 100644 --- a/highflip-server/src/main/resources/adaptor/highflip.service.properties +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -1,3 +1,3 @@ -highflip.adaptor.service.url=http://10.27.130.41:8380 -highflip.adaptor.service.party.id=9999 -highflip.adaptor.service.role=guest \ No newline at end of file +highflip.adaptor.service.url= +highflip.adaptor.service.party.id= +highflip.adaptor.service.role= \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index 975ae95..7f48543 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar +highflip.server.adaptor.path= ######################################################### # spring grpc ######################################################### diff --git a/pom.xml b/pom.xml index eb41ecc..468baf8 100644 --- a/pom.xml +++ b/pom.xml @@ -7,8 +7,8 @@ pom 1.0.0-SNAPSHOT highflip - - http://maven.apache.org + High layer Federated Learning Intercommunication Protocol + https://github.com/baidu/highflip @@ -29,6 +29,11 @@ 11 11 1.49.0 + + + 3.2.0 + 1.6 + 1.6.7 @@ -41,6 +46,36 @@ highflip-editor + + + dev + chenzhiyu + work123@Baidu + https://oss.sonatype.org/ + The HighFlip Project + https://oss.sonatype.org/ + + + + + + ossrh-snapshot + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + https://github.com/baidu/highflip + scm:git:git://github.com/baidu/highflip.git + scm:git:ssh://git@github.com/baidu/highflip.git + HEAD + + @@ -120,6 +155,55 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadoc + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-source + + jar + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + + report + test + + report + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.plugin.version} + @@ -130,23 +214,111 @@ org.jacoco jacoco-maven-plugin - 0.8.2 - - - - prepare-agent - - - - - report - test - - report - - - + + + org.apache.maven.plugins + maven-jar-plugin + + + + local + + true + + + + + release + + + + maven-source-plugin + + + create-source-jar + + jar + test-jar + + + + + + maven-jar-plugin + + + + test-jar + + + + true + + + + + + maven-javadoc-plugin + + + create-javadoc-jar + + javadoc + jar + + package + + + + ${java.version} + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven.gpg.plugin.version} + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus.staging.maven.plugin.version} + true + + sonatype-nexus-repository + https://oss.sonatype.org + true + + + + + + + sonatype-nexus-repository + https://oss.sonatype.org/content/repositories/snapshots + + + sonatype-nexus-repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + \ No newline at end of file From 928831828c824e62c49ee36291fc36decf492eef Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 3 Aug 2023 11:21:57 +0800 Subject: [PATCH 09/14] modify code after review --- .github/workflows/maven.yml | 23 ++++++++---------- .../console/commands/TaskCommand.java | 5 ++-- .../baidu/highflip/client/HighFlipClient.java | 13 ++++------ .../highflip/core/engine/HighFlipRuntime.java | 5 ++++ .../highflip/core/entity/runtime/Data.java | 2 +- .../highflip/core/entity/runtime/Task.java | 9 +++++++ highflip-editor/pom.xml | 18 +++++++------- highflip-proto/src/main/proto/highflip.proto | 5 ++-- .../server/adaptor/DefaultAdaptor.java | 22 +++++++++++++++-- .../highflip/server/config/AdaptorConfig.java | 4 +--- .../server/engine/HighFlipEngine.java | 21 ++++++++++------ .../engine/component/HighFlipRuntime.java | 18 +++++++++++++- .../server/rpc/v1/HighFlipRpcService.java | 24 +++++++------------ .../adaptor/highflip.service.properties | 6 ++--- .../src/main/resources/application.properties | 4 ++-- pom.xml | 17 ++++++++----- 16 files changed, 119 insertions(+), 77 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 20f2f7f..f4f0969 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -13,6 +13,11 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] + workflow_dispatch: + inputs: + branch: + description: 'The branch to build' + required: true jobs: build: @@ -29,16 +34,8 @@ jobs: cache: maven server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Build with Maven - run: mvn -B -Prelease -DskipTests package --file pom.xml - - name: Create release - uses: ncipollo/release-action@v1 - with: - allowUpdates: true - artifacts: "${{ github.workspace }}/target/*.jar" - token: ${{ secrets.GITHUB_TOKEN }} - name: Publish to the Maven Central Repository run: | mvn clean \ @@ -46,11 +43,11 @@ jobs: --batch-mode \ -Prelease \ -DskipTests \ - deploy + deploy --file pom.xml env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph diff --git a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java index 724295b..5c969f0 100644 --- a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java +++ b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java @@ -17,12 +17,13 @@ public class TaskCommand { @Autowired HighFlipClient client; - @ShellMethod(key = "task list", value = "List all task ids") + @ShellMethod(key = "task list", value = "List job all task ids") public Iterable list( + @ShellOption String jobId, @ShellOption(defaultValue = "0") Integer offset, @ShellOption(defaultValue = "0") Integer limit) { - return client.listTasks(offset, limit); + return client.listTasks(jobId, offset, limit); } @ShellMethod(key = "task get", value = "Get a task information") diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 8a4694e..3cd80e0 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -289,13 +289,17 @@ public Iterable getJobLog(String jobId){ /** * + * @param jobId job id in highflip * @param offset * @param limit * @return */ - public Iterable listTasks(int offset, int limit){ + public Iterable listTasks(String jobId, int offset, int limit) { Highflip.TaskListRequest request = Highflip.TaskListRequest .newBuilder() + .setJobId(jobId) + .setOffset(offset) + .setLimit(limit) .build(); Iterator response = getBlockingStub() @@ -584,11 +588,4 @@ public String createPartner(String name, String description){ return response.getPartnerId(); } - - public Highflip.GetServiceConfigResponse getServiceConfig() { - final Highflip.GetServiceConfigResponse serviceConfig = - getBlockingStub().getServiceConfig( - Highflip.Void.getDefaultInstance()); - return serviceConfig; - } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java index fc4ef77..21a56a6 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java @@ -6,6 +6,7 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; public interface HighFlipRuntime { @@ -28,4 +29,8 @@ public interface HighFlipRuntime { Operator getOperator(String operatorId); Data registerData(Data data); + + Iterable listTask(String jobId); + + void updateTask(Task task); } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index 9ce80a0..bc4239b 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -32,7 +32,7 @@ public class Data { @Id - @Column(name = "data_id", length = 72) + @Column(name = "data_id", length = 36) @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String dataId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java index 5feae39..f4ed6ad 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java @@ -35,6 +35,10 @@ public class Task { @GeneratedValue(generator = "id_gen") String taskid; + /** + * job id is not the job id in highflip but the job id in corresponding + * federated learning service + */ @Column(name = "job_id") String jobid; @@ -81,4 +85,9 @@ public class Task { @Type(type = "json") @Column(name = "binding") Map binding; + + @Type(type = "json") + @Column(name = "output_data") + List outputData; + } diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index 7f2e216..9919f91 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -11,16 +11,14 @@ highflip-editor - - 11 - 11 - 2.7.10 - 1.18.22 - 6.11 - 1.43 - 31.1-jre - 6.0.2 - + + + + + + + + diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index bdaabba..119c543 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -230,6 +230,8 @@ message TaskGetResponse{ repeated string input_tasks = 9; repeated string output_tasks = 10; + + repeated string output_data_ids = 11; } @@ -617,7 +619,4 @@ service HighFlip{ rpc getPartner(PartnerId) returns(PartnerGetResponse); rpc listPartner(PartnerListRequest) returns(stream PartnerListResponse); rpc controlPartner(PartnerControlRequest) returns(Void); - - // adaptor service config - rpc getServiceConfig(Void) returns(GetServiceConfigResponse); } \ No newline at end of file diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java index aa6dde4..e183cd5 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java @@ -2,6 +2,7 @@ import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.HighFlipAdaptor; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.impl.ConfigurableOperatorAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePartnerAdaptor; @@ -28,6 +29,12 @@ public class DefaultAdaptor implements HighFlipAdaptor { */ public static final String SERVICE_PROPERTIES = "/adaptor/highflip.service.properties"; + private HighFlipRuntime highFlipRuntime; + + public DefaultAdaptor(HighFlipRuntime highFlipRuntime) { + this.highFlipRuntime = highFlipRuntime; + } + @Override public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_JOB, new DumbJobAdaptor()); @@ -44,8 +51,19 @@ public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_PLATFORM, new ConfigurablePlatformAdaptor(HighFlipUtils.getProperty(PLATFORM_PROPERTIES))); - register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, - new ConfigurableServiceAdaptor(HighFlipUtils.getProperty(SERVICE_PROPERTIES))); + final ConfigurableServiceAdaptor configurableServiceAdaptor = + new ConfigurableServiceAdaptor( + HighFlipUtils.getProperty(SERVICE_PROPERTIES)); + register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, configurableServiceAdaptor); + + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.url", + configurableServiceAdaptor.getUrl()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.party.id", + configurableServiceAdaptor.getPartyId()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.role", + configurableServiceAdaptor.getRole()); + + register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } @Override diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java index af0ea2e..2a7827d 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java @@ -1,6 +1,5 @@ package com.baidu.highflip.server.config; -import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.DefaultAdaptor; import com.baidu.highflip.server.adaptor.loader.AdaptorLoader; @@ -77,8 +76,7 @@ void initialAdaptor() { } void initialDefaultAdaptor() { - DefaultAdaptor adaptor = new DefaultAdaptor(); + DefaultAdaptor adaptor = new DefaultAdaptor(highFlipRuntime); adaptor.setup(register); - register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java index 2085b0a..05443c3 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -64,7 +65,7 @@ public class HighFlipEngine { @Autowired PlatformTransactionManager transactionManager; - ConcurrentMap activeJobs; + ConcurrentMap activeJobs = new ConcurrentHashMap<>(); @Autowired AsyncTaskExecutor executor; @@ -313,11 +314,12 @@ public Job createJob(String name, String description, Graph graph) { getContext().getTaskRepository() .saveAll(news); } + activeJobs.putIfAbsent(job.getJobId(), job); return job; } // @Scheduled - protected void updateJob() { + public void updateJob() { JobAdaptor adaptor = getContext().getJobAdaptor(); @@ -326,6 +328,10 @@ protected void updateJob() { if (status != job.getStatus()) { job.setStatus(status); getContext().getJobRepository().save(job); + getContext().getJobAdaptor().updateJob(job); + } + if (status == Status.SUCCEEDED || status == Status.FAILED) { + activeJobs.remove(job.getJobId()); } }); } @@ -411,14 +417,15 @@ public void initializeTasks() { } // @Scheduled - private void updateTask() { - + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); } public Iterable listTask(String jobid) { - - return getContext().getTaskRepository() - .findAllByJobid(jobid); + final List tasks = getContext().getTaskRepository() + .findAllByJobid(jobid); + log.info("tasks size: {}", tasks.size()); + return tasks; } @Cacheable(value = "tasks") diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java index 934c929..791cce8 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java @@ -7,6 +7,8 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -18,13 +20,16 @@ public class HighFlipRuntime implements com.baidu.highflip.core.engine.HighFlipR @Autowired HighFlipContext context; + @Autowired + HighFlipConfiguration configuration; + HighFlipContext getContext() { return context; } @Override public Configuration getConfiguration() { - return null; + return configuration; } @Override @@ -75,4 +80,15 @@ public Data registerData(Data data) { Data dataSaved = getContext().getDataRepository().save(data); return dataSaved; } + + @Override + public Iterable listTask(String jobId) { + return getContext().getTaskRepository() + .findAllByJobid(jobId); + } + + @Override + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index e16cd85..0640b8f 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -226,6 +226,8 @@ public void checkJob(Highflip.JobId request, getEngine().getContext() .getJobAdaptor() .getJobStatus(job); + // check and update job + getEngine().updateJob(); Highflip.JobCheckResponse response = Highflip.JobCheckResponse .newBuilder() @@ -302,9 +304,11 @@ public void getJobLog(Highflip.JobLogRequest request, @Override public void listTask(Highflip.TaskListRequest request, StreamObserver responseObserver) { - + final Job job = getEngine().getJob(request.getJobId()); + log.info("job id: {}, job binging id: {}", job.getJobId(), + job.getBingingId()); Iterator response = Streams - .stream(getEngine().listTask(request.getJobId())) + .stream(getEngine().listTask(job.getBingingId())) .map(t -> Highflip.TaskListResponse .newBuilder() .setTaskId(t.getTaskid()) @@ -328,8 +332,10 @@ public void getTask(Highflip.TaskId request, .newBuilder() .setTaskId(task.getTaskid()) .setJobId(task.getJobid()) + .setNodeName(task.getNodeName()) .setCreateTime(task.getCreateTime().toString()) .setUpdateTime(task.getUpdateTime().toString()) + .addAllOutputDataIds(task.getOutputData()) .build(); returnOne(responseObserver, response); @@ -690,18 +696,4 @@ public void controlPartner(Highflip.PartnerControlRequest request, StreamObserver responseObserver) { } - - @Override - public void getServiceConfig(Highflip.Void request, - StreamObserver responseObserver) { - final ServiceAdaptor serviceAdaptor = - getEngine().getContext().getServiceAdaptor(); - Highflip.GetServiceConfigResponse response = - Highflip.GetServiceConfigResponse.newBuilder() - .setUrl(serviceAdaptor.getUrl()) - .setPartyId(serviceAdaptor.getPartyId()) - .setRole(serviceAdaptor.getRole()) - .build(); - returnOne(responseObserver, response); - } } diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties index 07cc227..b36b940 100644 --- a/highflip-server/src/main/resources/adaptor/highflip.service.properties +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -1,3 +1,3 @@ -highflip.adaptor.service.url= -highflip.adaptor.service.party.id= -highflip.adaptor.service.role= \ No newline at end of file +highflip.adaptor.service.url=http://10.27.130.41:8380 +highflip.adaptor.service.party.id=9999 +highflip.adaptor.service.role=guest \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index 7f48543..48ed02f 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path= +highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar ######################################################### # spring grpc ######################################################### @@ -32,7 +32,7 @@ spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect spring.datasource.driver-class-name=org.h2.Driver spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip spring.datasource.username=highflip -spring.datasource.password=HighFlip@123 +spring.datasource.password=changeMe ######################################################### # logging ######################################################### diff --git a/pom.xml b/pom.xml index 468baf8..e78d907 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,12 @@ 11 11 1.49.0 + 2.7.3 + 1.18.22 + 6.11 + 1.43 + 31.1-jre + 6.0.2 3.2.0 @@ -48,12 +54,11 @@ - dev - chenzhiyu - work123@Baidu - https://oss.sonatype.org/ - The HighFlip Project - https://oss.sonatype.org/ + Baidu dianshi + dianshi-help@baidu.com + https://dianshi.baidu.com/ + dianshi + https://dianshi.baidu.com/ From f5c6daf3d8147a0af26e25e6e266e17279cf472c Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 7 Aug 2023 10:59:04 +0800 Subject: [PATCH 10/14] config github actions and add distributionManagement --- .github/workflows/maven.yml | 54 +++++++++++-------- highflip-build/pom.xml | 2 +- highflip-clients/highflip-console/pom.xml | 10 ++-- highflip-clients/highflip-sdk/pom.xml | 10 ++-- highflip-clients/pom.xml | 4 +- highflip-core/pom.xml | 6 +-- highflip-doc/pom.xml | 2 +- highflip-editor/pom.xml | 17 ++---- highflip-proto/pom.xml | 15 +++++- highflip-server/pom.xml | 6 +-- .../highflip-adaptor-demo/pom.xml | 4 +- .../highflip-adaptor-fate/pom.xml | 4 +- highflip-vendors/pom.xml | 2 +- pom.xml | 45 +++++++--------- 14 files changed, 91 insertions(+), 90 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f4f0969..cb98617 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -14,10 +14,6 @@ on: pull_request: branches: [ "master" ] workflow_dispatch: - inputs: - branch: - description: 'The branch to build' - required: true jobs: build: @@ -25,25 +21,37 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: maven - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Publish to the Maven Central Repository - run: | - mvn clean \ - --no-transfer-progress \ - --batch-mode \ - -Prelease \ - -DskipTests \ - deploy --file pom.xml + - name: Checkout project + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build with Maven + run: mvn -B package -DskipTests -Prelease --file pom.xml + + - name: Set up Apache Maven Central + uses: actions/setup-java@v3 + with: # running setup-java again overwrites the settings.xml + java-version: '11' + distribution: 'temurin' + server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml + server-username: MAVEN_USERNAME # env variable for username in deploy + server-password: MAVEN_PASSWORD # env variable for token in deploy + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import + gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + + - name: Publish to the Maven Central Repository + run: + mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am env: MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} diff --git a/highflip-build/pom.xml b/highflip-build/pom.xml index 6d5cf4e..5ae3101 100644 --- a/highflip-build/pom.xml +++ b/highflip-build/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-build jar 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-console/pom.xml b/highflip-clients/highflip-console/pom.xml index 3047fff..1815e6d 100644 --- a/highflip-clients/highflip-console/pom.xml +++ b/highflip-clients/highflip-console/pom.xml @@ -11,16 +11,16 @@ --> - com.baidu + com.baidu.highflip highflip-console 1.0.0-SNAPSHOT highflip-console - com.baidu - highflip + com.baidu.highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml @@ -44,7 +44,7 @@ spring-boot-starter-json - com.baidu + com.baidu.highflip highflip-sdk 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index b44148a..49e9514 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-sdk jar 1.0.0-SNAPSHOT @@ -10,15 +10,15 @@ https://maven.apache.org - com.baidu - highflip + com.baidu.highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml - com.baidu + com.baidu.highflip highflip-proto ${project.version} diff --git a/highflip-clients/pom.xml b/highflip-clients/pom.xml index 3de5034..1457d0f 100644 --- a/highflip-clients/pom.xml +++ b/highflip-clients/pom.xml @@ -4,13 +4,13 @@ highflip - com.baidu + com.baidu.highflip 1.0.0-SNAPSHOT ../pom.xml - com.baidu + com.baidu.highflip highflip-clients 1.0.0-SNAPSHOT pom diff --git a/highflip-core/pom.xml b/highflip-core/pom.xml index ff39144..22b019e 100644 --- a/highflip-core/pom.xml +++ b/highflip-core/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-core jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -19,7 +19,7 @@ - com.baidu + com.baidu.highflip highflip-proto 1.0.0-SNAPSHOT diff --git a/highflip-doc/pom.xml b/highflip-doc/pom.xml index 0d8e57a..3ace508 100644 --- a/highflip-doc/pom.xml +++ b/highflip-doc/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-doc pom 1.0.0-SNAPSHOT diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index 9919f91..57aa44c 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -4,30 +4,21 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> highflip - com.baidu + com.baidu.highflip 1.0.0-SNAPSHOT 4.0.0 highflip-editor - - - - - - - - - - com.baidu + com.baidu.highflip highflip-sdk ${project.version} - com.baidu + com.baidu.highflip highflip-proto ${project.version} @@ -103,6 +94,4 @@ - - \ No newline at end of file diff --git a/highflip-proto/pom.xml b/highflip-proto/pom.xml index 1584c1c..868c091 100644 --- a/highflip-proto/pom.xml +++ b/highflip-proto/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-proto jar 1.0.0-SNAPSHOT @@ -16,6 +16,19 @@ 1.49.0 + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + io.grpc diff --git a/highflip-server/pom.xml b/highflip-server/pom.xml index 5055c3e..104b332 100644 --- a/highflip-server/pom.xml +++ b/highflip-server/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-server jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -70,7 +70,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT diff --git a/highflip-vendors/highflip-adaptor-demo/pom.xml b/highflip-vendors/highflip-adaptor-demo/pom.xml index a022666..871c819 100644 --- a/highflip-vendors/highflip-adaptor-demo/pom.xml +++ b/highflip-vendors/highflip-adaptor-demo/pom.xml @@ -3,7 +3,7 @@ 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-demo jar 1.0.0-SNAPSHOT @@ -18,7 +18,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/highflip-adaptor-fate/pom.xml b/highflip-vendors/highflip-adaptor-fate/pom.xml index 481c757..3fbcc3d 100644 --- a/highflip-vendors/highflip-adaptor-fate/pom.xml +++ b/highflip-vendors/highflip-adaptor-fate/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-fate jar 1.0.0-SNAPSHOT @@ -30,7 +30,7 @@ test - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/pom.xml b/highflip-vendors/pom.xml index 35c88ca..38a435b 100644 --- a/highflip-vendors/pom.xml +++ b/highflip-vendors/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-vendors pom 1.0.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index e78d907..5358d7a 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip pom 1.0.0-SNAPSHOT @@ -62,22 +62,10 @@ - - - ossrh-snapshot - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - https://github.com/baidu/highflip - scm:git:git://github.com/baidu/highflip.git - scm:git:ssh://git@github.com/baidu/highflip.git + scm:git:${project.scm.url} + scm:git:${project.scm.url} + https://github.com/baidu/highflip.git HEAD @@ -227,6 +215,19 @@ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + local @@ -307,23 +308,13 @@ ${nexus.staging.maven.plugin.version} true - sonatype-nexus-repository + ossrh https://oss.sonatype.org true - - - sonatype-nexus-repository - https://oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-repository - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - \ No newline at end of file From dad0fff56084a86d4587a88d2df538cc31034e8e Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 7 Aug 2023 15:03:28 +0800 Subject: [PATCH 11/14] add deploy.yml --- .github/workflows/deploy.yml | 60 ++++++++++++++++++++++++++++++++++++ .github/workflows/maven.yml | 32 +------------------ 2 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..9ab6985 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,60 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI with Maven + +on: + push: + branches: [ "master" ] + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout project + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build with Maven + run: mvn -B package -DskipTests -Prelease --file pom.xml + + - name: Set up Apache Maven Central + uses: actions/setup-java@v3 + with: # running setup-java again overwrites the settings.xml + java-version: '11' + distribution: 'temurin' + server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml + server-username: MAVEN_USERNAME # env variable for username in deploy + server-password: MAVEN_PASSWORD # env variable for token in deploy + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import + gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + + - name: Publish to the Maven Central Repository + run: + mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am + env: + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} + + # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive + # - name: Update dependency graph + # uses: advanced-security/maven-dependency-submission-action@v2.0.1 diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index cb98617..2431618 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -6,13 +6,11 @@ # separate terms of service, privacy policy, and support # documentation. -name: Java CI with Maven +name: Build Java Project on: push: - branches: [ "master" ] pull_request: - branches: [ "master" ] workflow_dispatch: jobs: @@ -29,34 +27,6 @@ jobs: with: java-version: '11' distribution: 'temurin' - server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build with Maven run: mvn -B package -DskipTests -Prelease --file pom.xml - - - name: Set up Apache Maven Central - uses: actions/setup-java@v3 - with: # running setup-java again overwrites the settings.xml - java-version: '11' - distribution: 'temurin' - server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml - server-username: MAVEN_USERNAME # env variable for username in deploy - server-password: MAVEN_PASSWORD # env variable for token in deploy - gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import - gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase - - - name: Publish to the Maven Central Repository - run: - mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am - env: - MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} - - # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - # - name: Update dependency graph - # uses: advanced-security/maven-dependency-submission-action@v2.0.1 From 6e2fd1f13b601445de4c0cc08ec3c8435a2fcc76 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 7 Aug 2023 15:32:27 +0800 Subject: [PATCH 12/14] modify the webhook of deploy java jar --- .github/workflows/deploy.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9ab6985..c02456b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,12 +6,11 @@ # separate terms of service, privacy policy, and support # documentation. -name: Java CI with Maven +name: Deploy Java jar with maven on: - push: - branches: [ "master" ] - workflow_dispatch: + release: + types: [published] jobs: build: From 505f72b6db564755180dd118ec051d85b1cce8a8 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Tue, 8 Aug 2023 11:10:50 +0800 Subject: [PATCH 13/14] add highflip fate vendors --- .../highflip-adaptor-fate/pom.xml | 58 ++- .../webank/ai/fate/adaptor/DataAdaptor.java | 159 ++++++- .../webank/ai/fate/adaptor/JobAdaptor.java | 129 ++++-- .../ai/fate/adaptor/OperatorAdaptor.java | 28 +- .../webank/ai/fate/adaptor/TaskAdaptor.java | 84 +++- .../com/webank/ai/fate/client/FateClient.java | 124 ++--- .../ai/fate/client/form/FateStatus.java | 45 ++ .../webank/ai/fate/client/form/data/Data.java | 21 + .../ai/fate/client/form/data/DataId.java | 12 + .../ai/fate/client/form/dsl/Component.java | 14 + .../client/form/dsl/ComponentParameters.java | 17 + .../ai/fate/client/form/dsl/Components.java | 12 + .../webank/ai/fate/client/form/dsl/Dsl.java | 15 + .../ai/fate/client/form/dsl/DslConf.java | 22 + .../webank/ai/fate/client/form/dsl/Input.java | 23 + .../ai/fate/client/form/dsl/Output.java | 21 + .../ai/fate/client/form/dsl/RoleConf.java | 21 + .../webank/ai/fate/client/form/dsl/Site.java | 12 + .../form/job/Adaptation_parameters.java | 22 + .../ai/fate/client/form/job/Common.java | 14 + .../ai/fate/client/form/job/Components.java | 49 ++ .../ai/fate/client/form/job/Computing.java | 16 + .../ai/fate/client/form/job/Dataio.java | 19 + .../ai/fate/client/form/job/Eggroll_run.java | 9 + .../fate/client/form/job/Engines_address.java | 18 + .../ai/fate/client/form/job/FateJob.java | 133 ++++++ .../ai/fate/client/form/job/Federation.java | 16 + .../form/job/Hetero_feature_binning.java | 19 + .../webank/ai/fate/client/form/job/Input.java | 21 + .../ai/fate/client/form/job/Intersection.java | 39 ++ .../ai/fate/client/form/job/JobData.java | 18 + .../fate/client/form/job/Job_parameters.java | 50 ++ .../ai/fate/client/form/job/Output.java | 21 + .../ai/fate/client/form/job/Pulsar_run.java | 9 + .../ai/fate/client/form/job/QueryJob.java | 106 +++++ .../ai/fate/client/form/job/Rabbitmq_run.java | 9 + .../ai/fate/client/form/job/Reader.java | 19 + .../webank/ai/fate/client/form/job/Roles.java | 18 + .../ai/fate/client/form/job/Runtime_conf.java | 34 ++ .../form/job/Runtime_conf_on_party.java | 33 ++ .../ai/fate/client/form/job/Spark_run.java | 9 + .../ai/fate/client/form/job/Storage.java | 16 + .../webank/ai/fate/client/form/job/Table.java | 17 + .../client/form/job/Train_runtime_conf.java | 9 + .../fate/client/form/job/Transform_param.java | 14 + .../webank/ai/fate/client/form/job/User.java | 18 + .../form/task/Adaptation_parameters.java | 14 + .../ai/fate/client/form/task/Class_path.java | 16 + .../ai/fate/client/form/task/Common.java | 28 ++ .../fate/client/form/task/ComponentParam.java | 31 ++ .../form/task/Component_parameters.java | 21 + .../ai/fate/client/form/task/Eggroll_run.java | 7 + .../fate/client/form/task/Encrypt_param.java | 13 + .../ai/fate/client/form/task/Engine_conf.java | 12 + .../client/form/task/Engines_address.java | 7 + .../webank/ai/fate/client/form/task/Env.java | 11 + .../ai/fate/client/form/task/FFateTask.java | 64 +++ .../ai/fate/client/form/task/FateTask.java | 69 +++ .../ai/fate/client/form/task/Initiator.java | 13 + .../fate/client/form/task/Job_parameters.java | 12 + .../ai/fate/client/form/task/Local.java | 13 + .../form/task/Optimal_binning_param.java | 18 + .../fate/client/form/task/Provider_info.java | 15 + .../ai/fate/client/form/task/Pulsar_run.java | 7 + .../fate/client/form/task/Rabbitmq_run.java | 7 + .../webank/ai/fate/client/form/task/Role.java | 14 + .../ai/fate/client/form/task/Spark_run.java | 7 + .../ai/fate/client/form/task/TaskData.java | 14 + .../client/form/task/Transform_param.java | 14 + .../webank/ai/fate/common/ConfigConst.java | 7 + .../ai/fate/common/DataMultipartFile.java | 76 +++ .../ai/fate/common/DecompressUtils.java | 39 ++ .../webank/ai/fate/common/FateConstants.java | 7 + .../com/webank/ai/fate/common/GZIPUtils.java | 84 ++++ .../deserializer/FeignSpringFormEncoder.java | 255 ++++++++++ .../deserializer/JsonListDeserializer.java | 34 ++ .../JsonMapComponentsDeserializer.java | 36 ++ .../JsonMapListStringDeserializer.java | 38 ++ .../deserializer/JsonMapMapDeserializer.java | 44 ++ .../JsonMapMapObjectDeserializer.java | 38 ++ .../JsonMapMapStringDeserializer.java | 44 ++ .../JsonMapStringDeserializer.java | 33 ++ .../deserializer/JsonStringDeserializer.java | 23 + .../common/deserializer/SerializerUtils.java | 416 +++++++++++++++++ .../webank/ai/fate/context/FateContext.java | 14 +- .../ai/fate/translator/DSLTranslator.java | 352 +++++++++++++- .../ai/fate/adaptor/DataAdaptorTest.java | 44 ++ .../webank/ai/fate/client/FateClientTest.java | 138 +++++- .../ai/fate/translator/DSLTranslatorTest.java | 434 ++++++++++++++++++ 89 files changed, 4070 insertions(+), 105 deletions(-) create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java diff --git a/highflip-vendors/highflip-adaptor-fate/pom.xml b/highflip-vendors/highflip-adaptor-fate/pom.xml index 3fbcc3d..c6622a8 100644 --- a/highflip-vendors/highflip-adaptor-fate/pom.xml +++ b/highflip-vendors/highflip-adaptor-fate/pom.xml @@ -17,6 +17,16 @@ + + com.baidu.highflip + highflip-proto + 1.0.0-SNAPSHOT + + + com.baidu.highflip + highflip-core + 1.0.0-SNAPSHOT + org.junit.jupiter junit-jupiter @@ -30,21 +40,42 @@ test - com.baidu.highflip - highflip-core - 1.0.0-SNAPSHOT - compile + com.google.protobuf + protobuf-java + 3.21.1 io.github.openfeign feign-core 11.10 + + org.slf4j + slf4j-api + 1.7.36 + + + + org.slf4j + slf4j-simple + 1.7.36 + io.github.openfeign feign-jackson 11.10 + + io.github.openfeign + feign-httpclient + 11.10 + + + + org.apache.commons + commons-compress + 1.21 + @@ -54,6 +85,25 @@ maven-surefire-plugin 2.22.2 + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java index 52b367f..45952eb 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java @@ -1,10 +1,167 @@ package com.webank.ai.fate.adaptor; +import static com.webank.ai.fate.common.FateConstants.DATA_ID_SEPARATOR; + +import com.baidu.highflip.core.entity.runtime.Data; +import com.baidu.highflip.core.entity.runtime.basic.DataCategory; +import com.baidu.highflip.core.entity.runtime.basic.KeyPair; +import com.baidu.highflip.core.utils.Foreach; +import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.common.DataMultipartFile; +import com.webank.ai.fate.common.DecompressUtils; import com.webank.ai.fate.context.FateContext; +import feign.Response; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +@Slf4j +@Getter +public class DataAdaptor implements com.baidu.highflip.core.adaptor.DataAdaptor { -public class DataAdaptor { + private final FateContext context; + + private final String DEFAULT_NAMESPACES = "HIGH-FLIP"; + + private final String DEFAULT_DELIMITER = ","; public DataAdaptor(FateContext context) { + this.context = context; + } + + @Override + public Data updateData(Data data) { + return null; + } + + @Override + public int getDataCount() { + return 0; + } + + @Override + public Data getDataByIndex(int index, Data data) { + return null; + } + + @Override + public void deleteData(Data data) { + getContext().getClient().deleteData(data.getBingingId(), DEFAULT_NAMESPACES); + } + + @Override + public InputStream readDataRaw(Data data) { + return null; + } + + @Override + public Iterator> readDataDense(Data data) { + if(data == null) { + throw new RuntimeException("data is null."); + } + log.info("data:{}", data); + + if (DataCategory.RESULT_DATA.equals(data.getCategory())) { + String jobId = (String) data.getBinding().get("jobId"); + String componentName = (String) data.getBinding().get("componentName"); + String role = (String) data.getBinding().get("role"); + String partyId = (String) data.getBinding().get("partyId"); + log.info("jobId: {}, componentName: {}, role: {}, partyId: {}", + jobId, componentName, role, partyId); + + try (Response response = getContext().getClient() + .downloadComponentResultData(jobId, componentName, + role, partyId)) { + String content = DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")).get("data.csv"); + return Arrays.stream(content.split("\n")) + .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) + .map(d -> (Object) d) + .collect(Collectors.toList())) + .collect(Collectors.toList()).iterator(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + try (Response response = getContext().getClient() + .downloadData(data.getName(), + DEFAULT_NAMESPACES)) { + String content = DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")).get("table.csv"); + return Arrays.stream(content.split("\n")) + .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) + .map(d -> (Object) d) + .collect(Collectors.toList())) + .collect(Collectors.toList()).iterator(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public Iterator> readDataSparse(Data data) { + return null; + } + + @Override + public Data createData(Data data) { + return null; + } + + @Override + public void writeDataRaw(Data data, InputStream body) { + + } + + @Override + public void writeDataDense(Data data, Iterator> body) { + StringBuilder stringBuilder = new StringBuilder(); + try { + String headers = + data.getColumns().stream().map(c -> c.getName()) + .collect(Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(headers).append("\n"); + final Foreach> foreach = Foreach.from(body); + for (List dataList : foreach) { + String column = dataList.stream().map(Object::toString).collect( + Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(column).append("\n"); + } + } catch (NoSuchElementException e) { + log.warn("body has no next element"); + } catch (Exception e) { + throw e; + } + String tableName = data.getName(); + log.info("push table:{} data:{}", tableName, data); + + MultipartFile multipartFile = new DataMultipartFile(tableName, + stringBuilder.toString().getBytes(StandardCharsets.UTF_8)); + // drop default 1, which means overwrite the dataset + ResultForm dataResultForm = + getContext().getClient() + .pushData(multipartFile, DEFAULT_DELIMITER, "1", + "4", tableName, DEFAULT_NAMESPACES, + null, null, + null, "1"); + data.setBingingId(dataResultForm.getData().getJob_id()); + } + + @Override + public void writeDataSparse(Data data, Iterator> body) { } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java index 6fb2c7b..319fb53 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java @@ -1,20 +1,29 @@ package com.webank.ai.fate.adaptor; import com.baidu.highflip.core.engine.HighFlipRuntime; -import com.baidu.highflip.core.entity.runtime.Job; +import com.baidu.highflip.core.entity.runtime.Data; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.basic.Action; +import com.baidu.highflip.core.entity.runtime.basic.DataCategory; +import com.baidu.highflip.core.entity.runtime.basic.DataMode; import com.baidu.highflip.core.entity.runtime.basic.Status; -import com.webank.ai.fate.client.form.JobQueryResponseForm; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.QueryJob; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.FateTask; import com.webank.ai.fate.context.FateContext; import com.webank.ai.fate.translator.DSLTranslator; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; @Slf4j @Getter @@ -28,11 +37,11 @@ public JobAdaptor(FateContext context) { @Override public List getFeatures() { - return null; + throw new UnsupportedOperationException(); } @Override - public Job createJob(Job job) { + public com.baidu.highflip.core.entity.runtime.Job createJob(com.baidu.highflip.core.entity.runtime.Job job) { DSLTranslator.FateDAG dag = getContext() .getTranslator() @@ -40,83 +49,151 @@ public Job createJob(Job job) { String bindId = getContext() .getClient() - .jobSumbit(dag.getDsl(), dag.getDsl()); + .jobSubmit(dag.getDsl(), dag.getConf()).getData().getJob_id(); job.setBingingId(bindId); return job; } @Override - public Job updateJob(Job job) { - return null; + public com.baidu.highflip.core.entity.runtime.Job updateJob(com.baidu.highflip.core.entity.runtime.Job job) { + + String bindId = job.getBingingId(); + final Iterable tasks = + getContext().getHighFlipRuntime().listTask(bindId); + for (Task task : tasks) { + ResultForm> result = + getContext().getClient().taskQuery(task.getBingingId()); + if (CollectionUtils.isEmpty(result.getData())) { + throw new RuntimeException("task not found, task: " + task); + } + final FFateTask fFateTask = result.getData().get(0); + final String f_status = fFateTask.getF_status(); + final Status taskStatus = convertToHighFlipStatus(f_status); + + if (task.getStatus().equals(taskStatus)) { + // task status has not changed + continue; + } + + if (taskStatus == Status.SUCCEEDED) { + // register data and bind to task + Data data = new Data(); + data.setBinding(Map.of("jobId", job.getBingingId(), + "taskId", task.getTaskid(), + "taskBingingId", task.getBingingId(), + "componentName", task.getName(), + "role",fFateTask.getF_role(), + "partyId",fFateTask.getF_party_id())); + data.setFormat(DataMode.DENSE); + data.setCategory(DataCategory.RESULT_DATA); + getContext().getHighFlipRuntime().registerData(data); + log.info("data: {}", data); + task.setOutputData(List.of(data.getDataId())); + } + task.setStatus(taskStatus.name()); + getContext().getHighFlipRuntime().updateTask(task); + } + return job; + } + + private Status convertToHighFlipStatus(String fateStatus) { + switch (fateStatus) { + case "success": + return Status.SUCCEEDED; + case "timeout": + return Status.FAILED; + case "failed": + return Status.FAILED; + default: + return Status.UNKNOWN; + } } @Override - public boolean hasJob(Job job) { - return false; + public boolean hasJob(com.baidu.highflip.core.entity.runtime.Job job) { + throw new UnsupportedOperationException(); } @Override - public Status getJobStatus(Job job) { + public Status getJobStatus(com.baidu.highflip.core.entity.runtime.Job job) { String bindId = job.getBingingId(); - ResultForm> result = getContext() + ResultForm> result = getContext() .getClient() .jobQuery(bindId); String status = result.getData() .get(0) - .getStatus(); + .getF_status(); - return Status.valueOf(status); + if ("SUCCESS".equals(status.toUpperCase())) { + return Status.SUCCEEDED; + } else { + return Status.valueOf(status.toUpperCase()); + } } @Override - public void deleteJob(Job job) { - + public void deleteJob(com.baidu.highflip.core.entity.runtime.Job job) { + throw new UnsupportedOperationException(); } @Override - public Job controlJob(Job job, Action action) { + public com.baidu.highflip.core.entity.runtime.Job controlJob(com.baidu.highflip.core.entity.runtime.Job job, Action action) { switch (action) { case STOP: getContext().getClient().jobStop(job.getBingingId()); break; + default: + throw new UnsupportedOperationException(); } return null; } @Override public int getJobCount() { - return 0; + log.info("client get job count"); + return getContext().getClient().listJob(1, Integer.MAX_VALUE).getData().getCount(); } @Override - public Job getJobByIndex(int index, Job job) { - return null; + public com.baidu.highflip.core.entity.runtime.Job getJobByIndex(int index, com.baidu.highflip.core.entity.runtime.Job job) { + log.info("client get job by index"); + FateJob response = getContext().getClient().listJob(index + 1, 1).getData().getJobs().get(0); + return FateJob.convertToEntity(response); } @Override - public Optional moreJob(Job job, HighFlipRuntime runtime) { + public Optional moreJob(com.baidu.highflip.core.entity.runtime.Job job, HighFlipRuntime runtime) { return Optional.empty(); } @Override - public int getTaskCount(Job job) { - return 0; + public int getTaskCount(com.baidu.highflip.core.entity.runtime.Job job) { + return getContext().getClient().listTask(job.getBingingId(), 1, Integer.MAX_VALUE).getData().getCount(); } @Override - public List getTaskList(Job job, List task) { - return null; + public List getTaskList(com.baidu.highflip.core.entity.runtime.Job job, List tasks) { + List queryResult = getContext().getClient().listTask(job.getBingingId(), 1, Integer.MAX_VALUE).getData().getTasks().stream() + .map(FateTask::convertToEntity) + .collect(Collectors.toList()); + for (int i = 0; i < tasks.size(); i++) { + Task queryTask = queryResult.get(i); + Task ret = tasks.get(i); + queryTask.setTaskid(ret.getTaskid()); + BeanUtils.copyProperties(queryTask, ret); + } + return tasks; } @Override - public int getJobLogCount(Job job) { + public int getJobLogCount(com.baidu.highflip.core.entity.runtime.Job job) { return 0; } @Override - public Iterator getJobLog(Job job, int offset, int limit) { + public Iterator getJobLog(com.baidu.highflip.core.entity.runtime.Job job, int offset, int limit) { return null; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java index 17c3625..bb66014 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java @@ -1,4 +1,30 @@ package com.webank.ai.fate.adaptor; -public class OperatorAdaptor { +import com.baidu.highflip.core.entity.runtime.Operator; + +public class OperatorAdaptor implements com.baidu.highflip.core.adaptor.OperatorAdaptor { + @Override + public int getOperatorCount() { + return 0; + } + + @Override + public Operator getOperatorByIndex(int index, Operator operator) { + return null; + } + + @Override + public Operator updateOperator(Operator operator) { + return null; + } + + @Override + public boolean hasOperator(Operator operator) { + return false; + } + + @Override + public void deleteOperator(Operator operator) { + + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java index 0fb81dc..b8a1e5d 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java @@ -1,16 +1,98 @@ package com.webank.ai.fate.adaptor; +import com.baidu.highflip.core.entity.runtime.Task; +import com.baidu.highflip.core.entity.runtime.basic.Action; +import com.baidu.highflip.core.entity.runtime.basic.Status; +import com.baidu.highflip.core.exception.HighFlipException; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.FateTask; import com.webank.ai.fate.context.FateContext; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.util.CollectionUtils; + @Slf4j @Getter -public class TaskAdaptor { +public class TaskAdaptor implements com.baidu.highflip.core.adaptor.TaskAdaptor { FateContext context; public TaskAdaptor(FateContext context) { this.context = context; } + + @Override + public Task updateTask(Task task) { + return null; + } + + @Override + public void deleteTask(Task task) { + + } + + @Override + public boolean hasTask(Task task) { + return false; + } + + @Override + public Status getTaskStatus(Task task) { + final List tasks = + getContext().getClient().taskQuery(task.getBingingId()) + .getData().stream() + .map(FFateTask::convertToEntity) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(tasks)) { + throw new RuntimeException("task not found: " + task); + } + final String status = tasks.get(0).getStatus(); + switch (status) { + case "success": + return Status.SUCCEEDED; + case "timeout": + return Status.FAILED; + case "failed": + return Status.FAILED; + default: + return Status.UNKNOWN; + } + } + + @Override + public Task controlTask(Task task, Action action, Map config) { + return null; + } + + @Override + public int getTaskCount() { + return getContext().getClient().listTask(null, 1, Integer.MAX_VALUE).getData().getCount(); + } + + @Override + public Task getTaskByIndex(int index, Task task) { + log.info("get task by index:{}", index); + return FateTask.convertToEntity(getContext().getClient().listTask(null, index+1, 1).getData().getTasks().get(0)); + } + + @Override + public void invokeTask(Task task) { + + } + + @Override + public int getTaskLogCount(Task task) { + return 0; + } + + @Override + public Iterator getTaskLog(Task task, int offset, int limit) { + return null; + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java index 116a10b..3ea0b0c 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java @@ -1,85 +1,93 @@ package com.webank.ai.fate.client; -import com.webank.ai.fate.client.form.JobQueryResponseForm; -import com.webank.ai.fate.client.form.JsonResultForm; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.data.Data; +import com.webank.ai.fate.client.form.data.DataId; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.JobData; +import com.webank.ai.fate.client.form.job.QueryJob; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.TaskData; +import com.webank.ai.fate.common.deserializer.FeignSpringFormEncoder; import feign.Feign; import feign.Headers; +import feign.Logger; import feign.Param; import feign.Request; import feign.RequestLine; +import feign.Response; +import feign.httpclient.ApacheHttpClient; import feign.jackson.JacksonDecoder; -import feign.jackson.JacksonEncoder; +import org.springframework.lang.Nullable; +import org.springframework.web.multipart.MultipartFile; -import java.io.File; import java.util.List; import java.util.concurrent.TimeUnit; -@Headers("Content-Type: application/json") public interface FateClient { static FateClient connect(String url) { - return Feign.builder() - .decoder(new JacksonDecoder()) - .encoder(new JacksonEncoder()) - .options(new Request.Options( - 10, TimeUnit.SECONDS, - 60, TimeUnit.SECONDS, - true)) + return Feign.builder().client(new ApacheHttpClient()).decoder(new JacksonDecoder()) + .encoder(new FeignSpringFormEncoder()).logLevel(Logger.Level.NONE) + .options(new Request.Options(10, TimeUnit.SECONDS, 60, TimeUnit.SECONDS, true)) .target(FateClient.class, url); } - @RequestLine("POST /v1/version/get") - JsonResultForm version(); - - @RequestLine("POST /v1/version/get") - JsonResultForm versionGet(@Param("module") String module); - @RequestLine("POST /v1/job/submit") - String jobSumbit(@Param("job_dsl") String dsl, - @Param("job_runtime_conf") String conf); - - - @RequestLine("POST /v1/job/list/job") - JsonResultForm jobList(@Param("limit") Integer limit); + @Headers("Content-Type: application/json") + ResultForm jobSubmit(@Param("job_dsl") Dsl dsl, @Param("job_runtime_conf") DslConf conf); @RequestLine("POST /v1/job/stop") + @Headers("Content-Type: application/json") void jobStop(@Param("job_id") String jobId); @RequestLine("POST /v1/job/query") - ResultForm> jobQuery(@Param("job_id") String jobId); - - - /** - * - * @param head - * @param id_delimiter - * @param partition - * @param table_name - * @param namespace - * @param storage_engine - * @param destory - * @param extend_sid - * @return - */ + @Headers("Content-Type: application/json") + ResultForm> jobQuery(@Param("job_id") String jobId); + + @RequestLine("POST /v1/job/list/job") + @Headers("Content-Type: application/json") + ResultForm listJob(@Param("page") int page, @Param("limit") int limit); + + @RequestLine("POST /v1/job/task/query") + @Headers("Content-Type: application/json") + ResultForm> taskQuery(@Param("task_id") String taskId); + + @RequestLine("POST /v1/job/list/task") + @Headers("Content-Type: application/json") + ResultForm listTask(@Param("job_id") @Nullable String jobId, @Param("page") int page, + @Param("limit") int limit); + + @RequestLine("POST /v1/table/delete") + @Headers("Content-Type: application/json") + ResultForm deleteData(@Param(value = "table_name") String tableName, + @Param(value = "namespace") String nameSpace); + + @RequestLine("GET /v1/table/download") + @Headers("Content-Type: application/json") + Response downloadData(@Param(value = "name") String tableName, @Param(value = "namespace") String nameSpace); + + @RequestLine("POST /v1/data/upload") @Headers("Content-Type: application/octet-stream") - @RequestLine("POST /data/upload") - JsonResultForm dataUpload( - @Param("file") File file, - @Param("head") Integer head, - @Param("id_delimiter") String idDelimiter, - @Param("partition") Integer partition, - @Param("table_name") String tableName, - @Param("namespace") String namespace, - @Param("storage_engine") String storageEngine, - @Param("destory") Integer destory, - @Param("extend_sid") Integer extendSid); - - - @RequestLine("POST /data/download") - JsonResultForm dataDownload( - @Param("table_name") String tableName, - @Param("namespace") String namespace, - @Param("output_path") String outputPath, - @Param("delimiter") String delimiter); + ResultForm pushData(@Param(value = "file") MultipartFile file, + @Param(value = "id_delimiter") String id_delimiter, @Param(value = "head") String head, + @Param(value = "partition") String partition, @Param(value = "table_name") String table_name, + @Param(value = "namespace") String namespace, @Param(value = "storage_engine") String storage_engine, + @Param(value = "destroy") String destroy, @Param(value = "extend_sid") String extend_sid, + @Param(value = "drop") String drop); + + @RequestLine("POST /v1/data/upload/history") + @Headers("Content-Type: application/json") + ResultForm getDataUploadHistory(@Param(value = "limit") int limit, + @Param(value = "job_id") String job_id); + + @RequestLine("GET /v1/tracking/component/output/data/download") + @Headers("Content-Type: application/json") + Response downloadComponentResultData(@Param(value = "job_id") String tableName, + @Param(value = "component_name") String component_name, + @Param(value = "role") String role, + @Param(value = "party_id") String party_id); + } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java new file mode 100644 index 0000000..2bacde1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java @@ -0,0 +1,45 @@ +package com.webank.ai.fate.client.form; + +import com.baidu.highflip.core.entity.runtime.basic.Status; + +public enum FateStatus { + + WAITING("waiting"), + RUNNING("running"), + SUCCESS("success"), + FAILED("failed"), + ; + FateStatus(String value) { + this.value = value; + } + + private final String value; + + public String getValue() { + return value; + } + + public static FateStatus fromValue(String value) { + for (FateStatus fateStatus : FateStatus.values()) { + if (fateStatus.getValue().equalsIgnoreCase(value)) { + return fateStatus; + } + } + throw new IllegalArgumentException(); + } + + public static Status convertToEntity(FateStatus fateStatus) { + switch (fateStatus) { + case WAITING: + return Status.APPENDING; + case RUNNING: + return Status.RUNNING; + case FAILED: + return Status.FAILED; + case SUCCESS: + return Status.SUCCEEDED; + } + throw new IllegalArgumentException(); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java new file mode 100644 index 0000000..51c7b64 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.data; + +import java.util.Map; + +@lombok.Data +public class Data { + + private String board_url; + private int code; + private String dsl_path; + private String job_id; + private String logs_directory; + private String message; + private Map model_info; + private String namespace; + private String pipeline_dsl_path; + private String runtime_conf_on_party_path; + private String runtime_conf_path; + private String table_name; + private String train_runtime_conf_path; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java new file mode 100644 index 0000000..a44fcf5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.data; + +import lombok.Data; + +@Data +public class DataId { + + private String table_name; + + private String namespaces; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java new file mode 100644 index 0000000..30b6179 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java @@ -0,0 +1,14 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +@Data +public class Component { + + private String module; + + private Input input; + + private Output output; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java new file mode 100644 index 0000000..5811b18 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java @@ -0,0 +1,17 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapMapObjectDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class ComponentParameters { + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> common; + + private RoleConf role; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java new file mode 100644 index 0000000..39f40fa --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Components { + + Map components; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java new file mode 100644 index 0000000..d90467c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java @@ -0,0 +1,15 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapComponentsDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Dsl { + + @JsonDeserialize(using = JsonMapComponentsDeserializer.class) + Map components; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java new file mode 100644 index 0000000..57498ef --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java @@ -0,0 +1,22 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapListStringDeserializer; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class DslConf { + + private String dsl_version; + + private Site initiator; + + @JsonDeserialize(using = JsonMapListStringDeserializer.class) + private Map> role; + + private ComponentParameters component_parameters; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java new file mode 100644 index 0000000..051fa04 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java @@ -0,0 +1,23 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import com.webank.ai.fate.common.deserializer.JsonMapListStringDeserializer; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class Input { + + @JsonDeserialize(using = JsonMapListStringDeserializer.class) + private Map> data; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List model; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List cache; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java new file mode 100644 index 0000000..c42906e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import lombok.Data; + +import java.util.List; + +@Data +public class Output { + + @JsonDeserialize(using = JsonListDeserializer.class) + private List data; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List model; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List cache; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java new file mode 100644 index 0000000..ecc2fbe --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapMapObjectDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class RoleConf { + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> host; + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> guest; + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> arbiter; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java new file mode 100644 index 0000000..6c048fd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +@Data +public class Site { + + private String role; + + private String party_id; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java new file mode 100644 index 0000000..8fe798b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java @@ -0,0 +1,22 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Adaptation_parameters { + + private boolean if_initiator_baseline; + + private int request_task_cores; + + private int task_cores_per_node; + + private int task_memory_per_node; + + private int task_nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java new file mode 100644 index 0000000..89cbcac --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java @@ -0,0 +1,14 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Common { + + private Hetero_feature_binning hetero_feature_binning_0; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java new file mode 100644 index 0000000..74fe323 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java @@ -0,0 +1,49 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Components { + + private Dataio dataio_0; + + private Hetero_feature_binning hetero_feature_binning_0; + + private Intersection intersection_0; + + private Reader reader_0; + + public void setDataio_0(Dataio dataio_0) { + this.dataio_0 = dataio_0; + } + + public Dataio getDataio_0() { + return dataio_0; + } + + public void setHetero_feature_binning_0(Hetero_feature_binning hetero_feature_binning_0) { + this.hetero_feature_binning_0 = hetero_feature_binning_0; + } + + public Hetero_feature_binning getHetero_feature_binning_0() { + return hetero_feature_binning_0; + } + + public void setIntersection_0(Intersection intersection_0) { + this.intersection_0 = intersection_0; + } + + public Intersection getIntersection_0() { + return intersection_0; + } + + public void setReader_0(Reader reader_0) { + this.reader_0 = reader_0; + } + + public Reader getReader_0() { + return reader_0; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java new file mode 100644 index 0000000..9456eae --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Computing { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java new file mode 100644 index 0000000..89c7093 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Dataio { + + private boolean with_label; + + public void setWith_label(boolean with_label) { + this.with_label = with_label; + } + + public boolean getWith_label() { + return with_label; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java new file mode 100644 index 0000000..bdd2bd9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Eggroll_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java new file mode 100644 index 0000000..97f5d33 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Engines_address { + + private Computing computing; + + private Federation federation; + + private Storage storage; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java new file mode 100644 index 0000000..6b640fd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java @@ -0,0 +1,133 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.baidu.highflip.core.entity.runtime.basic.Status; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.FateStatus; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import com.webank.ai.fate.common.deserializer.JsonStringDeserializer; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.joda.time.DateTime; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Slf4j +@Data +public class FateJob { + + private long apply_resource_time; + + private boolean cancel_signal; + + private String cancel_time; + + private int cores; + + private String create_date; + + private Date create_time; + + private String description; + + private Dsl dsl; + + private long elapsed; + + private String end_date; + + private int end_scheduling_updates; + + private Date end_time; + + private String engine_name; + + private String engine_type; + + private String initiator_party_id; + + private String initiator_role; + + private boolean is_initiator; + + private String job_id; + + private int memory; + + private String name; + + private List partners; + + private int party_id; + + private int progress; + + private boolean ready_signal; + + private String ready_time; + + private int remaining_cores; + + private int remaining_memory; + + private boolean rerun_signal; + + private boolean resource_in_use; + + private long return_resource_time; + + @JsonDeserialize(using = JsonStringDeserializer.class) + private String role; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map roles; + + private Runtime_conf runtime_conf; + + private Runtime_conf_on_party runtime_conf_on_party; + + private String start_date; + + private Date start_time; + + private String status; + + private String status_code; + + private String tag; + + private Train_runtime_conf train_runtime_conf; + + private String update_date; + + private Date update_time; + + private User user; + + private String user_id; + + public static com.baidu.highflip.core.entity.runtime.Job convertToEntity(FateJob response) { + com.baidu.highflip.core.entity.runtime.Job job = new com.baidu.highflip.core.entity.runtime.Job(); +// job.setJobId(); + job.setJobName(response.getName()); + job.setDescription(response.getDescription()); + job.setCreateTime(new DateTime(response.getCreate_time())); + job.setUpdateTime(new DateTime(response.getUpdate_time())); + job.setFinishTime(new DateTime(response.getEnd_time())); +// job.setGraph(); + job.setStatus(FateStatus.convertToEntity(FateStatus.fromValue(response.getStatus()))); +// job.setMessage(); +// job.setIsDeleted(); +// job.setInputTasks(); +// job.setOutputTasks(); + job.setBingingId(response.getJob_id()); +// job.setBinding(); + return job; + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java new file mode 100644 index 0000000..9fa754e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Federation { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java new file mode 100644 index 0000000..00920c5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Hetero_feature_binning { + + private Transform_param transform_param; + + public void setTransform_param(Transform_param transform_param) { + this.transform_param = transform_param; + } + + public Transform_param getTransform_param() { + return transform_param; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java new file mode 100644 index 0000000..5db61a9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java @@ -0,0 +1,21 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; +@Data +public class Input { + + private InputData data; + + @Data + public static class InputData { + + private List data; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java new file mode 100644 index 0000000..f0d298d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java @@ -0,0 +1,39 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Intersection { + + private Input input; + + private String module; + + private Output output; + + public void setInput(Input input) { + this.input = input; + } + + public Input getInput() { + return input; + } + + public void setModule(String module) { + this.module = module; + } + + public String getModule() { + return module; + } + + public void setOutput(Output output) { + this.output = output; + } + + public Output getOutput() { + return output; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java new file mode 100644 index 0000000..b9937ec --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; + +@Data +public class JobData { + + private int count; + + private List jobs; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java new file mode 100644 index 0000000..8631a12 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java @@ -0,0 +1,50 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Job_parameters { + + private Adaptation_parameters adaptation_parameters; + + private int auto_retries; + + private int auto_retry_delay; + + private String computing_engine; + + private int computing_partitions; + + private Eggroll_run eggroll_run; + + private Engines_address engines_address; + + private String federated_mode; + + private String federated_status_collect_type; + + private String federation_engine; + + private String job_type; + + private String model_id; + + private String model_version; + + private Map pulsar_run; + + private Map rabbitmq_run; + + private Map spark_run; + + private String storage_engine; + + private int task_parallelism; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java new file mode 100644 index 0000000..7844095 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java @@ -0,0 +1,21 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import java.util.List; + +public class Output { + + private List data; + + public void setData(List data) { + this.data = data; + } + + public List getData() { + return data; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java new file mode 100644 index 0000000..2837c57 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Pulsar_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java new file mode 100644 index 0000000..8e758c4 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java @@ -0,0 +1,106 @@ +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import com.webank.ai.fate.common.deserializer.JsonStringDeserializer; +import lombok.Data; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Data +public class QueryJob { + + private long f_apply_resource_time; + + private boolean f_cancel_signal; + + private String f_cancel_time; + + private int f_cores; + + private String f_create_date; + + private Date f_create_time; + + private String f_description; + + private Dsl f_dsl; + + private long f_elapsed; + + private String f_end_date; + + private int f_end_scheduling_updates; + + private Date f_end_time; + + private String f_engine_name; + + private String f_engine_type; + + private String f_initiator_party_id; + + private String f_initiator_role; + + private boolean f_is_initiator; + + private String f_job_id; + + private int f_memory; + + private String f_name; + + private List f_partners; + + private int f_party_id; + + private int f_progress; + + private boolean f_ready_signal; + + private String f_ready_time; + + private int f_remaining_cores; + + private int f_remaining_memory; + + private boolean f_rerun_signal; + + private boolean f_resource_in_use; + + private long f_return_resource_time; + + @JsonDeserialize(using = JsonStringDeserializer.class) + private String f_role; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map f_roles; + + private Runtime_conf f_runtime_conf; + + private Runtime_conf_on_party f_runtime_conf_on_party; + + private String f_start_date; + + private Date f_start_time; + + private String f_status; + + private String f_status_code; + + private String f_tag; + + private Train_runtime_conf f_train_runtime_conf; + + private String f_update_date; + + private Date f_update_time; + + private User f_user; + + private String f_user_id; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java new file mode 100644 index 0000000..69e7e6b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Rabbitmq_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java new file mode 100644 index 0000000..5ca5261 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Reader { + + private Table table; + + public void setTable(Table table) { + this.table = table; + } + + public Table getTable() { + return table; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java new file mode 100644 index 0000000..5b41ac9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; + +@Data +public class Roles { + + private List guest; + + private List host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java new file mode 100644 index 0000000..302937b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java @@ -0,0 +1,34 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Runtime_conf { + + private ComponentParameters component_parameters; + + private String conf_path; + + private String dsl_path; + + private int dsl_version; + + private Site initiator; + + private Job_parameters job_parameters; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map role; + + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java new file mode 100644 index 0000000..3a4c14c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java @@ -0,0 +1,33 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Runtime_conf_on_party { + + private ComponentParameters component_parameters; + + private String conf_path; + + private String dsl_path; + + private int dsl_version; + + private Site initiator; + + private Job_parameters job_parameters; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java new file mode 100644 index 0000000..09d7be4 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Spark_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java new file mode 100644 index 0000000..70df472 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Storage { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java new file mode 100644 index 0000000..93866f6 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java @@ -0,0 +1,17 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + + +import lombok.Data; + +@Data +public class Table { + + private String name; + + private String namespace; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java new file mode 100644 index 0000000..ae6c312 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Train_runtime_conf { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java new file mode 100644 index 0000000..166cc37 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java @@ -0,0 +1,14 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Transform_param { + + private String transform_type; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java new file mode 100644 index 0000000..04c9b02 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java @@ -0,0 +1,18 @@ +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class User { + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map guest; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java new file mode 100644 index 0000000..011bd6f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + +@Data +public class Adaptation_parameters { + + private boolean if_initiator_baseline; + private int request_task_cores; + private int task_cores_per_node; + private int task_memory_per_node; + private int task_nodes; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java new file mode 100644 index 0000000..b4228cf --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java @@ -0,0 +1,16 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + + +@Data +public class Class_path { + + private String feature_instance; + private String feature_vector; + private String homo_model_convert; + private String Interface; + private String model; + private String model_migrate; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java new file mode 100644 index 0000000..7afbc31 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java @@ -0,0 +1,28 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Common { + + private Adaptation_parameters adaptation_parameters; + private int auto_retries; + private int auto_retry_delay; + private String computing_engine; + private int computing_partitions; + private Eggroll_run eggroll_run; + private Engines_address engines_address; + private String federated_mode; + private String federated_status_collect_type; + private String federation_engine; + private String job_type; + private String model_id; + private String model_version; + private Pulsar_run pulsar_run; + private Rabbitmq_run rabbitmq_run; + private Spark_run spark_run; + private String storage_engine; + private int task_parallelism; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java new file mode 100644 index 0000000..b6de203 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java @@ -0,0 +1,31 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class ComponentParam { + + private List _feeded_deprecated_params; + private boolean _is_raw_conf; + private String _name; + private List _user_feeded_params; + private double adjustment_factor; + private int bin_indexes; + private String bin_names; + private int bin_num; + private String category_indexes; + private String category_names; + private int compress_thres; + private Encrypt_param encrypt_param; + private double error; + private int head_size; + private boolean local_only; + private String method; + private boolean need_run; + private Optimal_binning_param optimal_binning_param; + private boolean skip_static; + private Transform_param transform_param; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java new file mode 100644 index 0000000..51cde37 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java @@ -0,0 +1,21 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Component_parameters { + + private String CodePath; + private ComponentParam ComponentParam; + private String conf_path; + private String dsl_path; + private int dsl_version; + private Initiator initiator; + private Job_parameters job_parameters; + private Local local; + private String module; + private Role role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java new file mode 100644 index 0000000..bfe6f19 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Eggroll_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java new file mode 100644 index 0000000..d64a45e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Encrypt_param { + + private int key_length; + private String method; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java new file mode 100644 index 0000000..c710198 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java @@ -0,0 +1,12 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Engine_conf { + + private String computing_engine; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java new file mode 100644 index 0000000..bba10cf --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Engines_address { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java new file mode 100644 index 0000000..76b0cee --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java @@ -0,0 +1,11 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Env { + + private String PYTHONPATH; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java new file mode 100644 index 0000000..8aca632 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java @@ -0,0 +1,64 @@ +package com.webank.ai.fate.client.form.task; + +import java.util.Date; +import java.util.List; + +import org.joda.time.DateTime; + +import com.baidu.highflip.core.entity.runtime.Task; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; + +import lombok.Data; + +@Data +public class FFateTask { + private int f_auto_retries; + private int f_auto_retry_delay; + @JsonDeserialize(using = JsonListDeserializer.class) + private List f_cmd; + private String f_component_module; + private String f_component_name; + private Component_parameters f_component_parameters; + private Date f_create_time; + private int f_elapsed; + private Date f_end_time; + private Engine_conf f_engine_conf; + private String f_federated_mode; + private String f_federated_status_collect_type; + private String f_initiator_party_id; + private String f_initiator_role; + // job_id is the job id in FATE + private String f_job_id; + private String f_party_id; + private String f_party_status; + private Provider_info f_provider_info; + private String f_role; + private String f_run_ip; + private boolean f_run_on_this_party; + private int f_run_pid; + private Date f_start_time; + /** + * f_status is one of waiting,running,canceled,timeout,failed,pass,success + */ + private String f_status; + private String f_status_code; + private String f_task_id; + private int f_task_version; + private Date f_update_time; + private String f_worker_id; + + public static Task convertToEntity(FFateTask data) { + Task task = new Task(); + task.setJobid(data.getF_job_id()); + task.setName(data.getF_component_name()); + task.setCreateTime(new DateTime(data.getF_create_time())); + task.setUpdateTime(new DateTime(data.getF_update_time())); + task.setFinishTime(new DateTime(data.getF_end_time())); + task.setNodeName(data.getF_component_name()); + task.setStatus(data.getF_status()); + task.setBingingId(data.getF_task_id()); + return task; + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java new file mode 100644 index 0000000..975a4ff --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java @@ -0,0 +1,69 @@ + +package com.webank.ai.fate.client.form.task; + +import com.baidu.highflip.core.entity.runtime.Task; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import lombok.Data; +import org.joda.time.DateTime; + +import java.util.Date; +import java.util.List; + + +@Data +public class FateTask { + + private int auto_retries; + private int auto_retry_delay; + @JsonDeserialize(using = JsonListDeserializer.class) + private List cmd; + private String component_module; + private String component_name; + private Component_parameters component_parameters; + private Date create_time; + private int elapsed; + private Date end_time; + private Engine_conf engine_conf; + private String federated_mode; + private String federated_status_collect_type; + private String initiator_party_id; + private String initiator_role; + // job_id is the job id in FATE + private String job_id; + private String party_id; + private String party_status; + private Provider_info provider_info; + private String role; + private String run_ip; + private boolean run_on_this_party; + private int run_pid; + private Date start_time; + private String status; + private String status_code; + private String task_id; + private int task_version; + private Date update_time; + private String worker_id; + + public static Task convertToEntity(FateTask data) { + Task task = new Task(); +// task.setTaskid(); + task.setJobid(data.getJob_id()); + task.setName(data.getComponent_name()); +// task.setDescription(); + task.setCreateTime(new DateTime(data.getCreate_time())); + task.setUpdateTime(new DateTime(data.getUpdate_time())); + task.setFinishTime(new DateTime(data.getEnd_time())); + task.setNodeName(data.getComponent_name()); + task.setStatus(data.getStatus()); +// task.setMessage(); +// task.setIsDeleted(); +// task.setPrevious(); +// task.setNext(); + task.setBingingId(data.getTask_id()); +// task.setBinding(); + return task; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java new file mode 100644 index 0000000..d4746fc --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Initiator { + + private int party_id; + private String role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java new file mode 100644 index 0000000..35e1cb7 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java @@ -0,0 +1,12 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Job_parameters { + + private Common common; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java new file mode 100644 index 0000000..2e66e13 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Local { + + private int party_id; + private String role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java new file mode 100644 index 0000000..9cdaf7d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java @@ -0,0 +1,18 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Optimal_binning_param { + + private String adjustment_factor; + private int init_bin_nums; + private String init_bucket_method; + private String max_bin; + private int max_bin_pct; + private String metric_method; + private double min_bin_pct; + private boolean mixture; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java new file mode 100644 index 0000000..211fab1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java @@ -0,0 +1,15 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + + +@Data +public class Provider_info { + + private Class_path class_path; + private Env env; + private String name; + private String path; + private String version; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java new file mode 100644 index 0000000..a5b6e13 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Pulsar_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java new file mode 100644 index 0000000..ef0032b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Rabbitmq_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java new file mode 100644 index 0000000..51c34d1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class Role { + + private List guest; + private List host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java new file mode 100644 index 0000000..6412c4f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Spark_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java new file mode 100644 index 0000000..9c3a8c5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class TaskData { + + private int count; + private List tasks; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java new file mode 100644 index 0000000..5fefc0f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.Date; + + +@Data +public class Transform_param { + + private Date transform_cols; + private String transform_names; + private String transform_type; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java new file mode 100644 index 0000000..ab137c1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java @@ -0,0 +1,7 @@ +package com.webank.ai.fate.common; + +public interface ConfigConst { + + String FLOW_URL_CONFIG="highflip.adaptor.fate.fate-flow.url"; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java new file mode 100644 index 0000000..ed844dd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java @@ -0,0 +1,76 @@ +package com.webank.ai.fate.common; + +import org.springframework.lang.NonNull; +import org.springframework.util.Base64Utils; +import org.springframework.util.FileCopyUtils; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +public class DataMultipartFile implements MultipartFile { + + private String originalFilename = "file"; + + private final String contentType = "application/octet-stream"; + + private final byte[] content; + + public DataMultipartFile(String filename, byte[] bytes) { + if (filename != null) { + this.originalFilename = filename; + } + this.content = bytes; + } + + public DataMultipartFile(String filename, String base64str) { + if (filename != null) { + this.originalFilename = filename; + } + this.content = Base64Utils.decodeFromString(base64str); + } + + @Override + @NonNull + public String getName() { + return this.originalFilename; + } + + @Override + public String getOriginalFilename() { + return this.originalFilename; + } + + @Override + public String getContentType() { + return this.contentType; + } + + @Override + public boolean isEmpty() { + return (this.content.length == 0); + } + + @Override + public long getSize() { + return this.content.length; + } + + @Override + public byte[] getBytes() throws IOException { + return this.content; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.content); + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + FileCopyUtils.copy(this.content, dest); + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java new file mode 100644 index 0000000..aa2d2b1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java @@ -0,0 +1,39 @@ +package com.webank.ai.fate.common; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.springframework.lang.Nullable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + +public class DecompressUtils { + + public static Map decompressTarGzToStringMap(InputStream tarGzInputStream, @Nullable Function fileNameFilter) { + if (fileNameFilter == null) { + fileNameFilter = s -> true; + } + Map result = new HashMap<>(); + try (GZIPInputStream gis = new GZIPInputStream(tarGzInputStream);TarArchiveInputStream tais = new TarArchiveInputStream(gis)) { + int i = 0; + TarArchiveEntry entry; + while ((entry = tais.getNextTarEntry()) != null) { + if (fileNameFilter.apply(entry.getName())) { + String text = new BufferedReader(new InputStreamReader(tais, StandardCharsets.UTF_8)).lines().collect(Collectors.joining("\n")); + result.put(entry.getName(), text); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return result; + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java new file mode 100644 index 0000000..7931da6 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java @@ -0,0 +1,7 @@ +package com.webank.ai.fate.common; + +public class FateConstants { + public static final String DATA_ID_SEPARATOR = "\\+"; + + public static final String DATA_ID_CONNECTOR = "+"; +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java new file mode 100644 index 0000000..bbc891f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java @@ -0,0 +1,84 @@ +package com.webank.ai.fate.common; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class GZIPUtils { + public static final String GZIP_ENCODE_UTF_8 = "UTF-8"; + public static final String GZIP_ENCODE_ISO_8859_1 = "ISO-8859-1"; + + + public static byte[] compress(String str, String encoding) { + if (str == null || str.length() == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip; + try { + gzip = new GZIPOutputStream(out); + gzip.write(str.getBytes(encoding)); + gzip.close(); + } catch ( Exception e) { + e.printStackTrace(); + } + return out.toByteArray(); + } + + public static byte[] compress(String str) throws IOException { + return compress(str, GZIP_ENCODE_UTF_8); + } + + public static byte[] uncompress(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[256]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + } catch (Exception e) { + e.printStackTrace(); + } + return out.toByteArray(); + } + + public static String uncompressToString(byte[] bytes, String encoding) { + if (bytes == null || bytes.length == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[256]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + return out.toString(encoding); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static String uncompressToString(byte[] bytes) { + return uncompressToString(bytes, GZIP_ENCODE_UTF_8); + } + + public static void main(String[] args) throws IOException { + String s = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + System.out.println("字符串长度："+s.length()); + System.out.println("压缩后：："+compress(s).length); + System.out.println("解压后："+uncompress(compress(s)).length); + System.out.println("解压字符串后：："+uncompressToString(compress(s)).length()); + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java new file mode 100644 index 0000000..5da179c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java @@ -0,0 +1,255 @@ +package com.webank.ai.fate.common.deserializer; + +import feign.RequestTemplate; +import feign.codec.EncodeException; +import feign.codec.Encoder; +import feign.jackson.JacksonEncoder; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class FeignSpringFormEncoder extends JacksonEncoder implements Encoder { + + private final List> converters = new RestTemplate().getMessageConverters(); + + public static final Charset UTF_8 = Charset.forName("UTF-8"); + + public FeignSpringFormEncoder() {} + + /** + * 实现一个 HttpOutputMessage + */ + private class HttpOutputMessageImpl implements HttpOutputMessage { + /** + * 输出流，请求体 + */ + private final OutputStream body; + /** + * 请求头 + */ + private final HttpHeaders headers; + + public HttpOutputMessageImpl(OutputStream body, HttpHeaders headers) { + this.body = body; + this.headers = headers; + } + + @Override + public OutputStream getBody() throws IOException { + return body; + } + + @Override + public HttpHeaders getHeaders() { + return headers; + } + } + + /** + * 内部静态类,保存 MultipartFile 数据 + */ + static class MultipartFileResource extends InputStreamResource { + /** + * 文件名 + */ + private final String filename; + /** + * 文件大小 + */ + private final long size; + + /** + * 构造方法 + */ + public MultipartFileResource(InputStream inputStream, String filename, long size) { + super(inputStream); + this.filename = filename; + this.size = size; + } + + @Override + public String getFilename() { + return this.filename; + } + + @Override + public InputStream getInputStream() throws IOException, IllegalStateException { + return super.getInputStream(); + } + + @Override + public long contentLength() throws IOException { + return size; + } + } + + /** + * 重写编码器 + */ + @Override + public void encode(Object object, Type bodyType, RequestTemplate template) throws EncodeException { + if (isJsonRequest(template)){ + super.encode(object, bodyType, template); + } else { + final HttpHeaders multipartHeaders = new HttpHeaders(); + multipartHeaders.setContentType(MediaType.MULTIPART_FORM_DATA); + encodeMultipartFormRequest((Map) object, multipartHeaders, template); + } + } + + private boolean isJsonRequest(RequestTemplate requestTemplate) { + for (Map.Entry> header : requestTemplate.headers().entrySet()) { + if (header.getKey().equalsIgnoreCase("Content-Type")) { + for (String type : header.getValue()) { + if (type.contains("json")) { + return true; + } + } + } + } + return false; + } + + /** + * 对有文件、表单的进行编码 + */ + private void encodeMultipartFormRequest(Map formMap, HttpHeaders multipartHeaders, RequestTemplate template){ + if (formMap == null){ + throw new EncodeException("无法对格式为null的请求进行编码。"); + } + + LinkedMultiValueMap map = new LinkedMultiValueMap<>(); + //对每个参数进行检查校验 + for (Map.Entry entry : formMap.entrySet()){ + Object value = entry.getValue(); + //不同的数据类型进行不同的编码逻辑处理 + if (isMultipartFile(value)){ + //单个文件 + map.add(entry.getKey(), encodeMultipartFile((MultipartFile)value)); + + } else if (isMultipartFileArray(value)){ + //多个文件 + encodeMultipartFiles(map, (String) entry.getKey(), Arrays.asList((MultipartFile[]) value)); + + } else { + //普通请求数据 + map.add(entry.getKey(), encodeJsonObject(value)); + } + } + + encodeRequest(map, multipartHeaders, template); + } + + /** + * 对请求进行编码 + */ + private void encodeRequest(Object value, HttpHeaders requestHeaders, RequestTemplate template){ + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + HttpOutputMessage dummyRequest = new HttpOutputMessageImpl(outputStream, requestHeaders); + + try { + Class requestType = value.getClass(); + MediaType requestContentType = requestHeaders.getContentType(); + for (HttpMessageConverter messageConverter : converters){ + if (messageConverter.canWrite(requestType, requestContentType)){ + ((HttpMessageConverter) messageConverter).write(value, requestContentType, dummyRequest); + break; + } + } + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + + HttpHeaders headers = dummyRequest.getHeaders(); + if (headers != null){ + for (Map.Entry> entry : headers.entrySet()){ + template.header(entry.getKey(), entry.getValue()); + } + } + + /* + 请使用模板输出流。。。如果文件太大，这将导致问题，因为整个请求都将在内存中。 + */ + template.body(outputStream.toByteArray(), UTF_8); + } + + /** + * 编码为json对象 + */ + private HttpEntity encodeJsonObject(Object obj){ + HttpHeaders jsonPartHeaders = new HttpHeaders(); + jsonPartHeaders.setContentType(MediaType.APPLICATION_JSON); + return new HttpEntity<>(obj, jsonPartHeaders); + } + + /** + * 编码MultipartFile文件，将其转换为HttpEntity，同时设置 Content-type 为 application/octet-stream + * @param map 当前请求 map. + * @param name 数组字段的名称 + * @param fileList 要处理的文件 + */ + private void encodeMultipartFiles(LinkedMultiValueMap map, String name, List fileList){ + HttpHeaders filePartHeaders = new HttpHeaders(); + //设置 Content-type + filePartHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); + try { + for (MultipartFile file : fileList){ + Resource multipartFileResource = new MultipartFileResource(file.getInputStream(), file.getOriginalFilename(), file.getSize()); + map.add(name, new HttpEntity<>(multipartFileResource, filePartHeaders)); + } + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + } + + /** + * 编码MultipartFile文件，将其转换为HttpEntity，同时设置 Content-type 为 application/octet-stream + * @param file 要编码的文件 + */ + private HttpEntity encodeMultipartFile(MultipartFile file){ + HttpHeaders filePartHeaders = new HttpHeaders(); + //设置 Content-type + filePartHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); + try { + Resource multipartFileResource = new MultipartFileResource(file.getInputStream(), file.getOriginalFilename(), file.getSize()); + return new HttpEntity<>(multipartFileResource, filePartHeaders); + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + } + + /** + * 判断是否多个 MultipartFile + */ + private boolean isMultipartFileArray(Object object){ + return object != null && object.getClass().isArray() && MultipartFile.class.isAssignableFrom(object.getClass().getComponentType()); + } + + /** + * 判断是否MultipartFile文件 + * @param object 要判断的对象 + */ + private boolean isMultipartFile(Object object){ + return object instanceof MultipartFile; + } + + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java new file mode 100644 index 0000000..33f2b0d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java @@ -0,0 +1,34 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class JsonListDeserializer extends StdDeserializer> { + + public JsonListDeserializer() { + super(List.class); + } + + @Override + public List deserialize(JsonParser parser, DeserializationContext context) + throws IOException { + JsonNode node = parser.getCodec().readTree(parser); + List result = new ArrayList<>(); + if (node.isArray()) { + for (JsonNode element : node) { + result.add(element.asText()); + } + } else if (node.isObject()) { + result.add(node.asText()); + } else { + //maybe nothing? + } + return result; + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java new file mode 100644 index 0000000..580d056 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java @@ -0,0 +1,36 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.webank.ai.fate.client.form.dsl.Component; + +import java.io.IOException; +import java.util.Map; + +public class JsonMapComponentsDeserializer extends StdDeserializer> { + + protected JsonMapComponentsDeserializer() { + super(Map.class); + } + + @Override + public Map deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { +// Map ret = new HashMap<>(); +// Map temp = jsonParser.getCodec().readValue(jsonParser, new TypeReference>() { +// }); +// for (String key : temp.keySet()) { +// Object tempValue = temp.get(key); +// if (tempValue != null) { +// ObjectMapper objectMapper = new ObjectMapper(); +// String decode = objectMapper.writeValueAsString(tempValue); +// Component temp1 = objectMapper.readValue(decode, new TypeReference() { +// }); +// ret.put(key, temp1); +// } +// } +// return ret; + return SerializerUtils.mapObjectDeserialize(jsonParser.getCodec().readTree(jsonParser).toString(), Component.class); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java new file mode 100644 index 0000000..cce54a5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java @@ -0,0 +1,38 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JsonMapListStringDeserializer extends StdDeserializer>> { + + protected JsonMapListStringDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + Map> ret = new HashMap<>(); + Map temp = jsonParser.getCodec().readValue(jsonParser, new TypeReference>() { + }); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + List temp1 = objectMapper.readValue(decode, new TypeReference>() { + }); + ret.put(key, temp1); + } + } + return ret; + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java new file mode 100644 index 0000000..0f37715 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapDeserializer extends StdDeserializer>> { + + public JsonMapMapDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + Map map = new HashMap<>(); + for (String key1 : temp.keySet()) { + if (temp1.get(key1) != null) { + map.put(key, temp1.get(key1).toString()); + } + } + ret.put(key, map); + } + } + return ret; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java new file mode 100644 index 0000000..c83131d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java @@ -0,0 +1,38 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapObjectDeserializer extends StdDeserializer>> { + + public JsonMapMapObjectDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + ret.put(key, temp1); + } + } + return ret; + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java new file mode 100644 index 0000000..3e3b543 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapStringDeserializer extends StdDeserializer>> { + + public JsonMapMapStringDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + Map map = new HashMap<>(); + for (String key1 : temp1.keySet()) { + if (temp1.get(key1) != null) { + map.put(key, temp1.get(key1).toString()); + } + } + ret.put(key, map); + } + } + return ret; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java new file mode 100644 index 0000000..4c555ac --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java @@ -0,0 +1,33 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapStringDeserializer extends StdDeserializer> { + + public JsonMapStringDeserializer(){ + super(Map.class); + } + + @Override + public Map deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map map = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + if (temp.get(key) != null) { + map.put(key, temp.get(key).toString()); + } + } + return map; +// return SerializerUtils.mapObjectDeserialize(jsonParser.getCodec().readTree(jsonParser).toString(), String.class); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java new file mode 100644 index 0000000..ec89c61 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java @@ -0,0 +1,23 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; + +public class JsonStringDeserializer extends StdDeserializer { + + public JsonStringDeserializer() { + super(String.class); + } + + @Override + public String deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + return node.toString().replace("\"", ""); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java new file mode 100644 index 0000000..356a417 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java @@ -0,0 +1,416 @@ +package com.webank.ai.fate.common.deserializer; + +import com.baidu.highflip.core.entity.dag.Graph; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.translator.DSLTranslator; +import org.springframework.lang.Nullable; + +import java.util.HashMap; +import java.util.Map; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, TypeReference typeReference) throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } + + public static void main(String[] args) throws JsonProcessingException { + Dsl dsl = SerializerUtils.deserialize("{\n" + " \"components\": {\n" + " \"reader_0\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"reader_1\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"data_transform_0\": {\n" + + " \"module\": \"DataTransform\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_0.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"data_transform_1\": {\n" + " \"module\": \"DataTransform\",\n" + + " \"input\": {\n" + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_1.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"intersection_0\": {\n" + " \"module\": \"Intersection\",\n" + + " \"input\": {\n" + " \"data\": {\n" + " \"data\": [\n" + + " \"data_transform_0.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"cache\": [\n" + " \"cache\"\n" + " ]\n" + + " }\n" + " },\n" + " \"intersection_1\": {\n" + + " \"module\": \"Intersection\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"data_transform_1.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"cache\": [\n" + " \"cache\"\n" + " ]\n" + + " }\n" + " },\n" + " \"hetero_nn_0\": {\n" + + " \"module\": \"HeteroNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"train_data\": [\n" + + " \"intersection_0.data\"\n" + " ],\n" + + " \"validate_data\": [\n" + " \"intersection_1.data\"\n" + + " ]\n" + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"model\": [\n" + " \"model\"\n" + " ]\n" + + " }\n" + " },\n" + " \"hetero_nn_1\": {\n" + + " \"module\": \"HeteroNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"test_data\": [\n" + + " \"intersection_1.data\"\n" + " ]\n" + + " },\n" + " \"model\": [\n" + + " \"hetero_nn_0.model\"\n" + " ]\n" + " },\n" + + " \"output\": {\n" + " \"data\": [\n" + " \"data\"\n" + + " ],\n" + " \"model\": [\n" + " \"model\"\n" + + " ]\n" + " }\n" + " },\n" + " \"evaluation_0\": {\n" + + " \"module\": \"Evaluation\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"hetero_nn_0.data\",\n" + " \"hetero_nn_1.data\"\n" + + " ]\n" + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " }\n" + " }\n" + "}", Dsl.class); + DslConf dslConf = SerializerUtils.deserialize("{\n" + " \"dsl_version\": 2,\n" + " \"role\": {\n" + + " \"host\": [\n" + " 10000\n" + " ],\n" + " \"guest\": [\n" + + " 9999\n" + " ]\n" + " },\n" + " \"component_parameters\": {\n" + + " \"role\": {\n" + " \"guest\": {\n" + " \"0\": {\n" + + " \"data_transform_0\": {\n" + " \"with_label\": true\n" + + " },\n" + " \"hetero_nn_0\": {\n" + + " \"bottom_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 10\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 10\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 3,\n" + + " \"activation\": \"relu\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"top_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_2\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_2_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_2\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 1,\n" + + " \"activation\": \"sigmoid\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"interactive_layer_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_1\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_1_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_1\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 2,\n" + + " \"activation\": \"linear\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " }\n" + + " },\n" + " \"reader_1\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_hetero_guest\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " },\n" + " \"data_transform_1\": {\n" + + " \"with_label\": true\n" + " },\n" + + " \"reader_0\": {\n" + " \"table\": {\n" + + " \"name\": \"breast_hetero_guest\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " }\n" + " }\n" + " },\n" + " \"host\": {\n" + + " \"0\": {\n" + " \"data_transform_0\": {\n" + + " \"with_label\": false\n" + " },\n" + + " \"hetero_nn_0\": {\n" + " \"bottom_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_3\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 20\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_3_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_3\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 20\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 3,\n" + + " \"activation\": \"relu\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"interactive_layer_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_4\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_4_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_4\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 2,\n" + + " \"activation\": \"linear\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " }\n" + + " },\n" + " \"reader_1\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_hetero_host\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " },\n" + " \"data_transform_1\": {\n" + + " \"with_label\": false\n" + " },\n" + + " \"reader_0\": {\n" + " \"table\": {\n" + + " \"name\": \"breast_hetero_host\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " }\n" + " }\n" + " }\n" + " },\n" + + " \"common\": {\n" + " \"hetero_nn_0\": {\n" + " \"epochs\": 3,\n" + + " \"interactive_layer_lr\": 0.15,\n" + " \"batch_size\": -1,\n" + + " \"early_stop\": \"diff\",\n" + " \"callback_param\": {\n" + + " \"callbacks\": [\n" + " \"ModelCheckpoint\"\n" + + " ],\n" + " \"save_freq\": \"epoch\",\n" + + " \"validation_freqs\": 1,\n" + + " \"use_first_metric_only\": true,\n" + + " \"early_stopping_rounds\": 15,\n" + " \"metrics\": [\n" + + " \"AUC\"\n" + " ]\n" + " },\n" + + " \"optimizer\": {\n" + " \"learning_rate\": 0.15,\n" + + " \"decay\": 0.0,\n" + " \"momentum\": 0.0,\n" + + " \"nesterov\": false,\n" + " \"optimizer\": \"SGD\"\n" + + " },\n" + " \"loss\": \"binary_crossentropy\",\n" + + " \"bottom_nn_define\": null,\n" + " \"top_nn_define\": null,\n" + + " \"interactive_layer_define\": null,\n" + + " \"config_type\": \"keras\"\n" + " },\n" + + " \"hetero_nn_1\": {\n" + " \"optimizer\": null,\n" + + " \"bottom_nn_define\": null,\n" + " \"top_nn_define\": null,\n" + + " \"interactive_layer_define\": null,\n" + + " \"config_type\": \"keras\"\n" + " }\n" + " }\n" + " }\n" + "}", DslConf.class); + DSLTranslator.FateDAG fateDAG = new DSLTranslator.FateDAG(); + fateDAG.setConf(dslConf); + fateDAG.setDsl(dsl); + DSLTranslator dslTranslator = new DSLTranslator(); + Graph graph = dslTranslator.translate(fateDAG); + DSLTranslator.FateDAG fateDAG1 = dslTranslator.translate(graph); + System.out.println(toJsonString(fateDAG1)); + System.out.println(toJsonString(fateDAG)); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java index d0d7353..9219ba2 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java @@ -1,7 +1,9 @@ package com.webank.ai.fate.context; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.Configuration; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.webank.ai.fate.client.FateClient; import com.webank.ai.fate.translator.DSLTranslator; @@ -18,6 +20,8 @@ public class FateContext { Configuration config; + HighFlipRuntime highFlipRuntime; + FateClient client; DSLTranslator translator = new DSLTranslator(); @@ -26,8 +30,16 @@ public static FateContext from(InstanceRegister register) { FateContext context = new FateContext(); Configuration config = (Configuration) register.getInstance(InstanceNameList.HIGHFLIP_CONFIGURATION); + HighFlipRuntime highFlipRuntime = + (HighFlipRuntime) register.getInstance(InstanceNameList.HIGHFLIP_RUNTIME); context.setConfig(config); - + context.setHighFlipRuntime(highFlipRuntime); + ServiceAdaptor serviceAdaptor = + (ServiceAdaptor) register.getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + log.info("connecting to fate flow http service:{}", serviceAdaptor.getUrl()); + context.setClient(FateClient.connect(serviceAdaptor.getUrl())); + context.getTranslator().setPartyId(serviceAdaptor.getPartyId()); + context.getTranslator().setRole(serviceAdaptor.getRole()); return context; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java index 117f719..3b8af09 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java @@ -1,17 +1,360 @@ package com.webank.ai.fate.translator; import com.baidu.highflip.core.entity.dag.Graph; +import com.baidu.highflip.core.entity.dag.Node; +import com.baidu.highflip.core.entity.dag.Party; +import com.baidu.highflip.core.entity.dag.PartyNode; +import com.baidu.highflip.core.entity.dag.codec.TypeValue; +import com.baidu.highflip.core.entity.dag.common.NodeInputRef; +import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.webank.ai.fate.client.form.dsl.Component; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.dsl.Input; +import com.webank.ai.fate.client.form.dsl.Output; +import com.webank.ai.fate.client.form.dsl.RoleConf; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.SerializerUtils; + +import highflip.HighflipMeta; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.util.CollectionUtils; + @Slf4j -@NoArgsConstructor +@Data public class DSLTranslator { + String partyId; + + String role; + public FateDAG translate(Graph dag) { - return new FateDAG(); + Map componentMap = new HashMap<>(); + Map> componentConfMap = new HashMap<>(); + Map> role = new HashMap<>(); + Map> hostConf = new HashMap<>(); + Map> guestConf = new HashMap<>(); + Map> arbiterConf = new HashMap<>(); + for (Node node : dag.getNodes().values()) { + Component component = new Component(); + component.setInput(transInput(node.getInputs())); + component.setOutput(transOutput(node.getOutputs())); + component.setModule(node.getType()); + Map conf = new HashMap<>(); + for (String key : node.getAttributes().keySet()) { + conf.put(key, TypeValue.fromProto( + (HighflipMeta.TypedValueProto) node.getAttribute(key, null))); + } + componentConfMap.put(node.getName(), conf); + componentMap.put(node.getName(), component); + } + int guestIndex = 0; + int hostIndex = 0; + int arbiterIndex = 0; + Map> roleMap = new HashMap<>(); + List guest = new ArrayList<>(); + List host = new ArrayList<>(); + List arbiter = new ArrayList<>(); + for (Party party : dag.getParties().values()) { + role.putIfAbsent(party.getRole(), new ArrayList<>()); + role.get(party.getRole()).add(party.getName()); + Map componentAttr = new HashMap<>(); + for (PartyNode node : party.getNodes().values()) { + Map attr = node.getAttributes(); + componentAttr.put(node.getName(), attr); + } + if (party.getRole().equalsIgnoreCase("host")) { + hostConf.put(hostIndex + "", componentAttr); + host.add(party.getName()); + hostIndex++; + } + if (party.getRole().equalsIgnoreCase("guest")) { + guestConf.put(guestIndex + "", componentAttr); + guest.add(party.getName()); + guestIndex++; + } + if (party.getRole().equalsIgnoreCase("arbiter")) { + arbiterConf.put(arbiterIndex + "", componentAttr); + arbiter.add(party.getName()); + arbiterIndex++; + } + } + if (!guest.isEmpty()) { + roleMap.put("guest", guest); + } + if (!host.isEmpty()) { + roleMap.put("host", host); + } + if (!arbiter.isEmpty()) { + roleMap.put("arbiter", arbiter); + } + Dsl dsl = new Dsl(); + dsl.setComponents(componentMap); + DslConf dslConf = new DslConf(); + dslConf.setDsl_version("2"); + dslConf.setRole(roleMap); + ComponentParameters componentParameters = new ComponentParameters(); + componentParameters.setCommon(componentConfMap); + RoleConf roleConf = new RoleConf(); + if (!hostConf.isEmpty()) { + roleConf.setHost(hostConf); + } + if (!guestConf.isEmpty()) { + roleConf.setGuest(guestConf); + } + if (!arbiterConf.isEmpty()) { + roleConf.setArbiter(arbiterConf); + } + componentParameters.setRole(roleConf); + dslConf.setComponent_parameters(componentParameters); + FateDAG fateDAG = new FateDAG(); + fateDAG.setDsl(dsl); + fateDAG.setConf(dslConf); + // set initiator to fateDAG + Site initiator = new Site(); + initiator.setRole(this.role); + initiator.setParty_id(this.partyId); + fateDAG.getConf().setInitiator(initiator); + return fateDAG; + } + + public Graph translate(FateDAG dag) { + Graph graph = new Graph(); + Dsl dsl = dag.getDsl(); + Node pre = null; + Map nodeMap = new HashMap<>(); + Map partyMap = new HashMap<>(); + Map> rolePartyMap = new HashMap<>(); + graph.setNodes(nodeMap); + graph.setParties(partyMap); + for (Map.Entry entity : dsl.getComponents().entrySet()) { + String key = entity.getKey(); + Component component = entity.getValue(); + Node node = new Node(); + node.setName(key); + node.setType(component.getModule()); + if (component.getInput() != null) { + node.setInputs(transInput(component.getInput())); + } + if (component.getOutput() != null) { + node.setOutputs(transOutput(component.getOutput())); + } + if (pre != null) { + node.setParent(pre); + } + pre = node; + nodeMap.put(key, node); + } + DslConf dslConf = dag.getConf(); + for (Map.Entry> roleEntity : dslConf.getRole().entrySet()) { + String role = roleEntity.getKey(); + rolePartyMap.putIfAbsent(role.toLowerCase(), new ArrayList<>()); + for (String partyId : roleEntity.getValue()) { + Party party = new Party(); + party.setName(partyId); + party.setRole(role); + party.setNodes(new HashMap<>()); + partyMap.put(role + "-" + partyId, party); + rolePartyMap.get(role.toLowerCase()).add(party); + } + } + ComponentParameters componentParameters = dslConf.getComponent_parameters(); + for (Map.Entry> attrMap : componentParameters.getCommon().entrySet()) { + String componentName = attrMap.getKey(); + Map attr = attrMap.getValue(); + Node node = nodeMap.get(componentName); + if (node == null) { + throw new RuntimeException("dsl conf component not match"); + } + attr.forEach(node::setAttribute); + } + if (componentParameters.getRole().getGuest() != null) { + for (Map.Entry> attr : componentParameters.getRole().getGuest().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("guest").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + if (componentParameters.getRole().getHost() != null) { + for (Map.Entry> attr : componentParameters.getRole().getHost().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("host").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + if (componentParameters.getRole().getArbiter() != null) { + for (Map.Entry> attr : componentParameters.getRole().getArbiter().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("arbiter").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + return graph; + } + + private void addPartyNodes(Map componentsAttrs, Party party) { + if (party == null) { + throw new RuntimeException("dsl conf role not match"); + } + for (Map.Entry componentEntry : componentsAttrs.entrySet()) { + String componentName = componentEntry.getKey(); + Object componentAttr = componentEntry.getValue(); + PartyNode partyNode = new PartyNode(); + partyNode.setParent(party); + partyNode.setName(componentName); + Map attrMap; + try { + attrMap = SerializerUtils.deserializeType(SerializerUtils.toJsonString(componentAttr), + new TypeReference>() { + }); + if (attrMap != null) { + partyNode.setAttributes(attrMap); + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + party.getNodes().put(componentName, partyNode); + } + } + + private Map transInput(Input input) { + try { + Map map = new HashMap<>(); + if (input.getData() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(SerializerUtils.toJsonString(input.getData())) + .build(); + map.put("data", NodeInputRef.fromProto(nodeInputProto)); + } + if (input.getModel() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("model") + .setValue(SerializerUtils.toJsonString(input.getModel())) + .build(); + map.put("model", NodeInputRef.fromProto(nodeInputProto)); + } + if (input.getCache() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("cache") + .setValue(SerializerUtils.toJsonString(input.getCache())) + .build(); + map.put("cache", NodeInputRef.fromProto(nodeInputProto)); + } + return map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Input transInput(Map map) { + try { + if (map == null || CollectionUtils.isEmpty(map)) { + return null; + } + Input input = new Input(); + TypeReference>> typeReference = new TypeReference<>() { + }; + if (map.containsKey("data")) { + String content = map.get("data").getValue(); + Map> data = + SerializerUtils.deserializeType(content, typeReference); + input.setData(data); + } + TypeReference> typeReference2 = new TypeReference<>() { + }; + if (map.containsKey("model")) { + List model = SerializerUtils.deserializeType(map.get( + "model").getValue(), typeReference2); + input.setModel(model); + } + + if (map.containsKey("cache")) { + List cache = SerializerUtils.deserializeType(map.get( + "cache").getValue(), typeReference2); + input.setCache(cache); + } + return input; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map transOutput(Output output) { + try { + Map map = new HashMap<>(); + HighflipMeta.NodeOutputProto dataNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("data") + .setValue(SerializerUtils.toJsonString(output.getData())) + .build(); + map.put("data", NodeOutputRef.fromProto(dataNodeOutputProto)); + + HighflipMeta.NodeOutputProto modelNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("model") + .setValue(SerializerUtils.toJsonString(output.getModel())) + .build(); + map.put("model", NodeOutputRef.fromProto(modelNodeOutputProto)); + + HighflipMeta.NodeOutputProto cacheNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("cache") + .setValue(SerializerUtils.toJsonString(output.getCache())) + .build(); + map.put("cache", NodeOutputRef.fromProto(cacheNodeOutputProto)); + return map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Output transOutput(Map map) { + try { + if (map == null || CollectionUtils.isEmpty(map)) { + return null; + } + TypeReference> typeReference2 = new TypeReference<>() { + }; + Output output = new Output(); + if (map.containsKey("data")) { + List data = SerializerUtils.deserializeType( + map.get("data").getValue(), typeReference2); + output.setData(data); + } + if (map.containsKey("model")) { + List model = SerializerUtils.deserializeType( + map.get("model").getValue(), typeReference2); + output.setModel(model); + + } + if (map.containsKey("cache")) { + List cache = SerializerUtils.deserializeType( + map.get("cache").getValue(), typeReference2); + output.setCache(cache); + } + return output; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } } @Data @@ -19,8 +362,9 @@ public FateDAG translate(Graph dag) { @NoArgsConstructor public static class FateDAG { - String dsl; + Dsl dsl; + + DslConf conf; - String conf; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java new file mode 100644 index 0000000..c47cf48 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.adaptor; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; + +import com.baidu.highflip.core.utils.Foreach; + +public class DataAdaptorTest { + private final String DEFAULT_DELIMITER = ","; + + @Test + public void writeDataDenseTest() { + List> body = new ArrayList<>(); + List rowHeader = List.of("ID","name"); + List rowOne = List.of("1","bob"); + List rowTwo = List.of("2","joe"); + List rowThree = List.of("3","alice"); + body.add(rowHeader); + body.add(rowOne); + body.add(rowTwo); + body.add(rowThree); + Iterator> iterator = body.iterator(); + + StringBuilder stringBuilder = new StringBuilder(); + for (List dataList : Foreach.from(iterator)) { + String column = dataList.stream().map(Object::toString).collect( + Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(column).append("\n"); + } + } + + @Test + public void testSplit(){ + String dataId = "8a53e2c4-7e04-482b-9eef-4dd6c8376b1e+intersect_0" + + "+guest+9999"; + final String[] split = dataId.split("\\+"); + System.out.println(List.of(split)); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java index d445c3a..aad5c89 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java @@ -1,38 +1,158 @@ package com.webank.ai.fate.client; -import com.webank.ai.fate.client.form.JsonResultForm; +import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.JobData; +import com.webank.ai.fate.client.form.task.TaskData; +import com.webank.ai.fate.common.DataMultipartFile; +import com.webank.ai.fate.common.DecompressUtils; +import com.webank.ai.fate.common.deserializer.SerializerUtils; +import feign.Response; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; @Slf4j @Data @Disabled public class FateClientTest { - public static final String TEST_FLOW_URL = "http://127.0.0.1:9380"; + public static final String TEST_FLOW_URL = "http://10.27.130.41:8380"; FateClient client = FateClient.connect(TEST_FLOW_URL); + // @Test + // public void testVersionGet() { + // JsonResultForm result = getClient().version(); + // log.info("result = {}", result.getData().toPrettyString()); + // } + @Test - public void testVersionGet() { - JsonResultForm result = getClient().version(); - log.info("result = {}", result.getData().toPrettyString()); + public void testJobList() { + ResultForm result = getClient().listJob(1, 10); + log.info("result = {}", result); } @Test - public void testJobList() { - JsonResultForm result = getClient().jobList(10); - log.info("result = {}", result.getData().toPrettyString()); + public void testTaskList() { + ResultForm result = getClient().listTask(null, 18, 1); + log.info("result = {}", result); } @Test public void testJobQuery() { - ResultForm result = getClient().jobQuery("202210260752357956950"); + ResultForm result = getClient().jobQuery("202212011128410103660"); log.info("result = {}", result.getData().toString()); } + + @Test + public void testPushData() { + String data = "id,y,x0,x1,x2,x3,x4,x5\n" + "274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613\n" + + "199,0,0.426758,0.723479,0.316885,0.287273,1.000835,0.962702\n" + + "133,1,0.254879,-1.046633,0.209656,0.074214,-0.441366,-0.377645\n" + + "273,1,-1.142928,-0.781198,-1.166747,-0.923578,0.62823,-1.021418\n" + + "551,1,-0.879933,0.420589,-0.877527,-0.780484,-1.037534,-0.48388\n" + + "175,1,-1.451067,-1.406518,-1.456564,-1.092337,-0.708765,-1.168557\n"; + MultipartFile multipartFile = new DataMultipartFile("unittest_data3.csv", data.getBytes(StandardCharsets.UTF_8)); + ResultForm resultForm = getClient().pushData(multipartFile, ",", "1", "4", "highflip_test1", "experiment", null, null, null,"1"); + log.info("result = {}", resultForm); + } + + @Test + public void testJobSubmit() throws JsonProcessingException { + Dsl dsl = SerializerUtils.deserialize("{\n" + " \"components\": {\n" + " \"reader_0\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"data_transform_0\": {\n" + + " \"module\": \"DataTransform\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_0.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"homo_nn_0\": {\n" + " \"module\": \"HomoNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"train_data\": [\n" + + " \"data_transform_0.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"model\": [\n" + " \"model\"\n" + " ]\n" + + " }\n" + " }\n" + " }\n" + "}", Dsl.class); + DslConf dslConf = SerializerUtils.deserialize("{\n" + " \"dsl_version\": 2,\n" + " \"initiator\": {\n" + + " \"role\": \"guest\",\n" + " \"party_id\": 10000\n" + " },\n" + " \"role\": {\n" + + " \"arbiter\": [\n" + " 10000\n" + " ],\n" + " \"host\": [\n" + + " 10000\n" + " ],\n" + " \"guest\": [\n" + " 10000\n" + + " ]\n" + " },\n" + " \"component_parameters\": {\n" + " \"common\": {\n" + + " \"data_transform_0\": {\n" + " \"with_label\": true,\n" + + " \"label_name\": \"y\",\n" + " \"label_type\": \"int\",\n" + + " \"output_format\": \"dense\"\n" + " },\n" + + " \"homo_nn_0\": {\n" + " \"config_type\": \"pytorch\",\n" + + " \"nn_define\": [\n" + " {\n" + + " \"layer\": \"Linear\",\n" + " \"name\": \"line\",\n" + + " \"type\": \"normal\",\n" + " \"config\": [\n" + + " 30,\n" + " 1\n" + + " ]\n" + " },\n" + " {\n" + + " \"layer\": \"Sigmoid\",\n" + + " \"type\": \"activate\",\n" + + " \"name\": \"sigmoid\"\n" + " }\n" + " ],\n" + + " \"batch_size\": -1,\n" + " \"optimizer\": {\n" + + " \"optimizer\": \"Adam\",\n" + " \"lr\": 0.05\n" + + " },\n" + " \"early_stop\": {\n" + + " \"early_stop\": \"diff\",\n" + " \"eps\": 0.0001\n" + + " },\n" + " \"loss\": \"BCELoss\",\n" + + " \"metrics\": [\n" + " \"accuracy\"\n" + " ],\n" + + " \"max_iter\": 2\n" + " }\n" + " },\n" + " \"role\": {\n" + + " \"host\": {\n" + " \"0\": {\n" + " \"reader_0\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_homo_host\",\n" + + " \"namespace\": \"HIGH-FLIP\"\n" + " }\n" + + " }\n" + " }\n" + " },\n" + " \"guest\": {\n" + + " \"0\": {\n" + " \"reader_0\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_homo_guest\",\n" + + " \"namespace\": \"HIGH-FLIP\"\n" + " }\n" + + " }\n" + " }\n" + " }\n" + " }\n" + " }\n" + + "}", DslConf.class); + log.info("dsl:{}", SerializerUtils.toJsonString(dsl)); + log.info("dsl conf:{}", SerializerUtils.toJsonString(dslConf)); + ResultForm job = getClient().jobSubmit(dsl, dslConf); + log.info("result = {}", job); + } + + @Test + public void testDownloadData() { + try(Response response = getClient().downloadData("unittest_data3", "experiment")) { + InputStream inputStream = response.body().asInputStream(); + Map result = DecompressUtils.decompressTarGzToStringMap(inputStream, s -> s.contains("csv")); + log.info("result = {}, head = {}", result, response.headers()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testDownloadComponentResultData(){ + String jobId = "202305231618271012050"; + String componentName = "intersect_0"; + String role = "guest"; + String partyId = "9999"; + try (Response response = getClient().downloadComponentResultData(jobId, componentName, role, partyId)) { + InputStream inputStream = response.body().asInputStream(); + + Map result = DecompressUtils.decompressTarGzToStringMap(inputStream, s -> s.contains("csv")); + log.info("result = {}, head = {}", result, response.headers()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java new file mode 100644 index 0000000..7f4db49 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java @@ -0,0 +1,434 @@ +package com.webank.ai.fate.translator; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.baidu.highflip.core.entity.dag.Graph; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.ai.fate.client.FateClient; +import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.common.deserializer.SerializerUtils; + +import highflip.HighflipMeta; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DSLTranslatorTest { + + @Test + public void translateTest() { + // step 1: construct a highflip graph + HighflipMeta.GraphProto graphProto = constructHighflipGraph(); + + Graph graph = Graph.fromProto(graphProto); + // step 2: translate highflip graph to fate dag (dsl and conf) + DSLTranslator dslTranslator = new DSLTranslator(); + + DSLTranslator.FateDAG translate = dslTranslator.translate(graph); + + // fate flow http service + String TEST_FLOW_URL = "http://10.27.130.41:8380"; + + FateClient client = FateClient.connect(TEST_FLOW_URL); + + try { + log.info("conf = {}", + SerializerUtils.toJsonString(translate.getDsl())); + log.info("dsl = {}", + SerializerUtils.toJsonString(translate.getConf())); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + ResultForm job = client.jobSubmit(translate.getDsl(), + translate.getConf()); + log.info("result = {}", job); + } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + readerNodeOutputProtos.add(outputProto); + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model") + .setValue(getJsonStringOfModelOutput()) + .build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + dataTransformNodeInputProtos.add(dataTransformInputProto); + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs(dataTransformNodeInputProtos) + .addAllOutputs(dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + intersectNodeOutputProtos.add(intersectOutputProto); + List intersectNodeInputProtos = + new ArrayList<>(); + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + intersectNodeInputProtos.add(intersectInputProto); + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + // construct guest + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder() + .setRole(HighflipMeta.PartyRole.GUEST) + .addAllNodes(guestPartyNodes) + .setName("9999") + .build(); + + // construct host + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder() + .setRole(HighflipMeta.PartyRole.HOST) + .addAllNodes(hostPartyNodes) + .setName("10000") + .build(); + + // components + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + + @Test + public void testValueProto() { + Map tableMap2 = + new HashMap<>(); + tableMap2.put("name", HighflipMeta.ValueProto.newBuilder().setString( + "csv1").build()); + tableMap2.put("namespace", + HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + tableMap.put("bo", + HighflipMeta.ValueProto.newBuilder().setBool(true).build()); + + HighflipMeta.MapProto mapProto2 = HighflipMeta.MapProto.newBuilder().putAllMap(tableMap2).build(); + tableMap.put("mp", + HighflipMeta.ValueProto.newBuilder().setMap(mapProto2).build()); + + List list = new ArrayList<>(); + list.add(HighflipMeta.ValueProto.newBuilder().setMap(mapProto2).build()); + + HighflipMeta.ListProto listProto = + HighflipMeta.ListProto.newBuilder().addAllList(list).build(); + + tableMap.put("list", + HighflipMeta.ValueProto.newBuilder().setList(listProto).build()); + + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.MapProto map = typedValueProto.getValue().getMap(); + Map map1 = mapProtoConvert2Map(map); + try{ + String s = SerializerUtils.toJsonString(map1); + System.out.println(s); + }catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto){ + Map result = new HashMap<>(); + Map map = mapProto.getMapMap(); + for(Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private Object convertValueProto(HighflipMeta.ValueProto value){ + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + // Map + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + +} From 0264f8ea20d12a3adccc9c617b1a864f2dc57bda Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Wed, 9 Aug 2023 15:19:35 +0800 Subject: [PATCH 14/14] task bind job id in highflip, remove adaptor path configuration --- highflip-proto/src/main/proto/highflip.proto | 10 ---------- .../highflip/server/rpc/v1/HighFlipRpcService.java | 2 +- .../src/main/resources/application.properties | 2 +- .../java/com/webank/ai/fate/adaptor/JobAdaptor.java | 4 +++- .../com/webank/ai/fate/client/form/task/FFateTask.java | 1 - .../com/webank/ai/fate/client/form/task/FateTask.java | 8 -------- 6 files changed, 5 insertions(+), 22 deletions(-) diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index 119c543..9549533 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -561,16 +561,6 @@ message ConfigListResponse{ string key = 1; } -message GetServiceConfigResponse{ - - string url = 1; - - string party_id = 2; - - string role = 3; -} - - //////////////////////////////////////////////////////////////////////////////// // SERVICE //////////////////////////////////////////////////////////////////////////////// diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index 0640b8f..050051a 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -308,7 +308,7 @@ public void listTask(Highflip.TaskListRequest request, log.info("job id: {}, job binging id: {}", job.getJobId(), job.getBingingId()); Iterator response = Streams - .stream(getEngine().listTask(job.getBingingId())) + .stream(getEngine().listTask(request.getJobId())) .map(t -> Highflip.TaskListResponse .newBuilder() .setTaskId(t.getTaskid()) diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index 48ed02f..deced10 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar +highflip.server.adaptor.path= ######################################################### # spring grpc ######################################################### diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java index 319fb53..a8b992d 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java @@ -58,7 +58,7 @@ public com.baidu.highflip.core.entity.runtime.Job createJob(com.baidu.highflip.c @Override public com.baidu.highflip.core.entity.runtime.Job updateJob(com.baidu.highflip.core.entity.runtime.Job job) { - String bindId = job.getBingingId(); + String bindId = job.getJobId(); final Iterable tasks = getContext().getHighFlipRuntime().listTask(bindId); for (Task task : tasks) { @@ -181,6 +181,8 @@ public List getTaskList(com.baidu.highflip.core.entity.runtime.Job job, Li for (int i = 0; i < tasks.size(); i++) { Task queryTask = queryResult.get(i); Task ret = tasks.get(i); + // task bind job id in highflip + queryTask.setJobid(job.getJobId()); queryTask.setTaskid(ret.getTaskid()); BeanUtils.copyProperties(queryTask, ret); } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java index 8aca632..a639960 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java @@ -50,7 +50,6 @@ public class FFateTask { public static Task convertToEntity(FFateTask data) { Task task = new Task(); - task.setJobid(data.getF_job_id()); task.setName(data.getF_component_name()); task.setCreateTime(new DateTime(data.getF_create_time())); task.setUpdateTime(new DateTime(data.getF_update_time())); diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java index 975a4ff..e0a3b8b 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java @@ -48,21 +48,13 @@ public class FateTask { public static Task convertToEntity(FateTask data) { Task task = new Task(); -// task.setTaskid(); - task.setJobid(data.getJob_id()); task.setName(data.getComponent_name()); -// task.setDescription(); task.setCreateTime(new DateTime(data.getCreate_time())); task.setUpdateTime(new DateTime(data.getUpdate_time())); task.setFinishTime(new DateTime(data.getEnd_time())); task.setNodeName(data.getComponent_name()); task.setStatus(data.getStatus()); -// task.setMessage(); -// task.setIsDeleted(); -// task.setPrevious(); -// task.setNext(); task.setBingingId(data.getTask_id()); -// task.setBinding(); return task; }