From 505f72b6db564755180dd118ec051d85b1cce8a8 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Tue, 8 Aug 2023 11:10:50 +0800 Subject: [PATCH] add highflip fate vendors --- .../highflip-adaptor-fate/pom.xml | 58 ++- .../webank/ai/fate/adaptor/DataAdaptor.java | 159 ++++++- .../webank/ai/fate/adaptor/JobAdaptor.java | 129 ++++-- .../ai/fate/adaptor/OperatorAdaptor.java | 28 +- .../webank/ai/fate/adaptor/TaskAdaptor.java | 84 +++- .../com/webank/ai/fate/client/FateClient.java | 124 ++--- .../ai/fate/client/form/FateStatus.java | 45 ++ .../webank/ai/fate/client/form/data/Data.java | 21 + .../ai/fate/client/form/data/DataId.java | 12 + .../ai/fate/client/form/dsl/Component.java | 14 + .../client/form/dsl/ComponentParameters.java | 17 + .../ai/fate/client/form/dsl/Components.java | 12 + .../webank/ai/fate/client/form/dsl/Dsl.java | 15 + .../ai/fate/client/form/dsl/DslConf.java | 22 + .../webank/ai/fate/client/form/dsl/Input.java | 23 + .../ai/fate/client/form/dsl/Output.java | 21 + .../ai/fate/client/form/dsl/RoleConf.java | 21 + .../webank/ai/fate/client/form/dsl/Site.java | 12 + .../form/job/Adaptation_parameters.java | 22 + .../ai/fate/client/form/job/Common.java | 14 + .../ai/fate/client/form/job/Components.java | 49 ++ .../ai/fate/client/form/job/Computing.java | 16 + .../ai/fate/client/form/job/Dataio.java | 19 + .../ai/fate/client/form/job/Eggroll_run.java | 9 + .../fate/client/form/job/Engines_address.java | 18 + .../ai/fate/client/form/job/FateJob.java | 133 ++++++ .../ai/fate/client/form/job/Federation.java | 16 + .../form/job/Hetero_feature_binning.java | 19 + .../webank/ai/fate/client/form/job/Input.java | 21 + .../ai/fate/client/form/job/Intersection.java | 39 ++ .../ai/fate/client/form/job/JobData.java | 18 + .../fate/client/form/job/Job_parameters.java | 50 ++ .../ai/fate/client/form/job/Output.java | 21 + .../ai/fate/client/form/job/Pulsar_run.java | 9 + .../ai/fate/client/form/job/QueryJob.java | 106 +++++ .../ai/fate/client/form/job/Rabbitmq_run.java | 9 + .../ai/fate/client/form/job/Reader.java | 19 + .../webank/ai/fate/client/form/job/Roles.java | 18 + .../ai/fate/client/form/job/Runtime_conf.java | 34 ++ .../form/job/Runtime_conf_on_party.java | 33 ++ .../ai/fate/client/form/job/Spark_run.java | 9 + .../ai/fate/client/form/job/Storage.java | 16 + .../webank/ai/fate/client/form/job/Table.java | 17 + .../client/form/job/Train_runtime_conf.java | 9 + .../fate/client/form/job/Transform_param.java | 14 + .../webank/ai/fate/client/form/job/User.java | 18 + .../form/task/Adaptation_parameters.java | 14 + .../ai/fate/client/form/task/Class_path.java | 16 + .../ai/fate/client/form/task/Common.java | 28 ++ .../fate/client/form/task/ComponentParam.java | 31 ++ .../form/task/Component_parameters.java | 21 + .../ai/fate/client/form/task/Eggroll_run.java | 7 + .../fate/client/form/task/Encrypt_param.java | 13 + .../ai/fate/client/form/task/Engine_conf.java | 12 + .../client/form/task/Engines_address.java | 7 + .../webank/ai/fate/client/form/task/Env.java | 11 + .../ai/fate/client/form/task/FFateTask.java | 64 +++ .../ai/fate/client/form/task/FateTask.java | 69 +++ .../ai/fate/client/form/task/Initiator.java | 13 + .../fate/client/form/task/Job_parameters.java | 12 + .../ai/fate/client/form/task/Local.java | 13 + .../form/task/Optimal_binning_param.java | 18 + .../fate/client/form/task/Provider_info.java | 15 + .../ai/fate/client/form/task/Pulsar_run.java | 7 + .../fate/client/form/task/Rabbitmq_run.java | 7 + .../webank/ai/fate/client/form/task/Role.java | 14 + .../ai/fate/client/form/task/Spark_run.java | 7 + .../ai/fate/client/form/task/TaskData.java | 14 + .../client/form/task/Transform_param.java | 14 + .../webank/ai/fate/common/ConfigConst.java | 7 + .../ai/fate/common/DataMultipartFile.java | 76 +++ .../ai/fate/common/DecompressUtils.java | 39 ++ .../webank/ai/fate/common/FateConstants.java | 7 + .../com/webank/ai/fate/common/GZIPUtils.java | 84 ++++ .../deserializer/FeignSpringFormEncoder.java | 255 ++++++++++ .../deserializer/JsonListDeserializer.java | 34 ++ .../JsonMapComponentsDeserializer.java | 36 ++ .../JsonMapListStringDeserializer.java | 38 ++ .../deserializer/JsonMapMapDeserializer.java | 44 ++ .../JsonMapMapObjectDeserializer.java | 38 ++ .../JsonMapMapStringDeserializer.java | 44 ++ .../JsonMapStringDeserializer.java | 33 ++ .../deserializer/JsonStringDeserializer.java | 23 + .../common/deserializer/SerializerUtils.java | 416 +++++++++++++++++ .../webank/ai/fate/context/FateContext.java | 14 +- .../ai/fate/translator/DSLTranslator.java | 352 +++++++++++++- .../ai/fate/adaptor/DataAdaptorTest.java | 44 ++ .../webank/ai/fate/client/FateClientTest.java | 138 +++++- .../ai/fate/translator/DSLTranslatorTest.java | 434 ++++++++++++++++++ 89 files changed, 4070 insertions(+), 105 deletions(-) create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java create mode 100644 highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java diff --git a/highflip-vendors/highflip-adaptor-fate/pom.xml b/highflip-vendors/highflip-adaptor-fate/pom.xml index 3fbcc3d..c6622a8 100644 --- a/highflip-vendors/highflip-adaptor-fate/pom.xml +++ b/highflip-vendors/highflip-adaptor-fate/pom.xml @@ -17,6 +17,16 @@ + + com.baidu.highflip + highflip-proto + 1.0.0-SNAPSHOT + + + com.baidu.highflip + highflip-core + 1.0.0-SNAPSHOT + org.junit.jupiter junit-jupiter @@ -30,21 +40,42 @@ test - com.baidu.highflip - highflip-core - 1.0.0-SNAPSHOT - compile + com.google.protobuf + protobuf-java + 3.21.1 io.github.openfeign feign-core 11.10 + + org.slf4j + slf4j-api + 1.7.36 + + + + org.slf4j + slf4j-simple + 1.7.36 + io.github.openfeign feign-jackson 11.10 + + io.github.openfeign + feign-httpclient + 11.10 + + + + org.apache.commons + commons-compress + 1.21 + @@ -54,6 +85,25 @@ maven-surefire-plugin 2.22.2 + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java index 52b367f..45952eb 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java @@ -1,10 +1,167 @@ package com.webank.ai.fate.adaptor; +import static com.webank.ai.fate.common.FateConstants.DATA_ID_SEPARATOR; + +import com.baidu.highflip.core.entity.runtime.Data; +import com.baidu.highflip.core.entity.runtime.basic.DataCategory; +import com.baidu.highflip.core.entity.runtime.basic.KeyPair; +import com.baidu.highflip.core.utils.Foreach; +import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.common.DataMultipartFile; +import com.webank.ai.fate.common.DecompressUtils; import com.webank.ai.fate.context.FateContext; +import feign.Response; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +@Slf4j +@Getter +public class DataAdaptor implements com.baidu.highflip.core.adaptor.DataAdaptor { -public class DataAdaptor { + private final FateContext context; + + private final String DEFAULT_NAMESPACES = "HIGH-FLIP"; + + private final String DEFAULT_DELIMITER = ","; public DataAdaptor(FateContext context) { + this.context = context; + } + + @Override + public Data updateData(Data data) { + return null; + } + + @Override + public int getDataCount() { + return 0; + } + + @Override + public Data getDataByIndex(int index, Data data) { + return null; + } + + @Override + public void deleteData(Data data) { + getContext().getClient().deleteData(data.getBingingId(), DEFAULT_NAMESPACES); + } + + @Override + public InputStream readDataRaw(Data data) { + return null; + } + + @Override + public Iterator> readDataDense(Data data) { + if(data == null) { + throw new RuntimeException("data is null."); + } + log.info("data:{}", data); + + if (DataCategory.RESULT_DATA.equals(data.getCategory())) { + String jobId = (String) data.getBinding().get("jobId"); + String componentName = (String) data.getBinding().get("componentName"); + String role = (String) data.getBinding().get("role"); + String partyId = (String) data.getBinding().get("partyId"); + log.info("jobId: {}, componentName: {}, role: {}, partyId: {}", + jobId, componentName, role, partyId); + + try (Response response = getContext().getClient() + .downloadComponentResultData(jobId, componentName, + role, partyId)) { + String content = DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")).get("data.csv"); + return Arrays.stream(content.split("\n")) + .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) + .map(d -> (Object) d) + .collect(Collectors.toList())) + .collect(Collectors.toList()).iterator(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + try (Response response = getContext().getClient() + .downloadData(data.getName(), + DEFAULT_NAMESPACES)) { + String content = DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")).get("table.csv"); + return Arrays.stream(content.split("\n")) + .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) + .map(d -> (Object) d) + .collect(Collectors.toList())) + .collect(Collectors.toList()).iterator(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public Iterator> readDataSparse(Data data) { + return null; + } + + @Override + public Data createData(Data data) { + return null; + } + + @Override + public void writeDataRaw(Data data, InputStream body) { + + } + + @Override + public void writeDataDense(Data data, Iterator> body) { + StringBuilder stringBuilder = new StringBuilder(); + try { + String headers = + data.getColumns().stream().map(c -> c.getName()) + .collect(Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(headers).append("\n"); + final Foreach> foreach = Foreach.from(body); + for (List dataList : foreach) { + String column = dataList.stream().map(Object::toString).collect( + Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(column).append("\n"); + } + } catch (NoSuchElementException e) { + log.warn("body has no next element"); + } catch (Exception e) { + throw e; + } + String tableName = data.getName(); + log.info("push table:{} data:{}", tableName, data); + + MultipartFile multipartFile = new DataMultipartFile(tableName, + stringBuilder.toString().getBytes(StandardCharsets.UTF_8)); + // drop default 1, which means overwrite the dataset + ResultForm dataResultForm = + getContext().getClient() + .pushData(multipartFile, DEFAULT_DELIMITER, "1", + "4", tableName, DEFAULT_NAMESPACES, + null, null, + null, "1"); + data.setBingingId(dataResultForm.getData().getJob_id()); + } + + @Override + public void writeDataSparse(Data data, Iterator> body) { } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java index 6fb2c7b..319fb53 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/JobAdaptor.java @@ -1,20 +1,29 @@ package com.webank.ai.fate.adaptor; import com.baidu.highflip.core.engine.HighFlipRuntime; -import com.baidu.highflip.core.entity.runtime.Job; +import com.baidu.highflip.core.entity.runtime.Data; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.basic.Action; +import com.baidu.highflip.core.entity.runtime.basic.DataCategory; +import com.baidu.highflip.core.entity.runtime.basic.DataMode; import com.baidu.highflip.core.entity.runtime.basic.Status; -import com.webank.ai.fate.client.form.JobQueryResponseForm; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.QueryJob; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.FateTask; import com.webank.ai.fate.context.FateContext; import com.webank.ai.fate.translator.DSLTranslator; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; @Slf4j @Getter @@ -28,11 +37,11 @@ public JobAdaptor(FateContext context) { @Override public List getFeatures() { - return null; + throw new UnsupportedOperationException(); } @Override - public Job createJob(Job job) { + public com.baidu.highflip.core.entity.runtime.Job createJob(com.baidu.highflip.core.entity.runtime.Job job) { DSLTranslator.FateDAG dag = getContext() .getTranslator() @@ -40,83 +49,151 @@ public Job createJob(Job job) { String bindId = getContext() .getClient() - .jobSumbit(dag.getDsl(), dag.getDsl()); + .jobSubmit(dag.getDsl(), dag.getConf()).getData().getJob_id(); job.setBingingId(bindId); return job; } @Override - public Job updateJob(Job job) { - return null; + public com.baidu.highflip.core.entity.runtime.Job updateJob(com.baidu.highflip.core.entity.runtime.Job job) { + + String bindId = job.getBingingId(); + final Iterable tasks = + getContext().getHighFlipRuntime().listTask(bindId); + for (Task task : tasks) { + ResultForm> result = + getContext().getClient().taskQuery(task.getBingingId()); + if (CollectionUtils.isEmpty(result.getData())) { + throw new RuntimeException("task not found, task: " + task); + } + final FFateTask fFateTask = result.getData().get(0); + final String f_status = fFateTask.getF_status(); + final Status taskStatus = convertToHighFlipStatus(f_status); + + if (task.getStatus().equals(taskStatus)) { + // task status has not changed + continue; + } + + if (taskStatus == Status.SUCCEEDED) { + // register data and bind to task + Data data = new Data(); + data.setBinding(Map.of("jobId", job.getBingingId(), + "taskId", task.getTaskid(), + "taskBingingId", task.getBingingId(), + "componentName", task.getName(), + "role",fFateTask.getF_role(), + "partyId",fFateTask.getF_party_id())); + data.setFormat(DataMode.DENSE); + data.setCategory(DataCategory.RESULT_DATA); + getContext().getHighFlipRuntime().registerData(data); + log.info("data: {}", data); + task.setOutputData(List.of(data.getDataId())); + } + task.setStatus(taskStatus.name()); + getContext().getHighFlipRuntime().updateTask(task); + } + return job; + } + + private Status convertToHighFlipStatus(String fateStatus) { + switch (fateStatus) { + case "success": + return Status.SUCCEEDED; + case "timeout": + return Status.FAILED; + case "failed": + return Status.FAILED; + default: + return Status.UNKNOWN; + } } @Override - public boolean hasJob(Job job) { - return false; + public boolean hasJob(com.baidu.highflip.core.entity.runtime.Job job) { + throw new UnsupportedOperationException(); } @Override - public Status getJobStatus(Job job) { + public Status getJobStatus(com.baidu.highflip.core.entity.runtime.Job job) { String bindId = job.getBingingId(); - ResultForm> result = getContext() + ResultForm> result = getContext() .getClient() .jobQuery(bindId); String status = result.getData() .get(0) - .getStatus(); + .getF_status(); - return Status.valueOf(status); + if ("SUCCESS".equals(status.toUpperCase())) { + return Status.SUCCEEDED; + } else { + return Status.valueOf(status.toUpperCase()); + } } @Override - public void deleteJob(Job job) { - + public void deleteJob(com.baidu.highflip.core.entity.runtime.Job job) { + throw new UnsupportedOperationException(); } @Override - public Job controlJob(Job job, Action action) { + public com.baidu.highflip.core.entity.runtime.Job controlJob(com.baidu.highflip.core.entity.runtime.Job job, Action action) { switch (action) { case STOP: getContext().getClient().jobStop(job.getBingingId()); break; + default: + throw new UnsupportedOperationException(); } return null; } @Override public int getJobCount() { - return 0; + log.info("client get job count"); + return getContext().getClient().listJob(1, Integer.MAX_VALUE).getData().getCount(); } @Override - public Job getJobByIndex(int index, Job job) { - return null; + public com.baidu.highflip.core.entity.runtime.Job getJobByIndex(int index, com.baidu.highflip.core.entity.runtime.Job job) { + log.info("client get job by index"); + FateJob response = getContext().getClient().listJob(index + 1, 1).getData().getJobs().get(0); + return FateJob.convertToEntity(response); } @Override - public Optional moreJob(Job job, HighFlipRuntime runtime) { + public Optional moreJob(com.baidu.highflip.core.entity.runtime.Job job, HighFlipRuntime runtime) { return Optional.empty(); } @Override - public int getTaskCount(Job job) { - return 0; + public int getTaskCount(com.baidu.highflip.core.entity.runtime.Job job) { + return getContext().getClient().listTask(job.getBingingId(), 1, Integer.MAX_VALUE).getData().getCount(); } @Override - public List getTaskList(Job job, List task) { - return null; + public List getTaskList(com.baidu.highflip.core.entity.runtime.Job job, List tasks) { + List queryResult = getContext().getClient().listTask(job.getBingingId(), 1, Integer.MAX_VALUE).getData().getTasks().stream() + .map(FateTask::convertToEntity) + .collect(Collectors.toList()); + for (int i = 0; i < tasks.size(); i++) { + Task queryTask = queryResult.get(i); + Task ret = tasks.get(i); + queryTask.setTaskid(ret.getTaskid()); + BeanUtils.copyProperties(queryTask, ret); + } + return tasks; } @Override - public int getJobLogCount(Job job) { + public int getJobLogCount(com.baidu.highflip.core.entity.runtime.Job job) { return 0; } @Override - public Iterator getJobLog(Job job, int offset, int limit) { + public Iterator getJobLog(com.baidu.highflip.core.entity.runtime.Job job, int offset, int limit) { return null; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java index 17c3625..bb66014 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/OperatorAdaptor.java @@ -1,4 +1,30 @@ package com.webank.ai.fate.adaptor; -public class OperatorAdaptor { +import com.baidu.highflip.core.entity.runtime.Operator; + +public class OperatorAdaptor implements com.baidu.highflip.core.adaptor.OperatorAdaptor { + @Override + public int getOperatorCount() { + return 0; + } + + @Override + public Operator getOperatorByIndex(int index, Operator operator) { + return null; + } + + @Override + public Operator updateOperator(Operator operator) { + return null; + } + + @Override + public boolean hasOperator(Operator operator) { + return false; + } + + @Override + public void deleteOperator(Operator operator) { + + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java index 0fb81dc..b8a1e5d 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/TaskAdaptor.java @@ -1,16 +1,98 @@ package com.webank.ai.fate.adaptor; +import com.baidu.highflip.core.entity.runtime.Task; +import com.baidu.highflip.core.entity.runtime.basic.Action; +import com.baidu.highflip.core.entity.runtime.basic.Status; +import com.baidu.highflip.core.exception.HighFlipException; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.FateTask; import com.webank.ai.fate.context.FateContext; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.util.CollectionUtils; + @Slf4j @Getter -public class TaskAdaptor { +public class TaskAdaptor implements com.baidu.highflip.core.adaptor.TaskAdaptor { FateContext context; public TaskAdaptor(FateContext context) { this.context = context; } + + @Override + public Task updateTask(Task task) { + return null; + } + + @Override + public void deleteTask(Task task) { + + } + + @Override + public boolean hasTask(Task task) { + return false; + } + + @Override + public Status getTaskStatus(Task task) { + final List tasks = + getContext().getClient().taskQuery(task.getBingingId()) + .getData().stream() + .map(FFateTask::convertToEntity) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(tasks)) { + throw new RuntimeException("task not found: " + task); + } + final String status = tasks.get(0).getStatus(); + switch (status) { + case "success": + return Status.SUCCEEDED; + case "timeout": + return Status.FAILED; + case "failed": + return Status.FAILED; + default: + return Status.UNKNOWN; + } + } + + @Override + public Task controlTask(Task task, Action action, Map config) { + return null; + } + + @Override + public int getTaskCount() { + return getContext().getClient().listTask(null, 1, Integer.MAX_VALUE).getData().getCount(); + } + + @Override + public Task getTaskByIndex(int index, Task task) { + log.info("get task by index:{}", index); + return FateTask.convertToEntity(getContext().getClient().listTask(null, index+1, 1).getData().getTasks().get(0)); + } + + @Override + public void invokeTask(Task task) { + + } + + @Override + public int getTaskLogCount(Task task) { + return 0; + } + + @Override + public Iterator getTaskLog(Task task, int offset, int limit) { + return null; + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java index 116a10b..3ea0b0c 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/FateClient.java @@ -1,85 +1,93 @@ package com.webank.ai.fate.client; -import com.webank.ai.fate.client.form.JobQueryResponseForm; -import com.webank.ai.fate.client.form.JsonResultForm; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.data.Data; +import com.webank.ai.fate.client.form.data.DataId; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.JobData; +import com.webank.ai.fate.client.form.job.QueryJob; +import com.webank.ai.fate.client.form.task.FFateTask; +import com.webank.ai.fate.client.form.task.TaskData; +import com.webank.ai.fate.common.deserializer.FeignSpringFormEncoder; import feign.Feign; import feign.Headers; +import feign.Logger; import feign.Param; import feign.Request; import feign.RequestLine; +import feign.Response; +import feign.httpclient.ApacheHttpClient; import feign.jackson.JacksonDecoder; -import feign.jackson.JacksonEncoder; +import org.springframework.lang.Nullable; +import org.springframework.web.multipart.MultipartFile; -import java.io.File; import java.util.List; import java.util.concurrent.TimeUnit; -@Headers("Content-Type: application/json") public interface FateClient { static FateClient connect(String url) { - return Feign.builder() - .decoder(new JacksonDecoder()) - .encoder(new JacksonEncoder()) - .options(new Request.Options( - 10, TimeUnit.SECONDS, - 60, TimeUnit.SECONDS, - true)) + return Feign.builder().client(new ApacheHttpClient()).decoder(new JacksonDecoder()) + .encoder(new FeignSpringFormEncoder()).logLevel(Logger.Level.NONE) + .options(new Request.Options(10, TimeUnit.SECONDS, 60, TimeUnit.SECONDS, true)) .target(FateClient.class, url); } - @RequestLine("POST /v1/version/get") - JsonResultForm version(); - - @RequestLine("POST /v1/version/get") - JsonResultForm versionGet(@Param("module") String module); - @RequestLine("POST /v1/job/submit") - String jobSumbit(@Param("job_dsl") String dsl, - @Param("job_runtime_conf") String conf); - - - @RequestLine("POST /v1/job/list/job") - JsonResultForm jobList(@Param("limit") Integer limit); + @Headers("Content-Type: application/json") + ResultForm jobSubmit(@Param("job_dsl") Dsl dsl, @Param("job_runtime_conf") DslConf conf); @RequestLine("POST /v1/job/stop") + @Headers("Content-Type: application/json") void jobStop(@Param("job_id") String jobId); @RequestLine("POST /v1/job/query") - ResultForm> jobQuery(@Param("job_id") String jobId); - - - /** - * - * @param head - * @param id_delimiter - * @param partition - * @param table_name - * @param namespace - * @param storage_engine - * @param destory - * @param extend_sid - * @return - */ + @Headers("Content-Type: application/json") + ResultForm> jobQuery(@Param("job_id") String jobId); + + @RequestLine("POST /v1/job/list/job") + @Headers("Content-Type: application/json") + ResultForm listJob(@Param("page") int page, @Param("limit") int limit); + + @RequestLine("POST /v1/job/task/query") + @Headers("Content-Type: application/json") + ResultForm> taskQuery(@Param("task_id") String taskId); + + @RequestLine("POST /v1/job/list/task") + @Headers("Content-Type: application/json") + ResultForm listTask(@Param("job_id") @Nullable String jobId, @Param("page") int page, + @Param("limit") int limit); + + @RequestLine("POST /v1/table/delete") + @Headers("Content-Type: application/json") + ResultForm deleteData(@Param(value = "table_name") String tableName, + @Param(value = "namespace") String nameSpace); + + @RequestLine("GET /v1/table/download") + @Headers("Content-Type: application/json") + Response downloadData(@Param(value = "name") String tableName, @Param(value = "namespace") String nameSpace); + + @RequestLine("POST /v1/data/upload") @Headers("Content-Type: application/octet-stream") - @RequestLine("POST /data/upload") - JsonResultForm dataUpload( - @Param("file") File file, - @Param("head") Integer head, - @Param("id_delimiter") String idDelimiter, - @Param("partition") Integer partition, - @Param("table_name") String tableName, - @Param("namespace") String namespace, - @Param("storage_engine") String storageEngine, - @Param("destory") Integer destory, - @Param("extend_sid") Integer extendSid); - - - @RequestLine("POST /data/download") - JsonResultForm dataDownload( - @Param("table_name") String tableName, - @Param("namespace") String namespace, - @Param("output_path") String outputPath, - @Param("delimiter") String delimiter); + ResultForm pushData(@Param(value = "file") MultipartFile file, + @Param(value = "id_delimiter") String id_delimiter, @Param(value = "head") String head, + @Param(value = "partition") String partition, @Param(value = "table_name") String table_name, + @Param(value = "namespace") String namespace, @Param(value = "storage_engine") String storage_engine, + @Param(value = "destroy") String destroy, @Param(value = "extend_sid") String extend_sid, + @Param(value = "drop") String drop); + + @RequestLine("POST /v1/data/upload/history") + @Headers("Content-Type: application/json") + ResultForm getDataUploadHistory(@Param(value = "limit") int limit, + @Param(value = "job_id") String job_id); + + @RequestLine("GET /v1/tracking/component/output/data/download") + @Headers("Content-Type: application/json") + Response downloadComponentResultData(@Param(value = "job_id") String tableName, + @Param(value = "component_name") String component_name, + @Param(value = "role") String role, + @Param(value = "party_id") String party_id); + } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java new file mode 100644 index 0000000..2bacde1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/FateStatus.java @@ -0,0 +1,45 @@ +package com.webank.ai.fate.client.form; + +import com.baidu.highflip.core.entity.runtime.basic.Status; + +public enum FateStatus { + + WAITING("waiting"), + RUNNING("running"), + SUCCESS("success"), + FAILED("failed"), + ; + FateStatus(String value) { + this.value = value; + } + + private final String value; + + public String getValue() { + return value; + } + + public static FateStatus fromValue(String value) { + for (FateStatus fateStatus : FateStatus.values()) { + if (fateStatus.getValue().equalsIgnoreCase(value)) { + return fateStatus; + } + } + throw new IllegalArgumentException(); + } + + public static Status convertToEntity(FateStatus fateStatus) { + switch (fateStatus) { + case WAITING: + return Status.APPENDING; + case RUNNING: + return Status.RUNNING; + case FAILED: + return Status.FAILED; + case SUCCESS: + return Status.SUCCEEDED; + } + throw new IllegalArgumentException(); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java new file mode 100644 index 0000000..51c7b64 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/Data.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.data; + +import java.util.Map; + +@lombok.Data +public class Data { + + private String board_url; + private int code; + private String dsl_path; + private String job_id; + private String logs_directory; + private String message; + private Map model_info; + private String namespace; + private String pipeline_dsl_path; + private String runtime_conf_on_party_path; + private String runtime_conf_path; + private String table_name; + private String train_runtime_conf_path; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java new file mode 100644 index 0000000..a44fcf5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/data/DataId.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.data; + +import lombok.Data; + +@Data +public class DataId { + + private String table_name; + + private String namespaces; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java new file mode 100644 index 0000000..30b6179 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Component.java @@ -0,0 +1,14 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +@Data +public class Component { + + private String module; + + private Input input; + + private Output output; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java new file mode 100644 index 0000000..5811b18 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/ComponentParameters.java @@ -0,0 +1,17 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapMapObjectDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class ComponentParameters { + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> common; + + private RoleConf role; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java new file mode 100644 index 0000000..39f40fa --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Components.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Components { + + Map components; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java new file mode 100644 index 0000000..d90467c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Dsl.java @@ -0,0 +1,15 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapComponentsDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Dsl { + + @JsonDeserialize(using = JsonMapComponentsDeserializer.class) + Map components; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java new file mode 100644 index 0000000..57498ef --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/DslConf.java @@ -0,0 +1,22 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapListStringDeserializer; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class DslConf { + + private String dsl_version; + + private Site initiator; + + @JsonDeserialize(using = JsonMapListStringDeserializer.class) + private Map> role; + + private ComponentParameters component_parameters; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java new file mode 100644 index 0000000..051fa04 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Input.java @@ -0,0 +1,23 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import com.webank.ai.fate.common.deserializer.JsonMapListStringDeserializer; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class Input { + + @JsonDeserialize(using = JsonMapListStringDeserializer.class) + private Map> data; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List model; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List cache; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java new file mode 100644 index 0000000..c42906e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Output.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import lombok.Data; + +import java.util.List; + +@Data +public class Output { + + @JsonDeserialize(using = JsonListDeserializer.class) + private List data; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List model; + + @JsonDeserialize(using = JsonListDeserializer.class) + private List cache; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java new file mode 100644 index 0000000..ecc2fbe --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/RoleConf.java @@ -0,0 +1,21 @@ +package com.webank.ai.fate.client.form.dsl; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapMapObjectDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class RoleConf { + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> host; + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> guest; + + @JsonDeserialize(using = JsonMapMapObjectDeserializer.class) + private Map> arbiter; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java new file mode 100644 index 0000000..6c048fd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/dsl/Site.java @@ -0,0 +1,12 @@ +package com.webank.ai.fate.client.form.dsl; + +import lombok.Data; + +@Data +public class Site { + + private String role; + + private String party_id; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java new file mode 100644 index 0000000..8fe798b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Adaptation_parameters.java @@ -0,0 +1,22 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Adaptation_parameters { + + private boolean if_initiator_baseline; + + private int request_task_cores; + + private int task_cores_per_node; + + private int task_memory_per_node; + + private int task_nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java new file mode 100644 index 0000000..89cbcac --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Common.java @@ -0,0 +1,14 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Common { + + private Hetero_feature_binning hetero_feature_binning_0; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java new file mode 100644 index 0000000..74fe323 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Components.java @@ -0,0 +1,49 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Components { + + private Dataio dataio_0; + + private Hetero_feature_binning hetero_feature_binning_0; + + private Intersection intersection_0; + + private Reader reader_0; + + public void setDataio_0(Dataio dataio_0) { + this.dataio_0 = dataio_0; + } + + public Dataio getDataio_0() { + return dataio_0; + } + + public void setHetero_feature_binning_0(Hetero_feature_binning hetero_feature_binning_0) { + this.hetero_feature_binning_0 = hetero_feature_binning_0; + } + + public Hetero_feature_binning getHetero_feature_binning_0() { + return hetero_feature_binning_0; + } + + public void setIntersection_0(Intersection intersection_0) { + this.intersection_0 = intersection_0; + } + + public Intersection getIntersection_0() { + return intersection_0; + } + + public void setReader_0(Reader reader_0) { + this.reader_0 = reader_0; + } + + public Reader getReader_0() { + return reader_0; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java new file mode 100644 index 0000000..9456eae --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Computing.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Computing { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java new file mode 100644 index 0000000..89c7093 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Dataio.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Dataio { + + private boolean with_label; + + public void setWith_label(boolean with_label) { + this.with_label = with_label; + } + + public boolean getWith_label() { + return with_label; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java new file mode 100644 index 0000000..bdd2bd9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Eggroll_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Eggroll_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java new file mode 100644 index 0000000..97f5d33 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Engines_address.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Engines_address { + + private Computing computing; + + private Federation federation; + + private Storage storage; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java new file mode 100644 index 0000000..6b640fd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/FateJob.java @@ -0,0 +1,133 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.baidu.highflip.core.entity.runtime.basic.Status; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.FateStatus; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import com.webank.ai.fate.common.deserializer.JsonStringDeserializer; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.joda.time.DateTime; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Slf4j +@Data +public class FateJob { + + private long apply_resource_time; + + private boolean cancel_signal; + + private String cancel_time; + + private int cores; + + private String create_date; + + private Date create_time; + + private String description; + + private Dsl dsl; + + private long elapsed; + + private String end_date; + + private int end_scheduling_updates; + + private Date end_time; + + private String engine_name; + + private String engine_type; + + private String initiator_party_id; + + private String initiator_role; + + private boolean is_initiator; + + private String job_id; + + private int memory; + + private String name; + + private List partners; + + private int party_id; + + private int progress; + + private boolean ready_signal; + + private String ready_time; + + private int remaining_cores; + + private int remaining_memory; + + private boolean rerun_signal; + + private boolean resource_in_use; + + private long return_resource_time; + + @JsonDeserialize(using = JsonStringDeserializer.class) + private String role; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map roles; + + private Runtime_conf runtime_conf; + + private Runtime_conf_on_party runtime_conf_on_party; + + private String start_date; + + private Date start_time; + + private String status; + + private String status_code; + + private String tag; + + private Train_runtime_conf train_runtime_conf; + + private String update_date; + + private Date update_time; + + private User user; + + private String user_id; + + public static com.baidu.highflip.core.entity.runtime.Job convertToEntity(FateJob response) { + com.baidu.highflip.core.entity.runtime.Job job = new com.baidu.highflip.core.entity.runtime.Job(); +// job.setJobId(); + job.setJobName(response.getName()); + job.setDescription(response.getDescription()); + job.setCreateTime(new DateTime(response.getCreate_time())); + job.setUpdateTime(new DateTime(response.getUpdate_time())); + job.setFinishTime(new DateTime(response.getEnd_time())); +// job.setGraph(); + job.setStatus(FateStatus.convertToEntity(FateStatus.fromValue(response.getStatus()))); +// job.setMessage(); +// job.setIsDeleted(); +// job.setInputTasks(); +// job.setOutputTasks(); + job.setBingingId(response.getJob_id()); +// job.setBinding(); + return job; + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java new file mode 100644 index 0000000..9fa754e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Federation.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Federation { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java new file mode 100644 index 0000000..00920c5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Hetero_feature_binning.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Hetero_feature_binning { + + private Transform_param transform_param; + + public void setTransform_param(Transform_param transform_param) { + this.transform_param = transform_param; + } + + public Transform_param getTransform_param() { + return transform_param; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java new file mode 100644 index 0000000..5db61a9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Input.java @@ -0,0 +1,21 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; +@Data +public class Input { + + private InputData data; + + @Data + public static class InputData { + + private List data; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java new file mode 100644 index 0000000..f0d298d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Intersection.java @@ -0,0 +1,39 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Intersection { + + private Input input; + + private String module; + + private Output output; + + public void setInput(Input input) { + this.input = input; + } + + public Input getInput() { + return input; + } + + public void setModule(String module) { + this.module = module; + } + + public String getModule() { + return module; + } + + public void setOutput(Output output) { + this.output = output; + } + + public Output getOutput() { + return output; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java new file mode 100644 index 0000000..b9937ec --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/JobData.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; + +@Data +public class JobData { + + private int count; + + private List jobs; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java new file mode 100644 index 0000000..8631a12 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Job_parameters.java @@ -0,0 +1,50 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Job_parameters { + + private Adaptation_parameters adaptation_parameters; + + private int auto_retries; + + private int auto_retry_delay; + + private String computing_engine; + + private int computing_partitions; + + private Eggroll_run eggroll_run; + + private Engines_address engines_address; + + private String federated_mode; + + private String federated_status_collect_type; + + private String federation_engine; + + private String job_type; + + private String model_id; + + private String model_version; + + private Map pulsar_run; + + private Map rabbitmq_run; + + private Map spark_run; + + private String storage_engine; + + private int task_parallelism; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java new file mode 100644 index 0000000..7844095 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Output.java @@ -0,0 +1,21 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import java.util.List; + +public class Output { + + private List data; + + public void setData(List data) { + this.data = data; + } + + public List getData() { + return data; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java new file mode 100644 index 0000000..2837c57 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Pulsar_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Pulsar_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java new file mode 100644 index 0000000..8e758c4 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/QueryJob.java @@ -0,0 +1,106 @@ +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import com.webank.ai.fate.common.deserializer.JsonStringDeserializer; +import lombok.Data; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Data +public class QueryJob { + + private long f_apply_resource_time; + + private boolean f_cancel_signal; + + private String f_cancel_time; + + private int f_cores; + + private String f_create_date; + + private Date f_create_time; + + private String f_description; + + private Dsl f_dsl; + + private long f_elapsed; + + private String f_end_date; + + private int f_end_scheduling_updates; + + private Date f_end_time; + + private String f_engine_name; + + private String f_engine_type; + + private String f_initiator_party_id; + + private String f_initiator_role; + + private boolean f_is_initiator; + + private String f_job_id; + + private int f_memory; + + private String f_name; + + private List f_partners; + + private int f_party_id; + + private int f_progress; + + private boolean f_ready_signal; + + private String f_ready_time; + + private int f_remaining_cores; + + private int f_remaining_memory; + + private boolean f_rerun_signal; + + private boolean f_resource_in_use; + + private long f_return_resource_time; + + @JsonDeserialize(using = JsonStringDeserializer.class) + private String f_role; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map f_roles; + + private Runtime_conf f_runtime_conf; + + private Runtime_conf_on_party f_runtime_conf_on_party; + + private String f_start_date; + + private Date f_start_time; + + private String f_status; + + private String f_status_code; + + private String f_tag; + + private Train_runtime_conf f_train_runtime_conf; + + private String f_update_date; + + private Date f_update_time; + + private User f_user; + + private String f_user_id; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java new file mode 100644 index 0000000..69e7e6b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Rabbitmq_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Rabbitmq_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java new file mode 100644 index 0000000..5ca5261 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Reader.java @@ -0,0 +1,19 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Reader { + + private Table table; + + public void setTable(Table table) { + this.table = table; + } + + public Table getTable() { + return table; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java new file mode 100644 index 0000000..5b41ac9 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Roles.java @@ -0,0 +1,18 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +import java.util.List; + +@Data +public class Roles { + + private List guest; + + private List host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java new file mode 100644 index 0000000..302937b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf.java @@ -0,0 +1,34 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Runtime_conf { + + private ComponentParameters component_parameters; + + private String conf_path; + + private String dsl_path; + + private int dsl_version; + + private Site initiator; + + private Job_parameters job_parameters; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map role; + + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java new file mode 100644 index 0000000..3a4c14c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Runtime_conf_on_party.java @@ -0,0 +1,33 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class Runtime_conf_on_party { + + private ComponentParameters component_parameters; + + private String conf_path; + + private String dsl_path; + + private int dsl_version; + + private Site initiator; + + private Job_parameters job_parameters; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java new file mode 100644 index 0000000..09d7be4 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Spark_run.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Spark_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java new file mode 100644 index 0000000..70df472 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Storage.java @@ -0,0 +1,16 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Storage { + + private int cores_per_node; + + private int nodes; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java new file mode 100644 index 0000000..93866f6 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Table.java @@ -0,0 +1,17 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + + +import lombok.Data; + +@Data +public class Table { + + private String name; + + private String namespace; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java new file mode 100644 index 0000000..ae6c312 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Train_runtime_conf.java @@ -0,0 +1,9 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +public class Train_runtime_conf { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java new file mode 100644 index 0000000..166cc37 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/Transform_param.java @@ -0,0 +1,14 @@ +/** + * Copyright 2022 bejson.com + */ + +package com.webank.ai.fate.client.form.job; + +import lombok.Data; + +@Data +public class Transform_param { + + private String transform_type; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java new file mode 100644 index 0000000..04c9b02 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/job/User.java @@ -0,0 +1,18 @@ +package com.webank.ai.fate.client.form.job; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonMapStringDeserializer; +import lombok.Data; + +import java.util.Map; + +@Data +public class User { + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map guest; + + @JsonDeserialize(using = JsonMapStringDeserializer.class) + private Map host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java new file mode 100644 index 0000000..011bd6f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Adaptation_parameters.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + +@Data +public class Adaptation_parameters { + + private boolean if_initiator_baseline; + private int request_task_cores; + private int task_cores_per_node; + private int task_memory_per_node; + private int task_nodes; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java new file mode 100644 index 0000000..b4228cf --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Class_path.java @@ -0,0 +1,16 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + + +@Data +public class Class_path { + + private String feature_instance; + private String feature_vector; + private String homo_model_convert; + private String Interface; + private String model; + private String model_migrate; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java new file mode 100644 index 0000000..7afbc31 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Common.java @@ -0,0 +1,28 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Common { + + private Adaptation_parameters adaptation_parameters; + private int auto_retries; + private int auto_retry_delay; + private String computing_engine; + private int computing_partitions; + private Eggroll_run eggroll_run; + private Engines_address engines_address; + private String federated_mode; + private String federated_status_collect_type; + private String federation_engine; + private String job_type; + private String model_id; + private String model_version; + private Pulsar_run pulsar_run; + private Rabbitmq_run rabbitmq_run; + private Spark_run spark_run; + private String storage_engine; + private int task_parallelism; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java new file mode 100644 index 0000000..b6de203 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/ComponentParam.java @@ -0,0 +1,31 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class ComponentParam { + + private List _feeded_deprecated_params; + private boolean _is_raw_conf; + private String _name; + private List _user_feeded_params; + private double adjustment_factor; + private int bin_indexes; + private String bin_names; + private int bin_num; + private String category_indexes; + private String category_names; + private int compress_thres; + private Encrypt_param encrypt_param; + private double error; + private int head_size; + private boolean local_only; + private String method; + private boolean need_run; + private Optimal_binning_param optimal_binning_param; + private boolean skip_static; + private Transform_param transform_param; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java new file mode 100644 index 0000000..51cde37 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Component_parameters.java @@ -0,0 +1,21 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Component_parameters { + + private String CodePath; + private ComponentParam ComponentParam; + private String conf_path; + private String dsl_path; + private int dsl_version; + private Initiator initiator; + private Job_parameters job_parameters; + private Local local; + private String module; + private Role role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java new file mode 100644 index 0000000..bfe6f19 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Eggroll_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Eggroll_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java new file mode 100644 index 0000000..d64a45e --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Encrypt_param.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Encrypt_param { + + private int key_length; + private String method; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java new file mode 100644 index 0000000..c710198 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engine_conf.java @@ -0,0 +1,12 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Engine_conf { + + private String computing_engine; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java new file mode 100644 index 0000000..bba10cf --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Engines_address.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Engines_address { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java new file mode 100644 index 0000000..76b0cee --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Env.java @@ -0,0 +1,11 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Env { + + private String PYTHONPATH; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java new file mode 100644 index 0000000..8aca632 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FFateTask.java @@ -0,0 +1,64 @@ +package com.webank.ai.fate.client.form.task; + +import java.util.Date; +import java.util.List; + +import org.joda.time.DateTime; + +import com.baidu.highflip.core.entity.runtime.Task; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; + +import lombok.Data; + +@Data +public class FFateTask { + private int f_auto_retries; + private int f_auto_retry_delay; + @JsonDeserialize(using = JsonListDeserializer.class) + private List f_cmd; + private String f_component_module; + private String f_component_name; + private Component_parameters f_component_parameters; + private Date f_create_time; + private int f_elapsed; + private Date f_end_time; + private Engine_conf f_engine_conf; + private String f_federated_mode; + private String f_federated_status_collect_type; + private String f_initiator_party_id; + private String f_initiator_role; + // job_id is the job id in FATE + private String f_job_id; + private String f_party_id; + private String f_party_status; + private Provider_info f_provider_info; + private String f_role; + private String f_run_ip; + private boolean f_run_on_this_party; + private int f_run_pid; + private Date f_start_time; + /** + * f_status is one of waiting,running,canceled,timeout,failed,pass,success + */ + private String f_status; + private String f_status_code; + private String f_task_id; + private int f_task_version; + private Date f_update_time; + private String f_worker_id; + + public static Task convertToEntity(FFateTask data) { + Task task = new Task(); + task.setJobid(data.getF_job_id()); + task.setName(data.getF_component_name()); + task.setCreateTime(new DateTime(data.getF_create_time())); + task.setUpdateTime(new DateTime(data.getF_update_time())); + task.setFinishTime(new DateTime(data.getF_end_time())); + task.setNodeName(data.getF_component_name()); + task.setStatus(data.getF_status()); + task.setBingingId(data.getF_task_id()); + return task; + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java new file mode 100644 index 0000000..975a4ff --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/FateTask.java @@ -0,0 +1,69 @@ + +package com.webank.ai.fate.client.form.task; + +import com.baidu.highflip.core.entity.runtime.Task; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.webank.ai.fate.common.deserializer.JsonListDeserializer; +import lombok.Data; +import org.joda.time.DateTime; + +import java.util.Date; +import java.util.List; + + +@Data +public class FateTask { + + private int auto_retries; + private int auto_retry_delay; + @JsonDeserialize(using = JsonListDeserializer.class) + private List cmd; + private String component_module; + private String component_name; + private Component_parameters component_parameters; + private Date create_time; + private int elapsed; + private Date end_time; + private Engine_conf engine_conf; + private String federated_mode; + private String federated_status_collect_type; + private String initiator_party_id; + private String initiator_role; + // job_id is the job id in FATE + private String job_id; + private String party_id; + private String party_status; + private Provider_info provider_info; + private String role; + private String run_ip; + private boolean run_on_this_party; + private int run_pid; + private Date start_time; + private String status; + private String status_code; + private String task_id; + private int task_version; + private Date update_time; + private String worker_id; + + public static Task convertToEntity(FateTask data) { + Task task = new Task(); +// task.setTaskid(); + task.setJobid(data.getJob_id()); + task.setName(data.getComponent_name()); +// task.setDescription(); + task.setCreateTime(new DateTime(data.getCreate_time())); + task.setUpdateTime(new DateTime(data.getUpdate_time())); + task.setFinishTime(new DateTime(data.getEnd_time())); + task.setNodeName(data.getComponent_name()); + task.setStatus(data.getStatus()); +// task.setMessage(); +// task.setIsDeleted(); +// task.setPrevious(); +// task.setNext(); + task.setBingingId(data.getTask_id()); +// task.setBinding(); + return task; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java new file mode 100644 index 0000000..d4746fc --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Initiator.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Initiator { + + private int party_id; + private String role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java new file mode 100644 index 0000000..35e1cb7 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Job_parameters.java @@ -0,0 +1,12 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Job_parameters { + + private Common common; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java new file mode 100644 index 0000000..2e66e13 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Local.java @@ -0,0 +1,13 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Local { + + private int party_id; + private String role; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java new file mode 100644 index 0000000..9cdaf7d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Optimal_binning_param.java @@ -0,0 +1,18 @@ + +package com.webank.ai.fate.client.form.task; + + +import lombok.Data; + +@Data +public class Optimal_binning_param { + + private String adjustment_factor; + private int init_bin_nums; + private String init_bucket_method; + private String max_bin; + private int max_bin_pct; + private String metric_method; + private double min_bin_pct; + private boolean mixture; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java new file mode 100644 index 0000000..211fab1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Provider_info.java @@ -0,0 +1,15 @@ + +package com.webank.ai.fate.client.form.task; + +import lombok.Data; + + +@Data +public class Provider_info { + + private Class_path class_path; + private Env env; + private String name; + private String path; + private String version; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java new file mode 100644 index 0000000..a5b6e13 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Pulsar_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Pulsar_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java new file mode 100644 index 0000000..ef0032b --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Rabbitmq_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Rabbitmq_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java new file mode 100644 index 0000000..51c34d1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Role.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class Role { + + private List guest; + private List host; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java new file mode 100644 index 0000000..6412c4f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Spark_run.java @@ -0,0 +1,7 @@ + +package com.webank.ai.fate.client.form.task; + + +public class Spark_run { + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java new file mode 100644 index 0000000..9c3a8c5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/TaskData.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.List; + + +@Data +public class TaskData { + + private int count; + private List tasks; + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java new file mode 100644 index 0000000..5fefc0f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/client/form/task/Transform_param.java @@ -0,0 +1,14 @@ + +package com.webank.ai.fate.client.form.task; +import lombok.Data; + +import java.util.Date; + + +@Data +public class Transform_param { + + private Date transform_cols; + private String transform_names; + private String transform_type; +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java new file mode 100644 index 0000000..ab137c1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/ConfigConst.java @@ -0,0 +1,7 @@ +package com.webank.ai.fate.common; + +public interface ConfigConst { + + String FLOW_URL_CONFIG="highflip.adaptor.fate.fate-flow.url"; + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java new file mode 100644 index 0000000..ed844dd --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DataMultipartFile.java @@ -0,0 +1,76 @@ +package com.webank.ai.fate.common; + +import org.springframework.lang.NonNull; +import org.springframework.util.Base64Utils; +import org.springframework.util.FileCopyUtils; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +public class DataMultipartFile implements MultipartFile { + + private String originalFilename = "file"; + + private final String contentType = "application/octet-stream"; + + private final byte[] content; + + public DataMultipartFile(String filename, byte[] bytes) { + if (filename != null) { + this.originalFilename = filename; + } + this.content = bytes; + } + + public DataMultipartFile(String filename, String base64str) { + if (filename != null) { + this.originalFilename = filename; + } + this.content = Base64Utils.decodeFromString(base64str); + } + + @Override + @NonNull + public String getName() { + return this.originalFilename; + } + + @Override + public String getOriginalFilename() { + return this.originalFilename; + } + + @Override + public String getContentType() { + return this.contentType; + } + + @Override + public boolean isEmpty() { + return (this.content.length == 0); + } + + @Override + public long getSize() { + return this.content.length; + } + + @Override + public byte[] getBytes() throws IOException { + return this.content; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.content); + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + FileCopyUtils.copy(this.content, dest); + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java new file mode 100644 index 0000000..aa2d2b1 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/DecompressUtils.java @@ -0,0 +1,39 @@ +package com.webank.ai.fate.common; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.springframework.lang.Nullable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + +public class DecompressUtils { + + public static Map decompressTarGzToStringMap(InputStream tarGzInputStream, @Nullable Function fileNameFilter) { + if (fileNameFilter == null) { + fileNameFilter = s -> true; + } + Map result = new HashMap<>(); + try (GZIPInputStream gis = new GZIPInputStream(tarGzInputStream);TarArchiveInputStream tais = new TarArchiveInputStream(gis)) { + int i = 0; + TarArchiveEntry entry; + while ((entry = tais.getNextTarEntry()) != null) { + if (fileNameFilter.apply(entry.getName())) { + String text = new BufferedReader(new InputStreamReader(tais, StandardCharsets.UTF_8)).lines().collect(Collectors.joining("\n")); + result.put(entry.getName(), text); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return result; + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java new file mode 100644 index 0000000..7931da6 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/FateConstants.java @@ -0,0 +1,7 @@ +package com.webank.ai.fate.common; + +public class FateConstants { + public static final String DATA_ID_SEPARATOR = "\\+"; + + public static final String DATA_ID_CONNECTOR = "+"; +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java new file mode 100644 index 0000000..bbc891f --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/GZIPUtils.java @@ -0,0 +1,84 @@ +package com.webank.ai.fate.common; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class GZIPUtils { + public static final String GZIP_ENCODE_UTF_8 = "UTF-8"; + public static final String GZIP_ENCODE_ISO_8859_1 = "ISO-8859-1"; + + + public static byte[] compress(String str, String encoding) { + if (str == null || str.length() == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip; + try { + gzip = new GZIPOutputStream(out); + gzip.write(str.getBytes(encoding)); + gzip.close(); + } catch ( Exception e) { + e.printStackTrace(); + } + return out.toByteArray(); + } + + public static byte[] compress(String str) throws IOException { + return compress(str, GZIP_ENCODE_UTF_8); + } + + public static byte[] uncompress(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[256]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + } catch (Exception e) { + e.printStackTrace(); + } + return out.toByteArray(); + } + + public static String uncompressToString(byte[] bytes, String encoding) { + if (bytes == null || bytes.length == 0) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + GZIPInputStream ungzip = new GZIPInputStream(in); + byte[] buffer = new byte[256]; + int n; + while ((n = ungzip.read(buffer)) >= 0) { + out.write(buffer, 0, n); + } + return out.toString(encoding); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static String uncompressToString(byte[] bytes) { + return uncompressToString(bytes, GZIP_ENCODE_UTF_8); + } + + public static void main(String[] args) throws IOException { + String s = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + System.out.println("字符串长度："+s.length()); + System.out.println("压缩后：："+compress(s).length); + System.out.println("解压后："+uncompress(compress(s)).length); + System.out.println("解压字符串后：："+uncompressToString(compress(s)).length()); + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java new file mode 100644 index 0000000..5da179c --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/FeignSpringFormEncoder.java @@ -0,0 +1,255 @@ +package com.webank.ai.fate.common.deserializer; + +import feign.RequestTemplate; +import feign.codec.EncodeException; +import feign.codec.Encoder; +import feign.jackson.JacksonEncoder; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class FeignSpringFormEncoder extends JacksonEncoder implements Encoder { + + private final List> converters = new RestTemplate().getMessageConverters(); + + public static final Charset UTF_8 = Charset.forName("UTF-8"); + + public FeignSpringFormEncoder() {} + + /** + * 实现一个 HttpOutputMessage + */ + private class HttpOutputMessageImpl implements HttpOutputMessage { + /** + * 输出流，请求体 + */ + private final OutputStream body; + /** + * 请求头 + */ + private final HttpHeaders headers; + + public HttpOutputMessageImpl(OutputStream body, HttpHeaders headers) { + this.body = body; + this.headers = headers; + } + + @Override + public OutputStream getBody() throws IOException { + return body; + } + + @Override + public HttpHeaders getHeaders() { + return headers; + } + } + + /** + * 内部静态类,保存 MultipartFile 数据 + */ + static class MultipartFileResource extends InputStreamResource { + /** + * 文件名 + */ + private final String filename; + /** + * 文件大小 + */ + private final long size; + + /** + * 构造方法 + */ + public MultipartFileResource(InputStream inputStream, String filename, long size) { + super(inputStream); + this.filename = filename; + this.size = size; + } + + @Override + public String getFilename() { + return this.filename; + } + + @Override + public InputStream getInputStream() throws IOException, IllegalStateException { + return super.getInputStream(); + } + + @Override + public long contentLength() throws IOException { + return size; + } + } + + /** + * 重写编码器 + */ + @Override + public void encode(Object object, Type bodyType, RequestTemplate template) throws EncodeException { + if (isJsonRequest(template)){ + super.encode(object, bodyType, template); + } else { + final HttpHeaders multipartHeaders = new HttpHeaders(); + multipartHeaders.setContentType(MediaType.MULTIPART_FORM_DATA); + encodeMultipartFormRequest((Map) object, multipartHeaders, template); + } + } + + private boolean isJsonRequest(RequestTemplate requestTemplate) { + for (Map.Entry> header : requestTemplate.headers().entrySet()) { + if (header.getKey().equalsIgnoreCase("Content-Type")) { + for (String type : header.getValue()) { + if (type.contains("json")) { + return true; + } + } + } + } + return false; + } + + /** + * 对有文件、表单的进行编码 + */ + private void encodeMultipartFormRequest(Map formMap, HttpHeaders multipartHeaders, RequestTemplate template){ + if (formMap == null){ + throw new EncodeException("无法对格式为null的请求进行编码。"); + } + + LinkedMultiValueMap map = new LinkedMultiValueMap<>(); + //对每个参数进行检查校验 + for (Map.Entry entry : formMap.entrySet()){ + Object value = entry.getValue(); + //不同的数据类型进行不同的编码逻辑处理 + if (isMultipartFile(value)){ + //单个文件 + map.add(entry.getKey(), encodeMultipartFile((MultipartFile)value)); + + } else if (isMultipartFileArray(value)){ + //多个文件 + encodeMultipartFiles(map, (String) entry.getKey(), Arrays.asList((MultipartFile[]) value)); + + } else { + //普通请求数据 + map.add(entry.getKey(), encodeJsonObject(value)); + } + } + + encodeRequest(map, multipartHeaders, template); + } + + /** + * 对请求进行编码 + */ + private void encodeRequest(Object value, HttpHeaders requestHeaders, RequestTemplate template){ + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + HttpOutputMessage dummyRequest = new HttpOutputMessageImpl(outputStream, requestHeaders); + + try { + Class requestType = value.getClass(); + MediaType requestContentType = requestHeaders.getContentType(); + for (HttpMessageConverter messageConverter : converters){ + if (messageConverter.canWrite(requestType, requestContentType)){ + ((HttpMessageConverter) messageConverter).write(value, requestContentType, dummyRequest); + break; + } + } + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + + HttpHeaders headers = dummyRequest.getHeaders(); + if (headers != null){ + for (Map.Entry> entry : headers.entrySet()){ + template.header(entry.getKey(), entry.getValue()); + } + } + + /* + 请使用模板输出流。。。如果文件太大，这将导致问题，因为整个请求都将在内存中。 + */ + template.body(outputStream.toByteArray(), UTF_8); + } + + /** + * 编码为json对象 + */ + private HttpEntity encodeJsonObject(Object obj){ + HttpHeaders jsonPartHeaders = new HttpHeaders(); + jsonPartHeaders.setContentType(MediaType.APPLICATION_JSON); + return new HttpEntity<>(obj, jsonPartHeaders); + } + + /** + * 编码MultipartFile文件，将其转换为HttpEntity，同时设置 Content-type 为 application/octet-stream + * @param map 当前请求 map. + * @param name 数组字段的名称 + * @param fileList 要处理的文件 + */ + private void encodeMultipartFiles(LinkedMultiValueMap map, String name, List fileList){ + HttpHeaders filePartHeaders = new HttpHeaders(); + //设置 Content-type + filePartHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); + try { + for (MultipartFile file : fileList){ + Resource multipartFileResource = new MultipartFileResource(file.getInputStream(), file.getOriginalFilename(), file.getSize()); + map.add(name, new HttpEntity<>(multipartFileResource, filePartHeaders)); + } + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + } + + /** + * 编码MultipartFile文件，将其转换为HttpEntity，同时设置 Content-type 为 application/octet-stream + * @param file 要编码的文件 + */ + private HttpEntity encodeMultipartFile(MultipartFile file){ + HttpHeaders filePartHeaders = new HttpHeaders(); + //设置 Content-type + filePartHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); + try { + Resource multipartFileResource = new MultipartFileResource(file.getInputStream(), file.getOriginalFilename(), file.getSize()); + return new HttpEntity<>(multipartFileResource, filePartHeaders); + } catch (IOException e) { + throw new EncodeException("无法对请求进行编码:", e); + } + } + + /** + * 判断是否多个 MultipartFile + */ + private boolean isMultipartFileArray(Object object){ + return object != null && object.getClass().isArray() && MultipartFile.class.isAssignableFrom(object.getClass().getComponentType()); + } + + /** + * 判断是否MultipartFile文件 + * @param object 要判断的对象 + */ + private boolean isMultipartFile(Object object){ + return object instanceof MultipartFile; + } + + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java new file mode 100644 index 0000000..33f2b0d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonListDeserializer.java @@ -0,0 +1,34 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class JsonListDeserializer extends StdDeserializer> { + + public JsonListDeserializer() { + super(List.class); + } + + @Override + public List deserialize(JsonParser parser, DeserializationContext context) + throws IOException { + JsonNode node = parser.getCodec().readTree(parser); + List result = new ArrayList<>(); + if (node.isArray()) { + for (JsonNode element : node) { + result.add(element.asText()); + } + } else if (node.isObject()) { + result.add(node.asText()); + } else { + //maybe nothing? + } + return result; + } +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java new file mode 100644 index 0000000..580d056 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapComponentsDeserializer.java @@ -0,0 +1,36 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.webank.ai.fate.client.form.dsl.Component; + +import java.io.IOException; +import java.util.Map; + +public class JsonMapComponentsDeserializer extends StdDeserializer> { + + protected JsonMapComponentsDeserializer() { + super(Map.class); + } + + @Override + public Map deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { +// Map ret = new HashMap<>(); +// Map temp = jsonParser.getCodec().readValue(jsonParser, new TypeReference>() { +// }); +// for (String key : temp.keySet()) { +// Object tempValue = temp.get(key); +// if (tempValue != null) { +// ObjectMapper objectMapper = new ObjectMapper(); +// String decode = objectMapper.writeValueAsString(tempValue); +// Component temp1 = objectMapper.readValue(decode, new TypeReference() { +// }); +// ret.put(key, temp1); +// } +// } +// return ret; + return SerializerUtils.mapObjectDeserialize(jsonParser.getCodec().readTree(jsonParser).toString(), Component.class); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java new file mode 100644 index 0000000..cce54a5 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapListStringDeserializer.java @@ -0,0 +1,38 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JsonMapListStringDeserializer extends StdDeserializer>> { + + protected JsonMapListStringDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + Map> ret = new HashMap<>(); + Map temp = jsonParser.getCodec().readValue(jsonParser, new TypeReference>() { + }); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + List temp1 = objectMapper.readValue(decode, new TypeReference>() { + }); + ret.put(key, temp1); + } + } + return ret; + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java new file mode 100644 index 0000000..0f37715 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapDeserializer.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapDeserializer extends StdDeserializer>> { + + public JsonMapMapDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + Map map = new HashMap<>(); + for (String key1 : temp.keySet()) { + if (temp1.get(key1) != null) { + map.put(key, temp1.get(key1).toString()); + } + } + ret.put(key, map); + } + } + return ret; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java new file mode 100644 index 0000000..c83131d --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapObjectDeserializer.java @@ -0,0 +1,38 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapObjectDeserializer extends StdDeserializer>> { + + public JsonMapMapObjectDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + ret.put(key, temp1); + } + } + return ret; + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java new file mode 100644 index 0000000..3e3b543 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapMapStringDeserializer.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapMapStringDeserializer extends StdDeserializer>> { + + public JsonMapMapStringDeserializer() { + super(Map.class); + } + + @Override + public Map> deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map> ret = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + Object tempValue = temp.get(key); + if (tempValue != null) { + ObjectMapper objectMapper = new ObjectMapper(); + String decode = objectMapper.writeValueAsString(tempValue); + Map temp1 = objectMapper.readValue(decode, typeReference); + Map map = new HashMap<>(); + for (String key1 : temp1.keySet()) { + if (temp1.get(key1) != null) { + map.put(key, temp1.get(key1).toString()); + } + } + ret.put(key, map); + } + } + return ret; + } + +} \ No newline at end of file diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java new file mode 100644 index 0000000..4c555ac --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonMapStringDeserializer.java @@ -0,0 +1,33 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonMapStringDeserializer extends StdDeserializer> { + + public JsonMapStringDeserializer(){ + super(Map.class); + } + + @Override + public Map deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException { + Map map = new HashMap<>(); + TypeReference> typeReference = new TypeReference<>() { + }; + Map temp = jsonParser.getCodec().readValue(jsonParser, typeReference); + for (String key : temp.keySet()) { + if (temp.get(key) != null) { + map.put(key, temp.get(key).toString()); + } + } + return map; +// return SerializerUtils.mapObjectDeserialize(jsonParser.getCodec().readTree(jsonParser).toString(), String.class); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java new file mode 100644 index 0000000..ec89c61 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/JsonStringDeserializer.java @@ -0,0 +1,23 @@ +package com.webank.ai.fate.common.deserializer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; + +public class JsonStringDeserializer extends StdDeserializer { + + public JsonStringDeserializer() { + super(String.class); + } + + @Override + public String deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + return node.toString().replace("\"", ""); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java new file mode 100644 index 0000000..356a417 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/common/deserializer/SerializerUtils.java @@ -0,0 +1,416 @@ +package com.webank.ai.fate.common.deserializer; + +import com.baidu.highflip.core.entity.dag.Graph; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.translator.DSLTranslator; +import org.springframework.lang.Nullable; + +import java.util.HashMap; +import java.util.Map; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, TypeReference typeReference) throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } + + public static void main(String[] args) throws JsonProcessingException { + Dsl dsl = SerializerUtils.deserialize("{\n" + " \"components\": {\n" + " \"reader_0\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"reader_1\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"data_transform_0\": {\n" + + " \"module\": \"DataTransform\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_0.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"data_transform_1\": {\n" + " \"module\": \"DataTransform\",\n" + + " \"input\": {\n" + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_1.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"intersection_0\": {\n" + " \"module\": \"Intersection\",\n" + + " \"input\": {\n" + " \"data\": {\n" + " \"data\": [\n" + + " \"data_transform_0.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"cache\": [\n" + " \"cache\"\n" + " ]\n" + + " }\n" + " },\n" + " \"intersection_1\": {\n" + + " \"module\": \"Intersection\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"data_transform_1.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"cache\": [\n" + " \"cache\"\n" + " ]\n" + + " }\n" + " },\n" + " \"hetero_nn_0\": {\n" + + " \"module\": \"HeteroNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"train_data\": [\n" + + " \"intersection_0.data\"\n" + " ],\n" + + " \"validate_data\": [\n" + " \"intersection_1.data\"\n" + + " ]\n" + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"model\": [\n" + " \"model\"\n" + " ]\n" + + " }\n" + " },\n" + " \"hetero_nn_1\": {\n" + + " \"module\": \"HeteroNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"test_data\": [\n" + + " \"intersection_1.data\"\n" + " ]\n" + + " },\n" + " \"model\": [\n" + + " \"hetero_nn_0.model\"\n" + " ]\n" + " },\n" + + " \"output\": {\n" + " \"data\": [\n" + " \"data\"\n" + + " ],\n" + " \"model\": [\n" + " \"model\"\n" + + " ]\n" + " }\n" + " },\n" + " \"evaluation_0\": {\n" + + " \"module\": \"Evaluation\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"hetero_nn_0.data\",\n" + " \"hetero_nn_1.data\"\n" + + " ]\n" + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " }\n" + " }\n" + "}", Dsl.class); + DslConf dslConf = SerializerUtils.deserialize("{\n" + " \"dsl_version\": 2,\n" + " \"role\": {\n" + + " \"host\": [\n" + " 10000\n" + " ],\n" + " \"guest\": [\n" + + " 9999\n" + " ]\n" + " },\n" + " \"component_parameters\": {\n" + + " \"role\": {\n" + " \"guest\": {\n" + " \"0\": {\n" + + " \"data_transform_0\": {\n" + " \"with_label\": true\n" + + " },\n" + " \"hetero_nn_0\": {\n" + + " \"bottom_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 10\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 10\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 3,\n" + + " \"activation\": \"relu\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"top_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_2\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_2_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_2\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 1,\n" + + " \"activation\": \"sigmoid\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"interactive_layer_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_1\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_1_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_1\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 2,\n" + + " \"activation\": \"linear\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " }\n" + + " },\n" + " \"reader_1\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_hetero_guest\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " },\n" + " \"data_transform_1\": {\n" + + " \"with_label\": true\n" + " },\n" + + " \"reader_0\": {\n" + " \"table\": {\n" + + " \"name\": \"breast_hetero_guest\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " }\n" + " }\n" + " },\n" + " \"host\": {\n" + + " \"0\": {\n" + " \"data_transform_0\": {\n" + + " \"with_label\": false\n" + " },\n" + + " \"hetero_nn_0\": {\n" + " \"bottom_nn_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_3\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 20\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_3_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_3\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 20\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 3,\n" + + " \"activation\": \"relu\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " },\n" + + " \"interactive_layer_define\": {\n" + + " \"class_name\": \"Sequential\",\n" + + " \"config\": {\n" + + " \"name\": \"sequential_4\",\n" + + " \"layers\": [\n" + " {\n" + + " \"class_name\": \"InputLayer\",\n" + + " \"config\": {\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"sparse\": false,\n" + + " \"ragged\": false,\n" + + " \"name\": \"dense_4_input\"\n" + + " }\n" + " },\n" + + " {\n" + + " \"class_name\": \"Dense\",\n" + + " \"config\": {\n" + + " \"name\": \"dense_4\",\n" + + " \"trainable\": true,\n" + + " \"batch_input_shape\": [\n" + + " null,\n" + + " 2\n" + + " ],\n" + + " \"dtype\": \"float32\",\n" + + " \"units\": 2,\n" + + " \"activation\": \"linear\",\n" + + " \"use_bias\": true,\n" + + " \"kernel_initializer\": {\n" + + " \"class_name\": \"Constant\",\n" + + " \"config\": {\n" + + " \"value\": 1\n" + + " }\n" + + " },\n" + + " \"bias_initializer\": {\n" + + " \"class_name\": \"Zeros\",\n" + + " \"config\": {}\n" + + " },\n" + + " \"kernel_regularizer\": null,\n" + + " \"bias_regularizer\": null,\n" + + " \"activity_regularizer\": null,\n" + + " \"kernel_constraint\": null,\n" + + " \"bias_constraint\": null\n" + + " }\n" + " }\n" + + " ]\n" + " },\n" + + " \"keras_version\": \"2.4.0\",\n" + + " \"backend\": \"tensorflow\"\n" + " }\n" + + " },\n" + " \"reader_1\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_hetero_host\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " },\n" + " \"data_transform_1\": {\n" + + " \"with_label\": false\n" + " },\n" + + " \"reader_0\": {\n" + " \"table\": {\n" + + " \"name\": \"breast_hetero_host\",\n" + + " \"namespace\": \"experiment\"\n" + " }\n" + + " }\n" + " }\n" + " }\n" + " },\n" + + " \"common\": {\n" + " \"hetero_nn_0\": {\n" + " \"epochs\": 3,\n" + + " \"interactive_layer_lr\": 0.15,\n" + " \"batch_size\": -1,\n" + + " \"early_stop\": \"diff\",\n" + " \"callback_param\": {\n" + + " \"callbacks\": [\n" + " \"ModelCheckpoint\"\n" + + " ],\n" + " \"save_freq\": \"epoch\",\n" + + " \"validation_freqs\": 1,\n" + + " \"use_first_metric_only\": true,\n" + + " \"early_stopping_rounds\": 15,\n" + " \"metrics\": [\n" + + " \"AUC\"\n" + " ]\n" + " },\n" + + " \"optimizer\": {\n" + " \"learning_rate\": 0.15,\n" + + " \"decay\": 0.0,\n" + " \"momentum\": 0.0,\n" + + " \"nesterov\": false,\n" + " \"optimizer\": \"SGD\"\n" + + " },\n" + " \"loss\": \"binary_crossentropy\",\n" + + " \"bottom_nn_define\": null,\n" + " \"top_nn_define\": null,\n" + + " \"interactive_layer_define\": null,\n" + + " \"config_type\": \"keras\"\n" + " },\n" + + " \"hetero_nn_1\": {\n" + " \"optimizer\": null,\n" + + " \"bottom_nn_define\": null,\n" + " \"top_nn_define\": null,\n" + + " \"interactive_layer_define\": null,\n" + + " \"config_type\": \"keras\"\n" + " }\n" + " }\n" + " }\n" + "}", DslConf.class); + DSLTranslator.FateDAG fateDAG = new DSLTranslator.FateDAG(); + fateDAG.setConf(dslConf); + fateDAG.setDsl(dsl); + DSLTranslator dslTranslator = new DSLTranslator(); + Graph graph = dslTranslator.translate(fateDAG); + DSLTranslator.FateDAG fateDAG1 = dslTranslator.translate(graph); + System.out.println(toJsonString(fateDAG1)); + System.out.println(toJsonString(fateDAG)); + } +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java index d0d7353..9219ba2 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/context/FateContext.java @@ -1,7 +1,9 @@ package com.webank.ai.fate.context; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.Configuration; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.webank.ai.fate.client.FateClient; import com.webank.ai.fate.translator.DSLTranslator; @@ -18,6 +20,8 @@ public class FateContext { Configuration config; + HighFlipRuntime highFlipRuntime; + FateClient client; DSLTranslator translator = new DSLTranslator(); @@ -26,8 +30,16 @@ public static FateContext from(InstanceRegister register) { FateContext context = new FateContext(); Configuration config = (Configuration) register.getInstance(InstanceNameList.HIGHFLIP_CONFIGURATION); + HighFlipRuntime highFlipRuntime = + (HighFlipRuntime) register.getInstance(InstanceNameList.HIGHFLIP_RUNTIME); context.setConfig(config); - + context.setHighFlipRuntime(highFlipRuntime); + ServiceAdaptor serviceAdaptor = + (ServiceAdaptor) register.getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + log.info("connecting to fate flow http service:{}", serviceAdaptor.getUrl()); + context.setClient(FateClient.connect(serviceAdaptor.getUrl())); + context.getTranslator().setPartyId(serviceAdaptor.getPartyId()); + context.getTranslator().setRole(serviceAdaptor.getRole()); return context; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java index 117f719..3b8af09 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/translator/DSLTranslator.java @@ -1,17 +1,360 @@ package com.webank.ai.fate.translator; import com.baidu.highflip.core.entity.dag.Graph; +import com.baidu.highflip.core.entity.dag.Node; +import com.baidu.highflip.core.entity.dag.Party; +import com.baidu.highflip.core.entity.dag.PartyNode; +import com.baidu.highflip.core.entity.dag.codec.TypeValue; +import com.baidu.highflip.core.entity.dag.common.NodeInputRef; +import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.webank.ai.fate.client.form.dsl.Component; +import com.webank.ai.fate.client.form.dsl.ComponentParameters; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.dsl.Input; +import com.webank.ai.fate.client.form.dsl.Output; +import com.webank.ai.fate.client.form.dsl.RoleConf; +import com.webank.ai.fate.client.form.dsl.Site; +import com.webank.ai.fate.common.deserializer.SerializerUtils; + +import highflip.HighflipMeta; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.util.CollectionUtils; + @Slf4j -@NoArgsConstructor +@Data public class DSLTranslator { + String partyId; + + String role; + public FateDAG translate(Graph dag) { - return new FateDAG(); + Map componentMap = new HashMap<>(); + Map> componentConfMap = new HashMap<>(); + Map> role = new HashMap<>(); + Map> hostConf = new HashMap<>(); + Map> guestConf = new HashMap<>(); + Map> arbiterConf = new HashMap<>(); + for (Node node : dag.getNodes().values()) { + Component component = new Component(); + component.setInput(transInput(node.getInputs())); + component.setOutput(transOutput(node.getOutputs())); + component.setModule(node.getType()); + Map conf = new HashMap<>(); + for (String key : node.getAttributes().keySet()) { + conf.put(key, TypeValue.fromProto( + (HighflipMeta.TypedValueProto) node.getAttribute(key, null))); + } + componentConfMap.put(node.getName(), conf); + componentMap.put(node.getName(), component); + } + int guestIndex = 0; + int hostIndex = 0; + int arbiterIndex = 0; + Map> roleMap = new HashMap<>(); + List guest = new ArrayList<>(); + List host = new ArrayList<>(); + List arbiter = new ArrayList<>(); + for (Party party : dag.getParties().values()) { + role.putIfAbsent(party.getRole(), new ArrayList<>()); + role.get(party.getRole()).add(party.getName()); + Map componentAttr = new HashMap<>(); + for (PartyNode node : party.getNodes().values()) { + Map attr = node.getAttributes(); + componentAttr.put(node.getName(), attr); + } + if (party.getRole().equalsIgnoreCase("host")) { + hostConf.put(hostIndex + "", componentAttr); + host.add(party.getName()); + hostIndex++; + } + if (party.getRole().equalsIgnoreCase("guest")) { + guestConf.put(guestIndex + "", componentAttr); + guest.add(party.getName()); + guestIndex++; + } + if (party.getRole().equalsIgnoreCase("arbiter")) { + arbiterConf.put(arbiterIndex + "", componentAttr); + arbiter.add(party.getName()); + arbiterIndex++; + } + } + if (!guest.isEmpty()) { + roleMap.put("guest", guest); + } + if (!host.isEmpty()) { + roleMap.put("host", host); + } + if (!arbiter.isEmpty()) { + roleMap.put("arbiter", arbiter); + } + Dsl dsl = new Dsl(); + dsl.setComponents(componentMap); + DslConf dslConf = new DslConf(); + dslConf.setDsl_version("2"); + dslConf.setRole(roleMap); + ComponentParameters componentParameters = new ComponentParameters(); + componentParameters.setCommon(componentConfMap); + RoleConf roleConf = new RoleConf(); + if (!hostConf.isEmpty()) { + roleConf.setHost(hostConf); + } + if (!guestConf.isEmpty()) { + roleConf.setGuest(guestConf); + } + if (!arbiterConf.isEmpty()) { + roleConf.setArbiter(arbiterConf); + } + componentParameters.setRole(roleConf); + dslConf.setComponent_parameters(componentParameters); + FateDAG fateDAG = new FateDAG(); + fateDAG.setDsl(dsl); + fateDAG.setConf(dslConf); + // set initiator to fateDAG + Site initiator = new Site(); + initiator.setRole(this.role); + initiator.setParty_id(this.partyId); + fateDAG.getConf().setInitiator(initiator); + return fateDAG; + } + + public Graph translate(FateDAG dag) { + Graph graph = new Graph(); + Dsl dsl = dag.getDsl(); + Node pre = null; + Map nodeMap = new HashMap<>(); + Map partyMap = new HashMap<>(); + Map> rolePartyMap = new HashMap<>(); + graph.setNodes(nodeMap); + graph.setParties(partyMap); + for (Map.Entry entity : dsl.getComponents().entrySet()) { + String key = entity.getKey(); + Component component = entity.getValue(); + Node node = new Node(); + node.setName(key); + node.setType(component.getModule()); + if (component.getInput() != null) { + node.setInputs(transInput(component.getInput())); + } + if (component.getOutput() != null) { + node.setOutputs(transOutput(component.getOutput())); + } + if (pre != null) { + node.setParent(pre); + } + pre = node; + nodeMap.put(key, node); + } + DslConf dslConf = dag.getConf(); + for (Map.Entry> roleEntity : dslConf.getRole().entrySet()) { + String role = roleEntity.getKey(); + rolePartyMap.putIfAbsent(role.toLowerCase(), new ArrayList<>()); + for (String partyId : roleEntity.getValue()) { + Party party = new Party(); + party.setName(partyId); + party.setRole(role); + party.setNodes(new HashMap<>()); + partyMap.put(role + "-" + partyId, party); + rolePartyMap.get(role.toLowerCase()).add(party); + } + } + ComponentParameters componentParameters = dslConf.getComponent_parameters(); + for (Map.Entry> attrMap : componentParameters.getCommon().entrySet()) { + String componentName = attrMap.getKey(); + Map attr = attrMap.getValue(); + Node node = nodeMap.get(componentName); + if (node == null) { + throw new RuntimeException("dsl conf component not match"); + } + attr.forEach(node::setAttribute); + } + if (componentParameters.getRole().getGuest() != null) { + for (Map.Entry> attr : componentParameters.getRole().getGuest().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("guest").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + if (componentParameters.getRole().getHost() != null) { + for (Map.Entry> attr : componentParameters.getRole().getHost().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("host").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + if (componentParameters.getRole().getArbiter() != null) { + for (Map.Entry> attr : componentParameters.getRole().getArbiter().entrySet()) { + String index = attr.getKey(); + Map componentsAttrs = attr.getValue(); + Party party = rolePartyMap.get("arbiter").get(Integer.parseInt(index)); + addPartyNodes(componentsAttrs, party); + } + } + return graph; + } + + private void addPartyNodes(Map componentsAttrs, Party party) { + if (party == null) { + throw new RuntimeException("dsl conf role not match"); + } + for (Map.Entry componentEntry : componentsAttrs.entrySet()) { + String componentName = componentEntry.getKey(); + Object componentAttr = componentEntry.getValue(); + PartyNode partyNode = new PartyNode(); + partyNode.setParent(party); + partyNode.setName(componentName); + Map attrMap; + try { + attrMap = SerializerUtils.deserializeType(SerializerUtils.toJsonString(componentAttr), + new TypeReference>() { + }); + if (attrMap != null) { + partyNode.setAttributes(attrMap); + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + party.getNodes().put(componentName, partyNode); + } + } + + private Map transInput(Input input) { + try { + Map map = new HashMap<>(); + if (input.getData() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(SerializerUtils.toJsonString(input.getData())) + .build(); + map.put("data", NodeInputRef.fromProto(nodeInputProto)); + } + if (input.getModel() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("model") + .setValue(SerializerUtils.toJsonString(input.getModel())) + .build(); + map.put("model", NodeInputRef.fromProto(nodeInputProto)); + } + if (input.getCache() != null) { + HighflipMeta.NodeInputProto nodeInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("cache") + .setValue(SerializerUtils.toJsonString(input.getCache())) + .build(); + map.put("cache", NodeInputRef.fromProto(nodeInputProto)); + } + return map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Input transInput(Map map) { + try { + if (map == null || CollectionUtils.isEmpty(map)) { + return null; + } + Input input = new Input(); + TypeReference>> typeReference = new TypeReference<>() { + }; + if (map.containsKey("data")) { + String content = map.get("data").getValue(); + Map> data = + SerializerUtils.deserializeType(content, typeReference); + input.setData(data); + } + TypeReference> typeReference2 = new TypeReference<>() { + }; + if (map.containsKey("model")) { + List model = SerializerUtils.deserializeType(map.get( + "model").getValue(), typeReference2); + input.setModel(model); + } + + if (map.containsKey("cache")) { + List cache = SerializerUtils.deserializeType(map.get( + "cache").getValue(), typeReference2); + input.setCache(cache); + } + return input; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map transOutput(Output output) { + try { + Map map = new HashMap<>(); + HighflipMeta.NodeOutputProto dataNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("data") + .setValue(SerializerUtils.toJsonString(output.getData())) + .build(); + map.put("data", NodeOutputRef.fromProto(dataNodeOutputProto)); + + HighflipMeta.NodeOutputProto modelNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("model") + .setValue(SerializerUtils.toJsonString(output.getModel())) + .build(); + map.put("model", NodeOutputRef.fromProto(modelNodeOutputProto)); + + HighflipMeta.NodeOutputProto cacheNodeOutputProto = + HighflipMeta.NodeOutputProto.newBuilder() + .setName("cache") + .setValue(SerializerUtils.toJsonString(output.getCache())) + .build(); + map.put("cache", NodeOutputRef.fromProto(cacheNodeOutputProto)); + return map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Output transOutput(Map map) { + try { + if (map == null || CollectionUtils.isEmpty(map)) { + return null; + } + TypeReference> typeReference2 = new TypeReference<>() { + }; + Output output = new Output(); + if (map.containsKey("data")) { + List data = SerializerUtils.deserializeType( + map.get("data").getValue(), typeReference2); + output.setData(data); + } + if (map.containsKey("model")) { + List model = SerializerUtils.deserializeType( + map.get("model").getValue(), typeReference2); + output.setModel(model); + + } + if (map.containsKey("cache")) { + List cache = SerializerUtils.deserializeType( + map.get("cache").getValue(), typeReference2); + output.setCache(cache); + } + return output; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } } @Data @@ -19,8 +362,9 @@ public FateDAG translate(Graph dag) { @NoArgsConstructor public static class FateDAG { - String dsl; + Dsl dsl; + + DslConf conf; - String conf; } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java new file mode 100644 index 0000000..c47cf48 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/adaptor/DataAdaptorTest.java @@ -0,0 +1,44 @@ +package com.webank.ai.fate.adaptor; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; + +import com.baidu.highflip.core.utils.Foreach; + +public class DataAdaptorTest { + private final String DEFAULT_DELIMITER = ","; + + @Test + public void writeDataDenseTest() { + List> body = new ArrayList<>(); + List rowHeader = List.of("ID","name"); + List rowOne = List.of("1","bob"); + List rowTwo = List.of("2","joe"); + List rowThree = List.of("3","alice"); + body.add(rowHeader); + body.add(rowOne); + body.add(rowTwo); + body.add(rowThree); + Iterator> iterator = body.iterator(); + + StringBuilder stringBuilder = new StringBuilder(); + for (List dataList : Foreach.from(iterator)) { + String column = dataList.stream().map(Object::toString).collect( + Collectors.joining(DEFAULT_DELIMITER)); + stringBuilder.append(column).append("\n"); + } + } + + @Test + public void testSplit(){ + String dataId = "8a53e2c4-7e04-482b-9eef-4dd6c8376b1e+intersect_0" + + "+guest+9999"; + final String[] split = dataId.split("\\+"); + System.out.println(List.of(split)); + } + +} diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java index d445c3a..aad5c89 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/client/FateClientTest.java @@ -1,38 +1,158 @@ package com.webank.ai.fate.client; -import com.webank.ai.fate.client.form.JsonResultForm; +import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.dsl.Dsl; +import com.webank.ai.fate.client.form.dsl.DslConf; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.client.form.job.JobData; +import com.webank.ai.fate.client.form.task.TaskData; +import com.webank.ai.fate.common.DataMultipartFile; +import com.webank.ai.fate.common.DecompressUtils; +import com.webank.ai.fate.common.deserializer.SerializerUtils; +import feign.Response; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; @Slf4j @Data @Disabled public class FateClientTest { - public static final String TEST_FLOW_URL = "http://127.0.0.1:9380"; + public static final String TEST_FLOW_URL = "http://10.27.130.41:8380"; FateClient client = FateClient.connect(TEST_FLOW_URL); + // @Test + // public void testVersionGet() { + // JsonResultForm result = getClient().version(); + // log.info("result = {}", result.getData().toPrettyString()); + // } + @Test - public void testVersionGet() { - JsonResultForm result = getClient().version(); - log.info("result = {}", result.getData().toPrettyString()); + public void testJobList() { + ResultForm result = getClient().listJob(1, 10); + log.info("result = {}", result); } @Test - public void testJobList() { - JsonResultForm result = getClient().jobList(10); - log.info("result = {}", result.getData().toPrettyString()); + public void testTaskList() { + ResultForm result = getClient().listTask(null, 18, 1); + log.info("result = {}", result); } @Test public void testJobQuery() { - ResultForm result = getClient().jobQuery("202210260752357956950"); + ResultForm result = getClient().jobQuery("202212011128410103660"); log.info("result = {}", result.getData().toString()); } + + @Test + public void testPushData() { + String data = "id,y,x0,x1,x2,x3,x4,x5\n" + "274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613\n" + + "199,0,0.426758,0.723479,0.316885,0.287273,1.000835,0.962702\n" + + "133,1,0.254879,-1.046633,0.209656,0.074214,-0.441366,-0.377645\n" + + "273,1,-1.142928,-0.781198,-1.166747,-0.923578,0.62823,-1.021418\n" + + "551,1,-0.879933,0.420589,-0.877527,-0.780484,-1.037534,-0.48388\n" + + "175,1,-1.451067,-1.406518,-1.456564,-1.092337,-0.708765,-1.168557\n"; + MultipartFile multipartFile = new DataMultipartFile("unittest_data3.csv", data.getBytes(StandardCharsets.UTF_8)); + ResultForm resultForm = getClient().pushData(multipartFile, ",", "1", "4", "highflip_test1", "experiment", null, null, null,"1"); + log.info("result = {}", resultForm); + } + + @Test + public void testJobSubmit() throws JsonProcessingException { + Dsl dsl = SerializerUtils.deserialize("{\n" + " \"components\": {\n" + " \"reader_0\": {\n" + + " \"module\": \"Reader\",\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"data_transform_0\": {\n" + + " \"module\": \"DataTransform\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"data\": [\n" + + " \"reader_0.data\"\n" + " ]\n" + " }\n" + + " },\n" + " \"output\": {\n" + " \"data\": [\n" + + " \"data\"\n" + " ],\n" + " \"model\": [\n" + + " \"model\"\n" + " ]\n" + " }\n" + " },\n" + + " \"homo_nn_0\": {\n" + " \"module\": \"HomoNN\",\n" + " \"input\": {\n" + + " \"data\": {\n" + " \"train_data\": [\n" + + " \"data_transform_0.data\"\n" + " ]\n" + + " }\n" + " },\n" + " \"output\": {\n" + + " \"data\": [\n" + " \"data\"\n" + " ],\n" + + " \"model\": [\n" + " \"model\"\n" + " ]\n" + + " }\n" + " }\n" + " }\n" + "}", Dsl.class); + DslConf dslConf = SerializerUtils.deserialize("{\n" + " \"dsl_version\": 2,\n" + " \"initiator\": {\n" + + " \"role\": \"guest\",\n" + " \"party_id\": 10000\n" + " },\n" + " \"role\": {\n" + + " \"arbiter\": [\n" + " 10000\n" + " ],\n" + " \"host\": [\n" + + " 10000\n" + " ],\n" + " \"guest\": [\n" + " 10000\n" + + " ]\n" + " },\n" + " \"component_parameters\": {\n" + " \"common\": {\n" + + " \"data_transform_0\": {\n" + " \"with_label\": true,\n" + + " \"label_name\": \"y\",\n" + " \"label_type\": \"int\",\n" + + " \"output_format\": \"dense\"\n" + " },\n" + + " \"homo_nn_0\": {\n" + " \"config_type\": \"pytorch\",\n" + + " \"nn_define\": [\n" + " {\n" + + " \"layer\": \"Linear\",\n" + " \"name\": \"line\",\n" + + " \"type\": \"normal\",\n" + " \"config\": [\n" + + " 30,\n" + " 1\n" + + " ]\n" + " },\n" + " {\n" + + " \"layer\": \"Sigmoid\",\n" + + " \"type\": \"activate\",\n" + + " \"name\": \"sigmoid\"\n" + " }\n" + " ],\n" + + " \"batch_size\": -1,\n" + " \"optimizer\": {\n" + + " \"optimizer\": \"Adam\",\n" + " \"lr\": 0.05\n" + + " },\n" + " \"early_stop\": {\n" + + " \"early_stop\": \"diff\",\n" + " \"eps\": 0.0001\n" + + " },\n" + " \"loss\": \"BCELoss\",\n" + + " \"metrics\": [\n" + " \"accuracy\"\n" + " ],\n" + + " \"max_iter\": 2\n" + " }\n" + " },\n" + " \"role\": {\n" + + " \"host\": {\n" + " \"0\": {\n" + " \"reader_0\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_homo_host\",\n" + + " \"namespace\": \"HIGH-FLIP\"\n" + " }\n" + + " }\n" + " }\n" + " },\n" + " \"guest\": {\n" + + " \"0\": {\n" + " \"reader_0\": {\n" + + " \"table\": {\n" + + " \"name\": \"breast_homo_guest\",\n" + + " \"namespace\": \"HIGH-FLIP\"\n" + " }\n" + + " }\n" + " }\n" + " }\n" + " }\n" + " }\n" + + "}", DslConf.class); + log.info("dsl:{}", SerializerUtils.toJsonString(dsl)); + log.info("dsl conf:{}", SerializerUtils.toJsonString(dslConf)); + ResultForm job = getClient().jobSubmit(dsl, dslConf); + log.info("result = {}", job); + } + + @Test + public void testDownloadData() { + try(Response response = getClient().downloadData("unittest_data3", "experiment")) { + InputStream inputStream = response.body().asInputStream(); + Map result = DecompressUtils.decompressTarGzToStringMap(inputStream, s -> s.contains("csv")); + log.info("result = {}, head = {}", result, response.headers()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testDownloadComponentResultData(){ + String jobId = "202305231618271012050"; + String componentName = "intersect_0"; + String role = "guest"; + String partyId = "9999"; + try (Response response = getClient().downloadComponentResultData(jobId, componentName, role, partyId)) { + InputStream inputStream = response.body().asInputStream(); + + Map result = DecompressUtils.decompressTarGzToStringMap(inputStream, s -> s.contains("csv")); + log.info("result = {}, head = {}", result, response.headers()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java new file mode 100644 index 0000000..7f4db49 --- /dev/null +++ b/highflip-vendors/highflip-adaptor-fate/src/test/java/com/webank/ai/fate/translator/DSLTranslatorTest.java @@ -0,0 +1,434 @@ +package com.webank.ai.fate.translator; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.baidu.highflip.core.entity.dag.Graph; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.ai.fate.client.FateClient; +import com.webank.ai.fate.client.form.ResultForm; +import com.webank.ai.fate.client.form.job.FateJob; +import com.webank.ai.fate.common.deserializer.SerializerUtils; + +import highflip.HighflipMeta; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DSLTranslatorTest { + + @Test + public void translateTest() { + // step 1: construct a highflip graph + HighflipMeta.GraphProto graphProto = constructHighflipGraph(); + + Graph graph = Graph.fromProto(graphProto); + // step 2: translate highflip graph to fate dag (dsl and conf) + DSLTranslator dslTranslator = new DSLTranslator(); + + DSLTranslator.FateDAG translate = dslTranslator.translate(graph); + + // fate flow http service + String TEST_FLOW_URL = "http://10.27.130.41:8380"; + + FateClient client = FateClient.connect(TEST_FLOW_URL); + + try { + log.info("conf = {}", + SerializerUtils.toJsonString(translate.getDsl())); + log.info("dsl = {}", + SerializerUtils.toJsonString(translate.getConf())); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + ResultForm job = client.jobSubmit(translate.getDsl(), + translate.getConf()); + log.info("result = {}", job); + } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + readerNodeOutputProtos.add(outputProto); + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model") + .setValue(getJsonStringOfModelOutput()) + .build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + dataTransformNodeInputProtos.add(dataTransformInputProto); + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs(dataTransformNodeInputProtos) + .addAllOutputs(dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data") + .setValue(getJsonStringOfDataOutput()) + .build(); + intersectNodeOutputProtos.add(intersectOutputProto); + List intersectNodeInputProtos = + new ArrayList<>(); + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + intersectNodeInputProtos.add(intersectInputProto); + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + // construct guest + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder() + .setRole(HighflipMeta.PartyRole.GUEST) + .addAllNodes(guestPartyNodes) + .setName("9999") + .build(); + + // construct host + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder() + .setRole(HighflipMeta.PartyRole.HOST) + .addAllNodes(hostPartyNodes) + .setName("10000") + .build(); + + // components + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + + @Test + public void testValueProto() { + Map tableMap2 = + new HashMap<>(); + tableMap2.put("name", HighflipMeta.ValueProto.newBuilder().setString( + "csv1").build()); + tableMap2.put("namespace", + HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + tableMap.put("bo", + HighflipMeta.ValueProto.newBuilder().setBool(true).build()); + + HighflipMeta.MapProto mapProto2 = HighflipMeta.MapProto.newBuilder().putAllMap(tableMap2).build(); + tableMap.put("mp", + HighflipMeta.ValueProto.newBuilder().setMap(mapProto2).build()); + + List list = new ArrayList<>(); + list.add(HighflipMeta.ValueProto.newBuilder().setMap(mapProto2).build()); + + HighflipMeta.ListProto listProto = + HighflipMeta.ListProto.newBuilder().addAllList(list).build(); + + tableMap.put("list", + HighflipMeta.ValueProto.newBuilder().setList(listProto).build()); + + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.MapProto map = typedValueProto.getValue().getMap(); + Map map1 = mapProtoConvert2Map(map); + try{ + String s = SerializerUtils.toJsonString(map1); + System.out.println(s); + }catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto){ + Map result = new HashMap<>(); + Map map = mapProto.getMapMap(); + for(Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private Object convertValueProto(HighflipMeta.ValueProto value){ + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + // Map + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + +}