diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/crud/repository/BaseRepository.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/crud/repository/BaseRepository.java index ddb7984d..a4547676 100644 --- a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/crud/repository/BaseRepository.java +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/crud/repository/BaseRepository.java @@ -96,7 +96,10 @@ public Optional findOneById(ID id) { @Override public boolean existsById(ID id) { - return findOneById(id).isPresent(); + QueryWrapper wrapper = new QueryWrapper<>(); + TableInfo tableInfo = getTableInfo(); + wrapper.eq(tableInfo.getKeyColumn(), id); + return getBaseMapper().selectCount(wrapper) > 0; } @Override diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchColumnResolver.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchColumnResolver.java new file mode 100644 index 00000000..8a4384b6 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchColumnResolver.java @@ -0,0 +1,63 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.power4j.fist.boot.mybaits.crud.repository.Repository; + +import java.io.Serializable; +import java.util.Collection; +import java.util.function.BiConsumer; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public class BatchColumnResolver implements BatchUniqueResolver { + + private final Repository repository; + + private final BiConsumer, LambdaQueryWrapper> queryBuilder; + + public static BatchColumnResolver of(Repository repository, + BiConsumer, LambdaQueryWrapper> queryBuilder) { + return new BatchColumnResolver<>(repository, queryBuilder); + } + + public BatchColumnResolver(Repository repository, + BiConsumer, LambdaQueryWrapper> queryBuilder) { + this.queryBuilder = queryBuilder; + this.repository = repository; + } + + @Override + public long exists(Collection data) { + return repository.countBy(applyQuery(data)); + } + + @Override + public void removeExists(Collection data) { + repository.deleteAllBy(applyQuery(data)); + } + + protected LambdaQueryWrapper applyQuery(Collection example) { + LambdaQueryWrapper queryWrapper = repository.lambdaWrapper(); + queryBuilder.accept(example, queryWrapper); + return queryWrapper; + } + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchImportPipeline.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchImportPipeline.java new file mode 100644 index 00000000..ea209266 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchImportPipeline.java @@ -0,0 +1,117 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import com.power4j.fist.boot.mybaits.crud.repository.Repository; +import com.power4j.fist.data.migrate.ImportDataHandler; +import com.power4j.fist.data.migrate.ImportStatistic; +import lombok.extern.slf4j.Slf4j; + +import java.io.Serializable; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +@Slf4j +public class BatchImportPipeline implements ImportDataHandler

{ + + private final UniqueResolveEnum uniqueResolve; + + private final BatchUniqueResolver uniqueResolver; + + private final Repository repository; + + private final Function mapper; + + private final boolean cleanAll; + + private final long maxImportCount; + + private final ImportStatistic statistic = ImportStatistic.empty(); + + public BatchImportPipeline(UniqueResolveEnum uniqueResolve, BatchUniqueResolver uniqueResolver, + Repository repository, Function mapper, boolean cleanAll, long maxImportCount) { + if (!cleanAll && uniqueResolve == UniqueResolveEnum.SKIP) { + throw new IllegalArgumentException("cleanAll must be true when uniqueResolve is SKIP"); + } + this.uniqueResolve = uniqueResolve; + this.uniqueResolver = uniqueResolver; + this.repository = repository; + this.mapper = mapper; + this.cleanAll = cleanAll; + this.maxImportCount = maxImportCount; + } + + @Override + public void accept(Collection

data) { + final int size = data.size(); + if (log.isDebugEnabled()) { + log.debug("receive data,count :{},total :{}", size, statistic.getReceiveCount()); + } + statistic.addReceiveCount(size); + handleData(data); + } + + private void handleData(Collection

data) { + // 批量模式下不进行精确控制条数 + if (statistic.getInsertCount() >= maxImportCount && maxImportCount > 0) { + log.info("Import max count reached: {},skip", maxImportCount); + return; + } + long existCount; + List entities = data.stream().map(mapper).toList(); + int skipCount = 0; + int deleteCount = 0; + if (!cleanAll && (existCount = uniqueResolver.exists(entities)) > 0L) { + switch (uniqueResolve) { + case SKIP: + skipCount = (int) existCount; + break; + case REMOVE: + uniqueResolver.removeExists(entities); + deleteCount = (int) existCount; + break; + default: + throw new IllegalStateException("Unsupported unique resolve: " + uniqueResolve); + } + } + repository.saveAll(entities); + statistic.addSkipCount(skipCount); + statistic.addDeleteCount(deleteCount); + statistic.addInsertCount(entities.size() - skipCount); + } + + @Override + public ImportStatistic statistic() { + return statistic; + } + + @Override + public void beforeAll() { + statistic.reset(); + if (cleanAll) { + long deleteCount = repository.countAll(); + repository.deleteAll(); + statistic.setDeleteCount(deleteCount); + } + } + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchUniqueResolver.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchUniqueResolver.java new file mode 100644 index 00000000..2f33e650 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/BatchUniqueResolver.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import java.util.Collection; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public interface BatchUniqueResolver { + + /** + * 检测是记录否存在 + * @param data 数据 + * @return 已经存在数据条数 + */ + long exists(Collection data); + + /** + * 删除已经存在的记录 + * @param data 数据 + */ + void removeExists(Collection data); + + BatchUniqueResolver INVALID = new BatchUniqueResolver<>() { + @Override + public long exists(Collection data) { + throw new IllegalStateException("Invalid unique resolver"); + } + + @Override + public void removeExists(Collection data) { + throw new IllegalStateException("Invalid unique resolver"); + } + }; + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ColumnResolver.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ColumnResolver.java new file mode 100644 index 00000000..001d528c --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ColumnResolver.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.power4j.fist.boot.mybaits.crud.repository.Repository; + +import java.io.Serializable; +import java.util.function.BiConsumer; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public class ColumnResolver implements UniqueResolver { + + private final Repository repository; + + private final BiConsumer> queryBuilder; + + public static ColumnResolver of(Repository repository, + BiConsumer> queryBuilder) { + return new ColumnResolver<>(repository, queryBuilder); + } + + public ColumnResolver(Repository repository, + BiConsumer> queryBuilder) { + this.queryBuilder = queryBuilder; + this.repository = repository; + } + + @Override + public boolean exists(T example) { + return repository.countBy(applyQuery(example)) > 0; + } + + @Override + public void remove(T example) { + repository.deleteAllBy(applyQuery(example)); + } + + protected LambdaQueryWrapper applyQuery(T example) { + LambdaQueryWrapper queryWrapper = repository.lambdaWrapper(); + queryBuilder.accept(example, queryWrapper); + return queryWrapper; + } + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ImportPipeline.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ImportPipeline.java new file mode 100644 index 00000000..24a8752b --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ImportPipeline.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import com.power4j.fist.boot.mybaits.crud.repository.Repository; +import com.power4j.fist.data.migrate.ImportDataHandler; +import com.power4j.fist.data.migrate.ImportStatistic; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.io.Serializable; +import java.util.Collection; +import java.util.function.Function; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +@Slf4j +@RequiredArgsConstructor +public class ImportPipeline implements ImportDataHandler

{ + + private final UniqueResolveEnum uniqueResolve; + + private final UniqueResolver uniqueResolver; + + private final Repository repository; + + private final Function mapper; + + private final boolean cleanAll; + + private final long maxImportCount; + + private final ImportStatistic statistic = ImportStatistic.empty(); + + @Override + public void accept(Collection

data) { + final int size = data.size(); + if (log.isDebugEnabled()) { + log.debug("receive data,count :{},total :{}", size, statistic.getReceiveCount()); + } + statistic.addReceiveCount(data.size()); + handleData(data); + } + + private void handleData(Collection

data) { + for (P p : data) { + if (statistic.getInsertCount() >= maxImportCount && maxImportCount > 0) { + log.info("Import max count reached: {},skip", maxImportCount); + return; + } + T entity = mapper.apply(p); + if (!cleanAll && uniqueResolver.exists(entity)) { + switch (uniqueResolve) { + case SKIP: + statistic.addSkipCount(1); + return; + case REMOVE: + uniqueResolver.remove(entity); + statistic.addDeleteCount(1); + break; + default: + throw new IllegalStateException("Unsupported unique resolve: " + uniqueResolve); + } + } + repository.saveOne(entity); + statistic.addInsertCount(1); + } + + } + + @Override + public ImportStatistic statistic() { + return statistic; + } + + @Override + public void beforeAll() { + statistic.reset(); + if (cleanAll) { + long deleteCount = repository.countAll(); + repository.deleteAll(); + statistic.setDeleteCount(deleteCount); + } + } + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ImportPipelineBuilder.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ImportPipelineBuilder.java new file mode 100644 index 00000000..83a03b20 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/ImportPipelineBuilder.java @@ -0,0 +1,96 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import com.power4j.fist.boot.mybaits.crud.repository.Repository; +import com.power4j.fist.data.migrate.ImportDataHandler; + +import java.io.Serializable; +import java.util.function.Function; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public class ImportPipelineBuilder { + + private final Repository repository; + + private final Function mapper; + + private UniqueResolveEnum uniqueResolve; + + private BatchUniqueResolver batchUniqueResolver; + + private UniqueResolver uniqueResolver; + + private boolean cleanAll; + + private long maxImportCount; + + public static ImportPipelineBuilder of(Repository repository, + Function mapper) { + return new ImportPipelineBuilder<>(repository, mapper); + } + + public ImportPipelineBuilder(Repository repository, Function mapper) { + this.repository = repository; + this.mapper = mapper; + } + + public ImportPipelineBuilder conflictResolve(UniqueResolveEnum conflictResolve) { + this.uniqueResolve = conflictResolve; + return this; + } + + public ImportPipelineBuilder batchUniqueResolver(BatchUniqueResolver resolver) { + this.uniqueResolver = null; + this.batchUniqueResolver = resolver; + return this; + } + + public ImportPipelineBuilder uniqueResolver(UniqueResolver resolver) { + this.batchUniqueResolver = null; + this.uniqueResolver = resolver; + return this; + } + + public ImportPipelineBuilder cleanAll(boolean cleanAll) { + this.cleanAll = cleanAll; + return this; + } + + public ImportPipelineBuilder maxImportCount(long maxImportCount) { + this.maxImportCount = maxImportCount; + return this; + } + + public ImportDataHandler

build() { + if (!cleanAll && uniqueResolve == null) { + throw new IllegalStateException("must set unique resolve strategy when cleanAll is false"); + } + if (uniqueResolver == null && batchUniqueResolver == null) { + throw new IllegalStateException("must set uniqueResolver or batchUniqueResolver"); + } + if (batchUniqueResolver != null) { + return new BatchImportPipeline<>(uniqueResolve, batchUniqueResolver, repository, mapper, cleanAll, + maxImportCount); + } + return new ImportPipeline<>(uniqueResolve, uniqueResolver, repository, mapper, cleanAll, maxImportCount); + } + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/PrimaryResolver.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/PrimaryResolver.java new file mode 100644 index 00000000..1d7a3895 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/PrimaryResolver.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import com.power4j.fist.boot.mybaits.crud.repository.Repository; + +import java.io.Serializable; +import java.util.function.Function; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public class PrimaryResolver implements UniqueResolver { + + private final Repository repository; + + private final Function idExtractor; + + public PrimaryResolver(Repository repository, Function idExtractor) { + this.repository = repository; + this.idExtractor = idExtractor; + } + + @Override + public boolean exists(T example) { + ID id = idExtractor.apply(example); + return repository.existsById(id); + } + + @Override + public void remove(T example) { + ID id = idExtractor.apply(example); + repository.deleteOneById(id); + } + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/UniqueResolveEnum.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/UniqueResolveEnum.java new file mode 100644 index 00000000..6b010ed5 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/UniqueResolveEnum.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public enum UniqueResolveEnum { + + SKIP, REMOVE, + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/UniqueResolver.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/UniqueResolver.java new file mode 100644 index 00000000..689e477e --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/UniqueResolver.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public interface UniqueResolver { + + /** + * 检测是记录否存在 + * @param example 数据 + * @return true 表示存在 + */ + boolean exists(T example); + + /** + * 删除记录 + * @param example 数据 + */ + void remove(T example); + + UniqueResolver INVALID = new UniqueResolver<>() { + @Override + public boolean exists(Object example) { + throw new IllegalStateException("Invalid unique resolver"); + } + + @Override + public void remove(Object example) { + throw new IllegalStateException("Invalid unique resolver"); + } + }; + +} diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/package-info.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/package-info.java new file mode 100644 index 00000000..5ef02125 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/main/java/com/power4j/fist/boot/mybaits/migrate/package-info.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +@NonNullApi +package com.power4j.fist.boot.mybaits.migrate; + +import org.springframework.lang.NonNullApi; diff --git a/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/test/java/com/power4j/fist/boot/mybaits/migrate/ImportPipelineBuilderTest.java b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/test/java/com/power4j/fist/boot/mybaits/migrate/ImportPipelineBuilderTest.java new file mode 100644 index 00000000..a81e40b4 --- /dev/null +++ b/fist-kit-app/fist-data/fist-boot-crud-mybatis/src/test/java/com/power4j/fist/boot/mybaits/migrate/ImportPipelineBuilderTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.boot.mybaits.migrate; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +class ImportPipelineBuilderTest { + + @Test + void mustSetOneResolver() { + ImportPipelineBuilder builder = ImportPipelineBuilder.of(null, null) + .batchUniqueResolver(null) + .uniqueResolver(null); + Assertions.assertThrows(IllegalStateException.class, builder::build); + } + + @Test + void mustSetUniqueResolve() { + ImportPipelineBuilder builder = ImportPipelineBuilder.of(null, null); + Assertions.assertThrows(IllegalStateException.class, builder::build); + } + +} diff --git a/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/ImportDataHandler.java b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/ImportDataHandler.java new file mode 100644 index 00000000..57c111a9 --- /dev/null +++ b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/ImportDataHandler.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.data.migrate; + +import java.util.Collection; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +public interface ImportDataHandler { + + /** + * 处理输入数据 + * @param data 输入 + */ + void accept(Collection data); + + /** + * 读取统计数据 + * @return ImportStatistic + */ + ImportStatistic statistic(); + + /** + * 回调方法,在执行导入前调用 + */ + default void beforeAll() { + // do nothing + } + + /** + * 回调方法,在执行导入后调用 + */ + default void afterAll() { + // do nothing + } + +} diff --git a/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/ImportStatistic.java b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/ImportStatistic.java new file mode 100644 index 00000000..2634b701 --- /dev/null +++ b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/ImportStatistic.java @@ -0,0 +1,100 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.data.migrate; + +import com.power4j.coca.kit.common.text.Display; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +@Getter +@Setter +@Builder +public class ImportStatistic implements Display { + + private long receiveCount; + + private long skipCount; + + private long insertCount; + + private long deleteCount; + + private long updateCount; + + public static ImportStatistic empty() { + return ImportStatistic.builder().build(); + } + + public static ImportStatistic copyOf(ImportStatistic other) { + return empty().merge(other); + } + + public ImportStatistic merge(ImportStatistic other) { + this.receiveCount += other.getReceiveCount(); + this.skipCount += other.getSkipCount(); + this.insertCount += other.getInsertCount(); + this.deleteCount += other.getDeleteCount(); + this.updateCount += other.getUpdateCount(); + return this; + } + + public void addSkipCount(int val) { + this.skipCount += val; + } + + public void addInsertCount(int val) { + this.insertCount += val; + } + + public void addDeleteCount(int val) { + this.deleteCount += val; + } + + public void addUpdateCount(int val) { + this.updateCount += val; + } + + public void addReceiveCount(int val) { + this.receiveCount += val; + } + + public void reset() { + this.receiveCount = 0; + this.skipCount = 0; + this.insertCount = 0; + this.deleteCount = 0; + this.updateCount = 0; + } + + @Override + public String display() { + return "{" + "receiveCount=" + receiveCount + ", skipCount=" + skipCount + ", insertCount=" + insertCount + + ", deleteCount=" + deleteCount + ", updateCount=" + updateCount + '}'; + } + + @Override + public String toString() { + return "ImportStatistic{" + "receiveCount=" + receiveCount + ", skipCount=" + skipCount + ", insertCount=" + + insertCount + ", deleteCount=" + deleteCount + ", updateCount=" + updateCount + '}'; + } + +} diff --git a/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/MigrateUtil.java b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/MigrateUtil.java new file mode 100644 index 00000000..9d306049 --- /dev/null +++ b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/MigrateUtil.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024. ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.power4j.fist.data.migrate; + +import com.alibaba.excel.context.AnalysisContext; +import com.power4j.fist.data.excel.ExcelParser; +import lombok.experimental.UtilityClass; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +@UtilityClass +public class MigrateUtil { + + private final static int DEFAULT_BATCH_SIZE = 1_000; + + public ImportStatistic importExcel(InputStream stream, Class docType, int batchSize, + ImportDataHandler consumer) { + int batch = batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; + AtomicLong total = new AtomicLong(); + List list = new ArrayList<>(batch); + BiConsumer handler = (T data, AnalysisContext context) -> { + total.incrementAndGet(); + list.add(data); + if (list.size() >= batch) { + processAndClear(list, consumer); + } + }; + consumer.beforeAll(); + // @formatter:off + ExcelParser.builder() + .docType(docType) + .handler(handler) + .build() + .process(stream); + // @formatter:on + processAndClear(list, consumer); + consumer.afterAll(); + return consumer.statistic(); + } + + static void processAndClear(Collection data, ImportDataHandler consumer) { + if (data.isEmpty()) { + return; + } + consumer.accept(new ArrayList<>(data)); + data.clear(); + } + +} diff --git a/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/package-info.java b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/package-info.java new file mode 100644 index 00000000..65409aa4 --- /dev/null +++ b/fist-kit-app/fist-data/fist-support-data/src/main/java/com/power4j/fist/data/migrate/package-info.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 ChenJun (power4j@outlook.com & https://github.com/John-Chan) + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl.html + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @author CJ (power4j@outlook.com) + * @since 1.0 + */ +@NonNullApi +package com.power4j.fist.data.migrate; + +import org.springframework.lang.NonNullApi; diff --git a/pom.xml b/pom.xml index 95be3a6d..0dc0df70 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ false - 3.3.0 + 3.4-RC1 UTF-8 UTF-8 17 @@ -66,11 +66,11 @@ 0.2.0 0.0.43 1.6.0 - 3.0.2 + 4.0.1 0.8.12 3.3.1 9.0.1 - 3.5.0 + 3.5.1 3.13.0 3.8.0 @@ -89,10 +89,10 @@ 3.0.0 2.2.20 2.6.0 - 3.3.3 + 3.3.4 1.3.2 1.1.1.RELEASE - 11.20 + 11.20.1 2.14.5 3.3.1 5.5.1