Merge pull request #974 from syucream/feature/import-limit-on-entries

Limit freqnent import jobs for same entity by default

airone/lib/drf.py

@@ -84,6 +84,10 @@ class InvalidValueError(ValidationError):
     default_code = "AE-250000"
 
 
+class FrequentImportError(ValidationError):
+    default_code = "AE-260000"
+
+
 def custom_exception_handler(exc, context):
     def _convert_error_code(detail):
         if isinstance(detail, list):

apiclient/typescript-fetch/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dmm-com/airone-apiclient-typescript-fetch",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "AirOne APIv2 client in TypeScript",
   "main": "src/autogenerated/index.ts",
   "scripts": {

entry/api_v2/views.py

@@ -1,4 +1,6 @@
 import re
+from datetime import datetime, timedelta
+from typing import Optional
 
 from django.db.models import Q
 from drf_spectacular.types import OpenApiTypes
@@ -13,6 +15,7 @@
 from airone.lib.acl import ACLType
 from airone.lib.drf import (
     DuplicatedObjectExistsError,
+    FrequentImportError,
     IncorrectTypeError,
     ObjectNotExistsError,
     RequiredParameterError,
@@ -39,7 +42,7 @@
 from entry.settings import CONFIG
 from entry.settings import CONFIG as ENTRY_CONFIG
 from group.models import Group
-from job.models import Job
+from job.models import Job, JobOperation
 from role.models import Role
 from user.models import User
 
@@ -477,20 +480,45 @@ def get_queryset(self):
             raise IncorrectTypeError(f"unsupported attr type: {entity_attr.type}")
 
 
+@extend_schema(
+    parameters=[
+        OpenApiParameter("force", OpenApiTypes.BOOL, OpenApiParameter.QUERY, default=False),
+    ],
+)
 class EntryImportAPI(generics.GenericAPIView):
     parser_classes = [YAMLParser]
     serializer_class = serializers.Serializer
 
+    def get_queryset(self):
+        import_data = self.request.data
+        entity_names = [d["entity"] for d in import_data]
+        return Entity.objects.filter(name__in=entity_names, is_active=True)
+
     def post(self, request):
         import_datas = request.data
         user: User = request.user
         serializer = EntryImportSerializer(data=import_datas)
         serializer.is_valid(raise_exception=True)
-
-        job_ids = []
-        error_list = []
+        entities = self.get_queryset()
+
+        # limit import job to deny accidental frequent import for same entity
+        if request.query_params.get("force", "") not in ["true", "True"]:
+            valid_statuses = [Job.STATUS["PREPARING"], Job.STATUS["PROCESSING"], Job.STATUS["DONE"]]
+            yesterday = datetime.now() - timedelta(days=1)
+            if Job.objects.filter(
+                status__in=valid_statuses,
+                operation=JobOperation.IMPORT_ENTRY_V2.value,
+                target__in=entities,
+                created_at__gte=yesterday,
+            ).exists():
+                raise FrequentImportError("Import job for each entity can apply once a day.")
+
+        job_ids: list[int] = []
+        error_list: list[str] = []
         for import_data in import_datas:
-            entity = Entity.objects.filter(name=import_data["entity"], is_active=True).first()
+            entity: Optional[Entity] = next(
+                (e for e in entities if e.name == import_data["entity"]), None
+            )
             if not entity:
                 error_list.append("%s: Entity does not exists." % import_data["entity"])
                 continue

entry/tests/test_api_v2.py

@@ -2490,7 +2490,7 @@ def test_import_update_entry(self):
 
         # Update only some attributes
         fp = self.open_fixture_file("import_data_v2_update_some.yaml")
-        resp = self.client.post("/entry/api/v2/import/", fp.read(), "application/yaml")
+        resp = self.client.post("/entry/api/v2/import/?force=true", fp.read(), "application/yaml")
         fp.close()
 
         self.assertEqual(resp.status_code, 200)
@@ -2504,7 +2504,7 @@ def test_import_update_entry(self):
 
         # Update remove attribute value
         fp = self.open_fixture_file("import_data_v2_update_remove.yaml")
-        resp = self.client.post("/entry/api/v2/import/", fp.read(), "application/yaml")
+        resp = self.client.post("/entry/api/v2/import/?force=true", fp.read(), "application/yaml")
         fp.close()
 
         self.assertEqual(resp.status_code, 200)
@@ -2616,9 +2616,10 @@ def test_import_entry_has_referrals_with_entities(self):
 
         with self.assertLogs(logger=Logger, level=logging.WARNING) as warning_log:
             fp = self.open_fixture_file("import_data_v2.yaml")
-            resp = self.client.post("/entry/api/v2/import/", fp.read(), "application/yaml")
+            resp = self.client.post(
+                "/entry/api/v2/import/?force=true", fp.read(), "application/yaml"
+            )
             fp.close()
-            print(warning_log.output)
             self.assertEqual(
                 warning_log.output,
                 [
@@ -2844,6 +2845,25 @@ def test_import_cancel(self):
         self.assertEqual(job.text, "Now importing... (progress: [    1/    1])")
         self.assertFalse(Entry.objects.filter(name="test-entry").exists())
 
+    @patch("entry.tasks.import_entries_v2.delay", Mock(side_effect=tasks.import_entries_v2))
+    def test_import_frequent_jobs(self):
+        fp = self.open_fixture_file("import_data_v2.yaml")
+        resp = self.client.post("/entry/api/v2/import/", fp.read(), "application/yaml")
+        fp.close()
+        self.assertEqual(resp.status_code, 200)
+
+        # without force param, it should exceed the limit
+        fp = self.open_fixture_file("import_data_v2.yaml")
+        resp = self.client.post("/entry/api/v2/import/?force=false", fp.read(), "application/yaml")
+        fp.close()
+        self.assertEqual(resp.status_code, 400)
+
+        # with force param, it should ignore the limit
+        fp = self.open_fixture_file("import_data_v2.yaml")
+        resp = self.client.post("/entry/api/v2/import/?force=true", fp.read(), "application/yaml")
+        fp.close()
+        self.assertEqual(resp.status_code, 200)
+
     def test_advanced_search(self):
         entry1: Entry = self.add_entry(
             self.user,

frontend/src/components/common/ImportForm.tsx

@@ -32,6 +32,7 @@ export const ImportForm: FC<Props> = ({ handleImport, handleCancel }) => {
           await handleImport(fileReader.result);
           history.go(0);
         } catch (e) {
+          // TODO check the error code returned from the backend to get error details
           setErrorMessage("ファイルのアップロードに失敗しました。");
           enqueueSnackbar("ファイルのアップロードに失敗しました", {
             variant: "error",
@@ -42,32 +43,30 @@ export const ImportForm: FC<Props> = ({ handleImport, handleCancel }) => {
   };
 
   return (
-    <Box>
-      <Box display="flex" flexDirection="column">
-        <Input type="file" onChange={onChange} />
+    <Box display="flex" flexDirection="column">
+      <Input type="file" onChange={onChange} />
 
-        <Typography color="error" variant="caption" my="4px">
-          {errorMessage}
-        </Typography>
-        <Box display="flex" justifyContent="flex-end">
-          <Button
-            type="submit"
-            variant="contained"
-            color="secondary"
-            onClick={onClick}
-            sx={{ m: "4px" }}
-          >
-            インポート
-          </Button>
-          <Button
-            variant="contained"
-            color="info"
-            onClick={handleCancel}
-            sx={{ m: "4px" }}
-          >
-            キャンセル
-          </Button>
-        </Box>
+      <Typography color="error" variant="caption" my="4px">
+        {errorMessage}
+      </Typography>
+      <Box display="flex" justifyContent="flex-end">
+        <Button
+          type="submit"
+          variant="contained"
+          color="secondary"
+          onClick={onClick}
+          sx={{ m: "4px" }}
+        >
+          インポート
+        </Button>
+        <Button
+          variant="contained"
+          color="info"
+          onClick={handleCancel}
+          sx={{ m: "4px" }}
+        >
+          キャンセル
+        </Button>
       </Box>
     </Box>
   );

frontend/src/components/entry/EntryImportModal.tsx

@@ -1,6 +1,6 @@
-import { Box, Modal, Typography } from "@mui/material";
+import { Box, Checkbox, Modal, Typography } from "@mui/material";
 import { styled } from "@mui/material/styles";
-import React, { FC } from "react";
+import React, { FC, useState } from "react";
 
 import { ImportForm } from "components/common/ImportForm";
 import { aironeApiClientV2 } from "repository/AironeApiClientV2";
@@ -30,6 +30,8 @@ export const EntryImportModal: FC<Props> = ({
   openImportModal,
   closeImportModal,
 }) => {
+  const [forceImport, setForceImport] = useState(false);
+
   return (
     <StyledModal
       aria-labelledby="transition-modal-title"
@@ -47,12 +49,23 @@ export const EntryImportModal: FC<Props> = ({
         <Typography variant={"caption"} my="4px">
           ※CSV形式のファイルは選択できません。
         </Typography>
-        <ImportForm
-          handleImport={(data: string | ArrayBuffer) =>
-            aironeApiClientV2.importEntries(data)
-          }
-          handleCancel={closeImportModal}
-        />
+        <Box display="flex" alignItems="center">
+          <Checkbox
+            checked={forceImport}
+            onChange={(event) => setForceImport(event.target.checked)}
+          />
+          <Typography variant={"body2"}>
+            強制的にインポートする(短期間にインポートを繰り返したい場合に使用してください)
+          </Typography>
+        </Box>
+        <Box my="8px">
+          <ImportForm
+            handleImport={(data: string | ArrayBuffer) =>
+              aironeApiClientV2.importEntries(data, forceImport)
+            }
+            handleCancel={closeImportModal}
+          />
+        </Box>
       </Paper>
     </StyledModal>
   );

frontend/src/repository/AironeApiClientV2.ts

@@ -862,14 +862,22 @@ class AironeApiClientV2 {
     );
   }
 
-  async importEntries(data: string | ArrayBuffer): Promise<void> {
-    return await this.entry.entryApiV2ImportCreate({
-      headers: {
-        "Content-Type": "application/yaml",
-        "X-CSRFToken": getCsrfToken(),
+  async importEntries(
+    data: string | ArrayBuffer,
+    force: boolean
+  ): Promise<void> {
+    return await this.entry.entryApiV2ImportCreate(
+      {
+        force,
       },
-      body: new Blob([data]),
-    });
+      {
+        headers: {
+          "Content-Type": "application/yaml",
+          "X-CSRFToken": getCsrfToken(),
+        },
+        body: new Blob([data]),
+      }
+    );
   }
 
   async resetPassword(username: string): Promise<void> {

frontend/src/services/AironeAPIErrorUtil.ts

@@ -24,6 +24,7 @@ const aironeAPIErrors: Record<string, string> = {
   "AE-122000": "入力データが大きすぎます",
   "AE-210000": "操作に必要な権限が不足しています",
   "AE-220000": "入力データが既存のデータと重複しています",
+  "AE-260000": "短期間に同じターゲットに対してインポートが発生しました",
 };
 
 const extractErrorDetail = (errorDetail: ErrorDetail): string =>

package.json

@@ -77,7 +77,7 @@
     "zod": "^3.20.6"
   },
   "dependencies": {
-    "@dmm-com/airone-apiclient-typescript-fetch": "^0.0.6",
+    "@dmm-com/airone-apiclient-typescript-fetch": "^0.0.7",
     "@emotion/react": "^11.10.5",
     "@emotion/styled": "^11.10.5",
     "@jest/globals": "^27.0.6",