Merge pull request #269 from akawashiro/check-DB-update

Check db update

Dockerfile

@@ -13,6 +13,7 @@ RUN npm run check:prettier
 RUN npm run build
 RUN npm run test
 RUN npm run scan_test_pdfs
+RUN ./scripts/check_update_all_generated_DBs.sh
 RUN [ -f "edit_and_run.sh" ] && cat edit_and_run.sh && exit 1 || echo "Build DB succeeded"
 
 WORKDIR /jendeley/jendeley-frontend

jendeley-backend/.npmignore

@@ -18,3 +18,4 @@ benchmark.svg
 perf.data*
 processed-isolate-*.txt
 processed-isolate-*.txt
+generated_DBs

jendeley-backend/scripts/check_update_all_generated_DBs.sh

@@ -0,0 +1,17 @@
+#! /bin/bash
+set -eux
+
+rootdir=$(git rev-parse --show-toplevel)
+workdir=$(mktemp -d -t jendeley_check_update_all_generated_DBs_XXXXX)
+cd ${rootdir}/jendeley-backend
+
+ok=0
+for db in $(find ${rootdir}/jendeley-backend/generated_DBs -name "*.json" | sort); do
+    echo Checking $db
+    node --require source-map-support/register dist/index.js update_db --db1 ${db} --db2 ${workdir}/updated_$(basename $db)
+    if [[ ! -f ${workdir}/updated_$(basename $db) ]]; then
+        ok=1
+    fi
+done
+echo Check ${workdir} for updated DBs
+exit $ok

jendeley-backend/scripts/generated_DBs_for_all_versions.sh

@@ -0,0 +1,79 @@
+#!/bin/bash -eux
+
+rootdir=$(git rev-parse --show-toplevel)
+versions=(
+  # TODO: Do not skip pre 1.0.0 versions
+  # "0.0.1"
+  # "0.0.2"
+  # "0.0.3"
+  # "0.0.4"
+  # "0.0.5"
+  # "0.0.6"
+  # "0.0.7"
+  # "0.0.8"
+  # "0.0.9"
+  # "0.0.10"
+  # "0.0.11"
+  # "0.0.12"
+  # "0.0.13"
+  # "0.0.14"
+  # "0.0.15"
+  # "0.0.16"
+  # "0.0.17"
+  # "0.0.18"
+  # "0.0.19"
+  # "0.0.20"
+  # "0.0.21"
+  # "0.0.22"
+  # "0.0.23"
+  # "0.0.24"
+  # "0.0.25"
+  # "0.0.26"
+  # "0.0.27"
+  "1.0.0"
+  "1.0.1"
+  "1.0.2"
+  "1.0.3"
+  "1.0.4"
+  "1.0.5"
+  "1.0.6"
+  "1.0.7"
+  "1.0.8"
+  "1.0.9"
+  "1.0.10"
+  "1.1.0"
+  "1.1.1"
+  "1.2.0"
+  "1.3.0"
+  "2.0.2"
+  "2.0.4"
+  "2.0.5"
+  "2.0.6"
+  "2.0.7"
+  "2.0.9"
+  "2.0.10"
+  "2.0.12"
+  "2.1.1"
+  "2.1.2"
+  "2.1.3"
+  "2.2.0"
+)
+
+mkdir -p ${rootdir}/jendeley-backend/generated_DBs
+process_list=()
+for version in "${versions[@]}"; do
+    echo "Generating DB for version $version"
+    workdir=$(mktemp -d -t gen_DBs-for-all-versions-${version}-XXXXXXXXXX)
+    cp -r ${rootdir}/jendeley-backend/test_pdfs ${workdir}/test_pdfs
+    rm -rf ${workdir}/test_pdfs/jendeley_db.json
+    echo "#! /bin/bash -eux" > ${workdir}/gen_DB.sh
+    echo "npm install -g @a_kawashiro/jendeley@${version}" >> ${workdir}/gen_DB.sh
+    echo "jendeley scan --papers_dir /workdir/test_pdfs --book_dirs /workdir/test_pdfs/dummyTapl" >> ${workdir}/gen_DB.sh
+    docker run --volume /${workdir}:/workdir --rm node:21 bash /workdir/gen_DB.sh && cp ${workdir}/test_pdfs/jendeley_db.json ${rootdir}/jendeley-backend/generated_DBs/jendeley_db_${version}.json &
+    process_id=$!
+    process_list+=(${process_id})
+done
+
+for p in "${process_list[@]}"; do
+    wait ${p}
+done

jendeley-backend/src/update_db.ts

@@ -7,7 +7,12 @@ import fetch from "node-fetch";
 import { Request as NFRequest } from "node-fetch";
 import { Either, genLeft, genRight } from "./either";
 import { validateJsonDB } from "./validate_db";
-import { ENTRY_AUTHORS, ENTRY_TEXT, ENTRY_TITLE } from "./constants";
+import {
+  ENTRY_AUTHORS,
+  ENTRY_TEXT,
+  DB_META_KEY,
+  JENDELEY_VERSION,
+} from "./constants";
 
 async function getTextsFromPDF(
   pdfFullpath: string,
@@ -79,6 +84,7 @@ async function update_db(dbPathVer1: string[], dbPathVer2: string[]) {
       jsonDB[id][ENTRY_AUTHORS] = [];
     }
   }
+  jsonDB[DB_META_KEY]["version"] = JENDELEY_VERSION;
 
   if (fs.existsSync(concatDirs(dbPathVer2))) {
     logger.fatal(dbPathVer2 + " already exists.");

run-CI.sh

@@ -1,5 +1,5 @@
 #! /bin/bash -eux
 
-docker build . -f ./Dockerfile
-docker build . -f ./Releasable.Dockerfile
-docker build . -f ./shellcheck.Dockerfile
+docker build . -f ./Dockerfile --network=host
+docker build . -f ./Releasable.Dockerfile --network=host
+docker build . -f ./shellcheck.Dockerfile --network=host