Fix AmazonReviews and fixed mkdocs version (#2725)

* Fix AmazonReviews The amazon reviews has a dependency on another s3 bucket that was removed, breaking the CI. This fixes it by pulling from a local cache. * Try to fix mkdocs version
deepjavalibrary · Jul 28, 2023 · 03e2865 · 03e2865
1 parent ae84c25
commit 03e2865
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 21 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Install CN fonts
         run: sudo apt-get update && sudo apt-get install fonts-arphic-uming
       - name: install Python Dependencies
-        run: pip3 install nbconvert mkdocs mkdocs-exclude mknotebooks mkdocs-material jupyter Pygments Markdown
+        run: pip3 install nbconvert mkdocs==1.4.3 mkdocs-exclude mknotebooks mkdocs-material jupyter Pygments Markdown
       - name: Install IJava kernel
         run: |
           git clone https://github.com/frankfliu/IJava.git

diff --git a/...et/src/test/resources/mlrepo/dataset/nlp/ai/djl/basicdataset/amazon_reviews/metadata.json b/...et/src/test/resources/mlrepo/dataset/nlp/ai/djl/basicdataset/amazon_reviews/metadata.json
@@ -17,24 +17,9 @@
       },
       "files": {
         "amazon_reviews": {
-          "uri": "https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Digital_Software_v1_00.tsv.gz",
-          "sha1Hash": "b8390100b92579ed814eede4112514417e339902",
-          "size": 18997559
-        }
-      }
-    },
-    {
-      "version": "1.0",
-      "snapshot": false,
-      "name": "amazon_reviews_us_Software",
-      "properties": {
-        "dataset": "us_Software"
-      },
-      "files": {
-        "amazon_reviews": {
-          "uri": "https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Software_v1_00.tsv.gz",
-          "sha1Hash": "e48346dd356698ce680e385e3ecf07501de695b8",
-          "size": 94010685
+          "uri": "1.0/amazon_reviews_us_Digital_Software_v1_00.tsv.gz",
+          "sha1Hash": "098fb62c5731161dd1e10298a5d11636253609a1",
+          "size": 18997604
         }
       }
     }

diff --git a/...les/src/main/java/ai/djl/examples/training/transferlearning/TrainAmazonReviewRanking.java b/...les/src/main/java/ai/djl/examples/training/transferlearning/TrainAmazonReviewRanking.java
@@ -125,7 +125,7 @@ public static TrainingResult runExample(String[] args)
     private static CsvDataset getDataset(
             Arguments arguments, BertFullTokenizer tokenizer, int maxLength) {
         String amazonReview =
-                "https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Digital_Software_v1_00.tsv.gz";
+                "https://mlrepo.djl.ai/dataset/nlp/ai/djl/basicdataset/amazon_reviews/1.0/amazon_reviews_us_Digital_Software_v1_00.tsv.gz";
         float paddingToken = tokenizer.getVocabulary().getIndex("[PAD]");
         return CsvDataset.builder()
                 .optCsvUrl(amazonReview)

diff --git a/jupyter/rank_classification_using_BERT_on_Amazon_Review.ipynb b/jupyter/rank_classification_using_BERT_on_Amazon_Review.ipynb
@@ -159,7 +159,7 @@
    "source": [
     "CsvDataset getDataset(int batchSize, BertFullTokenizer tokenizer, int maxLength, int limit) {\n",
     "    String amazonReview =\n",
-    "            \"https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Digital_Software_v1_00.tsv.gz\";\n",
+    "            \"https://mlrepo.djl.ai/dataset/nlp/ai/djl/basicdataset/amazon_reviews/1.0/amazon_reviews_us_Digital_Software_v1_00.tsv.gz\";\n",
     "    float paddingToken = tokenizer.getVocabulary().getIndex(\"[PAD]\");\n",
     "    return CsvDataset.builder()\n",
     "            .optCsvUrl(amazonReview) // load from Url\n",