Show warning when local storages are not empty (#10)

* show warning in case of local storages are not empty

* Revert "show warning in case of local storages are not empty"

This reverts commit 13b3103

* move warning for non-empty local storages to _ensureDir() functions

* continue when dirent is not a directory, move nested code to higher level, add test checking warnings

* bump package version to 1.0.3

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@apify/storage-local",
-    "version": "1.0.2",
+    "version": "1.0.3",
     "description": "Drop in replacement of Apify API with a local SQLite and filesystem implementation. Not all API features are supported.",
     "engines": {
         "node": ">=10.17.0"
@@ -37,6 +37,7 @@
         "lint:fix": "eslint ./src ./test --ext .js --fix"
     },
     "dependencies": {
+        "apify-shared": "^0.5.7",
         "better-sqlite3-with-prebuilds": "^7.1.1",
         "content-type": "^1.0.4",
         "mime-types": "^2.1.27",

src/consts.js

@@ -10,6 +10,16 @@ exports.REQUEST_ID_LENGTH = 15;
  */
 exports.TIMESTAMP_SQL = "STRFTIME('%Y-%m-%dT%H:%M:%fZ', 'NOW')";
 
+/**
+ * Types of all emulated storages (currently used for warning messages only).
+ * @type {{REQUEST_QUEUE: string, KEY_VALUE_STORE: string, string, DATASET: string}}
+ */
+exports.STORAGE_TYPES = {
+    REQUEST_QUEUE: 'Request queue',
+    KEY_VALUE_STORE: 'Key-value store',
+    DATASET: 'Dataset',
+};
+
 /**
  * Names of all emulated storages.
  * @type {{REQUEST_QUEUES: string, KEY_VALUE_STORES: string, REQUEST_QUEUE_REQUESTS: string, DATASETS: string}}

src/index.js

@@ -1,7 +1,9 @@
 const fs = require('fs-extra');
 const ow = require('ow');
 const path = require('path');
-const { STORAGE_NAMES } = require('./consts');
+const log = require('apify-shared/log');
+const { KEY_VALUE_STORE_KEYS } = require('apify-shared/consts');
+const { STORAGE_NAMES, STORAGE_TYPES } = require('./consts');
 const DatabaseConnectionCache = require('./database_connection_cache');
 const DatasetClient = require('./resource_clients/dataset');
 const DatasetCollectionClient = require('./resource_clients/dataset_collection');
@@ -144,6 +146,7 @@ class ApifyStorageLocal {
     _ensureDatasetDir() {
         if (!this.isDatasetDirInitialized) {
             fs.ensureDirSync(this.datasetDir);
+            this._checkIfStorageIsEmpty(STORAGE_TYPES.DATASET, this.datasetDir);
             this.isDatasetDirInitialized = true;
         }
     }
@@ -154,6 +157,7 @@ class ApifyStorageLocal {
     _ensureKeyValueStoreDir() {
         if (!this.isKeyValueStoreDirInitialized) {
             fs.ensureDirSync(this.keyValueStoreDir);
+            this._checkIfStorageIsEmpty(STORAGE_TYPES.KEY_VALUE_STORE, this.keyValueStoreDir);
             this.isKeyValueStoreDirInitialized = true;
         }
     }
@@ -164,9 +168,37 @@ class ApifyStorageLocal {
     _ensureRequestQueueDir() {
         if (!this.isRequestQueueDirInitialized) {
             fs.ensureDirSync(this.requestQueueDir);
+            this._checkIfStorageIsEmpty(STORAGE_TYPES.REQUEST_QUEUE, this.requestQueueDir);
             this.isRequestQueueDirInitialized = true;
         }
     }
+
+    _checkIfStorageIsEmpty(storageType, storageDir) {
+        const dirsWithPreviousState = [];
+
+        const dirents = fs.readdirSync(storageDir, { withFileTypes: true });
+        for (const dirent of dirents) {
+            if (!dirent.isDirectory()) continue; // eslint-disable-line
+
+            const innerStorageDir = path.resolve(storageDir, dirent.name);
+            let innerDirents = fs.readdirSync(innerStorageDir).filter((fileName) => !(/(^|\/)\.[^/.]/g).test(fileName));
+            if (storageType === STORAGE_TYPES.KEY_VALUE_STORE) {
+                innerDirents = innerDirents.filter((fileName) => !RegExp(KEY_VALUE_STORE_KEYS.INPUT).test(fileName));
+            }
+
+            if (innerDirents.length) {
+                dirsWithPreviousState.push(innerStorageDir);
+            }
+        }
+
+        const dirsNo = dirsWithPreviousState.length;
+        if (dirsNo) {
+            log.warning(`The following ${storageType} director${dirsNo === 1 ? 'y' : 'ies'} contain${dirsNo === 1 ? 's' : ''} a previous state:`
+                + `\n      ${dirsWithPreviousState.join('\n      ')}`
+                + '\n      If you did not intend to persist the state - '
+                + `please clear the respective director${dirsNo === 1 ? 'y' : 'ies'} and re-start the actor.`);
+        }
+    }
 }
 
 module.exports = ApifyStorageLocal;

test/index.test.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const path = require('path');
+const log = require('apify-shared/log');
 const ApifyStorageLocal = require('../src/index');
 const { STORAGE_NAMES } = require('../src/consts');
 const { prepareTestDir, removeTestDir } = require('./_tools');
@@ -26,7 +27,7 @@ test('does not create folders immediately', () => {
 });
 
 test('creates folders lazily', () => {
-    const storageLocal = new ApifyStorageLocal({ // eslint-disable-line
+    const storageLocal = new ApifyStorageLocal({
         storageDir: STORAGE_DIR,
     });
     const requestQueueDir = path.join(STORAGE_DIR, STORAGE_NAMES.REQUEST_QUEUES);
@@ -39,3 +40,38 @@ test('creates folders lazily', () => {
         expect(fs.statSync(dir).isDirectory()).toBe(true);
     }
 });
+
+test('warning is shown when storage is non-empty', () => {
+    const storageLocal = new ApifyStorageLocal({
+        storageDir: STORAGE_DIR,
+    });
+
+    const requestQueueDir = path.join(STORAGE_DIR, STORAGE_NAMES.REQUEST_QUEUES);
+    const keyValueStoreDir = path.join(STORAGE_DIR, STORAGE_NAMES.KEY_VALUE_STORES);
+    const datasetDir = path.join(STORAGE_DIR, STORAGE_NAMES.DATASETS);
+
+    const fileData = JSON.stringify({ foo: 'bar' });
+    const innerDirName = 'default';
+
+    const innerRequestQueueDir = path.join(requestQueueDir, innerDirName);
+    fs.ensureDirSync(innerRequestQueueDir);
+    fs.writeFileSync(path.join(innerRequestQueueDir, '000000001.json'), fileData);
+
+    const innerKeyValueStoreDir = path.join(keyValueStoreDir, innerDirName);
+    fs.ensureDirSync(innerKeyValueStoreDir);
+    fs.writeFileSync(path.join(innerKeyValueStoreDir, 'INPUT.json'), fileData);
+
+    const innerDatasetDir = path.join(datasetDir, innerDirName);
+    fs.ensureDirSync(innerDatasetDir);
+    fs.writeFileSync(path.join(innerDatasetDir, '000000001.json'), fileData);
+
+    const warnings = jest.spyOn(log, 'warning');
+
+    storageLocal.keyValueStores();
+    storageLocal.requestQueues();
+    storageLocal.datasets();
+
+    // warning is expected to be shown 2 times only (for Dataset and Request queue)
+    // as it should not be shown when INPUT.json in the only file in Key-value store
+    expect(warnings).toHaveBeenCalledTimes(2);
+});