Merge pull request #129 from botpress/cloud

Cloud

.dockerignore

@@ -3,4 +3,10 @@ node_modules
 Dockerfile
 .dockerignore
 .vscode
-.github
+.github
+
+# Other
+*.config.json
+*.ignore.me
+ignore.me.*
+*.ignore.me.*

.eslintrc.js

@@ -11,10 +11,10 @@ module.exports = {
     tsconfigRootDir: __dirname,
     sourceType: 'module'
   },
-  ignorePatterns: ['**/index.d.ts', '**/global.d.ts', '**/*.test.ts', '*.js', '**/dist/**'],
+  ignorePatterns: ['**/index.d.ts', '**/global.d.ts', '*.js', '**/dist/**'],
   plugins: ['eslint-plugin-import', 'eslint-plugin-jsdoc', '@typescript-eslint'],
   rules: {
-    '@typescript-eslint/consistent-type-definitions': 'error',
+    '@typescript-eslint/consistent-type-definitions': ['warn', 'type'],
     '@typescript-eslint/member-delimiter-style': [
       'error',
       {
@@ -100,6 +100,19 @@ module.exports = {
     'no-trailing-spaces': 'error',
     'no-var': 'error',
     'object-shorthand': 'error',
-    'prefer-const': 'warn'
+    'prefer-const': 'warn',
+    '@typescript-eslint/explicit-member-accessibility': [
+      'warn',
+      {
+        accessibility: 'explicit',
+        overrides: {
+          accessors: 'explicit',
+          constructors: 'no-public',
+          methods: 'explicit',
+          properties: 'off',
+          parameterProperties: 'explicit'
+        }
+      }
+    ]
   }
 }

.github/actions/setup-e2e/action.yml

@@ -0,0 +1,51 @@
+name: Setup E2E
+description: Setup E2E NLU Usage
+
+inputs:
+  lang_server_dim:
+    description: 'Dimension in which to run the language server'
+    required: false
+    type: 'string'
+    default: '25'
+
+  lang_server_lang:
+    description: 'Language to download in the language server'
+    required: false
+    type: 'string'
+    default: 'en'
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-node@v2
+      with:
+        node-version-file: '.nvmrc'
+        cache: 'yarn'
+    - name: Fetch Node Packages
+      shell: bash
+      run: yarn
+    - name: Build
+      shell: bash
+      run: yarn build
+    - name: package
+      shell: bash
+      run: yarn package --linux
+    - name: Rename binary
+      id: rename_binary
+      shell: bash
+      run: |
+        bin_original_name=$(node -e "console.log(require('./scripts/utils/binary').getFileName())")
+        echo "Moving ./dist/$bin_original_name to ./nlu ..."
+        mv ./dist/$bin_original_name ./nlu
+    - name: Download language models
+      shell: bash
+      run: ./nlu lang download --lang ${{ inputs.lang_server_lang }} --dim ${{ inputs.lang_server_dim }}
+    - name: Start Language Server
+      shell: bash
+      run: |
+        ./nlu lang --dim ${{ inputs.lang_server_dim }} &
+        echo "Lang Server started on pid $!"
+    - name: Sleep
+      uses: jakejarvis/wait-action@master
+      with:
+        time: '15s'

.github/workflows/bench.yml

@@ -0,0 +1,34 @@
+name: Bench
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+jobs:
+  benchmark:
+    name: Run benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '16.13.0'
+      - name: Fetch Node Packages
+        run: |
+          yarn --verbose
+      - name: Build
+        run: |
+          yarn build
+      - name: Download language models
+        run: |
+          yarn start lang download --lang fr --dim 100
+          yarn start lang download --lang en --dim 100
+      - name: Run Regression Test
+        run: |
+          yarn start lang --dim 100 &
+          sleep 15s && yarn start nlu --doc false --log-level "critical" --ducklingEnabled false --languageURL http://localhost:3100 &
+          sleep 25s && yarn bench --skip="clinc150"

.github/workflows/codestyle.yml

@@ -1,5 +1,12 @@
 name: Codestyle
-on: [pull_request]
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
 jobs:
   run_prettier:
     name: Run Prettier on codebase
@@ -9,7 +16,7 @@ jobs:
         uses: actions/checkout@master
       - uses: actions/setup-node@v1
         with:
-          node-version: '12.13.0'
+          node-version: '16.13.0'
       - name: Install Dependencies
         run: |
           yarn
@@ -24,7 +31,7 @@ jobs:
         uses: actions/checkout@master
       - uses: actions/setup-node@v1
         with:
-          node-version: '12.13.0'
+          node-version: '16.13.0'
       - name: Install Dependencies
         run: |
           yarn

.github/workflows/e2e.yml

@@ -0,0 +1,150 @@
+name: E2E
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+jobs:
+  fs:
+    name: file system
+    runs-on: ubuntu-latest
+    services:
+      duckling:
+        image: rasa/duckling
+        ports:
+          - 8000:8000
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+      - name: Setup E2E
+        uses: ./.github/actions/setup-e2e
+      - name: Start NLU Server
+        run: |
+          ./nlu \
+            --log-level "critical" \
+            --ducklingURL http://localhost:8000 \
+            --languageURL http://localhost:3100 \
+            --modelTransferEnabled \
+            --port 3200  &
+          nlu_pid=$!
+          echo "NLU Server started on pid $nlu_pid"
+      - name: Sleep
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '15s'
+      - name: Run Tests
+        run: |
+          yarn e2e --nlu-endpoint http://localhost:3200
+  db:
+    name: database
+    runs-on: ubuntu-latest
+    services:
+      duckling:
+        image: rasa/duckling
+        ports:
+          - 8000:8000
+      postgres:
+        # Docker Hub image
+        image: postgres
+        env:
+          POSTGRES_DB: botpress-nlu-1
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PORT: 5432
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+      - name: Setup E2E
+        uses: ./.github/actions/setup-e2e
+      - name: Start NLU Server
+        run: |
+          ./nlu \
+            --log-level "critical" \
+            --ducklingURL http://localhost:8000 \
+            --languageURL http://localhost:3100 \
+            --modelTransferEnabled \
+            --port 3201 \
+            --dbURL postgres://postgres:postgres@localhost:5432/botpress-nlu-1 & \
+          nlu_pid=$!
+          echo "NLU Server started on pid $nlu_pid"
+      - name: Sleep
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '15s'
+      - name: Run Tests
+        run: |
+          yarn e2e --nlu-endpoint http://localhost:3201
+  cluster:
+    name: cluster
+    runs-on: ubuntu-latest
+    services:
+      duckling:
+        image: rasa/duckling
+        ports:
+          - 8000:8000
+      postgres:
+        # Docker Hub image
+        image: postgres
+        env:
+          POSTGRES_DB: botpress-nlu-2
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PORT: 5432
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+      - name: Setup E2E
+        uses: ./.github/actions/setup-e2e
+      - name: Start First NLU Server on port 3202
+        run: |
+          ./nlu \
+            --maxTraining 0 \
+            --maxLinting 0 \
+            --log-level "critical" \
+            --ducklingURL http://localhost:8000 \
+            --languageURL http://localhost:3100 \
+            --modelTransferEnabled \
+            --port 3202 \
+            --dbURL postgres://postgres:postgres@localhost:5432/botpress-nlu-2 & \
+          nlu_pid1=$!
+          echo "NLU Server started on pid $nlu_pid1"
+      - name: Sleep
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '5s'
+      - name: Start Second NLU Server on port 3203
+        run: |
+          ./nlu \
+            --log-level "critical" \
+            --ducklingURL http://localhost:8000 \
+            --languageURL http://localhost:3100 \
+            --port 3203 \
+            --dbURL postgres://postgres:postgres@localhost:5432/botpress-nlu-2 & \
+          nlu_pid2=$!
+          echo "NLU Server started on pid $nlu_pid2"
+      - name: Sleep
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '15s'
+      - name: Run Tests
+        run: |
+          yarn e2e --nlu-endpoint http://localhost:3202

.github/workflows/unit-tests.yml

@@ -1,5 +1,12 @@
 name: Tests
-on: [pull_request]
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
 jobs:
   unit:
     name: Run unit tests