Github actions for testing and coverage (#124)

.github/workflows/coverage.yml

@@ -0,0 +1,33 @@
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate coverage report
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+      - run: npm test
+      - run: docker exec node-etl-runner-1 bash -c "npx lcov-badge2 .tap/report/lcov.info -o.tap/report/lcov-report/badge.svg"
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: '.tap/report/lcov-report'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/publish.yml

@@ -0,0 +1,19 @@
+name: Publish to NPM
+on:
+  release:
+    types: [created]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Publish package on NPM 📦
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Unit tests
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+      - run: npm test

.gitignore

@@ -3,3 +3,4 @@
 /.nyc_output/
 /coverage/
 /.tap/
+package-lock.json

README.md

@@ -1,16 +1,12 @@
 [![NPM Version][npm-image]][npm-url]
 [![NPM Downloads][downloads-image]][downloads-url]
-[![Test Coverage][circle-image]][circle-url]
-[![Coverage][coverage-image]][coverage-url]
+[![code coverage](https://zjonsson.github.io/node-etl/badge.svg)](https://zjonsson.github.io/node-etl/) 
+
 
 [npm-image]: https://img.shields.io/npm/v/etl.svg
 [npm-url]: https://npmjs.org/package/etl
-[circle-image]: https://circleci.com/gh/ZJONSSON/node-etl.png?style=shield
-[circle-url]: https://circleci.com/gh/ZJONSSON/node-etl/tree/master
 [downloads-image]: https://img.shields.io/npm/dm/etl.svg
 [downloads-url]: https://npmjs.org/package/etl
-[coverage-image]: https://3tjjj5abqi.execute-api.us-east-1.amazonaws.com/prod/node-etl/badge
-[coverage-url]: https://3tjjj5abqi.execute-api.us-east-1.amazonaws.com/prod/node-etl/url
 
 ETL is a collection of stream based components that can be piped together to form a complete ETL pipeline with buffering, bulk-inserts and concurrent database streams. See the `test` directory for live examples.
 

docker-compose.yml

@@ -29,40 +29,30 @@ services:
       - "9229:9229"
     expose:
       - 9229
-    networks:
-      etl-network:
-        ipv4_address: 173.23.0.100
 
   mongodb:
     image: 'mongo:7'
-    networks:
-      - etl-network
 
   mysql:
-    image: mysql:8
-    command: --default-authentication-plugin=mysql_native_password
+    image: mysql:8.0
     restart: always
     environment:
-      MYSQL_ROOT_PASSWORD: example
-    networks:
-      - etl-network
+      - MYSQL_ROOT_PASSWORD=example
+
+    command: ["mysqld", "--default-authentication-plugin=mysql_native_password", "--sql_mode="]
+    volumes:
+      - mysql-data:/var/lib/mysql
 
   postgres:
     image: postgres:16
     restart: always
     environment:
       POSTGRES_PASSWORD: example
-    networks:
-      - etl-network
 
   elasticsearch:
     image: elasticsearch:8.12.0
     environment: ['http.host=0.0.0.0', 'transport.host=127.0.0.1','xpack.security.enabled=false', 'xpack.security.enrollment.enabled=false']
-    networks:
-      - etl-network
 
-networks:
-  etl-network:
-    ipam:
-     config:
-       - subnet: 173.23.0.0/16
+
+volumes:
+  mysql-data:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "etl",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Collection of stream-based components that form an ETL pipeline",
   "main": "index.js",
   "author": "Ziggy Jonsson (http://github.com/zjonsson/)",
@@ -9,8 +9,8 @@
     "url": "http://github.com/ZJONSSON/node-etl"
   },
   "scripts": {
-    "test": "docker-compose -p etl up -d --no-recreate;docker exec -it etl-runner-1 bash ./test.sh",
-    "docker": "docker-compose -p etl up -d --no-recreate;docker exec -it etl-runner-1 bash"
+    "test": "docker compose up -d --no-recreate --quiet-pull;docker exec node-etl-runner-1 bash ./test.sh",
+    "docker": "docker compose up -d --no-recreate --quiet-pull;docker exec node-etl-runner-1 bash"
   },
   "license": "MIT",
   "dependencies": {
@@ -26,7 +26,7 @@
     "mysql": "^2.18.1",
     "pg": "^8.11.3",
     "pg-query-stream": "~1.0.0",
-    "tap": "^18.6.1",
+    "tap": "^19.0.2",
     "url-js": "^0.2.5"
   }
 }

test.sh

@@ -4,4 +4,4 @@ wait-for-it mongodb:27017
 wait-for-it mysql:3306
 wait-for-it postgres:5432
 # cluster test needs to be run from master thread
-node test/cluster-test.js | tap - test --exclude=*cluster* jobs=5 --passes
+node test/cluster-test.js | tap test --exclude=*cluster* jobs=5  --coverage-report=lcov --allow-incomplete-coverage

test/lib/dataStream.js

@@ -2,10 +2,13 @@ const PassThrough = require('stream').PassThrough;
 
 module.exports = function dataStream(data) {
   const s = PassThrough({objectMode:true});
-  data.forEach((d,i) => setTimeout( () => {
-    s.write(d);
-    if (i == data.length-1)
-      s.end();
-  },i));
+  // Simulate async data stream
+  Promise.resolve().then(async() => {
+    for (let i = 0; i < data.length; i++) {
+      s.write(data[i]);
+      await new Promise(resolve => setTimeout(resolve,10))
+    }
+    s.end();
+  });
   return s;
 };