From 6a0a7db71ae525ca9f0992e527500cb1dde923d5 Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Mon, 27 May 2024 20:07:32 +0000 Subject: [PATCH] Github actions for testing and coverage (#124) --- .github/workflows/coverage.yml | 33 +++++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 19 +++++++++++++++++++ .github/workflows/test.yml | 20 ++++++++++++++++++++ .gitignore | 1 + README.md | 8 ++------ circle.yml | 11 ----------- docker-compose.yml | 28 +++++++++------------------- package.json | 8 ++++---- test.sh | 2 +- test/lib/dataStream.js | 13 ++++++++----- 10 files changed, 97 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml delete mode 100644 circle.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..bf309fb --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,33 @@ +name: Node.js CI + +on: + push: + branches: [ master ] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +jobs: + coverage: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Generate coverage report + uses: actions/setup-node@v3 + with: + node-version: 18.x + - run: npm test + - run: docker exec node-etl-runner-1 bash -c "npx lcov-badge2 .tap/report/lcov.info -o.tap/report/lcov-report/badge.svg" + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: '.tap/report/lcov-report' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..000c866 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,19 @@ +name: Publish to NPM +on: + release: + types: [created] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: '18.x' + registry-url: 'https://registry.npmjs.org' + - name: Publish package on NPM 📦 + run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c4e3a0f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: Node.js CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Unit tests + uses: actions/setup-node@v3 + with: + node-version: 18.x + - run: npm test diff --git a/.gitignore b/.gitignore index 79d86ad..7b2e134 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /.nyc_output/ /coverage/ /.tap/ +package-lock.json \ No newline at end of file diff --git a/README.md b/README.md index 7e3f819..e689a7f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,12 @@ [![NPM Version][npm-image]][npm-url] [![NPM Downloads][downloads-image]][downloads-url] -[![Test Coverage][circle-image]][circle-url] -[![Coverage][coverage-image]][coverage-url] +[![code coverage](https://zjonsson.github.io/node-etl/badge.svg)](https://zjonsson.github.io/node-etl/) + [npm-image]: https://img.shields.io/npm/v/etl.svg [npm-url]: https://npmjs.org/package/etl -[circle-image]: https://circleci.com/gh/ZJONSSON/node-etl.png?style=shield -[circle-url]: https://circleci.com/gh/ZJONSSON/node-etl/tree/master [downloads-image]: https://img.shields.io/npm/dm/etl.svg [downloads-url]: https://npmjs.org/package/etl -[coverage-image]: https://3tjjj5abqi.execute-api.us-east-1.amazonaws.com/prod/node-etl/badge -[coverage-url]: https://3tjjj5abqi.execute-api.us-east-1.amazonaws.com/prod/node-etl/url ETL is a collection of stream based components that can be piped together to form a complete ETL pipeline with buffering, bulk-inserts and concurrent database streams. See the `test` directory for live examples. diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 9c64160..0000000 --- a/circle.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 # CircleCI version -jobs: - build: - machine: true # Use a Linux VM instead of docker environment - working_directory: ~/node-etl # Default working directory, where your project will be cloned - steps: - - checkout - - run: docker-compose -p etl up -d - - run: npm test - - store_artifacts: - path: coverage/lcov-report \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ff28c28..32897ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,40 +29,30 @@ services: - "9229:9229" expose: - 9229 - networks: - etl-network: - ipv4_address: 173.23.0.100 mongodb: image: 'mongo:7' - networks: - - etl-network mysql: - image: mysql:8 - command: --default-authentication-plugin=mysql_native_password + image: mysql:8.0 restart: always environment: - MYSQL_ROOT_PASSWORD: example - networks: - - etl-network + - MYSQL_ROOT_PASSWORD=example + + command: ["mysqld", "--default-authentication-plugin=mysql_native_password", "--sql_mode="] + volumes: + - mysql-data:/var/lib/mysql postgres: image: postgres:16 restart: always environment: POSTGRES_PASSWORD: example - networks: - - etl-network elasticsearch: image: elasticsearch:8.12.0 environment: ['http.host=0.0.0.0', 'transport.host=127.0.0.1','xpack.security.enabled=false', 'xpack.security.enrollment.enabled=false'] - networks: - - etl-network -networks: - etl-network: - ipam: - config: - - subnet: 173.23.0.0/16 \ No newline at end of file + +volumes: + mysql-data: \ No newline at end of file diff --git a/package.json b/package.json index d299b89..915cfe5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "etl", - "version": "0.7.1", + "version": "0.8.0", "description": "Collection of stream-based components that form an ETL pipeline", "main": "index.js", "author": "Ziggy Jonsson (http://github.com/zjonsson/)", @@ -9,8 +9,8 @@ "url": "http://github.com/ZJONSSON/node-etl" }, "scripts": { - "test": "docker-compose -p etl up -d --no-recreate;docker exec -it etl-runner-1 bash ./test.sh", - "docker": "docker-compose -p etl up -d --no-recreate;docker exec -it etl-runner-1 bash" + "test": "docker compose up -d --no-recreate --quiet-pull;docker exec node-etl-runner-1 bash ./test.sh", + "docker": "docker compose up -d --no-recreate --quiet-pull;docker exec node-etl-runner-1 bash" }, "license": "MIT", "dependencies": { @@ -26,7 +26,7 @@ "mysql": "^2.18.1", "pg": "^8.11.3", "pg-query-stream": "~1.0.0", - "tap": "^18.6.1", + "tap": "^19.0.2", "url-js": "^0.2.5" } } diff --git a/test.sh b/test.sh index efde105..7f92206 100644 --- a/test.sh +++ b/test.sh @@ -4,4 +4,4 @@ wait-for-it mongodb:27017 wait-for-it mysql:3306 wait-for-it postgres:5432 # cluster test needs to be run from master thread -node test/cluster-test.js | tap - test --exclude=*cluster* jobs=5 --passes \ No newline at end of file +node test/cluster-test.js | tap test --exclude=*cluster* jobs=5 --coverage-report=lcov --allow-incomplete-coverage \ No newline at end of file diff --git a/test/lib/dataStream.js b/test/lib/dataStream.js index c5fbcda..e4ec0de 100644 --- a/test/lib/dataStream.js +++ b/test/lib/dataStream.js @@ -2,10 +2,13 @@ const PassThrough = require('stream').PassThrough; module.exports = function dataStream(data) { const s = PassThrough({objectMode:true}); - data.forEach((d,i) => setTimeout( () => { - s.write(d); - if (i == data.length-1) - s.end(); - },i)); + // Simulate async data stream + Promise.resolve().then(async() => { + for (let i = 0; i < data.length; i++) { + s.write(data[i]); + await new Promise(resolve => setTimeout(resolve,10)) + } + s.end(); + }); return s; }; \ No newline at end of file