From 937faa8cf5b797733e5890f3f8ccc8df6cbda732 Mon Sep 17 00:00:00 2001 From: Ashish Modi Date: Sun, 2 Feb 2020 12:16:36 +0000 Subject: [PATCH] used latest version of mongodb for tests bec of npm audit vulnerability (#100) * used latest version of mongodb for tests bec of vulnerability * closed mongo connection * used correct mongo connection string. * removed console --- package.json | 4 +-- test/lib/mongo.js | 76 +++++++++++++++++++-------------------- test/mongo-insert-test.js | 48 +++++++++++-------------- test/mongo-update-test.js | 40 +++++++++++---------- 4 files changed, 80 insertions(+), 88 deletions(-) diff --git a/package.json b/package.json index c587e3c..f3bdbfc 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,12 @@ }, "devDependencies": { "@elastic/elasticsearch": "~7.1.0", - "mongodb": "~2.2.27", + "mongodb": "^3.3.5", "mysql": "^2.17.1", "pg": "~6.2.3", "pg-query-stream": "~1.0.0", - "tap": "^14.6.9", "sqlite3": "^4.1.0", + "tap": "^14.6.9", "url-js": "^0.2.5" } } diff --git a/test/lib/mongo.js b/test/lib/mongo.js index d8ee010..7dd5119 100644 --- a/test/lib/mongo.js +++ b/test/lib/mongo.js @@ -1,42 +1,38 @@ -var Promise = require('bluebird'), - mongo = require('mongodb'), - data = require('../data'); - - -Promise.promisifyAll(mongo); -Promise.promisifyAll(mongo.MongoClient); - -module.exports = () => ({ - db : mongo.connectAsync('mongodb://mongodb:27017/etl_tests'), - - getCollection : function(name) { - var self = this,collection; - - if (this.collections[name]) - return Promise.resolve(this.collections[name]); - - return this.db - .then(function(db) { - return db.collection(name); - }) - .then(function(d) { - collection = self.collections[name] = d; - return collection.removeAsync({}); - }) - .then(function() { - return collection; - }); - }, - - collections: {}, - - cleanup : function() { - return this.db - .then(function(db) { - return Promise.map(Object.keys(this.collections),function(key) { - return db.dropCollectionAsync(key); - }); - }); + +const mongodbClient = require("mongodb").MongoClient; + +let client; + +async function getMongodbDriver() { + + if (!client) { + client = await mongodbClient.connect('mongodb://mongodb:27017/etl_tests', {"useNewUrlParser": true, "useUnifiedTopology": true}); } -}); + + return client.db(); +} + +async function getCollection(collectionName) { + const db = await getMongodbDriver(); + return db.collection(collectionName); +} + +async function clear() { + const db = await getMongodbDriver(); + await Promise.all( + [ + db.collection("insert").deleteMany({}), + db.collection("update-empty").deleteMany({}), + db.collection("update-populated").deleteMany({}), + db.collection("upsert").deleteMany({}), + db.collection("upsert2").deleteMany({}), + db.collection("upsert3").deleteMany({}) + ]); + await client.close(); +} + +module.exports = { + getCollection, + clear +}; diff --git a/test/mongo-insert-test.js b/test/mongo-insert-test.js index e9e72de..cc6aef8 100644 --- a/test/mongo-insert-test.js +++ b/test/mongo-insert-test.js @@ -1,37 +1,33 @@ const etl = require('../index'); const data = require('./data'); -const mongo = require('./lib/mongo')(); +const {getCollection, clear} = require('./lib/mongo'); const t = require('tap'); const Promise = require('bluebird'); t.test('mongo.insert', async t => { - await mongo.db; + + t.teardown(() => t.end()); t.test('piping data into mongo.insert',async t => { - const d = await mongo.getCollection('insert') - .then(collection => { - return data.stream() - .pipe(etl.mongo.insert(collection,{pushResult:true})) - .promise(); - }); - + const collection = await getCollection('insert'); + const d = await data.stream() + .pipe(etl.mongo.insert(collection,{pushResult:true})) + .promise(); d.forEach(d => t.same(d,{ok:1,n:1},'inserts each record')); }); t.test('mongo collection',async t => { - const collection = await mongo.getCollection('insert',true); - const d = await collection.find({},{_id:false}).toArrayAsync(); + const collection = await getCollection('insert'); + const d = await collection.find({},{ projection: {_id:0}}).toArray(); t.same(d,data.data,'reveals data'); }); t.test('pushResults == false and collection as promise',async t => { - const d = await mongo.getCollection('insert') - .then(collection => { - return data.stream(etl.mongo.insert(Promise.resolve(collection))) - .pipe(etl.mongo.insert(Promise.resolve(collection))) - .promise(); - }); + const collection = await getCollection('insert'); + const d = await data.stream(etl.mongo.insert(collection)) + .pipe(etl.mongo.insert(collection)) + .promise(); t.same(d,[],'returns nothing'); }); @@ -47,15 +43,13 @@ t.test('mongo.insert', async t => { t.same(e.message,'CONNECTION_ERROR','should bubble down'); }); }) -.then( - () => mongo.db.then( db => db.close()), - e => { - if (e.message.includes('ECONNREFUSED')) - console.warn('Warning: MongoDB server not available'); - else - throw e; - } -); - +.then(() => clear()) +.then(() => t.end()) +.catch(e => { + if (e.message.includes('ECONNREFUSED')) + console.warn('Warning: MongoDB server not available'); + else + console.warn(e.message); +}); \ No newline at end of file diff --git a/test/mongo-update-test.js b/test/mongo-update-test.js index 0a7aeb8..d399ea1 100644 --- a/test/mongo-update-test.js +++ b/test/mongo-update-test.js @@ -1,13 +1,15 @@ const etl = require('../index'); const Promise = require('bluebird'); const data = require('./data'); -const mongo = require('./lib/mongo')(); +const {getCollection, clear} = require('./lib/mongo'); const t = require('tap'); t.test('mongo update', {autoend: true}, t => { + t.teardown(() => t.end()); + t.test('single record', {autoend: true}, async t => { - const collection = mongo.getCollection('update-empty'); + const collection = await getCollection('update-empty'); t.test('missing keys',async t => { const e = await Promise.try(() => etl.mongo.update(collection)) @@ -40,7 +42,7 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('bulk', {autoend:true}, async t => { - const collection = await mongo.getCollection('update-empty'); + const collection = await getCollection('update-empty'); t.test('on an empty collection', async t => { const update = etl.mongo.update(collection,['name'],{pushResult:true}); @@ -57,7 +59,7 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('with pushresults == false',async t => { - const collection = await mongo.getCollection('update-empty'); + const collection = await getCollection('update-empty'); const update = etl.mongo.update(collection,['name']); const d = await data.stream() @@ -68,10 +70,10 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('on a populated collection', {autoend: true}, async t => { - const collection = await mongo.getCollection('update-populated',true); + const collection = await getCollection('update-populated'); t.test('update', async t => { - await collection.insertAsync(data.copy()); + await collection.insertMany(data.copy()); const update = etl.mongo.update(collection,['name'],{pushResult:true}); let d = await data.stream() @@ -95,20 +97,19 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('using find', async t => { - const collection = await mongo.getCollection('update-populated',true); - const d = await collection.find({},{_id:false}).toArrayAsync(); + const collection = await getCollection('update-populated'); + const d = await collection.find({},{projection: {_id:false}}).toArray(); const expected = data.copy().map(function(d,i) { if (i) d.newfield = 'newfield'; return d; }); - t.same(d,expected,'results were saved'); }); }); - t.test('using upsert option', {autoend: true}, t => { - const collection = mongo.getCollection('upsert'); + t.test('using upsert option', {autoend: true}, async t => { + const collection = await getCollection('upsert'); t.test('upsert', async t => { const upsert = etl.mongo.update(collection,['name'],{pushResult:true,upsert:true}); @@ -123,15 +124,15 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('find',async t => { - const collection = await mongo.getCollection('upsert'); - const d = await collection.find({},{_id:false}).toArrayAsync(); + const collection = await getCollection('upsert'); + const d = await collection.find({},{projection : {_id:false}}).toArray(); t.same(d,data.data,'results are saved'); }); }); t.test('using upsert function',{autoend: true}, async t => { - const collection = await mongo.getCollection('upsert2'); + const collection = await getCollection('upsert2'); t.test('should populate', async t => { const upsert = etl.mongo.upsert(collection,['name'],{pushResult:true}); @@ -147,14 +148,14 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('find',async t => { - const d = await collection.find({},{_id:false}).toArrayAsync(); + const d = await collection.find({},{projection: {_id:false}}).toArray(); t.same(d,data.data,'results are saved'); }); }); }); t.test('using $update with upsert function',{autoend: true}, async t => { - const collection = await mongo.getCollection('upsert3'); + const collection = await getCollection('upsert3'); t.test('should populate', async t => { const upsert = etl.mongo.upsert(collection,['name'],{pushResult:true}); @@ -171,7 +172,7 @@ t.test('mongo update', {autoend: true}, t => { }); t.test('find',async t => { - const d = await collection.find({}, {_id: false}).toArrayAsync(); + const d = await collection.find({}, {projection: {_id: false}}).toArray(); t.same(d,data.data,'results are saved'); }); }); @@ -185,10 +186,11 @@ t.test('mongo update', {autoend: true}, t => { t.same(e.message,'CONNECTION_ERROR','passes down'); }); }) +.then(() => clear()) +.then(() => t.end()) .catch(e => { if (e.message.includes('ECONNREFUSED')) console.warn('Warning: MongoDB server not available'); else console.warn(e.message); -}) -.then(() => mongo.db.then( db => db.close())); \ No newline at end of file +}); \ No newline at end of file