Skip to content

Commit

Permalink
used latest version of mongodb for tests bec of npm audit vulnerabili…
Browse files Browse the repository at this point in the history 
…ty (#100)

* used latest version of mongodb for tests bec of vulnerability

* closed mongo connection

* used correct mongo connection string.

* removed console
  • Loading branch information
@montumodi
montumodi authored Feb 2, 2020
1 parent 33799e5 commit 937faa8
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 88 deletions.
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
},
"devDependencies": {
"@elastic/elasticsearch": "~7.1.0",
"mongodb": "~2.2.27",
"mongodb": "^3.3.5",
"mysql": "^2.17.1",
"pg": "~6.2.3",
"pg-query-stream": "~1.0.0",
"tap": "^14.6.9",
"sqlite3": "^4.1.0",
"tap": "^14.6.9",
"url-js": "^0.2.5"
}
}
76 changes: 36 additions & 40 deletions test/lib/mongo.js
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,38 @@
var Promise = require('bluebird'),
mongo = require('mongodb'),
data = require('../data');


Promise.promisifyAll(mongo);
Promise.promisifyAll(mongo.MongoClient);

module.exports = () => ({
db : mongo.connectAsync('mongodb://mongodb:27017/etl_tests'),

getCollection : function(name) {
var self = this,collection;

if (this.collections[name])
return Promise.resolve(this.collections[name]);

return this.db
.then(function(db) {
return db.collection(name);
})
.then(function(d) {
collection = self.collections[name] = d;
return collection.removeAsync({});
})
.then(function() {
return collection;
});
},

collections: {},

cleanup : function() {
return this.db
.then(function(db) {
return Promise.map(Object.keys(this.collections),function(key) {
return db.dropCollectionAsync(key);
});
});

const mongodbClient = require("mongodb").MongoClient;

let client;

async function getMongodbDriver() {

if (!client) {
client = await mongodbClient.connect('mongodb://mongodb:27017/etl_tests', {"useNewUrlParser": true, "useUnifiedTopology": true});
}
});

return client.db();
}

async function getCollection(collectionName) {
const db = await getMongodbDriver();
return db.collection(collectionName);
}

async function clear() {
const db = await getMongodbDriver();
await Promise.all(
[
db.collection("insert").deleteMany({}),
db.collection("update-empty").deleteMany({}),
db.collection("update-populated").deleteMany({}),
db.collection("upsert").deleteMany({}),
db.collection("upsert2").deleteMany({}),
db.collection("upsert3").deleteMany({})
]);
await client.close();
}

module.exports = {
getCollection,
clear
};

48 changes: 21 additions & 27 deletions test/mongo-insert-test.js
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,33 @@
const etl = require('../index');
const data = require('./data');
const mongo = require('./lib/mongo')();
const {getCollection, clear} = require('./lib/mongo');
const t = require('tap');
const Promise = require('bluebird');

t.test('mongo.insert', async t => {
await mongo.db;

t.teardown(() => t.end());

t.test('piping data into mongo.insert',async t => {
const d = await mongo.getCollection('insert')
.then(collection => {
return data.stream()
.pipe(etl.mongo.insert(collection,{pushResult:true}))
.promise();
});

const collection = await getCollection('insert');
const d = await data.stream()
.pipe(etl.mongo.insert(collection,{pushResult:true}))
.promise();
d.forEach(d => t.same(d,{ok:1,n:1},'inserts each record'));
});

t.test('mongo collection',async t => {
const collection = await mongo.getCollection('insert',true);
const d = await collection.find({},{_id:false}).toArrayAsync();
const collection = await getCollection('insert');
const d = await collection.find({},{ projection: {_id:0}}).toArray();

t.same(d,data.data,'reveals data');
});

t.test('pushResults == false and collection as promise',async t => {
const d = await mongo.getCollection('insert')
.then(collection => {
return data.stream(etl.mongo.insert(Promise.resolve(collection)))
.pipe(etl.mongo.insert(Promise.resolve(collection)))
.promise();
});
const collection = await getCollection('insert');
const d = await data.stream(etl.mongo.insert(collection))
.pipe(etl.mongo.insert(collection))
.promise();

t.same(d,[],'returns nothing');
});
Expand All @@ -47,15 +43,13 @@ t.test('mongo.insert', async t => {
t.same(e.message,'CONNECTION_ERROR','should bubble down');
});
})
.then(
() => mongo.db.then( db => db.close()),
e => {
if (e.message.includes('ECONNREFUSED'))
console.warn('Warning: MongoDB server not available');
else
throw e;
}
);

.then(() => clear())
.then(() => t.end())
.catch(e => {
if (e.message.includes('ECONNREFUSED'))
console.warn('Warning: MongoDB server not available');
else
console.warn(e.message);
});


40 changes: 21 additions & 19 deletions test/mongo-update-test.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
const etl = require('../index');
const Promise = require('bluebird');
const data = require('./data');
const mongo = require('./lib/mongo')();
const {getCollection, clear} = require('./lib/mongo');
const t = require('tap');

t.test('mongo update', {autoend: true}, t => {

t.teardown(() => t.end());

t.test('single record', {autoend: true}, async t => {
const collection = mongo.getCollection('update-empty');
const collection = await getCollection('update-empty');

t.test('missing keys',async t => {
const e = await Promise.try(() => etl.mongo.update(collection))
Expand Down Expand Up @@ -40,7 +42,7 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('bulk', {autoend:true}, async t => {
const collection = await mongo.getCollection('update-empty');
const collection = await getCollection('update-empty');

t.test('on an empty collection', async t => {
const update = etl.mongo.update(collection,['name'],{pushResult:true});
Expand All @@ -57,7 +59,7 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('with pushresults == false',async t => {
const collection = await mongo.getCollection('update-empty');
const collection = await getCollection('update-empty');
const update = etl.mongo.update(collection,['name']);

const d = await data.stream()
Expand All @@ -68,10 +70,10 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('on a populated collection', {autoend: true}, async t => {
const collection = await mongo.getCollection('update-populated',true);
const collection = await getCollection('update-populated');

t.test('update', async t => {
await collection.insertAsync(data.copy());
await collection.insertMany(data.copy());
const update = etl.mongo.update(collection,['name'],{pushResult:true});

let d = await data.stream()
Expand All @@ -95,20 +97,19 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('using find', async t => {
const collection = await mongo.getCollection('update-populated',true);
const d = await collection.find({},{_id:false}).toArrayAsync();
const collection = await getCollection('update-populated');
const d = await collection.find({},{projection: {_id:false}}).toArray();

const expected = data.copy().map(function(d,i) {
if (i) d.newfield = 'newfield';
return d;
});

t.same(d,expected,'results were saved');
});
});

t.test('using upsert option', {autoend: true}, t => {
const collection = mongo.getCollection('upsert');
t.test('using upsert option', {autoend: true}, async t => {
const collection = await getCollection('upsert');

t.test('upsert', async t => {
const upsert = etl.mongo.update(collection,['name'],{pushResult:true,upsert:true});
Expand All @@ -123,15 +124,15 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('find',async t => {
const collection = await mongo.getCollection('upsert');
const d = await collection.find({},{_id:false}).toArrayAsync();
const collection = await getCollection('upsert');
const d = await collection.find({},{projection : {_id:false}}).toArray();

t.same(d,data.data,'results are saved');
});
});

t.test('using upsert function',{autoend: true}, async t => {
const collection = await mongo.getCollection('upsert2');
const collection = await getCollection('upsert2');

t.test('should populate', async t => {
const upsert = etl.mongo.upsert(collection,['name'],{pushResult:true});
Expand All @@ -147,14 +148,14 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('find',async t => {
const d = await collection.find({},{_id:false}).toArrayAsync();
const d = await collection.find({},{projection: {_id:false}}).toArray();
t.same(d,data.data,'results are saved');
});
});
});

t.test('using $update with upsert function',{autoend: true}, async t => {
const collection = await mongo.getCollection('upsert3');
const collection = await getCollection('upsert3');
t.test('should populate', async t => {
const upsert = etl.mongo.upsert(collection,['name'],{pushResult:true});

Expand All @@ -171,7 +172,7 @@ t.test('mongo update', {autoend: true}, t => {
});

t.test('find',async t => {
const d = await collection.find({}, {_id: false}).toArrayAsync();
const d = await collection.find({}, {projection: {_id: false}}).toArray();
t.same(d,data.data,'results are saved');
});
});
Expand All @@ -185,10 +186,11 @@ t.test('mongo update', {autoend: true}, t => {
t.same(e.message,'CONNECTION_ERROR','passes down');
});
})
.then(() => clear())
.then(() => t.end())
.catch(e => {
if (e.message.includes('ECONNREFUSED'))
console.warn('Warning: MongoDB server not available');
else
console.warn(e.message);
})
.then(() => mongo.db.then( db => db.close()));
});

0 comments on commit 937faa8

Please sign in to comment.