From 8a734248b46bf968f80bc825a64e43f63cb9aed1 Mon Sep 17 00:00:00 2001 From: Kyle Date: Tue, 6 Dec 2022 10:30:40 -0500 Subject: [PATCH] add ability to create indexes (#5) * update * add logging * update readme * run indexes on connection * indexes have been run * consistency in private createIndexes * add comment * remove readme change * remove catch block * fix duplicate calls * fix options * await save correctly in super class * update error message for duplicate key * remove useless semicolon * fix app error * Removing double imports (#18) Co-authored-by: Marcus Veloso <44785425+velosomarcus@users.noreply.github.com> --- README.md | 2 - forkrun | Bin 50420 -> 17064 bytes src/adapters/datasources/datasource-file.js | 4 +- .../datasources/datasource-meshlink.js | 4 +- .../datasources/datasource-mongodb.js | 43 +++++++++++++----- src/domain/datasource-factory.js | 12 ++++- src/domain/datasource.js | 10 ++++ src/domain/shared-memory.js | 4 +- src/domain/util/app-error.js | 4 ++ 9 files changed, 62 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 52691967..2f33d3f8 100644 --- a/README.md +++ b/README.md @@ -121,5 +121,3 @@ yarn demo [downloads-url]: https://npmjs.org/package/@module-federation/aegis [gitpod-image]: https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod [gitpod-url]: https://gitpod.io/github.com/module-federation/aegis - - diff --git a/forkrun b/forkrun index 72b79895a4dc9e8771370ad87d0e3bb1426ac413..a76a9a006fe29e134febfa623a899cf977a4657f 100755 GIT binary patch literal 17064 zcmeHOeQX@X6`%9@Vgt$9fdrhSB?~200`kQUj_ZI@?rfjEZtWyBHXkZseZD)}CwzDA z-CnS13#klnLE9I-|NAgDz2qIk6h;#X!Jh zyJ>;y!Ib4zc?udsQsmvD51|4irgl4w&2+sKi|M^CN^v>PjiGuWjCM0*H$!$z7pXj^ z98YwER|~bnO%;$~Qsi9-x7}@I=cetX&lJnRbn{s~@mb0{z;=psBU)UP%VEmpeGTj| zF8^-gZtkV>R_li`>IbG2Ua@mDHQ3y=b2L#un#vT$>&IJ~>zkWG`D|#HtOC{x%fK<$3cefgtKCkJhyILX_ce>;crt!Sj3slqY)%Yib9V_%%j>cDU0QtPE^R25 z8WqDyJ()QKczisj4W%-%(bPRjV9^Qc=5#F`OJzhpGnPwb^dZsN-PPKr?Fuznxm}^% zLhI`5(GtmAayXUOlexa0w$W@R*%uoeMQ6k5Y=)d^rfOM5b3FKARNz=JFEfwL@&V!l z%Hw@QDzy$rkw#cybLt|rN})0&eh%^cpQq>iQF_iWK0|nQSejYS3+8KGc%@?M)w%Fm z!e6?{l;ar2HgKA^;qv)Rf|3ol&-+Ci&UHnTb2gmk2I7wsjz0?+Cr-z0IL}#>ov`6J zKRM0XaQcanqUUWm1lmfcZ1}|%>SeqLcoFa-;6=cTfENKT0{{C6yjOGON9yET)#{PJ z%R7ZoADGoEj8p35Z>vwr${D+J052Ka@<3{~3e+D#$*CM2 z_&?kE#<#nQr|(pYud9=PT|Cg&Jr(GH7N}F}{sawEr}oZ4^^A?b0F(Epfn9)%Iu&>w z$)0oiMhM*_n1|3^WX#uWJq+sNBvH_RQtIzUdv(VJwYaFxy>pW~ch0Z+o>fm@&^LgC zBsr)y=7(fE*&f?}cy9s3qHtZmI=QzMvug3IzE(Z5w*#2c3l|Kdlz`wp8|VSvcMxo? z^3i_bZV+OoAMBvB8xrb~%5B@FpXj;9+0q-}p%{I2^4qV9ng@;{ekAZ7`~jgBpDTS6 zWNL9vo!a;|EG(YJmuP5H0Q%ZrvlGFMb%<-9^&|v zw2YS2=c0?~QJn&~_$(AMgRQ#}96m5Ru><%lcr0xJ{>?2|)rlZr!-!QORwWB~GjIZG z2o2~zmr$?YAX}*iO3$1(j2AF27>3BL-BWwNf2|OaTe^xbM*6#oA4K{h#ZUUxsrvf? zzPWqHd7Kla*FJ%^&z<+{TN_`Y{_HL;br;`hFTN2mHoT@zp7g1E&J_NJ^X2OYBX>j& zM(&JgCx?bYHCyk;#>nz;CSkd9o>?aKXqTKSP`1c#Tr3M3ncS!rG4HytnNsQ!MwY zVz!R>jYOG{nV6>ial99BniOz-hKcvsU173fo;Cq8zDO10dM}b5@4@5puOyuHJ~qtu zL0lySUk+t^XUQLYDV03O9hQTVe~je+AfDqw?*C(i_j$V=Hc)?ui5?(2Ky-}gVWMvP zPvfh@rbeY=Qr@@O+17T0vb}$>kkJduo={V$p}w&oxyE~UHG~?PLXA62+_d{wcR!5c zIvxNvMcmuU`_*kQ>|*r`-jD9YE5*0U`UzO$)~Ub78m~^gO7OUL;?;ut+lgN!c$_%# ziv^EICw_@FZk;&H;xhf@hf@_wSv)AF%i=X+rYsI1?ca5lyI$~f!imEUYsHS3dB3>K z+MjQi`r%)c!)5h1TKkKg`j-oS-Z=3qM7iV#T*JIMp25~=D*pO!^e* zgEN_SlK2zAQC|rYK%~0D^uIv*@;_OSt&_M@7rCnd$NIbd)B~=pR2Q9qUtI=|175Bl z53PcWcuvpj1=#Tu7t1F-G(D}w@vKK4&UqxV+VE(0FgB_s^lUD##R}sho=uO9!XcAH zsHLH~vAhxTrM`DB{O<%LJZ|%>7)Saqut|eE>bqc_qM z)uQ{_H8^MDR80u2{q}v4o~|~hM4oX02#zIbQI$NX?Y%DKmlble>6)A}N< z-BI)^PsYUa1zBl2KNAk~v6xRi43kX6^cc8tp2Jyo&}1F8S;o*QoYFZI%W3g+g0r#Q zFjr^!ppMfBx&%rykKTZ;^Bj(r$Y-^YSSEpIbh`F~Rw9+r3i%|2YPppdZ{P>>d2%C< z1DR)v+=aoh9}YgA2P%8W$9dAo=|P@Xazc1&$%%+iej=^M27&51la6o>PADaFVdDY>rVtv-WudD2gbf-jq~NSuDnU@BwX0r_4GXD063dT>P+}qjMVM62nUX`vTt1b} zI2a9Nx#VaJ4JbD@s*8{eSqQSB;ViJ2$tU9?q$kIL#jy_BSvd|u$q^deBMI=voaw_H z=B5Sbp{Q6o1NB#qVPF#LsEQx*%h>=l5r(3M&$I+usWKhp_hH`t$mb zDVNV@wcPcOfjyqfVtZacGF33Rkg$%JWqV%7;rTUeGTZZeJ5yeNf{IM!j#=>y$l=3fR*&ovXWL{9ldhE)D4x#sx2=hg{CNTvLi`i1PN&?=dwty9`{c_$MYy BN!9=W literal 50420 zcmeI5Yitx%6vxl*wq>zEH&6%)q^k)8gn$T`8fY1W@>B|?Kt<)UwA)A8?&9tifuasC zMT`kc5F5!x2uid(LP`imgBX@r@tm32c6Lxmz;FIfdgh-0 zeVo1bcW3w8Nl)Ck@>ee+7qPk7`?8m$6Kx_tM@09t&tz9>-jo>yOA3k#r8m`##i?Dl zD9N*#Cn{B3P*R*aY^>K(`^eI~6%tRghLws1SH&#BjWgewJo7NitWAlQb9Gr7>0L1g zN=0LJ%cEUg&U}+5oB4!+m}q-Gua%D(8m3ffG#04}Rdngp^KP{AJ!*}Mwwpb-^ea^t zdZ8**rm8~aVMC|pJL1@%XlD+4ez}JZBhR!c#Z%PO!nq5~+t93+fLp^Ht%p0sCAW`V zsfw`e8d(2xzFeMbm}aiG6lHKAo;2nb4n?~&I`idf=Kcm+3q;F){>@U)X`7`x&dJ&^ z&1~%FlpdwZ0vX6`T zXN!lOUUn49nyu^mE%XmAVJ6jUN zKmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l z00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1pb!zwL}b4_BSx2Kly zy0k>YhMenP93OJzX`=1zBdJNtpb1rMuTr*ns(U12gdpt{Vay93Duz}ikdETESdv5ULTp!7Ec>MXVh>B&ee4$%uZGK$V zisrh_?2eIIlh@^#b09xHl;?0AXttc0i#fB+v+6|+tR-$Z{@js{OlmSd-(s|HgSzf@ z^}O6-eO`N}*3?;Fu#cI>n?33zpM5*mb2j?iT7uj;CyhA67Wc8meQogoTl|0-PcJN* zQ?zidn)=iD3rS`n!V z#mZG_WwoJWfzoObsyt9tliE{J7l@RR3a$#4t|V0+j_`_#U@RD7wIUdagd?2Or6`@! zRc)m)uPzo;mBBz+Fhb*N163i)<=HC(5k+}{NChn{tu3Qxc#GtrGxbDn(X(WvFpYHG zO=+~*chixuSXD znLKkmpEXNKW!PMP+VwJrjyg2sPMNPe^k#?Vn>F5rp4{HYp2^;eJ=rfubB=Kr`T44D zWv&FJ5~#}JZgg0&(Qn+!fBz!f_^DSj-%d7fEy*sM`}n%#FHIV`x@GJo>|(u%j&YE^CtuI*`RnESA9GK4On7VQd|%twgRbP}e7o(V zmk%5&3AbJN&=jQlUJYbFYw7&s)*me>C diff --git a/src/adapters/datasources/datasource-file.js b/src/adapters/datasources/datasource-file.js index 995a7b67..9cae6180 100644 --- a/src/adapters/datasources/datasource-file.js +++ b/src/adapters/datasources/datasource-file.js @@ -9,8 +9,8 @@ export class DataSourceFile extends DataSource { /** * @param {Set} map */ - constructor (map, name, options = {}) { - super(map, name, options) + constructor (map, name, namespace, options = {}) { + super(map, name, namespace, options) this.file = this.getFilePath() this.className = DataSourceFile.name } diff --git a/src/adapters/datasources/datasource-meshlink.js b/src/adapters/datasources/datasource-meshlink.js index b1f85ce0..c57368a8 100644 --- a/src/adapters/datasources/datasource-meshlink.js +++ b/src/adapters/datasources/datasource-meshlink.js @@ -14,8 +14,8 @@ async function fetchSharedObject (name) { } export class DataSourceMeshLink extends DataSource { - constructor (dataSource, name, options = {}) { - super(dataSource, name, options) + constructor (map, name, namespace, options = {}) { + super(map, name, namespace, options) } /** diff --git a/src/adapters/datasources/datasource-mongodb.js b/src/adapters/datasources/datasource-mongodb.js index c5f78533..5a8c7d4f 100644 --- a/src/adapters/datasources/datasource-mongodb.js +++ b/src/adapters/datasources/datasource-mongodb.js @@ -1,13 +1,13 @@ 'use strict' -import DataSource from '../../domain/datasource' +import { ObjectId, MongoClient } from 'mongodb'; +import { Transform, Writable } from 'stream'; +import qpm from 'query-params-mongo'; +import DataSource from '../../domain/datasource'; +import { DataSourceMemory } from './datasource-memory' const HIGHWATERMARK = 50 -const mongodb = require('mongodb') -const { MongoClient } = mongodb -const { Transform, Writable } = require("stream") -const qpm = require("query-params-mongo") const processQuery = qpm({ autoDetect: [ { valuePattern: /^null$/i, dataType: 'nullstring' } @@ -19,11 +19,13 @@ const processQuery = qpm({ const url = process.env.MONGODB_URL || "mongodb://localhost:27017" const configRoot = require("../../config").hostConfig + const dsOptions = configRoot.adapters.datasources.DataSourceMongoDb.options || { - runOffline: true, + runOffline: false, numConns: 2 } -const cacheSize = configRoot.adapters.cacheSize || 3000 + +const cacheSize = configRoot.adapters.cacheSize || 3000; /** * @type {Map} @@ -45,6 +47,7 @@ export class DataSourceMongoDb extends DataSource { super(map, name, namespace, options) this.cacheSize = cacheSize this.mongoOpts = mongoOpts + this.options = options this.runOffline = dsOptions.runOffline this.url = url } @@ -61,6 +64,13 @@ export class DataSourceMongoDb extends DataSource { } const client = connections.shift() connections.push(client) + + if(!this.options?.connOpts?.indexesHaveBeenRun && this.options?.connOpts?.indexes) { + console.info(`running indexes for datasource ${this.name} with index values`, this.options.connOpts.indexes) + await this.createIndexes(client) + this.options.connOpts.indexesHaveBeenRun = true; + } + return client } catch (error) { console.error({ fn: this.connection.name, error }) @@ -71,11 +81,23 @@ export class DataSourceMongoDb extends DataSource { return (await this.connection()).db(this.namespace).collection(this.name) } + async createIndexes(client) { + const indexOperations = this.options.connOpts.indexes.map((index) => { + return { + name: index.fields.join("_"), + key: index.fields.reduce((a, v) => ({ ...a, [v]: 1 }), {}), + ...index.options, + } + }); + + return await client.db(this.namespace).collection(this.name).createIndexes(indexOperations); + } + async find (id) { try { return (await this.collection()).findOne({ _id: id }) } catch (error) { - console.error({ fn: this.findDb.name, error }) + console.error({ fn: this.find.name, error }) } } @@ -91,12 +113,11 @@ export class DataSourceMongoDb extends DataSource { */ async save (id, data) { try { - const col = await this.collection() - col.replaceOne({ _id: id }, { ...data, _id: id }, { upsert: true }) + await (await this.collection()).replaceOne({ _id: id }, { ...data, _id: id }, { upsert: true }) } catch (error) { // default is if (!this.runOffline) { - throw new Error('db trans failed,', error) + throw new Error(`DB Transaction failed: ${error.message}`) } // run while db is down - cache will be ahead console.error('db trans failed, sync it later', error) diff --git a/src/domain/datasource-factory.js b/src/domain/datasource-factory.js index 8e431aab..7c4f2e3f 100644 --- a/src/domain/datasource-factory.js +++ b/src/domain/datasource-factory.js @@ -37,6 +37,10 @@ const DefaultDataSource = */ const DsCoreExtensions = superclass => class extends superclass { + constructor(map, name, namespace, options) { + super(map, name, namespace, options) + } + set factory (value) { this[FACTORY] = value } @@ -64,6 +68,7 @@ const DsCoreExtensions = superclass => console.error({ fn: this.save.name, error }) throw error } + } /** @@ -210,7 +215,7 @@ const DataSourceFactory = (() => { if (adapterName) return adapters[adapterName] || DefaultDataSource - if (spec?.datasource?.adapterFactory) { + if (spec?.datasource?.factory) { const url = spec.datasource.url const cacheSize = spec.datasource.cacheSize const adapterFactory = spec.datasource.factory @@ -249,9 +254,12 @@ const DataSourceFactory = (() => { const DsClass = createDataSourceClass(spec, options) const DsExtendedClass = extendDataSourceClass(DsClass, options) + if(spec.datasource) { + options = {...options, connOpts: {...spec.datasource}} + } + const newDs = new DsExtendedClass(dsMap, name, namespace, options) newDs.factory = this // setter to avoid exposing in ctor - if (!options.ephemeral) dataSources.set(name, newDs) debug && console.debug({ newDs }) diff --git a/src/domain/datasource.js b/src/domain/datasource.js index 8921b091..e97066b3 100644 --- a/src/domain/datasource.js +++ b/src/domain/datasource.js @@ -94,6 +94,16 @@ export default class DataSource { return this.dsMap.set(id, data) } + /** + * Create indexes as defined in the datasource object in the model + * @param {Array} + * @returns {Promise} result + */ + + async createIndexes(indexes) { + throw new Error('abstract method not implemented') + } + /** * Find model instance by ID * @param {*} id record id diff --git a/src/domain/shared-memory.js b/src/domain/shared-memory.js index f44315b6..5fa5476b 100644 --- a/src/domain/shared-memory.js +++ b/src/domain/shared-memory.js @@ -46,8 +46,8 @@ const dataType = { */ const SharedMemoryMixin = superclass => class extends superclass { - constructor (map, name, options) { - super(map, name, options) + constructor (map, name, namespace, options) { + super(map, name, namespace, options) } /** diff --git a/src/domain/util/app-error.js b/src/domain/util/app-error.js index 84b91148..20fd8ee7 100644 --- a/src/domain/util/app-error.js +++ b/src/domain/util/app-error.js @@ -6,6 +6,10 @@ * @returns */ export function AppError (error, code = 400, cause = null) { + if(error.code > 500) { + error.code = 400 + } + return { name: error.name, stack: error.stack,