Skip to content

Commit

Permalink
Merge branch 'dev-alex/excel-upload' EXCEL - MONGODB
Browse files Browse the repository at this point in the history 
# Conflicts:
#	eda/eda_api/lib/module/dashboard/dashboard.controller.ts
#	eda/eda_api/lib/router.ts
#	eda/eda_api/lib/services/connection/manager-connection.service.ts
  • Loading branch information
@jortilles
jortilles committed Aug 23, 2024
2 parents 517cb3a + a9f8b4b commit a6de4e5
Show file tree
Hide file tree
Showing 20 changed files with 1,015 additions and 40 deletions.
17 changes: 9 additions & 8 deletions eda/eda_api/lib/module/dashboard/dashboard.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1071,21 +1071,22 @@ export class DashboardController {
} and Panel:${req.body.dashboard.panel_id} `
)
console.log(query)
console.log(
'\n-------------------------------------------------------------------------------\n'
)
console.log('\n-------------------------------------------------------------------------------\n');

/**cached query */
let cacheEnabled =
dataModelObject.ds.metadata.cache_config &&
dataModelObject.ds.metadata.cache_config.enabled === true
let cacheEnabled = false;
// dataModelObject.ds.metadata.cache_config &&
// dataModelObject.ds.metadata.cache_config.enabled === true;

console.log('cacheEnabled', cacheEnabled)
const cachedQuery = cacheEnabled
? await CachedQueryService.checkQuery(req.body.model_id, query)
: null

if (!cachedQuery) {
connection.client = await connection.getclient()
const getResults = await connection.execQuery(query)
connection.client = await connection.getclient();
const getResults = await connection.execQuery(query);
console.log('getResults', getResults);

let numerics = []
/** si es oracle o alguns mysql haig de fer una merda per tornar els numeros normals. */
Expand Down
4 changes: 2 additions & 2 deletions eda/eda_api/lib/module/datasource/datasource.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,10 @@ export class DataSourceController {

//aparto las relaciones ocultas para optimizar el modelo.
ds.ds.model.tables.forEach(t => {
t.no_relations = t.relations.filter(r => r.visible == false)
t.no_relations = t ? t.relations.filter(r => r.visible == false) : [];
});
ds.ds.model.tables.forEach(t => {
t.relations = t.relations.filter(r => r.visible !== false)
t.relations = t ? t.relations.filter(r => r.visible !== false) : [];
});
/** Comprobacionde la reciprocidad de las relaciones */
ds.ds.model.tables.forEach(tabla => {
Expand Down
233 changes: 233 additions & 0 deletions eda/eda_api/lib/module/excel/excel-sheet.controller.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
import { NextFunction, Request, Response } from "express";
import { HttpException } from '../global/model/index';
import { EnCrypterService } from "../../services/encrypter/encrypter.service";
import { MongoDBConnection } from "../../services/connection/db-systems/mongodb-connection";
import DataSource, { IDataSource } from "../datasource/model/datasource.model";
import ExcelSheetModel from "./model/excel-sheet.model";
import { AggregationTypes } from "../global/model/aggregation-types";
import { DateUtil } from "../../utils/date.util";

const databaseUrl = require('../../../config/database.config');


export class ExcelSheetController {
static async GenerateCollectionFromJSON(req: Request, res: Response, next: NextFunction) {
return await ExcelSheetController.FromJSONToCollection(req, res, next);
}

static async FromJSONToCollection(req: Request, res: Response, next: NextFunction) {
//Guarda una nueva colección con el nombre pasado desde el frontal, si esta ya existe sustituye los campos del excel por los nuevos.
try {
const excelName = req.body?.name, optimize = req.body?.optimize, cacheAllowed = req.body?.allowCache;
let excelFields = req.body?.fields;

if (!excelName || !excelFields) {
return res.status(400).json({ ok: false, message: 'Nombre o campos incorrectos en la solicitud' });
}

const excelModel = ExcelSheetModel(excelName)
const excelDocs = await excelModel.findOne({});

if (excelDocs) {
excelDocs.key = excelFields
excelDocs.save()
} else {

const parsedUrl = new URL(databaseUrl?.url);
//Transformar a datasource con todo inicializado a vacio
const { host, port, password } = parsedUrl;
const config = {
type: "mongodb",
host: host.substring(0, host.indexOf(':')),
port: Number(port),
database: parsedUrl.pathname.substring(1),
user: parsedUrl.username,
password,
};

const mongoConnection = new MongoDBConnection(config);
const client = await mongoConnection.getclient();

try {
const database = client.db(config.database);
const collection = database.collection('xls_' + excelName);

const formatedFields = JSON.parse(JSON.stringify(excelFields));

for (const obj of formatedFields) {
for (let key in obj) {
let field: any = obj[key];

if (isNaN(Number(field))) {
const isDateValue = DateUtil.convertDate(field);

if (isDateValue) {
obj[key] = isDateValue; //{ "$date": field }
}
} else {
obj[key] = Number(field);
}
}
}

// Insertar los datos
const result = await collection.insertMany(formatedFields);
} catch (err) {
console.error('JSON to COllection Error: ', err);
throw err;
} finally {
await client.close();
}
}

await this.ExcelCollectionToDataSource(excelName, excelFields, optimize, cacheAllowed, res, next);
} catch (error) {
console.error('Error al crear o actualizar el ExcelSheet:', error);
next(new HttpException(500, 'Error al crear o actualizar el ExcelSheet'));
}
}

static async ExistsExcelData(req: Request, res: Response, next: NextFunction) {
//Checkea si hay documentos, en el nombre pasado por el frontal. Si los hay devuelve true para confirmar en el front
try {
if (!req.body?.name) return res.status(400).json({ ok: false, message: 'Nombre o campos incorrectos en la solicitud' });
const excelModelChecker = ExcelSheetModel(req.body?.name), existentExcelDoc = await excelModelChecker.find({});
if (existentExcelDoc.length > 0) return res.status(200).json({ ok: true, message: 'Modelo existe', existence: true });
return res.status(200).json({ ok: true, message: 'Modelo existe', existence: false });
} catch (error) {
console.log("Error: ", error);
return false;
}
}

static async ExcelCollectionToDataSource(excelName, excelFields, optimized, cacheAllowed, res: Response, next: NextFunction) {
try {
//Declaramos un objeto que va a contener los tipos y nombres de los campos del Excel
const propertiesAndTypes = {};
excelFields.forEach(object => {
Object.entries(object).forEach(([property, value]) => {

if (!isNaN(Number(value))) {
propertiesAndTypes[property] = 'numeric';
} else {
const isDateValue = DateUtil.convertDate(value);

if (isDateValue) {
propertiesAndTypes[property] = 'date';
} else if (typeof value === 'string') {
propertiesAndTypes[property] = 'text';
}
}

});
});
const propertiesAndTypesArray = Object.entries(propertiesAndTypes).map(([name, type]) => ({ name, type })), columnsEntry = [];
//Mapeado de las columnas
propertiesAndTypesArray.forEach((column) => {
let newCol: any = {
column_name: column.name,
column_type: String(column.type),
display_name: {
default: column.name,
localized: []
},
description: {
default: column.name,
localized: []
},
minimumFractionDigits: 0,
column_granted_roles: [],
row_granted_roles: [],
visible: true,
tableCount: 0,
valueListSource: {},
}

if (newCol.column_type === 'numeric') {
newCol.aggregation_type = AggregationTypes.getValuesForNumbers();
} else if (newCol.column_type === 'text') {
newCol.aggregation_type = AggregationTypes.getValuesForText();
} else {
newCol.aggregation_type = AggregationTypes.getValuesForOthers();
}

columnsEntry.push(newCol);
});
//Construcción del objeto table
const dsTableObject =
[
{
table_name: excelName,
display_name: {
default: excelName,
localized: []
},
description: {
default: excelName,
localized: []
},
table_granted_roles: [],
table_type: [],
columns: columnsEntry,
relations: [],
visible: true,
tableCount: 0,
no_relations: []
}
];

if (!databaseUrl?.url) return res.status(400).json({ ok: false, message: 'La connexión a la base de datos no existe' });
const parsedUrl = new URL(databaseUrl?.url);
//Transformar a datasource con todo inicializado a vacio
const database = parsedUrl.pathname.substring(1);
const { host, port, password } = parsedUrl;
const datasource: IDataSource = new DataSource({
ds: {
connection: {
type: "mongodb",
host: host.substring(0, host.indexOf(':')),
port: Number(port),
database,
schema: "public",
searchPath: "public",
user: parsedUrl.username,
password: EnCrypterService.encrypt(password),
poolLimit: null,
sid: null,
warehouse: null,
ssl: false
},
metadata: {
model_name: excelName,
model_id: "",
model_granted_roles: [],
optimized: optimized ?? false,
cache_config: {
units: "",
quantity: 1,
hours: "",
minutes: "",
enabled: cacheAllowed ?? false,
},
filter: null,
model_owner: "",
tags: [],
external: {}
},
model: {
tables: dsTableObject
}
}
});

datasource.save((err, data_source) => {
if (err) { return next(new HttpException(500, `Error saving the datasource`)); }
return res.status(201).json({ ok: true, data_source_id: data_source._id });
});
} catch (error) {
console.log("Error al parsear el excel: ", error);
throw error;
}
}

}
33 changes: 33 additions & 0 deletions eda/eda_api/lib/module/excel/excel-sheet.router.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import * as express from 'express';
import { authGuard } from '../../guards/auth-guard';
import { roleGuard } from '../../guards/role-guard';
import { ExcelSheetController } from './excel-sheet.controller';

const router = express.Router();

/**
* @openapi
* /excel-sheets/add-json-data-source
* post:
* description: Adds/updates the excel sheet object passed within the body/form, checking by the name field.
* parameters:
* - name: token
* in: path
* required: true
* type: string
* - name: excelsheet
* in: body
* required: true
* type: object
* responses:
* 200:
* description: Creation/Update of the excel sheet successfull
* 500:
* description: Error trying to create the new excel sheet
* tags:
* - Excel Sheets Routes
*/
router.post('/add-json-data-source',authGuard,roleGuard,ExcelSheetController.GenerateCollectionFromJSON);
//TODO
router.post('/existent-json-data-source',authGuard,roleGuard,ExcelSheetController.ExistsExcelData);
export default router;
15 changes: 15 additions & 0 deletions eda/eda_api/lib/module/excel/model/excel-sheet.model.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import * as mongoose from 'mongoose';

export interface IExcelSheet extends mongoose.Document {
key:any
}

const ExcelSheetSchema = new mongoose.Schema({
key:{type: Object},
});
//Función que recibe el nombre del modelo, el esquema y posteriormente el nombre de la colección donde se va a guardar el modelo
const ExcelSheetModel = (name:string) =>{
return mongoose.model<IExcelSheet>( name, ExcelSheetSchema, `xls_${name}`);
}

export default ExcelSheetModel;
6 changes: 5 additions & 1 deletion eda/eda_api/lib/router.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import AddTableRouter from './module/addtabletomodel/addtable.router';
import DataSourceRouter from './module/datasource/datasource.router';
import UploadsRouter from './module/uploads/uploads.router';
import MailRouter from './module/mail/mail.router';
import DocuRouter from './routes/api/api-docs'
import DocuRouter from './routes/api/api-docs';
import ExcelRouter from './module/excel/excel-sheet.router';
import ThirdPartyRouter from './module/thirdParty/thirdParty.router';


const router = express.Router();

router.use('/admin', AdminRouter);
Expand All @@ -22,6 +24,8 @@ router.use('/addTable', AddTableRouter );

router.use('/mail', MailRouter);

router.use('/excel-sheets',ExcelRouter);

router.use('/tp', ThirdPartyRouter);

/* ruta per documentació*/
Expand Down
Loading

0 comments on commit a6de4e5

Please sign in to comment.