diff --git a/eda/eda_api/lib/module/dashboard/dashboard.controller.ts b/eda/eda_api/lib/module/dashboard/dashboard.controller.ts index a32382f1..efd7095e 100644 --- a/eda/eda_api/lib/module/dashboard/dashboard.controller.ts +++ b/eda/eda_api/lib/module/dashboard/dashboard.controller.ts @@ -1071,21 +1071,22 @@ export class DashboardController { } and Panel:${req.body.dashboard.panel_id} ` ) console.log(query) - console.log( - '\n-------------------------------------------------------------------------------\n' - ) + console.log('\n-------------------------------------------------------------------------------\n'); /**cached query */ - let cacheEnabled = - dataModelObject.ds.metadata.cache_config && - dataModelObject.ds.metadata.cache_config.enabled === true + let cacheEnabled = false; + // dataModelObject.ds.metadata.cache_config && + // dataModelObject.ds.metadata.cache_config.enabled === true; + + console.log('cacheEnabled', cacheEnabled) const cachedQuery = cacheEnabled ? await CachedQueryService.checkQuery(req.body.model_id, query) : null if (!cachedQuery) { - connection.client = await connection.getclient() - const getResults = await connection.execQuery(query) + connection.client = await connection.getclient(); + const getResults = await connection.execQuery(query); + console.log('getResults', getResults); let numerics = [] /** si es oracle o alguns mysql haig de fer una merda per tornar els numeros normals. */ diff --git a/eda/eda_api/lib/module/datasource/datasource.controller.ts b/eda/eda_api/lib/module/datasource/datasource.controller.ts index 79ccd598..af0d5ee8 100644 --- a/eda/eda_api/lib/module/datasource/datasource.controller.ts +++ b/eda/eda_api/lib/module/datasource/datasource.controller.ts @@ -256,10 +256,10 @@ export class DataSourceController { //aparto las relaciones ocultas para optimizar el modelo. ds.ds.model.tables.forEach(t => { - t.no_relations = t.relations.filter(r => r.visible == false) + t.no_relations = t ? t.relations.filter(r => r.visible == false) : []; }); ds.ds.model.tables.forEach(t => { - t.relations = t.relations.filter(r => r.visible !== false) + t.relations = t ? t.relations.filter(r => r.visible !== false) : []; }); /** Comprobacionde la reciprocidad de las relaciones */ ds.ds.model.tables.forEach(tabla => { diff --git a/eda/eda_api/lib/module/excel/excel-sheet.controller.ts b/eda/eda_api/lib/module/excel/excel-sheet.controller.ts new file mode 100644 index 00000000..b101f3a2 --- /dev/null +++ b/eda/eda_api/lib/module/excel/excel-sheet.controller.ts @@ -0,0 +1,233 @@ +import { NextFunction, Request, Response } from "express"; +import { HttpException } from '../global/model/index'; +import { EnCrypterService } from "../../services/encrypter/encrypter.service"; +import { MongoDBConnection } from "../../services/connection/db-systems/mongodb-connection"; +import DataSource, { IDataSource } from "../datasource/model/datasource.model"; +import ExcelSheetModel from "./model/excel-sheet.model"; +import { AggregationTypes } from "../global/model/aggregation-types"; +import { DateUtil } from "../../utils/date.util"; + +const databaseUrl = require('../../../config/database.config'); + + +export class ExcelSheetController { + static async GenerateCollectionFromJSON(req: Request, res: Response, next: NextFunction) { + return await ExcelSheetController.FromJSONToCollection(req, res, next); + } + + static async FromJSONToCollection(req: Request, res: Response, next: NextFunction) { + //Guarda una nueva colección con el nombre pasado desde el frontal, si esta ya existe sustituye los campos del excel por los nuevos. + try { + const excelName = req.body?.name, optimize = req.body?.optimize, cacheAllowed = req.body?.allowCache; + let excelFields = req.body?.fields; + + if (!excelName || !excelFields) { + return res.status(400).json({ ok: false, message: 'Nombre o campos incorrectos en la solicitud' }); + } + + const excelModel = ExcelSheetModel(excelName) + const excelDocs = await excelModel.findOne({}); + + if (excelDocs) { + excelDocs.key = excelFields + excelDocs.save() + } else { + + const parsedUrl = new URL(databaseUrl?.url); + //Transformar a datasource con todo inicializado a vacio + const { host, port, password } = parsedUrl; + const config = { + type: "mongodb", + host: host.substring(0, host.indexOf(':')), + port: Number(port), + database: parsedUrl.pathname.substring(1), + user: parsedUrl.username, + password, + }; + + const mongoConnection = new MongoDBConnection(config); + const client = await mongoConnection.getclient(); + + try { + const database = client.db(config.database); + const collection = database.collection('xls_' + excelName); + + const formatedFields = JSON.parse(JSON.stringify(excelFields)); + + for (const obj of formatedFields) { + for (let key in obj) { + let field: any = obj[key]; + + if (isNaN(Number(field))) { + const isDateValue = DateUtil.convertDate(field); + + if (isDateValue) { + obj[key] = isDateValue; //{ "$date": field } + } + } else { + obj[key] = Number(field); + } + } + } + + // Insertar los datos + const result = await collection.insertMany(formatedFields); + } catch (err) { + console.error('JSON to COllection Error: ', err); + throw err; + } finally { + await client.close(); + } + } + + await this.ExcelCollectionToDataSource(excelName, excelFields, optimize, cacheAllowed, res, next); + } catch (error) { + console.error('Error al crear o actualizar el ExcelSheet:', error); + next(new HttpException(500, 'Error al crear o actualizar el ExcelSheet')); + } + } + + static async ExistsExcelData(req: Request, res: Response, next: NextFunction) { + //Checkea si hay documentos, en el nombre pasado por el frontal. Si los hay devuelve true para confirmar en el front + try { + if (!req.body?.name) return res.status(400).json({ ok: false, message: 'Nombre o campos incorrectos en la solicitud' }); + const excelModelChecker = ExcelSheetModel(req.body?.name), existentExcelDoc = await excelModelChecker.find({}); + if (existentExcelDoc.length > 0) return res.status(200).json({ ok: true, message: 'Modelo existe', existence: true }); + return res.status(200).json({ ok: true, message: 'Modelo existe', existence: false }); + } catch (error) { + console.log("Error: ", error); + return false; + } + } + + static async ExcelCollectionToDataSource(excelName, excelFields, optimized, cacheAllowed, res: Response, next: NextFunction) { + try { + //Declaramos un objeto que va a contener los tipos y nombres de los campos del Excel + const propertiesAndTypes = {}; + excelFields.forEach(object => { + Object.entries(object).forEach(([property, value]) => { + + if (!isNaN(Number(value))) { + propertiesAndTypes[property] = 'numeric'; + } else { + const isDateValue = DateUtil.convertDate(value); + + if (isDateValue) { + propertiesAndTypes[property] = 'date'; + } else if (typeof value === 'string') { + propertiesAndTypes[property] = 'text'; + } + } + + }); + }); + const propertiesAndTypesArray = Object.entries(propertiesAndTypes).map(([name, type]) => ({ name, type })), columnsEntry = []; + //Mapeado de las columnas + propertiesAndTypesArray.forEach((column) => { + let newCol: any = { + column_name: column.name, + column_type: String(column.type), + display_name: { + default: column.name, + localized: [] + }, + description: { + default: column.name, + localized: [] + }, + minimumFractionDigits: 0, + column_granted_roles: [], + row_granted_roles: [], + visible: true, + tableCount: 0, + valueListSource: {}, + } + + if (newCol.column_type === 'numeric') { + newCol.aggregation_type = AggregationTypes.getValuesForNumbers(); + } else if (newCol.column_type === 'text') { + newCol.aggregation_type = AggregationTypes.getValuesForText(); + } else { + newCol.aggregation_type = AggregationTypes.getValuesForOthers(); + } + + columnsEntry.push(newCol); + }); + //Construcción del objeto table + const dsTableObject = + [ + { + table_name: excelName, + display_name: { + default: excelName, + localized: [] + }, + description: { + default: excelName, + localized: [] + }, + table_granted_roles: [], + table_type: [], + columns: columnsEntry, + relations: [], + visible: true, + tableCount: 0, + no_relations: [] + } + ]; + + if (!databaseUrl?.url) return res.status(400).json({ ok: false, message: 'La connexión a la base de datos no existe' }); + const parsedUrl = new URL(databaseUrl?.url); + //Transformar a datasource con todo inicializado a vacio + const database = parsedUrl.pathname.substring(1); + const { host, port, password } = parsedUrl; + const datasource: IDataSource = new DataSource({ + ds: { + connection: { + type: "mongodb", + host: host.substring(0, host.indexOf(':')), + port: Number(port), + database, + schema: "public", + searchPath: "public", + user: parsedUrl.username, + password: EnCrypterService.encrypt(password), + poolLimit: null, + sid: null, + warehouse: null, + ssl: false + }, + metadata: { + model_name: excelName, + model_id: "", + model_granted_roles: [], + optimized: optimized ?? false, + cache_config: { + units: "", + quantity: 1, + hours: "", + minutes: "", + enabled: cacheAllowed ?? false, + }, + filter: null, + model_owner: "", + tags: [], + external: {} + }, + model: { + tables: dsTableObject + } + } + }); + + datasource.save((err, data_source) => { + if (err) { return next(new HttpException(500, `Error saving the datasource`)); } + return res.status(201).json({ ok: true, data_source_id: data_source._id }); + }); + } catch (error) { + console.log("Error al parsear el excel: ", error); + throw error; + } + } + +} \ No newline at end of file diff --git a/eda/eda_api/lib/module/excel/excel-sheet.router.ts b/eda/eda_api/lib/module/excel/excel-sheet.router.ts new file mode 100644 index 00000000..85a4e3fb --- /dev/null +++ b/eda/eda_api/lib/module/excel/excel-sheet.router.ts @@ -0,0 +1,33 @@ +import * as express from 'express'; +import { authGuard } from '../../guards/auth-guard'; +import { roleGuard } from '../../guards/role-guard'; +import { ExcelSheetController } from './excel-sheet.controller'; + +const router = express.Router(); + +/** + * @openapi + * /excel-sheets/add-json-data-source + * post: + * description: Adds/updates the excel sheet object passed within the body/form, checking by the name field. + * parameters: + * - name: token + * in: path + * required: true + * type: string + * - name: excelsheet + * in: body + * required: true + * type: object + * responses: + * 200: + * description: Creation/Update of the excel sheet successfull + * 500: + * description: Error trying to create the new excel sheet + * tags: + * - Excel Sheets Routes + */ +router.post('/add-json-data-source',authGuard,roleGuard,ExcelSheetController.GenerateCollectionFromJSON); +//TODO +router.post('/existent-json-data-source',authGuard,roleGuard,ExcelSheetController.ExistsExcelData); +export default router; \ No newline at end of file diff --git a/eda/eda_api/lib/module/excel/model/excel-sheet.model.ts b/eda/eda_api/lib/module/excel/model/excel-sheet.model.ts new file mode 100644 index 00000000..911316e4 --- /dev/null +++ b/eda/eda_api/lib/module/excel/model/excel-sheet.model.ts @@ -0,0 +1,15 @@ +import * as mongoose from 'mongoose'; + +export interface IExcelSheet extends mongoose.Document { + key:any +} + +const ExcelSheetSchema = new mongoose.Schema({ + key:{type: Object}, +}); +//Función que recibe el nombre del modelo, el esquema y posteriormente el nombre de la colección donde se va a guardar el modelo +const ExcelSheetModel = (name:string) =>{ + return mongoose.model( name, ExcelSheetSchema, `xls_${name}`); +} + +export default ExcelSheetModel; \ No newline at end of file diff --git a/eda/eda_api/lib/router.ts b/eda/eda_api/lib/router.ts index dc3d6644..1da975f6 100644 --- a/eda/eda_api/lib/router.ts +++ b/eda/eda_api/lib/router.ts @@ -5,9 +5,11 @@ import AddTableRouter from './module/addtabletomodel/addtable.router'; import DataSourceRouter from './module/datasource/datasource.router'; import UploadsRouter from './module/uploads/uploads.router'; import MailRouter from './module/mail/mail.router'; -import DocuRouter from './routes/api/api-docs' +import DocuRouter from './routes/api/api-docs'; +import ExcelRouter from './module/excel/excel-sheet.router'; import ThirdPartyRouter from './module/thirdParty/thirdParty.router'; + const router = express.Router(); router.use('/admin', AdminRouter); @@ -22,6 +24,8 @@ router.use('/addTable', AddTableRouter ); router.use('/mail', MailRouter); +router.use('/excel-sheets',ExcelRouter); + router.use('/tp', ThirdPartyRouter); /* ruta per documentació*/ diff --git a/eda/eda_api/lib/services/connection/db-systems/mongodb-connection.ts b/eda/eda_api/lib/services/connection/db-systems/mongodb-connection.ts new file mode 100644 index 00000000..75f94a12 --- /dev/null +++ b/eda/eda_api/lib/services/connection/db-systems/mongodb-connection.ts @@ -0,0 +1,145 @@ +import { MongoDBBuilderService } from '../../query-builder/qb-systems/mongodb-builder-service'; +import { AbstractConnection } from '../abstract-connection'; +import { AggregationTypes } from '../../../module/global/model/aggregation-types'; +import { MongoClient } from "mongodb"; +import _ from 'lodash'; + +export class MongoDBConnection extends AbstractConnection { + + GetDefaultSchema(): string { + return 'public'; + } + + private connectUrl: string; + private queryBuilder: MongoDBBuilderService; + + async getclient() { + try { + const type = this.config.type; + const host = this.config.host; + const port = this.config.port; + const db = this.config.database; + const user = this.config.user; + const password = this.config.password; + + this.connectUrl = `${type}://${user}:${password}@${host}:${port}/${db}?authSource=${db}`; + + const options = { useNewUrlParser: true, useUnifiedTopology: true }; + const connection = await MongoClient.connect(this.connectUrl, options); + return connection; + } catch (error) { + throw error; + } + } + + async tryConnection(): Promise { + try { + this.client = await this.getclient(); + console.log('\x1b[32m%s\x1b[0m', 'Connecting to MongoDB🍃 database...\n'); + await this.client.connect(); + this.itsConnected(); + await this.client.close(); + return; + } catch (err) { + throw err; + } + } + + public async generateDataModel(optimize: number, filters: string, name?: string): Promise { + return ''; + } + + async execQuery(query: any): Promise { + const client = await this.getclient() + + try { + // db and collection + const database = client.db(this.config.database); + const collection = database.collection('xls_' + query.collectionName); + + const aggregations = query.aggregations || {}; + + // prevent to display all the fields with projection (select) + const projection = query.columns.reduce((acc: any, field: string) => { + if (aggregations['count_distinct']?.includes(field)) { + acc[field] = { $size: `$${field}` }; + } else if (_.isEmpty(query.dateProjection) && query.dateFormat[field] === 'No') { + acc[field] = { $dateToString: { format: "%Y-%m-%d", date: { $dateFromString: { dateString: `$${field}` } } } }; + } else if (!_.isEmpty(query.dateProjection) && query.dateProjection[field]) { + acc[field] = query.dateProjection[field]; + } else { + acc[field] = 1; + } + return acc; + }, {}); + + let data: any; + // Format and sort + let formatData = []; + + if (!_.isEmpty(query.dateProjection)) { + query.pipeline.unshift({ $project: projection }); + } + + query.pipeline.push({ $project: projection }); + + // Filters always before $group + if (query.filters && !_.isEmpty(query.filters)) { + query.pipeline.unshift({ $match: query.filters }); + } + + // HavingFilters always after $group + if (query.havingFilters && !_.isEmpty(query.havingFilters)) { + query.pipeline.push({ $match: query.havingFilters }); + } + + console.log("Info de la consulta: ", JSON.stringify(query.pipeline)); + + data = await collection.aggregate(query.pipeline).toArray(); + + if (data.length > 0) { + formatData = data.map(doc => { + const ordenado = query.columns.map(col => { + // Verificar si el campo está en _id (caso de agrupación) + if (doc._id && doc._id.hasOwnProperty(col)) { + return doc._id[col]; + } + // De lo contrario, usar el campo de agregación directamente + return doc[col]; + }); + return ordenado; + }); + } + return formatData; + } catch (err) { + console.error(err); + throw err; + } finally { + await client.close(); + } + } + + + async execSqlQuery(query: string): Promise { + return this.execQuery(query); + } + + override async getQueryBuilded(queryData: any, dataModel: any, user: any) { + this.queryBuilder = new MongoDBBuilderService(queryData, dataModel, user); + return this.queryBuilder.builder(); + } + + public BuildSqlQuery(queryData: any, dataModel: any, user: any): string { + return ''; + } + + public createTable(queryData: any, user: any): string { + return ''; + } + + public generateInserts(queryData: any, user: any): string { + return ''; + } + + +} diff --git a/eda/eda_api/lib/services/connection/manager-connection.service.ts b/eda/eda_api/lib/services/connection/manager-connection.service.ts index f5bf4a17..4cf42fa2 100644 --- a/eda/eda_api/lib/services/connection/manager-connection.service.ts +++ b/eda/eda_api/lib/services/connection/manager-connection.service.ts @@ -5,10 +5,11 @@ import { VerticaConnection } from './db-systems/vertica-connection'; import { MysqlConnection } from './db-systems/mysql-connection'; import { PgConnection } from './db-systems/pg-connection'; import { AbstractConnection } from './abstract-connection'; -import DataSource from '../../module/datasource/model/datasource.model'; import { EnCrypterService } from '../encrypter/encrypter.service'; import { SQLserverConnection } from './db-systems/slqserver-connection'; import { JSONWebServiceConnection } from './db-systems/json-webservice-connection'; +import { MongoDBConnection } from './db-systems/mongodb-connection'; +import DataSource from '../../module/datasource/model/datasource.model'; export const MS_CONNECTION = 'mssql', @@ -19,7 +20,8 @@ export const ORACLE_CONNECTION = 'oracle', BIGQUERY_CONNECTION = 'bigquery', SNOWFLAKE_CONNECTION = 'snowflake', - WEB_SERVICE = 'jsonwebservice' + WEB_SERVICE = 'jsonwebservice', + MONGODB_CONNECTION = 'mongodb' @@ -59,13 +61,14 @@ export class ManagerConnectionService { return new SnowflakeConnection(config); case WEB_SERVICE: return new JSONWebServiceConnection(config); + case MONGODB_CONNECTION: + return new MongoDBConnection(config); default: return null; } } static async testConnection(config: any): Promise { - switch (config.type) { case MS_CONNECTION: //return new MsConnection(config, secondary); @@ -85,6 +88,8 @@ export class ManagerConnectionService { return new SnowflakeConnection(config); case WEB_SERVICE: return new JSONWebServiceConnection(config); + case MONGODB_CONNECTION: + return new MongoDBConnection(config); default: return null; } diff --git a/eda/eda_api/lib/services/query-builder/qb-systems/mongodb-builder-service.ts b/eda/eda_api/lib/services/query-builder/qb-systems/mongodb-builder-service.ts new file mode 100644 index 00000000..a4278928 --- /dev/null +++ b/eda/eda_api/lib/services/query-builder/qb-systems/mongodb-builder-service.ts @@ -0,0 +1,304 @@ +import * as _ from 'lodash'; + + +export class MongoDBBuilderService { + + public queryTODO: any; + public dataModel: any; + public user: string; + + constructor(queryTODO: any, dataModel: any, user: any) { + this.queryTODO = queryTODO; + this.dataModel = dataModel; + this.user = user._id; + } + + public builder(): any { + try { + const collectionName = this.queryTODO.fields[0].table_id; + const fields = this.queryTODO.fields; + + const mongoQuery: any = { + collectionName, + criteria: {}, + columns: [], + aggregations: {}, + filters: [], + dateFormat: {}, + dateProjection: {} + }; + + fields.forEach((column: any) => { + mongoQuery.columns.push(column.column_name); + + if (column.column_type == 'date') { + mongoQuery.dateFormat[column.column_name] = column.format || 'No'; + } + }); + + mongoQuery.filters = this.getFilters(); + + mongoQuery.havingFilters = this.getHavingFilters(); + + const pipeline = this.getPipeline(); + mongoQuery.pipeline = pipeline?.pipeline; + mongoQuery.aggregations = pipeline?.aggregations; + mongoQuery.dateProjection = pipeline?.dateProjection; + + return mongoQuery; + } catch (err) { + console.error('Error:', err); + throw err; + } + } + + public getFilters() { + const columns = this.queryTODO.fields; + + const filters = this.queryTODO.filters.filter((f: any) => { + const column = columns.find((c: any) => f.filter_table == c.table_id && f.filter_column == c.column_name); + f.column_type = column?.column_type || 'text'; + + if (column && column?.aggregation_type && column?.aggregation_type === 'none') { + return true; + } else { + return false; + } + }); + + if (filters.length > 0) { + return this.formatFilter(filters); + } else { + return null; + } + + } + + public getHavingFilters() { + const columns = this.queryTODO.fields; + //TO HAVING CLAUSE + const havingFilters = this.queryTODO.filters.filter((f: any) => { + const column = columns.find((c: any) => c.table_id === f.filter_table && f.filter_column === c.column_name); + f.column_type = column?.column_type || 'text'; + + if (column && column?.column_type == 'numeric' && column?.aggregation_type !== 'none') { + return true; + } else { + return false; + } + }); + + if (havingFilters.length > 0) { + return this.formatFilter(havingFilters); + } else { + return null; + } + } + + + public formatFilter(filters: any[]) { + const formatedFilter = { + $and: [] + }; + + for (const filter of filters) { + // if (['=', '!=', '>', '<', '<=', '>=', 'like', 'not_like'].includes(filter)) return 0; + // else if (['not_in', 'in'].includes(filter)) return 1; + // else if (filter === 'between') return 2; + // else if (filter === 'not_null') return 3; + + const filterType = filter.filter_type; + const columnType = filter.column_type; + + if (!['not_null'].includes(filterType)) { + + const value = filter.filter_elements[0].value1; + const firstValue = columnType == 'numeric' ? Number(value[0]) : value[0]; + + if (filterType == '=') { + formatedFilter['$and'].push({ [filter.filter_column]: firstValue }) + } else if (filterType == '!=') { + formatedFilter['$and'].push({ [filter.filter_column]: { $ne: firstValue } }); + } else if (filterType == '>') { + formatedFilter['$and'].push({ [filter.filter_column]: { $gt: firstValue } }); + } else if (filterType == '<') { + formatedFilter['$and'].push({ [filter.filter_column]: { $lt: firstValue } }); + } else if (filterType == '>=') { + formatedFilter['$and'].push({ [filter.filter_column]: { $gte: firstValue } }); + } else if (filterType == '<=') { + formatedFilter['$and'].push({ [filter.filter_column]: { $lte: firstValue } }); + } else if (filterType == 'like') { + formatedFilter['$and'].push({ [filter.filter_column]: { $regex: firstValue, $options: 'i' } }); + } else if (filterType == 'not_like') { + formatedFilter['$and'].push({ [filter.filter_column]: { $not: { $regex: firstValue, $options: 'i' } } }); + } + + if (filterType == 'in') { + formatedFilter['$and'].push({ [filter.filter_column]: { $in: value } }); + } else if (filterType == 'not_in') { + formatedFilter['$and'].push({ [filter.filter_column]: { $not: { $in: value } } }); + } + + if (filterType == 'between') { + const value2 = filter.filter_elements[1].value2; + const secondValue = columnType == 'numeric' ? Number(value2[0]) : value2[0]; + + formatedFilter['$and'].push({ [filter.filter_column]: { $gte: firstValue, $lte: secondValue } }); + } + + } else { + if (filterType == 'not_null') { + formatedFilter['$and'].push({ [filter.filter_column]: { $exists: true, $ne: null } }); + } + } + + } + + return formatedFilter; + } + + public getPipeline() { + const fields = this.queryTODO.fields; + const pipeline = { + $group: { + _id: {} + } + }; + + const agg = { + 'sum': '$sum', + 'avg': '$avg', + 'max': '$max', + 'min': '$min', + 'count': '$sum', + 'count_distinct': '$addToSet', + 'none': '', + }; + + const aggregations = {}; + const dateProjection = {}; + for (const column of fields) { + if (column.aggregation_type !== 'none') { + + if (column.aggregation_type == 'count') { + pipeline['$group'][column.column_name] = { '$sum': 1 }; + } else { + pipeline['$group'][column.column_name] = { [agg[column.aggregation_type]]: `$${column.column_name}` }; + } + + aggregations[column.aggregation_type] = aggregations[column.aggregation_type] || []; + aggregations[column.aggregation_type].push(column.column_name); + } else { + + if (column.column_type === 'date') { + const format = column.format || 'No'; + + if (format == 'year') { + pipeline['$group']._id[column.column_name] = `$${column.column_name}`; + dateProjection[column.column_name] = { $year: `$${column.column_name}` }; + } else if (format == 'month') { + pipeline['$group']._id[column.column_name] = `$${column.column_name}`; + dateProjection[column.column_name] = { $dateToString: { format: "%Y-%m", date: `$${column.column_name}` } }; + } else if (format == 'No') { + pipeline['$group']._id[column.column_name] = `$${column.column_name}`; + dateProjection[column.column_name] = { $dateToString: { format: "%Y-%m-%d", date: `$${column.column_name}` } }; //{ format: "%Y-%m-%d", date: `$${column.column_name}` }; + } + + } else if (fields.length > 1 || column.column_type != 'numeric') { + pipeline['$group']._id[column.column_name] = `$${column.column_name}`; + } + } + } + + return { + aggregations, + dateProjection, + pipeline: [pipeline] + } + } + + public sqlBuilder(userQuery: any, filters: any[]): string { + let sql_query = ''; + + return sql_query; + } + + + public normalQuery(columns: string[], origin: string, dest: any[], joinTree: any[], grouping: any[], filters: any[], havingFilters: any[], + tables: Array, limit: number, joinType: string, valueListJoins: Array, schema: string, database: string, forSelector: any) { + + } + + /** + * + * @param filter filter element + * @param columnType column type + * @returns firght side of the filter + */ + + public processFilter(filter: any, columnType: string) { + filter = filter.map(elem => { + if (elem === null || elem === undefined) return 'ihatenulos'; + else return elem; + }); + if (!Array.isArray(filter)) { + switch (columnType) { + case 'text': return `'${filter}'`; + //case 'text': return `'${filter}'`; + case 'numeric': return filter; + case 'date': return `to_date('${filter}','YYYY-MM-DD')` + } + } else { + let str = ''; + filter.forEach(value => { + const tail = columnType === 'date' + ? `to_date('${value}','YYYY-MM-DD')` + : columnType === 'numeric' ? value : `'${String(value).replace(/'/g, "''")}'`; + str = str + tail + ',' + }); + + // En el cas dels filtres de seguretat si l'usuari no pot veure res.... + filter.forEach(f => { + if (f == '(x => None)') { + switch (columnType) { + case 'text': str = `'(x => None)' `; break; + case 'numeric': str = 'null '; break; + case 'date': str = `to_date('4092-01-01','YYYY-MM-DD') `; break; + } + } + }); + + return str.substring(0, str.length - 1); + } + } + + + + /** this funciton is done to get the end of a date time range 2010-01-01 23:59:59 */ + public processFilterEndRange(filter: any, columnType: string) { + filter = filter.map(elem => { + if (elem === null || elem === undefined) return 'ihatenulos'; + else return elem; + }); + if (!Array.isArray(filter)) { + switch (columnType) { + case 'text': return `'${filter}'`; + //case 'text': return `'${filter}'`; + case 'numeric': return filter; + case 'date': return `to_timestamp('${filter} 23:59:59','YYYY-MM-DD HH24:MI:SS')` + } + } else { + let str = ''; + filter.forEach(value => { + const tail = columnType === 'date' + ? `to_timestamp('${value} 23:59:59','YYYY-MM-DD HH24:MI:SS')` + : columnType === 'numeric' ? value : `'${String(value).replace(/'/g, "''")}'`; + str = str + tail + ',' + }); + return str.substring(0, str.length - 1); + } + } + +} + + diff --git a/eda/eda_api/lib/utils/date.util.ts b/eda/eda_api/lib/utils/date.util.ts new file mode 100644 index 00000000..ee0ed0d8 --- /dev/null +++ b/eda/eda_api/lib/utils/date.util.ts @@ -0,0 +1,30 @@ + + +export class DateUtil { + + + // Función para validar y convertir dates + public static isValidDate(date: string, formato: string) { + try { + const [day, month, year] = date.split(formato.includes('/') ? '/' : '-'); + const formatDate: any = new Date(Number(year), Number(month) - 1, Number(day)); + return formatDate instanceof Date && !isNaN(Number(formatDate)); + } catch (err) { + console.error('isValidDateError:', date); + } + } + + public static convertDate(date: any) { + if (DateUtil.isValidDate(date, 'DD/MM/YYYY')) { + const [year, month, day] = date.split('/').reverse() + return new Date(Number(year), Number(month) - 1, Number(day)); + } else if (DateUtil.isValidDate(date, 'YYYY-MM-DD')) { + const [year, month, day] = date.split('-'); + return new Date(Number(year), Number(month) - 1, Number(day)); + } else { + return null; + } + + } + +} \ No newline at end of file diff --git a/eda/eda_app/src/app/module/components/eda-panels/eda-blank-panel/column-dialog/column-dialog.component.ts b/eda/eda_app/src/app/module/components/eda-panels/eda-blank-panel/column-dialog/column-dialog.component.ts index 5f7fbc94..6b824393 100644 --- a/eda/eda_app/src/app/module/components/eda-panels/eda-blank-panel/column-dialog/column-dialog.component.ts +++ b/eda/eda_app/src/app/module/components/eda-panels/eda-blank-panel/column-dialog/column-dialog.component.ts @@ -83,7 +83,7 @@ export class ColumnDialogComponent extends EdaDialogAbstract { const title = this.selectedColumn.display_name.default; const col = $localize`:@@col:Atributo`, from = $localize`:@@table:de la entidad`; this.dialog.title = `${col} ${title} ${from} ${this.controller.params.table}`; - + console.log(this.controller); this.carregarValidacions(); const columnType = this.selectedColumn.column_type; diff --git a/eda/eda_app/src/app/module/pages/data-sources/data-source-detail/data-source-detail.component.ts b/eda/eda_app/src/app/module/pages/data-sources/data-source-detail/data-source-detail.component.ts index 4ebac30e..b816cb71 100644 --- a/eda/eda_app/src/app/module/pages/data-sources/data-source-detail/data-source-detail.component.ts +++ b/eda/eda_app/src/app/module/pages/data-sources/data-source-detail/data-source-detail.component.ts @@ -102,7 +102,8 @@ export class DataSourceDetailComponent implements OnInit, OnDestroy { { label: 'Oracle', value: 'oracle' }, { label: 'BigQuery', value: 'bigquery' }, { label: 'SnowFlake', value: 'snowflake'}, - { label: 'JsonWebService', value: 'jsonwebservice'} + { label: 'JsonWebService', value: 'jsonwebservice'}, + { label: 'Mongo', value: 'mongodb'} ]; public selectedTipoBD: SelectItem; diff --git a/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.html b/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.html index 7c652c03..6f2fc015 100644 --- a/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.html +++ b/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.html @@ -37,7 +37,7 @@ -
@@ -45,7 +45,7 @@
-
+
@@ -147,8 +147,14 @@
- - +
+
+ + + +
+
-
\ No newline at end of file + + diff --git a/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.ts b/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.ts index 369702b1..c2f72666 100644 --- a/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.ts +++ b/eda/eda_app/src/app/module/pages/data-sources/dsconfig-wrapper.component.ts @@ -1,8 +1,9 @@ -import { Component, OnInit, ViewChild } from '@angular/core'; +import { Component, ElementRef, Input, OnInit, ViewChild } from '@angular/core'; import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; -import { SidebarService, DataSourceService, SpinnerService, AlertService, StyleProviderService } from '@eda/services/service.index'; +import { SidebarService, DataSourceService, SpinnerService, AlertService, StyleProviderService, ExcelFormatterService } from '@eda/services/service.index'; import { UploadFileComponent } from './data-source-detail/upload-file/upload-file.component'; +import { ConfirmationService } from 'primeng/api'; import Swal from 'sweetalert2'; @Component({ @@ -13,7 +14,7 @@ import Swal from 'sweetalert2'; export class DsConfigWrapperComponent implements OnInit { @ViewChild('fileUploader', { static: false }) fileUploader: UploadFileComponent; - + @ViewChild('excelFile', { static: false }) excelFile: ElementRef; public dbTypes: any[] = [ { name: 'Postgres', value: 'postgres' }, { name: 'Sql Server', value: 'sqlserver' }, @@ -22,7 +23,9 @@ export class DsConfigWrapperComponent implements OnInit { { name: 'Oracle', value: 'oracle' }, { name: 'BigQuery', value: 'bigquery' }, { name: 'SnowFlake', value: 'snowflake' }, - { name: 'jsonWebService', value: 'jsonwebservice' } + { name: 'jsonWebService', value: 'jsonwebservice' }, + { name: 'Excel', value: 'excel' } + ]; public sidOpts: any[] = [ @@ -38,10 +41,16 @@ export class DsConfigWrapperComponent implements OnInit { public allowCacheSTR: string = $localize`:@@allowCache: Habilitar caché`; public filterTooltip: string = $localize`:@@filterTooltip:Puedes añadir palabras separadas por comas, que se aplicarán como filtros de tipo LIKE a la hora de recuperar las tablas de tu base de datos`; public allowSSLSTR: string = $localize`:@@allowSSL:Conexión mediante SSL`; + public excelFileName:string = ""; public optimize: boolean = true; public allowCache: boolean = true; public ssl: boolean; private project_id: string; + public canBeClosed = false; + public uploading = false; + public uploadSuccessful = false; + public excelFileData:JSON[] = []; + @@ -52,7 +61,10 @@ export class DsConfigWrapperComponent implements OnInit { private spinnerService: SpinnerService, private alertService: AlertService, private router: Router, - public styleProviderService: StyleProviderService) { + public styleProviderService: StyleProviderService, + private excelFormatterService:ExcelFormatterService, + private confirmationService:ConfirmationService + ) { this.form = this.formBuilder.group({ name: [null, Validators.required], @@ -79,19 +91,29 @@ export class DsConfigWrapperComponent implements OnInit { } - switchTypes() { - - if (this.form.invalid) { - this.alertService.addError('Formulario incorrecto, revise los campos'); - } - else if (this.form.value.type.value !== 'bigquery') { - this.addDataSource(); - - } - else { - this.addBigQueryDataSource(); - } - } + async switchTypes() { + const type = this.form.value.type.value; + if (this.form.invalid) { this.alertService.addError('Formulario incorrecto, revise los campos'); } + else if (!['bigquery', 'excel'].includes(type)) { + this.addDataSource(); + } else if (type === 'excel') { + if (!this.form.value?.name) this.alertService.addError("No name provided"); + const checker = await this.checkExcelCollection(); + if (checker.existence) { + this.confirmationService.confirm({ + message: $localize`:@@confirmationExcelMessage:¿Estás seguro de que quieres sobreescribir este modelo de datos?`, + header: $localize`:@@confirmationExcel:Confirmación`, + acceptLabel: $localize`:@@si:Si`, + rejectLabel: $localize`:@@no:No`, + icon: 'pi pi-exclamation-triangle', + accept: () => this.sendJSONCollection(), + }) + } else this.sendJSONCollection(); + } + else { + this.addBigQueryDataSource(); + } + } public async addBigQueryDataSource(): Promise { this.spinnerService.on(); @@ -164,6 +186,42 @@ export class DsConfigWrapperComponent implements OnInit { } } + public async sendJSONCollection(): Promise { + this.spinnerService.on(); + if(!this.form.value?.name) this.alertService.addError("No name provided"); + if(Object.keys(this.excelFileData).length > 0 ){ + try { + const fileData = + { + name: this.form.value?.name, + fields: this.excelFileData, + optimize:this.form.value?.optimize, + allowCache: this.form.value?.allowCache + }; + const res = await this.excelFormatterService.addNewCollectionFromJSON(fileData).toPromise(); + + this.spinnerService.off(); + this.alertService.addSuccess($localize`:@@CollectionText:Colección creada correctamente`,); + this.router.navigate(['/data-source/', res.data_source_id]); + } catch (err) { + this.spinnerService.off(); + this.alertService.addError(err); + throw err; + } + } + } + + public async checkExcelCollection():Promise{ + try{ + const nameData = { name:this.form.value?.name} + const existenceCheck = await this.excelFormatterService.checkExistenceFromJSON(nameData).toPromise(); + return existenceCheck; + }catch(error){ + console.log(error); + return false; + } + } + selectDefaultPort() { const type = this.form.value.type.value; switch (type) { @@ -176,7 +234,7 @@ export class DsConfigWrapperComponent implements OnInit { case 'sqlserver': this.form.patchValue({ port: 1433 }); break; - case 'mongo': + case 'mongo' && 'excel': this.form.patchValue({ port: 27017 }); break; case 'mysql': @@ -210,5 +268,26 @@ export class DsConfigWrapperComponent implements OnInit { } + addExcelFile() { + this.excelFile.nativeElement.value = ""; + this.excelFile.nativeElement.click(); + } + + async excelFileLoaded(event: any) { + const file = event.target.files[0]; + + if (file) { + this.excelFileName = file.name; + try { + const jsonData = await this.excelFormatterService.readExcelToJson(file); + + jsonData === null ? this.alertService.addError($localize`:@@ErrorExcel:Cargue un archivo .xls o .xlsx`) : this.excelFileData = jsonData; + } catch (error) { + console.error('Error al leer el archivo Excel:', error); + } + } + } + + } \ No newline at end of file diff --git a/eda/eda_app/src/app/services/api/excel-formatter.service.ts b/eda/eda_app/src/app/services/api/excel-formatter.service.ts new file mode 100644 index 00000000..6cbcb4f4 --- /dev/null +++ b/eda/eda_app/src/app/services/api/excel-formatter.service.ts @@ -0,0 +1,62 @@ +import { Injectable } from '@angular/core'; +import * as XLSX from 'xlsx'; +import * as _ from 'lodash'; +import { Observable } from 'rxjs/internal/Observable'; +import { ApiService } from './api.service'; +import { HttpClient } from '@angular/common/http'; + + +@Injectable({ + providedIn: 'root' +}) +export class ExcelFormatterService extends ApiService { + + constructor(protected http: HttpClient) { + super(http); + } + private globalExcelRoute = '/excel-sheets'; + /** + * Reads an Excel file and converts the data to JSON. + * @param filePath The path to the Excel file. + */ + async readExcelToJson(file: File): Promise { + if (!file.name.endsWith('.xls') && !file.name.endsWith('.xlsx')) return null; + + return new Promise((resolve, reject) => { + const fileReader = new FileReader(); + fileReader.onload = (event) => { + const data = event.target.result; + const workbook: XLSX.WorkBook = XLSX.read(data, { type: 'binary', cellDates: true }); + + console.log('workobook', workbook); + const sheetName: string = workbook.SheetNames[0]; + const sheet: XLSX.WorkSheet = workbook.Sheets[sheetName]; + + const jsonData: JSON[] = XLSX.utils.sheet_to_json(sheet, { raw:false, dateNF:'yyyy-mm-dd' }); + console.log('jsonData -->', jsonData) + resolve(jsonData); + }; + fileReader.onerror = (error) => { + console.error('Error reading file', error); + reject(error); + }; + fileReader.readAsArrayBuffer(file); + }); + } + /** + * Receives a json and sends it to given route + * @param json + */ + addNewCollectionFromJSON(jsonData): Observable { + return this.post(`${this.globalExcelRoute}/add-json-data-source`, jsonData); + } + + /** + * Receives a name and checks the existence of given json name + * @param json + */ + checkExistenceFromJSON(nameData): Observable { + return this.post(`${this.globalExcelRoute}/existent-json-data-source`, nameData); + } + +} diff --git a/eda/eda_app/src/app/services/service.index.ts b/eda/eda_app/src/app/services/service.index.ts index ce97f343..ed40184d 100644 --- a/eda/eda_app/src/app/services/service.index.ts +++ b/eda/eda_app/src/app/services/service.index.ts @@ -30,6 +30,7 @@ export * from './api/global-filters.service'; // Global filter export * from './api/group.service'; // Group export * from './api/createTable.service'; export * from './api/mail.service'; +export * from './api/excel-formatter.service'; diff --git a/eda/eda_app/src/app/services/utils/chart-utils.service.ts b/eda/eda_app/src/app/services/utils/chart-utils.service.ts index 171292dd..472b7ee7 100644 --- a/eda/eda_app/src/app/services/utils/chart-utils.service.ts +++ b/eda/eda_app/src/app/services/utils/chart-utils.service.ts @@ -108,6 +108,7 @@ export class ChartUtilsService { { display_name: $localize`:@@dates7:FECHA COMPLETA`, value: 'timestamp', selected: false }, { display_name: $localize`:@@dates4:NO`, value: 'No', selected: false } ]; + public histoGramRangesTxt: string = $localize`:@@histoGramRangesTxt:Rango`; diff --git a/eda/eda_app/src/locale/messages.ca.xlf b/eda/eda_app/src/locale/messages.ca.xlf index d0d2b77f..54101e52 100644 --- a/eda/eda_app/src/locale/messages.ca.xlf +++ b/eda/eda_app/src/locale/messages.ca.xlf @@ -2067,6 +2067,24 @@ Entrar + +Confirmación +Confirmació + + app/module/pages/data-sources/dsconfig-wrapper.component.ts + 107 + + + + +¿Estás seguro de que quieres sobreescribir este modelo de datos? +¿Segur que vols sobreescriure aquest model de dades? + + app/module/pages/data-sources/dsconfig-wrapper.component.ts + 106 + + + Añadir Tag Afegir Tag diff --git a/eda/eda_app/src/locale/messages.en.xlf b/eda/eda_app/src/locale/messages.en.xlf index 34f7a6e7..f9d7f80b 100644 --- a/eda/eda_app/src/locale/messages.en.xlf +++ b/eda/eda_app/src/locale/messages.en.xlf @@ -2070,6 +2070,24 @@ Login + +Confirmación +Confirmation + + app/module/pages/data-sources/dsconfig-wrapper.component.ts + 107 + + + + +¿Estás seguro de que quieres sobreescribir este modelo de datos? +Are you sure you want to overwrite this data model? + + app/module/pages/data-sources/dsconfig-wrapper.component.ts + 106 + + + Este tag esta vacío This tag is empty diff --git a/eda/eda_app/src/locale/messages.pl.xlf b/eda/eda_app/src/locale/messages.pl.xlf index c680b8b5..1643b26b 100644 --- a/eda/eda_app/src/locale/messages.pl.xlf +++ b/eda/eda_app/src/locale/messages.pl.xlf @@ -2074,6 +2074,24 @@ Login + +Confirmación +Potwierdzenie + + app/module/pages/data-sources/dsconfig-wrapper.component.ts + 107 + + + + +¿Estás seguro de que quieres sobreescribir este modelo de datos? +Czy na pewno chcesz nadpisać ten model danych? + + app/module/pages/data-sources/dsconfig-wrapper.component.ts + 106 + + + Este tag esta vacío To znacznik jest pusty