diff --git a/dbptk-core/pom.xml b/dbptk-core/pom.xml index 23c7c176..89cabff4 100644 --- a/dbptk-core/pom.xml +++ b/dbptk-core/pom.xml @@ -143,6 +143,11 @@ dbptk-module-postgresql + + com.databasepreservation + dbptk-module-normalize-1nf-config + + com.databasepreservation dbptk-module-siard diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomColumnConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomColumnConfiguration.java new file mode 100644 index 00000000..cfa3f769 --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomColumnConfiguration.java @@ -0,0 +1,46 @@ +package com.databasepreservation.model.modules.configuration; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({"name", "description", "nillable", "merkle", "inventory", "externalLOB"}) +public class CustomColumnConfiguration extends ColumnConfiguration { + private Boolean nillable; + private String description; + + public Boolean getNillable() { + return nillable; + } + + public void setNillable(Boolean nillable) { + this.nillable = nillable; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + if (!super.equals(o)) + return false; + CustomColumnConfiguration that = (CustomColumnConfiguration) o; + return Objects.equals(getNillable(), that.getNillable()) && Objects.equals(getDescription(), that.getDescription()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getNillable(), getDescription()); + } +} diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomViewConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomViewConfiguration.java index 3f0eb5de..15ccb16c 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomViewConfiguration.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/CustomViewConfiguration.java @@ -7,29 +7,35 @@ */ package com.databasepreservation.model.modules.configuration; -import com.databasepreservation.Constants; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - import java.util.ArrayList; import java.util.List; import java.util.Objects; +import com.databasepreservation.Constants; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + /** * @author Miguel Guimarães */ -@JsonPropertyOrder({"name", "description", "query"}) +@JsonPropertyOrder({"name", "simulateTable", "description", "query", "columns", "primaryKey", "foreignKeys"}) +@JsonInclude(JsonInclude.Include.NON_NULL) public class CustomViewConfiguration { private String name; + private boolean simulateTable = false; private String description; private String query; - private List columns; + private List columns; + private PrimaryKeyConfiguration primaryKey; + private List foreignKeys; public CustomViewConfiguration() { name = Constants.EMPTY; columns = new ArrayList<>(); description = Constants.EMPTY; query = Constants.EMPTY; + foreignKeys = new ArrayList<>(); } public String getName() { @@ -40,6 +46,22 @@ public void setName(String name) { this.name = name; } + /** + * Should the custom view simulate a table in the archive? + *

+ * This will remove the prefix from the name. The view will still be included in + * the archive with the prefix to document the archive. + * + * @return Boolean + */ + public boolean isSimulateTable() { + return simulateTable; + } + + public void setSimulateTable(boolean simulateTable) { + this.simulateTable = simulateTable; + } + public String getDescription() { return description; } @@ -56,14 +78,30 @@ public void setQuery(String query) { this.query = query; } - public List getColumns() { + public List getColumns() { return columns; } - public void setColumns(List columns) { + public void setColumns(List columns) { this.columns = columns; } + public PrimaryKeyConfiguration getPrimaryKey() { + return primaryKey; + } + + public void setPrimaryKey(PrimaryKeyConfiguration primaryKey) { + this.primaryKey = primaryKey; + } + + public List getForeignKeys() { + return foreignKeys; + } + + public void setForeignKeys(List foreignKeys) { + this.foreignKeys = foreignKeys; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -72,7 +110,8 @@ public boolean equals(Object o) { return Objects.equals(name, that.name) && Objects.equals(description, that.description) && Objects.equals(query, that.query) && - Objects.equals(columns, that.columns); + Objects.equals(columns, that.columns) && Objects.equals(primaryKey, that.primaryKey) + && Objects.equals(foreignKeys, that.foreignKeys); } @Override diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ForeignKeyConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ForeignKeyConfiguration.java new file mode 100644 index 00000000..7718bf3b --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ForeignKeyConfiguration.java @@ -0,0 +1,65 @@ +package com.databasepreservation.model.modules.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.List; +import java.util.Objects; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ForeignKeyConfiguration { + + private String name; + private String referencedTable; + private List references; + private String description; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getReferencedTable() { + return referencedTable; + } + + public void setReferencedTable(String referencedTable) { + this.referencedTable = referencedTable; + } + + public List getReferences() { + return references; + } + + public void setReferences(List references) { + this.references = references; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ForeignKeyConfiguration that = (ForeignKeyConfiguration) o; + return Objects.equals(name, that.name) && Objects.equals(referencedTable, that.referencedTable) && Objects.equals( + references, that.references) && Objects.equals(description, that.description); + } + + @Override + public int hashCode() { + return Objects.hash(name, referencedTable, references); + } + +} diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java index 55cee5de..8ccf2882 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java @@ -8,16 +8,7 @@ package com.databasepreservation.model.modules.configuration; import static com.databasepreservation.Constants.VIEW_NAME_PREFIX; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CANDIDATE_KEYS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CHECK_CONSTRAINTS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.FOREIGN_KEYS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIMARY_KEYS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIVILEGES; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROLES; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROUTINES; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.TRIGGERS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.USERS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.VIEWS; +import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.*; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -26,19 +17,20 @@ import com.databasepreservation.Constants; import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.*; /** * @author Miguel Guimarães */ @JsonPropertyOrder({"import", "schemas", "ignore"}) @JsonIgnoreProperties(value = {"fetchRows"}) +@JsonInclude(JsonInclude.Include.NON_NULL) public class ModuleConfiguration { private ImportModuleConfiguration importModuleConfiguration; + // Statements to run before importing database, e.g. creating temporary tables for use in custom views exported as + // tables. + private List setupStatements; private Map schemaConfigurations; private Map ignore; private boolean fetchRows; @@ -372,6 +364,15 @@ public ImportModuleConfiguration getImportModuleConfiguration() { return importModuleConfiguration; } + @JsonProperty("setupStatements") + public List getSetupStatements() { + return setupStatements; + } + + public void setSetupStatements(List setupStatements) { + this.setupStatements = setupStatements; + } + public void setImportModuleConfiguration(ImportModuleConfiguration importModuleConfiguration) { this.importModuleConfiguration = importModuleConfiguration; } diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/PrimaryKeyConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/PrimaryKeyConfiguration.java new file mode 100644 index 00000000..34e33c25 --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/PrimaryKeyConfiguration.java @@ -0,0 +1,54 @@ +package com.databasepreservation.model.modules.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.List; +import java.util.Objects; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PrimaryKeyConfiguration { + + private String name; + private List columnNames; + private String description; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getColumnNames() { + return columnNames; + } + + public void setColumnNames(List columnNames) { + this.columnNames = columnNames; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + PrimaryKeyConfiguration that = (PrimaryKeyConfiguration) o; + return Objects.equals(name, that.name) && Objects.equals(columnNames, that.columnNames) + && Objects.equals(description, that.description); + } + + @Override + public int hashCode() { + return Objects.hash(name, columnNames, description); + } +} diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ReferenceConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ReferenceConfiguration.java new file mode 100644 index 00000000..65b729e6 --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ReferenceConfiguration.java @@ -0,0 +1,40 @@ +package com.databasepreservation.model.modules.configuration; + +import java.util.Objects; + +public class ReferenceConfiguration { + + private String column; + private String referenced; + + public String getColumn() { + return column; + } + + public void setColumn(String column) { + this.column = column; + } + + public String getReferenced() { + return referenced; + } + + public void setReferenced(String referenced) { + this.referenced = referenced; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ReferenceConfiguration that = (ReferenceConfiguration) o; + return Objects.equals(column, that.column) && Objects.equals(referenced, that.referenced); + } + + @Override + public int hashCode() { + return Objects.hash(column, referenced); + } +} diff --git a/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java b/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java index 9d2ca779..c846cdaf 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java +++ b/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java @@ -7,30 +7,11 @@ */ package com.databasepreservation.utils; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CANDIDATE_KEYS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CHECK_CONSTRAINTS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.FOREIGN_KEYS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIMARY_KEYS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIVILEGES; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROLES; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROUTINES; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.TRIGGERS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.USERS; -import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.VIEWS; - -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import com.databasepreservation.Constants; -import com.databasepreservation.model.modules.configuration.ColumnConfiguration; -import com.databasepreservation.model.modules.configuration.CustomViewConfiguration; -import com.databasepreservation.model.modules.configuration.ImportModuleConfiguration; -import com.databasepreservation.model.modules.configuration.ModuleConfiguration; -import com.databasepreservation.model.modules.configuration.SchemaConfiguration; -import com.databasepreservation.model.modules.configuration.TableConfiguration; -import com.databasepreservation.model.modules.configuration.ViewConfiguration; +import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.*; + +import java.util.*; + +import com.databasepreservation.model.modules.configuration.*; import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures; import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.TableStructure; @@ -55,7 +36,9 @@ public static ModuleConfiguration getDefaultModuleConfiguration() { } public static void addCustomViewConfiguration(ModuleConfiguration moduleConfiguration, String schemaName, String name, - String description, String query) { + boolean simulateTable, String description, String query, List columns, + PrimaryKeyConfiguration primaryKey, List foreignKeys) { + SchemaConfiguration schemaConfiguration = moduleConfiguration.getSchemaConfigurations().get(schemaName); if (schemaConfiguration == null) { schemaConfiguration = new SchemaConfiguration(); @@ -63,8 +46,12 @@ public static void addCustomViewConfiguration(ModuleConfiguration moduleConfigur CustomViewConfiguration customViewConfiguration = new CustomViewConfiguration(); customViewConfiguration.setName(name); + customViewConfiguration.setSimulateTable(simulateTable); customViewConfiguration.setDescription(description); customViewConfiguration.setQuery(query); + customViewConfiguration.setColumns(columns); + customViewConfiguration.setPrimaryKey(primaryKey); + customViewConfiguration.setForeignKeys(new ArrayList<>(foreignKeys)); // Ensure that items can be added. schemaConfiguration.getCustomViewConfigurations().add(customViewConfiguration); moduleConfiguration.getSchemaConfigurations().put(schemaName, schemaConfiguration); diff --git a/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java b/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java index 9b69d5ee..45c44149 100644 --- a/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java +++ b/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java @@ -38,9 +38,10 @@ * @author Miguel Guimarães */ public class ImportConfiguration implements DatabaseFilterModule { - private DatabaseStructure dbStructure; - private SchemaStructure currentSchema; - private ModuleConfiguration moduleConfiguration; + protected DatabaseStructure dbStructure; + protected SchemaStructure currentSchema; + protected TableStructure currentTable; + protected ModuleConfiguration moduleConfiguration; private Path outputFile; private ObjectMapper mapper; @@ -119,7 +120,7 @@ public void handleDataOpenSchema(String schemaName) throws ModuleException { */ @Override public void handleDataOpenTable(String tableId) throws ModuleException { - TableStructure currentTable = dbStructure.getTableById(tableId); + currentTable = dbStructure.getTableById(tableId); if (currentTable == null) { throw new ModuleException().withMessage("Couldn't find table with id: " + tableId); } diff --git a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java index eb82545d..91ef7151 100644 --- a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java +++ b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java @@ -17,30 +17,12 @@ import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.Array; -import java.sql.Blob; -import java.sql.Connection; -import java.sql.DatabaseMetaData; +import java.sql.*; import java.sql.Date; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.sql.Statement; -import java.sql.Struct; -import java.sql.Time; -import java.sql.Timestamp; -import java.sql.Types; import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; @@ -51,55 +33,22 @@ import com.databasepreservation.Constants; import com.databasepreservation.managers.ModuleConfigurationManager; import com.databasepreservation.managers.RemoteConnectionManager; -import com.databasepreservation.model.data.ArrayCell; -import com.databasepreservation.model.data.BinaryCell; -import com.databasepreservation.model.data.Cell; -import com.databasepreservation.model.data.NullCell; -import com.databasepreservation.model.data.Row; -import com.databasepreservation.model.data.SimpleCell; +import com.databasepreservation.model.data.*; import com.databasepreservation.model.exception.InvalidDataException; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.exception.SQLParseException; import com.databasepreservation.model.exception.TableNotFoundException; import com.databasepreservation.model.modules.DatabaseImportModule; import com.databasepreservation.model.modules.DatatypeImporter; -import com.databasepreservation.model.modules.configuration.CustomViewConfiguration; -import com.databasepreservation.model.modules.configuration.ModuleConfiguration; +import com.databasepreservation.model.modules.configuration.*; import com.databasepreservation.model.modules.filters.DatabaseFilterModule; import com.databasepreservation.model.reporters.Reporter; -import com.databasepreservation.model.structure.CandidateKey; -import com.databasepreservation.model.structure.CheckConstraint; -import com.databasepreservation.model.structure.ColumnStructure; -import com.databasepreservation.model.structure.DatabaseStructure; -import com.databasepreservation.model.structure.ForeignKey; -import com.databasepreservation.model.structure.PrimaryKey; -import com.databasepreservation.model.structure.PrivilegeStructure; -import com.databasepreservation.model.structure.Reference; -import com.databasepreservation.model.structure.RoleStructure; -import com.databasepreservation.model.structure.RoutineStructure; -import com.databasepreservation.model.structure.SchemaStructure; -import com.databasepreservation.model.structure.TableStructure; -import com.databasepreservation.model.structure.Trigger; -import com.databasepreservation.model.structure.UserStructure; -import com.databasepreservation.model.structure.ViewStructure; -import com.databasepreservation.model.structure.type.ComposedTypeArray; -import com.databasepreservation.model.structure.type.ComposedTypeStructure; -import com.databasepreservation.model.structure.type.SimpleTypeBinary; -import com.databasepreservation.model.structure.type.SimpleTypeBoolean; -import com.databasepreservation.model.structure.type.SimpleTypeDateTime; -import com.databasepreservation.model.structure.type.SimpleTypeNumericApproximate; -import com.databasepreservation.model.structure.type.SimpleTypeNumericExact; -import com.databasepreservation.model.structure.type.Type; -import com.databasepreservation.model.structure.type.UnsupportedDataType; +import com.databasepreservation.model.structure.*; +import com.databasepreservation.model.structure.type.*; import com.databasepreservation.modules.DefaultExceptionNormalizer; import com.databasepreservation.modules.SQLHelper; import com.databasepreservation.modules.jdbc.JDBCModuleFactory; -import com.databasepreservation.utils.ConfigUtils; -import com.databasepreservation.utils.JodaUtils; -import com.databasepreservation.utils.MapUtils; -import com.databasepreservation.utils.MiscUtils; -import com.databasepreservation.utils.PortUtils; -import com.databasepreservation.utils.RemoteConnectionUtils; +import com.databasepreservation.utils.*; import com.jcraft.jsch.Session; /** @@ -636,14 +585,11 @@ protected List getTables(SchemaStructure schema) throws SQLExcep if (!customViewConfigurations.isEmpty()) { for (CustomViewConfiguration custom : customViewConfigurations) { - String description = custom.getDescription(); - String query = custom.getQuery(); String name = custom.getName(); LOGGER.info("Obtaining table structure for custom view {}", name); try { - TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, name, tableIndex, - description, query); + TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, tableIndex, custom); tables.add(customViewStructureAsTable); tableIndex++; } catch (SQLException e) { @@ -758,13 +704,19 @@ protected List getCustomViews(String schemaName) throws ModuleExc if (!customViewConfigurations.isEmpty()) { for (CustomViewConfiguration custom : customViewConfigurations) { + // If simulating table, do not export as view, only as table. + if (custom.isSimulateTable()) + continue; + ViewStructure view = new ViewStructure(); + // Use prefix view.setName(CUSTOM_VIEW_NAME_PREFIX + custom.getName()); view.setDescription(custom.getDescription()); view.setQueryOriginal(custom.getQuery()); try { - view.setColumns(getColumnsFromCustomView(custom.getName(), custom.getQuery())); + // Don't set primary key on views (this makes only sense in as-table context) + view.setColumns(getColumnsFromCustomView(custom.getName(), custom.getQuery(), null, null)); } catch (SQLException e) { reporter.ignored("Columns from custom view " + custom.getName() + " in schema " + schemaName, "there was a problem retrieving them form the database"); @@ -926,18 +878,29 @@ protected TableStructure getViewStructure(SchemaStructure schema, String tableNa return view; } - protected TableStructure getCustomViewStructureAsTable(SchemaStructure schema, String viewName, int tableIndex, - String description, String query) throws SQLException, ModuleException { + protected TableStructure getCustomViewStructureAsTable(SchemaStructure schema, int tableIndex, + CustomViewConfiguration custom) throws SQLException, ModuleException { + String viewName = custom.getName(); + String description = custom.getDescription(); + String query = custom.getQuery(); + PrimaryKey primaryKey = custom.getPrimaryKey() != null + ? getPrimaryKeyConfigurationAsPrimaryKey(custom.getPrimaryKey(), viewName) + : null; + + String name = (custom.isSimulateTable() ? "" : CUSTOM_VIEW_NAME_PREFIX) + viewName; + TableStructure view = new TableStructure(); - view.setId(schema.getName() + "." + CUSTOM_VIEW_NAME_PREFIX + viewName); - view.setName(CUSTOM_VIEW_NAME_PREFIX + viewName); + view.setId(schema.getName() + "." + name); + view.setName(name); view.setSchema(schema); view.setIndex(tableIndex); view.setDescription(description); - view.setColumns(getColumnsFromCustomView(viewName, query)); - view.setPrimaryKey(null); - view.setForeignKeys(new ArrayList<>()); + view.setColumns(getColumnsFromCustomView(viewName, query, custom.getPrimaryKey(), custom.getColumns())); + view.setPrimaryKey(primaryKey); + view.setForeignKeys(custom.getForeignKeys() == null ? new ArrayList<>() + : custom.getForeignKeys().stream().map(c -> getForeignKeyConfigurationAsForeignKey(c, name)) + .collect(Collectors.toList())); view.setCandidateKeys(new ArrayList<>()); view.setCheckConstraints(new ArrayList<>()); view.setTriggers(new ArrayList<>()); @@ -948,6 +911,50 @@ protected TableStructure getCustomViewStructureAsTable(SchemaStructure schema, S return view; } + private PrimaryKey getPrimaryKeyConfigurationAsPrimaryKey(PrimaryKeyConfiguration configuration, String tableName) { + PrimaryKey pk = new PrimaryKey(); + + if (configuration.getName() != null) { + pk.setName(configuration.getName()); + } else { + pk.setName(getPrimaryKeyName(tableName)); + } + + pk.setDescription(configuration.getDescription()); + pk.setColumnNames(configuration.getColumnNames()); + + return pk; + } + + private ForeignKey getForeignKeyConfigurationAsForeignKey(ForeignKeyConfiguration configuration, String tableName) { + ForeignKey fk = new ForeignKey(); + + if (configuration.getReferences() == null || configuration.getReferences().isEmpty()) { + throw new IllegalArgumentException("Empty reference list in foreignKey configuration on " + tableName); + } + + if (configuration.getName() != null) { + fk.setName(configuration.getName()); + } else { + fk.setName(getForeignKeyName(tableName, configuration.getReferences().getFirst().getColumn())); + } + + fk.setReferencedTable(configuration.getReferencedTable()); + fk.setReferences(configuration.getReferences().stream().map(this::getReferenceConfigurationAsReference).toList()); + fk.setDescription(configuration.getDescription()); + + return fk; + } + + private Reference getReferenceConfigurationAsReference(ReferenceConfiguration configuration) { + Reference ref = new Reference(); + + ref.setColumn(configuration.getColumn()); + ref.setReferenced(configuration.getReferenced()); + + return ref; + } + private int getRows(String schemaName, String tableName) throws SQLException, ModuleException { String query = sqlHelper.getRowsSQL(schemaName, tableName); LOGGER.debug("count query: {}", query); @@ -1148,8 +1155,14 @@ protected List getColumns(String schemaName, String tableName) return columns; } - protected List getColumnsFromCustomView(String viewName, String query) + protected List getColumnsFromCustomView(String viewName, String query, + PrimaryKeyConfiguration primaryKey, List columnConfigurations) throws ModuleException, SQLException { + + Map columnConfigurationMap = columnConfigurations != null + ? columnConfigurations.stream().collect(Collectors.toMap(CustomColumnConfiguration::getName, Function.identity())) + : Collections.emptyMap(); + List columns = new ArrayList<>(); try (PreparedStatement preparedStatement = getConnection().prepareStatement(query)) { @@ -1163,12 +1176,17 @@ protected List getColumnsFromCustomView(String viewName, String String columnTypeName = metaData.getColumnTypeName(i); int columnDisplaySize = metaData.getColumnDisplaySize(i); int precision = metaData.getPrecision(i); + CustomColumnConfiguration columnConfiguration = columnConfigurationMap.get(columnName); + boolean nillable = (primaryKey == null || !primaryKey.getColumnNames().contains(columnName)) + && (columnConfiguration == null || columnConfiguration.getNillable() == null + || columnConfiguration.getNillable()); + String description = columnConfiguration != null ? columnConfiguration.getDescription() : ""; Type checkedType = datatypeImporter.getCheckedType(dbStructure, actualSchema, tableName, columnName, columnType, columnTypeName, columnDisplaySize, precision, 10); - ColumnStructure column = new ColumnStructure(viewName + "." + columnName, columnName, checkedType, true, "", "", - false); + ColumnStructure column = new ColumnStructure(viewName + "." + columnName, columnName, checkedType, nillable, + description, "", false); columns.add(column); } @@ -1310,7 +1328,7 @@ protected PrimaryKey getPrimaryKey(String schemaName, String tableName) throws S } if (pkName == null) { - pkName = tableName + "_pkey"; + pkName = getPrimaryKeyName(tableName); } PrimaryKey pk = new PrimaryKey(); @@ -1319,6 +1337,10 @@ protected PrimaryKey getPrimaryKey(String schemaName, String tableName) throws S return pkColumns.isEmpty() ? null : pk; } + private static String getPrimaryKeyName(String tableName) { + return tableName + "_pkey"; + } + /** * Get the table foreign keys * @@ -1341,8 +1363,7 @@ protected List getForeignKeys(String schemaName, String tableName) t String fkeyName = rs.getString("FK_NAME"); if (fkeyName == null) { - fkeyName = "FK_" + rs.getString("PKTABLE_NAME") + "_" + rs.getString("FKTABLE_NAME") + "_" - + rs.getString("FKCOLUMN_NAME"); + fkeyName = getForeignKeyName(rs.getString("FKTABLE_NAME"), rs.getString("FKCOLUMN_NAME")); } for (ForeignKey key : foreignKeys) { @@ -1373,6 +1394,13 @@ protected List getForeignKeys(String schemaName, String tableName) t return foreignKeys; } + private String getForeignKeyName(String fkTableName, String fkColumnName) { + // The original foreign key scheme, before specifying foreign keys on custom views was implemented, included + // PKTABLE_NAME. It was only used if the database did not return any name. That scheme is vulnerable to exceeding + // name length limits in SIARD XSD's, and the fk table and column are sufficient to create a unique id. + return "FK_" + fkTableName + "_" + fkColumnName; + } + protected String getReferencedSchema(String s) throws SQLException, ModuleException { return s; } @@ -2065,11 +2093,33 @@ protected Set getIgnoredExportedSchemas() { return ignore; } + private void executeSetupStatements() throws ModuleException { + ModuleConfiguration configuration = getModuleConfiguration(); + if (configuration.getSetupStatements() == null) + return; + + int i = 0; + int count = configuration.getSetupStatements().size(); + + try { + for (String sql : configuration.getSetupStatements()) { + LOGGER.info("Executing setup statement {} of {}", ++i, count); + Statement st = getStatement(); + st.execute(sql); + } + } catch (SQLException e) { + closeConnection(); + throw new ModuleException().withCause(e).withMessage(e.getMessage()); + } + } + @Override public DatabaseFilterModule migrateDatabaseTo(DatabaseFilterModule exportModule) throws ModuleException { try { exportModule.initDatabase(); + executeSetupStatements(); + exportModule.setIgnoredSchemas(getIgnoredExportedSchemas()); exportModule.handleStructure(getDatabaseStructure()); diff --git a/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java b/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java index d0e03468..d6cb23d1 100644 --- a/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java +++ b/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java @@ -210,14 +210,11 @@ protected List getTables(SchemaStructure schema) throws SQLExcep if (!customViewConfigurations.isEmpty()) { for (CustomViewConfiguration custom : customViewConfigurations) { - String description = custom.getDescription(); - String query = custom.getQuery(); String name = custom.getName(); LOGGER.info("Obtaining table structure for custom view {}", name); try { - TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, name, tableIndex, - description, query); + TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, tableIndex, custom); tables.add(customViewStructureAsTable); tableIndex++; } catch (SQLException e) { diff --git a/dbptk-modules/dbptk-module-normalize-1nf-config/pom.xml b/dbptk-modules/dbptk-module-normalize-1nf-config/pom.xml new file mode 100644 index 00000000..3db3a97c --- /dev/null +++ b/dbptk-modules/dbptk-module-normalize-1nf-config/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + normalize-1nf-config + dbptk-module-normalize-1nf-config + 3.1.0-SNAPSHOT + + com.databasepreservation + dbptk-modules + 3.1.0-SNAPSHOT + .. + + + + + + com.databasepreservation + dbptk-model + + + com.databasepreservation + dbptk-module-import-config + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.16.1 + + + com.fasterxml.jackson.core + jackson-databind + 2.16.1 + + + diff --git a/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfiguration.java b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfiguration.java new file mode 100644 index 00000000..eef64a44 --- /dev/null +++ b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfiguration.java @@ -0,0 +1,357 @@ +package com.databasepreservation.modules.config; + +import static com.databasepreservation.modules.config.Normalize1NFConfiguration.NormalizedColumnType.ARRAY; +import static com.databasepreservation.modules.config.Normalize1NFConfiguration.NormalizedColumnType.JSON; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.Constants; +import com.databasepreservation.managers.ModuleConfigurationManager; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.modules.configuration.*; +import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures; +import com.databasepreservation.model.structure.ColumnStructure; +import com.databasepreservation.model.structure.PrimaryKey; +import com.databasepreservation.utils.MapUtils; +import com.databasepreservation.utils.ModuleConfigurationUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * @author Daniel Lundsgaard Skovenborg + */ +public class Normalize1NFConfiguration extends ImportConfiguration { + + private static final Logger LOGGER = LoggerFactory.getLogger(Normalize1NFConfiguration.class); + + // Value to use for query in merge file to exclude from normalization view + // creation. Empty string will not work because Constants.EMPTY will be assigned + // if "query" key is left out to + // override other fields. + private static final String EXCLUDE_CUSTOM_VIEW_QUERY = "--"; // + private static final String DEFAULT_ARRAY_NAME_PATTERN = "${table}__${column}"; + private static final String DEFAULT_ARRAY_FOREIGN_KEY_COLUMN_PATTERN = "${table}_${column}"; + private static final String DEFAULT_ARRAY_INDEX_COLUMN_NAME_PATTERN = "array_index"; + private static final String DEFAULT_ARRAY_ITEM_COLUMN_NAME_PATTERN = "${column}_item"; + private static final String DEFAULT_ARRAY_TABLE_ALIAS = "a"; + + private static final String DEFAULT_JSON_NAME_PATTERN = DEFAULT_ARRAY_NAME_PATTERN; + private static final String DEFAULT_JSON_FOREIGN_KEY_COLUMN_PATTERN = DEFAULT_ARRAY_FOREIGN_KEY_COLUMN_PATTERN; + + // TODO: Allow overriding all patterns. + private String foreignIdColumnDescriptionPattern; + + private String arrayNamePattern = DEFAULT_ARRAY_NAME_PATTERN; + private String arrayForeignKeyColumnPattern = DEFAULT_ARRAY_FOREIGN_KEY_COLUMN_PATTERN; + private String arrayDescriptionPattern; + private String arrayIndexColumnDescriptionPattern; + private String arrayItemColumnDescriptionPattern; + private String arrayIndexColumnNamePattern = DEFAULT_ARRAY_INDEX_COLUMN_NAME_PATTERN; + private String arrayItemColumnNamePattern = DEFAULT_ARRAY_ITEM_COLUMN_NAME_PATTERN; + private String arrayTableAlias = DEFAULT_ARRAY_TABLE_ALIAS; + + private String jsonNamePattern = DEFAULT_JSON_NAME_PATTERN; + private String jsonForeignKeyColumnPattern = DEFAULT_JSON_FOREIGN_KEY_COLUMN_PATTERN; + private String jsonDescriptionPattern; + + private final ModuleConfiguration mergeConfiguration; + private final boolean noSQLQuotes; + + public Normalize1NFConfiguration(Path outputFile, Path mergeFile, boolean noSQLQuotes, String arrayDescriptionPattern, + String jsonDescriptionPattern, String foreignIdColumnDescriptionPattern, String arrayIndexColumnDescriptionPattern, + String arrayItemColumnDescriptionPattern) + throws ModuleException { + super(outputFile); + this.noSQLQuotes = noSQLQuotes; + this.arrayDescriptionPattern = arrayDescriptionPattern; + this.jsonDescriptionPattern = jsonDescriptionPattern; + this.foreignIdColumnDescriptionPattern = foreignIdColumnDescriptionPattern; + this.arrayIndexColumnDescriptionPattern = arrayIndexColumnDescriptionPattern; + this.arrayItemColumnDescriptionPattern = arrayItemColumnDescriptionPattern; + + if (mergeFile == null) { + // Create empty configuration so that we don't have to check for null everywhere. + mergeConfiguration = new ModuleConfiguration(); + } else { + try { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + mergeConfiguration = mapper.readValue(mergeFile.toFile(), ModuleConfiguration.class); + } catch (IOException e) { + throw new ModuleException() + .withMessage("Could not read the merge configuration from file " + mergeFile.normalize().toAbsolutePath()) + .withCause(e); + } + } + } + + @Override + public void initDatabase() { + super.initDatabase(); + + ModuleConfiguration dbConfiguration = ModuleConfigurationManager.getInstance().getModuleConfiguration(); + Map ignore = dbConfiguration.getIgnore(); + ignore.put(DatabaseTechnicalFeatures.PRIMARY_KEYS, false); + dbConfiguration.setIgnore(ignore); + } + + @Override + public void handleDataOpenTable(String tableId) throws ModuleException { + super.handleDataOpenTable(tableId); + + currentTable.getColumns().forEach(this::handleColumn); + + // TODO: add support for using merge configuration to add foreign keys to table, + // e.g., foreign key from an enum column to a code table constructed with a + // custom view in the merge configuration. + } + + private void handleColumn(ColumnStructure column) { + + boolean isArray = column.getType().getSql99TypeName().endsWith("ARRAY"); + boolean isJson = column.getType().getOriginalTypeName().equalsIgnoreCase("json") + || column.getType().getOriginalTypeName().equalsIgnoreCase("jsonb"); + + if (!isArray && !isJson) + return; + + String schemaName = currentSchema.getName(); + String tableName = currentTable.getName(); + + NormalizedColumnType ncType = isArray ? ARRAY : JSON; + String columnName = column.getName(); + String viewName = formatTblCol(ncType == ARRAY ? arrayNamePattern : jsonNamePattern, tableName, columnName); + + // Remove normalized column from table configuration. + removeColumnFromConfiguration(schemaName, tableName, columnName); + + // Allow overriding creation of view (e.g., if making a manual normalization). + CustomViewConfiguration merge = mergeConfiguration.getCustomViewConfiguration(schemaName, viewName); + if (merge != null && EXCLUDE_CUSTOM_VIEW_QUERY.equals(merge.getQuery())) { + LOGGER.info("Normalization of {}.{}.{} ({}) is excluded by merge file", schemaName, tableName, columnName, + viewName); + return; + } + + addNormalizationViewConfiguration(ncType, schemaName, tableName, columnName, viewName); + mergeCustomViewConfiguration(schemaName, viewName); + } + + private void removeColumnFromConfiguration(String schemaName, String tableName, String columnName) { + // Assume no copy. + LOGGER.info("Removing non-1NF column {}.{}.{} from configuration", schemaName, tableName, columnName); + TableConfiguration tableConfiguration = moduleConfiguration.getTableConfiguration(schemaName, tableName); + List newColumns = tableConfiguration.getColumns().stream() + .filter(c -> !c.getName().equals(columnName)).collect(Collectors.toList()); + tableConfiguration.setColumns(newColumns); + } + + private void addNormalizationViewConfiguration(NormalizedColumnType ncType, String schemaName, String tableName, + String columnName, String viewName) { + + PrimaryKey primaryKey = currentTable.getPrimaryKey(); + + if (primaryKey == null) { + LOGGER.warn("Table {}.{} has no primary key. Cannot create normalization of {} column {}", ncType, schemaName, + tableName, columnName); + return; + } + + LOGGER.info("Creating normalization view of {} column {}.{}.{}", ncType, schemaName, tableName, columnName); + + String description = formatTblCol(ncType == ARRAY ? arrayDescriptionPattern : jsonDescriptionPattern, tableName, + columnName); + List columns = new ArrayList<>(); + String query = ncType == ARRAY ? getArrayNormalizationSQL(schemaName, tableName, columnName, primaryKey, columns) + : getJsonNormalizationSQL(schemaName, tableName, primaryKey, columns); + PrimaryKeyConfiguration primaryKeyConfiguration = getPrimaryKeyConfiguration(ncType, primaryKey, tableName, + columnName); + ForeignKeyConfiguration foreignKeyConfiguration = getForeignKeyConfiguration(ncType, primaryKey, tableName); + + ModuleConfigurationUtils.addCustomViewConfiguration(moduleConfiguration, schemaName, viewName, true, description, + query, columns, primaryKeyConfiguration, Collections.singletonList(foreignKeyConfiguration)); + } + + @Override + public void finishDatabase() throws ModuleException { + // Add all custom views from the merge configuration if not present. + moduleConfiguration.getSchemaConfigurations().forEach((schemaName, schemaConfiguration) -> { + SchemaConfiguration schemaToMerge = mergeConfiguration.getSchemaConfigurations().get(schemaName); + if (schemaToMerge == null) + return; + + List customViewConfigurations = schemaConfiguration.getCustomViewConfigurations(); + + for (CustomViewConfiguration custom : schemaToMerge.getCustomViewConfigurations()) { + if (!EXCLUDE_CUSTOM_VIEW_QUERY.equals(custom.getQuery()) + && moduleConfiguration.getCustomViewConfiguration(schemaName, custom.getName()) == null) { + LOGGER.info("Adding custom view {}.{} from merge configuration file", schemaName, custom.getName()); + customViewConfigurations.add(custom); + } + } + + customViewConfigurations.sort(Comparator.comparing(CustomViewConfiguration::getName)); + }); + + super.finishDatabase(); + } + + private void mergeCustomViewConfiguration(String schemaName, String viewName) { + CustomViewConfiguration merge = mergeConfiguration.getCustomViewConfiguration(schemaName, viewName); + if (merge == null) + return; + + LOGGER.info("Merging configuration of custom view {}.{}", schemaName, viewName); + + // Assume no copy on getters. + // Allow setting description and columns, overriding query and primary key, and + // adding foreign keys. + CustomViewConfiguration view = moduleConfiguration.getCustomViewConfiguration(schemaName, viewName); + if (merge.getDescription() != null) { + view.setDescription(merge.getDescription()); + } + if (!Constants.EMPTY.equals(merge.getQuery())) { + view.setQuery(merge.getQuery()); + } + if (!merge.getColumns().isEmpty()) { + view.setColumns(merge.getColumns()); + } + if (merge.getPrimaryKey() != null) { + view.setPrimaryKey(merge.getPrimaryKey()); + } + if (!merge.getForeignKeys().isEmpty()) { + // Add, not replace! + view.getForeignKeys().addAll(merge.getForeignKeys()); + } + } + + private static String formatTblCol(String pattern, String tableName, String columnName) { + return StringSubstitutor.replace(pattern, MapUtils.buildMapFromObjects("table", tableName, "column", columnName)); + } + + private String quoteSQL(String sqlName) { + return noSQLQuotes ? sqlName : '"' + sqlName + '"'; + } + + private String getArrayNormalizationSQL(String schemaName, String tableName, String columnName, + PrimaryKey primaryKey, List columns) { + + // Resulting SQL is only tested in PostgreSQL, but "UNNEST ... WITH ORDINALITY" should be standard SQL. + String indexColumnName = formatTblCol(arrayIndexColumnNamePattern, tableName, columnName); + String itemColumnName = formatTblCol(arrayItemColumnNamePattern, tableName, columnName); + String qColumnName = quoteSQL(columnName); + String qIndexColumnName = quoteSQL(indexColumnName); + String qItemColumnName = quoteSQL(itemColumnName); + + StringBuilder sb = getNormalizationSQLStringBuilder(ARRAY, tableName, primaryKey, columns).append(", ") + .append(qIndexColumnName).append(", ").append(qItemColumnName).append(" "); + addNormalizationSQLFrom(sb, schemaName, tableName) // + .append(" cross join unnest(").append(qColumnName).append(") with ordinality as ") // + .append(arrayTableAlias).append("(").append(qItemColumnName).append(", ").append(qIndexColumnName).append(")"); + + addCustomColumnConfiguration(columns, indexColumnName, null, + formatTblCol(arrayIndexColumnDescriptionPattern, tableName, columnName)); + // Item nullability defaults to false (assume no null items in array). + addCustomColumnConfiguration(columns, itemColumnName, false, + formatTblCol(arrayItemColumnDescriptionPattern, tableName, columnName)); + + return sb.toString(); + } + + private String getJsonNormalizationSQL(String schemaName, String tableName, PrimaryKey primaryKey, + List columns) { + // Get a template only; do not attempt to calculate columns which would require processing all rows in the table. + StringBuilder sb = getNormalizationSQLStringBuilder(JSON, tableName, primaryKey, columns).append(" "); + addNormalizationSQLFrom(sb, schemaName, tableName); + + return sb.toString(); + } + + private StringBuilder getNormalizationSQLStringBuilder(NormalizedColumnType ncType, String tableName, + PrimaryKey primaryKey, List columns) { + + StringBuilder sb = new StringBuilder("select"); + boolean first = true; + + for (String pkColumnName : primaryKey.getColumnNames()) { + String name = formatTblCol(ncType == ARRAY ? arrayForeignKeyColumnPattern : jsonForeignKeyColumnPattern, + tableName, pkColumnName); + sb.append(first ? " " : ", ").append(pkColumnName).append(" as ").append(quoteSQL(name)); + addCustomColumnConfiguration(columns, name, null, + formatTblCol(foreignIdColumnDescriptionPattern, tableName, pkColumnName)); + first = false; + } + + return sb; + } + + private StringBuilder addNormalizationSQLFrom(StringBuilder sb, String schemaName, String tableName) { + String qSchemaName = quoteSQL(schemaName); + String qTableName = quoteSQL(tableName); + + sb.append("from ").append(qSchemaName).append(".").append(qTableName); + + return sb; + } + + private PrimaryKeyConfiguration getPrimaryKeyConfiguration(NormalizedColumnType ncType, PrimaryKey primaryKey, + String tableName, + String columnName) { + PrimaryKeyConfiguration primaryKeyConfiguration = new PrimaryKeyConfiguration(); + List columnNames = new ArrayList<>(2); + + for (String pkColumnName : primaryKey.getColumnNames()) { + columnNames.add(formatTblCol(ncType == ARRAY ? arrayForeignKeyColumnPattern : jsonForeignKeyColumnPattern, + tableName, pkColumnName)); + } + + if (ncType == ARRAY) + columnNames.add(formatTblCol(arrayIndexColumnNamePattern, tableName, columnName)); + + primaryKeyConfiguration.setColumnNames(columnNames); + + return primaryKeyConfiguration; + } + + private ForeignKeyConfiguration getForeignKeyConfiguration(NormalizedColumnType ncType, PrimaryKey primaryKey, + String tableName) { + ForeignKeyConfiguration foreignKeyConfiguration = new ForeignKeyConfiguration(); + + foreignKeyConfiguration.setReferencedTable(tableName); + + List refererences = new ArrayList<>(2); + + for (String pkColumnName : primaryKey.getColumnNames()) { + ReferenceConfiguration ref = new ReferenceConfiguration(); + ref.setColumn(formatTblCol(ncType == ARRAY ? arrayForeignKeyColumnPattern : jsonForeignKeyColumnPattern, + tableName, pkColumnName)); + ref.setReferenced(pkColumnName); + refererences.add(ref); + } + foreignKeyConfiguration.setReferences(refererences); + + return foreignKeyConfiguration; + } + + private void addCustomColumnConfiguration(List list, String name, Boolean nillable, + String description) { + + CustomColumnConfiguration customColumnConfiguration = new CustomColumnConfiguration(); + customColumnConfiguration.setName(name); + customColumnConfiguration.setMerkle(false); + customColumnConfiguration.setNillable(nillable); + customColumnConfiguration.setDescription(description); + + list.add(customColumnConfiguration); + } + + enum NormalizedColumnType { + ARRAY, JSON + } +} diff --git a/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfigurationModuleFactory.java b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfigurationModuleFactory.java new file mode 100644 index 00000000..e6c421c7 --- /dev/null +++ b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfigurationModuleFactory.java @@ -0,0 +1,176 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/db-preservation-toolkit + */ +package com.databasepreservation.modules.config; + +import static java.util.stream.Collectors.toMap; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import com.databasepreservation.managers.ModuleConfigurationManager; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.exception.UnsupportedModuleException; +import com.databasepreservation.model.modules.DatabaseImportModule; +import com.databasepreservation.model.modules.DatabaseModuleFactory; +import com.databasepreservation.model.modules.configuration.ModuleConfiguration; +import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures; +import com.databasepreservation.model.modules.filters.DatabaseFilterModule; +import com.databasepreservation.model.parameters.Parameter; +import com.databasepreservation.model.parameters.Parameters; +import com.databasepreservation.model.reporters.Reporter; +import com.databasepreservation.utils.ModuleConfigurationUtils; + +/** + * Exposes an export module which extends the import-config module to generate a + * configuration which uses custom views normalize the database to be 1NF as + * needed for SIARD DK. + * + * @author Daniel Lundsgaard Skovenborg + */ +public class Normalize1NFConfigurationModuleFactory implements DatabaseModuleFactory { + public static final String PARAMETER_FILE = "file"; + public static final String MERGE_FILE = "merge-file"; + public static final String NO_SQL_QUOTES = "no-sql-quotes"; + public static final String ARRAY_DESCRIPTION_PATTERN = "pattern-array-description"; + public static final String JSON_DESCRIPTION_PATTERN = "pattern-json-description"; + private static final String FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN = "pattern-foreign-id-column-description"; + private static final String ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN = "pattern-array-index-column-description-pattern"; + private static final String ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN = "pattern-array-item-column-description-pattern"; + + private static final String DEFAULT_ARRAY_DESCRIPTION_PATTERN = "Normalized array column ${table}.${column}"; + private static final String DEFAULT_JSON_DESCRIPTION_PATTERN = "Normalized JSON column ${table}.${column}."; + private static final String DEFAULT_FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN = "Id of ${table} row."; + private static final String DEFAULT_ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN = "Index of item in normalized array."; + private static final String DEFAULT_ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN = "Value of item in normalized array."; + + private static final Parameter file = new Parameter().shortName("f").longName(PARAMETER_FILE) + .description("Path to the import configuration file").hasArgument(true).setOptionalArgument(false).required(true); + private static final Parameter mergeFile = new Parameter().shortName("mf").longName(MERGE_FILE) + .description("Path a configuration file to merge with the output").hasArgument(true).setOptionalArgument(false) + .required(false); + private static final Parameter noSQLQuotes = new Parameter().shortName("nqc").longName(NO_SQL_QUOTES) + .description( + "Don't quote SQL identifiers in normalization view queries (use if applicable to get more readable queries)") + .hasArgument(false).required(false).valueIfNotSet("false").valueIfSet("true"); + + // TODO: Allow overriding all patterns. + private static final Parameter arrayDescriptionPattern = new Parameter().shortName("pad") + .longName(ARRAY_DESCRIPTION_PATTERN) + .description(withDefault("Pattern for description of normalized array columns.", DEFAULT_ARRAY_DESCRIPTION_PATTERN)) + .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet(DEFAULT_ARRAY_DESCRIPTION_PATTERN); + private static final Parameter jsonDescriptionPattern = new Parameter().shortName("pjd") + .longName(JSON_DESCRIPTION_PATTERN) + .description(withDefault("Pattern for description of normalized JSON columns.", DEFAULT_JSON_DESCRIPTION_PATTERN)) + .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet(DEFAULT_JSON_DESCRIPTION_PATTERN); + private static final Parameter foreignIdColumnDescriptionPattern = new Parameter().shortName("pfid") + .longName(FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN) + .description(withDefault("Pattern for description of foreign id column for normalized columns.", + DEFAULT_FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN)) + .hasArgument(true).setOptionalArgument(false).required(false) + .valueIfNotSet(DEFAULT_FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN); + private static final Parameter arrayIndexColumnDescriptionPattern = new Parameter().shortName("paicd") + .longName(ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN) + .description(withDefault("Pattern for description of array index column for normalized array columns.", + DEFAULT_ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN)) + .hasArgument(true).setOptionalArgument(false).required(false) + .valueIfNotSet(DEFAULT_ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN); + private static final Parameter arrayItemColumnDescriptionPattern = new Parameter().shortName("patcd") + .longName(ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN) + .description(withDefault("Pattern for description of array value column for normalized array columns.", + DEFAULT_ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN)) + .hasArgument(true).setOptionalArgument(false).required(false) + .valueIfNotSet(DEFAULT_ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN); + + private static final List parameters = Arrays.asList(file, mergeFile, noSQLQuotes, + arrayDescriptionPattern, jsonDescriptionPattern, foreignIdColumnDescriptionPattern, + arrayIndexColumnDescriptionPattern, arrayItemColumnDescriptionPattern); + + private static String withDefault(String description, String defaultValue) { + return String.format("%s Default: \"%s\"", description, defaultValue); + } + + @Override + public boolean producesImportModules() { + return false; + } + + @Override + public boolean producesExportModules() { + return true; + } + + @Override + public String getModuleName() { + return "normalize-1nf-config"; + } + + @Override + public boolean isEnabled() { + return true; + } + + @Override + public Map getAllParameters() { + return parameters.stream().collect(toMap(Parameter::longName, Function.identity())); + } + + @Override + public Parameters getConnectionParameters() throws UnsupportedModuleException { + throw ExceptionBuilder.UnsupportedModuleExceptionForImportModule(); + } + + @Override + public Parameters getImportModuleParameters() throws UnsupportedModuleException { + throw ExceptionBuilder.UnsupportedModuleExceptionForImportModule(); + } + + @Override + public Parameters getExportModuleParameters() { + return new Parameters(parameters, null); + } + + @Override + public DatabaseImportModule buildImportModule(Map parameters, Reporter reporter) + throws UnsupportedModuleException { + throw ExceptionBuilder.UnsupportedModuleExceptionForImportModule(); + } + + @Override + public DatabaseFilterModule buildExportModule(Map parameters, Reporter reporter) + throws ModuleException { + Path pFile = Paths.get(parameters.get(file)); + Path pMergeFile = parameters.get(mergeFile) != null ? Paths.get(parameters.get(mergeFile)) : null; + boolean pNoSQLQuotes = Boolean.parseBoolean(parameters.get(noSQLQuotes)); + + reporter.exportModuleParameters(this.getModuleName(), PARAMETER_FILE, + pFile.normalize().toAbsolutePath().toString()); + + final ModuleConfiguration defaultModuleConfiguration = ModuleConfigurationUtils.getDefaultModuleConfiguration(); + defaultModuleConfiguration.setFetchRows(false); + defaultModuleConfiguration + .setIgnore(ModuleConfigurationUtils.createIgnoreListExcept(true, DatabaseTechnicalFeatures.VIEWS)); + + ModuleConfigurationManager.getInstance().setup(defaultModuleConfiguration); + + reporter.exportModuleParameters(getModuleName(), PARAMETER_FILE, pFile.normalize().toAbsolutePath().toString()); + + return new Normalize1NFConfiguration(pFile, pMergeFile, pNoSQLQuotes, + getOrDefault(parameters, arrayDescriptionPattern), getOrDefault(parameters, jsonDescriptionPattern), + getOrDefault(parameters, foreignIdColumnDescriptionPattern), + getOrDefault(parameters, arrayIndexColumnDescriptionPattern), + getOrDefault(parameters, arrayItemColumnDescriptionPattern)); + } + + private String getOrDefault(Map parameters, Parameter parameter) { + return parameters.getOrDefault(parameter, parameter.valueIfNotSet()); + } +} diff --git a/dbptk-modules/pom.xml b/dbptk-modules/pom.xml index f5c26b76..d9e6ce23 100644 --- a/dbptk-modules/pom.xml +++ b/dbptk-modules/pom.xml @@ -29,6 +29,7 @@ dbptk-module-openedge dbptk-module-oracle dbptk-module-postgresql + dbptk-module-normalize-1nf-config dbptk-module-siard dbptk-module-sql-server dbptk-module-sybase diff --git a/pom.xml b/pom.xml index a721cb25..799e57f0 100644 --- a/pom.xml +++ b/pom.xml @@ -255,6 +255,12 @@ 3.1.0-SNAPSHOT + + com.databasepreservation + dbptk-module-normalize-1nf-config + 3.1.0-SNAPSHOT + + com.databasepreservation dbptk-module-siard