foreignKeys;
public CustomViewConfiguration() {
name = Constants.EMPTY;
columns = new ArrayList<>();
description = Constants.EMPTY;
query = Constants.EMPTY;
+ foreignKeys = new ArrayList<>();
}
public String getName() {
@@ -40,6 +46,22 @@ public void setName(String name) {
this.name = name;
}
+ /**
+ * Should the custom view simulate a table in the archive?
+ *
+ * This will remove the prefix from the name. The view will still be included in
+ * the archive with the prefix to document the archive.
+ *
+ * @return Boolean
+ */
+ public boolean isSimulateTable() {
+ return simulateTable;
+ }
+
+ public void setSimulateTable(boolean simulateTable) {
+ this.simulateTable = simulateTable;
+ }
+
public String getDescription() {
return description;
}
@@ -56,14 +78,30 @@ public void setQuery(String query) {
this.query = query;
}
- public List getColumns() {
+ public List getColumns() {
return columns;
}
- public void setColumns(List columns) {
+ public void setColumns(List columns) {
this.columns = columns;
}
+ public PrimaryKeyConfiguration getPrimaryKey() {
+ return primaryKey;
+ }
+
+ public void setPrimaryKey(PrimaryKeyConfiguration primaryKey) {
+ this.primaryKey = primaryKey;
+ }
+
+ public List getForeignKeys() {
+ return foreignKeys;
+ }
+
+ public void setForeignKeys(List foreignKeys) {
+ this.foreignKeys = foreignKeys;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -72,7 +110,8 @@ public boolean equals(Object o) {
return Objects.equals(name, that.name) &&
Objects.equals(description, that.description) &&
Objects.equals(query, that.query) &&
- Objects.equals(columns, that.columns);
+ Objects.equals(columns, that.columns) && Objects.equals(primaryKey, that.primaryKey)
+ && Objects.equals(foreignKeys, that.foreignKeys);
}
@Override
diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ForeignKeyConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ForeignKeyConfiguration.java
new file mode 100644
index 00000000..7718bf3b
--- /dev/null
+++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ForeignKeyConfiguration.java
@@ -0,0 +1,65 @@
+package com.databasepreservation.model.modules.configuration;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.List;
+import java.util.Objects;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ForeignKeyConfiguration {
+
+ private String name;
+ private String referencedTable;
+ private List references;
+ private String description;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getReferencedTable() {
+ return referencedTable;
+ }
+
+ public void setReferencedTable(String referencedTable) {
+ this.referencedTable = referencedTable;
+ }
+
+ public List getReferences() {
+ return references;
+ }
+
+ public void setReferences(List references) {
+ this.references = references;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ ForeignKeyConfiguration that = (ForeignKeyConfiguration) o;
+ return Objects.equals(name, that.name) && Objects.equals(referencedTable, that.referencedTable) && Objects.equals(
+ references, that.references) && Objects.equals(description, that.description);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, referencedTable, references);
+ }
+
+}
diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java
index 55cee5de..8ccf2882 100644
--- a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java
+++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ModuleConfiguration.java
@@ -8,16 +8,7 @@
package com.databasepreservation.model.modules.configuration;
import static com.databasepreservation.Constants.VIEW_NAME_PREFIX;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CANDIDATE_KEYS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CHECK_CONSTRAINTS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.FOREIGN_KEYS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIMARY_KEYS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIVILEGES;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROLES;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROUTINES;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.TRIGGERS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.USERS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.VIEWS;
+import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -26,19 +17,20 @@
import com.databasepreservation.Constants;
import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.*;
/**
* @author Miguel Guimarães
*/
@JsonPropertyOrder({"import", "schemas", "ignore"})
@JsonIgnoreProperties(value = {"fetchRows"})
+@JsonInclude(JsonInclude.Include.NON_NULL)
public class ModuleConfiguration {
private ImportModuleConfiguration importModuleConfiguration;
+ // Statements to run before importing database, e.g. creating temporary tables for use in custom views exported as
+ // tables.
+ private List setupStatements;
private Map schemaConfigurations;
private Map ignore;
private boolean fetchRows;
@@ -372,6 +364,15 @@ public ImportModuleConfiguration getImportModuleConfiguration() {
return importModuleConfiguration;
}
+ @JsonProperty("setupStatements")
+ public List getSetupStatements() {
+ return setupStatements;
+ }
+
+ public void setSetupStatements(List setupStatements) {
+ this.setupStatements = setupStatements;
+ }
+
public void setImportModuleConfiguration(ImportModuleConfiguration importModuleConfiguration) {
this.importModuleConfiguration = importModuleConfiguration;
}
diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/PrimaryKeyConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/PrimaryKeyConfiguration.java
new file mode 100644
index 00000000..34e33c25
--- /dev/null
+++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/PrimaryKeyConfiguration.java
@@ -0,0 +1,54 @@
+package com.databasepreservation.model.modules.configuration;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.List;
+import java.util.Objects;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PrimaryKeyConfiguration {
+
+ private String name;
+ private List columnNames;
+ private String description;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public List getColumnNames() {
+ return columnNames;
+ }
+
+ public void setColumnNames(List columnNames) {
+ this.columnNames = columnNames;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ PrimaryKeyConfiguration that = (PrimaryKeyConfiguration) o;
+ return Objects.equals(name, that.name) && Objects.equals(columnNames, that.columnNames)
+ && Objects.equals(description, that.description);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, columnNames, description);
+ }
+}
diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ReferenceConfiguration.java b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ReferenceConfiguration.java
new file mode 100644
index 00000000..65b729e6
--- /dev/null
+++ b/dbptk-model/src/main/java/com/databasepreservation/model/modules/configuration/ReferenceConfiguration.java
@@ -0,0 +1,40 @@
+package com.databasepreservation.model.modules.configuration;
+
+import java.util.Objects;
+
+public class ReferenceConfiguration {
+
+ private String column;
+ private String referenced;
+
+ public String getColumn() {
+ return column;
+ }
+
+ public void setColumn(String column) {
+ this.column = column;
+ }
+
+ public String getReferenced() {
+ return referenced;
+ }
+
+ public void setReferenced(String referenced) {
+ this.referenced = referenced;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ ReferenceConfiguration that = (ReferenceConfiguration) o;
+ return Objects.equals(column, that.column) && Objects.equals(referenced, that.referenced);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(column, referenced);
+ }
+}
diff --git a/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java b/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java
index 9d2ca779..c846cdaf 100644
--- a/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java
+++ b/dbptk-model/src/main/java/com/databasepreservation/utils/ModuleConfigurationUtils.java
@@ -7,30 +7,11 @@
*/
package com.databasepreservation.utils;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CANDIDATE_KEYS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.CHECK_CONSTRAINTS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.FOREIGN_KEYS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIMARY_KEYS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.PRIVILEGES;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROLES;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.ROUTINES;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.TRIGGERS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.USERS;
-import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.VIEWS;
-
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.databasepreservation.Constants;
-import com.databasepreservation.model.modules.configuration.ColumnConfiguration;
-import com.databasepreservation.model.modules.configuration.CustomViewConfiguration;
-import com.databasepreservation.model.modules.configuration.ImportModuleConfiguration;
-import com.databasepreservation.model.modules.configuration.ModuleConfiguration;
-import com.databasepreservation.model.modules.configuration.SchemaConfiguration;
-import com.databasepreservation.model.modules.configuration.TableConfiguration;
-import com.databasepreservation.model.modules.configuration.ViewConfiguration;
+import static com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures.*;
+
+import java.util.*;
+
+import com.databasepreservation.model.modules.configuration.*;
import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures;
import com.databasepreservation.model.structure.ColumnStructure;
import com.databasepreservation.model.structure.TableStructure;
@@ -55,7 +36,9 @@ public static ModuleConfiguration getDefaultModuleConfiguration() {
}
public static void addCustomViewConfiguration(ModuleConfiguration moduleConfiguration, String schemaName, String name,
- String description, String query) {
+ boolean simulateTable, String description, String query, List columns,
+ PrimaryKeyConfiguration primaryKey, List foreignKeys) {
+
SchemaConfiguration schemaConfiguration = moduleConfiguration.getSchemaConfigurations().get(schemaName);
if (schemaConfiguration == null) {
schemaConfiguration = new SchemaConfiguration();
@@ -63,8 +46,12 @@ public static void addCustomViewConfiguration(ModuleConfiguration moduleConfigur
CustomViewConfiguration customViewConfiguration = new CustomViewConfiguration();
customViewConfiguration.setName(name);
+ customViewConfiguration.setSimulateTable(simulateTable);
customViewConfiguration.setDescription(description);
customViewConfiguration.setQuery(query);
+ customViewConfiguration.setColumns(columns);
+ customViewConfiguration.setPrimaryKey(primaryKey);
+ customViewConfiguration.setForeignKeys(new ArrayList<>(foreignKeys)); // Ensure that items can be added.
schemaConfiguration.getCustomViewConfigurations().add(customViewConfiguration);
moduleConfiguration.getSchemaConfigurations().put(schemaName, schemaConfiguration);
diff --git a/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java b/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java
index 9b69d5ee..45c44149 100644
--- a/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java
+++ b/dbptk-modules/dbptk-module-import-config/src/main/java/com/databasepreservation/modules/config/ImportConfiguration.java
@@ -38,9 +38,10 @@
* @author Miguel Guimarães
*/
public class ImportConfiguration implements DatabaseFilterModule {
- private DatabaseStructure dbStructure;
- private SchemaStructure currentSchema;
- private ModuleConfiguration moduleConfiguration;
+ protected DatabaseStructure dbStructure;
+ protected SchemaStructure currentSchema;
+ protected TableStructure currentTable;
+ protected ModuleConfiguration moduleConfiguration;
private Path outputFile;
private ObjectMapper mapper;
@@ -119,7 +120,7 @@ public void handleDataOpenSchema(String schemaName) throws ModuleException {
*/
@Override
public void handleDataOpenTable(String tableId) throws ModuleException {
- TableStructure currentTable = dbStructure.getTableById(tableId);
+ currentTable = dbStructure.getTableById(tableId);
if (currentTable == null) {
throw new ModuleException().withMessage("Couldn't find table with id: " + tableId);
}
diff --git a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java
index eb82545d..91ef7151 100644
--- a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java
+++ b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java
@@ -17,30 +17,12 @@
import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.sql.Array;
-import java.sql.Blob;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
+import java.sql.*;
import java.sql.Date;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.SQLFeatureNotSupportedException;
-import java.sql.Statement;
-import java.sql.Struct;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.sql.Types;
import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
@@ -51,55 +33,22 @@
import com.databasepreservation.Constants;
import com.databasepreservation.managers.ModuleConfigurationManager;
import com.databasepreservation.managers.RemoteConnectionManager;
-import com.databasepreservation.model.data.ArrayCell;
-import com.databasepreservation.model.data.BinaryCell;
-import com.databasepreservation.model.data.Cell;
-import com.databasepreservation.model.data.NullCell;
-import com.databasepreservation.model.data.Row;
-import com.databasepreservation.model.data.SimpleCell;
+import com.databasepreservation.model.data.*;
import com.databasepreservation.model.exception.InvalidDataException;
import com.databasepreservation.model.exception.ModuleException;
import com.databasepreservation.model.exception.SQLParseException;
import com.databasepreservation.model.exception.TableNotFoundException;
import com.databasepreservation.model.modules.DatabaseImportModule;
import com.databasepreservation.model.modules.DatatypeImporter;
-import com.databasepreservation.model.modules.configuration.CustomViewConfiguration;
-import com.databasepreservation.model.modules.configuration.ModuleConfiguration;
+import com.databasepreservation.model.modules.configuration.*;
import com.databasepreservation.model.modules.filters.DatabaseFilterModule;
import com.databasepreservation.model.reporters.Reporter;
-import com.databasepreservation.model.structure.CandidateKey;
-import com.databasepreservation.model.structure.CheckConstraint;
-import com.databasepreservation.model.structure.ColumnStructure;
-import com.databasepreservation.model.structure.DatabaseStructure;
-import com.databasepreservation.model.structure.ForeignKey;
-import com.databasepreservation.model.structure.PrimaryKey;
-import com.databasepreservation.model.structure.PrivilegeStructure;
-import com.databasepreservation.model.structure.Reference;
-import com.databasepreservation.model.structure.RoleStructure;
-import com.databasepreservation.model.structure.RoutineStructure;
-import com.databasepreservation.model.structure.SchemaStructure;
-import com.databasepreservation.model.structure.TableStructure;
-import com.databasepreservation.model.structure.Trigger;
-import com.databasepreservation.model.structure.UserStructure;
-import com.databasepreservation.model.structure.ViewStructure;
-import com.databasepreservation.model.structure.type.ComposedTypeArray;
-import com.databasepreservation.model.structure.type.ComposedTypeStructure;
-import com.databasepreservation.model.structure.type.SimpleTypeBinary;
-import com.databasepreservation.model.structure.type.SimpleTypeBoolean;
-import com.databasepreservation.model.structure.type.SimpleTypeDateTime;
-import com.databasepreservation.model.structure.type.SimpleTypeNumericApproximate;
-import com.databasepreservation.model.structure.type.SimpleTypeNumericExact;
-import com.databasepreservation.model.structure.type.Type;
-import com.databasepreservation.model.structure.type.UnsupportedDataType;
+import com.databasepreservation.model.structure.*;
+import com.databasepreservation.model.structure.type.*;
import com.databasepreservation.modules.DefaultExceptionNormalizer;
import com.databasepreservation.modules.SQLHelper;
import com.databasepreservation.modules.jdbc.JDBCModuleFactory;
-import com.databasepreservation.utils.ConfigUtils;
-import com.databasepreservation.utils.JodaUtils;
-import com.databasepreservation.utils.MapUtils;
-import com.databasepreservation.utils.MiscUtils;
-import com.databasepreservation.utils.PortUtils;
-import com.databasepreservation.utils.RemoteConnectionUtils;
+import com.databasepreservation.utils.*;
import com.jcraft.jsch.Session;
/**
@@ -636,14 +585,11 @@ protected List getTables(SchemaStructure schema) throws SQLExcep
if (!customViewConfigurations.isEmpty()) {
for (CustomViewConfiguration custom : customViewConfigurations) {
- String description = custom.getDescription();
- String query = custom.getQuery();
String name = custom.getName();
LOGGER.info("Obtaining table structure for custom view {}", name);
try {
- TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, name, tableIndex,
- description, query);
+ TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, tableIndex, custom);
tables.add(customViewStructureAsTable);
tableIndex++;
} catch (SQLException e) {
@@ -758,13 +704,19 @@ protected List getCustomViews(String schemaName) throws ModuleExc
if (!customViewConfigurations.isEmpty()) {
for (CustomViewConfiguration custom : customViewConfigurations) {
+ // If simulating table, do not export as view, only as table.
+ if (custom.isSimulateTable())
+ continue;
+
ViewStructure view = new ViewStructure();
+ // Use prefix
view.setName(CUSTOM_VIEW_NAME_PREFIX + custom.getName());
view.setDescription(custom.getDescription());
view.setQueryOriginal(custom.getQuery());
try {
- view.setColumns(getColumnsFromCustomView(custom.getName(), custom.getQuery()));
+ // Don't set primary key on views (this makes only sense in as-table context)
+ view.setColumns(getColumnsFromCustomView(custom.getName(), custom.getQuery(), null, null));
} catch (SQLException e) {
reporter.ignored("Columns from custom view " + custom.getName() + " in schema " + schemaName,
"there was a problem retrieving them form the database");
@@ -926,18 +878,29 @@ protected TableStructure getViewStructure(SchemaStructure schema, String tableNa
return view;
}
- protected TableStructure getCustomViewStructureAsTable(SchemaStructure schema, String viewName, int tableIndex,
- String description, String query) throws SQLException, ModuleException {
+ protected TableStructure getCustomViewStructureAsTable(SchemaStructure schema, int tableIndex,
+ CustomViewConfiguration custom) throws SQLException, ModuleException {
+ String viewName = custom.getName();
+ String description = custom.getDescription();
+ String query = custom.getQuery();
+ PrimaryKey primaryKey = custom.getPrimaryKey() != null
+ ? getPrimaryKeyConfigurationAsPrimaryKey(custom.getPrimaryKey(), viewName)
+ : null;
+
+ String name = (custom.isSimulateTable() ? "" : CUSTOM_VIEW_NAME_PREFIX) + viewName;
+
TableStructure view = new TableStructure();
- view.setId(schema.getName() + "." + CUSTOM_VIEW_NAME_PREFIX + viewName);
- view.setName(CUSTOM_VIEW_NAME_PREFIX + viewName);
+ view.setId(schema.getName() + "." + name);
+ view.setName(name);
view.setSchema(schema);
view.setIndex(tableIndex);
view.setDescription(description);
- view.setColumns(getColumnsFromCustomView(viewName, query));
- view.setPrimaryKey(null);
- view.setForeignKeys(new ArrayList<>());
+ view.setColumns(getColumnsFromCustomView(viewName, query, custom.getPrimaryKey(), custom.getColumns()));
+ view.setPrimaryKey(primaryKey);
+ view.setForeignKeys(custom.getForeignKeys() == null ? new ArrayList<>()
+ : custom.getForeignKeys().stream().map(c -> getForeignKeyConfigurationAsForeignKey(c, name))
+ .collect(Collectors.toList()));
view.setCandidateKeys(new ArrayList<>());
view.setCheckConstraints(new ArrayList<>());
view.setTriggers(new ArrayList<>());
@@ -948,6 +911,50 @@ protected TableStructure getCustomViewStructureAsTable(SchemaStructure schema, S
return view;
}
+ private PrimaryKey getPrimaryKeyConfigurationAsPrimaryKey(PrimaryKeyConfiguration configuration, String tableName) {
+ PrimaryKey pk = new PrimaryKey();
+
+ if (configuration.getName() != null) {
+ pk.setName(configuration.getName());
+ } else {
+ pk.setName(getPrimaryKeyName(tableName));
+ }
+
+ pk.setDescription(configuration.getDescription());
+ pk.setColumnNames(configuration.getColumnNames());
+
+ return pk;
+ }
+
+ private ForeignKey getForeignKeyConfigurationAsForeignKey(ForeignKeyConfiguration configuration, String tableName) {
+ ForeignKey fk = new ForeignKey();
+
+ if (configuration.getReferences() == null || configuration.getReferences().isEmpty()) {
+ throw new IllegalArgumentException("Empty reference list in foreignKey configuration on " + tableName);
+ }
+
+ if (configuration.getName() != null) {
+ fk.setName(configuration.getName());
+ } else {
+ fk.setName(getForeignKeyName(tableName, configuration.getReferences().getFirst().getColumn()));
+ }
+
+ fk.setReferencedTable(configuration.getReferencedTable());
+ fk.setReferences(configuration.getReferences().stream().map(this::getReferenceConfigurationAsReference).toList());
+ fk.setDescription(configuration.getDescription());
+
+ return fk;
+ }
+
+ private Reference getReferenceConfigurationAsReference(ReferenceConfiguration configuration) {
+ Reference ref = new Reference();
+
+ ref.setColumn(configuration.getColumn());
+ ref.setReferenced(configuration.getReferenced());
+
+ return ref;
+ }
+
private int getRows(String schemaName, String tableName) throws SQLException, ModuleException {
String query = sqlHelper.getRowsSQL(schemaName, tableName);
LOGGER.debug("count query: {}", query);
@@ -1148,8 +1155,14 @@ protected List getColumns(String schemaName, String tableName)
return columns;
}
- protected List getColumnsFromCustomView(String viewName, String query)
+ protected List getColumnsFromCustomView(String viewName, String query,
+ PrimaryKeyConfiguration primaryKey, List columnConfigurations)
throws ModuleException, SQLException {
+
+ Map columnConfigurationMap = columnConfigurations != null
+ ? columnConfigurations.stream().collect(Collectors.toMap(CustomColumnConfiguration::getName, Function.identity()))
+ : Collections.emptyMap();
+
List columns = new ArrayList<>();
try (PreparedStatement preparedStatement = getConnection().prepareStatement(query)) {
@@ -1163,12 +1176,17 @@ protected List getColumnsFromCustomView(String viewName, String
String columnTypeName = metaData.getColumnTypeName(i);
int columnDisplaySize = metaData.getColumnDisplaySize(i);
int precision = metaData.getPrecision(i);
+ CustomColumnConfiguration columnConfiguration = columnConfigurationMap.get(columnName);
+ boolean nillable = (primaryKey == null || !primaryKey.getColumnNames().contains(columnName))
+ && (columnConfiguration == null || columnConfiguration.getNillable() == null
+ || columnConfiguration.getNillable());
+ String description = columnConfiguration != null ? columnConfiguration.getDescription() : "";
Type checkedType = datatypeImporter.getCheckedType(dbStructure, actualSchema, tableName, columnName, columnType,
columnTypeName, columnDisplaySize, precision, 10);
- ColumnStructure column = new ColumnStructure(viewName + "." + columnName, columnName, checkedType, true, "", "",
- false);
+ ColumnStructure column = new ColumnStructure(viewName + "." + columnName, columnName, checkedType, nillable,
+ description, "", false);
columns.add(column);
}
@@ -1310,7 +1328,7 @@ protected PrimaryKey getPrimaryKey(String schemaName, String tableName) throws S
}
if (pkName == null) {
- pkName = tableName + "_pkey";
+ pkName = getPrimaryKeyName(tableName);
}
PrimaryKey pk = new PrimaryKey();
@@ -1319,6 +1337,10 @@ protected PrimaryKey getPrimaryKey(String schemaName, String tableName) throws S
return pkColumns.isEmpty() ? null : pk;
}
+ private static String getPrimaryKeyName(String tableName) {
+ return tableName + "_pkey";
+ }
+
/**
* Get the table foreign keys
*
@@ -1341,8 +1363,7 @@ protected List getForeignKeys(String schemaName, String tableName) t
String fkeyName = rs.getString("FK_NAME");
if (fkeyName == null) {
- fkeyName = "FK_" + rs.getString("PKTABLE_NAME") + "_" + rs.getString("FKTABLE_NAME") + "_"
- + rs.getString("FKCOLUMN_NAME");
+ fkeyName = getForeignKeyName(rs.getString("FKTABLE_NAME"), rs.getString("FKCOLUMN_NAME"));
}
for (ForeignKey key : foreignKeys) {
@@ -1373,6 +1394,13 @@ protected List getForeignKeys(String schemaName, String tableName) t
return foreignKeys;
}
+ private String getForeignKeyName(String fkTableName, String fkColumnName) {
+ // The original foreign key scheme, before specifying foreign keys on custom views was implemented, included
+ // PKTABLE_NAME. It was only used if the database did not return any name. That scheme is vulnerable to exceeding
+ // name length limits in SIARD XSD's, and the fk table and column are sufficient to create a unique id.
+ return "FK_" + fkTableName + "_" + fkColumnName;
+ }
+
protected String getReferencedSchema(String s) throws SQLException, ModuleException {
return s;
}
@@ -2065,11 +2093,33 @@ protected Set getIgnoredExportedSchemas() {
return ignore;
}
+ private void executeSetupStatements() throws ModuleException {
+ ModuleConfiguration configuration = getModuleConfiguration();
+ if (configuration.getSetupStatements() == null)
+ return;
+
+ int i = 0;
+ int count = configuration.getSetupStatements().size();
+
+ try {
+ for (String sql : configuration.getSetupStatements()) {
+ LOGGER.info("Executing setup statement {} of {}", ++i, count);
+ Statement st = getStatement();
+ st.execute(sql);
+ }
+ } catch (SQLException e) {
+ closeConnection();
+ throw new ModuleException().withCause(e).withMessage(e.getMessage());
+ }
+ }
+
@Override
public DatabaseFilterModule migrateDatabaseTo(DatabaseFilterModule exportModule) throws ModuleException {
try {
exportModule.initDatabase();
+ executeSetupStatements();
+
exportModule.setIgnoredSchemas(getIgnoredExportedSchemas());
exportModule.handleStructure(getDatabaseStructure());
diff --git a/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java b/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java
index d0e03468..d6cb23d1 100644
--- a/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java
+++ b/dbptk-modules/dbptk-module-ms-access/src/main/java/com/databasepreservation/modules/msAccess/in/MsAccessUCanAccessImportModule.java
@@ -210,14 +210,11 @@ protected List getTables(SchemaStructure schema) throws SQLExcep
if (!customViewConfigurations.isEmpty()) {
for (CustomViewConfiguration custom : customViewConfigurations) {
- String description = custom.getDescription();
- String query = custom.getQuery();
String name = custom.getName();
LOGGER.info("Obtaining table structure for custom view {}", name);
try {
- TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, name, tableIndex,
- description, query);
+ TableStructure customViewStructureAsTable = getCustomViewStructureAsTable(schema, tableIndex, custom);
tables.add(customViewStructureAsTable);
tableIndex++;
} catch (SQLException e) {
diff --git a/dbptk-modules/dbptk-module-normalize-1nf-config/pom.xml b/dbptk-modules/dbptk-module-normalize-1nf-config/pom.xml
new file mode 100644
index 00000000..3db3a97c
--- /dev/null
+++ b/dbptk-modules/dbptk-module-normalize-1nf-config/pom.xml
@@ -0,0 +1,39 @@
+
+
+ 4.0.0
+ normalize-1nf-config
+ dbptk-module-normalize-1nf-config
+ 3.1.0-SNAPSHOT
+
+ com.databasepreservation
+ dbptk-modules
+ 3.1.0-SNAPSHOT
+ ..
+
+
+
+
+
+ com.databasepreservation
+ dbptk-model
+
+
+ com.databasepreservation
+ dbptk-module-import-config
+
+
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+ 2.16.1
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ 2.16.1
+
+
+
diff --git a/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfiguration.java b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfiguration.java
new file mode 100644
index 00000000..eef64a44
--- /dev/null
+++ b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfiguration.java
@@ -0,0 +1,357 @@
+package com.databasepreservation.modules.config;
+
+import static com.databasepreservation.modules.config.Normalize1NFConfiguration.NormalizedColumnType.ARRAY;
+import static com.databasepreservation.modules.config.Normalize1NFConfiguration.NormalizedColumnType.JSON;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.commons.text.StringSubstitutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.databasepreservation.Constants;
+import com.databasepreservation.managers.ModuleConfigurationManager;
+import com.databasepreservation.model.exception.ModuleException;
+import com.databasepreservation.model.modules.configuration.*;
+import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures;
+import com.databasepreservation.model.structure.ColumnStructure;
+import com.databasepreservation.model.structure.PrimaryKey;
+import com.databasepreservation.utils.MapUtils;
+import com.databasepreservation.utils.ModuleConfigurationUtils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * @author Daniel Lundsgaard Skovenborg
+ */
+public class Normalize1NFConfiguration extends ImportConfiguration {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Normalize1NFConfiguration.class);
+
+ // Value to use for query in merge file to exclude from normalization view
+ // creation. Empty string will not work because Constants.EMPTY will be assigned
+ // if "query" key is left out to
+ // override other fields.
+ private static final String EXCLUDE_CUSTOM_VIEW_QUERY = "--"; //
+ private static final String DEFAULT_ARRAY_NAME_PATTERN = "${table}__${column}";
+ private static final String DEFAULT_ARRAY_FOREIGN_KEY_COLUMN_PATTERN = "${table}_${column}";
+ private static final String DEFAULT_ARRAY_INDEX_COLUMN_NAME_PATTERN = "array_index";
+ private static final String DEFAULT_ARRAY_ITEM_COLUMN_NAME_PATTERN = "${column}_item";
+ private static final String DEFAULT_ARRAY_TABLE_ALIAS = "a";
+
+ private static final String DEFAULT_JSON_NAME_PATTERN = DEFAULT_ARRAY_NAME_PATTERN;
+ private static final String DEFAULT_JSON_FOREIGN_KEY_COLUMN_PATTERN = DEFAULT_ARRAY_FOREIGN_KEY_COLUMN_PATTERN;
+
+ // TODO: Allow overriding all patterns.
+ private String foreignIdColumnDescriptionPattern;
+
+ private String arrayNamePattern = DEFAULT_ARRAY_NAME_PATTERN;
+ private String arrayForeignKeyColumnPattern = DEFAULT_ARRAY_FOREIGN_KEY_COLUMN_PATTERN;
+ private String arrayDescriptionPattern;
+ private String arrayIndexColumnDescriptionPattern;
+ private String arrayItemColumnDescriptionPattern;
+ private String arrayIndexColumnNamePattern = DEFAULT_ARRAY_INDEX_COLUMN_NAME_PATTERN;
+ private String arrayItemColumnNamePattern = DEFAULT_ARRAY_ITEM_COLUMN_NAME_PATTERN;
+ private String arrayTableAlias = DEFAULT_ARRAY_TABLE_ALIAS;
+
+ private String jsonNamePattern = DEFAULT_JSON_NAME_PATTERN;
+ private String jsonForeignKeyColumnPattern = DEFAULT_JSON_FOREIGN_KEY_COLUMN_PATTERN;
+ private String jsonDescriptionPattern;
+
+ private final ModuleConfiguration mergeConfiguration;
+ private final boolean noSQLQuotes;
+
+ public Normalize1NFConfiguration(Path outputFile, Path mergeFile, boolean noSQLQuotes, String arrayDescriptionPattern,
+ String jsonDescriptionPattern, String foreignIdColumnDescriptionPattern, String arrayIndexColumnDescriptionPattern,
+ String arrayItemColumnDescriptionPattern)
+ throws ModuleException {
+ super(outputFile);
+ this.noSQLQuotes = noSQLQuotes;
+ this.arrayDescriptionPattern = arrayDescriptionPattern;
+ this.jsonDescriptionPattern = jsonDescriptionPattern;
+ this.foreignIdColumnDescriptionPattern = foreignIdColumnDescriptionPattern;
+ this.arrayIndexColumnDescriptionPattern = arrayIndexColumnDescriptionPattern;
+ this.arrayItemColumnDescriptionPattern = arrayItemColumnDescriptionPattern;
+
+ if (mergeFile == null) {
+ // Create empty configuration so that we don't have to check for null everywhere.
+ mergeConfiguration = new ModuleConfiguration();
+ } else {
+ try {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ mergeConfiguration = mapper.readValue(mergeFile.toFile(), ModuleConfiguration.class);
+ } catch (IOException e) {
+ throw new ModuleException()
+ .withMessage("Could not read the merge configuration from file " + mergeFile.normalize().toAbsolutePath())
+ .withCause(e);
+ }
+ }
+ }
+
+ @Override
+ public void initDatabase() {
+ super.initDatabase();
+
+ ModuleConfiguration dbConfiguration = ModuleConfigurationManager.getInstance().getModuleConfiguration();
+ Map ignore = dbConfiguration.getIgnore();
+ ignore.put(DatabaseTechnicalFeatures.PRIMARY_KEYS, false);
+ dbConfiguration.setIgnore(ignore);
+ }
+
+ @Override
+ public void handleDataOpenTable(String tableId) throws ModuleException {
+ super.handleDataOpenTable(tableId);
+
+ currentTable.getColumns().forEach(this::handleColumn);
+
+ // TODO: add support for using merge configuration to add foreign keys to table,
+ // e.g., foreign key from an enum column to a code table constructed with a
+ // custom view in the merge configuration.
+ }
+
+ private void handleColumn(ColumnStructure column) {
+
+ boolean isArray = column.getType().getSql99TypeName().endsWith("ARRAY");
+ boolean isJson = column.getType().getOriginalTypeName().equalsIgnoreCase("json")
+ || column.getType().getOriginalTypeName().equalsIgnoreCase("jsonb");
+
+ if (!isArray && !isJson)
+ return;
+
+ String schemaName = currentSchema.getName();
+ String tableName = currentTable.getName();
+
+ NormalizedColumnType ncType = isArray ? ARRAY : JSON;
+ String columnName = column.getName();
+ String viewName = formatTblCol(ncType == ARRAY ? arrayNamePattern : jsonNamePattern, tableName, columnName);
+
+ // Remove normalized column from table configuration.
+ removeColumnFromConfiguration(schemaName, tableName, columnName);
+
+ // Allow overriding creation of view (e.g., if making a manual normalization).
+ CustomViewConfiguration merge = mergeConfiguration.getCustomViewConfiguration(schemaName, viewName);
+ if (merge != null && EXCLUDE_CUSTOM_VIEW_QUERY.equals(merge.getQuery())) {
+ LOGGER.info("Normalization of {}.{}.{} ({}) is excluded by merge file", schemaName, tableName, columnName,
+ viewName);
+ return;
+ }
+
+ addNormalizationViewConfiguration(ncType, schemaName, tableName, columnName, viewName);
+ mergeCustomViewConfiguration(schemaName, viewName);
+ }
+
+ private void removeColumnFromConfiguration(String schemaName, String tableName, String columnName) {
+ // Assume no copy.
+ LOGGER.info("Removing non-1NF column {}.{}.{} from configuration", schemaName, tableName, columnName);
+ TableConfiguration tableConfiguration = moduleConfiguration.getTableConfiguration(schemaName, tableName);
+ List newColumns = tableConfiguration.getColumns().stream()
+ .filter(c -> !c.getName().equals(columnName)).collect(Collectors.toList());
+ tableConfiguration.setColumns(newColumns);
+ }
+
+ private void addNormalizationViewConfiguration(NormalizedColumnType ncType, String schemaName, String tableName,
+ String columnName, String viewName) {
+
+ PrimaryKey primaryKey = currentTable.getPrimaryKey();
+
+ if (primaryKey == null) {
+ LOGGER.warn("Table {}.{} has no primary key. Cannot create normalization of {} column {}", ncType, schemaName,
+ tableName, columnName);
+ return;
+ }
+
+ LOGGER.info("Creating normalization view of {} column {}.{}.{}", ncType, schemaName, tableName, columnName);
+
+ String description = formatTblCol(ncType == ARRAY ? arrayDescriptionPattern : jsonDescriptionPattern, tableName,
+ columnName);
+ List columns = new ArrayList<>();
+ String query = ncType == ARRAY ? getArrayNormalizationSQL(schemaName, tableName, columnName, primaryKey, columns)
+ : getJsonNormalizationSQL(schemaName, tableName, primaryKey, columns);
+ PrimaryKeyConfiguration primaryKeyConfiguration = getPrimaryKeyConfiguration(ncType, primaryKey, tableName,
+ columnName);
+ ForeignKeyConfiguration foreignKeyConfiguration = getForeignKeyConfiguration(ncType, primaryKey, tableName);
+
+ ModuleConfigurationUtils.addCustomViewConfiguration(moduleConfiguration, schemaName, viewName, true, description,
+ query, columns, primaryKeyConfiguration, Collections.singletonList(foreignKeyConfiguration));
+ }
+
+ @Override
+ public void finishDatabase() throws ModuleException {
+ // Add all custom views from the merge configuration if not present.
+ moduleConfiguration.getSchemaConfigurations().forEach((schemaName, schemaConfiguration) -> {
+ SchemaConfiguration schemaToMerge = mergeConfiguration.getSchemaConfigurations().get(schemaName);
+ if (schemaToMerge == null)
+ return;
+
+ List customViewConfigurations = schemaConfiguration.getCustomViewConfigurations();
+
+ for (CustomViewConfiguration custom : schemaToMerge.getCustomViewConfigurations()) {
+ if (!EXCLUDE_CUSTOM_VIEW_QUERY.equals(custom.getQuery())
+ && moduleConfiguration.getCustomViewConfiguration(schemaName, custom.getName()) == null) {
+ LOGGER.info("Adding custom view {}.{} from merge configuration file", schemaName, custom.getName());
+ customViewConfigurations.add(custom);
+ }
+ }
+
+ customViewConfigurations.sort(Comparator.comparing(CustomViewConfiguration::getName));
+ });
+
+ super.finishDatabase();
+ }
+
+ private void mergeCustomViewConfiguration(String schemaName, String viewName) {
+ CustomViewConfiguration merge = mergeConfiguration.getCustomViewConfiguration(schemaName, viewName);
+ if (merge == null)
+ return;
+
+ LOGGER.info("Merging configuration of custom view {}.{}", schemaName, viewName);
+
+ // Assume no copy on getters.
+ // Allow setting description and columns, overriding query and primary key, and
+ // adding foreign keys.
+ CustomViewConfiguration view = moduleConfiguration.getCustomViewConfiguration(schemaName, viewName);
+ if (merge.getDescription() != null) {
+ view.setDescription(merge.getDescription());
+ }
+ if (!Constants.EMPTY.equals(merge.getQuery())) {
+ view.setQuery(merge.getQuery());
+ }
+ if (!merge.getColumns().isEmpty()) {
+ view.setColumns(merge.getColumns());
+ }
+ if (merge.getPrimaryKey() != null) {
+ view.setPrimaryKey(merge.getPrimaryKey());
+ }
+ if (!merge.getForeignKeys().isEmpty()) {
+ // Add, not replace!
+ view.getForeignKeys().addAll(merge.getForeignKeys());
+ }
+ }
+
+ private static String formatTblCol(String pattern, String tableName, String columnName) {
+ return StringSubstitutor.replace(pattern, MapUtils.buildMapFromObjects("table", tableName, "column", columnName));
+ }
+
+ private String quoteSQL(String sqlName) {
+ return noSQLQuotes ? sqlName : '"' + sqlName + '"';
+ }
+
+ private String getArrayNormalizationSQL(String schemaName, String tableName, String columnName,
+ PrimaryKey primaryKey, List columns) {
+
+ // Resulting SQL is only tested in PostgreSQL, but "UNNEST ... WITH ORDINALITY" should be standard SQL.
+ String indexColumnName = formatTblCol(arrayIndexColumnNamePattern, tableName, columnName);
+ String itemColumnName = formatTblCol(arrayItemColumnNamePattern, tableName, columnName);
+ String qColumnName = quoteSQL(columnName);
+ String qIndexColumnName = quoteSQL(indexColumnName);
+ String qItemColumnName = quoteSQL(itemColumnName);
+
+ StringBuilder sb = getNormalizationSQLStringBuilder(ARRAY, tableName, primaryKey, columns).append(", ")
+ .append(qIndexColumnName).append(", ").append(qItemColumnName).append(" ");
+ addNormalizationSQLFrom(sb, schemaName, tableName) //
+ .append(" cross join unnest(").append(qColumnName).append(") with ordinality as ") //
+ .append(arrayTableAlias).append("(").append(qItemColumnName).append(", ").append(qIndexColumnName).append(")");
+
+ addCustomColumnConfiguration(columns, indexColumnName, null,
+ formatTblCol(arrayIndexColumnDescriptionPattern, tableName, columnName));
+ // Item nullability defaults to false (assume no null items in array).
+ addCustomColumnConfiguration(columns, itemColumnName, false,
+ formatTblCol(arrayItemColumnDescriptionPattern, tableName, columnName));
+
+ return sb.toString();
+ }
+
+ private String getJsonNormalizationSQL(String schemaName, String tableName, PrimaryKey primaryKey,
+ List columns) {
+ // Get a template only; do not attempt to calculate columns which would require processing all rows in the table.
+ StringBuilder sb = getNormalizationSQLStringBuilder(JSON, tableName, primaryKey, columns).append(" ");
+ addNormalizationSQLFrom(sb, schemaName, tableName);
+
+ return sb.toString();
+ }
+
+ private StringBuilder getNormalizationSQLStringBuilder(NormalizedColumnType ncType, String tableName,
+ PrimaryKey primaryKey, List columns) {
+
+ StringBuilder sb = new StringBuilder("select");
+ boolean first = true;
+
+ for (String pkColumnName : primaryKey.getColumnNames()) {
+ String name = formatTblCol(ncType == ARRAY ? arrayForeignKeyColumnPattern : jsonForeignKeyColumnPattern,
+ tableName, pkColumnName);
+ sb.append(first ? " " : ", ").append(pkColumnName).append(" as ").append(quoteSQL(name));
+ addCustomColumnConfiguration(columns, name, null,
+ formatTblCol(foreignIdColumnDescriptionPattern, tableName, pkColumnName));
+ first = false;
+ }
+
+ return sb;
+ }
+
+ private StringBuilder addNormalizationSQLFrom(StringBuilder sb, String schemaName, String tableName) {
+ String qSchemaName = quoteSQL(schemaName);
+ String qTableName = quoteSQL(tableName);
+
+ sb.append("from ").append(qSchemaName).append(".").append(qTableName);
+
+ return sb;
+ }
+
+ private PrimaryKeyConfiguration getPrimaryKeyConfiguration(NormalizedColumnType ncType, PrimaryKey primaryKey,
+ String tableName,
+ String columnName) {
+ PrimaryKeyConfiguration primaryKeyConfiguration = new PrimaryKeyConfiguration();
+ List columnNames = new ArrayList<>(2);
+
+ for (String pkColumnName : primaryKey.getColumnNames()) {
+ columnNames.add(formatTblCol(ncType == ARRAY ? arrayForeignKeyColumnPattern : jsonForeignKeyColumnPattern,
+ tableName, pkColumnName));
+ }
+
+ if (ncType == ARRAY)
+ columnNames.add(formatTblCol(arrayIndexColumnNamePattern, tableName, columnName));
+
+ primaryKeyConfiguration.setColumnNames(columnNames);
+
+ return primaryKeyConfiguration;
+ }
+
+ private ForeignKeyConfiguration getForeignKeyConfiguration(NormalizedColumnType ncType, PrimaryKey primaryKey,
+ String tableName) {
+ ForeignKeyConfiguration foreignKeyConfiguration = new ForeignKeyConfiguration();
+
+ foreignKeyConfiguration.setReferencedTable(tableName);
+
+ List refererences = new ArrayList<>(2);
+
+ for (String pkColumnName : primaryKey.getColumnNames()) {
+ ReferenceConfiguration ref = new ReferenceConfiguration();
+ ref.setColumn(formatTblCol(ncType == ARRAY ? arrayForeignKeyColumnPattern : jsonForeignKeyColumnPattern,
+ tableName, pkColumnName));
+ ref.setReferenced(pkColumnName);
+ refererences.add(ref);
+ }
+ foreignKeyConfiguration.setReferences(refererences);
+
+ return foreignKeyConfiguration;
+ }
+
+ private void addCustomColumnConfiguration(List list, String name, Boolean nillable,
+ String description) {
+
+ CustomColumnConfiguration customColumnConfiguration = new CustomColumnConfiguration();
+ customColumnConfiguration.setName(name);
+ customColumnConfiguration.setMerkle(false);
+ customColumnConfiguration.setNillable(nillable);
+ customColumnConfiguration.setDescription(description);
+
+ list.add(customColumnConfiguration);
+ }
+
+ enum NormalizedColumnType {
+ ARRAY, JSON
+ }
+}
diff --git a/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfigurationModuleFactory.java b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfigurationModuleFactory.java
new file mode 100644
index 00000000..e6c421c7
--- /dev/null
+++ b/dbptk-modules/dbptk-module-normalize-1nf-config/src/main/java/com/databasepreservation/modules/config/Normalize1NFConfigurationModuleFactory.java
@@ -0,0 +1,176 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE file at the root of the source
+ * tree and available online at
+ *
+ * https://github.com/keeps/db-preservation-toolkit
+ */
+package com.databasepreservation.modules.config;
+
+import static java.util.stream.Collectors.toMap;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import com.databasepreservation.managers.ModuleConfigurationManager;
+import com.databasepreservation.model.exception.ModuleException;
+import com.databasepreservation.model.exception.UnsupportedModuleException;
+import com.databasepreservation.model.modules.DatabaseImportModule;
+import com.databasepreservation.model.modules.DatabaseModuleFactory;
+import com.databasepreservation.model.modules.configuration.ModuleConfiguration;
+import com.databasepreservation.model.modules.configuration.enums.DatabaseTechnicalFeatures;
+import com.databasepreservation.model.modules.filters.DatabaseFilterModule;
+import com.databasepreservation.model.parameters.Parameter;
+import com.databasepreservation.model.parameters.Parameters;
+import com.databasepreservation.model.reporters.Reporter;
+import com.databasepreservation.utils.ModuleConfigurationUtils;
+
+/**
+ * Exposes an export module which extends the import-config module to generate a
+ * configuration which uses custom views normalize the database to be 1NF as
+ * needed for SIARD DK.
+ *
+ * @author Daniel Lundsgaard Skovenborg
+ */
+public class Normalize1NFConfigurationModuleFactory implements DatabaseModuleFactory {
+ public static final String PARAMETER_FILE = "file";
+ public static final String MERGE_FILE = "merge-file";
+ public static final String NO_SQL_QUOTES = "no-sql-quotes";
+ public static final String ARRAY_DESCRIPTION_PATTERN = "pattern-array-description";
+ public static final String JSON_DESCRIPTION_PATTERN = "pattern-json-description";
+ private static final String FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN = "pattern-foreign-id-column-description";
+ private static final String ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN = "pattern-array-index-column-description-pattern";
+ private static final String ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN = "pattern-array-item-column-description-pattern";
+
+ private static final String DEFAULT_ARRAY_DESCRIPTION_PATTERN = "Normalized array column ${table}.${column}";
+ private static final String DEFAULT_JSON_DESCRIPTION_PATTERN = "Normalized JSON column ${table}.${column}.";
+ private static final String DEFAULT_FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN = "Id of ${table} row.";
+ private static final String DEFAULT_ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN = "Index of item in normalized array.";
+ private static final String DEFAULT_ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN = "Value of item in normalized array.";
+
+ private static final Parameter file = new Parameter().shortName("f").longName(PARAMETER_FILE)
+ .description("Path to the import configuration file").hasArgument(true).setOptionalArgument(false).required(true);
+ private static final Parameter mergeFile = new Parameter().shortName("mf").longName(MERGE_FILE)
+ .description("Path a configuration file to merge with the output").hasArgument(true).setOptionalArgument(false)
+ .required(false);
+ private static final Parameter noSQLQuotes = new Parameter().shortName("nqc").longName(NO_SQL_QUOTES)
+ .description(
+ "Don't quote SQL identifiers in normalization view queries (use if applicable to get more readable queries)")
+ .hasArgument(false).required(false).valueIfNotSet("false").valueIfSet("true");
+
+ // TODO: Allow overriding all patterns.
+ private static final Parameter arrayDescriptionPattern = new Parameter().shortName("pad")
+ .longName(ARRAY_DESCRIPTION_PATTERN)
+ .description(withDefault("Pattern for description of normalized array columns.", DEFAULT_ARRAY_DESCRIPTION_PATTERN))
+ .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet(DEFAULT_ARRAY_DESCRIPTION_PATTERN);
+ private static final Parameter jsonDescriptionPattern = new Parameter().shortName("pjd")
+ .longName(JSON_DESCRIPTION_PATTERN)
+ .description(withDefault("Pattern for description of normalized JSON columns.", DEFAULT_JSON_DESCRIPTION_PATTERN))
+ .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet(DEFAULT_JSON_DESCRIPTION_PATTERN);
+ private static final Parameter foreignIdColumnDescriptionPattern = new Parameter().shortName("pfid")
+ .longName(FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN)
+ .description(withDefault("Pattern for description of foreign id column for normalized columns.",
+ DEFAULT_FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN))
+ .hasArgument(true).setOptionalArgument(false).required(false)
+ .valueIfNotSet(DEFAULT_FOREIGN_ID_COLUMN_DESCRIPTION_PATTERN);
+ private static final Parameter arrayIndexColumnDescriptionPattern = new Parameter().shortName("paicd")
+ .longName(ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN)
+ .description(withDefault("Pattern for description of array index column for normalized array columns.",
+ DEFAULT_ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN))
+ .hasArgument(true).setOptionalArgument(false).required(false)
+ .valueIfNotSet(DEFAULT_ARRAY_INDEX_COLUMN_DESCRIPTION_PATTERN);
+ private static final Parameter arrayItemColumnDescriptionPattern = new Parameter().shortName("patcd")
+ .longName(ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN)
+ .description(withDefault("Pattern for description of array value column for normalized array columns.",
+ DEFAULT_ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN))
+ .hasArgument(true).setOptionalArgument(false).required(false)
+ .valueIfNotSet(DEFAULT_ARRAY_ITEM_COLUMN_DESCRIPTION_PATTERN);
+
+ private static final List parameters = Arrays.asList(file, mergeFile, noSQLQuotes,
+ arrayDescriptionPattern, jsonDescriptionPattern, foreignIdColumnDescriptionPattern,
+ arrayIndexColumnDescriptionPattern, arrayItemColumnDescriptionPattern);
+
+ private static String withDefault(String description, String defaultValue) {
+ return String.format("%s Default: \"%s\"", description, defaultValue);
+ }
+
+ @Override
+ public boolean producesImportModules() {
+ return false;
+ }
+
+ @Override
+ public boolean producesExportModules() {
+ return true;
+ }
+
+ @Override
+ public String getModuleName() {
+ return "normalize-1nf-config";
+ }
+
+ @Override
+ public boolean isEnabled() {
+ return true;
+ }
+
+ @Override
+ public Map getAllParameters() {
+ return parameters.stream().collect(toMap(Parameter::longName, Function.identity()));
+ }
+
+ @Override
+ public Parameters getConnectionParameters() throws UnsupportedModuleException {
+ throw ExceptionBuilder.UnsupportedModuleExceptionForImportModule();
+ }
+
+ @Override
+ public Parameters getImportModuleParameters() throws UnsupportedModuleException {
+ throw ExceptionBuilder.UnsupportedModuleExceptionForImportModule();
+ }
+
+ @Override
+ public Parameters getExportModuleParameters() {
+ return new Parameters(parameters, null);
+ }
+
+ @Override
+ public DatabaseImportModule buildImportModule(Map parameters, Reporter reporter)
+ throws UnsupportedModuleException {
+ throw ExceptionBuilder.UnsupportedModuleExceptionForImportModule();
+ }
+
+ @Override
+ public DatabaseFilterModule buildExportModule(Map parameters, Reporter reporter)
+ throws ModuleException {
+ Path pFile = Paths.get(parameters.get(file));
+ Path pMergeFile = parameters.get(mergeFile) != null ? Paths.get(parameters.get(mergeFile)) : null;
+ boolean pNoSQLQuotes = Boolean.parseBoolean(parameters.get(noSQLQuotes));
+
+ reporter.exportModuleParameters(this.getModuleName(), PARAMETER_FILE,
+ pFile.normalize().toAbsolutePath().toString());
+
+ final ModuleConfiguration defaultModuleConfiguration = ModuleConfigurationUtils.getDefaultModuleConfiguration();
+ defaultModuleConfiguration.setFetchRows(false);
+ defaultModuleConfiguration
+ .setIgnore(ModuleConfigurationUtils.createIgnoreListExcept(true, DatabaseTechnicalFeatures.VIEWS));
+
+ ModuleConfigurationManager.getInstance().setup(defaultModuleConfiguration);
+
+ reporter.exportModuleParameters(getModuleName(), PARAMETER_FILE, pFile.normalize().toAbsolutePath().toString());
+
+ return new Normalize1NFConfiguration(pFile, pMergeFile, pNoSQLQuotes,
+ getOrDefault(parameters, arrayDescriptionPattern), getOrDefault(parameters, jsonDescriptionPattern),
+ getOrDefault(parameters, foreignIdColumnDescriptionPattern),
+ getOrDefault(parameters, arrayIndexColumnDescriptionPattern),
+ getOrDefault(parameters, arrayItemColumnDescriptionPattern));
+ }
+
+ private String getOrDefault(Map parameters, Parameter parameter) {
+ return parameters.getOrDefault(parameter, parameter.valueIfNotSet());
+ }
+}
diff --git a/dbptk-modules/pom.xml b/dbptk-modules/pom.xml
index f5c26b76..d9e6ce23 100644
--- a/dbptk-modules/pom.xml
+++ b/dbptk-modules/pom.xml
@@ -29,6 +29,7 @@
dbptk-module-openedge
dbptk-module-oracle
dbptk-module-postgresql
+ dbptk-module-normalize-1nf-config
dbptk-module-siard
dbptk-module-sql-server
dbptk-module-sybase
diff --git a/pom.xml b/pom.xml
index a721cb25..799e57f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -255,6 +255,12 @@
3.1.0-SNAPSHOT