Merge pull request #162 from OHDSI/sheet-name-hotfix

Preventing sheet name clashes for long tables. Closes #14 and #113

rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Database.java

@@ -1,14 +1,14 @@
 /*******************************************************************************
  * Copyright 2019 Observational Health Data Sciences and Informatics
- * 
+ *
  * This file is part of WhiteRabbit
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -176,11 +176,13 @@ public static Database generateModelFromScanReport(String filename) {
 
 	private static String[][] getValueCounts(QuickAndDirtyXlsxReader workbook, String tableName, String fieldName) {
 		Sheet tableSheet = null;
-		for (Sheet sheet : workbook)
-			if (sheet.getName().equals(tableName)) {
+		String targetSheetName = Table.createSheetNameFromTableName(tableName);
+		for (Sheet sheet : workbook) {
+			if (sheet.getName().equals(targetSheetName)) {
 				tableSheet = sheet;
 				break;
 			}
+		}
 		if (tableSheet == null) // Sheet not found for table, return empty array
 			return new String[0][0];
 

rabbit-core/src/main/java/org/ohdsi/rabbitInAHat/dataModel/Table.java

@@ -102,4 +102,26 @@ public boolean isStem() {
 	public void setStem(boolean isStem) {
 		this.isStem = isStem;
 	}
+
+	public static String indexTableNameForSheet(String tableName, int index) {
+		String name = tableName;
+		// Prepend index for long table names (to make sheet name unique later)
+		if (name.length() > 31) {
+			name = Integer.toString(index) + '_' + name;
+		}
+		return name;
+	}
+
+	public static String createSheetNameFromTableName(String tableName) {
+		String name = tableName;
+
+		// Excel sheet names have a maximum of 31 characters
+		if (name.length() > 31) {
+			name = name.substring(0, 31);
+		}
+
+		// Backslash causes issues in excel
+		name = name.replace('/','_');
+		return name;
+	}
 }

whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java

@@ -37,6 +37,7 @@
 import org.ohdsi.databases.DbType;
 import org.ohdsi.databases.RichConnection;
 import org.ohdsi.databases.RichConnection.QueryResult;
+import org.ohdsi.rabbitInAHat.dataModel.Table;
 import org.ohdsi.utilities.StringUtilities;
 import org.ohdsi.utilities.collections.CountingSet;
 import org.ohdsi.utilities.collections.CountingSet.Count;
@@ -117,28 +118,38 @@ private void generateReport(Map<String, List<FieldInfo>> tableToFieldInfos, Stri
 		SXSSFWorkbook workbook = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk
 
 		// Create overview sheet
-		Sheet sheet = workbook.createSheet("Overview");
+		Sheet overviewSheet = workbook.createSheet("Overview");
 		if (!scanValues) {
-			addRow(sheet, "Table", "Field", "Type", "N rows");
+			addRow(overviewSheet, "Table", "Field", "Type", "N rows");
 			for (String table : tables) {
 				for (FieldInfo fieldInfo : tableToFieldInfos.get(table))
-					addRow(sheet, table, fieldInfo.name, fieldInfo.getTypeDescription(), Long.valueOf(fieldInfo.rowCount));
-				addRow(sheet, "");
+					addRow(overviewSheet, table, fieldInfo.name, fieldInfo.getTypeDescription(), Long.valueOf(fieldInfo.rowCount));
+				addRow(overviewSheet, "");
 			}
 		} else {
-			addRow(sheet, "Table", "Field", "Type", "Max length", "N rows", "N rows checked", "Fraction empty");
-			for (String table : tables) {
-				for (FieldInfo fieldInfo : tableToFieldInfos.get(table))
-					addRow(sheet, table, fieldInfo.name, fieldInfo.getTypeDescription(), Integer.valueOf(fieldInfo.maxLength), Long.valueOf(fieldInfo.rowCount),
+			addRow(overviewSheet, "Table", "Field", "Type", "Max length", "N rows", "N rows checked", "Fraction empty");
+			int sheetIndex = 0;
+			Map<String, String> sheetNameLookup = new HashMap<>();
+			for (String tableName : tables) {
+				// Make tablename unique
+				String tableNameIndexed = Table.indexTableNameForSheet(tableName, sheetIndex);
+
+				String sheetName = Table.createSheetNameFromTableName(tableNameIndexed);
+				sheetNameLookup.put(tableName, sheetName);
+
+				for (FieldInfo fieldInfo : tableToFieldInfos.get(tableName))
+					addRow(overviewSheet, tableNameIndexed, fieldInfo.name, fieldInfo.getTypeDescription(), Integer.valueOf(fieldInfo.maxLength), Long.valueOf(fieldInfo.rowCount),
 							Long.valueOf(fieldInfo.nProcessed), fieldInfo.getFractionEmpty());
-				addRow(sheet, "");
+				addRow(overviewSheet, "");
+				sheetIndex += 1;
 			}
 
-			// Create per table sheets
-			for (String table : tables) {
-				sheet = workbook.createSheet(table);
-				List<FieldInfo> fieldInfos = tableToFieldInfos.get(table);
-				List<List<Pair<String, Integer>>> valueCounts = new ArrayList<List<Pair<String, Integer>>>();
+			// Create per table scan values
+			for (String tableName : tables) {
+				Sheet valueSheet = workbook.createSheet(sheetNameLookup.get(tableName));
+
+				List<FieldInfo> fieldInfos = tableToFieldInfos.get(tableName);
+				List<List<Pair<String, Integer>>> valueCounts = new ArrayList<>();
 				Object[] header = new Object[fieldInfos.size() * 2];
 				int maxCount = 0;
 				for (int i = 0; i < fieldInfos.size(); i++) {
@@ -153,7 +164,7 @@ private void generateReport(Map<String, List<FieldInfo>> tableToFieldInfos, Stri
 					if (counts.size() > maxCount)
 						maxCount = counts.size();
 				}
-				addRow(sheet, header);
+				addRow(valueSheet, header);
 				for (int i = 0; i < maxCount; i++) {
 					Object[] row = new Object[fieldInfos.size() * 2];
 					for (int j = 0; j < fieldInfos.size(); j++) {
@@ -166,10 +177,10 @@ private void generateReport(Map<String, List<FieldInfo>> tableToFieldInfos, Stri
 							row[(j * 2) + 1] = "";
 						}
 					}
-					addRow(sheet, row);
+					addRow(valueSheet, row);
 				}
 				// Save some memory by derefencing tables already included in the report:
-				tableToFieldInfos.remove(table);
+				tableToFieldInfos.remove(tableName);
 			}
 		}