-
Notifications
You must be signed in to change notification settings - Fork 27
Commit
Signed-off-by: Milan Gyalai <[email protected]>
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,87 +1,121 @@ | ||
package org.opensearch.migrations.dashboards; | ||
|
||
import java.io.*; | ||
import java.util.Scanner; | ||
import java.util.HashMap; | ||
import java.util.HashSet; | ||
import java.util.Map; | ||
import java.util.Queue; | ||
import java.util.Set; | ||
|
||
import com.google.gson.Gson; | ||
|
||
import org.opensearch.migrations.dashboards.model.Dashboard; | ||
import org.opensearch.migrations.dashboards.converter.DashboardConverter; | ||
import org.opensearch.migrations.dashboards.converter.IndexPatternConverter; | ||
import org.opensearch.migrations.dashboards.converter.QueryConverter; | ||
import org.opensearch.migrations.dashboards.converter.SavedObjectConverter; | ||
import org.opensearch.migrations.dashboards.converter.SearchConverter; | ||
import org.opensearch.migrations.dashboards.converter.UrlConverter; | ||
import org.opensearch.migrations.dashboards.converter.VisualizationConverter; | ||
import org.opensearch.migrations.dashboards.savedobjects.SavedObject; | ||
import org.opensearch.migrations.dashboards.savedobjects.SavedObjectParser; | ||
import org.opensearch.migrations.dashboards.util.Stats; | ||
|
||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
|
||
import lombok.Getter; | ||
import lombok.extern.slf4j.Slf4j; | ||
import picocli.CommandLine; | ||
import picocli.CommandLine.Command; | ||
|
||
@Command(name = "Dashboard Sanitizer", version = "0.1", mixinStandardHelpOptions = true) | ||
@Slf4j | ||
public class Sanitizer implements Runnable{ | ||
public class Sanitizer { | ||
|
||
@CommandLine.Option(names = {"-V", "--version"}, versionHelp = true, description = "display version info") | ||
boolean versionInfoRequested; | ||
private static final Sanitizer INSTANCE = new Sanitizer(); | ||
|
||
@CommandLine.Option(names = {"?", "-h", "--help"}, usageHelp = true, description = "display this help message") | ||
boolean usageHelpRequested; | ||
private final SavedObjectParser savedObjectParser = new SavedObjectParser(); | ||
|
||
@CommandLine.Option(names = {"-s", "--source"}, required = true, description = "The Elastic dashboard object file in ndjson.") | ||
private String sourceFile; | ||
private Queue<SavedObject> processingQueue = new java.util.LinkedList<>(); | ||
|
||
@CommandLine.Option(names = {"-o", "--output"}, required = true, description = "The sanitized OpenSearch dashboard object file in ndjson.", defaultValue = "os-dashboards.ndjson") | ||
private String outputFile; | ||
@Getter | ||
private Stats stats = new Stats(); | ||
|
||
@Override | ||
public void run() { | ||
//check for sourceFile, if empty, print usage and return | ||
if (sourceFile.isEmpty()) { | ||
CommandLine.usage(this, System.out); | ||
} | ||
@SuppressWarnings("rawtypes") | ||
private Map<String, SavedObjectConverter> typeConverters = new HashMap<>() {{ | ||
put("index-pattern", new IndexPatternConverter()); | ||
put("search", new SearchConverter()); | ||
put("dashboard", new DashboardConverter()); | ||
put("visualization", new VisualizationConverter()); | ||
put("url", new UrlConverter()); | ||
put("query", new QueryConverter()); | ||
}}; | ||
|
||
private Set<String> notSupportedTypes = new HashSet<>() {{ | ||
add("map"); | ||
add("canvas-workpad"); | ||
add("canvas-element"); | ||
add("graph-workspace"); | ||
add("connector"); | ||
add("rule"); | ||
add("action"); | ||
add("config"); | ||
add("lens"); | ||
}}; | ||
|
||
public static Sanitizer getInstance() { | ||
return INSTANCE; | ||
} | ||
|
||
@SuppressWarnings({"unchecked", "rawtypes"}) | ||
public String sanitize(String jsonString) { | ||
String result = null; | ||
try { | ||
Scanner scanner = new Scanner(new BufferedInputStream(new FileInputStream(sourceFile))); | ||
BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile)); | ||
Stats stats = sanitizeDashboardsFromFile(scanner, writer); | ||
System.out.printf("%s file is sanitized and output available at %s%n", sourceFile, outputFile); | ||
stats.printStats(); | ||
} catch (IOException e) { | ||
throw new RuntimeException(e); | ||
final SavedObject savedObject = savedObjectParser.load(jsonString); | ||
|
||
if (savedObject == null) { | ||
return null; | ||
Check warning on line 70 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java#L70
|
||
} | ||
|
||
if (typeConverters.containsKey(savedObject.getObjectType())) { | ||
this.addNewObjectToQueue(savedObject); | ||
result = this.processQueue(); | ||
|
||
stats.registerProcessed(); | ||
|
||
} else if (notSupportedTypes.contains(savedObject.getObjectType())) { | ||
log.warn("The object type {} is not supported.", savedObject.getObjectType()); | ||
stats.registerSkipped(savedObject.getObjectType()); | ||
Check warning on line 81 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java#L80-L81
|
||
} else { | ||
log.warn("No converter found for the object type {}.", savedObject.getObjectType()); | ||
stats.registerSkipped(savedObject.getObjectType()); | ||
Check warning on line 84 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java#L83-L84
|
||
} | ||
} catch (JsonProcessingException e) { | ||
log.error("Failed to parse the provided string as a json.", e); | ||
} catch (IllegalArgumentException e) { | ||
log.error("Failed to load the saved object.", e); | ||
Check warning on line 89 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/Sanitizer.java#L86-L89
|
||
} | ||
return result; | ||
} | ||
|
||
public static void main(String[] args) { | ||
CommandLine cmd = new CommandLine(new Sanitizer()); | ||
cmd.parseArgs(args); | ||
if (cmd.isUsageHelpRequested() ) { | ||
cmd.usage(System.out); | ||
return; | ||
} else if (cmd.isVersionHelpRequested()) { | ||
cmd.printVersionHelp(System.out); | ||
return; | ||
|
||
@SuppressWarnings("unchecked") | ||
public String processQueue() { | ||
final StringBuffer buffer = new StringBuffer(); | ||
while (!processingQueue.isEmpty()) { | ||
final SavedObject savedObject = processingQueue.poll(); | ||
|
||
buffer.append(typeConverters.get(savedObject.getObjectType()).convert(savedObject).jsonAsString()); | ||
buffer.append(System.lineSeparator()); | ||
} | ||
int exitCode = cmd.execute(args); | ||
System.exit(exitCode); | ||
|
||
buffer.deleteCharAt(buffer.length() - System.lineSeparator().length()); | ||
return buffer.toString(); | ||
} | ||
|
||
public static Stats sanitizeDashboardsFromFile(Scanner source, BufferedWriter writer) throws IOException { | ||
|
||
Gson gson = new Gson(); | ||
Stats counter = new Stats(); | ||
|
||
while (source.hasNextLine()) { | ||
String line = source.nextLine(); | ||
Dashboard dashboardObject = gson.fromJson(line, Dashboard.class); | ||
// if dashboard id is null, it could be summary line, skip the line | ||
if (dashboardObject.getId() == null) { | ||
counter.registerSkipped(dashboardObject); | ||
continue; | ||
} else if (!dashboardObject.isCompatibleType()) { | ||
counter.registerSkipped(dashboardObject); | ||
continue; | ||
} | ||
public void addNewObjectToQueue(SavedObject savedObject) { | ||
processingQueue.add(savedObject); | ||
} | ||
|
||
dashboardObject.makeCompatibleToOS(); | ||
writer.write(gson.toJson(dashboardObject)); | ||
writer.newLine(); | ||
writer.flush(); | ||
counter.registerProcessed(dashboardObject); | ||
} | ||
return counter; | ||
// supporting Unit testing | ||
public void clearQueue() { | ||
processingQueue.clear(); | ||
} | ||
// supporting Unit testing | ||
public int getQueueSize() { | ||
return processingQueue.size(); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,84 @@ | ||
package org.opensearch.migrations.dashboards; | ||
|
||
import java.io.*; | ||
import java.util.Optional; | ||
import java.util.Scanner; | ||
|
||
import org.opensearch.migrations.dashboards.util.Stats; | ||
|
||
import lombok.extern.slf4j.Slf4j; | ||
import picocli.CommandLine; | ||
import picocli.CommandLine.Command; | ||
|
||
@Command(name = "Dashboard Sanitizer", version = "0.1", mixinStandardHelpOptions = true) | ||
@Slf4j | ||
public class SanitizerCli implements Runnable{ | ||
Check warning on line 15 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L14-L15
|
||
|
||
@CommandLine.Option(names = {"-V", "--version"}, versionHelp = true, description = "display version info") | ||
boolean versionInfoRequested; | ||
|
||
@CommandLine.Option(names = {"?", "-h", "--help"}, usageHelp = true, description = "display this help message") | ||
boolean usageHelpRequested; | ||
|
||
@CommandLine.Option(names = {"-s", "--source"}, required = true, description = "The Elastic dashboard object file in ndjson.") | ||
private String sourceFile; | ||
|
||
@CommandLine.Option(names = {"-o", "--output"}, required = true, description = "The sanitized OpenSearch dashboard object file in ndjson.", defaultValue = "os-dashboards.ndjson") | ||
private String outputFile; | ||
|
||
@Override | ||
public void run() { | ||
//check for sourceFile, if empty, print usage and return | ||
if (sourceFile.isEmpty()) { | ||
CommandLine.usage(this, System.out); | ||
Check warning on line 33 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L33
|
||
} | ||
try (Scanner scanner = new Scanner(new BufferedInputStream(new FileInputStream(sourceFile))); | ||
BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) { | ||
Check warning on line 36 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L35-L36
|
||
|
||
Stats stats = sanitizeDashboardsFromFile(scanner, writer); | ||
log.info("Input file {} is sanitized and output available at %", sourceFile, outputFile); | ||
log.info(stats.printStats()); | ||
} catch (IOException e) { | ||
throw new RuntimeException(e); | ||
} | ||
} | ||
Check warning on line 44 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L38-L44
|
||
|
||
public static void main(String[] args) { | ||
CommandLine cmd = new CommandLine(new SanitizerCli()); | ||
Check warning on line 47 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L47
|
||
try { | ||
cmd.parseArgs(args); | ||
Check warning on line 49 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L49
|
||
|
||
if (cmd.isUsageHelpRequested() ) { | ||
cmd.usage(System.out); | ||
return; | ||
Check warning on line 53 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L52-L53
|
||
} else if (cmd.isVersionHelpRequested()) { | ||
cmd.printVersionHelp(System.out); | ||
return; | ||
Check warning on line 56 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L55-L56
|
||
} | ||
int exitCode = cmd.execute(args); | ||
System.exit(exitCode); | ||
} catch (CommandLine.ParameterException e) { | ||
System.err.println(e.getMessage()); | ||
cmd.usage(System.err); | ||
System.exit(1); | ||
} | ||
Check warning on line 64 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L58-L64
|
||
|
||
} | ||
Check warning on line 66 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L66
|
||
|
||
public static Stats sanitizeDashboardsFromFile(Scanner source, BufferedWriter writer) throws IOException { | ||
Sanitizer sanitizer = Sanitizer.getInstance(); | ||
Check warning on line 69 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L69
|
||
|
||
while (source.hasNextLine()) { | ||
String line = source.nextLine(); | ||
String sanitizedLine = sanitizer.sanitize(line); | ||
Check warning on line 73 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L72-L73
|
||
|
||
if (sanitizedLine == null) { | ||
continue; | ||
Check warning on line 76 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L76
|
||
} | ||
writer.write(sanitizedLine); | ||
writer.newLine(); | ||
writer.flush(); | ||
} | ||
return sanitizer.getStats(); | ||
Check warning on line 82 in dashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java Codecov / codecov/patchdashboardsSanitizer/src/main/java/org/opensearch/migrations/dashboards/SanitizerCli.java#L78-L82
|
||
} | ||
} |