diff --git a/.gitignore b/.gitignore index 75543ad..ae3ef88 100644 --- a/.gitignore +++ b/.gitignore @@ -283,3 +283,4 @@ local.properties # .nfs files are created when an open file is removed but is still being accessed .nfs* +docs/readthedocs/_build/ \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..90e935b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,21 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-20.04 + tools: + python: "3.9" + +# Configure python +python: + install: + - requirements: docs/readthedocs/requirements.txt + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/readthedocs/conf.py \ No newline at end of file diff --git a/README.md b/README.md index a15ef56..61ddfc5 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ the whole area. |vertical levels | 60 level | |upper boundary | 22.5km | -![](doc/img/icon-eu-nest.png) +![](docs/readthedocs/_static/img/icon-eu-nest.png) In contrast, the converter area can be set in the code to the area of interest. @@ -88,7 +88,7 @@ In the future it would make sense to provide the coordinates via a config file instead of hard-coding them into the binary. If you want to provide this future feel free to hand in an issue or a pull request. -![](doc/img/converter_covered_area.jpg) +![](docs/readthedocs/_static/img/converter_covered_area.jpg) ### Requirements diff --git a/build.gradle b/build.gradle index cb5a55f..7697501 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,6 @@ +import com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer +import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer + plugins { id "com.jfrog.artifactory" version "4.24.20" //artifactory support id 'groovy' // groovy support @@ -11,6 +14,8 @@ plugins { id 'com.github.onslip.gradle-one-jar' version '1.0.6' // pack a self contained jar id 'jacoco' // java code coverage plugin id "org.sonarqube" version "3.3" // sonarqube + id "kr.motd.sphinx" version "2.10.0" + id "com.github.johnrengelman.shadow" version "6.1.0" // fat jar } ext { @@ -31,13 +36,11 @@ defaultTasks 'build' sourceCompatibility = javaVersion targetCompatibility = javaVersion - apply from: scriptsLocation + 'pmd.gradle' apply from: scriptsLocation + 'spotbugs.gradle' apply from: scriptsLocation + 'spotless.gradle' apply from: scriptsLocation + 'modernizer.gradle' apply from: scriptsLocation + 'checkJavaVersion.gradle' -apply from: scriptsLocation + 'fatJar.gradle' apply from: scriptsLocation + 'tests.gradle' apply from: scriptsLocation + 'jacoco.gradle' // jacoco java code coverage apply from: scriptsLocation + 'sonarqube.gradle' // sonarqube config @@ -125,7 +128,7 @@ publishing { from components.java artifact sourcesJar artifact javadocJar - artifact fatJar + artifact shadowJar versionMapping { usage('java-api') { fromResolutionOf('runtimeClasspath') @@ -151,7 +154,7 @@ tasks.withType(JavaCompile) { } application { - mainClassName = mainClass + mainClassName = "edu.ie3.tools.Main" } jar { @@ -166,3 +169,10 @@ run { // Gradle uses an empty Input as default, leading to a non-blocking behaviour and thus an immediate shutdown standardInput = System.in } + +shadowJar { + transform(AppendingTransformer) { + resource = 'reference.conf' + } + zip64 = true +} diff --git a/docs/readthedocs/Makefile b/docs/readthedocs/Makefile new file mode 100644 index 0000000..bfd7a7e --- /dev/null +++ b/docs/readthedocs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= python3 -msphinx +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/readthedocs/_static/css/theme_override.css b/docs/readthedocs/_static/css/theme_override.css new file mode 100644 index 0000000..914a128 --- /dev/null +++ b/docs/readthedocs/_static/css/theme_override.css @@ -0,0 +1,13 @@ +/* Suppress spacing after multi-line entries in tables + * Inspired by this issue: https://github.com/readthedocs/sphinx_rtd_theme/issues/117 */ +.wy-table-responsive table td div.line-block { + margin-bottom: 0; + font-size: 90%; +} +.wy-table-responsive table th div.line-block { + margin-bottom: 0; + font-size: 90%; +} +.wy-table-responsive table th p { + margin-bottom: 0; +} \ No newline at end of file diff --git a/doc/img/converter_covered_area.jpg b/docs/readthedocs/_static/img/converter_covered_area.jpg similarity index 100% rename from doc/img/converter_covered_area.jpg rename to docs/readthedocs/_static/img/converter_covered_area.jpg diff --git a/doc/img/icon-eu-nest.png b/docs/readthedocs/_static/img/icon-eu-nest.png similarity index 100% rename from doc/img/icon-eu-nest.png rename to docs/readthedocs/_static/img/icon-eu-nest.png diff --git a/docs/readthedocs/conf.py b/docs/readthedocs/conf.py new file mode 100644 index 0000000..a276c49 --- /dev/null +++ b/docs/readthedocs/conf.py @@ -0,0 +1,72 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'DWDWeatherTools' +copyright = u'2022. TU Dortmund University, Institute of Energy Systems, Energy Efficiency and Energy Economics, Research group Distribution grid planning and operation ' +author = 'Johannes Hiry, Debopama Sen Sarma, Chris Kittl, Sebastian Peter, Thomas Oberließen' + +# The full version, including alpha/beta/rc tags +version = '0.1.' +release = '0.1.1-SNAPSHOT' + +pygments_style = 'tango' +add_function_parentheses = True +# Will point sphinx to use 'index.rst' as the master document +master_doc = 'index' + +# -- General configuration --------------------------------------------------- +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autosectionlabel', + 'myst_parser' +] + +# Prefix all autogenerated labels wit the document to get unique labels (e.g. `index:Hello`) +autosectionlabel_prefix_document = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'venv'] + +source_encoding = 'utf-8-sig' + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' +html_short_title = "DWDWeatherTools" +htmlhelp_basename = 'DWDWeatherTools-doc' +html_use_index = True +html_show_sourcelink = False + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +def setup(app): + app.add_css_file("css/theme_override.css") diff --git a/docs/readthedocs/index.rst b/docs/readthedocs/index.rst new file mode 100644 index 0000000..242feec --- /dev/null +++ b/docs/readthedocs/index.rst @@ -0,0 +1,31 @@ +Documentation of the DWDTools +========================================= + +Hello weather! + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + user/users + + + +Contact the (Main) Maintainers +------------------------------ +If you feel, something is missing, wrong or misleading, please contact one of our main contributors: + + * `@sensarmad `_ + * `@johanneshiry `_ + * `@ckittl `_ + * `@t-ober `_ + * `@sebastian-peter `_ + +Hat tip to all other contributors! + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/readthedocs/make.bat b/docs/readthedocs/make.bat new file mode 100644 index 0000000..922152e --- /dev/null +++ b/docs/readthedocs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/readthedocs/requirements.txt b/docs/readthedocs/requirements.txt new file mode 100644 index 0000000..e014332 --- /dev/null +++ b/docs/readthedocs/requirements.txt @@ -0,0 +1,3 @@ +myst-parser==0.16.1 +Sphinx==4.2.0 +sphinx-rtd-theme==1.0.0 \ No newline at end of file diff --git a/docs/readthedocs/user/users.md b/docs/readthedocs/user/users.md new file mode 100644 index 0000000..a6be67b --- /dev/null +++ b/docs/readthedocs/user/users.md @@ -0,0 +1,3 @@ +# Users Guide + +You got this! diff --git a/gradle/scripts/fatJar.gradle b/gradle/scripts/fatJar.gradle deleted file mode 100644 index 5f3823d..0000000 --- a/gradle/scripts/fatJar.gradle +++ /dev/null @@ -1,13 +0,0 @@ -tasks.register('fatJar', Jar){ - getArchiveClassifier().set('fat') - manifest { - attributes( - 'Main-Class': mainClass - ) - } - //baseName = project.archivesBaseName + '-fat' - from { - configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } - } - with jar -} diff --git a/src/main/java/edu/ie3/tools/Converter.java b/src/main/java/edu/ie3/tools/Converter.java index c8545bf..4da029c 100644 --- a/src/main/java/edu/ie3/tools/Converter.java +++ b/src/main/java/edu/ie3/tools/Converter.java @@ -35,6 +35,8 @@ * @version 4.0 */ public class Converter implements Runnable { + private ZonedDateTime convertFrom; + private ZonedDateTime convertUntil; public static final Logger logger = LogManager.getLogger(Converter.class); public static final Logger fileStatusLogger = LogManager.getLogger("FileStatus"); @@ -50,23 +52,11 @@ public class Converter implements Runnable { private final ExecutorService fileEraserExecutor = Executors.newFixedThreadPool((int) Math.ceil(noOfProcessors / 3d)); - /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */ - public static String getFormattedTimestep(@NotNull ZonedDateTime modelrun, int timestep) { - return "MR " - + MODEL_RUN_FORMATTER.format(modelrun) - + " - TS " - + String.format("%02d", timestep) - + " | "; - } + public Converter() {} - /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 | ") */ - public static String getFormattedModelrun(@NotNull ZonedDateTime modelrun) { - return "MR " + MODEL_RUN_FORMATTER.format(modelrun) + " | "; - } - - /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */ - public static String getFormattedTimestep(@NotNull FileModel file) { - return getFormattedTimestep(file.getModelrun(), file.getTimestep()); + public Converter(ZonedDateTime convertFrom, ZonedDateTime convertUntil) { + this.convertFrom = convertFrom; + this.convertUntil = convertUntil; } @Override @@ -76,10 +66,21 @@ public void run() { logger.setLevel(Main.debug ? Level.ALL : Level.INFO); printInit(); validateConnectionProperties(); - convert(); + if (this.convertFrom != null && this.convertUntil != null) { + convert(this.convertFrom, this.convertUntil); + } else { + convertUntilNewest(); + } } else logger.info("Converter is already running."); } + public void printInit() { + logger.info("________________________________________________________________________________"); + logger.info("Converter started"); + logger.trace("Program arguments:"); + Main.printProgramArguments().forEach(s -> logger.trace(" " + s)); + } + /** Validates Connection Properties from user input */ private void validateConnectionProperties() { Properties receivedProperties = new Properties(); @@ -103,51 +104,47 @@ private void validateConnectionProperties() { dbController = new DatabaseController(PERSISTENCE_UNIT_NAME, receivedProperties); } - public void printInit() { - logger.info("________________________________________________________________________________"); - logger.info("Converter started"); - logger.trace("Program arguments:"); - Main.printProgramArguments().forEach(s -> logger.trace(" " + s)); + private void convertUntilNewest() { + // retrieves the starting modelrun ( = oldest modelrun where persisted==false or no converter + // run info available) + ZonedDateTime currentModelrun = + (ZonedDateTime) + dbController.execSingleResultNamedQuery( + FileModel.OldestModelrunWithUnprocessedFiles, Collections.emptyList()); + + // retrieves the newest possible modelrun ( = newest downloaded modelrun) + ZonedDateTime newestPossibleModelrun = + (ZonedDateTime) + dbController.execSingleResultNamedQuery( + FileModel.NewestDownloadedModelrun, Collections.emptyList()); + + convert(currentModelrun, newestPossibleModelrun); } - private void convert() { + private void convert(ZonedDateTime start, ZonedDateTime end) { String formattedModelrun = ""; try { fileEraser = new FileEraser(edu.ie3.tools.Main.directory, dbController); - // retrieves the newest possible modelrun ( = newest downloaded modelrun) - ZonedDateTime newestPossibleModelrun = - (ZonedDateTime) - dbController.execSingleResultNamedQuery( - FileModel.NewestDownloadedModelrun, Collections.emptyList()); - - // retrieves the starting modelrun ( = oldest modelrun where persisted==false or no converter - // run info available) - ZonedDateTime currentModelrun = - (ZonedDateTime) - dbController.execSingleResultNamedQuery( - FileModel.OldestModelrunWithUnprocessedFiles, Collections.emptyList()); - - if (currentModelrun != null) { + if (start != null) { coordinates = getCoordinates(); - while (currentModelrun.isBefore(newestPossibleModelrun) - || currentModelrun.isEqual(newestPossibleModelrun)) { + while (start.isBefore(end) || start.isEqual(end)) { logger.info( "############################### " - + MODEL_RUN_FORMATTER.format(currentModelrun) + + MODEL_RUN_FORMATTER.format(start) + " ###############################"); - formattedModelrun = getFormattedModelrun(currentModelrun); + formattedModelrun = getFormattedModelrun(start); long tic; long toc; tic = System.currentTimeMillis(); for (int timestep = 0; timestep < edu.ie3.tools.Main.timesteps; timestep++) { - handleTimestep(currentModelrun, timestep); + handleTimestep(start, timestep); dbController.flush(); } toc = System.currentTimeMillis(); logger.debug(formattedModelrun + "This modelrun took " + (toc - tic) / 60000 + "m \n"); - currentModelrun = currentModelrun.plusHours(3); // increment modelrun + start = start.plusHours(3); // increment modelrun } } } catch (Exception e) { @@ -157,6 +154,11 @@ private void convert() { } } + /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 | ") */ + public static String getFormattedModelrun(@NotNull ZonedDateTime modelrun) { + return "MR " + MODEL_RUN_FORMATTER.format(modelrun) + " | "; + } + /** opens archive files and converts the data for one timestep */ private void handleTimestep(ZonedDateTime currentModelrun, int timestep) { String formattedTimestep = getFormattedTimestep(currentModelrun, timestep); @@ -165,7 +167,7 @@ private void handleTimestep(ZonedDateTime currentModelrun, int timestep) { logger.info(formattedTimestep + "Opening of archive files started"); long tic, toc; tic = System.currentTimeMillis(); - openArchiveFiles(currentModelrun, timestep); + decompressGribFiles(currentModelrun, timestep); toc = System.currentTimeMillis(); logger.info( formattedTimestep + "Opening of archive files finished (" + (toc - tic) / 1000 + "s)"); @@ -175,13 +177,33 @@ private void handleTimestep(ZonedDateTime currentModelrun, int timestep) { logger.info(formattedTimestep + "Timestep finished"); } - private void openArchiveFiles(ZonedDateTime currentModelrun, int timestep) { + /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */ + public static String getFormattedTimestep(@NotNull ZonedDateTime modelrun, int timestep) { + return "MR " + + MODEL_RUN_FORMATTER.format(modelrun) + + " - TS " + + String.format("%02d", timestep) + + " | "; + } + + /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */ + public static String getFormattedTimestep(@NotNull FileModel file) { + return getFormattedTimestep(file.getModelrun(), file.getTimestep()); + } + + /** + * Decompresses all grib files that are intact for every parameter. + * + * @param currentModelrun the model run for which to decompress files + * @param timestep the time step that is currently handled + */ + private void decompressGribFiles(ZonedDateTime currentModelrun, int timestep) { String folderpath = Main.directory + File.separator + FILENAME_DATE_FORMATTER.format(currentModelrun) + File.separator; - List tasks = new ArrayList<>(); + List fileDecompressors = new ArrayList<>(); List files = new ArrayList<>(); for (Parameter param : Parameter.values()) { FileModel file = @@ -189,30 +211,18 @@ private void openArchiveFiles(ZonedDateTime currentModelrun, int timestep) { FileModel.class, FileModel.createFileName(currentModelrun, timestep, param)); if (file != null) { files.add(file); - if (file.isSufficient_size() && (file.isValid_file() == null || file.isValid_file())) { + // add file to the files to decompress if it's valid + if (isFileIntact(file)) { if (!file.isPersisted() && !file.isArchivefile_deleted() && !file.isDecompressed()) { - tasks.add(new Decompressor(file, folderpath)); - } - } else if (file.getDownload_fails() > 3 - || file.getModelrun().isBefore(ZonedDateTime.now().minusDays(1))) { - if (edu.ie3.tools.Main.deleteDownloadedFiles) { - logger.trace( - "Delete file " - + file.getName() - + " because it did not have a valid size or content."); - fileEraserExecutor.submit(fileEraser.eraseCallable(file)); - } else { - logger.trace( - "File " - + file.getName() - + " did not have a valid size or content. If you want to delete it pass -del as argument!"); + fileDecompressors.add(new Decompressor(file, folderpath)); } - } + } else handleBrokenFile(file); } } try { - decompressionExecutor.invokeAll(tasks); + // actual decompression of files + decompressionExecutor.invokeAll(fileDecompressors); } catch (InterruptedException e) { e.printStackTrace(); } @@ -228,6 +238,37 @@ private void openArchiveFiles(ZonedDateTime currentModelrun, int timestep) { }); } + /** + * Checks if file is sufficiently large and is valid + * + * @param file the file to check + * @return whether it is intact or not + */ + private Boolean isFileIntact(FileModel file) { + return file.isSufficient_size() && (file.isValid_file() == null || file.isValid_file()); + } + + /** + * Either delete broken file or log it depending on configuration. + * + * @param file the broken file to handle + */ + private void handleBrokenFile(FileModel file) { + if (file.getDownloadFails() > 3 + || file.getModelrun().isBefore(ZonedDateTime.now().minusDays(1))) { + if (edu.ie3.tools.Main.deleteDownloadedFiles) { + logger.trace( + "Delete file " + file.getName() + " because it did not have a valid size or content."); + fileEraserExecutor.submit(fileEraser.eraseCallable(file)); + } else { + logger.trace( + "File " + + file.getName() + + " did not have a valid size or content. If you want to delete it pass -del as argument!"); + } + } + } + /** * Calls {@link Converter#convertTimeStep(ZonedDateTime, int, String)} with default folderpath *
@@ -485,10 +526,10 @@ private void shutdownAllExecutors() { private Collection getCoordinates() { HashMap namedCoordinateParams = new HashMap<>(); - namedCoordinateParams.put("minLatitude", Main.minLatitude); - namedCoordinateParams.put("maxLatitude", Main.maxLatitude); - namedCoordinateParams.put("minLongitude", Main.minLongitude); - namedCoordinateParams.put("maxLongitude", Main.maxLongitude); + namedCoordinateParams.put("minLatitude", Main.MIN_LATITUDE); + namedCoordinateParams.put("maxLatitude", Main.MAX_LATITUDE); + namedCoordinateParams.put("minLongitude", Main.MIN_LONGITUDE); + namedCoordinateParams.put("maxLongitude", Main.MAX_LONGITUDE); return dbController.execNamedQuery( CoordinateModel.CoordinatesInRectangle, namedCoordinateParams); } diff --git a/src/main/java/edu/ie3/tools/Downloader.java b/src/main/java/edu/ie3/tools/Downloader.java index 9894309..6d1f0a2 100644 --- a/src/main/java/edu/ie3/tools/Downloader.java +++ b/src/main/java/edu/ie3/tools/Downloader.java @@ -181,7 +181,7 @@ public boolean download(ZonedDateTime modelrun, Parameter param) { */ public boolean downloadFile(String folder, FileModel filemodel) { boolean success = false; - if (!filemodel.isSufficient_size() && filemodel.getDownload_fails() < 3) { + if (!filemodel.isSufficient_size() && filemodel.getDownloadFails() < 3) { String url = filemodel.getURL(); try { if (isUrlReachable(url)) { diff --git a/src/main/java/edu/ie3/tools/Extractor.java b/src/main/java/edu/ie3/tools/Extractor.java index 4191312..6ee73b5 100644 --- a/src/main/java/edu/ie3/tools/Extractor.java +++ b/src/main/java/edu/ie3/tools/Extractor.java @@ -42,9 +42,15 @@ public Extractor( formattedTimestep = Converter.getFormattedTimestep(file); } + /** + * Checks headline of grib file whether it is valid. Since eccodes v2.21.0 the data extraction + * expects a headline w/o commas but with whitespaces see ECC-1197 - + * https://jira.ecmwf.int/browse/ECC-1197 + * + * @param headlineString the headline string + * @return whether it is valid or not + */ private boolean validHeadline(String headlineString) { - // since eccodes v2.21.0 the data extraction expects a headline w/o commas but with whitespaces - // see ECC-1197 - https://jira.ecmwf.int/browse/ECC-1197 String headline = "Latitude Longitude Value"; String oldHeadline = "Latitude, Longitude, Value"; @@ -52,6 +58,14 @@ private boolean validHeadline(String headlineString) { && (headlineString.trim().equals(headline) || headlineString.trim().equals(oldHeadline)); } + /** + * Parses the output of a parameter grib file. And returns a map from coordinate where the value + * was measured to actual value fixme: coordinate without id? + * + * @param reader the reader of the grib file input stream + * @return map from coordinate where the value was measured to actual value + * @throws IOException + */ protected HashMap parse(BufferedReader reader) throws IOException { HashMap coordinateToValue = new HashMap<>(720729); try { @@ -161,7 +175,9 @@ private ExtractorResult extractParameters() throws IOException { if (e.getMessage().contains("Cannot run program")) { logger.error( e - + ". Are eccodes (https://confluence.ecmwf.int/display/ECC) installed and did you pass the correct path of the eccodes for a custom install location (-eccodes=)? "); + + ". Are eccodes (https://confluence.ecmwf.int/display/ECC) installed and did you pass the correct path of the eccodes (current path is " + + eccodesLocation + + " ) for a custom install location (-eccodes=)? "); } else { logger.error(e); validFile = false; diff --git a/src/main/java/edu/ie3/tools/Main.java b/src/main/java/edu/ie3/tools/Main.java index 1acccd2..5ef1bfb 100644 --- a/src/main/java/edu/ie3/tools/Main.java +++ b/src/main/java/edu/ie3/tools/Main.java @@ -7,7 +7,11 @@ package edu.ie3.tools; import java.io.File; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.HashSet; +import java.util.Locale; import java.util.Set; import java.util.TimeZone; import picocli.CommandLine; @@ -57,6 +61,16 @@ public class Main { description = "Converts grib2 files") public static boolean doConvert; + @CommandLine.Option( + names = {"-convert_from"}, + description = "Start datetime of conversion. Format: yyyy-MM-dd HH:mm:ss") + public static String convertFrom; + + @CommandLine.Option( + names = {"-convert_until"}, + description = "End datetime of conversion. Format: yyyy-MM-dd HH:mm:ss") + public static String convertUntil; + @CommandLine.Option( names = {"download"}, description = "Downloads grib2 files") @@ -114,10 +128,10 @@ public class Main { public static boolean debug = false; // Coordinate-Rectangle to convert - public static final double minLongitude = 4.29694; - public static final double maxLongitude = 18.98635; - public static final double minLatitude = 45.71457; - public static final double maxLatitude = 57.65129; + public static final double MIN_LONGITUDE = 4.29694; + public static final double MAX_LONGITUDE = 18.98635; + public static final double MIN_LATITUDE = 45.71457; + public static final double MAX_LATITUDE = 57.65129; public static void main(String[] args) { // ICON-EU data timezone is UTC time, @@ -143,8 +157,23 @@ public static void main(String[] args) { if (!eccodes.isEmpty()) eccodes += File.separator; eccodes += "grib_get_data"; } - if (doDownload) new Downloader().run(); - if (doConvert) new Converter().run(); + + if (doConvert) { + if (convertFrom != null && convertUntil != null) { + DateTimeFormatter dateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + .withZone(ZoneId.of("UTC")) + .withLocale(Locale.GERMANY); + ZonedDateTime from = ZonedDateTime.parse(convertFrom, dateTimeFormatter); + ZonedDateTime until = ZonedDateTime.parse(convertUntil, dateTimeFormatter); + new Converter(from, until).run(); + } else if (convertFrom != null || convertUntil != null) { + throw new IllegalArgumentException( + "Either convertFrom or convertTo is missing. We need both to convert data for a specific interval"); + } else { + new Downloader().run(); + } + } } public static Set printProgramArguments() { diff --git a/src/main/java/edu/ie3/tools/models/persistence/FileModel.java b/src/main/java/edu/ie3/tools/models/persistence/FileModel.java index ba4f6e9..89d7d54 100644 --- a/src/main/java/edu/ie3/tools/models/persistence/FileModel.java +++ b/src/main/java/edu/ie3/tools/models/persistence/FileModel.java @@ -106,6 +106,15 @@ public void setName(String name) { this.name = name; } + /** + * Creates the correct file name within the icon model for a parameter within a given time step + * for a given model run + * + * @param modelrun the model run to consider + * @param timestep the time step to check + * @param parameter the considered parameter + * @return a String of the expected file name + */ public static String createFileName(ZonedDateTime modelrun, int timestep, Parameter parameter) { String name = parameter.getPrefix(); // ie icon-eu_europe_regular-lat-lon_single-level_ name += FILENAME_DATE_FORMATTER.format(modelrun) + "_"; // ie 2018090512_ @@ -134,11 +143,11 @@ public void setParameter(Parameter parameter) { this.parameter = parameter; } - public int getDownload_fails() { + public int getDownloadFails() { return download_fails; } - public void setDownload_fails(int download_fails) { + public void setDownloadFails(int download_fails) { this.download_fails = download_fails; } @@ -198,6 +207,11 @@ public void setPersisted(boolean persisted) { this.persisted = persisted; } + /** + * Checks if the raw file downloaded from the model run is deleted. + * + * @return whether it is deleted or not + */ public boolean isArchivefile_deleted() { return archivefile_deleted; } diff --git a/src/main/java/edu/ie3/tools/utils/DatabaseController.java b/src/main/java/edu/ie3/tools/utils/DatabaseController.java index da0cf53..2f205d4 100644 --- a/src/main/java/edu/ie3/tools/utils/DatabaseController.java +++ b/src/main/java/edu/ie3/tools/utils/DatabaseController.java @@ -275,6 +275,14 @@ public void persist(Serializable entity) { manager.joinTransaction(); } + /** + * Looks for an object within the database via using its class and primary key + * + * @param clazz the class representation of the object to look for + * @param id the primary key + * @param the class of the object to look for + * @return the entity if it is found or null otherwise + */ public C find(Class clazz, Object id) { C entity = null; try { @@ -318,6 +326,14 @@ public List execNamedQuery(String queryName, Map namedParams) { return objs; } + /** + * Execute a named query. Named queries are defined in @NamedQueries sections and are SQL queries + * that can be used as arguments via their respective name. + * + * @param queryName the name of the query to execute + * @param params additional parameters to set within the named query + * @return the result of the query as a generic object + */ public Object execSingleResultNamedQuery(String queryName, List params) { Object res = null; try {