Add option to use local files instead of download.

.gitignore

@@ -153,4 +153,4 @@ testData/download
 !testData/download/targetFolderExistsAndIsFile.zip
 
 # Downloader productive folder
-inputData/download
+input/download

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Option to use local files instead of download [#256](https://github.com/ie3-institute/simBench2psdm/issues/256)
+
+
 ## [1.0.0] - 2021-08-03
 ### Added
 -   Basic functionality to convert SimBench data sets to [PowerSystemDataModel](https://github.com/ie3-institute/powersystemdatamodel)

build.gradle

@@ -18,7 +18,7 @@ ext {
 
 	scalaVersion = '2.13'
 	scalaBinaryVersion = '2.13.14'
-	tscfgVersion = '0.9.986'
+	tscfgVersion = '1.0.0'
 	slf4jVersion = '2.0.13'
 
 	scriptsLocation = 'gradle' + File.separator + 'scripts' + File.separator //location of script plugins
@@ -87,8 +87,8 @@ dependencies {
 	implementation 'org.mockito:mockito-core:5.12.0' // mocking framework
 
 	// config //
-	implementation 'com.typesafe:config:+'
-	implementation "com.github.carueda:tscfg_2.13:${tscfgVersion}"
+	implementation 'com.typesafe:config:1.4.3'
+	implementation "com.github.carueda:tscfg_2.13:$tscfgVersion"
 
 	// cmd args parser //
 	implementation "com.github.scopt:scopt_${scalaVersion}:+"

gradle/scripts/tscfg.gradle

@@ -5,7 +5,7 @@ task genConfigClass {
 	doLast {
 		def tscfgJarFile = project.file('build/tscfg-' + tscfgVersion + '.jar')
 		if (!tscfgJarFile.exists() || !tscfgJarFile.isFile()) {
-			download {
+			download.run {
 				src 'https://github.com/carueda/tscfg/releases/download/v' + tscfgVersion + '/tscfg-' + tscfgVersion + '.jar'
 				dest buildDir
 			}

input/config/localFileSourceConfig.conf

@@ -0,0 +1,34 @@
+# Simple config to convert only one simple lv grid
+io {
+  input {
+    folder = "input/local/"
+
+    localFile = {
+        isZipped = true
+    }
+
+    csv = {
+      fileEncoding = "UTF-8"
+      fileEnding = ".csv"
+      separator = ";"
+      directoryHierarchy = false
+    }
+  }
+
+  output {
+    csv = {
+      fileEncoding = "UTF-8"
+      fileEnding = ".csv"
+      separator = ";"
+      directoryHierarchy = false
+    }
+
+    targetFolder = "convertedData/local/"
+    compress = false
+  }
+
+  # code for local files
+  simbenchCodes = [
+    "1-LV-rural1--0-no_sw"
+  ]
+}

src/main/resources/config-template.conf

@@ -9,9 +9,18 @@ CsvConfig {
 io {
   simbenchCodes = ["String"]
   input {
-    download.baseUrl = "String" | "http://141.51.193.167/simbench/gui/usecase/download"
-    download.folder = "String" | "inputData/download/"
-    download.failOnExistingFiles = "Boolean" | false
+    folder = "String" | "input/download/"
+
+    #@optional
+    localFile {
+        isZipped = "Boolean" | true
+        failOnExistingFiles = "Boolean" | false
+    }
+    #@optional
+    download {
+        baseUrl = "String" | "http://141.51.193.167/simbench/gui/usecase/download"
+        failOnExistingFiles = "Boolean" | false
+    }
     csv = CsvConfig
   }
   output {

src/main/scala/edu/ie3/simbench/config/ConfigValidator.scala

@@ -36,6 +36,7 @@ case object ConfigValidator {
   @throws[SimbenchException]
   private def checkValidity(io: SimbenchConfig.Io): Unit = {
     checkSimbenchCodes(io.simbenchCodes)
+    checkFileSource(io.input)
   }
 
   /** Checks the validity of the provided codes with the help of the permissible
@@ -47,11 +48,33 @@ case object ConfigValidator {
     */
   @throws[CodeValidationException]
   private def checkSimbenchCodes(codes: List[java.lang.String]): Unit = {
+    if (codes.isEmpty) {
+      throw new SimbenchConfigException(s"No simbench codes were provided!")
+    }
+
     for (code <- codes) {
       SimbenchCode.isValid(code) match {
         case Success(_)         =>
         case Failure(exception) => throw exception
       }
     }
   }
+
+  @throws[SimbenchException]
+  private def checkFileSource(cfg: SimbenchConfig.Io.Input): Unit = {
+    val sources = Vector(
+      cfg.localFile,
+      cfg.download
+    ).filter(_.isDefined)
+
+    sources.size match {
+      case 0 =>
+        throw new SimbenchConfigException(s"No file sources defined in: $cfg")
+      case 1 =>
+      case n =>
+        throw new SimbenchConfigException(
+          s"Too many file sources ($n) defined in: $cfg"
+        )
+    }
+  }
 }

src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala

@@ -1,4 +1,4 @@
-// generated by tscfg 0.9.986 on Mon Aug 09 20:12:24 CEST 2021
+// generated by tscfg 1.0.0 on Thu Aug 08 13:29:09 CEST 2024
 // source: src/main/resources/config-template.conf
 
 package edu.ie3.simbench.config
@@ -47,9 +47,8 @@ object SimbenchConfig {
         $tsCfgValidator: $TsCfgValidator
     ): SimbenchConfig.Conversion = {
       SimbenchConfig.Conversion(
-        removeSwitches = c.hasPathOrNull("removeSwitches") && c.getBoolean(
-          "removeSwitches"
-        )
+        removeSwitches =
+          c.hasPathOrNull("removeSwitches") && c.getBoolean("removeSwitches")
       )
     }
   }
@@ -62,13 +61,14 @@ object SimbenchConfig {
   object Io {
     final case class Input(
         csv: SimbenchConfig.CsvConfig,
-        download: SimbenchConfig.Io.Input.Download
+        download: scala.Option[SimbenchConfig.Io.Input.Download],
+        folder: java.lang.String,
+        localFile: scala.Option[SimbenchConfig.Io.Input.LocalFile]
     )
     object Input {
       final case class Download(
           baseUrl: java.lang.String,
-          failOnExistingFiles: scala.Boolean,
-          folder: java.lang.String
+          failOnExistingFiles: scala.Boolean
       )
       object Download {
         def apply(
@@ -80,13 +80,30 @@ object SimbenchConfig {
             baseUrl =
               if (c.hasPathOrNull("baseUrl")) c.getString("baseUrl")
               else "http://141.51.193.167/simbench/gui/usecase/download",
+            failOnExistingFiles =
+              c.hasPathOrNull("failOnExistingFiles") && c.getBoolean(
+                "failOnExistingFiles"
+              )
+          )
+        }
+      }
+
+      final case class LocalFile(
+          failOnExistingFiles: scala.Boolean,
+          isZipped: scala.Boolean
+      )
+      object LocalFile {
+        def apply(
+            c: com.typesafe.config.Config,
+            parentPath: java.lang.String,
+            $tsCfgValidator: $TsCfgValidator
+        ): SimbenchConfig.Io.Input.LocalFile = {
+          SimbenchConfig.Io.Input.LocalFile(
             failOnExistingFiles =
               c.hasPathOrNull("failOnExistingFiles") && c.getBoolean(
                 "failOnExistingFiles"
               ),
-            folder =
-              if (c.hasPathOrNull("folder")) c.getString("folder")
-              else "inputData/download/"
+            isZipped = !c.hasPathOrNull("isZipped") || c.getBoolean("isZipped")
           )
         }
       }
@@ -103,12 +120,29 @@ object SimbenchConfig {
             parentPath + "csv.",
             $tsCfgValidator
           ),
-          download = SimbenchConfig.Io.Input.Download(
-            if (c.hasPathOrNull("download")) c.getConfig("download")
-            else com.typesafe.config.ConfigFactory.parseString("download{}"),
-            parentPath + "download.",
-            $tsCfgValidator
-          )
+          download =
+            if (c.hasPathOrNull("download"))
+              scala.Some(
+                SimbenchConfig.Io.Input.Download(
+                  c.getConfig("download"),
+                  parentPath + "download.",
+                  $tsCfgValidator
+                )
+              )
+            else None,
+          folder =
+            if (c.hasPathOrNull("folder")) c.getString("folder")
+            else "input/download/",
+          localFile =
+            if (c.hasPathOrNull("localFile"))
+              scala.Some(
+                SimbenchConfig.Io.Input.LocalFile(
+                  c.getConfig("localFile"),
+                  parentPath + "localFile.",
+                  $tsCfgValidator
+                )
+              )
+            else None
         )
       }
     }
@@ -208,7 +242,7 @@ object SimbenchConfig {
     java.lang.String.valueOf(cv.unwrapped())
   }
 
-  private final class $TsCfgValidator {
+  final class $TsCfgValidator {
     private val badPaths =
       scala.collection.mutable.ArrayBuffer[java.lang.String]()
 

src/main/scala/edu/ie3/simbench/io/Downloader.scala

@@ -10,7 +10,6 @@ import edu.ie3.simbench.model.SimbenchCode
 
 import scala.language.postfixOps
 import scala.sys.process._
-import scala.util.{Failure, Success, Try}
 
 final case class Downloader(
     downloadFolder: String,

src/main/scala/edu/ie3/simbench/io/SimbenchReader.scala

@@ -87,6 +87,12 @@ final case class SimbenchReader(
      * Await is okay here */
     val modelClassToRawData = getFieldToValueMaps
 
+    if (modelClassToRawData.forall(_._2.isEmpty)) {
+      throw SimbenchDataModelException(
+        s"Model is empty. Please check the provided data."
+      )
+    }
+
     /* Extracting all profiles */
     val loadProfiles = buildModels(modelClassToRawData, LoadProfile)
     val powerPlantProfiles = buildModels(modelClassToRawData, PowerPlantProfile)
@@ -330,7 +336,7 @@ final case class SimbenchReader(
               read(clazz, fields)
             }
         ),
-        Duration("10 s")
+        Duration("30 s")
       )
       .toMap
   }