Merge branch 'dev' into mh/#898-updating-diagram

# Conflicts: # CHANGELOG.md # docs/uml/main/ExtEvSimulationClasses.puml
ie3-institute · Oct 6, 2024 · eeee724 · eeee724
2 parents 7571eaa + 14e2284
commit eeee724
Show file tree

Hide file tree

Showing 57 changed files with 1,293 additions and 803 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -101,31 +101,32 @@ node {
         }
       }
 
+      // Build the project
+      stage('build') {
+        gradle('clean assemble', projectName)
+      }
+
       // test the project
       stage('run tests') {
 
         sh 'java -version'
 
-        gradle('--refresh-dependencies clean spotlessCheck pmdMain pmdTest reportScoverage checkScoverage', projectName)
+        gradle('--refresh-dependencies spotlessCheck pmdMain pmdTest', projectName)
 
         sh(script: """set +x && cd $projectName""" + ''' set +x; ./gradlew javadoc''', returnStdout: true)
       }
 
-      // sonarqube analysis
-      stage('sonarqube analysis') {
+      // sonarqube analysis & quality gate
+      stage('sonarqube') {
         String sonarqubeCurrentBranchName = prFromFork() ? prJsonObj.head.repo.full_name : currentBranchName // forks needs to be handled differently
         String sonarqubeCmd = determineSonarqubeGradleCmd(sonarqubeProjectKey, sonarqubeCurrentBranchName, targetBranchName, orgName, projectName, projectName)
         withSonarQubeEnv() {
           // will pick the global server connection from jenkins for sonarqube
           gradle(sonarqubeCmd, projectName)
         }
-      }
-
-      // sonarqube quality gate
-      stage("quality gate") {
         timeout(time: 1, unit: 'HOURS') {
-          // just in case something goes wrong, pipeline will be killed after a timeout
-          def qg = waitForQualityGate() // reuse taskId previously collected by withSonarQubeEnv
+          // Just in case something goes wrong, pipeline will be killed after a timeout
+          def qg = waitForQualityGate() // Reuse taskId previously collected by withSonarQubeEnv
           if (qg.status != 'OK') {
             error "Pipeline aborted due to quality gate failure: ${qg.status}"
           }
@@ -684,4 +685,4 @@ def getBranchType(String branchName) {
   } else {
     return null
   }
-}
+}
diff --git a/build.gradle b/build.gradle
@@ -25,12 +25,12 @@ ext {
   javaVersion = JavaVersion.VERSION_17
 
   scalaVersion = '2.13'
-  scalaBinaryVersion = '2.13.14'
-  pekkoVersion = '1.1.0'
+  scalaBinaryVersion = '2.13.15'
+  pekkoVersion = '1.1.1'
   jtsVersion = '1.20.0'
   confluentKafkaVersion = '7.4.0'
   tscfgVersion = '1.1.3'
-  scapegoatVersion = '3.0.0'
+  scapegoatVersion = '3.0.3'
 
   testContainerVersion = '0.41.4'
 
@@ -103,7 +103,7 @@ dependencies {
   /* testing */
   testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
   testImplementation 'org.scalatestplus:mockito-3-4_2.13:3.2.10.0'
-  testImplementation 'org.mockito:mockito-core:5.13.0' // mocking framework
+  testImplementation 'org.mockito:mockito-core:5.14.1' // mocking framework
   testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.19"
   testRuntimeOnly 'com.vladsch.flexmark:flexmark-all:0.64.8' //scalatest html output
   testImplementation group: 'org.pegdown', name: 'pegdown', version: '1.6.0'
@@ -149,7 +149,7 @@ dependencies {
   implementation 'javax.measure:unit-api:2.2'
   implementation 'tech.units:indriya:2.2' // quantities
   implementation "org.typelevel:squants_${scalaVersion}:1.8.3"
-  implementation 'org.apache.commons:commons-csv:1.11.0'
+  implementation 'org.apache.commons:commons-csv:1.12.0'
   implementation 'org.scalanlp:breeze_2.13:2.1.0' // scientific calculations (http://www.scalanlp.org/)
   implementation 'de.lmu.ifi.dbs.elki:elki:0.7.5' // Statistics (for random load model)
   implementation 'org.jgrapht:jgrapht-core:1.5.2'

diff --git a/docs/readthedocs/_static/bibliography/bibtexAll.bib b/docs/readthedocs/_static/bibliography/bibtexAll.bib
@@ -189,4 +189,10 @@ @Book{Kittl_2022
   address   = {Düren},
   year      = {2022},
   doi       = {10.17877/DE290R-22548}
+}
+
+@MISC{Radiation_ECMWF,
+  author = {Robin Hogan},
+  title = {Radiation Quantities in the ECMWF model and MARS},
+  howpublished = {\url{https://www.ecmwf.int/sites/default/files/elibrary/2015/18490-radiation-quantities-ecmwf-model-and-mars.pdf}}
 }
diff --git a/docs/readthedocs/config.md b/docs/readthedocs/config.md
@@ -94,9 +94,13 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 ```
 
+While using a csv sink, the raw data output files can be zipped directly when `zipFiles = true` is used.
+
+
 #### Output configuration of the grid
 
 The grid output configuration defines for which grid components simulation values are to be output.

diff --git a/docs/readthedocs/howto/weatherDataHowToCopernicusERA5.md b/docs/readthedocs/howto/weatherDataHowToCopernicusERA5.md
@@ -0,0 +1,34 @@
+(weatherDataHowToCopernicusERA5)=
+
+# How To use Copernicus ERA5 weather data in SIMONA
+
+To use weather data from the past within SIMONA we recommend to use the dataset [ERA5 hourly data on single levels from 1940 to present](https://cds-beta.climate.copernicus.eu/datasets/reanalysis-era5-single-levels?tab=download) of [Copernicus Climate Data Store](https://cds-beta.climate.copernicus.eu/).
+
+The following data parameter should be used
+
+- Wind
+    - 100m u-component of wind
+    - 100m v-component of wind
+- Radiation
+  - Total sky direct solar radiation at surface (FDIR)
+  - Surface solar radiation downwards (SSRD)
+- Temperature
+  - 2m temperature
+
+Since SIMONAs [PV Model](pv_model) requires direct and diffuse solar radiation, the diffuse solar radiation need to be determined from the ERA5 data.
+
+## Pre-Processing solar radiation weather data
+
+To obtain diffuse solar radiation data from ERA5 weather data, the necessary diffuse solar radiation (FDIFF) at surface can be calculated by
+
+$$
+ FDIFF = SSRD - FDIR
+$$
+
+*with*\
+**SSRD** = Surface solar radiation downwards\
+**FDIR** = Total sky direct solar radiation at surface
+
+
+**References:**
+* {cite:cts}`Radiation_ECMWF`
diff --git a/docs/readthedocs/models/em.md b/docs/readthedocs/models/em.md
@@ -6,11 +6,11 @@ Energy Management Agents (EmAgents) control power feed-in and load of system par
 
 ## Protocol
 
-During simulation, EmAgents send `RequestFlexOptions` and `IssueFlexControl` messages and receive `ProvideFlexOptions` and `FlexCompletion` messages.
-After having been requested to calculate flex options via `RequestFlexOptions`, controllable assets send back their flex options to the controlling unit using `ProvideFlexOptions`. 
+During simulation, EmAgents send `FlexActivation` and `IssueFlexControl` messages and receive `ProvideFlexOptions` and `FlexCompletion` messages.
+After being requested to calculate flex options via `FlexActivation`, controllable assets send back their flex options to the controlling unit using `ProvideFlexOptions`. 
 Eventually the controlling EmAgent responds with some type of `IssueFlexControl` messages, setting a power set point for operation.
 The asset then tries to realize the set power as best as it can and replies with a `FlexCompletion` messages.
-If an EmAgent is itself controlled by another EmAgent, it also behaves like a system participant (sends `RequestFlexOptions` and `IssueFlexControl` messages etc.).
+If an EmAgent is itself controlled by another EmAgent, it also behaves like a system participant (sends `FlexActivation` and `IssueFlexControl` messages etc.).
 
 
 Every EmAgent aggregates flex options and power of its connected assets and disaggregates flex control among the connected assets. 

diff --git a/docs/readthedocs/requirements.txt b/docs/readthedocs/requirements.txt
@@ -4,4 +4,4 @@ sphinxcontrib-plantuml==0.30
 myst-parser==4.0.0
 markdown-it-py==3.0.0
 sphinx-hoverxref==1.4.1
-sphinxcontrib-bibtex==2.6.2
+sphinxcontrib-bibtex==2.6.3
diff --git a/docs/readthedocs/usersguide.md b/docs/readthedocs/usersguide.md
@@ -119,6 +119,13 @@ Besides a configuration and the actual grid and grid participants, SIMONA also e
 There is an option to use sample weather data, but if you want sensible results, definitely consider supplying suitable data.
 Information on the expected data format and different supported sources are given in the input parameters section of the {doc}`config` file.
 
+The following How-To's are available:
+```{toctree}
+---
+maxdepth: 1
+---
+howto/weatherDataHowToCopernicusERA5
+```
 
 ## Simulation Outputs
 

diff --git a/docs/uml/main/ParticipantInheritance.puml b/docs/uml/main/ParticipantInheritance.puml
@@ -26,7 +26,7 @@
     + getAndCheckNodalVoltage(...): ComparableQuantity[Dimensionless]
     + initializeParticipant(...): FSM.State[AgentState, ParticipantStateData]
     + {abstract} determineBaseStateData(...): BaseStateData[ApparentPower]
-    + determineTriggersThroughoutSimulation(...): Option[List[ScheduleTriggerMessage]]
+    + determineTriggersThroughoutSimulation(...): Option[List[ScheduleActivation]]
     + baseStateDataForExternalDataProvision(...): FromOutsideBaseStateData
     + queryPrimaryDataAndChangeState(...): FSM.State[AgentState, ParticipantStateData]
     + querySecondaryDataAndChangeState(...): FSM.State[AgentState, ParticipantStateData]

diff --git a/docs/uml/protocol/ExtEvSimulationSequence.puml b/docs/uml/protocol/ExtEvSimulationSequence.puml
@@ -43,11 +43,11 @@ Scheduler -> ExtEvDataService: <font color=red>!</font> Activation(t1)
 deactivate Scheduler
 activate ExtEvDataService
 
-ExtSimulation -> ExtSimAdapter: <font color=red>!</font> CompletionMessage(newTriggers)
+ExtSimulation -> ExtSimAdapter: <font color=red>!</font> CompletionMessage(newTick)
 deactivate ExtSimulation
 activate ExtSimAdapter
 
-ExtSimAdapter -> Scheduler: <font color=red>!</font> <font color=green>Completion(newTriggers)</font>
+ExtSimAdapter -> Scheduler: <font color=red>!</font> <font color=green>Completion(newTick)</font>
 deactivate ExtSimAdapter
 
 ExtEvDataService -> EvcsAgent1: <font color=red>!</font> RegistrationSuccessfulMessage(t1)

diff --git a/docs/uml/protocol/InitializationPhase.puml b/docs/uml/protocol/InitializationPhase.puml
@@ -1,19 +1,19 @@
 @startuml
-MainController -> Listener: create()
-MainController -> SimScheduler: create()
-MainController -> Services: create()
-MainController -> SimScheduler: <font color=red>!</font> ScheduleTriggerMessage(\n\tInitializeServiceTrigger(0),\n\tweatherActorRef)
-MainController -> GridAgents: create()
-MainController -> SimScheduler: <font color=red>!</font> ScheduleTriggerMessage(\n\tInitializeGridAgentTrigger(0),\n\tgridAgentInitData,\n\tgridAgentRef)
-MainController -> SimScheduler: <font color=red>?</font> StartScheduleMessage()
-SimScheduler -> Services: <font color=red>!</font> TriggerWithIdMessage(\n\tInitializeServiceTrigger(_),\n\ttriggerId,\n\t_)
-Services --> SimScheduler: <font color=red>!</font> CompletionMessage(\n\ttriggerId,\n\tScheduleTriggerMessage[...])
-SimScheduler -> GridAgents: <font color=red>!</font> TriggerWithIdMessage(\n\tInitializeGridAgentTrigger(_),\n\ttriggerId,\n\t_)
-GridAgents -> SimScheduler: <font color=red>!</font> ScheduleTriggerMessage(\n\tInitializeSystemComponentsTrigger(0),\n\tassetAgentRef)
-GridAgents --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, ScheduleTriggerMessage[...])
-SimScheduler -> ParticipantAgents: <font color=red>!</font> TriggerWithIdMessage(\n\tIInitializeSystemComponentsTrigger(_),\n\ttriggerId,\n\t_)
+SimonaSim -> Listener: create()
+SimonaSim -> Scheduler: create()
+SimonaSim -> Services: create()
+Services -> Scheduler: <font color=red>!</font> ScheduleActivation(\n\tweatherActorRef, \n\tINIT_SIM_TICK)
+SimonaSim -> GridAgents: create()
+GridAgents -> Scheduler: <font color=red>!</font> ScheduleActivation(\n\tgridAgentRef, \n\tINIT_SIM_TICK)
+SimonaSim -> Scheduler: <font color=red>?</font> StartScheduleMessage()
+Scheduler -> Services: <font color=red>!</font> Activation(\n\tINIT_SIM_TICK)
+Services --> Scheduler: <font color=red>!</font> Completion(\n\tweatherActorRef, \n\ttick)
+Scheduler -> GridAgents: <font color=red>!</font> Activation(\n\tINIT_SIM_TICK)
+GridAgents -> Scheduler: <font color=red>!</font> ScheduleActivation(\n\tassetAgentRef, \n\tINIT_SIM_TICK)
+GridAgents --> Scheduler: <font color=red>!</font> Completion(\n\tgridAgentRef,\n\ttick)
+Scheduler -> ParticipantAgents: <font color=red>!</font> Activation(\n\tINIT_SIM_TICK)
 ParticipantAgents -> Services: <font color=red>!</font> RegisterForWeatherMessage(lat, lon)
 ParticipantAgents -> Services: <font color=red>!</font> RegisterFor...
 Services --> ParticipantAgents: <font color=red>!</font> RegistrationResponseMessages
-ParticipantAgents --> SimScheduler: <font color=red>!</font> CompletionMessage(\n\ttriggerId,\n\tScheduleTriggerMessage[\n\t\tActivityStartTrigger(tick)\n\t])
+ParticipantAgents --> Scheduler: <font color=red>!</font> Completion(\n\tassetAgentRef,\n\ttick)
 @enduml
diff --git a/docs/uml/protocol/ParticipantTriggeredByItself.puml b/docs/uml/protocol/ParticipantTriggeredByItself.puml
@@ -1,6 +1,6 @@
 @startuml
-SimScheduler -> ParticipantAgents: <font color=red>!</font> TriggerWithIdMessage(\n\tIInitializeSystemComponentsTrigger(_),\n\ttriggerId,\n\t_)
-ParticipantAgents --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, ScheduleTriggerMessage[...])
-SimScheduler -> ParticipantAgents: <font color=red>!</font> TriggerWithIdMessage(\n\tActivityStartTrigger(tick),\n\t_,\n\t_)
-ParticipantAgents --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, None)
+Scheduler -> ParticipantAgents: <font color=red>!</font> Activation(\n\tINIT_SIM_TICK)
+ParticipantAgents --> Scheduler: <font color=red>!</font> Completion(\n\tactor, tick)
+Scheduler -> ParticipantAgents: <font color=red>!</font> Activation(\n\ttick)
+ParticipantAgents --> Scheduler: <font color=red>!</font> Completion(\n\tactor, newTick)
 @enduml
diff --git a/docs/uml/protocol/ParticipantTriggeredByPrimaryData.puml b/docs/uml/protocol/ParticipantTriggeredByPrimaryData.puml
@@ -1,7 +1,7 @@
 @startuml
-SimScheduler -> PrimaryServiceWorker: <font color=red>!</font> TriggerWithIdMessage(\n\tActivityStartTrigger(tick),\n\t_,\n\t_)
+Scheduler -> PrimaryServiceWorker: <font color=red>!</font> Activation(\n\tInit_SIM_TICK)
 PrimaryServiceWorker -> ParticipantAgents: <font color=red>!</font> <i>ProvisionMessage</i>
-PrimaryServiceWorker --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, None)
-SimScheduler -> ParticipantAgents: <font color=red>!</font> TriggerWithIdMessage(\n\tActivityStartTrigger(tick),\n\t_,\n\t_)
-ParticipantAgents --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, None)
+PrimaryServiceWorker --> Scheduler: <font color=red>!</font> Completion(\n\tactor,None)
+Scheduler -> ParticipantAgents: <font color=red>!</font> Activation(\n\ttick)
+ParticipantAgents --> Scheduler: <font color=red>!</font> Completion(\n\tactor, newTick)
 @enduml
diff --git a/docs/uml/protocol/ParticipantTriggeredBySecondaryData.puml b/docs/uml/protocol/ParticipantTriggeredBySecondaryData.puml
@@ -1,10 +1,10 @@
 @startuml
-SimScheduler -> Services: <font color=red>!</font> TriggerWithIdMessage(\n\tActivityStartTrigger(tick),\n\t_,\n\t_)
+Scheduler -> Services: <font color=red>!</font> Activation(\n\tINIT_SIM_TICK)
 Services -> ParticipantAgents: <font color=red>!</font> ProvideWeatherMessage(\n\ttick,\n\tweatherData,\n\tnextDataTick)
-SimScheduler -> ParticipantAgents: <font color=red>!</font> TriggerWithIdMessage(\n\tActivityStartTrigger(tick),\n\t_,\n\t_)
+Scheduler -> ParticipantAgents: <font color=red>!</font> Activation(\n\ttick)
 Services -> ParticipantAgents: <font color=red>!</font> <i>ProvisionMessage</i>
 Services -> ParticipantAgents: <font color=red>!</font> <i>ProvisionMessage</i>
-Services --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, None)
+Services --> Scheduler: <font color=red>!</font> Completion(\n\tactor, None)
 ParticipantAgents -> ParticipantAgents: <font color=red>!</font> StartCalculationTrigger(tick)
-ParticipantAgents --> SimScheduler: <font color=red>!</font> CompletionMessage(triggerId, None)
+ParticipantAgents --> Scheduler: <font color=red>!</font> Completion(\n\tactor, newTick)
 @enduml
diff --git a/docs/uml/protocol/em/ControlledEm.puml b/docs/uml/protocol/em/ControlledEm.puml
@@ -54,13 +54,13 @@ deactivate WeatherService
 Scheduler -> EmAgent1: Activation(tick=0)
 activate EmAgent1
 
-EmAgent1 -> EmAgent2: RequestFlexOptions(tick=0)
+EmAgent1 -> EmAgent2: FlexActivation(tick=0)
 activate EmAgent2
 
-EmAgent2 -> StorageAgent: RequestFlexOptions(tick=0)
+EmAgent2 -> StorageAgent: FlexActivation(tick=0)
 activate StorageAgent
 
-EmAgent2 -> PvAgent: RequestFlexOptions(tick=0)
+EmAgent2 -> PvAgent: FlexActivation(tick=0)
 activate PvAgent
 
 PvAgent -> EmAgent2: ProvideFlexOptions
@@ -81,12 +81,15 @@ activate PvAgent
 EmAgent2 -> StorageAgent: IssuePowerControl(tick=0)
 activate StorageAgent
 
+PvAgent -> EmAgent2: FlexResult
 PvAgent -> EmAgent2: FlexCompletion(nextTick=3600)
 deactivate PvAgent
 
+StorageAgent -> EmAgent2: FlexResult
 StorageAgent -> EmAgent2: FlexCompletion(nextTick=1805)
 deactivate StorageAgent
 
+EmAgent2 -> EmAgent1: FlexResult
 EmAgent2 -> EmAgent1: FlexCompletion(nextTick=1805)
 deactivate EmAgent2
 
@@ -98,10 +101,10 @@ deactivate EmAgent1
 Scheduler -> EmAgent1: Activation(tick=1805)
 activate EmAgent1
 
-EmAgent1 -> EmAgent2: RequestFlexOptions(tick=1805)
+EmAgent1 -> EmAgent2: FlexActivation(tick=1805)
 activate EmAgent2
 
-EmAgent2 -> StorageAgent: RequestFlexOptions(tick=1805)
+EmAgent2 -> StorageAgent: FlexActivation(tick=1805)
 activate StorageAgent
 
 StorageAgent -> EmAgent2: ProvideFlexOptions
@@ -116,9 +119,11 @@ activate EmAgent2
 EmAgent2 -> StorageAgent: IssuePowerControl(tick=1805)
 activate StorageAgent
 
+StorageAgent -> EmAgent2: FlexResult
 StorageAgent -> EmAgent2: FlexCompletion(nextTick=10800)
 deactivate StorageAgent
 
+EmAgent2 -> EmAgent1: FlexResult
 EmAgent2 -> EmAgent1: FlexCompletion(nextTick=3600)
 deactivate EmAgent2
 

diff --git a/docs/uml/protocol/em/UncontrolledEm.puml b/docs/uml/protocol/em/UncontrolledEm.puml
@@ -50,10 +50,10 @@ deactivate WeatherService
 Scheduler -> EmAgent: Activation(tick=0)
 activate EmAgent
 
-EmAgent -> StorageAgent: RequestFlexOptions(tick=0)
+EmAgent -> StorageAgent: FlexActivation(tick=0)
 activate StorageAgent
 
-EmAgent -> PvAgent: RequestFlexOptions(tick=0)
+EmAgent -> PvAgent: FlexActivation(tick=0)
 activate PvAgent
 
 PvAgent -> EmAgent: ProvideFlexOptions
@@ -68,9 +68,11 @@ activate PvAgent
 EmAgent -> StorageAgent: IssuePowerControl(tick=0)
 activate StorageAgent
 
+PvAgent -> EmAgent: FlexResult
 PvAgent -> EmAgent: FlexCompletion(nextTick=3600)
 deactivate PvAgent
 
+StorageAgent -> EmAgent: FlexResult
 StorageAgent -> EmAgent: FlexCompletion(nextTick=8400)
 deactivate StorageAgent
 

diff --git a/gradle/scripts/scoverage.gradle b/gradle/scripts/scoverage.gradle
@@ -3,7 +3,7 @@
 // https://github.com/scoverage/gradle-scoverage/issues/109 for details
 
 scoverage {
-  scoverageVersion = "2.1.1"
+  scoverageVersion = "2.2.1"
   scoverageScalaVersion = scalaBinaryVersion
   coverageOutputHTML = false
   coverageOutputXML = true

diff --git a/input/samples/vn_simona/vn_simona.conf b/input/samples/vn_simona/vn_simona.conf
@@ -51,6 +51,7 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 
 simona.output.grid = {

diff --git a/src/main/resources/config/config-template.conf b/src/main/resources/config/config-template.conf
@@ -269,6 +269,7 @@ simona.output.sink.csv {
   isHierarchic = Boolean | false
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = "Boolean" | false
 }
 #@optional
 simona.output.sink.influxDb1x {