Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Multiple samples per day for tiff #269

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
+35 −4
Draft

Multiple samples per day for tiff #269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.AccumulatorV2
import org.openeo.geotrellis
import org.openeo.geotrellis.creo.CreoS3Utils
import org.openeo.geotrellis.netcdf.NetCDFRDDWriter.fixedTimeOffset
import org.openeo.geotrellis.stac.STACItem
import org.openeo.geotrellis.tile_grid.TileGrid
import org.slf4j.LoggerFactory
Expand All @@ -32,6 +33,7 @@ import spire.math.Integral
import spire.syntax.cfor.cfor

import java.nio.file.{Path, Paths}
import java.time.Duration
import java.time.format.DateTimeFormatter
import java.util.{ArrayList, Collections, Map, List => JList}
import scala.collection.JavaConverters._
Expand All @@ -41,6 +43,8 @@ package object geotiff {

private val logger = LoggerFactory.getLogger(getClass)

val secondsPerDay = 86400L

class SetAccumulator[T](var value: Set[T]) extends AccumulatorV2[T, Set[T]] {
def this() = this(Set.empty[T])
override def isZero: Boolean = value.isEmpty
Expand Down Expand Up @@ -95,6 +99,14 @@ package object geotiff {
val croppedExtent: Extent = preProcessResult._2
val preprocessedRdd: RDD[(SpaceTimeKey, MultibandTile)] with Metadata[TileLayerMetadata[SpaceTimeKey]] = preProcessResult._3

val temporalResolution = if (preprocessedRdd.keys.filter({
case key: SpaceTimeKey =>
// true if not exactly n days:
Duration.between(fixedTimeOffset, key.time).getSeconds % secondsPerDay != 0
case _ =>
false
}).isEmpty()) TemporalResolution.days else TemporalResolution.seconds

val tileLayout = preprocessedRdd.metadata.tileLayout

val totalCols = math.ceil(gridBounds.width.toDouble / tileLayout.tileCols).toInt
Expand Down Expand Up @@ -124,7 +136,13 @@ package object geotiff {
(index, (multibandTile.cellType, compressedBytes))
})
}.map(tuple => {
val filename = s"${formatOptions.filenamePrefix}_${DateTimeFormatter.ISO_DATE.format(tuple._1.time)}.tif"
val filename = temporalResolution match {
case TemporalResolution.days =>
s"${formatOptions.filenamePrefix}_${DateTimeFormatter.ISO_DATE.format(tuple._1.time)}.tif"
case TemporalResolution.seconds =>
// ':' is not valid in a Windows filename
s"${formatOptions.filenamePrefix}_${DateTimeFormatter.ISO_ZONED_DATE_TIME.format(tuple._1.time).replace(":", "-")}.tif"
}
val timestamp = tuple._1.time format DateTimeFormatter.ISO_ZONED_DATE_TIME
((filename, timestamp), tuple._2)
}).groupByKey().map((tuple: ((String, String), Iterable[Vector[(Int, (CellType, Array[Byte]))]])) => {
Expand Down Expand Up @@ -682,14 +700,28 @@ package object geotiff {
val layout = rdd.metadata.layout
val crs = rdd.metadata.crs

val temporalResolution = if (rdd.keys.filter({
case key: SpaceTimeKey =>
// true if not exactly n days:
Duration.between(fixedTimeOffset, key.time).getSeconds % secondsPerDay != 0
case _ =>
false
}).isEmpty()) TemporalResolution.days else TemporalResolution.seconds

rdd
.flatMap { case (key, tile) => featuresBC.value
.filter { case (_, geometry) => layout.mapTransform.keysForGeometry(geometry) contains key.spatialKey }
.map { case (name, geometry) => ((name, (geometry, key.time)), (key.spatialKey, tile)) }
}
.groupByKey()
.map { case ((name, (geometry, time)), tiles) =>
val filename = s"${filenamePrefix.getOrElse("openEO")}_${DateTimeFormatter.ISO_DATE.format(time)}_$name.tif"
val filename = temporalResolution match {
case TemporalResolution.days =>
s"${filenamePrefix.getOrElse("openEO")}_${DateTimeFormatter.ISO_DATE.format(time)}_$name.tif"
case TemporalResolution.seconds =>
// ':' is not valid in a Windows filename
s"${filenamePrefix.getOrElse("openEO")}_${DateTimeFormatter.ISO_ZONED_DATE_TIME.format(time).replace(":", "-")}_$name.tif"
}
val filePath = Paths.get(path).resolve(filename).toString
val timestamp = time format DateTimeFormatter.ISO_ZONED_DATE_TIME
(stitchAndWriteToTiff(tiles, filePath, layout, crs, geometry, croppedExtent, cropDimensions, compression),
Expand Down
3 changes: 1 addition & 2 deletions openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/WriteRDDToGeotiffTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,6 @@ class WriteRDDToGeotiffTest {
assertArrayEquals(croppedReference.toArray(), result2.band(0).toArrayTile().crop(2 * 256, 0, layoutCols * 256, layoutRows * 256).toArray())
}

@Ignore("Fix this test: https://github.com/Open-EO/openeo-geotrellis-extensions/issues/257")
@Test
def testWriteMultibandTemporalHourlyRDDWithGaps(): Unit = {
val layoutCols = 8
Expand Down Expand Up @@ -392,7 +391,7 @@ class WriteRDDToGeotiffTest {

val ret = saveSamples(tileLayerRDD, outDir.toString, tiltedRectangle, sampleNames,
DeflateCompression(BEST_COMPRESSION))
assertTrue(ret.get(0)._2.contains("T"))
assertTrue(ret.get(0)._1.contains("T"))
}

@Test
Expand Down