Get time coverage from dataset metadata

Author: dlindhol

docs/configuring-datasets.md

@@ -22,9 +22,9 @@ as a function of time, stored in a [CSV file][csv].
 
 ```
 #time,value
-2018-01-01T00:00:00Z,0
-2018-01-01T00:00:01Z,1
-2018-01-01T00:00:02Z,2
+2018-01-01,0
+2018-01-02,1
+2018-01-03,2
 ```
 
 We will need to provide an FDML file that describes this dataset.
@@ -170,15 +170,16 @@ implemented by the `latis.time.Time` class.
 Finally, we need to add metadata for our dataset. HAPI requires that
 we define the following:
 
+- time coverage
 - units for all variables
-- coverage for time variables
 
 This is how we specify the required metadata in FDML:
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
 
 <dataset id="simple_dataset"
+    temporalCoverage="2018-01-01/2018-01-03"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:noNamespaceSchemaLocation="http://latis-data.io/schemas/1.0/fdml-with-text-adapter.xsd">
 
@@ -192,21 +193,22 @@ This is how we specify the required metadata in FDML:
         <scalar id="time"
                 type="string"
                 class="latis.time.Time"
-                units="yyyy-MM-dd'T'hh:mm:ss'Z'"
-                coverage="2018-01-01T00:00:00Z/2018-01-01T00:00:02Z"/>
+                units="yyyy-MM-dd"/>
         <scalar id="value" type="int" units="units"/>
     </function>
 </dataset>
 ```
 
-We can add metadata to the model by adding attributes:
+We can add metadata to the dataset by adding attributes:
 
+- `temporalCoverage` : Adds time coverage (specified as `<start>/<stop>`)
+  - The `start` and `stop` times must be in ISO 8601 format. If the `stop`
+    time is not present, the current date will be used.
 - `units` : Adds units
   - For times represented by formatted text, this is a [time format
-    string][java-sdf]. This is so LaTiS can read and convert times.
+    string][java-dtf]. This is so LaTiS can read and convert times.
     The server will report "UTC" as the units as required by the HAPI
     specification.
-- `coverage` : Adds coverage (specified as `<start>/<stop>`)
 
 The snippet above is a complete FDML file that describes the simple
 CSV data. Both the FDML and the CSV files are available in the
@@ -233,7 +235,7 @@ If the FDML file is invalid, you'll get a message with the error.
 [fdm]: https://github.com/latis-data/latis/wiki/LaTiS-Data-Model
 [fdml]: ../examples/fdml/simple_dataset.fdml
 [hapi-spec]: https://github.com/hapi-server/data-specification/blob/master/hapi-2.1.0/HAPI-data-access-spec-2.1.0.md
-[java-sdf]: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/text/SimpleDateFormat.html
+[java-dtf]: https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/time/format/DateTimeFormatter.html
 [latis-3]: https://github.com/latis-data/latis3
 [readme]: ../README.md
 [releases]: https://github.com/lasp/hapi-server/releases

examples/fdml/simple_dataset.csv

@@ -1,3 +1,3 @@
-2018-01-01T00:00:00Z,0
-2018-01-01T00:00:01Z,1
-2018-01-01T00:00:02Z,2
+2018-01-01,0
+2018-01-02,1
+2018-01-03,2

examples/fdml/simple_dataset.fdml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <dataset id="simple_dataset"
+    temporalCoverage="2018-01-01/2018-01-03"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:noNamespaceSchemaLocation="http://latis-data.io/schemas/1.0/fdml-with-text-adapter.xsd">
 
@@ -14,8 +15,7 @@
         <scalar id="time"
                 type="string"
                 class="latis.time.Time"
-                units="yyyy-MM-dd'T'hh:mm:ss'Z'"
-                coverage="2018-01-01T00:00:00Z/2018-01-01T00:00:02Z"/>
+                units="yyyy-MM-dd"/>
         <scalar id="value" type="int" units="units"/>
     </function>
 </dataset>

src/main/scala/latis/service/hapi/Latis3Interpreter.scala

@@ -126,15 +126,11 @@ class Latis3Interpreter(catalog: Catalog) extends HapiInterpreter[IO] {
       NonEmptyList.fromList(params),
       UnsupportedDataset("")
     )
-    tFmt   <- Either.fromOption(
-      tVar.metadata.getProperty("units"),
-      MetadataError("Time parameter missing metadata property 'units'")
-    )
     tCov   <- Either.fromOption(
-      tVar.metadata.getProperty("coverage"),
-      MetadataError("Time parameter missing metadata property 'coverage'")
+      ds.metadata.getProperty("temporalCoverage"),
+      MetadataError("Dataset missing metadata property 'temporalCoverage'")
     )
-    tCovP  <- parseCoverage(tCov, tFmt)
+    tCovP  <- parseCoverage(tCov)
   } yield Metadata(
     psNel,
     tCovP._1,

src/main/scala/latis/util/HapiUtils.scala

@@ -1,34 +1,43 @@
 package latis.util
 
-import cats.implicits._
+import cats.syntax.all._
 
 import latis.service.hapi._
 import latis.time.TimeFormat
-import latis.time.TimeScale
-import latis.units.UnitConverter
 
 object HapiUtils {
 
-  /** Convert a time value to a HAPI time given its original units. */
-  def toHapiTime(fmt: String, value: String): Either[InfoError, String] = for {
-    fromTime <- TimeFormat.fromExpression(fmt)
-      .flatMap(_.parse(value).map(_.toDouble))
-      .orElse(Either.catchNonFatal(value.toDouble))
-      .leftMap(_ => MetadataError("Unsupported time units"))
-    scale    <- TimeScale.fromExpression(fmt)
-      .leftMap(_ => MetadataError("Unsupported time units"))
-    converter = UnitConverter(scale, TimeScale.Default)
-    toTime    = converter.convert(fromTime).toLong
-  } yield TimeFormat.formatIso(toTime)
-
-  /** Parse coverage metadata from FDML and convert to HAPI time. */
-  def parseCoverage(coverage: String, fmt: String): Either[InfoError, (String, String)] = {
+  /**
+   * Parse temporalCoverage from Dataset metadata to be used for HAPI info.
+   *
+   * The coverage is expected to be "/" separated ISO 8601 times. These will
+   * be converted to the form yyyy-MM-ddZ for the HAPI info response. A coverage
+   * without a value after the "/" will use the current date as the end time.
+   */
+  def parseCoverage(coverage: String): Either[InfoError, (String, String)] = {
     coverage.split('/').toList match {
-      case s :: e :: Nil if s.nonEmpty && e.nonEmpty => for {
-        sF <- toHapiTime(fmt, s)
-        eF <- toHapiTime(fmt, e)
-      } yield (sF, eF)
-      case _ => MetadataError("Invalid coverage").asLeft
+      case s :: e :: Nil if s.nonEmpty && e.nonEmpty =>
+        (for {
+          targetFomat <- TimeFormat.fromExpression("yyyy-MM-dd'Z'")
+          startMillis <- TimeFormat.parseIso(s)
+          endMillis   <- TimeFormat.parseIso(e)
+          startDate    = targetFomat.format(startMillis)
+          endDate      = targetFomat.format(endMillis)
+        } yield (startDate, endDate)).leftMap { _ =>
+          MetadataError(s"Invalid temporalCoverage: $coverage")
+        }
+      // Replace open end (e.g. "2000-01-01/") with current date
+      case s :: Nil if s.nonEmpty && coverage.endsWith("/") =>
+        (for {
+          targetFomat <- TimeFormat.fromExpression("yyyy-MM-dd'Z'")
+          startMillis <- TimeFormat.parseIso(s)
+          endMillis    = System.currentTimeMillis() //TODO: beware side effect
+          startDate    = targetFomat.format(startMillis)
+          endDate      = targetFomat.format(endMillis)
+        } yield (startDate, endDate)).leftMap { _ =>
+          MetadataError(s"Invalid temporalCoverage: $coverage")
+        }
+      case _ => MetadataError(s"Invalid temporalCoverage: $coverage").asLeft
     }
   }
 }

src/test/scala/latis/service/hapi/DataServiceSuite.scala

@@ -29,14 +29,13 @@ import latis.util.Identifier._
 
 class DataServiceSuite extends CatsEffectSuite {
 
-  /** Build a simple test DataService[IO] with a time -> int dataset using the Latis3Interpreter*/
+  /** Build a simple test DataService[IO] with a time -> int dataset using the Latis3Interpreter */
   private lazy val dataset = (for {
     time <- Time.fromMetadata(
       Metadata(
         "id" -> "time",
         "type" -> "string",
-        "units" -> "yyyy-MM-dd",
-        "coverage" -> "2000-01-01/2000-01-05"
+        "units" -> "yyyy-MM-dd"
       )
     )
     disp <- Scalar.fromMetadata(
@@ -54,7 +53,11 @@ class DataServiceSuite extends CatsEffectSuite {
       Sample(DomainData("2000-01-04"), RangeData(2)),
       Sample(DomainData("2000-01-05"), RangeData(0)),
     ))
-    dataset = new MemoizedDataset(Metadata("id" -> "testdataset"), model, data)
+    md = Metadata(
+      "id" -> "testdataset",
+      "temporalCoverage" -> "2000-01-01/2000-01-05"
+    )
+    dataset = new MemoizedDataset(md, model, data)
   } yield dataset).fold(err => fail(err.message), identity)
   private lazy val latisInterp = new Latis3Interpreter(Catalog(dataset))
   private lazy val dataService = new DataService[IO](latisInterp).service
@@ -261,8 +264,8 @@ class DataServiceSuite extends CatsEffectSuite {
           ("fill", Json.Null),
         ),
       )),
-      ("startDate", Json.fromString("2000-01-01T00:00:00.000Z")),
-      ("stopDate", Json.fromString("2000-01-05T00:00:00.000Z")),
+      ("startDate", Json.fromString("2000-01-01Z")),
+      ("stopDate", Json.fromString("2000-01-05Z")),
       ("format", Json.fromString("json")),
       ("data", Json.arr(
         Json.arr(Json.fromString("2000-01-01T00:00:00.000Z"), Json.fromInt(1)),