diff --git a/build.sbt b/build.sbt index 237d6a5..42b3315 100644 --- a/build.sbt +++ b/build.sbt @@ -3,8 +3,8 @@ ThisBuild / scalaVersion := "3.3.8" val fs2DataVersion = "1.8.1" val http4sVersion = "0.23.36" -val latisVersion = "b66ec47" -val latisHapiVersion = "1a507d1" +val latisVersion = "c1531e77" +val latisHapiVersion = "1d65ba50" lazy val root = (project in file(".")) .settings( diff --git a/src/main/scala/latis/ops/ConvertHapiTypes.scala b/src/main/scala/latis/ops/ConvertHapiTypes.scala index e0cf68d..a6bb99c 100644 --- a/src/main/scala/latis/ops/ConvertHapiTypes.scala +++ b/src/main/scala/latis/ops/ConvertHapiTypes.scala @@ -10,23 +10,38 @@ import latis.util.LatisException /** * Converts scalar values to be consistent with supported HAPI types. * - * HAPI supports only double, int, and string types. This will convert - * some types that can safely be converted. Datasets with other types - * will be excluded from the Catalog by HapiService.filteredCatalog. - * This operation needs to be consistent with that filter. + * HAPI supports only doubles, 32-bit integers, and string types. + * This will convert LaTiS types that can safely be converted. Longs + * are an exception. Although HAPI does not support 64-bit integers, + * they are common enough in data sources (even when an int is sufficient) + * that this attempts to convert long values to 32-bit integers. If the + * long value exceeds the range of an Int, a fill value will be used + * if defined for that variable. Otherwise, an exception will be thrown. * - * This is only needed for the binary output. + * Datasets with other types will be excluded from the Catalog by + * HapiService.filteredCatalog. This operation needs to be consistent + * with that filter. + * + * This assumes flat datasets with no nesting. + * + * This is only needed for the binary output, so it is otherwise not + * applicable. */ class ConvertHapiTypes extends MapOperation { - //TODO: assumes non-nested functions or tuples def mapFunction(model: DataType): Sample => Sample = { // Note, domain can only be time and it is handled elsewhere - case Sample(d, r) => Sample(d, RangeData(r.map(convertValue))) + case Sample(d, r) => + val rdata = model.getScalars.tail.zip(r).map(convertValue) + Sample(d, RangeData(rdata)) } - private def convertValue(data: Data): Data = data match { + private def convertValue(scalar: Scalar, data: Data): Data = data match { case v: ShortValue => IntValue(v.value.toInt) + case v: LongValue => + if (v.value > Int.MaxValue.toLong || v.value < Int.MinValue.toLong) + scalar.fillValue.getOrElse(throw LatisException("Integer overflow")) + else IntValue(v.value.toInt) case v: FloatValue => DoubleValue(v.value.toDouble) case _ => data //no-op, shouldn't get here due to catalog filter } @@ -45,6 +60,10 @@ class ConvertHapiTypes extends MapOperation { Scalar.fromMetadata( scalar.metadata + ("type" -> "int") ).fold(throw _, identity) //should not fail + case LongValueType => + Scalar.fromMetadata( + scalar.metadata + ("type" -> "int") + ).fold(throw _, identity) //should not fail case _ => scalar //no-op, shouldn't get here due to catalog filter } } diff --git a/src/main/scala/latis/service/hapi/HapiService.scala b/src/main/scala/latis/service/hapi/HapiService.scala index 8a93310..bcbd4cd 100644 --- a/src/main/scala/latis/service/hapi/HapiService.scala +++ b/src/main/scala/latis/service/hapi/HapiService.scala @@ -42,16 +42,18 @@ class HapiService(catalog: Catalog) extends ServiceInterface(catalog, OperationR // This checks that: // - The Dataset metadata has temporalCoverage // - The single domain variable is of type Time - // - Each scalar in the range has a supported type + // - Each scalar in the range has a supported or convertible type // - If the type of a scalar is string, its size is defined + // - If the type is "long" a fillValue must be defined private val filteredCatalog: Catalog = { val covP: Metadata => Boolean = _.getProperty("temporalCoverage").isDefined val typeP: Metadata => Boolean = md => md.getProperty("type").exists { case "string" => md.getProperty("size").isDefined case "double" => true case "int" => true - case "float" => true //may be converted to double by ConvertHapiTypes - case "short" => true //may be converted to int by ConvertHapiTypes + case "long" => md.getProperty("fillValue").isDefined + case "float" => true //converted to double by ConvertHapiTypes + case "short" => true //converted to int by ConvertHapiTypes case _ => false }