From b66ae897a4ba7a0f529b88a00a4551f550183a9f Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Tue, 28 Jul 2026 12:31:25 -0600 Subject: [PATCH 1/7] Update dependencies --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 237d6a5..42b3315 100644 --- a/build.sbt +++ b/build.sbt @@ -3,8 +3,8 @@ ThisBuild / scalaVersion := "3.3.8" val fs2DataVersion = "1.8.1" val http4sVersion = "0.23.36" -val latisVersion = "b66ec47" -val latisHapiVersion = "1a507d1" +val latisVersion = "c1531e77" +val latisHapiVersion = "1d65ba50" lazy val root = (project in file(".")) .settings( From f5a1e957df4d8b66431687676a4d84a8b2ebbaef Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Tue, 28 Jul 2026 12:36:38 -0600 Subject: [PATCH 2/7] Allow serving LaTiS datasets containing long data Add longs to the type conversion operation that is applied before encoding and streaming. Disable filtering out datasets that have long data types from the catalog. --- src/main/scala/latis/ops/ConvertHapiTypes.scala | 5 +++++ src/main/scala/latis/service/hapi/HapiService.scala | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/scala/latis/ops/ConvertHapiTypes.scala b/src/main/scala/latis/ops/ConvertHapiTypes.scala index e0cf68d..b11a320 100644 --- a/src/main/scala/latis/ops/ConvertHapiTypes.scala +++ b/src/main/scala/latis/ops/ConvertHapiTypes.scala @@ -27,6 +27,7 @@ class ConvertHapiTypes extends MapOperation { private def convertValue(data: Data): Data = data match { case v: ShortValue => IntValue(v.value.toInt) + case v: LongValue => IntValue(v.value.toInt) //risk of overflow but no exception case v: FloatValue => DoubleValue(v.value.toDouble) case _ => data //no-op, shouldn't get here due to catalog filter } @@ -45,6 +46,10 @@ class ConvertHapiTypes extends MapOperation { Scalar.fromMetadata( scalar.metadata + ("type" -> "int") ).fold(throw _, identity) //should not fail + case LongValueType => + Scalar.fromMetadata( + scalar.metadata + ("type" -> "int") + ).fold(throw _, identity) //should not fail case _ => scalar //no-op, shouldn't get here due to catalog filter } } diff --git a/src/main/scala/latis/service/hapi/HapiService.scala b/src/main/scala/latis/service/hapi/HapiService.scala index 8a93310..e5fced1 100644 --- a/src/main/scala/latis/service/hapi/HapiService.scala +++ b/src/main/scala/latis/service/hapi/HapiService.scala @@ -42,7 +42,7 @@ class HapiService(catalog: Catalog) extends ServiceInterface(catalog, OperationR // This checks that: // - The Dataset metadata has temporalCoverage // - The single domain variable is of type Time - // - Each scalar in the range has a supported type + // - Each scalar in the range has a supported or convertible type // - If the type of a scalar is string, its size is defined private val filteredCatalog: Catalog = { val covP: Metadata => Boolean = _.getProperty("temporalCoverage").isDefined @@ -50,8 +50,9 @@ class HapiService(catalog: Catalog) extends ServiceInterface(catalog, OperationR case "string" => md.getProperty("size").isDefined case "double" => true case "int" => true - case "float" => true //may be converted to double by ConvertHapiTypes - case "short" => true //may be converted to int by ConvertHapiTypes + case "long" => true //converted to int by ConvertHapiTypes + case "float" => true //converted to double by ConvertHapiTypes + case "short" => true //converted to int by ConvertHapiTypes case _ => false } From 11fbbc8f5fd25208d04be2cfcbc7889e5490682d Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Tue, 4 Aug 2026 08:42:05 -0600 Subject: [PATCH 3/7] Update comments --- src/main/scala/latis/ops/ConvertHapiTypes.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/scala/latis/ops/ConvertHapiTypes.scala b/src/main/scala/latis/ops/ConvertHapiTypes.scala index b11a320..0d897c9 100644 --- a/src/main/scala/latis/ops/ConvertHapiTypes.scala +++ b/src/main/scala/latis/ops/ConvertHapiTypes.scala @@ -11,14 +11,19 @@ import latis.util.LatisException * Converts scalar values to be consistent with supported HAPI types. * * HAPI supports only double, int, and string types. This will convert - * some types that can safely be converted. Datasets with other types - * will be excluded from the Catalog by HapiService.filteredCatalog. - * This operation needs to be consistent with that filter. + * some types that can safely be converted. Longs are an exception. + * Although HAPI does not support 64-bit integers, they are common enough + * in data sources that they are converted to 32-bit integers here at + * the risk of integer overflow (wrapped to negative numbers, not an error). + * Datasets with other types will be excluded from the Catalog by + * HapiService.filteredCatalog. This operation needs to be consistent + * with that filter. + * + * This assumes flat datasets with no nesting. * * This is only needed for the binary output. */ class ConvertHapiTypes extends MapOperation { - //TODO: assumes non-nested functions or tuples def mapFunction(model: DataType): Sample => Sample = { // Note, domain can only be time and it is handled elsewhere From 2070b918af28eba25b1a904c755eed7b802aaa63 Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Tue, 4 Aug 2026 13:08:16 -0600 Subject: [PATCH 4/7] Check for long to int overflow --- .../scala/latis/ops/ConvertHapiTypes.scala | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/main/scala/latis/ops/ConvertHapiTypes.scala b/src/main/scala/latis/ops/ConvertHapiTypes.scala index 0d897c9..d0d2fd5 100644 --- a/src/main/scala/latis/ops/ConvertHapiTypes.scala +++ b/src/main/scala/latis/ops/ConvertHapiTypes.scala @@ -10,29 +10,38 @@ import latis.util.LatisException /** * Converts scalar values to be consistent with supported HAPI types. * - * HAPI supports only double, int, and string types. This will convert - * some types that can safely be converted. Longs are an exception. - * Although HAPI does not support 64-bit integers, they are common enough - * in data sources that they are converted to 32-bit integers here at - * the risk of integer overflow (wrapped to negative numbers, not an error). + * HAPI supports only doubles, 32-bit integers, and string types. + * This will convert LaTiS types that can safely be converted. Longs + * are an exception. Although HAPI does not support 64-bit integers, + * they are common enough in data sources that this attempts to convert + * long values to 32-bit integers. If the long value exceeds the max Int, + * a fill value will be used if defined for that variable. Otherwise, an + * overflow exception will be thrown. + * * Datasets with other types will be excluded from the Catalog by * HapiService.filteredCatalog. This operation needs to be consistent * with that filter. * * This assumes flat datasets with no nesting. * - * This is only needed for the binary output. + * This is only needed for the binary output, so it is otherwise not + * applicable. */ class ConvertHapiTypes extends MapOperation { def mapFunction(model: DataType): Sample => Sample = { // Note, domain can only be time and it is handled elsewhere - case Sample(d, r) => Sample(d, RangeData(r.map(convertValue))) + case Sample(d, r) => + val rdata = model.getScalars.tail.zip(r).map(convertValue) + Sample(d, RangeData(rdata)) } - private def convertValue(data: Data): Data = data match { + private def convertValue(scalar: Scalar, data: Data): Data = data match { case v: ShortValue => IntValue(v.value.toInt) - case v: LongValue => IntValue(v.value.toInt) //risk of overflow but no exception + case v: LongValue => + if (v.value > Int.MaxValue.toLong) + scalar.fillValue.getOrElse(throw LatisException("Integer overflow")) + else IntValue(v.value.toInt) case v: FloatValue => DoubleValue(v.value.toDouble) case _ => data //no-op, shouldn't get here due to catalog filter } From 44dabd9cb240ff7a1baa1ce0e20cc47aaa79fb65 Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Tue, 4 Aug 2026 13:11:56 -0600 Subject: [PATCH 5/7] note that some data source (such as ialirt) use longs when not needed --- src/main/scala/latis/ops/ConvertHapiTypes.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/latis/ops/ConvertHapiTypes.scala b/src/main/scala/latis/ops/ConvertHapiTypes.scala index d0d2fd5..062b8b6 100644 --- a/src/main/scala/latis/ops/ConvertHapiTypes.scala +++ b/src/main/scala/latis/ops/ConvertHapiTypes.scala @@ -13,10 +13,10 @@ import latis.util.LatisException * HAPI supports only doubles, 32-bit integers, and string types. * This will convert LaTiS types that can safely be converted. Longs * are an exception. Although HAPI does not support 64-bit integers, - * they are common enough in data sources that this attempts to convert - * long values to 32-bit integers. If the long value exceeds the max Int, - * a fill value will be used if defined for that variable. Otherwise, an - * overflow exception will be thrown. + * they are common enough in data sources (even when an int is sufficient) + * that this attempts to convert long values to 32-bit integers. If the + * long value exceeds the max Int, a fill value will be used if defined + * for that variable. Otherwise, an overflow exception will be thrown. * * Datasets with other types will be excluded from the Catalog by * HapiService.filteredCatalog. This operation needs to be consistent From a55d002e144c779c16801befa0bf37437cc8e69c Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Tue, 4 Aug 2026 14:45:10 -0600 Subject: [PATCH 6/7] Don't include a dataset with a long without a fillValue --- src/main/scala/latis/service/hapi/HapiService.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/latis/service/hapi/HapiService.scala b/src/main/scala/latis/service/hapi/HapiService.scala index e5fced1..bcbd4cd 100644 --- a/src/main/scala/latis/service/hapi/HapiService.scala +++ b/src/main/scala/latis/service/hapi/HapiService.scala @@ -44,13 +44,14 @@ class HapiService(catalog: Catalog) extends ServiceInterface(catalog, OperationR // - The single domain variable is of type Time // - Each scalar in the range has a supported or convertible type // - If the type of a scalar is string, its size is defined + // - If the type is "long" a fillValue must be defined private val filteredCatalog: Catalog = { val covP: Metadata => Boolean = _.getProperty("temporalCoverage").isDefined val typeP: Metadata => Boolean = md => md.getProperty("type").exists { case "string" => md.getProperty("size").isDefined case "double" => true case "int" => true - case "long" => true //converted to int by ConvertHapiTypes + case "long" => md.getProperty("fillValue").isDefined case "float" => true //converted to double by ConvertHapiTypes case "short" => true //converted to int by ConvertHapiTypes case _ => false From a779b4ad9993e868df9ba85e0109011b6eda0649 Mon Sep 17 00:00:00 2001 From: Doug Lindholm Date: Wed, 5 Aug 2026 11:06:03 -0600 Subject: [PATCH 7/7] Test negative int overflow --- src/main/scala/latis/ops/ConvertHapiTypes.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/latis/ops/ConvertHapiTypes.scala b/src/main/scala/latis/ops/ConvertHapiTypes.scala index 062b8b6..a6bb99c 100644 --- a/src/main/scala/latis/ops/ConvertHapiTypes.scala +++ b/src/main/scala/latis/ops/ConvertHapiTypes.scala @@ -15,8 +15,8 @@ import latis.util.LatisException * are an exception. Although HAPI does not support 64-bit integers, * they are common enough in data sources (even when an int is sufficient) * that this attempts to convert long values to 32-bit integers. If the - * long value exceeds the max Int, a fill value will be used if defined - * for that variable. Otherwise, an overflow exception will be thrown. + * long value exceeds the range of an Int, a fill value will be used + * if defined for that variable. Otherwise, an exception will be thrown. * * Datasets with other types will be excluded from the Catalog by * HapiService.filteredCatalog. This operation needs to be consistent @@ -39,7 +39,7 @@ class ConvertHapiTypes extends MapOperation { private def convertValue(scalar: Scalar, data: Data): Data = data match { case v: ShortValue => IntValue(v.value.toInt) case v: LongValue => - if (v.value > Int.MaxValue.toLong) + if (v.value > Int.MaxValue.toLong || v.value < Int.MinValue.toLong) scalar.fillValue.getOrElse(throw LatisException("Integer overflow")) else IntValue(v.value.toInt) case v: FloatValue => DoubleValue(v.value.toDouble)