Added support for Parquet TIME (#4775)

deephaven · Nov 7, 2023 · 24f8182 · 24f8182
1 parent 914825d
commit 24f8182
Show file tree

Hide file tree

Showing 27 changed files with 621 additions and 131 deletions.
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/CodecLookup.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/CodecLookup.java
@@ -19,6 +19,7 @@
 import java.math.BigInteger;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.LocalTime;
 
 /**
  * Utility class to concentrate {@link ObjectCodec} lookups.
@@ -74,6 +75,7 @@ private static boolean noCodecRequired(@NotNull final Class<?> dataType) {
         return dataType == Boolean.class ||
                 dataType == Instant.class ||
                 dataType == LocalDate.class ||
+                dataType == LocalTime.class ||
                 dataType == String.class ||
                 // A BigDecimal column maps to a logical type of decimal, with
                 // appropriate precision and scale calculated from column data,

diff --git a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java
@@ -876,6 +876,45 @@ public static LocalTime toLocalTime(@Nullable final ZonedDateTime dateTime) {
         return dateTime.toLocalTime();
     }
 
+    /**
+     * Converts the number of milliseconds from midnight to a {@link LocalTime}
+     *
+     * @param millis milliseconds from midnight
+     * @return the {@link LocalTime}, or {@code null} if any input is {@link QueryConstants#NULL_INT NULL_INT}
+     */
+    public static @Nullable LocalTime millisOfDayToLocalTime(final int millis) {
+        if (millis == NULL_INT) {
+            return null;
+        }
+        return LocalTime.ofNanoOfDay(millis * MILLI);
+    }
+
+    /**
+     * Converts the number of microseconds from midnight to a {@link LocalTime}
+     *
+     * @param micros microseconds from midnight
+     * @return the {@link LocalTime}, or {@code null} if any input is {@link QueryConstants#NULL_LONG NULL_LONG}
+     */
+    public static @Nullable LocalTime microsOfDayToLocalTime(final long micros) {
+        if (micros == NULL_LONG) {
+            return null;
+        }
+        return LocalTime.ofNanoOfDay(micros * MICRO);
+    }
+
+    /**
+     * Converts the number of nanoseconds from midnight to a {@link LocalTime}
+     *
+     * @param nanos nanoseconds from midnight
+     * @return the {@link LocalTime}, or {@code null} if any input is {@link QueryConstants#NULL_LONG NULL_LONG}
+     */
+    public static @Nullable LocalTime nanosOfDayToLocalTime(final long nanos) {
+        if (nanos == NULL_LONG) {
+            return null;
+        }
+        return LocalTime.ofNanoOfDay(nanos);
+    }
+
     /**
      * Converts an {@link Instant} to a {@link Date}. {@code instant} will be truncated to millisecond resolution.
      *
@@ -2350,7 +2389,7 @@ public static int minuteOfHour(@Nullable final ZonedDateTime dateTime) {
      *
      * @param instant time
      * @param timeZone time zone
-     * @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of nanoseconds that
+     * @return {@link QueryConstants#NULL_LONG} if either input is {@code null}; otherwise, number of nanoseconds that
      *         have elapsed since the top of the day
      */
     @ScriptApi
@@ -2370,7 +2409,7 @@ public static long nanosOfDay(@Nullable final Instant instant, @Nullable final Z
      * upon if the daylight savings time adjustment is forwards or backwards.
      *
      * @param dateTime time
-     * @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of nanoseconds that
+     * @return {@link QueryConstants#NULL_LONG} if either input is {@code null}; otherwise, number of nanoseconds that
      *         have elapsed since the top of the day
      */
     @ScriptApi
@@ -2382,6 +2421,21 @@ public static long nanosOfDay(@Nullable final ZonedDateTime dateTime) {
         return epochNanos(dateTime) - epochNanos(atMidnight(dateTime));
     }
 
+    /**
+     * Returns the number of nanoseconds that have elapsed since the top of the day.
+     *
+     * @param localTime time
+     * @return {@link QueryConstants#NULL_LONG} if input is {@code null}; otherwise, number of nanoseconds that have
+     *         elapsed since the top of the day
+     */
+    public static long nanosOfDay(@Nullable final LocalTime localTime) {
+        if (localTime == null) {
+            return NULL_LONG;
+        }
+
+        return localTime.toNanoOfDay();
+    }
+
     /**
      * Returns the number of milliseconds that have elapsed since the top of the day.
      * <p>

diff --git a/engine/time/src/test/java/io/deephaven/time/TestDateTimeUtils.java b/engine/time/src/test/java/io/deephaven/time/TestDateTimeUtils.java
@@ -1350,6 +1350,19 @@ public void testToLocalTime() {
         TestCase.assertEquals(lt, DateTimeUtils.toLocalTime(dt3));
         // noinspection ConstantConditions
         TestCase.assertNull(DateTimeUtils.toLocalTime(null));
+
+        final LocalTime someTimeInMillis = LocalTime.of(6, 33, 9, (int) (123 * DateTimeUtils.MILLI));
+        TestCase.assertEquals(someTimeInMillis,
+                DateTimeUtils.millisOfDayToLocalTime((int) (someTimeInMillis.toNanoOfDay() / DateTimeUtils.MILLI)));
+        TestCase.assertNull(DateTimeUtils.millisOfDayToLocalTime(NULL_INT));
+
+        final LocalTime someTimeInMicros = LocalTime.of(6, 33, 9, (int) (123456 * DateTimeUtils.MICRO));
+        TestCase.assertEquals(someTimeInMicros,
+                DateTimeUtils.microsOfDayToLocalTime(someTimeInMicros.toNanoOfDay() / DateTimeUtils.MICRO));
+        TestCase.assertNull(DateTimeUtils.microsOfDayToLocalTime(NULL_LONG));
+
+        TestCase.assertEquals(lt, DateTimeUtils.nanosOfDayToLocalTime(lt.toNanoOfDay()));
+        TestCase.assertNull(DateTimeUtils.nanosOfDayToLocalTime(NULL_LONG));
     }
 
     public void testToZonedDateTime() {
@@ -2665,52 +2678,68 @@ public void testNanosOfMilli() {
     public void testNanosOfDay() {
         final Instant dt2 = DateTimeUtils.parseInstant("2023-01-02T11:23:45.123456789 JP");
         final ZonedDateTime dt3 = dt2.atZone(TZ_JP);
+        final LocalTime lt = dt3.toLocalTime();
+        final long expectedNanos = 123456789L + 1_000_000_000L * (45 + 23 * 60 + 11 * 60 * 60);
 
-        TestCase.assertEquals(123456789L + 1_000_000_000L * (45 + 23 * 60 + 11 * 60 * 60),
-                DateTimeUtils.nanosOfDay(dt2, TZ_JP));
+        TestCase.assertEquals(expectedNanos, DateTimeUtils.nanosOfDay(dt2, TZ_JP));
         TestCase.assertEquals(NULL_LONG, DateTimeUtils.nanosOfDay(dt2, null));
         TestCase.assertEquals(NULL_LONG, DateTimeUtils.nanosOfDay(null, TZ_JP));
 
-        TestCase.assertEquals(123456789L + 1_000_000_000L * (45 + 23 * 60 + 11 * 60 * 60),
-                DateTimeUtils.nanosOfDay(dt3));
-        TestCase.assertEquals(NULL_LONG, DateTimeUtils.nanosOfDay(null));
+        TestCase.assertEquals(expectedNanos, DateTimeUtils.nanosOfDay(dt3));
+        TestCase.assertEquals(NULL_LONG, DateTimeUtils.nanosOfDay((ZonedDateTime) null));
+
+        TestCase.assertEquals(expectedNanos, DateTimeUtils.nanosOfDay(lt));
+        TestCase.assertEquals(NULL_LONG, DateTimeUtils.nanosOfDay((LocalTime) null));
 
         // Test daylight savings time
 
         final Instant dstMid1 = DateTimeUtils.parseInstant("2023-03-12T00:00:00 America/Denver");
 
         final Instant dstI11 = DateTimeUtils.plus(dstMid1, DateTimeUtils.HOUR);
         final ZonedDateTime dstZdt11 = DateTimeUtils.toZonedDateTime(dstI11, ZoneId.of("America/Denver"));
+        final LocalTime dstLt11 = dstZdt11.toLocalTime();
         TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstI11, ZoneId.of("America/Denver")));
         TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstZdt11));
+        TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstLt11));
+
 
         final Instant dstI12 = DateTimeUtils.plus(dstMid1, 2 * DateTimeUtils.HOUR);
         final ZonedDateTime dstZdt12 = DateTimeUtils.toZonedDateTime(dstI12, ZoneId.of("America/Denver"));
+        final LocalTime dstLt12 = dstZdt12.toLocalTime();
         TestCase.assertEquals(2 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstI12, ZoneId.of("America/Denver")));
         TestCase.assertEquals(2 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstZdt12));
+        TestCase.assertEquals(3 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstLt12)); // Adjusted
 
         final Instant dstI13 = DateTimeUtils.plus(dstMid1, 3 * DateTimeUtils.HOUR);
         final ZonedDateTime dstZdt13 = DateTimeUtils.toZonedDateTime(dstI13, ZoneId.of("America/Denver"));
+        final LocalTime dstLt13 = dstZdt13.toLocalTime();
         TestCase.assertEquals(3 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstI13, ZoneId.of("America/Denver")));
         TestCase.assertEquals(3 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstZdt13));
+        TestCase.assertEquals(4 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstLt13)); // Adjusted
 
 
         final Instant dstMid2 = DateTimeUtils.parseInstant("2023-11-05T00:00:00 America/Denver");
 
         final Instant dstI21 = DateTimeUtils.plus(dstMid2, DateTimeUtils.HOUR);
         final ZonedDateTime dstZdt21 = DateTimeUtils.toZonedDateTime(dstI21, ZoneId.of("America/Denver"));
+        final LocalTime dstLt21 = dstZdt21.toLocalTime();
         TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstI21, ZoneId.of("America/Denver")));
         TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstZdt21));
+        TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstLt21));
 
         final Instant dstI22 = DateTimeUtils.plus(dstMid2, 2 * DateTimeUtils.HOUR);
         final ZonedDateTime dstZdt22 = DateTimeUtils.toZonedDateTime(dstI22, ZoneId.of("America/Denver"));
+        final LocalTime dstLt22 = dstZdt22.toLocalTime();
         TestCase.assertEquals(2 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstI22, ZoneId.of("America/Denver")));
         TestCase.assertEquals(2 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstZdt22));
+        TestCase.assertEquals(DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstLt22)); // Adjusted
 
         final Instant dstI23 = DateTimeUtils.plus(dstMid2, 3 * DateTimeUtils.HOUR);
         final ZonedDateTime dstZdt23 = DateTimeUtils.toZonedDateTime(dstI23, ZoneId.of("America/Denver"));
+        final LocalTime dstLt23 = dstZdt23.toLocalTime();
         TestCase.assertEquals(3 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstI23, ZoneId.of("America/Denver")));
         TestCase.assertEquals(3 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstZdt23));
+        TestCase.assertEquals(2 * DateTimeUtils.HOUR, DateTimeUtils.nanosOfDay(dstLt23)); // Adjusted
     }
 
     public void testMillisOfSecond() {

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java
@@ -26,6 +26,7 @@
 import java.math.BigInteger;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.LocalTime;
 import java.util.*;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
@@ -343,10 +344,7 @@ public Optional<Class<?>> visit(final LogicalTypeAnnotation.DateLogicalTypeAnnot
 
             @Override
             public Optional<Class<?>> visit(final LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
-                if (timeLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS) {
-                    return Optional.of(int.class);
-                }
-                return Optional.of(long.class);
+                return Optional.of(LocalTime.class);
             }
 
             @Override

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java
@@ -25,6 +25,7 @@
 import java.math.BigInteger;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.LocalTime;
 import java.util.*;
 import java.util.function.Supplier;
 
@@ -48,7 +49,8 @@ public class TypeInfos {
             StringType.INSTANCE,
             InstantType.INSTANCE,
             BigIntegerType.INSTANCE,
-            LocalDateType.INSTANCE
+            LocalDateType.INSTANCE,
+            LocalTimeType.INSTANCE,
     };
 
     private static final Map<Class<?>, TypeInfo> BY_CLASS;
@@ -399,6 +401,28 @@ public PrimitiveBuilder<PrimitiveType> getBuilder(boolean required, boolean repe
         }
     }
 
+    private enum LocalTimeType implements TypeInfo {
+        INSTANCE;
+
+        private static final Set<Class<?>> clazzes = Collections.singleton(LocalTime.class);
+
+        @Override
+        public Set<Class<?>> getTypes() {
+            return clazzes;
+        }
+
+        @Override
+        public PrimitiveBuilder<PrimitiveType> getBuilder(boolean required, boolean repeating, Class<?> dataType) {
+            if (!isValidFor(dataType)) {
+                throw new IllegalArgumentException("Invalid data type " + dataType);
+            }
+            // Always write in (isAdjustedToUTC = true, unit = NANOS) format
+            return type(PrimitiveTypeName.INT64, required, repeating)
+                    .as(LogicalTypeAnnotation.timeType(true, LogicalTypeAnnotation.TimeUnit.NANOS));
+        }
+    }
+
+
     /**
      * We will encode BigIntegers as Decimal types. Parquet has no special type for BigIntegers, but we can maintain
      * external compatibility by encoding them as fixed length decimals of scale 1. Internally, we'll record that we

diff --git a/...arquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java b/...arquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java
@@ -706,23 +706,23 @@ private static class LogicalTypeVisitor<ATTR extends Any>
         private final ColumnChunkReader columnChunkReader;
         private final Class<?> componentType;
 
-        LogicalTypeVisitor(@NotNull String name, @NotNull ColumnChunkReader columnChunkReader,
-                Class<?> componentType) {
+        LogicalTypeVisitor(@NotNull final String name, @NotNull final ColumnChunkReader columnChunkReader,
+                final Class<?> componentType) {
             this.name = name;
             this.columnChunkReader = columnChunkReader;
             this.componentType = componentType;
         }
 
         @Override
         public Optional<ToPage<ATTR, ?>> visit(
-                LogicalTypeAnnotation.StringLogicalTypeAnnotation stringLogicalType) {
+                final LogicalTypeAnnotation.StringLogicalTypeAnnotation stringLogicalType) {
             return Optional
                     .of(ToStringPage.create(componentType, columnChunkReader.getDictionarySupplier()));
         }
 
         @Override
         public Optional<ToPage<ATTR, ?>> visit(
-                LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
+                final LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
             // TODO(deephaven-core#976): Unable to read parquet TimestampLogicalTypeAnnotation that is not adjusted
             // to UTC
             if (timestampLogicalType.isAdjustedToUTC()) {
@@ -733,8 +733,7 @@ private static class LogicalTypeVisitor<ATTR extends Any>
         }
 
         @Override
-        public Optional<ToPage<ATTR, ?>> visit(
-                LogicalTypeAnnotation.IntLogicalTypeAnnotation intLogicalType) {
+        public Optional<ToPage<ATTR, ?>> visit(final LogicalTypeAnnotation.IntLogicalTypeAnnotation intLogicalType) {
 
             if (intLogicalType.isSigned()) {
                 switch (intLogicalType.getBitWidth()) {
@@ -761,18 +760,14 @@ private static class LogicalTypeVisitor<ATTR extends Any>
         }
 
         @Override
-        public Optional<ToPage<ATTR, ?>> visit(
-                LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) {
+        public Optional<ToPage<ATTR, ?>> visit(final LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) {
             return Optional.of(ToDatePageFromInt.create(componentType));
         }
 
         @Override
-        public Optional<ToPage<ATTR, ?>> visit(
-                LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
-            if (timeLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS) {
-                return Optional.of(ToIntPage.create(componentType));
-            }
-            return Optional.of(ToLongPage.create(componentType));
+        public Optional<ToPage<ATTR, ?>> visit(final LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
+            return Optional
+                    .of(ToTimePage.create(componentType, timeLogicalType.getUnit(), timeLogicalType.isAdjustedToUTC()));
         }
 
         @Override