fix: all-null chunks for every column type

CHANGELOG.md

@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - An all-null chunk is written as one null `vortex.constant`, as Rust writes it, instead of a masked constant `0`. ([#594](https://github.com/dfa1/vortex-java/pull/594))
 - An all-null chunk stored as a null constant (as vortex-jni writes it) reads as a masked array with every row null, like any nullable chunk, instead of a `NullArray` that the CLI, SQL and Parquet export failed on. ([#594](https://github.com/dfa1/vortex-java/pull/594))
+- An all-null date, time, timestamp or uuid chunk is written as a bare null `vortex.constant`, as Rust writes it, without a `vortex.ext` around it. ([#595](https://github.com/dfa1/vortex-java/pull/595))
+- An all-null decimal chunk reads as a masked array too, and a filter selects every row of an all-null uuid or list chunk for `IS NULL`, where it selected none. ([#595](https://github.com/dfa1/vortex-java/pull/595))
 - `filter` and `select --where` on the command line work on a nullable column, where they failed; a null row matches no comparison, as in SQL. ([#593](https://github.com/dfa1/vortex-java/pull/593))
 
 ## [0.18.1] — 2026-10-22

core/src/main/java/io/github/dfa1/vortex/core/model/DType.java

@@ -255,6 +255,26 @@ public static Decimal of(int precision, int scale, boolean nullable) {
             return new Decimal((byte) precision, (byte) scale, nullable);
         }
 
+        /// The bytes of one value: the smallest of 1, 2, 4, 8, 16 or 32 whose integer holds every
+        /// unscaled value of this precision, as Rust's `DecimalType::smallest_decimal_value_type`.
+        ///
+        /// @return the value width in bytes
+        public int byteWidth() {
+            if (precision <= 2) {
+                return 1;
+            }
+            if (precision <= 4) {
+                return 2;
+            }
+            if (precision <= 9) {
+                return 4;
+            }
+            if (precision <= 18) {
+                return 8;
+            }
+            return precision <= 38 ? 16 : 32;
+        }
+
         // A negative scale (e.g. -2 for hundreds) is legal.
         private static void requireValid(int precision, int scale) {
             if (precision < 1 || precision > 76) {

core/src/test/java/io/github/dfa1/vortex/core/model/DTypeDecimalTest.java

@@ -50,6 +50,20 @@ void constructor_enforcesTheSameBounds() {
                 .hasMessageContaining("out of range");
     }
 
+    @ParameterizedTest(name = "precision {0} -> {1} bytes")
+    @CsvSource({"1, 1", "2, 1", "3, 2", "4, 2", "5, 4", "9, 4", "10, 8", "18, 8", "19, 16", "38, 16", "39, 32", "76, 32"})
+    void byteWidth_matchesRustPrecisionBoundaries(int precision, int expectedBytes) {
+        // Given: Rust stores a decimal in the smallest integer its precision fits; the writer packs
+        // and the reader sizes placeholders by this, so a wrong edge misreads every value
+        DType.Decimal sut = DType.Decimal.of(precision, 0, false);
+
+        // When
+        int result = sut.byteWidth();
+
+        // Then
+        assertThat(result).isEqualTo(expectedBytes);
+    }
+
     @Test
     void factory_enforcesTheSameBounds() {
         // Given / When / Then: the public decimal(int, int) shortcut the writer's users call

integration/src/test/java/io/github/dfa1/vortex/integration/NullSparseRunEndInteropIntegrationTest.java

@@ -11,6 +11,9 @@
 import io.github.dfa1.vortex.core.model.ColumnName;
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.writer.WriteOptions;
+import io.github.dfa1.vortex.writer.encode.DateExtensionEncoder;
+import io.github.dfa1.vortex.writer.encode.TimeExtensionEncoder;
+import io.github.dfa1.vortex.writer.encode.TimestampExtensionEncoder;
 import io.github.dfa1.vortex.inspect.InspectorTree;
 import io.github.dfa1.vortex.reader.ReadRegistry;
 import io.github.dfa1.vortex.reader.ScanOptions;
@@ -282,19 +285,56 @@ void javaAllNullColumn_isANullConstantAsJniWritesIt(ArrowType arrowType, DType d
             writer.writeChunk(chunk -> chunk.put(ColumnName.of("v"), nulls));
         }
 
-        // Then: the data array is what Rust writes, and vortex-jni reads every row back as null
+        // Then: the data array is what Rust writes, vortex-jni reads every row back as null, and
+        // our reader reads every row of both files as null, masked like any nullable chunk
         assertThat(dataEncodings(javaFile)).isEqualTo(dataEncodings(jniFile)).isEqualTo(List.of("vortex.constant"));
         assertThat(jniNullCount(javaFile)).isEqualTo(ROWS);
+        assertThat(javaNullCount(javaFile)).isEqualTo(ROWS);
+        assertThat(javaNullCount(jniFile)).isEqualTo(ROWS);
     }
 
     static Stream<Arguments> allNullColumns() {
+        List<Object> noValues = java.util.Collections.nCopies(ROWS, null);
         return Stream.of(
-                Arguments.of(new ArrowType.Int(64, true), DType.I64.asNullable(), new Long[ROWS]),
+                Arguments.of(new ArrowType.Int(8, true), DType.I8.asNullable(), new Byte[ROWS]),
                 Arguments.of(new ArrowType.Int(16, true), DType.I16.asNullable(), new Short[ROWS]),
+                Arguments.of(new ArrowType.Int(32, true), DType.I32.asNullable(), new Integer[ROWS]),
+                Arguments.of(new ArrowType.Int(64, true), DType.I64.asNullable(), new Long[ROWS]),
+                Arguments.of(new ArrowType.Int(8, false), DType.U8.asNullable(), new Byte[ROWS]),
+                Arguments.of(new ArrowType.Int(32, false), DType.U32.asNullable(), new Integer[ROWS]),
+                Arguments.of(new ArrowType.Int(64, false), DType.U64.asNullable(), new Long[ROWS]),
+                Arguments.of(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), DType.F32.asNullable(),
+                        new Float[ROWS]),
                 Arguments.of(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), DType.F64.asNullable(),
                         new Double[ROWS]),
                 Arguments.of(ArrowType.Bool.INSTANCE, DType.BOOL.asNullable(), new Boolean[ROWS]),
-                Arguments.of(ArrowType.Utf8.INSTANCE, DType.UTF8.asNullable(), new String[ROWS]));
+                Arguments.of(ArrowType.Utf8.INSTANCE, DType.UTF8.asNullable(), new String[ROWS]),
+                Arguments.of(ArrowType.Binary.INSTANCE, DType.BINARY.asNullable(), new byte[ROWS][]),
+                Arguments.of(new ArrowType.Decimal(10, 2, 128), new DType.Decimal((byte) 10, (byte) 2, true),
+                        new java.math.BigDecimal[ROWS]),
+                Arguments.of(new ArrowType.Date(org.apache.arrow.vector.types.DateUnit.DAY),
+                        DateExtensionEncoder.INSTANCE.dtype(true), noValues),
+                Arguments.of(new ArrowType.Time(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, 32),
+                        TimeExtensionEncoder.INSTANCE.dtype(true), noValues),
+                Arguments.of(new ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null),
+                        TimestampExtensionEncoder.INSTANCE.dtype(true), noValues));
+    }
+
+    /// The rows of column `v` our reader reads as null.
+    private static long javaNullCount(Path file) throws IOException {
+        long[] nulls = {0};
+        try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll());
+             var iter = reader.scan(ScanOptions.all())) {
+            iter.forEachRemaining(chunk -> {
+                MaskedArray column = chunk.column(ColumnName.of("v"));
+                for (long i = 0; i < column.length(); i++) {
+                    if (!column.isValid(i)) {
+                        nulls[0]++;
+                    }
+                }
+            });
+        }
+        return nulls[0];
     }
 
     // ── JNI write helper ──────────────────────────────────────────────────────

reader/src/main/java/io/github/dfa1/vortex/reader/compute/NumericColumns.java

@@ -6,8 +6,10 @@
 import io.github.dfa1.vortex.reader.array.BoolArray;
 import io.github.dfa1.vortex.reader.array.ByteArray;
 import io.github.dfa1.vortex.reader.array.IntArray;
+import io.github.dfa1.vortex.reader.array.LazyConstantBoolArray;
 import io.github.dfa1.vortex.reader.array.LongArray;
 import io.github.dfa1.vortex.reader.array.MaskedArray;
+import io.github.dfa1.vortex.reader.array.NullArray;
 import io.github.dfa1.vortex.reader.array.ShortArray;
 
 /// Value-access helpers for numeric primitive columns, shared by the fused filter-and-aggregate
@@ -30,12 +32,18 @@ static Array unwrap(Array array) {
         return array instanceof MaskedArray masked ? masked.inner() : array;
     }
 
-    /// Returns the validity bitmap of a [MaskedArray], or `null` when the array carries no nulls.
+    /// Returns the validity bitmap of a [MaskedArray], an all-false one for a [NullArray] (an
+    /// all-null chunk of a type with no masked form, such as a uuid or a list), or `null` when the
+    /// array carries no nulls.
     ///
     /// @param array the array to inspect
     /// @return the validity bitmap, or `null`
     static BoolArray validityOf(Array array) {
-        return array instanceof MaskedArray masked ? masked.validity() : null;
+        return switch (array) {
+            case MaskedArray masked -> masked.validity();
+            case NullArray nulls -> new LazyConstantBoolArray(DType.BOOL, nulls.length(), false);
+            default -> null;
+        };
     }
 
     /// Reports whether the unwrapped array is a specialized long-domain primitive.

reader/src/main/java/io/github/dfa1/vortex/reader/compute/Values.java

@@ -29,14 +29,15 @@ private Values() {
 
     /// Reports whether the value at position `i` is null.
     ///
-    /// Only a [MaskedArray] carries nulls; any other array is non-nullable, so every position is
-    /// present. A masked array delegates to [MaskedArray#isValid(long)].
+    /// Only an array with a validity ([NumericColumns#validityOf(Array)]) carries nulls; any other
+    /// array is non-nullable, so every position is present.
     ///
     /// @param array the array to test
     /// @param i     the zero-based position
     /// @return `true` if the value at `i` is null
     static boolean isNullAt(Array array, long i) {
-        return array instanceof MaskedArray masked && !masked.isValid(i);
+        BoolArray validity = NumericColumns.validityOf(array);
+        return validity != null && !validity.getBoolean(i);
     }
 
     /// Reads the value at position `i` as a boxed scalar through the array's typed accessor.

reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java

@@ -22,6 +22,7 @@
 
 import java.io.IOException;
 import java.lang.foreign.MemorySegment;
+import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
 
 /// Read-only decoder for `vortex.constant`.
@@ -63,11 +64,10 @@ private static Array arrayFromScalar(DecodeContext ctx, ProtoScalarValue scalar,
         // canonicalizes it to the dtype's own array with every row invalid, so a reader sees an
         // all-null I64? chunk as any other masked I64? chunk, not as a NullArray it cannot cast.
         if (scalar.null_value() != null) {
-            if (dtype instanceof DType.Decimal || dtype instanceof DType.Variant) {
-                // ponytail: no placeholder value built for these yet, they stay a NullArray
+            Array placeholder = placeholder(ctx, dtype.withNullable(false), n);
+            if (placeholder == null) {
                 return new NullArray(dtype, n);
             }
-            Array placeholder = arrayFromScalar(ctx, NO_VALUE, dtype.withNullable(false), n);
             return new MaskedArray(placeholder, new LazyConstantBoolArray(DType.BOOL, n, false));
         }
         if (dtype instanceof DType.Variant) {
@@ -129,6 +129,22 @@ private static Array constantPrimitive(DType outDtype, PType ptype, ProtoScalarV
         };
     }
 
+    /// The value under a null constant's all-invalid validity: a constant `0`, `false`, `""` or
+    /// zero decimal of `dtype`, or `null` for a dtype with no such scalar.
+    ///
+    /// ponytail: lists, structs, variants and extensions over non-primitive storage (uuid) have
+    /// no placeholder yet and stay a NullArray; add one when a consumer needs them masked.
+    private static Array placeholder(DecodeContext ctx, DType dtype, long n) {
+        return switch (dtype) {
+            case DType.Bool _, DType.Primitive _, DType.Utf8 _, DType.Binary _ -> arrayFromScalar(ctx, NO_VALUE, dtype, n);
+            case DType.Decimal decimal -> new LazyConstantDecimalArray(dtype, n,
+                    BigDecimal.ZERO.setScale(decimal.scale()), decimal.byteWidth());
+            case DType.Extension ext when ext.storageDType() instanceof DType.Primitive ->
+                    arrayFromScalar(ctx, NO_VALUE, dtype, n);
+            default -> null;
+        };
+    }
+
     private static Array decodeDecimal(DType dtype, ProtoScalarValue scalar, long n) {
         byte[] elemBytes = scalar.bytes_value();
         if (elemBytes == null) {

reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeTest.java

@@ -9,6 +9,7 @@
 import io.github.dfa1.vortex.reader.array.Array;
 import io.github.dfa1.vortex.reader.array.LazyDecimalBytePartsArray;
 import io.github.dfa1.vortex.reader.array.MaskedArray;
+import io.github.dfa1.vortex.reader.array.NullArray;
 import io.github.dfa1.vortex.reader.array.VarBinArray;
 import io.github.dfa1.vortex.reader.array.VarBinDictArray;
 import io.github.dfa1.vortex.reader.array.VarBinSlicedArray;
@@ -78,6 +79,43 @@ void filteredAggregateRejectsNullArguments() {
                 .isThrownBy(() -> Compute.filteredAggregate(chunk, null, ColumnName.of("v")));
     }
 
+    /// An all-null chunk of a type with no masked form (a uuid, a list) reads as a [NullArray]; the
+    /// filter tested it as a non-nullable array, so IS NULL selected none of its null rows.
+    @Nested
+    class AllNullChunk {
+
+        private static final ColumnName COL = ColumnName.of("c");
+        private static final long ROWS = 5;
+
+        static Stream<Arguments> predicates() {
+            return Stream.of(
+                    Arguments.of(named("is null", new Predicate.IsNull()), true),
+                    Arguments.of(named("is not null", new Predicate.IsNotNull()), false),
+                    Arguments.of(named("eq", new Predicate.Eq(1L)), false),
+                    Arguments.of(named("neq", new Predicate.Neq(1L)), false),
+                    Arguments.of(named("gt", new Predicate.Gt(1L)), false),
+                    Arguments.of(named("lte", new Predicate.Lte(1L)), false));
+        }
+
+        @ParameterizedTest(name = "{0}")
+        @MethodSource("predicates")
+        void onlyIsNullSelectsItsRows(Predicate predicate, boolean selected) {
+            // Given
+            Chunk chunk = mock(Chunk.class);
+            given(chunk.<Array>column(COL)).willReturn(new NullArray(DType.I64.asNullable(), ROWS));
+            RowFilter filter = new RowFilter.Column(COL, predicate);
+
+            // When
+            LongPredicate result = Compute.matcher(chunk, filter);
+
+            // Then: the lowered matcher and the per-row oracle agree on every row
+            for (long i = 0; i < ROWS; i++) {
+                assertThat(result.test(i)).as("row %d", i).isEqualTo(selected);
+                assertThat(Compute.matches(chunk, filter, i)).as("row %d", i).isEqualTo(selected);
+            }
+        }
+    }
+
     /// [Compute#matcher(Chunk, RowFilter)] lowers a filter once per chunk; over a string dictionary
     /// it evaluates the predicate per dictionary entry instead of per row (#552). The per-row
     /// [Compute#matches(Chunk, RowFilter, long)] is the oracle: both must select the same rows.

reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java

@@ -6,6 +6,7 @@
 import io.github.dfa1.vortex.reader.array.Array;
 import io.github.dfa1.vortex.reader.array.GenericArray;
 import io.github.dfa1.vortex.reader.array.MaskedArray;
+import io.github.dfa1.vortex.reader.array.NullArray;
 import io.github.dfa1.vortex.reader.array.MaterializedByteArray;
 import io.github.dfa1.vortex.reader.array.MaterializedIntArray;
 import io.github.dfa1.vortex.reader.array.MaterializedShortArray;
@@ -119,6 +120,16 @@ void maskedArrayReadsThroughItsPayload() {
         assertThat(Values.isNullAt(masked, 1)).isTrue();
     }
 
+    @Test
+    void aNullArrayIsNullAtEveryRow() {
+        // Given an all-null chunk of a type with no masked form (a uuid, a list)
+        Array nulls = new NullArray(DType.I64.asNullable(), 3);
+
+        // When / Then
+        assertThat(Values.isNullAt(nulls, 0)).isTrue();
+        assertThat(Values.isNullAt(nulls, 2)).isTrue();
+    }
+
     private static Array byteArray(DType dtype, byte... values) {
         MemorySegment seg = ARENA.allocate(Math.max(1L, values.length));
         for (int i = 0; i < values.length; i++) {