fix: all-null chunks as a null constant, read back masked

CHANGELOG.md

@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Breaking:** CSV import stores an empty cell in a number or boolean column as null instead of `0` or `false`, so inferred number and boolean columns are now nullable (read through `MaskedArray`), and an empty cell or a missing column fails the import where the schema declares the column non-nullable. ([#592](https://github.com/dfa1/vortex-java/issues/592))
 
 ### Fixed
+- An all-null chunk is written as one null `vortex.constant`, as Rust writes it, instead of a masked constant `0`. ([#594](https://github.com/dfa1/vortex-java/pull/594))
+- An all-null chunk stored as a null constant (as vortex-jni writes it) reads as a masked array with every row null, like any nullable chunk, instead of a `NullArray` that the CLI, SQL and Parquet export failed on. ([#594](https://github.com/dfa1/vortex-java/pull/594))
 - `filter` and `select --where` on the command line work on a nullable column, where they failed; a null row matches no comparison, as in SQL. ([#593](https://github.com/dfa1/vortex-java/pull/593))
 
 ## [0.18.1] — 2026-10-22

integration/src/test/java/io/github/dfa1/vortex/integration/ListViewInteropIntegrationTest.java

@@ -111,9 +111,15 @@ void javaWriter_jniReader_rowRangeAndFiltered(Element element, Nulls nulls, Elem
                 ListInteropIntegrationTest.expected(element, nulls, elementNulls, 333, 1_777));
         assertThat(filtered).containsExactlyInAnyOrderElementsOf(
                 ListInteropIntegrationTest.expected(element, nulls, elementNulls, (int) THRESHOLD, ROWS));
+        // an all-null column is one null constant before any encoding is tried, as Rust's compressor
+        // makes it, so only the other shapes hold the list view
+        String encoding = "vortex.listview";
+        if (nulls == Nulls.ALL_NULL) {
+            encoding = "vortex.constant";
+        }
         try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll())) {
             assertThat(VortexInspector.inspect(reader)).as("the file must hold the encoding under test")
-                    .contains("vortex.listview");
+                    .contains(encoding);
         }
     }
 

integration/src/test/java/io/github/dfa1/vortex/integration/NullSparseRunEndInteropIntegrationTest.java

@@ -1,10 +1,16 @@
 package io.github.dfa1.vortex.integration;
 
+import dev.vortex.api.DataSource;
+import dev.vortex.api.Expression;
+import dev.vortex.api.Partition;
+import dev.vortex.api.Scan;
 import dev.vortex.api.Session;
 import dev.vortex.api.VortexWriter;
 import dev.vortex.arrow.ArrowAllocation;
 import dev.vortex.jni.NativeLoader;
 import io.github.dfa1.vortex.core.model.ColumnName;
+import io.github.dfa1.vortex.core.model.DType;
+import io.github.dfa1.vortex.writer.WriteOptions;
 import io.github.dfa1.vortex.inspect.InspectorTree;
 import io.github.dfa1.vortex.reader.ReadRegistry;
 import io.github.dfa1.vortex.reader.ScanOptions;
@@ -13,7 +19,6 @@
 import io.github.dfa1.vortex.reader.array.DoubleArray;
 import io.github.dfa1.vortex.reader.array.LongArray;
 import io.github.dfa1.vortex.reader.array.MaskedArray;
-import io.github.dfa1.vortex.reader.array.NullArray;
 import io.github.dfa1.vortex.reader.array.ShortArray;
 import io.github.dfa1.vortex.reader.array.VarBinArray;
 import io.github.dfa1.vortex.reader.array.VarBinSparseArray;
@@ -26,20 +31,27 @@
 import org.apache.arrow.vector.SmallIntVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.ArrowReader;
 import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 import java.util.function.ObjIntConsumer;
+import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -231,9 +243,9 @@ void jniNullConstant_i64_allRowsDecodeNull(@TempDir Path tmp) throws IOException
             iter.forEachRemaining(chunk -> {
                 Array col = chunk.column(ColumnName.of("v"));
                 rowCount[0] += col.length();
-                if (col instanceof NullArray) {
-                    nullCount[0] += col.length();
-                } else if (col instanceof MaskedArray m) {
+                // masked like any nullable chunk: a NullArray here failed every consumer that
+                // unwraps a MaskedArray and casts its payload
+                if (col instanceof MaskedArray m) {
                     for (long i = 0; i < m.length(); i++) {
                         if (!m.isValid(i)) {
                             nullCount[0]++;
@@ -252,6 +264,39 @@ void jniNullConstant_i64_allRowsDecodeNull(@TempDir Path tmp) throws IOException
         assertThat(usedEncodings(file)).contains("vortex.constant");
     }
 
+    @ParameterizedTest
+    @MethodSource("allNullColumns")
+    void javaAllNullColumn_isANullConstantAsJniWritesIt(ArrowType arrowType, DType dtype, Object nulls,
+                                                        @TempDir Path tmp) throws IOException {
+        // Given: the same all-null column written by both writers. Rust's compressor turns an
+        // all-invalid array into one null-scalar vortex.constant (compressor/cascade.rs); Java
+        // wrote vortex.masked(constant 0, constant false), a values child that carries nothing
+        Path jniFile = tmp.resolve("jni.vtx");
+        writeJni(jniFile, new Schema(List.of(Field.nullable("v", arrowType))), (root, i) -> { });
+        Path javaFile = tmp.resolve("java.vtx");
+
+        // When
+        try (FileChannel channel = FileChannel.open(javaFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+             var writer = io.github.dfa1.vortex.writer.VortexWriter.create(channel,
+                     DType.structBuilder().field(ColumnName.of("v"), dtype).build(), WriteOptions.defaults())) {
+            writer.writeChunk(chunk -> chunk.put(ColumnName.of("v"), nulls));
+        }
+
+        // Then: the data array is what Rust writes, and vortex-jni reads every row back as null
+        assertThat(dataEncodings(javaFile)).isEqualTo(dataEncodings(jniFile)).isEqualTo(List.of("vortex.constant"));
+        assertThat(jniNullCount(javaFile)).isEqualTo(ROWS);
+    }
+
+    static Stream<Arguments> allNullColumns() {
+        return Stream.of(
+                Arguments.of(new ArrowType.Int(64, true), DType.I64.asNullable(), new Long[ROWS]),
+                Arguments.of(new ArrowType.Int(16, true), DType.I16.asNullable(), new Short[ROWS]),
+                Arguments.of(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), DType.F64.asNullable(),
+                        new Double[ROWS]),
+                Arguments.of(ArrowType.Bool.INSTANCE, DType.BOOL.asNullable(), new Boolean[ROWS]),
+                Arguments.of(ArrowType.Utf8.INSTANCE, DType.UTF8.asNullable(), new String[ROWS]));
+    }
+
     // ── JNI write helper ──────────────────────────────────────────────────────
 
     private static void writeJni(Path file, Schema schema, ObjIntConsumer<VectorSchemaRoot> fill) throws IOException {
@@ -282,6 +327,36 @@ private static List<String> usedEncodings(Path file) throws IOException {
         }
     }
 
+    /// The encodings of column `v`'s data, without its zone-map table: the zoned layout's first
+    /// child, or the column itself when it is not zoned.
+    private static List<String> dataEncodings(Path file) throws IOException {
+        try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll())) {
+            InspectorTree.Node column = InspectorTree.build(reader).root().children().getFirst();
+            InspectorTree.Node data = column;
+            if (column.layout().isZoned()) {
+                data = column.children().getFirst();
+            }
+            return data.usedEncodings().stream().sorted().toList();
+        }
+    }
+
+    /// The null rows vortex-jni reads in column `v`.
+    private static long jniNullCount(Path file) throws IOException {
+        var options = dev.vortex.api.ScanOptions.builder()
+                .projection(Expression.select(new String[]{"v"}, Expression.root()));
+        long nulls = 0;
+        Scan scan = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString()).scan(options.build());
+        while (scan.hasNext()) {
+            Partition partition = scan.next();
+            try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
+                while (reader.loadNextBatch()) {
+                    nulls += reader.getVectorSchemaRoot().getVector("v").getNullCount();
+                }
+            }
+        }
+        return nulls;
+    }
+
     private static List<Double> readF64(Path file) throws IOException {
         var out = new ArrayList<Double>();
         forEachValue(file, (masked, inner, i) -> out.add(masked.isValid(i) ? ((DoubleArray) inner).getDouble(i) : null));

reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java

@@ -16,6 +16,7 @@
 import io.github.dfa1.vortex.reader.array.LazyConstantLongArray;
 import io.github.dfa1.vortex.reader.array.LazyConstantShortArray;
 import io.github.dfa1.vortex.reader.array.LazyDecimalArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
 import io.github.dfa1.vortex.reader.array.NullArray;
 import io.github.dfa1.vortex.reader.array.VarBinConstantArray;
 
@@ -26,6 +27,10 @@
 /// Read-only decoder for `vortex.constant`.
 public final class ConstantEncodingDecoder implements EncodingDecoder {
 
+    /// A scalar with no value set: decodes as `0`, `false` or empty bytes, the placeholder under
+    /// a null constant's all-invalid validity.
+    private static final ProtoScalarValue NO_VALUE = ProtoScalarValue.ofNullValue(null);
+
     /// Creates the decoder.
     public ConstantEncodingDecoder() {
     }
@@ -54,9 +59,16 @@ private static Array arrayFromScalar(DecodeContext ctx, ProtoScalarValue scalar,
         if (dtype instanceof DType.Null) {
             return new NullArray(dtype, n);
         }
-        // Null scalar (proto null_value tag present) means every row is null (#246).
+        // Null scalar (proto null_value tag present) means every row is null (#246). Rust
+        // canonicalizes it to the dtype's own array with every row invalid, so a reader sees an
+        // all-null I64? chunk as any other masked I64? chunk, not as a NullArray it cannot cast.
         if (scalar.null_value() != null) {
-            return new NullArray(dtype, n);
+            if (dtype instanceof DType.Decimal || dtype instanceof DType.Variant) {
+                // ponytail: no placeholder value built for these yet, they stay a NullArray
+                return new NullArray(dtype, n);
+            }
+            Array placeholder = arrayFromScalar(ctx, NO_VALUE, dtype.withNullable(false), n);
+            return new MaskedArray(placeholder, new LazyConstantBoolArray(DType.BOOL, n, false));
         }
         if (dtype instanceof DType.Variant) {
             // A constant variant wraps a typed inner scalar (Scalar::variant(inner)); the

reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java

@@ -3,16 +3,25 @@
 import io.github.dfa1.vortex.core.error.VortexException;
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.model.EncodingId;
+import io.github.dfa1.vortex.core.proto.ProtoNullValue;
 import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
 import io.github.dfa1.vortex.reader.ReadRegistry;
 import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.BoolArray;
+import io.github.dfa1.vortex.reader.array.DoubleArray;
 import io.github.dfa1.vortex.reader.array.LongArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
+import io.github.dfa1.vortex.reader.array.ShortArray;
 import io.github.dfa1.vortex.reader.array.VarBinArray;
 import io.github.dfa1.vortex.reader.array.VarBinConstantArray;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
+import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -42,6 +51,34 @@ void primitiveScalar_missingAllValueFields_decodesAsZero() {
         assertThat(longs.getLong(2)).isZero();
     }
 
+    @ParameterizedTest
+    @MethodSource("nullableTypes")
+    void nullScalar_decodesAsAMaskedArrayWithEveryRowInvalid(DType dtype, Class<?> payload) {
+        // Given: a null constant, which Rust and our writer emit for an all-null chunk; it read as
+        // a NullArray, which every consumer that unwraps a MaskedArray and casts failed on
+        ProtoScalarValue scalar = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE);
+
+        // When
+        Array result = decode(scalar, dtype, 3);
+
+        // Then
+        assertThat(result).isInstanceOf(MaskedArray.class);
+        assertThat(result.dtype()).isEqualTo(dtype);
+        MaskedArray masked = (MaskedArray) result;
+        assertThat(masked.inner()).isInstanceOf(payload);
+        assertThat(masked.isValid(0)).isFalse();
+        assertThat(masked.isValid(2)).isFalse();
+    }
+
+    static Stream<Arguments> nullableTypes() {
+        return Stream.of(
+                Arguments.of(DType.I64.asNullable(), LongArray.class),
+                Arguments.of(DType.I16.asNullable(), ShortArray.class),
+                Arguments.of(DType.F64.asNullable(), DoubleArray.class),
+                Arguments.of(DType.BOOL.asNullable(), BoolArray.class),
+                Arguments.of(DType.UTF8.asNullable(), VarBinArray.class));
+    }
+
     /// A scalar whose oneof tag doesn't match the declared Decimal dtype (e.g. only
     /// int64_value set, `bytes_value` absent) previously threw a raw NullPointerException
     /// reading `bytes_value().length` instead of a [VortexException] (ADR 0003).

writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java

@@ -1,5 +1,6 @@
 package io.github.dfa1.vortex.writer.encode;
 
+import io.github.dfa1.vortex.core.proto.ProtoNullValue;
 import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.error.VortexException;
@@ -50,6 +51,12 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
             throw new VortexException(EncodingId.VORTEX_MASKED,
                     "expected NullableData, got " + (data == null ? "null" : data.getClass().getName()));
         }
+        // Rust's compressor turns an all-invalid array into one null-scalar constant before trying
+        // any scheme (compressor/cascade.rs); masking a values child would store nothing useful
+        if (ctx.allowedCascading() > 0 && validity.length > 0 && !validity[0]
+                && SimdOperationsSupport.preferred().allEqual(validity)) {
+            return nullConstant();
+        }
         DType nonNullable = dtype.withNullable(false);
         EncodeResult valuesResult = encodeValues(nonNullable, values, ctx);
         EncodeResult validityResult = encodeValidity(validity, ctx);
@@ -164,6 +171,12 @@ static EncodeResult encodeValidity(boolean[] validity, EncodeContext ctx) {
         return best;
     }
 
+    /// A `vortex.constant` whose scalar is null: every row null, and no min/max.
+    private static EncodeResult nullConstant() {
+        byte[] scalar = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE).encode();
+        return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalar)));
+    }
+
     private static long totalBytes(EncodeResult result) {
         long total = 0;
         for (MemorySegment buf : result.buffers()) {

writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java

@@ -17,7 +17,7 @@
 import io.github.dfa1.vortex.reader.array.LazyConstantIntArray;
 import io.github.dfa1.vortex.reader.array.LazyConstantLongArray;
 import io.github.dfa1.vortex.reader.array.LongArray;
-import io.github.dfa1.vortex.reader.array.NullArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
 import io.github.dfa1.vortex.reader.array.ShortArray;
 import io.github.dfa1.vortex.core.testing.DTypes;
 import io.github.dfa1.vortex.reader.decode.DecodeContext;
@@ -467,7 +467,7 @@ private static ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException
     }
 
     /// Rust can write a constant array whose scalar is null (proto null_value tag).
-    /// The decoder must return a [NullArray] — not 0 / false (#246).
+    /// The decoder must return every row null — not 0 / false (#246).
     @Nested
     class NullScalar {
 
@@ -477,7 +477,7 @@ static Stream<DType> nullableDtypes() {
 
         @ParameterizedTest
         @MethodSource("nullableDtypes")
-        void decode_nullScalar_returnsNullArray(DType dtype) {
+        void decode_nullScalar_returnsEveryRowInvalid(DType dtype) {
             // Given — scalar proto with only null_value tag set (Rust-written null constant)
             ProtoScalarValue nullScalar = ProtoScalarValue.ofNullValue(ProtoNullValue.NULL_VALUE);
             EncodeResult encoded = EncodeResult.simple(
@@ -488,9 +488,12 @@ void decode_nullScalar_returnsNullArray(DType dtype) {
             DecodeContext ctx = DecodeTestHelper.toDecodeContext(encoded, rowCount, dtype, REGISTRY);
             Array result = DECODER.decode(ctx);
 
-            // Then — must be NullArray, not LazyConstant*(value=0) or LazyConstantBool(false)
-            assertThat(result).isInstanceOf(NullArray.class);
+            // Then — every row null, not LazyConstant*(value=0) or LazyConstantBool(false); masked
+            // like any nullable chunk, as Rust canonicalizes a null constant
+            assertThat(result).isInstanceOf(MaskedArray.class);
             assertThat(result.length()).isEqualTo(rowCount);
+            assertThat(((MaskedArray) result).isValid(0)).isFalse();
+            assertThat(((MaskedArray) result).isValid(rowCount - 1)).isFalse();
         }
     }