Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public boolean read(VectorSchemaRoot root) throws IOException {
JsonToken t = parser.nextToken();
if (t == START_OBJECT) {
{
int count = readNextField("count", Integer.class);
int count = readCount();
nextFieldIs("columns");
readToken(START_ARRAY);
{
Expand Down Expand Up @@ -224,7 +224,7 @@ public VectorSchemaRoot read() throws IOException {
if (t == START_OBJECT) {
VectorSchemaRoot recordBatch = VectorSchemaRoot.create(schema, allocator);
{
int count = readNextField("count", Integer.class);
int count = readCount();
recordBatch.setRowCount(count);
nextFieldIs("columns");
readToken(START_ARRAY);
Expand Down Expand Up @@ -885,7 +885,7 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws IOEx
/* Initialize the vector with required capacity but don't allocateNew since we would
* be doing loadFieldBuffers.
*/
int valueCount = readNextField("count", Integer.class);
int valueCount = readCount();

vector.setInitialCapacity(valueCount);

Expand Down Expand Up @@ -975,6 +975,14 @@ private <T> T readNextField(String expectedFieldName, Class<T> c)
return parser.readValueAs(c);
}

private int readCount() throws IOException {
int count = readNextField("count", Integer.class);
if (count < 0) {
throw new IllegalArgumentException("Expected count >= 0, got " + count);
}
return count;
}

private void nextFieldIs(String expectedFieldName) throws IOException, JsonParseException {
String name = parser.nextFieldName();
if (name == null || !name.equals(expectedFieldName)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@
*/
package org.apache.arrow.vector.ipc.message;

import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;

import com.google.flatbuffers.FlatBufferBuilder;
import org.apache.arrow.flatbuf.FieldNode;
import org.apache.arrow.util.Preconditions;

/** Metadata about Vectors/Arrays that is written to a channel. */
public class ArrowFieldNode implements FBSerializable {
Expand All @@ -31,12 +30,14 @@ public class ArrowFieldNode implements FBSerializable {
* Constructs a new instance.
*
* @param length The number of values written.
* @param nullCount The number of null values.
* @param nullCount The number of null values, or -1 if unknown.
*/
public ArrowFieldNode(long length, long nullCount) {
super();
this.length = checkedCastToInt(length);
this.nullCount = checkedCastToInt(nullCount);
Preconditions.checkArgument(length >= 0, "Expected length >= 0, got %s", length);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check runs on the long before it's narrowed, but the checkedCastToInt only uses Math.toIntExact when bounds checking is on.
With arrow.enable_unsafe_memory_access=true it's a plain (int) cast. A positive long whose low 32 bits are negative passes the check and is stored as a negative int, in the same mode this PR is supposed to protect.

IPC is safe because MessageSerializer.deserializeRecordBatch rejects values outside int range before it calls this. C Data is not: ArrayImporter passes snapshot.length, an int64 from the producer, straight in.
For example length = 4294967196 (2^32 - 100) passes the check and becomes -100.

Narrowing unconditionally should fixes it. Doing the same for nullCount still allows the -1 sentinel:

Suggested change
Preconditions.checkArgument(length >= 0, "Expected length >= 0, got %s", length);
Preconditions.checkArgument(length >= 0, "Expected length >= 0, got %s", length);
this.length = Math.toIntExact(length);
this.nullCount = Math.toIntExact(nullCount);

The static import of checkedCastToInt becomes unused after this.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the thread above (#1422 (comment)); addressed there.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check now throws inside MessageSerializer.deserializeRecordBatch(RecordBath, ArrowBuf), before that method releases body at the end. A negative node length now leaks the body buffer. Before this change the same input loaded without error and didn't leak.

I reproduced it with a stream whose FieldNode.length is patched to -100. ArrowStreamReader and ArrowFileReader both throw and leave 8192 bytes allocated, and allocator.close() fails with "Memory was leaked". Flight goes through the same call: ArrowMessage.asRecordBatch() retains underlying and relies on deserializeRecordBatch to release it.

Could we release body in a finally? ArrowRecordBatch retains the slices it keeps, so the release is correct on both paths:

  public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB, ArrowBuf body)
      throws IOException {
    try {
      // ... existing node/buffer/compression/variadic parsing, unchanged ...
      return new ArrowRecordBatch(
          checkedCastToInt(recordBatchFB.length()),
          nodes,
          buffers,
          bodyCompression,
          variadicBufferCounts,
          /*alignBuffers*/ true);
    } finally {
      body.getReferenceManager().release();
    }
  }

This also fixes the same leak on the existing IOException paths in that method (for example node length > INT_MAX), and covers the dictionary batch and Flight callers. ArrowMessage doesn't need a change. Releasing there too would release the buffer twice.

Preconditions.checkArgument(nullCount >= -1, "Expected nullCount >= -1, got %s", nullCount);
this.length = Math.toIntExact(length);
this.nullCount = Math.toIntExact(nullCount);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -48,6 +49,11 @@

public class TestJSONFile extends BaseFileTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TestJSONFile.class);
private static final String NEGATIVE_BATCH_COUNT_JSON =
"{\"schema\": {\"fields\": [{\"name\": \"i\", \"type\": {\"name\": \"int\","
+ " \"isSigned\": true, \"bitWidth\": 32}, \"nullable\": true, \"children\": []}]},"
+ " \"batches\": [{\"count\": -100, \"columns\": [{\"name\": \"i\", \"count\": 0,"
+ " \"VALIDITY\": [], \"DATA\": []}]}]}";

@Test
public void testNoBatches() throws IOException {
Expand Down Expand Up @@ -521,4 +527,47 @@ public void testRoundtripEmptyVector() throws Exception {
}
}
}

@Test
public void testNegativeColumnCountRejected() throws IOException {
File file =
writeJson(
"{\"schema\": {\"fields\": [{\"name\": \"s\", \"type\": {\"name\": \"struct\"},"
+ " \"nullable\": true, \"children\": []}]},"
+ " \"batches\": [{\"count\": 0, \"columns\": [{\"name\": \"s\", \"count\": -100,"
+ " \"VALIDITY\": []}]}]}");
// Small limit, so that sizing buffers from a negative count fails fast
try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, 1 << 20);
JsonFileReader reader = new JsonFileReader(file, readerAllocator)) {
reader.start();
assertThrows(IllegalArgumentException.class, () -> reader.read());
}
}

@Test
public void testNegativeBatchCountRejected() throws IOException {
File file = writeJson(NEGATIVE_BATCH_COUNT_JSON);
try (JsonFileReader reader = new JsonFileReader(file, allocator)) {
reader.start();
assertThrows(IllegalArgumentException.class, () -> reader.read());
}
}

@Test
public void testNegativeBatchCountRejectedWhenReadingIntoRoot() throws IOException {
File file = writeJson(NEGATIVE_BATCH_COUNT_JSON);
try (JsonFileReader reader = new JsonFileReader(file, allocator)) {
Schema schema = reader.start();
try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
assertThrows(IllegalArgumentException.class, () -> reader.read(root));
}
}
}

private static File writeJson(String json) throws IOException {
Path path = Files.createTempFile("arrow-", ".json");
path.toFile().deleteOnExit();
Files.writeString(path, json);
return path.toFile();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.arrow.vector.ipc.message;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;

import org.junit.jupiter.api.Test;

public class TestArrowFieldNode {

@Test
public void testNegativeLengthRejected() {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These tests cover the constructor, but not the deserialization path the PR is about. Could we add a test that deserializes a record batch with a negative node length and checks that nothing leaks? For example, in MessageSerializerTest:

  @Test
  public void testNegativeNodeLengthDoesNotLeakBody() {
    FlatBufferBuilder builder = new FlatBufferBuilder();
    RecordBatch.startNodesVector(builder, 1);
    FieldNode.createFieldNode(builder, -100, 0);
    int nodes = builder.endVector();
    RecordBatch.startBuffersVector(builder, 0);
    int buffers = builder.endVector();
    RecordBatch.startRecordBatch(builder);
    RecordBatch.addLength(builder, 0);
    RecordBatch.addNodes(builder, nodes);
    RecordBatch.addBuffers(builder, buffers);
    builder.finish(RecordBatch.endRecordBatch(builder));
    RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(builder.dataBuffer());

    try (BufferAllocator allocator = new RootAllocator()) {
      ArrowBuf body = allocator.buffer(8);
      assertThrows(
          IllegalArgumentException.class,
          () -> MessageSerializer.deserializeRecordBatch(recordBatchFB, body));
      assertEquals(0, allocator.getAllocatedMemory());
    }
  }

It needs three imports: com.google.flatbuffers.FlatBufferBuilder, org.apache.arrow.flatbuf.FieldNode and org.apache.arrow.flatbuf.RecordBatch.

IllegalArgumentException e =
assertThrows(IllegalArgumentException.class, () -> new ArrowFieldNode(-100, 0));
assertEquals("Expected length >= 0, got -100", e.getMessage());
}

@Test
public void testLargeNegativeLengthRejected() {
assertThrows(IllegalArgumentException.class, () -> new ArrowFieldNode(-5_000_000_000L, 0));
}

@Test
public void testLengthAboveIntRangeRejected() {
assertThrows(ArithmeticException.class, () -> new ArrowFieldNode((1L << 32) - 100, 0));
}

@Test
public void testZeroAndPositiveLengthAccepted() {
assertEquals(0, new ArrowFieldNode(0, 0).getLength());
assertEquals(42, new ArrowFieldNode(42, 0).getLength());
}

@Test
public void testNegativeNullCountSentinelPreserved() {
ArrowFieldNode node = new ArrowFieldNode(10, -1);
assertEquals(10, node.getLength());
assertEquals(-1, node.getNullCount());
}

@Test
public void testNullCountBelowSentinelRejected() {
assertThrows(IllegalArgumentException.class, () -> new ArrowFieldNode(10, -2));
}

@Test
public void testNullCountAboveIntRangeRejected() {
assertThrows(ArithmeticException.class, () -> new ArrowFieldNode(10, (1L << 32) - 2));
}
}
Loading