Skip to content

GH-3900: Replace Guava with JDK APIs in parquet-cli #3906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Fokko merged 1 commit into from
Oct 29, 2026
Merged

GH-3900: Replace Guava with JDK APIs in parquet-cli #3906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion parquet-cli/src/main/java/org/apache/parquet/cli/BaseCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import com.beust.jcommander.internal.Lists;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.io.CharStreams;
import com.google.common.io.Resources;
import java.io.File;
Expand Down Expand Up @@ -51,6 +50,7 @@
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.HadoopReadOptions;
import org.apache.parquet.Preconditions;
import org.apache.parquet.avro.AvroParquetReader;
import org.apache.parquet.avro.AvroReadSupport;
import org.apache.parquet.cli.json.AvroJsonReader;
Expand Down
4 changes: 2 additions & 2 deletions parquet-cli/src/main/java/org/apache/parquet/cli/Help.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterDescription;
import com.beust.jcommander.Parameters;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand All @@ -33,7 +33,7 @@
@Parameters(commandDescription = "Retrieves details on the functions of other commands")
public class Help implements Command {
@Parameter(description = "<commands>")
List<String> helpCommands = Lists.newArrayList();
List<String> helpCommands = new ArrayList<>();

private final JCommander jc;
private final Logger console;
Expand Down
3 changes: 1 addition & 2 deletions parquet-cli/src/main/java/org/apache/parquet/cli/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.List;
Expand Down Expand Up @@ -89,7 +88,7 @@ public class Main extends Configured implements Tool {
@VisibleForTesting
static final String DEFAULT_PROGRAM_NAME = "parquet";

private static Set<String> HELP_ARGS = ImmutableSet.of("-h", "-help", "--help", "help");
private static Set<String> HELP_ARGS = Set.of("-h", "-help", "--help", "help");

private final Logger console;
private final Help help;
Expand Down
13 changes: 4 additions & 9 deletions parquet-cli/src/main/java/org/apache/parquet/cli/Util.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,9 @@
import static org.apache.parquet.format.Encoding.DELTA_LENGTH_BYTE_ARRAY;

import com.google.common.base.Ascii;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.hash.HashCode;
import java.util.Set;
import org.apache.parquet.Preconditions;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.EncodingStats;
Expand Down Expand Up @@ -240,16 +237,14 @@ public static String encodingsAsString(Set<Encoding> encodings, ColumnDescriptor
return sb.toString();
}

private static final Splitter DOT = Splitter.on('.');

public static ColumnDescriptor descriptor(String column, MessageType schema) {
String[] path = Iterables.toArray(DOT.split(column), String.class);
String[] path = column.split("\\.", -1);
Preconditions.checkArgument(schema.containsPath(path), "Schema doesn't have column: %s", column);
return schema.getColumnDescription(path);
}

public static String columnName(ColumnDescriptor desc) {
return Joiner.on('.').join(desc.getPath());
return String.join(".", desc.getPath());
}

public static PrimitiveType primitive(MessageType schema, String[] path) {
Expand All @@ -264,7 +259,7 @@ public static PrimitiveType primitive(MessageType schema, String[] path) {
}

public static PrimitiveType primitive(String column, MessageType schema) {
String[] path = Iterables.toArray(DOT.split(column), String.class);
String[] path = column.split("\\.", -1);
Preconditions.checkArgument(schema.containsPath(path), "Schema doesn't have column: %s", column);
return primitive(schema, path);
}
Expand Down
7 changes: 3 additions & 4 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/CSVSchemaCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,11 @@
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.internal.Lists;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Set;
import org.apache.parquet.Preconditions;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.cli.csv.AvroCSV;
import org.apache.parquet.cli.csv.CSVProperties;
Expand Down Expand Up @@ -101,9 +100,9 @@ public int run() throws IOException {
.charset(charsetName)
.build();

Set<String> required = ImmutableSet.of();
Set<String> required = Set.of();
if (requiredFields != null) {
required = ImmutableSet.copyOf(requiredFields);
required = Set.copyOf(requiredFields);
}

// assume fields are nullable by default, users can easily change this
Expand Down
6 changes: 3 additions & 3 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/CatCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.avro.Schema;
import org.apache.avro.SchemaParseException;
import org.apache.parquet.Preconditions;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.cli.util.Expressions;
import org.apache.parquet.example.data.Group;
Expand Down Expand Up @@ -150,7 +150,7 @@ private void runWithGroupReader(String source) throws IOException {

@Override
public List<String> getExamples() {
return Lists.newArrayList(
return Arrays.asList(
"# Show the first 10 records in file \"data.avro\":",
"data.avro",
"# Show the first 50 records in file \"data.parquet\":",
Expand Down
15 changes: 5 additions & 10 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/CheckParquet251Command.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,10 @@
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.internal.Lists;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.CorruptStatistics;
import org.apache.parquet.Version;
Expand Down Expand Up @@ -97,13 +94,11 @@ private String check(String file) throws IOException {

// get just the binary columns
List<ColumnDescriptor> columns = Lists.newArrayList();
Iterables.addAll(
columns, Iterables.filter(meta.getSchema().getColumns(), new Predicate<ColumnDescriptor>() {
@Override
public boolean apply(@Nullable ColumnDescriptor input) {
return input != null && input.getType() == BINARY;
}
}));
for (ColumnDescriptor input : meta.getSchema().getColumns()) {
if (input != null && input.getType() == BINARY) {
columns.add(input);
}
}

// now check to see if the data is actually corrupt
try (ParquetFileReader reader =
Expand Down
6 changes: 3 additions & 3 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/ColumnMaskingCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.HadoopReadOptions;
import org.apache.parquet.Preconditions;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetFileWriter;
Expand Down Expand Up @@ -109,7 +109,7 @@ public int run() throws IOException {

@Override
public List<String> getExamples() {
return Lists.newArrayList(
return Arrays.asList(
"# Replace columns with masked values and write to a new Parquet file",
"-m nullify -i input.parquet -o output.parquet -c col1_name");
}
Expand Down
6 changes: 3 additions & 3 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/ColumnSizeCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,17 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.Preconditions;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileReader;
Expand Down Expand Up @@ -143,7 +143,7 @@ private String formatRatio(float ratio) {

@Override
public List<String> getExamples() {
return Lists.newArrayList(
return Arrays.asList(
"# Print every column size in byte and ratio for a Parquet file",
"sample.parquet",
"sample.parquet -c col_1",
Expand Down
11 changes: 5 additions & 6 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/ConvertCSVCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,16 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import org.apache.avro.Schema;
import org.apache.avro.SchemaNormalization;
import org.apache.avro.generic.GenericData;
import org.apache.parquet.Preconditions;
import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.cli.csv.AvroCSV;
Expand Down Expand Up @@ -141,9 +140,9 @@ public int run() throws IOException {
if (avroSchemaFile != null) {
csvSchema = Schemas.fromAvsc(open(avroSchemaFile));
} else {
Set<String> required = ImmutableSet.of();
Set<String> required = Set.of();
if (requiredFields != null) {
required = ImmutableSet.copyOf(requiredFields);
required = Set.copyOf(requiredFields);
}

String filename = new File(targets.get(0)).getName();
Expand Down Expand Up @@ -199,7 +198,7 @@ public int run() throws IOException {

@Override
public List<String> getExamples() {
return Lists.newArrayList(
return Arrays.asList(
"# Create a Parquet file from a CSV file",
"sample.csv -o sample.parquet --schema schema.avsc",
"# Create a Parquet file in HDFS from local CSV",
Expand Down
6 changes: 3 additions & 3 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/ConvertCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,16 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.Preconditions;
import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.cli.util.Codecs;
Expand Down Expand Up @@ -153,7 +153,7 @@ public int run() throws IOException {

@Override
public List<String> getExamples() {
return Lists.newArrayList(
return Arrays.asList(
"# Create a Parquet file from an Avro file",
"sample.avro -o sample.parquet",
"# Create a Parquet file in S3 from a local Avro file",
Expand Down
23 changes: 10 additions & 13 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/ParquetMetadataCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,14 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.commons.text.TextStringBuilder;
import org.apache.parquet.Preconditions;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Encoding;
Expand Down Expand Up @@ -102,7 +100,7 @@ public int run() throws IOException {

@Override
public List<String> getExamples() {
return Lists.newArrayList();
return new ArrayList<>();
}

private int maxSize(Iterable<String> strings) {
Expand Down Expand Up @@ -131,12 +129,11 @@ private void printRowGroup(Logger console, int index, BlockMetaData rowGroup, Me
filePath != null ? "path: " + filePath : "",
new TextStringBuilder(80).appendPadding(80, '-')));

int size = maxSize(Iterables.transform(rowGroup.getColumns(), new Function<ColumnChunkMetaData, String>() {
@Override
public String apply(@Nullable ColumnChunkMetaData input) {
return input == null ? "" : input.getPath().toDotString();
}
}));
List<String> columnNames = new ArrayList<>();
for (ColumnChunkMetaData input : rowGroup.getColumns()) {
columnNames.add(input == null ? "" : input.getPath().toDotString());
}
int size = maxSize(columnNames);

console.info(String.format(
"%-" + size + "s %-9s %-9s %-9s %-10s %-7s %s",
Expand All @@ -155,7 +152,7 @@ public String apply(@Nullable ColumnChunkMetaData input) {
private void printColumnChunk(Logger console, int width, ColumnChunkMetaData column, MessageType schema) {
String[] path = column.getPath().toArray();
PrimitiveType type = primitive(schema, path);
Preconditions.checkNotNull(type);
Objects.requireNonNull(type);

ColumnDescriptor desc = schema.getColumnDescription(path);
long size = column.getTotalSize();
Expand Down
6 changes: 3 additions & 3 deletions parquet-cli/src/main/java/org/apache/parquet/cli/commands/PruneColumnsCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.Preconditions;
import org.apache.parquet.cli.BaseCommand;
import org.apache.parquet.hadoop.util.ColumnPruner;
import org.slf4j.Logger;
Expand Down Expand Up @@ -74,7 +74,7 @@ public int run() throws IOException {

@Override
public List<String> getExamples() {
return Lists.newArrayList(
return Arrays.asList(
"# Removes specified columns and write to a new Parquet file",
"-i input.parquet -o output.parquet -c col1_name");
}
Expand Down
Loading
Loading