Skip to content

fix(table): reuse partition field IDs on v3 tables #4110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinjqliu merged 3 commits into from
Oct 24, 2026
+25 −1
Merged

fix(table): reuse partition field IDs on v3 tables #4110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
32a3b4e
fix(table): reuse partition field IDs on v3 tables
porkedchop Oct 15, 2026
9ba780e
Merge remote-tracking branch 'origin/main' into fix/v3-partition-fiel…
porkedchop Oct 18, 2026
2b28cad
test: cover removing and re-adding a partition field in one update
kevinjqliu Oct 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyiceberg/table/update/spec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ def _add_new_field(
return PartitionSpec(*partition_fields, spec_id=new_spec_id)

def _partition_field(self, transform_key: tuple[int, Transform[Any, Any]], name: str | None) -> PartitionField:
if self._transaction.table_metadata.format_version == 2:
if self._transaction.table_metadata.format_version >= 2:

@kevinjqliu kevinjqliu Oct 24, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like we need to follow up and fix a number of these in the code base

source_id, transform = transform_key
historical_fields = []
for spec in self._transaction.table_metadata.specs().values():
Expand Down
24 changes: 24 additions & 0 deletions tests/table/test_init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1454,6 +1454,30 @@ def test_last_assigned_partition_id(table_v2: Table) -> None:
AssertLastAssignedPartitionId(last_assigned_partition_id=1).validate(base_metadata)


@pytest.mark.parametrize("table", [lf("table_v2"), lf("table_v3")])
def test_update_spec_reuses_removed_partition_field_id(table: Table) -> None:
txn = table.transaction()
txn.update_spec().remove_field("x").add_field("y", BucketTransform(16), "y_bucket").commit()
txn.update_spec().add_identity("x").commit()

assert txn.table_metadata.spec().fields == (
PartitionField(source_id=2, field_id=1001, transform=BucketTransform(16), name="y_bucket"),
PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),
)
assert txn.table_metadata.last_partition_id == 1001


@pytest.mark.parametrize("table", [lf("table_v2"), lf("table_v3")])
def test_update_spec_remove_and_add_same_field_reuses_id(table: Table) -> None:
txn = table.transaction()
txn.update_spec().remove_field("x").add_identity("x").commit()

assert txn.table_metadata.spec().fields == (
PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),
)
assert txn.table_metadata.last_partition_id == 1000


def test_assert_default_spec_id(table_v2: Table) -> None:
base_metadata = table_v2.metadata
AssertDefaultSpecId(default_spec_id=base_metadata.default_spec_id).validate(base_metadata)
Expand Down
Loading