Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions pyiceberg/catalog/rest/__init__.py

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The bug was using the wrong helper in the request body. The body should not be escaped, the URL should be. We should probably do some refactoring here (esp. with the naming).

Helper Escaped? Returns for ("ns", "my view") Use for
_identifier_to_validated_tuple(identifier) No ("ns", "my view") JSON body, error messages. Raises NoSuchIdentifierError if there's no namespace
_split_identifier_for_path(identifier, kind) Yes {"namespace": "ns", "view": "my%20view"} URL path only

Original file line number Diff line number Diff line change
Expand Up @@ -1312,7 +1312,7 @@ def create_view(
location = location.rstrip("/")

request = CreateViewRequest(
name=namespace_and_view["view"],
name=self._identifier_to_validated_tuple(identifier)[-1],

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use _identifier_to_validated_tuple for request bodies so names aren't escaped, and escape only in URL paths.

mirrors CreateTableRequest

request = CreateTableRequest(
name=self._identifier_to_validated_tuple(identifier)[-1],
location=location,
table_schema=fresh_schema,
partition_spec=fresh_partition_spec,
write_order=fresh_sort_order,
stage_create=stage_create,
properties=properties,
)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Idea (not necessarily for this PR): what if Identifier still acted like a tuple but had named methods on it as well? I really don't like the -1 here, since it's not always intuitive why we're doing that.

Could you add a comment explaining what you're trying to do?

location=location,
view_schema=fresh_schema,
view_version=view_version,
Expand Down Expand Up @@ -1826,15 +1826,14 @@ def view_exists(self, identifier: str | Identifier) -> bool:
def register_view(self, identifier: str | Identifier, metadata_location: str) -> View:
self._check_endpoint(Capability.V1_REGISTER_VIEW)
namespace_and_view = self._split_identifier_for_path(identifier, IdentifierKind.VIEW)
namespace = namespace_and_view["namespace"]
view = namespace_and_view["view"]
identifier_tuple = self._identifier_to_validated_tuple(identifier)
if self.table_exists(identifier):
raise TableAlreadyExistsError(f"Table {namespace}.{view} already exists")
raise TableAlreadyExistsError(f"Table {'.'.join(identifier_tuple)} already exists")

request = RegisterViewRequest(name=view, metadata_location=metadata_location)
request = RegisterViewRequest(name=identifier_tuple[-1], metadata_location=metadata_location)
serialized_json = request.model_dump_json().encode(UTF8)
response = self._session.post(
self.url(Endpoints.register_view, namespace=namespace),
self.url(Endpoints.register_view, namespace=namespace_and_view["namespace"]),
Comment on lines +1829 to +1836

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

follows register_table

namespace_and_table = self._split_identifier_for_path(identifier)
request = RegisterTableRequest(
name=self._identifier_to_validated_tuple(identifier)[-1],
metadata_location=metadata_location,
overwrite=overwrite,
)
serialized_json = request.model_dump_json().encode(UTF8)
response = self._session.post(
self.url(Endpoints.register_table, namespace=namespace_and_table["namespace"]),

data=serialized_json,
)
try:
Expand Down
41 changes: 40 additions & 1 deletion tests/catalog/test_rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from contextlib import contextmanager
from typing import Any, cast
from unittest import mock
from urllib.parse import parse_qs, urlsplit
from urllib.parse import parse_qs, quote, urlsplit

import pytest
from requests import Request
Expand Down Expand Up @@ -1805,6 +1805,27 @@ def test_create_view_200(rest_mock: Mocker, table_schema_simple: Schema, example
assert actual.config == example_view_metadata_rest_json["config"]


def test_create_view_name_is_not_encoded_in_body(
rest_mock: Mocker, table_schema_simple: Schema, example_view_metadata_rest_json: dict[str, Any]
) -> None:
rest_mock.post(
f"{TEST_URI}v1/namespaces/fokko/views",
json=example_view_metadata_rest_json,
status_code=200,
request_headers=TEST_HEADERS,
)
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN)
view_version = ViewVersion(
schema_id=1,
summary={"engine-name": "spark", "engineVersion": "3.3"},
representations=[],
default_namespace=["default"],
)
for view_name in ["my view", "dept/view", "100%", "\u65e5\u672c"]:
catalog.create_view(identifier=("fokko", view_name), schema=table_schema_simple, view_version=view_version)
assert rest_mock.last_request.json()["name"] == view_name


def test_create_view_409(
rest_mock: Mocker,
table_schema_simple: Schema,
Expand Down Expand Up @@ -2817,6 +2838,24 @@ def test_register_view_200(rest_mock: Mocker, example_view_metadata_rest_json: d
assert actual.config == example_view_metadata_rest_json["config"]


def test_register_view_name_is_not_encoded_in_body(rest_mock: Mocker, example_view_metadata_rest_json: dict[str, Any]) -> None:
rest_mock.post(
f"{TEST_URI}v1/namespaces/default/register-view",
json=example_view_metadata_rest_json,
status_code=200,
request_headers=TEST_HEADERS,
)
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN)
for view_name in ["my view", "dept/view", "100%", "\u65e5\u672c"]:
rest_mock.head(
f"{TEST_URI}v1/namespaces/default/tables/{quote(view_name, safe='')}",
status_code=404,
request_headers=TEST_HEADERS,
)
catalog.register_view(identifier=("default", view_name), metadata_location="s3://warehouse/database/view/metadata.json")
assert rest_mock.last_request.json()["name"] == view_name


def test_register_view_409_view(rest_mock: Mocker) -> None:
rest_mock.head(
f"{TEST_URI}v1/namespaces/default/tables/registered_view",
Expand Down
47 changes: 47 additions & 0 deletions tests/integration/test_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,53 @@ def test_rest_drop_view(
assert not rest_catalog.view_exists(identifier)


@pytest.mark.integration
def test_rest_identifiers_with_special_characters(

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

integration test that checks special character handling on relevant endpoints

rest_catalog: RestCatalog, table_schema_simple: Schema, example_view_metadata_v1: dict[str, Any]
) -> None:
"""Names that need URL encoding must round-trip unchanged through path, query, and body parameters."""
# "%" is omitted because the REST fixture's Jetty rejects "%25" in URL paths ("400 Ambiguous URI path encoding")
for name in ["with space", "a+b&c=d", "\u65e5\u672c"]:
parent = (f"{name}_{uuid.uuid4().hex}",)
namespace = (*parent, name)
table_identifier = (*namespace, f"table {name}")
view_identifier = (*namespace, f"view {name}")
registered_view_identifier = (*namespace, f"registered view {name}")

rest_catalog.create_namespace(parent)
rest_catalog.create_namespace(namespace, properties={"key": name})
assert parent in rest_catalog.list_namespaces()
assert rest_catalog.list_namespaces(parent) == [namespace]
assert rest_catalog.namespace_exists(namespace)
assert rest_catalog.load_namespace_properties(namespace)["key"] == name

rest_catalog.create_table(table_identifier, table_schema_simple)
assert rest_catalog.table_exists(table_identifier)
assert rest_catalog.list_tables(namespace) == [table_identifier]
table = rest_catalog.load_table(table_identifier)
assert table.name() == table_identifier
with table.transaction() as tx:
tx.set_properties(key=name)
assert rest_catalog.load_table(table_identifier).properties["key"] == name

view = View(view_identifier, ViewMetadata.model_validate(example_view_metadata_v1))
created_view = rest_catalog.create_view(view_identifier, view.schema(), view.current_version())
assert rest_catalog.view_exists(view_identifier)
assert rest_catalog.list_views(namespace) == [view_identifier]

assert created_view.metadata_location is not None
rest_catalog.register_view(registered_view_identifier, created_view.metadata_location)
assert rest_catalog.view_exists(registered_view_identifier)
assert sorted(rest_catalog.list_views(namespace)) == sorted([view_identifier, registered_view_identifier])

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

two views here because one is from create_view, another from register_review


rest_catalog.drop_view(registered_view_identifier)
rest_catalog.drop_view(view_identifier)
rest_catalog.drop_table(table_identifier)
rest_catalog.drop_namespace(namespace)
rest_catalog.drop_namespace(parent)
assert not rest_catalog.namespace_exists(parent)


@pytest.mark.integration
def test_rest_custom_namespace_separator(rest_catalog: RestCatalog, table_schema_simple: Schema) -> None:
"""
Expand Down
Loading