From 58ce1d84f503959f6ac7b7a98cd853e1a1a6d410 Mon Sep 17 00:00:00 2001 From: Jathavedhan M Date: Mon, 26 Oct 2026 12:03:50 +0530 Subject: [PATCH] Handle pod logs cut mid-character by limitBytes read_namespaced_pod_log(limit_bytes=N) returns the first N bytes of the log, and the API server does not align that cut to a character boundary. When it falls inside a multi-byte UTF-8 character, response_deserialize() raised UnicodeDecodeError and the caller lost the whole log. When the strict decode fails, decode again with an incremental decoder and final=False. That leaves out only an incomplete sequence at the very end and still raises for any other invalid byte, so valid responses and genuinely bad data behave as before. The sync and asyncio api_client.py files are generated, so the change is also kept in scripts/api_client_patch.diff and scripts/api_client_asyncio_patch.diff, which the update scripts apply. Signed-off-by: Jathavedhan M --- kubernetes/aio/client/api_client.py | 10 ++++++++- kubernetes/aio/test/test_generated_api.py | 22 ++++++++++++++++++ kubernetes/client/api_client.py | 10 ++++++++- kubernetes/test/test_api_client.py | 25 +++++++++++++++++++++ scripts/api_client_asyncio_patch.diff | 27 +++++++++++++++++++++++ scripts/api_client_patch.diff | 27 +++++++++++++++++++++++ scripts/update-client-asyncio.sh | 3 +++ scripts/update-client.sh | 3 +++ 8 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 scripts/api_client_asyncio_patch.diff create mode 100644 scripts/api_client_patch.diff diff --git a/kubernetes/aio/client/api_client.py b/kubernetes/aio/client/api_client.py index 3595abd6bb..20edcd3639 100644 --- a/kubernetes/aio/client/api_client.py +++ b/kubernetes/aio/client/api_client.py @@ -11,6 +11,7 @@ +import codecs import datetime from dateutil.parser import parse from enum import Enum @@ -353,7 +354,14 @@ def response_deserialize( if content_type is not None: match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) encoding = match.group(1) if match else "utf-8" - response_text = response_data.data.decode(encoding) + try: + response_text = response_data.data.decode(encoding) + except UnicodeDecodeError: + # limitBytes on pod logs cuts at a byte boundary and can split a + # multi-byte character. final=False drops only that incomplete + # tail; any other invalid byte still raises. + decoder = codecs.getincrementaldecoder(encoding)() + response_text = decoder.decode(response_data.data, final=False) return_data = self.deserialize(response_text, response_type, content_type) finally: if not 200 <= response_data.status <= 299: diff --git a/kubernetes/aio/test/test_generated_api.py b/kubernetes/aio/test/test_generated_api.py index 374267b208..f42316c1dd 100644 --- a/kubernetes/aio/test/test_generated_api.py +++ b/kubernetes/aio/test/test_generated_api.py @@ -48,6 +48,7 @@ async def asyncSetUp(self): self.response_status = 200 self.response_headers = {} self.responses = None + self.log_body = b'' app = web.Application() app.router.add_route('*', '/{path:.*}', self._handle_request) self.runner = web.AppRunner(app) @@ -93,6 +94,8 @@ async def _handle_request(self, request): }).encode() + b'\n') await response.write_eof() return response + if request.path.endswith('/log'): + return web.Response(body=self.log_body, content_type='text/plain') if self.responses is not None: index = len(self.requests) - 1 response, status, headers = self.responses[ @@ -371,3 +374,22 @@ async def test_models_preserve_int_or_string_and_python_field_names(self): 'resource_version', inspect.signature(V1ObjectMeta).parameters, ) + + async def test_pod_log_cut_mid_character_by_limit_bytes(self): + # The last two bytes are the start of a three-byte euro sign. + self.log_body = 'caf\u00e9 \u20ac'.encode()[:-1] + + log = await CoreV1Api(self.api_client).read_namespaced_pod_log( + 'pod', 'default', limit_bytes=len(self.log_body), + ) + + self.assertEqual('caf\u00e9 ', log) + + async def test_pod_log_with_invalid_utf8_still_raises(self): + # A Latin-1 e-acute is not valid UTF-8 wherever it appears. + self.log_body = b'caf\xe9 \xe2\x82' + + with self.assertRaises(UnicodeDecodeError): + await CoreV1Api(self.api_client).read_namespaced_pod_log( + 'pod', 'default', limit_bytes=len(self.log_body), + ) diff --git a/kubernetes/client/api_client.py b/kubernetes/client/api_client.py index 40eaa6ea65..5a53db44d2 100644 --- a/kubernetes/client/api_client.py +++ b/kubernetes/client/api_client.py @@ -11,6 +11,7 @@ +import codecs import datetime from dateutil.parser import parse from enum import Enum @@ -425,7 +426,14 @@ def response_deserialize( if content_type is not None: match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) encoding = match.group(1) if match else "utf-8" - response_text = response_data.data.decode(encoding) + try: + response_text = response_data.data.decode(encoding) + except UnicodeDecodeError: + # limitBytes on pod logs cuts at a byte boundary and can split a + # multi-byte character. final=False drops only that incomplete + # tail; any other invalid byte still raises. + decoder = codecs.getincrementaldecoder(encoding)() + response_text = decoder.decode(response_data.data, final=False) return_data = self.deserialize(response_text, response_type, content_type) finally: if not 200 <= response_data.status <= 299: diff --git a/kubernetes/test/test_api_client.py b/kubernetes/test/test_api_client.py index 91cbe45bbc..b5eec30f7a 100644 --- a/kubernetes/test/test_api_client.py +++ b/kubernetes/test/test_api_client.py @@ -68,6 +68,31 @@ def test_rest_proxycare(self): client = kubernetes.client.ApiClient(configuration=config) self.assertEqual( expected_pool, type(client.rest_client.pool_manager) ) + def _read_pod_log(self, body): + with kubernetes.client.ApiClient() as client: + response = urllib3.HTTPResponse( + body=body, status=200, reason='OK', + headers={'Content-Type': 'text/plain'}, + ) + with mock.patch.object( + client.rest_client.pool_manager, 'request', + return_value=response, + ): + return kubernetes.client.CoreV1Api(client).read_namespaced_pod_log( + 'pod', 'default', limit_bytes=len(body), + ) + + def test_pod_log_cut_mid_character_by_limit_bytes(self): + # The last two bytes are the start of a three-byte euro sign. + log = self._read_pod_log('caf\u00e9 \u20ac'.encode()[:-1]) + + self.assertEqual('caf\u00e9 ', log) + + def test_pod_log_with_invalid_utf8_still_raises(self): + # A Latin-1 e-acute is not valid UTF-8 wherever it appears. + with self.assertRaises(UnicodeDecodeError): + self._read_pod_log(b'caf\xe9 \xe2\x82') + def test_client_go_read_retries_do_not_override_write_retries(self): config = Configuration(proxy='', no_proxy='', retries=urllib3.Retry(total=2)) config.client_go_retries = True diff --git a/scripts/api_client_asyncio_patch.diff b/scripts/api_client_asyncio_patch.diff new file mode 100644 index 0000000000..90a1582906 --- /dev/null +++ b/scripts/api_client_asyncio_patch.diff @@ -0,0 +1,27 @@ +diff --git a/kubernetes/aio/client/api_client.py b/kubernetes/aio/client/api_client.py +--- a/kubernetes/aio/client/api_client.py ++++ b/kubernetes/aio/client/api_client.py +@@ -11,6 +11,7 @@ + + + ++import codecs + import datetime + from dateutil.parser import parse + from enum import Enum +@@ -353,7 +354,14 @@ class ApiClient: + if content_type is not None: + match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) + encoding = match.group(1) if match else "utf-8" +- response_text = response_data.data.decode(encoding) ++ try: ++ response_text = response_data.data.decode(encoding) ++ except UnicodeDecodeError: ++ # limitBytes on pod logs cuts at a byte boundary and can split a ++ # multi-byte character. final=False drops only that incomplete ++ # tail; any other invalid byte still raises. ++ decoder = codecs.getincrementaldecoder(encoding)() ++ response_text = decoder.decode(response_data.data, final=False) + return_data = self.deserialize(response_text, response_type, content_type) + finally: + if not 200 <= response_data.status <= 299: diff --git a/scripts/api_client_patch.diff b/scripts/api_client_patch.diff new file mode 100644 index 0000000000..9d37498e66 --- /dev/null +++ b/scripts/api_client_patch.diff @@ -0,0 +1,27 @@ +diff --git a/kubernetes/client/api_client.py b/kubernetes/client/api_client.py +--- a/kubernetes/client/api_client.py ++++ b/kubernetes/client/api_client.py +@@ -11,6 +11,7 @@ + + + ++import codecs + import datetime + from dateutil.parser import parse + from enum import Enum +@@ -425,7 +426,14 @@ class ApiClient: + if content_type is not None: + match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) + encoding = match.group(1) if match else "utf-8" +- response_text = response_data.data.decode(encoding) ++ try: ++ response_text = response_data.data.decode(encoding) ++ except UnicodeDecodeError: ++ # limitBytes on pod logs cuts at a byte boundary and can split a ++ # multi-byte character. final=False drops only that incomplete ++ # tail; any other invalid byte still raises. ++ decoder = codecs.getincrementaldecoder(encoding)() ++ response_text = decoder.decode(response_data.data, final=False) + return_data = self.deserialize(response_text, response_type, content_type) + finally: + if not 200 <= response_data.status <= 299: diff --git a/scripts/update-client-asyncio.sh b/scripts/update-client-asyncio.sh index 2de63e9c8e..b00cd4b6e5 100755 --- a/scripts/update-client-asyncio.sh +++ b/scripts/update-client-asyncio.sh @@ -61,6 +61,9 @@ mv "${CLIENT_ROOT}/swagger.json" "${SCRIPT_ROOT}/swagger.json" echo ">>> restoring Kubernetes client-go retry integration..." git apply "${SCRIPT_ROOT}/client_go_retry_asyncio_patch.diff" +echo ">>> restoring truncated UTF-8 response decoding..." +git apply "${SCRIPT_ROOT}/api_client_asyncio_patch.diff" + echo ">>> updating version information..." sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup-asyncio.py" sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py" diff --git a/scripts/update-client.sh b/scripts/update-client.sh index 102db9a1b7..1cdfe4ca1e 100755 --- a/scripts/update-client.sh +++ b/scripts/update-client.sh @@ -68,6 +68,9 @@ git apply "${SCRIPT_ROOT}/client_go_retry_patch.diff" echo ">>> restoring Kubernetes TCP keepalive option..." git apply "${SCRIPT_ROOT}/keepalive_patch.diff" +echo ">>> restoring truncated UTF-8 response decoding..." +git apply "${SCRIPT_ROOT}/api_client_patch.diff" + echo ">>> updating version information..." sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup.py" sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py"