diff --git a/kubernetes/aio/client/api_client.py b/kubernetes/aio/client/api_client.py index 3595abd6bb..20edcd3639 100644 --- a/kubernetes/aio/client/api_client.py +++ b/kubernetes/aio/client/api_client.py @@ -11,6 +11,7 @@ +import codecs import datetime from dateutil.parser import parse from enum import Enum @@ -353,7 +354,14 @@ def response_deserialize( if content_type is not None: match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) encoding = match.group(1) if match else "utf-8" - response_text = response_data.data.decode(encoding) + try: + response_text = response_data.data.decode(encoding) + except UnicodeDecodeError: + # limitBytes on pod logs cuts at a byte boundary and can split a + # multi-byte character. final=False drops only that incomplete + # tail; any other invalid byte still raises. + decoder = codecs.getincrementaldecoder(encoding)() + response_text = decoder.decode(response_data.data, final=False) return_data = self.deserialize(response_text, response_type, content_type) finally: if not 200 <= response_data.status <= 299: diff --git a/kubernetes/aio/test/test_generated_api.py b/kubernetes/aio/test/test_generated_api.py index 374267b208..f42316c1dd 100644 --- a/kubernetes/aio/test/test_generated_api.py +++ b/kubernetes/aio/test/test_generated_api.py @@ -48,6 +48,7 @@ async def asyncSetUp(self): self.response_status = 200 self.response_headers = {} self.responses = None + self.log_body = b'' app = web.Application() app.router.add_route('*', '/{path:.*}', self._handle_request) self.runner = web.AppRunner(app) @@ -93,6 +94,8 @@ async def _handle_request(self, request): }).encode() + b'\n') await response.write_eof() return response + if request.path.endswith('/log'): + return web.Response(body=self.log_body, content_type='text/plain') if self.responses is not None: index = len(self.requests) - 1 response, status, headers = self.responses[ @@ -371,3 +374,22 @@ async def test_models_preserve_int_or_string_and_python_field_names(self): 'resource_version', inspect.signature(V1ObjectMeta).parameters, ) + + async def test_pod_log_cut_mid_character_by_limit_bytes(self): + # The last two bytes are the start of a three-byte euro sign. + self.log_body = 'caf\u00e9 \u20ac'.encode()[:-1] + + log = await CoreV1Api(self.api_client).read_namespaced_pod_log( + 'pod', 'default', limit_bytes=len(self.log_body), + ) + + self.assertEqual('caf\u00e9 ', log) + + async def test_pod_log_with_invalid_utf8_still_raises(self): + # A Latin-1 e-acute is not valid UTF-8 wherever it appears. + self.log_body = b'caf\xe9 \xe2\x82' + + with self.assertRaises(UnicodeDecodeError): + await CoreV1Api(self.api_client).read_namespaced_pod_log( + 'pod', 'default', limit_bytes=len(self.log_body), + ) diff --git a/kubernetes/client/api_client.py b/kubernetes/client/api_client.py index 40eaa6ea65..5a53db44d2 100644 --- a/kubernetes/client/api_client.py +++ b/kubernetes/client/api_client.py @@ -11,6 +11,7 @@ +import codecs import datetime from dateutil.parser import parse from enum import Enum @@ -425,7 +426,14 @@ def response_deserialize( if content_type is not None: match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) encoding = match.group(1) if match else "utf-8" - response_text = response_data.data.decode(encoding) + try: + response_text = response_data.data.decode(encoding) + except UnicodeDecodeError: + # limitBytes on pod logs cuts at a byte boundary and can split a + # multi-byte character. final=False drops only that incomplete + # tail; any other invalid byte still raises. + decoder = codecs.getincrementaldecoder(encoding)() + response_text = decoder.decode(response_data.data, final=False) return_data = self.deserialize(response_text, response_type, content_type) finally: if not 200 <= response_data.status <= 299: diff --git a/kubernetes/test/test_api_client.py b/kubernetes/test/test_api_client.py index 91cbe45bbc..b5eec30f7a 100644 --- a/kubernetes/test/test_api_client.py +++ b/kubernetes/test/test_api_client.py @@ -68,6 +68,31 @@ def test_rest_proxycare(self): client = kubernetes.client.ApiClient(configuration=config) self.assertEqual( expected_pool, type(client.rest_client.pool_manager) ) + def _read_pod_log(self, body): + with kubernetes.client.ApiClient() as client: + response = urllib3.HTTPResponse( + body=body, status=200, reason='OK', + headers={'Content-Type': 'text/plain'}, + ) + with mock.patch.object( + client.rest_client.pool_manager, 'request', + return_value=response, + ): + return kubernetes.client.CoreV1Api(client).read_namespaced_pod_log( + 'pod', 'default', limit_bytes=len(body), + ) + + def test_pod_log_cut_mid_character_by_limit_bytes(self): + # The last two bytes are the start of a three-byte euro sign. + log = self._read_pod_log('caf\u00e9 \u20ac'.encode()[:-1]) + + self.assertEqual('caf\u00e9 ', log) + + def test_pod_log_with_invalid_utf8_still_raises(self): + # A Latin-1 e-acute is not valid UTF-8 wherever it appears. + with self.assertRaises(UnicodeDecodeError): + self._read_pod_log(b'caf\xe9 \xe2\x82') + def test_client_go_read_retries_do_not_override_write_retries(self): config = Configuration(proxy='', no_proxy='', retries=urllib3.Retry(total=2)) config.client_go_retries = True diff --git a/scripts/api_client_asyncio_patch.diff b/scripts/api_client_asyncio_patch.diff new file mode 100644 index 0000000000..90a1582906 --- /dev/null +++ b/scripts/api_client_asyncio_patch.diff @@ -0,0 +1,27 @@ +diff --git a/kubernetes/aio/client/api_client.py b/kubernetes/aio/client/api_client.py +--- a/kubernetes/aio/client/api_client.py ++++ b/kubernetes/aio/client/api_client.py +@@ -11,6 +11,7 @@ + + + ++import codecs + import datetime + from dateutil.parser import parse + from enum import Enum +@@ -353,7 +354,14 @@ class ApiClient: + if content_type is not None: + match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) + encoding = match.group(1) if match else "utf-8" +- response_text = response_data.data.decode(encoding) ++ try: ++ response_text = response_data.data.decode(encoding) ++ except UnicodeDecodeError: ++ # limitBytes on pod logs cuts at a byte boundary and can split a ++ # multi-byte character. final=False drops only that incomplete ++ # tail; any other invalid byte still raises. ++ decoder = codecs.getincrementaldecoder(encoding)() ++ response_text = decoder.decode(response_data.data, final=False) + return_data = self.deserialize(response_text, response_type, content_type) + finally: + if not 200 <= response_data.status <= 299: diff --git a/scripts/api_client_patch.diff b/scripts/api_client_patch.diff new file mode 100644 index 0000000000..9d37498e66 --- /dev/null +++ b/scripts/api_client_patch.diff @@ -0,0 +1,27 @@ +diff --git a/kubernetes/client/api_client.py b/kubernetes/client/api_client.py +--- a/kubernetes/client/api_client.py ++++ b/kubernetes/client/api_client.py +@@ -11,6 +11,7 @@ + + + ++import codecs + import datetime + from dateutil.parser import parse + from enum import Enum +@@ -425,7 +426,14 @@ class ApiClient: + if content_type is not None: + match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) + encoding = match.group(1) if match else "utf-8" +- response_text = response_data.data.decode(encoding) ++ try: ++ response_text = response_data.data.decode(encoding) ++ except UnicodeDecodeError: ++ # limitBytes on pod logs cuts at a byte boundary and can split a ++ # multi-byte character. final=False drops only that incomplete ++ # tail; any other invalid byte still raises. ++ decoder = codecs.getincrementaldecoder(encoding)() ++ response_text = decoder.decode(response_data.data, final=False) + return_data = self.deserialize(response_text, response_type, content_type) + finally: + if not 200 <= response_data.status <= 299: diff --git a/scripts/update-client-asyncio.sh b/scripts/update-client-asyncio.sh index 2de63e9c8e..b00cd4b6e5 100755 --- a/scripts/update-client-asyncio.sh +++ b/scripts/update-client-asyncio.sh @@ -61,6 +61,9 @@ mv "${CLIENT_ROOT}/swagger.json" "${SCRIPT_ROOT}/swagger.json" echo ">>> restoring Kubernetes client-go retry integration..." git apply "${SCRIPT_ROOT}/client_go_retry_asyncio_patch.diff" +echo ">>> restoring truncated UTF-8 response decoding..." +git apply "${SCRIPT_ROOT}/api_client_asyncio_patch.diff" + echo ">>> updating version information..." sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup-asyncio.py" sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py" diff --git a/scripts/update-client.sh b/scripts/update-client.sh index 102db9a1b7..1cdfe4ca1e 100755 --- a/scripts/update-client.sh +++ b/scripts/update-client.sh @@ -68,6 +68,9 @@ git apply "${SCRIPT_ROOT}/client_go_retry_patch.diff" echo ">>> restoring Kubernetes TCP keepalive option..." git apply "${SCRIPT_ROOT}/keepalive_patch.diff" +echo ">>> restoring truncated UTF-8 response decoding..." +git apply "${SCRIPT_ROOT}/api_client_patch.diff" + echo ">>> updating version information..." sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup.py" sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py"