Handle pod logs cut mid-character by limitBytes

kubernetes/aio/client/api_client.py

@@ -11,6 +11,7 @@
 
 
 
+import codecs
 import datetime
 from dateutil.parser import parse
 from enum import Enum
@@ -353,7 +354,14 @@ def response_deserialize(
                 if content_type is not None:
                     match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type)
                 encoding = match.group(1) if match else "utf-8"
-                response_text = response_data.data.decode(encoding)
+                try:
+                    response_text = response_data.data.decode(encoding)
+                except UnicodeDecodeError:
+                    # limitBytes on pod logs cuts at a byte boundary and can split a
+                    # multi-byte character. final=False drops only that incomplete
+                    # tail; any other invalid byte still raises.
+                    decoder = codecs.getincrementaldecoder(encoding)()
+                    response_text = decoder.decode(response_data.data, final=False)
                 return_data = self.deserialize(response_text, response_type, content_type)
         finally:
             if not 200 <= response_data.status <= 299:

kubernetes/aio/test/test_generated_api.py

@@ -48,6 +48,7 @@ async def asyncSetUp(self):
         self.response_status = 200
         self.response_headers = {}
         self.responses = None
+        self.log_body = b''
         app = web.Application()
         app.router.add_route('*', '/{path:.*}', self._handle_request)
         self.runner = web.AppRunner(app)
@@ -93,6 +94,8 @@ async def _handle_request(self, request):
             }).encode() + b'\n')
             await response.write_eof()
             return response
+        if request.path.endswith('/log'):
+            return web.Response(body=self.log_body, content_type='text/plain')
         if self.responses is not None:
             index = len(self.requests) - 1
             response, status, headers = self.responses[
@@ -371,3 +374,22 @@ async def test_models_preserve_int_or_string_and_python_field_names(self):
             'resource_version',
             inspect.signature(V1ObjectMeta).parameters,
         )
+
+    async def test_pod_log_cut_mid_character_by_limit_bytes(self):
+        # The last two bytes are the start of a three-byte euro sign.
+        self.log_body = 'caf\u00e9 \u20ac'.encode()[:-1]
+
+        log = await CoreV1Api(self.api_client).read_namespaced_pod_log(
+            'pod', 'default', limit_bytes=len(self.log_body),
+        )
+
+        self.assertEqual('caf\u00e9 ', log)
+
+    async def test_pod_log_with_invalid_utf8_still_raises(self):
+        # A Latin-1 e-acute is not valid UTF-8 wherever it appears.
+        self.log_body = b'caf\xe9 \xe2\x82'
+
+        with self.assertRaises(UnicodeDecodeError):
+            await CoreV1Api(self.api_client).read_namespaced_pod_log(
+                'pod', 'default', limit_bytes=len(self.log_body),
+            )

kubernetes/client/api_client.py

@@ -11,6 +11,7 @@
 
 
 
+import codecs
 import datetime
 from dateutil.parser import parse
 from enum import Enum
@@ -425,7 +426,14 @@ def response_deserialize(
                 if content_type is not None:
                     match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type)
                 encoding = match.group(1) if match else "utf-8"
-                response_text = response_data.data.decode(encoding)
+                try:
+                    response_text = response_data.data.decode(encoding)
+                except UnicodeDecodeError:
+                    # limitBytes on pod logs cuts at a byte boundary and can split a
+                    # multi-byte character. final=False drops only that incomplete
+                    # tail; any other invalid byte still raises.
+                    decoder = codecs.getincrementaldecoder(encoding)()
+                    response_text = decoder.decode(response_data.data, final=False)
                 return_data = self.deserialize(response_text, response_type, content_type)
         finally:
             if not 200 <= response_data.status <= 299:

kubernetes/test/test_api_client.py

@@ -68,6 +68,31 @@ def test_rest_proxycare(self):
             client = kubernetes.client.ApiClient(configuration=config)
             self.assertEqual( expected_pool, type(client.rest_client.pool_manager) )
 
+    def _read_pod_log(self, body):
+        with kubernetes.client.ApiClient() as client:
+            response = urllib3.HTTPResponse(
+                body=body, status=200, reason='OK',
+                headers={'Content-Type': 'text/plain'},
+            )
+            with mock.patch.object(
+                client.rest_client.pool_manager, 'request',
+                return_value=response,
+            ):
+                return kubernetes.client.CoreV1Api(client).read_namespaced_pod_log(
+                    'pod', 'default', limit_bytes=len(body),
+                )
+
+    def test_pod_log_cut_mid_character_by_limit_bytes(self):
+        # The last two bytes are the start of a three-byte euro sign.
+        log = self._read_pod_log('caf\u00e9 \u20ac'.encode()[:-1])
+
+        self.assertEqual('caf\u00e9 ', log)
+
+    def test_pod_log_with_invalid_utf8_still_raises(self):
+        # A Latin-1 e-acute is not valid UTF-8 wherever it appears.
+        with self.assertRaises(UnicodeDecodeError):
+            self._read_pod_log(b'caf\xe9 \xe2\x82')
+
     def test_client_go_read_retries_do_not_override_write_retries(self):
         config = Configuration(proxy='', no_proxy='', retries=urllib3.Retry(total=2))
         config.client_go_retries = True

scripts/api_client_asyncio_patch.diff

@@ -0,0 +1,27 @@
+diff --git a/kubernetes/aio/client/api_client.py b/kubernetes/aio/client/api_client.py
+--- a/kubernetes/aio/client/api_client.py
++++ b/kubernetes/aio/client/api_client.py
+@@ -11,6 +11,7 @@
+
+
+
++import codecs
+ import datetime
+ from dateutil.parser import parse
+ from enum import Enum
+@@ -353,7 +354,14 @@ class ApiClient:
+                 if content_type is not None:
+                     match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type)
+                 encoding = match.group(1) if match else "utf-8"
+-                response_text = response_data.data.decode(encoding)
++                try:
++                    response_text = response_data.data.decode(encoding)
++                except UnicodeDecodeError:
++                    # limitBytes on pod logs cuts at a byte boundary and can split a
++                    # multi-byte character. final=False drops only that incomplete
++                    # tail; any other invalid byte still raises.
++                    decoder = codecs.getincrementaldecoder(encoding)()
++                    response_text = decoder.decode(response_data.data, final=False)
+                 return_data = self.deserialize(response_text, response_type, content_type)
+         finally:
+             if not 200 <= response_data.status <= 299:

scripts/api_client_patch.diff

@@ -0,0 +1,27 @@
+diff --git a/kubernetes/client/api_client.py b/kubernetes/client/api_client.py
+--- a/kubernetes/client/api_client.py
++++ b/kubernetes/client/api_client.py
+@@ -11,6 +11,7 @@
+
+
+
++import codecs
+ import datetime
+ from dateutil.parser import parse
+ from enum import Enum
+@@ -425,7 +426,14 @@ class ApiClient:
+                 if content_type is not None:
+                     match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type)
+                 encoding = match.group(1) if match else "utf-8"
+-                response_text = response_data.data.decode(encoding)
++                try:
++                    response_text = response_data.data.decode(encoding)
++                except UnicodeDecodeError:
++                    # limitBytes on pod logs cuts at a byte boundary and can split a
++                    # multi-byte character. final=False drops only that incomplete
++                    # tail; any other invalid byte still raises.
++                    decoder = codecs.getincrementaldecoder(encoding)()
++                    response_text = decoder.decode(response_data.data, final=False)
+                 return_data = self.deserialize(response_text, response_type, content_type)
+         finally:
+             if not 200 <= response_data.status <= 299:

scripts/update-client-asyncio.sh

@@ -61,6 +61,9 @@ mv "${CLIENT_ROOT}/swagger.json" "${SCRIPT_ROOT}/swagger.json"
 echo ">>> restoring Kubernetes client-go retry integration..."
 git apply "${SCRIPT_ROOT}/client_go_retry_asyncio_patch.diff"
 
+echo ">>> restoring truncated UTF-8 response decoding..."
+git apply "${SCRIPT_ROOT}/api_client_asyncio_patch.diff"
+
 echo ">>> updating version information..."
 sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup-asyncio.py"
 sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py"

scripts/update-client.sh

@@ -68,6 +68,9 @@ git apply "${SCRIPT_ROOT}/client_go_retry_patch.diff"
 echo ">>> restoring Kubernetes TCP keepalive option..."
 git apply "${SCRIPT_ROOT}/keepalive_patch.diff"
 
+echo ">>> restoring truncated UTF-8 response decoding..."
+git apply "${SCRIPT_ROOT}/api_client_patch.diff"
+
 echo ">>> updating version information..."
 sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup.py"
 sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py"