feat(CI): Set up CodSpeed

.github/workflows/codspeed.yml

@@ -0,0 +1,38 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read # Check out the repository and its submodules
+  id-token: write # Authenticate result uploads to CodSpeed via OIDC
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: codspeed-macro
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          submodules: recursive
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: '3.13'
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0
+        with:
+          mode: walltime
+          run: make benchmark

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "tests/data/sdk-benchmarks"]
+	path = tests/data/sdk-benchmarks
+	url = https://github.com/Flagsmith/sdk-benchmarks.git
+	branch = 0.0.2

Makefile

@@ -0,0 +1,9 @@
+SDK_BENCHMARKS := tests/data/sdk-benchmarks
+
+$(SDK_BENCHMARKS)/environments:
+	git submodule update --init -- $(SDK_BENCHMARKS)
+
+.PHONY: benchmark
+benchmark: | $(SDK_BENCHMARKS)/environments
+	poetry install --with dev,benchmark
+	SDK_BENCHMARKS_ENVIRONMENTS_DIR=$(SDK_BENCHMARKS)/environments poetry run pytest benchmarks --codspeed

benchmarks/test_sdk.py

@@ -0,0 +1,96 @@
+"""
+Benchmarks a locally evaluating SDK client against environment documents tiered by size.
+"""
+
+import os
+import typing
+from pathlib import Path
+
+import pytest
+import responses
+from flag_engine.engine import ContextValue
+
+from flagsmith import Flagsmith
+
+API_URL = "http://flagsmith.benchmark/api/v1/"
+ENVIRONMENTS_DIR = Path(os.environ["SDK_BENCHMARKS_ENVIRONMENTS_DIR"])
+IDENTITY_OVERRIDES_IDENTIFIER = "identity-override-1"
+IDENTIFIER = "user-1"
+TRAITS: typing.Mapping[str, ContextValue] = {
+    "plan": "plan-7",
+    "age": 30,
+    "country": "GB",
+    "version": "2.5.1",
+    "email": "[email protected]",
+    "user_id": 12,
+    "beta": True,
+    "score": 95,
+    "tier": "pro",
+    "device": "iPhone 15",
+}
+
+
+@pytest.fixture(scope="module", params=["small", "medium", "large"])
+def client(request: pytest.FixtureRequest) -> Flagsmith:
+    """
+    Build a locally evaluating client serving the environment document from memory.
+    """
+    environment_document = (ENVIRONMENTS_DIR / f"{request.param}.json").read_bytes()
+
+    with responses.RequestsMock() as mocked_responses:
+        mocked_responses.get(
+            f"{API_URL}environment-document/",
+            body=environment_document,
+            content_type="application/json",
+        )
+        client = Flagsmith(
+            environment_key="ser.benchmark",
+            api_url=API_URL,
+            enable_local_evaluation=True,
+        )
+        client.environment_data_polling_manager_thread.stop()  # Refreshed once, at startup only
+        client.environment_data_polling_manager_thread.join()
+
+    return client
+
+
+@pytest.mark.benchmark
+def test_get_environment_flags(client: Flagsmith) -> None:
+    """
+    Get a feature value from the environment flags.
+    """
+    client.get_environment_flags().get_feature_value("feature_1")
+
+
+@pytest.mark.benchmark
+def test_get_identity_flags_with_identity_overrides(client: Flagsmith) -> None:
+    """
+    Check a feature for an identity with identity overrides.
+    """
+    client.get_identity_flags(IDENTITY_OVERRIDES_IDENTIFIER).is_feature_enabled(
+        "feature_8"
+    )
+
+
+@pytest.mark.benchmark
+def test_get_identity_flags_no_traits(client: Flagsmith) -> None:
+    """
+    Get a feature value for an identity without traits.
+    """
+    client.get_identity_flags(IDENTIFIER).get_feature_value("feature_1")
+
+
+@pytest.mark.benchmark
+def test_get_identity_flags_with_traits(client: Flagsmith) -> None:
+    """
+    Get a feature value for an identity with traits.
+    """
+    client.get_identity_flags(IDENTIFIER, TRAITS).get_feature_value("feature_1")
+
+
+@pytest.mark.benchmark
+def test_get_identity_segments_with_traits(client: Flagsmith) -> None:
+    """
+    List the segments of an identity with traits.
+    """
+    client.get_identity_segments(IDENTIFIER, TRAITS)

pyproject.toml

@@ -31,6 +31,15 @@ responses = "^0.24.1"
 types-requests = "^2.32"
 pyfakefs = "^5.9.2"
 
+[tool.poetry.group.benchmark]
+optional = true
+
+[tool.poetry.group.benchmark.dependencies]
+pytest-codspeed = "^5.0.3"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
 [tool.mypy]
 exclude = ["example/*"]