From 16a845af282db2386537d92139befc6513b40b4d Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 15:40:45 +0900 Subject: [PATCH 1/2] JVM wheels: py3-none-, one wheel per platform The wheel bundles a platform-specific JRE but nothing compiled against CPython, yet has_ext_modules() = True also stamped cpXY-cpXY tags, so a release was 16 wheels of about 65 MB (1 GB; PyPI's default project limit is 10 GB). setup.py now subclasses bdist_wheel and keeps only the platform tag (--plat-name, _PYTHON_HOST_PLATFORM or the host's), so the manylinux_2_28, macosx_11_0_arm64 and win_amd64 tags work as before and the wheel is py3-none-, like the native arcadedb wheel. A release is 4 wheels. - pyproject.toml: requires-python >=3.11,<3.15. The tag no longer keeps pip off 3.15, and jpype1 has no cp315 wheel yet (jpype-project/jpype#1501); the cap goes with it. setuptools >= 70.1 (its own bdist_wheel) for the build. - test-jvm.yml: a build job builds each platform once (wheel-); the test job downloads that wheel and installs it, builds nothing. Test legs are the shape of test-native.yml: every Python on linux/amd64, 3.11 and 3.14 on the other platforms, 10 legs on every event (scripts/select_matrix.py, --wheels gives the build legs). - release-jvm.yml: verify expects 4 wheels, one per platform, all py3-none; the tag check (scripts/verify_wheel_tag.py, also run by build.sh) reads the name and the WHEEL file. - after_tag_verify.sh reads a previous release's cpXY files as the py3-none set. - tests: test_wheel_tags.py, test_select_matrix.py, test_release_name_guards.py. - docs and CLAUDE.md: one wheel per platform, the 10 legs, the 3.15 cap; the JVM manylinux tag in the docs was manylinux_2_34, the wheels are manylinux_2_28. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release-jvm.yml | 39 ++--- .github/workflows/test-jvm.yml | 143 +++++++++++++----- CLAUDE.md | 2 +- docs/modules/ROOT/pages/development/ci.adoc | 23 ++- .../ROOT/pages/development/release.adoc | 2 +- .../ROOT/pages/how-to/concurrency.adoc | 2 +- docs/modules/ROOT/pages/installation.adoc | 10 +- docs/modules/ROOT/pages/which-package.adoc | 4 +- pyproject.toml | 12 +- scripts/Dockerfile.build | 3 +- scripts/after_tag_verify.sh | 36 ++--- scripts/build.sh | 7 +- scripts/select_matrix.py | 81 ++++++---- scripts/verify_wheel_tag.py | 85 +++++++++++ setup.py | 27 +++- tests/conftest.py | 1 + tests/test_release_name_guards.py | 57 ++++++- tests/test_select_matrix.py | 50 ++++-- tests/test_wheel_tags.py | 107 +++++++++++++ 19 files changed, 530 insertions(+), 161 deletions(-) create mode 100644 scripts/verify_wheel_tag.py create mode 100644 tests/test_wheel_tags.py diff --git a/.github/workflows/release-jvm.yml b/.github/workflows/release-jvm.yml index c275cc4..866ea5f 100644 --- a/.github/workflows/release-jvm.yml +++ b/.github/workflows/release-jvm.yml @@ -92,8 +92,9 @@ jobs: TAG: ${{ github.ref_name }} run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME" - # Builds a wheel per platform and Python version and runs the unit tests on each. The called - # workflow uploads the wheels as wheel--py artifacts. + # Builds one wheel for each platform (py3-none-, the same file for every Python) and + # runs the unit tests on it, on the Pythons scripts/select_matrix.py picks. The called workflow + # uploads the wheels as wheel- artifacts. test: name: Unit tests and wheels needs: validate-version @@ -123,10 +124,10 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Download wheels for all tested Python versions + - name: Download the wheels of the four platforms uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: wheel-*-py* + pattern: wheel-* path: dist/ merge-multiple: true @@ -166,34 +167,31 @@ jobs: run: | ls -lh dist/ python3 -I - <<'PY' - import os, pathlib, re, subprocess, sys, tomllib, zipfile + import os, pathlib, re, subprocess, sys, zipfile tag = os.environ["TAG"] wheels = sorted(pathlib.Path("dist").glob("*.whl")) - # One wheel per platform and supported interpreter. The interpreters are the classifiers of - # pyproject.toml, which the full matrix of scripts/select_matrix.py mirrors (the release runs the full - # matrix, not the small pull request one); the four platforms are the matrix platforms. Adding either to - # the matrix changes the expected count here. - prefix = "Programming Language :: Python :: 3." - classifiers = tomllib.load(open("pyproject.toml", "rb"))["project"]["classifiers"] - pythons = [c.rsplit(" ", 1)[1] for c in classifiers if c.startswith(prefix)] + # One wheel for each platform, py3-none- (setup.py): nothing in the wheel is compiled against + # CPython, so the Python versions it installs on are requires-python and the classifiers of pyproject.toml, + # not extra files. The four platforms are the platforms of the build matrix (scripts/select_matrix.py --wheels). + # Adding a platform to the matrix changes the expected count here. platforms = { "linux/amd64": r"manylinux.*x86_64", "linux/arm64": r"manylinux.*aarch64", "darwin/arm64": r"macosx.*arm64", "windows/amd64": r"win_amd64", } - expected = len(pythons) * len(platforms) - print(f"wheels: {len(wheels)} (expected {expected} = {len(pythons)} Python versions x {len(platforms)} platforms)") + expected = len(platforms) + print(f"wheels: {len(wheels)} (expected {expected} = one for each of the {len(platforms)} platforms)") problems = [] if len(wheels) != expected: problems.append(f"expected {expected} wheels, got {len(wheels)}") for name, pattern in platforms.items(): n = sum(1 for w in wheels if re.search(pattern, w.name)) print(f" {name}: {n}") - if n != len(pythons): - problems.append(f"{name}: expected {len(pythons)} wheels, got {n}") + if n != 1: + problems.append(f"{name}: expected 1 wheel, got {n}") names = set() for w in wheels: @@ -222,10 +220,13 @@ jobs: with open(os.environ["GITHUB_OUTPUT"], "a") as out: out.write(f"import-name={imports[0]}\n") - # Size policy: every wheel stays under 100 MB (decimal); build.sh checks the same limit. + # Tag policy: every wheel is py3-none-, in its name and in its WHEEL file (a cpXY wheel would be one of sixteen). + # Size policy: every wheel stays under 100 MB (decimal); build.sh checks the same two limits. # ARCADEDB_WHEEL_MAX_MB is deliberately not set: the release has no override. - rc = subprocess.run([sys.executable, "scripts/verify_wheel_size.py", *map(str, wheels)]).returncode - sys.exit(rc) + for script in ("verify_wheel_tag.py", "verify_wheel_size.py"): + rc = subprocess.run([sys.executable, f"scripts/{script}", *map(str, wheels)]).returncode + if rc: + sys.exit(rc) PY { echo "## Wheels" diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml index 3249f09..f075c70 100644 --- a/.github/workflows/test-jvm.yml +++ b/.github/workflows/test-jvm.yml @@ -1,4 +1,6 @@ # The JVM build: the `arcadedb-jvm` distribution. release-jvm.yml calls this workflow on a version tag and publishes the wheels it builds. +# One wheel per platform, py3-none- (setup.py): the `build` job builds each once, and the `test` job installs that one file +# into every Python (3.11 to 3.14) and runs the suite. The jars and the bundled JRE do not depend on the Python, so nothing is built per Python. # The native build, `arcadedb`, is test-native.yml / release-native.yml. name: Test JVM (arcadedb-jvm) @@ -200,6 +202,7 @@ jobs: outputs: arcadedb-tag: ${{ steps.jars.outputs.arcadedb-tag }} matrix: ${{ steps.matrix.outputs.matrix }} + wheel-matrix: ${{ steps.matrix.outputs.wheel-matrix }} steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -207,6 +210,8 @@ jobs: # The platform x Python legs of this run (scripts/select_matrix.py): a small matrix on # pull requests, the full one on every other event. The organisation shares one runner # cap with ArcadeData/arcadedb, so a pull request does not run every combination. + # The wheel matrix is the platforms of those legs, one build each: the wheel is + # py3-none-, the same file for every Python. # Done first, so a failure here costs no Docker pull. - name: Select the test matrix id: matrix @@ -215,8 +220,11 @@ jobs: EVENT_NAME: ${{ github.event_name }} run: | matrix="$(python3 scripts/select_matrix.py jvm --event "$EVENT_NAME")" + wheel_matrix="$(python3 scripts/select_matrix.py jvm --event "$EVENT_NAME" --wheels)" echo "$matrix" + echo "$wheel_matrix" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "wheel-matrix=$wheel_matrix" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 @@ -264,9 +272,11 @@ jobs: path: src/arcadedb/jars/*.jar retention-days: 1 - # Second job: Build and test on each platform - test: - name: Test arcadedb (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + # Second job: build the wheel of each platform, once. The wheel is py3-none- (setup.py), so the file built here is + # the one every Python of the test job below installs: nothing is built per Python. build.sh takes a Python only for the Linux + # container that builds and smoke-tests the wheel; any of them gives the same file. + build: + name: Build arcadedb-jvm wheel (${{ matrix.platform }}) runs-on: ${{ matrix.runs-on }} needs: download-jars if: github.event_name != 'pull_request' || github.event.pull_request.draft == false @@ -275,12 +285,9 @@ jobs: strategy: # Don't cancel other matrix jobs if one fails fail-fast: false - # Chosen by the "Select the test matrix" step of download-jars from the event that - # started the run (scripts/select_matrix.py holds the platform and Python lists): - # small on pull requests, the full 4 platforms x 4 Pythons on every other event. - # Every leg carries platform, runs-on and python-version, so the job names stay - # "Test arcadedb (, Python )". - matrix: ${{ fromJSON(needs.download-jars.outputs.matrix) }} + # One leg per platform of the test matrix, from the "Select the test matrix" step of download-jars + # (scripts/select_matrix.py --wheels): {"include": [{"platform": ..., "runs-on": ...}, ...]}. + matrix: ${{ fromJSON(needs.download-jars.outputs.wheel-matrix) }} steps: - name: Checkout code @@ -289,7 +296,7 @@ jobs: - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Setup UV package manager shell: bash @@ -342,23 +349,93 @@ jobs: # ln -s "$PYTHON_DIR/python.exe" "$PYTHON_DIR/python3.exe" || true # python3 --version - - name: Build and test arcadedb (${{ matrix.platform }}) + - name: Build arcadedb (${{ matrix.platform }}) shell: bash env: # The engine version download-jars read from the ARCADEDB_IMAGE_TAG jars; # the Docker build takes its base image by this tag, the jars from the artifact ARCADEDB_TAG: ${{ needs.download-jars.outputs.arcadedb-tag }} run: | - echo "🔨 Building arcadedb for ${{ matrix.platform }} with Python ${{ matrix.python-version }}..." + echo "🔨 Building arcadedb-jvm for ${{ matrix.platform }}..." # The jars download-jars fetched, on every platform, so all wheels of this # run carry the same engine build (and build.sh checks the integration jar) - ./scripts/build.sh ${{ matrix.platform }} ${{ matrix.python-version }} src/arcadedb/jars + ./scripts/build.sh ${{ matrix.platform }} 3.12 src/arcadedb/jars - - name: Extract wheel for additional testing + # build.sh already ran this; it is repeated here so the tag shows in this job's log and summary, and so a change to build.sh + # cannot drop it unnoticed. The wheel must be py3-none-, the file the test job installs on every Python. + - name: Check the wheel tag shell: bash run: | - mkdir -p test-install - cp dist/*.whl test-install/ + python scripts/verify_wheel_tag.py dist/*.whl + { + echo "## 📦 Wheel: arcadedb-jvm (${{ matrix.platform }})" + echo + for w in dist/*.whl; do + echo "- \`$(basename "$w")\`, $(du -m "$w" | cut -f1) MB" + done + } >> "$GITHUB_STEP_SUMMARY" + + # One wheel per platform: the test job of the platform installs this file on every Python, and the release workflow + # publishes it (verify-wheels downloads wheel-). + - name: Upload wheel artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }} + path: dist/*.whl + if-no-files-found: error + retention-days: 7 + + # Third job: install the wheel of the platform into each Python and run the suite. Nothing is built here: the wheel is the file the + # build job made (py3-none-), so each Python is tested against the same file that is published. + test: + name: Test arcadedb (${{ matrix.platform }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.runs-on }} + needs: [download-jars, build] + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + strategy: + # Don't cancel other matrix jobs if one fails + fail-fast: false + # Chosen by the "Select the test matrix" step of download-jars from the event that + # started the run (scripts/select_matrix.py holds the platform and Python lists): + # small on pull requests, the full 4 platforms x 4 Pythons on every other event. + # Every leg carries platform, runs-on and python-version, so the job names stay + # "Test arcadedb (, Python )". + matrix: ${{ fromJSON(needs.download-jars.outputs.matrix) }} + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup UV package manager + shell: bash + run: | + # Retry transient network failures (a connection reset failed a whole examples job on #8897); the + # script is downloaded first so a retried transfer is never piped half-written into sh. + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" https://astral.sh/uv/install.sh + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + uv --version + + # The wheel of this platform, built once by the build job and the same file for every Python. + - name: Download the wheel of ${{ matrix.platform }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }} + path: test-install + + - name: Set UTF-8 encoding (Windows only) + if: matrix.platform == 'windows/amd64' || matrix.platform == 'windows/arm64' + shell: bash + run: | + echo "PYTHONIOENCODING=utf-8" >> $GITHUB_ENV + echo "PYTHONUTF8=1" >> $GITHUB_ENV # The tests below run on the runner (glibc 2.39). This step runs the wheel where its tag says it must # work: a manylinux_2_28 container (glibc 2.28, AlmaLinux 8, the oldest system the Linux wheels are tagged for). @@ -397,11 +474,6 @@ jobs: PYSMOKE CONTAINER - - name: Set up Python for host testing - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: ${{ matrix.python-version }} - # Note: Java is NOT required for arcadedb (JRE is bundled) # This package works without any external Java installation! @@ -411,9 +483,11 @@ jobs: # This list is hand-maintained and mirrors the `test` extra. It had # already drifted: pyarrow and pandas were missing, so to_arrow() and # to_dataframe() tests skipped themselves in this job while it - # reported green. The guard step below now fails the build if any + # reported green. setuptools is there for tests/test_wheel_packages.py and + # tests/test_wheel_tags.py, which run setup.py on a copy of the sources (this job + # no longer installs build tools: it builds nothing). The guard step below now fails the build if any # test skips for a missing import, so the next drift is loud. - uv pip install --system test-install/*.whl pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas polars "adbc-driver-postgresql>=1.12" "duckdb>=1.4.2" "ladybug>=0.21.2" "networkx>=3.4" + uv pip install --system test-install/*.whl "setuptools>=84.1.0" pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas polars "adbc-driver-postgresql>=1.12" "duckdb>=1.4.2" "ladybug>=0.21.2" "networkx>=3.4" - name: Run pytest on host id: pytest @@ -557,7 +631,7 @@ jobs: f.write("\n") # Get wheel size and analyze contents - wheel_files = glob.glob("dist/*.whl") + wheel_files = glob.glob("test-install/*.whl") if wheel_files: wheel_file = wheel_files[0] @@ -640,25 +714,10 @@ jobs: .coverage retention-days: 7 - - name: Upload wheel artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-py${{ matrix.python-version }} - path: dist/*.whl - retention-days: 7 - - - name: Upload wheel artifact for release - if: matrix.python-version == '3.12' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || matrix.platform == 'windows/amd64' && 'windows-amd64' || 'windows-arm64' }}-test - path: dist/*.whl - retention-days: 7 - # Summary job that checks all platforms test-summary: name: Test Summary - needs: test + needs: [build, test] runs-on: ubuntu-24.04 # Skipped (not red) when the matrix was skipped for a draft PR. if: always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) @@ -668,10 +727,10 @@ jobs: echo "## 🎯 Overall Test Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - if [ "${{ needs.test.result }}" = "success" ]; then + if [ "${{ needs.build.result }}" = "success" ] && [ "${{ needs.test.result }}" = "success" ]; then echo "✅ **All platforms passed testing!**" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "The arcadedb package has been successfully built and tested." >> $GITHUB_STEP_SUMMARY + echo "The arcadedb package has been successfully built (one py3-none wheel for each platform) and tested." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Package**: arcadedb" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY diff --git a/CLAUDE.md b/CLAUDE.md index 73e419a..5730ee4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,7 +71,7 @@ python -m bandit -c pyproject.toml -r examples --severity-level medium --confide Each package has one test and one release workflow: `test-jvm.yml` and `release-jvm.yml` for `arcadedb-jvm`, `test-native.yml` and `release-native.yml` for `arcadedb` (the release workflows call the test workflows with `workflow_call`; the PyPI trusted publishers are keyed on the release file names and the environments `pypi-jvm` and `pypi`, so never rename them without changing PyPI first). `test-python-examples.yml` runs `examples/`. -`.github/workflows/test-jvm.yml` builds wheels for linux/amd64, linux/arm64, darwin/arm64, windows/amd64 and tests Python 3.11–3.14; also runs nightly against `arcadedata/arcadedb:latest`. The org shares a runner cap with upstream, so a pull request runs a small matrix (jvm: linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14 = 10 legs; examples: all four platforms on 3.14 = 4 legs, 20 checks in all), while push to main, the nightly run, dispatch and the release run all 16 legs of each workflow (38 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit resolves extras with `--resolution lowest-direct` for each Python classifier and runs pip-audit, so raise lower bounds when a floor has an advisory. +`.github/workflows/test-jvm.yml` builds ONE wheel per platform (linux/amd64, linux/arm64, darwin/arm64, windows/amd64), tagged `py3-none-` by `setup.py` (`bdist_wheel.get_tag`; `scripts/verify_wheel_tag.py` checks it), and tests that wheel on Python 3.11–3.14; also runs nightly against `arcadedata/arcadedb:latest`. `requires-python` is `>=3.11,<3.15` because the tag no longer keeps pip off 3.15 and JPype has no 3.15 wheel; drop the cap with it. The org shares a runner cap with upstream, so the unit tests run 10 legs on every event (linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14, the shape of test-native.yml), and the examples run all four platforms on 3.14 on a pull request (4 legs, 24 checks in all) and 16 legs on push to main, the nightly run, dispatch and the release (36 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit resolves extras with `--resolution lowest-direct` for each Python classifier and runs pip-audit, so raise lower bounds when a floor has an advisory. `.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/after_tag_verify.sh` checks the finished release. The rule is stated once, on the docs page `development/release.adoc`. diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index 65cb003..3a4d00c 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -18,30 +18,39 @@ Each package has one test workflow and one release workflow: The release workflows call the test workflows (`workflow_call`) and then publish; see xref:development/release.adoc[Release]. Two workflows test the `arcadedb-jvm` package: `test-jvm.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheels on a native runner of each platform. `test-native.yml` builds and tests the `arcadedb` (native) package; see <<_native_build,Native build>>. -The package supports four platforms and four Python versions (3.11 to 3.14). That is 16 combinations for each workflow. A pull request does not run all of them, because the organisation shares one runner limit with ArcadeData/arcadedb. Each macOS or Windows job on a pull request delays the upstream jobs too. +The package supports four platforms and four Python versions (3.11 to 3.14). That is 16 combinations. The organisation shares one runner limit with ArcadeData/arcadedb, so each macOS or Windows job delays the upstream jobs too, and not every combination runs. + +== One wheel for each platform + +The wheel of `arcadedb-jvm` is tagged `py3-none-` (`setup.py`), like the native wheel: it carries a JRE for its platform, but nothing in it is compiled against CPython. So a Python version is a test, not a build. `test-jvm.yml` has two jobs: + +* `Build arcadedb-jvm wheel ()` builds the wheel once for each platform and uploads it as `wheel-`. `scripts/verify_wheel_tag.py` fails the build when the wheel is not `py3-none-`. +* `Test arcadedb (, Python )` downloads that wheel and installs it into one Python. It builds nothing. The release publishes these same four files. + +The `requires-python` of the package is `>=3.11,<3.15`. The wheel tag no longer keeps pip off Python 3.15, so the cap does it. JPype has no 3.15 wheel yet; remove the cap, and add 3.15 to the matrix, when it has one. == What runs when -The script `scripts/select_matrix.py` chooses the combinations. The first job of each workflow, `Download ArcadeDB JARs`, runs it and passes the result to the test job. +The script `scripts/select_matrix.py` chooses the combinations. The first job of each workflow, `Download ArcadeDB JARs`, runs it and passes the result to the next jobs: the test legs, and for `test-jvm.yml` also the platforms to build (`--wheels`). [cols="2,3,3"] |=== | Event | Unit tests (`Test arcadedb`) | Examples (`Test Python Examples`) | Pull request -| `linux/amd64` on all four Python versions. `linux/arm64`, `darwin/arm64` and `windows/amd64` on Python 3.11 and 3.14. +| 10 legs: `linux/amd64` on all four Python versions. `linux/arm64`, `darwin/arm64` and `windows/amd64` on Python 3.11 and 3.14. | All four platforms on Python 3.14. | Push to `main`, the nightly run, a manual run, a release -| All 16 combinations. +| The same 10 legs. | All 16 combinations. |=== -A pull request runs 20 checks: 14 test jobs (10 and 4), the two `Download ArcadeDB JARs` jobs, the two summary jobs, `Bandit security scan` and `Dependency floor audit`. The full matrix runs 38. +The unit tests run the same 10 legs on every event, like the native package (all Python versions on one Linux platform, the oldest and the newest elsewhere). They were 16 legs on `main`, the nightly run and a release before the wheel became `py3-none`; the extra legs tested the same file again. A pull request runs 24 checks: 14 test jobs (10 and 4), 4 wheel builds, the two `Download ArcadeDB JARs` jobs, the two summary jobs, `Bandit security scan` and `Dependency floor audit`. The other events run 36: the examples run all 16 combinations there. The names of the test jobs do not change: `Test arcadedb (, Python )`. A check keeps its history when a pull request runs fewer of them. -A release always runs the full matrix, because `release-jvm.yml` expects one wheel for each platform and Python version. The nightly run also tests against the newest `arcadedata/arcadedb:latest` image. +`release-jvm.yml` expects exactly four wheels, one for each platform, all `py3-none`. It checks the count, the tags, the sizes and the names before it publishes. The nightly run also tests against the newest `arcadedata/arcadedb:latest` image. == Other rules @@ -60,4 +69,4 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package == Change the selection -Edit `PULL_REQUEST_LEGS` in `scripts/select_matrix.py`. The test `tests/test_select_matrix.py` fixes the full matrix at 16 combinations, so a change that shrinks it fails. When you add a Python version, add it to `PYTHONS` in the script and to the classifiers in `pyproject.toml`. A test checks that both lists are the same, and that `requires-python` is the oldest of them. When you drop one, remove it from the same places and update the counts in the test. +Edit `JVM_LEGS` (unit tests, every event) or `PULL_REQUEST_LEGS` (examples on a pull request) in `scripts/select_matrix.py`. The test `tests/test_select_matrix.py` fixes the legs, so a change that shrinks them fails. When you add a Python version, add it to `PYTHONS` in the script and to the classifiers in `pyproject.toml`. A test checks that both lists are the same, that `requires-python` starts at the oldest of them and stops before the minor after the newest. When you drop one, remove it from the same places and update the counts in the test. diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index 3bc648d..d09b7dd 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -58,7 +58,7 @@ scripts/after_tag_verify.sh 26.11.1 26.10.1 The second argument is the previous release. Use `-` for the first release of the package. The script checks: * The workflow run, and the conclusion of the publish job and of its upload step. -* The files on PyPI: one wheel per supported Python version and platform, each under 100 MB, none yanked. +* The files on PyPI: one `py3-none` wheel for each platform (the same file for every Python version), each under 100 MB, none yanked. * The sha256 of every file on PyPI against the wheels the workflow checked. * A `pip download` for each of the four platforms, and a fresh install on Python 3.12 and 3.14 that opens a database and runs one SQL and one openCypher query. diff --git a/docs/modules/ROOT/pages/how-to/concurrency.adoc b/docs/modules/ROOT/pages/how-to/concurrency.adoc index f322624..baccb37 100644 --- a/docs/modules/ROOT/pages/how-to/concurrency.adoc +++ b/docs/modules/ROOT/pages/how-to/concurrency.adoc @@ -252,4 +252,4 @@ If a process must stop and it does not react to a signal, `SIGKILL` cost no ackn The native wheel is a `py3-none` wheel, and it installed and ran on free-threaded Python 3.14.5 with the GIL off. The Python 3.14t rows in the tables above are those runs: threads, several writers, asyncio and the signal and kill probes. The probe on several processes was not run there. The runs are shorter than those on Python 3.12, as listed at the top of this page. -The JVM wheel does not install on Python 3.14t. `uv` could not resolve `{jvm-pkg-name}` 26.10.2.post1 there: it reported no wheel with a matching Python version tag. +The JVM package was not run on Python 3.14t. Its wheels up to 26.11.1.dev0 were tagged for 3.11 to 3.14 (`cp314`), so `uv` found no wheel for 3.14t (it reported no wheel with a matching Python version tag for `{jvm-pkg-name}` 26.10.2.post1). The newer wheels are `py3-none`, and JPype has a 3.14t wheel on Linux, so pip can now install the package on 3.14t. No JVM test run has been made there, so treat it as untested. diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index abae1af..da93fe9 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -36,22 +36,22 @@ The JVM package is self-contained. It carries a Java 25 runtime for its platform [[requirements]] == Requirements -* Python 3.11 to 3.14 for both packages. The native package also supports Python 3.15: it is one `py3-none` wheel that loads its library with `ctypes`, and the CI runs a subset of the tests on 3.15. `{jvm-pkg-name}` does not install on 3.15 yet, because its dependency JPype has no 3.15 wheel. The CI runs all four versions 3.11 to 3.14 on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). +* Python 3.11 to 3.14 for both packages. The native package also supports Python 3.15: it is one `py3-none` wheel that loads its library with `ctypes`, and the CI runs a subset of the tests on 3.15. `{jvm-pkg-name}` does not install on 3.15 yet, because its dependency JPype has no 3.15 wheel: its `requires-python` is `>=3.11,<3.15`, so pip says that no version fits instead of building JPype from source. The cap goes when JPype ships a 3.15 wheel. The CI tests the JVM wheel on all four versions 3.11 to 3.14 on Linux x86_64, and on 3.11 and 3.14 on the other platforms (xref:development/ci.adoc[Continuous integration]). * One of the four platforms below. The Windows wheel of the native build is the newest, and its test run is the least proven of the four. It needs the Microsoft Visual C++ runtime (`VCRUNTIME140.dll`), which every CPython for Windows already brings. -Every wheel is platform-specific and built on a native runner of its platform, so pip needs no hint to pick the right one. The JVM wheel also carries a JRE for its platform. The native wheel is one file for each platform, and it serves every Python version because it loads its library with `ctypes`. +Every wheel is platform-specific and built on a native runner of its platform, so pip needs no hint to pick the right one. Each package is one file for each platform, tagged `py3-none-`, and it serves every Python version it supports. The native wheel does so because it loads its library with `ctypes`. The JVM wheel carries a JRE for its platform, and nothing in it is compiled against CPython: JPype, its dependency, has its own wheel for each Python version. [cols="2,3,3"] |=== -| Platform | Wheel tag, `{native-pkg-name}` | Wheel tag, `{jvm-pkg-name}` (Python 3.12) +| Platform | Platform tag, `{native-pkg-name}` | Platform tag, `{jvm-pkg-name}` | Linux x86_64 | `manylinux_2_28_x86_64`: glibc 2.28 or later (RHEL 8 and Rocky 8, Debian 10, Ubuntu 20.04 and newer) -| `manylinux_2_34_x86_64`: glibc 2.34 or later (RHEL 9, Debian 12, Ubuntu 22.04 and newer) +| `manylinux_2_28_x86_64`: glibc 2.28 or later | Linux ARM64 | `manylinux_2_28_aarch64`: glibc 2.28 or later -| `manylinux_2_34_aarch64`: glibc 2.34 or later +| `manylinux_2_28_aarch64`: glibc 2.28 or later | macOS Apple Silicon | `macosx_11_0_arm64` diff --git a/docs/modules/ROOT/pages/which-package.adoc b/docs/modules/ROOT/pages/which-package.adoc index 12e4cd4..5f94a35 100644 --- a/docs/modules/ROOT/pages/which-package.adoc +++ b/docs/modules/ROOT/pages/which-package.adoc @@ -78,11 +78,11 @@ The API is the same. The runtime is not. These are the differences you can plan | Python versions | 3.11 to 3.15. On 3.15 the CI runs a subset of the tests. -| 3.11 to 3.14. JPype has no 3.15 wheel yet. +| 3.11 to 3.14. JPype has no 3.15 wheel yet, so pip refuses the package on 3.15. | Platforms | Linux x86_64 and aarch64 (glibc 2.28 or later), macOS arm64, Windows x64. The Windows wheel is the newest: it is built in CI, and its test run is the least proven of the four. Not built: Intel Mac, Alpine (musl), Windows on ARM. See xref:installation.adoc#requirements[Requirements]. -| Linux x86_64 and aarch64 (glibc 2.34 or later), macOS arm64, Windows x64. Not built: Intel Mac, Alpine (musl), Windows on ARM. +| Linux x86_64 and aarch64 (glibc 2.28 or later), macOS arm64, Windows x64. Not built: Intel Mac, Alpine (musl), Windows on ARM. | Java-object APIs (`get_java_database`, `get_java_document`, `get_java_*`) | Not available. They raise `NotOnNativeYet`. diff --git a/pyproject.toml b/pyproject.toml index 67892ad..3561e34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,9 @@ # setup.py imports only setuptools, so the build needs no JPype. A build-time jpype1 was compiled from # source on a Python without a JPype wheel (3.15, until JPype > 1.7.1) and failed there. `build` is a # frontend, not a backend requirement. -requires = ["setuptools>=61.0", "wheel"] +# setuptools 70.1 is the first release with its own bdist_wheel command, which setup.py subclasses to +# tag the wheel py3-none-. +requires = ["setuptools>=70.1", "wheel"] build-backend = "setuptools.build_meta" # This file defines the package metadata for arcadedb. @@ -14,7 +16,13 @@ name = "arcadedb-jvm" version = "0.0.0" description = "ArcadeDB embedded multi-model database, JVM build with a bundled JRE (no Java installation needed). The native build is the package arcadedb; both import as arcadedb" readme = "README.md" -requires-python = ">=3.11" +# The upper bound is JPype: jpype1 1.7.1 has wheels for CPython 3.11 to 3.14 and none for 3.15 (JPype +# merged 3.15 support in jpype-project/jpype#1501, not released yet). The wheels of this package are +# py3-none-, so the wheel tag no longer keeps pip off an interpreter: without this cap pip +# would install the wheel on 3.15 and then try to build JPype from source. +# REMOVE THE CAP when jpype1 ships cp315 wheels: then add the 3.15 classifier below, 3.15 to PYTHONS in +# scripts/select_matrix.py and the cap in tests/test_select_matrix.py. +requires-python = ">=3.11,<3.15" license = { text = "Apache-2.0" } authors = [ { name = "Taewoon Kim", email = "taewoon@humem.ai" } diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index 742e977..3d21d20 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -6,7 +6,8 @@ # Excludes JARs listed in scripts/jar_exclusions.txt to optimize size # # REQUIRED BUILD ARGS (passed from build.sh): -# PYTHON_VERSION - Python version for wheel (e.g., 3.11, 3.12, 3.13, 3.14) +# PYTHON_VERSION - Python that builds the wheel and runs its smoke test (e.g., 3.12). The wheel is py3-none- +# (setup.py), so any of 3.11 to 3.14 gives the same file; the test workflow runs it on each # PACKAGE_NAME - always: arcadedb-jvm # PACKAGE_DESCRIPTION - package description # TARGET_PLATFORM - Linux target platform for bundled JRE (linux-x64, linux-arm64) diff --git a/scripts/after_tag_verify.sh b/scripts/after_tag_verify.sh index a029de2..730333f 100755 --- a/scripts/after_tag_verify.sh +++ b/scripts/after_tag_verify.sh @@ -7,10 +7,10 @@ # # is the release whose PyPI file set this one must match in shape. Pass - when # there is none (the first release of the distribution): the file set is then checked against -# pyproject.toml, one wheel per supported Python version and platform. +# the four platforms, one wheel (py3-none-) for each platform. # # Run it under bash (not zsh). Every step prints PASS or FAIL; the exit code is the number of FAILs. -# Set PIN=4-5 to pin the smoke runs to those cores. It downloads about 4 GB (the 20 checked wheels of the run and four from PyPI): TMPDIR or WORK should point at a disk, not a tmpfs. Needs: gh, curl, python3, uv. +# Set PIN=4-5 to pin the smoke runs to those cores. It downloads about 0.6 GB (the 4 checked wheels of the run and four from PyPI): TMPDIR or WORK should point at a disk, not a tmpfs. Needs: gh, curl, python3, uv. set -uo pipefail V="${1:?version}" PREV="${2:?previous version}" @@ -44,35 +44,29 @@ EOF if awk -F'\t' '$2=="completed" && $3=="success" && $4=="success"{ok=1} END{exit ok?0:1}' "$W/publish-job.tsv"; then pass "publish job and its 'Publish to PyPI' step both succeeded"; else fail "publish job or its upload step did not succeed"; fi fi -# (b) PyPI file set against the previous release: the same files as the previous release with the new version, minus wheels for Python versions the release dropped, each under 100 MB (decimal). With "-" as the previous version: one wheel per Python version of the pyproject.toml classifiers and per platform +# (b) PyPI file set against the previous release: the same files as the previous release with the new version (a previous release with one cpXY wheel for each Python counts as one py3-none wheel for each platform), each under 100 MB (decimal). With "-" as the previous version: one py3-none wheel for each platform curl -fsS "https://pypi.org/pypi/$DIST/$V/json" -o "$W/pypi-new.json" || fail "PyPI has no $V yet" if [ "$PREV" = "-" ]; then echo '{"urls": []}' > "$W/pypi-prev.json" else curl -fsS "https://pypi.org/pypi/$DIST/$PREV/json" -o "$W/pypi-prev.json" || fail "cannot read PyPI $PREV" fi -python3 - "$W/pypi-new.json" "$W/pypi-prev.json" "$V" "$PREV" "$(dirname "$0")/../pyproject.toml" << 'EOF' || FAILS=$((FAILS + 1)) -import json,re,sys,tomllib +python3 - "$W/pypi-new.json" "$W/pypi-prev.json" "$V" "$PREV" << 'EOF' || FAILS=$((FAILS + 1)) +import json,re,sys new,prev,v,pv=json.load(open(sys.argv[1])),json.load(open(sys.argv[2])),sys.argv[3],sys.argv[4] -project=tomllib.load(open(sys.argv[5],"rb"))["project"] -# A previous release may carry wheels for an interpreter this release dropped: they are not expected again. -floor=int(re.search(r">=\s*3\.(\d+)",project["requires-python"]).group(1)) -def kept(f): - m=re.search(r"-cp3(\d+)-",f) - return m is None or int(m.group(1))>=floor -want={u["filename"].replace(pv,v) for u in prev["urls"] if kept(u["filename"])} +# Releases up to 26.11.1.dev0 had one cpXY-cpXY wheel for each Python and platform; since then the wheel is py3-none-, +# one for each platform. Read the previous file set in the new shape, so the Pythons a release drops or adds change no file. +want={re.sub(r"-cp3\d+-cp3\d+-","-py3-none-",u["filename"]).replace(pv,v) for u in prev["urls"]} have={u["filename"]:u for u in new["urls"]} gaps=0 if pv=="-": - # No release to compare with: one wheel per supported Python version and platform, as release-jvm.yml counts them. - pythons=[c.rsplit(".",1)[1] for c in project["classifiers"] if c.startswith("Programming Language :: Python :: 3.")] + # No release to compare with: one py3-none wheel for each platform, as release-jvm.yml counts them. plats=[r"manylinux.*x86_64",r"manylinux.*aarch64",r"macosx.*arm64",r"win_amd64"] - for py in pythons: - for plat in plats: - hits=[f for f in have if re.search(r"-cp3%s-"%py,f) and re.search(plat,f)] - want.update(hits) - if len(hits)!=1: - print("EXPECTED one wheel for Python 3.%s on %s, got %d"%(py,plat,len(hits))); gaps+=1 + for plat in plats: + hits=[f for f in have if re.search(r"-py3-none-",f) and re.search(plat,f)] + want.update(hits) + if len(hits)!=1: + print("EXPECTED one wheel on %s, got %d"%(plat,len(hits))); gaps+=1 missing=sorted(want-set(have)); extra=sorted(set(have)-want) print("files on PyPI for %s: %d (previous release had %d)"%(v,len(have),len(prev["urls"]))) for m in missing: print("MISSING",m) @@ -106,7 +100,7 @@ fi # (d) every platform resolves, and a fresh environment installs and works (3.12 and 3.14) for plat in manylinux_2_34_x86_64 manylinux_2_34_aarch64 macosx_11_0_arm64 win_amd64; do - if uvx pip download --no-deps --only-binary=:all: --platform "$plat" --python-version 3.12 "$DIST==$V" -d "$W/dl-$plat" > "$W/dl-$plat.log" 2>&1; then pass "pip download $plat (cp312)"; else fail "pip download $plat: $(tail -1 "$W/dl-$plat.log")"; fi + if uvx pip download --no-deps --only-binary=:all: --platform "$plat" --python-version 3.12 "$DIST==$V" -d "$W/dl-$plat" > "$W/dl-$plat.log" 2>&1; then pass "pip download $plat (Python 3.12)"; else fail "pip download $plat: $(tail -1 "$W/dl-$plat.log")"; fi done for py in 3.12 3.14; do [ -n "${SKIP_SMOKE:-}" ] && { diff --git a/scripts/build.sh b/scripts/build.sh index 2b11470..7ba4e54 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -59,7 +59,8 @@ print_usage() { echo " windows/amd64 Windows x86_64 (native build on Windows)" echo "" echo "PYTHON_VERSION:" - echo " Python version for wheel (default: 3.12); used by Linux (Docker) builds only" + echo " Python of the Linux (Docker) build and its smoke test (default: 3.12); ignored on macOS and Windows." + echo " The wheel is py3-none-: the same file whichever Python builds it" echo " Examples: 3.11, 3.12, 3.13, 3.14" echo "" echo "JAR_LIB_DIR (optional):" @@ -532,6 +533,10 @@ else "$SIZE_PY" "$SCRIPT_DIR/verify_wheel_licences.py" "$NEWEST_WHEEL" || exit 1 fi +# THE TAG POLICY: one wheel for each platform, py3-none- (setup.py), the same file for every Python. A cpXY wheel +# would be one of sixteen for a release, and an -any wheel would install where its JRE does not run. +"$SIZE_PY" "$SCRIPT_DIR/verify_wheel_tag.py" "$NEWEST_WHEEL" || exit 1 + echo -e "${GREEN}🎉 Build completed successfully!${NC}" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" diff --git a/scripts/select_matrix.py b/scripts/select_matrix.py index 3774036..13c1ba3 100644 --- a/scripts/select_matrix.py +++ b/scripts/select_matrix.py @@ -1,25 +1,29 @@ #!/usr/bin/env python3 """Pick the platform x Python legs a CI workflow runs for the event that started it. -The wheels support four platforms and four Python versions, which is 16 legs per workflow. A -pull request does not need all of them: the organisation shares one runner cap with -ArcadeData/arcadedb, so every extra macOS or Windows leg on a pull request delays upstream too. +The wheels support four platforms and four Python versions. -- `pull_request` runs the small matrix below. -- Every other event runs the full matrix: a push to main, the nightly schedule, a manual - dispatch, and the release workflow (which calls the workflows as `workflow_call`; the event - name seen there is the caller's, a tag push). The release needs one wheel per platform and - Python version, and `release-jvm.yml` counts them. +- `jvm` (test-jvm.yml) runs the same 10 legs on every event: a pull request, a push to main, the nightly + schedule, a manual dispatch and the release workflow (which calls the workflow as `workflow_call`). The JVM + wheel is one file for each platform (py3-none-, built once by the `build` job), so a Python version + is a test leg, not a build. Like the native package (test-native.yml), the Pythons are all four on + linux/amd64 and the oldest and the newest on the other platforms: a bug that depends on the interpreter shows + on the ends of the range, and one that depends on the platform shows on any version of it. The organisation + shares one runner cap with ArcadeData/arcadedb, so every extra macOS or Windows leg delays upstream too. +- `examples` (test-python-examples.yml) runs the newest Python on every platform on a pull request, and every + combination (16 legs) on every other event. -The selection fails open to the full matrix: an event name this script does not know runs -everything, never less. +The selection fails open to the full matrix: an event name this script does not know runs everything, never less +(for `jvm`, whose legs do not depend on the event, that is the same 10 legs). usage: - select_matrix.py jvm [--event NAME] # test-jvm.yml - select_matrix.py examples [--event NAME] # test-python-examples.yml + select_matrix.py jvm [--event NAME] # test-jvm.yml: the test legs + select_matrix.py jvm [--event NAME] --wheels # test-jvm.yml: the build legs, one for each platform of the test legs + select_matrix.py examples [--event NAME] # test-python-examples.yml `--event` defaults to $GITHUB_EVENT_NAME. The output is one line of JSON for -`strategy.matrix`: {"include": [{"platform": ..., "runs-on": ..., "python-version": ...}, ...]}. +`strategy.matrix`: {"include": [{"platform": ..., "runs-on": ..., "python-version": ...}, ...]}, and with +`--wheels` {"include": [{"platform": ..., "runs-on": ...}, ...]}. The names of the jobs stay "Test arcadedb (, Python )", so the history of a check is comparable before and after the selection. """ @@ -46,27 +50,28 @@ OLDEST = PYTHONS[0] NEWEST = PYTHONS[-1] -# The legs of a pull request, per workflow: platform -> Python versions. -# -# jvm: all Pythons on linux/amd64 (the cheap, fast runner and the one most contributors -# use), the oldest and the newest on the three other platforms. A bug that depends on the -# interpreter shows on the ends of the range, and one that depends on the platform shows on -# any version of it. +# The legs of `jvm`, the same for every event: all Pythons on linux/amd64 (the cheap, fast runner and the one most +# contributors use), the oldest and the newest on the three other platforms (the shape of test-native.yml). The wheel +# is py3-none-, one file for each platform, so these are test legs against that file, not builds. +JVM_LEGS = { + "linux/amd64": PYTHONS, + "linux/arm64": (OLDEST, NEWEST), + "darwin/arm64": (OLDEST, NEWEST), + "windows/amd64": (OLDEST, NEWEST), +} + +# The legs of a pull request, for the workflows whose legs depend on the event. # # examples: the newest Python only, on every platform. The examples are slow (torch, # sentence-transformers), and they carry platform-specific code (`os.name == "nt"` branches in # examples 07 to 12) and per-platform steps in the workflow (a CPU torch index on macOS, UTF-8 # on Windows), so all four platforms stay. PULL_REQUEST_LEGS = { - "jvm": { - "linux/amd64": PYTHONS, - "linux/arm64": (OLDEST, NEWEST), - "darwin/arm64": (OLDEST, NEWEST), - "windows/amd64": (OLDEST, NEWEST), - }, "examples": {platform: (NEWEST,) for platform in RUNNERS}, } +WORKFLOWS = ("jvm", "examples") + def _legs(per_platform: dict[str, tuple[str, ...]]) -> list[dict[str, str]]: return [ @@ -78,25 +83,39 @@ def _legs(per_platform: dict[str, tuple[str, ...]]) -> list[dict[str, str]]: def select(workflow: str, event: str) -> dict[str, list[dict[str, str]]]: """The `strategy.matrix` of `workflow` ("jvm" or "examples") for `event`.""" - if workflow not in PULL_REQUEST_LEGS: - raise ValueError( - f"unknown workflow {workflow!r}; use one of {sorted(PULL_REQUEST_LEGS)}" - ) + if workflow not in WORKFLOWS: + raise ValueError(f"unknown workflow {workflow!r}; use one of {sorted(WORKFLOWS)}") + if workflow == "jvm": + return {"include": _legs(JVM_LEGS)} if event == "pull_request": return {"include": _legs(PULL_REQUEST_LEGS[workflow])} return {"include": _legs({platform: PYTHONS for platform in RUNNERS})} +def select_wheels(workflow: str, event: str) -> dict[str, list[dict[str, str]]]: + """The build legs of `workflow` for `event`: one for each platform that has a test leg, whatever the Python.""" + runners: dict[str, str] = {} + for leg in select(workflow, event)["include"]: + runners.setdefault(leg["platform"], leg["runs-on"]) + return {"include": [{"platform": p, "runs-on": r} for p, r in runners.items()]} + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument("workflow", choices=sorted(PULL_REQUEST_LEGS)) + parser.add_argument("workflow", choices=sorted(WORKFLOWS)) parser.add_argument( "--event", default=os.environ.get("GITHUB_EVENT_NAME", ""), help="the event that started the workflow (default: $GITHUB_EVENT_NAME)", ) + parser.add_argument( + "--wheels", + action="store_true", + help="print the build legs (one for each platform) instead of the test legs", + ) args = parser.parse_args(argv) - print(json.dumps(select(args.workflow, args.event), separators=(",", ":"))) + chosen = select_wheels if args.wheels else select + print(json.dumps(chosen(args.workflow, args.event), separators=(",", ":"))) return 0 diff --git a/scripts/verify_wheel_tag.py b/scripts/verify_wheel_tag.py new file mode 100644 index 0000000..7242129 --- /dev/null +++ b/scripts/verify_wheel_tag.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Fail a wheel that is not tagged py3-none-. + +arcadedb-jvm ships one wheel for each platform, not one for each Python version: the JRE inside is platform-specific, +but nothing in the wheel is compiled against CPython (the Java side is reached through JPype, a dependency with its +own wheels). setup.py tags the wheel py3-none-; a wheel tagged cpXY-cpXY- would be one of sixteen +files for a release (about 65 MB each) where four do, and a wheel tagged -any would install on a platform whose JRE +it does not carry. Which Pythons the wheel installs on is `requires-python` in pyproject.toml. + +A wheel passes when its file name says py3-none- with a real platform, and its WHEEL file agrees: every +`Tag:` line is that tag and `Root-Is-Purelib` is false (the files go to platlib, next to the JRE). + +Usage: + python verify_wheel_tag.py WHEEL [WHEEL ...] + +Exit status: 0 every wheel passes, 1 a wheel does not, 2 usage error or a wheel that is missing or not a zip file. +""" + +from __future__ import annotations + +import argparse +import sys +import zipfile +from pathlib import Path + + +def problems(wheel: Path) -> list[str]: + """What is wrong with the tag of `wheel`; empty when it is py3-none- in both the name and the WHEEL file.""" + found: list[str] = [] + if not wheel.name.endswith(".whl"): + return [f"{wheel.name}: not a .whl file"] + # name-version(-build)-python-abi-platform.whl + parts = wheel.name[: -len(".whl")].split("-") + if len(parts) not in (5, 6): + return [f"{wheel.name}: not a wheel file name (name-version-python-abi-platform.whl)"] + python, abi, platform = parts[-3:] + if (python, abi) != ("py3", "none"): + found.append(f"{wheel.name}: tagged {python}-{abi}, expected py3-none") + if platform == "any" or not platform: + found.append(f"{wheel.name}: platform tag {platform!r}; the wheel bundles a platform-specific JRE") + tag = f"{python}-{abi}-{platform}" + + with zipfile.ZipFile(wheel) as zf: + names = [n for n in zf.namelist() if n.endswith(".dist-info/WHEEL")] + if len(names) != 1: + return found + [f"{wheel.name}: expected one .dist-info/WHEEL, found {names}"] + # strip: a wheel built on Windows has CRLF line endings + lines = [line.strip() for line in zf.read(names[0]).decode("utf-8").splitlines()] + tags = [line.split(":", 1)[1].strip() for line in lines if line.startswith("Tag:")] + if tags != [tag]: + found.append(f"{wheel.name}: the WHEEL file says Tag {tags}, the file name says {tag}") + purelib = [line.split(":", 1)[1].strip().lower() for line in lines if line.startswith("Root-Is-Purelib:")] + if purelib != ["false"]: + found.append(f"{wheel.name}: Root-Is-Purelib is {purelib}, expected false (a platform wheel)") + return found + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("wheels", nargs="+", type=Path) + args = parser.parse_args(argv) + + status = 0 + for wheel in args.wheels: + try: + found = problems(wheel) + except (OSError, zipfile.BadZipFile) as error: + print(f"ERROR: {wheel}: {error}", file=sys.stderr) + return 2 + if found: + status = 1 + for line in found: + print(f"ERROR: {line}", file=sys.stderr) + else: + print(f"OK: {wheel.name} is tagged {'-'.join(wheel.name[: -len('.whl')].split('-')[-3:])}") + if status: + print( + " The wheel must be py3-none-: see setup.py (bdist_wheel.get_tag). Do not publish it.", + file=sys.stderr, + ) + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py index dfae8bb..643c823 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,17 @@ """ -Setup configuration for arcadedb. +Setup configuration for arcadedb-jvm. -This file forces the wheel to be platform-specific (not py3-none-any) -because we bundle platform-specific JRE binaries. +The wheel is platform-specific because it bundles a platform-specific JRE, but it is not tied to a +Python version: nothing in it is compiled against CPython (the Java side is reached through JPype, +a dependency with its own wheels). So the tag is py3-none-, one wheel per platform and the +same shape as the native `arcadedb` wheel, not cpXY-cpXY-, one wheel per Python. + +The Pythons the wheel installs on are decided by `requires-python` and the `jpype1` dependency in +pyproject.toml, not by the wheel tag. """ from setuptools import setup +from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools.dist import Distribution @@ -13,10 +19,19 @@ class BinaryDistribution(Distribution): """Mark package as having platform-specific content.""" def has_ext_modules(self): - # Return True to indicate this is a platform-specific package - # even though we don't have C extensions + # Return True to get a platform tag (and a platlib wheel) even though we have no C + # extensions. It would also stamp the interpreter and ABI tags of the building Python + # (cp312-cp312); bdist_wheel.get_tag below replaces those. return True +class bdist_wheel(_bdist_wheel): + """Keep the platform tag (--plat-name, _PYTHON_HOST_PLATFORM or the host's), drop the Python and ABI tags.""" + + def get_tag(self): + _python, _abi, plat = super().get_tag() + return "py3", "none", plat + + # All other configuration is in pyproject.toml -setup(distclass=BinaryDistribution) +setup(distclass=BinaryDistribution, cmdclass={"bdist_wheel": bdist_wheel}) diff --git a/tests/conftest.py b/tests/conftest.py index c3eb374..f37b3fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -73,6 +73,7 @@ "test_jar_closure.py": "jar closure of the JVM wheel", "test_java_package_shadowing.py": "JPype package import hooks", "test_wheel_platform_tag.py": "JVM wheel tag", + "test_wheel_tags.py": "JVM wheel tag (py3-none-)", "test_wheel_size.py": "JVM wheel size", "test_wheel_licences.py": "JVM wheel licences", "test_wheel_packages.py": "JVM wheel contents", diff --git a/tests/test_release_name_guards.py b/tests/test_release_name_guards.py index 6d4c88a..9f69c35 100644 --- a/tests/test_release_name_guards.py +++ b/tests/test_release_name_guards.py @@ -59,6 +59,12 @@ def _wheel( ) with zipfile.ZipFile(path, "w") as zf: zf.writestr(f"{dist}-{TAG}.dist-info/METADATA", meta) + # the WHEEL file of a platform wheel, as setup.py builds it (scripts/verify_wheel_tag.py reads it) + wheel_tag = "-".join(file_name.split("-")[-3:]) + zf.writestr( + f"{dist}-{TAG}.dist-info/WHEEL", + newline.join(["Wheel-Version: 1.0", "Root-Is-Purelib: false", f"Tag: {wheel_tag}", ""]), + ) if top_level is not None: zf.writestr(f"{dist}-{TAG}.dist-info/top_level.txt", top_level + newline) return path @@ -145,24 +151,21 @@ def test_the_native_workflow_refuses_a_jvm_wheel_and_the_other_way_round(tmp_pat def _jvm_release_tree(root: Path, windows_metadata_name: str | None = None) -> Path: - """What the JVM "Verify wheel count, versions and sizes" step reads: pyproject.toml, a size script and 16 wheels.""" + """What the JVM "Verify wheel count, versions and sizes" step reads: a size script, the real tag script and 4 wheels (one for each platform).""" (root / "scripts").mkdir() (root / "scripts" / "verify_wheel_size.py").write_text("import sys\nsys.exit(0)\n", encoding="utf-8") - (root / "pyproject.toml").write_bytes((REPO / "pyproject.toml").read_bytes()) + (root / "scripts" / "verify_wheel_tag.py").write_bytes((REPO / "scripts" / "verify_wheel_tag.py").read_bytes()) dist = root / "dist" dist.mkdir() - pythons = re.findall(r"Programming Language :: Python :: (3\.\d+)", (REPO / "pyproject.toml").read_text(encoding="utf-8")) platforms = { "manylinux_2_28_x86_64": "\n", "manylinux_2_28_aarch64": "\n", "macosx_11_0_arm64": "\n", "win_amd64": "\r\n", # built on Windows: CRLF, as in 26.11.1.dev0 } - for py in pythons: - tag = "cp" + py.replace(".", "") - for platform, newline in platforms.items(): - name = windows_metadata_name if (platform == "win_amd64" and windows_metadata_name) else "arcadedb-jvm" - _wheel(dist, f"arcadedb_jvm-{TAG}-{tag}-{tag}-{platform}", name, newline, top_level="arcadedb") + for platform, newline in platforms.items(): + name = windows_metadata_name if (platform == "win_amd64" and windows_metadata_name) else "arcadedb-jvm" + _wheel(dist, f"arcadedb_jvm-{TAG}-py3-none-{platform}", name, newline, top_level="arcadedb") return root @@ -181,3 +184,41 @@ def test_the_jvm_wheel_check_still_refuses_wheels_of_two_distributions(tmp_path) result = _run(_script("release-jvm.yml", VERIFY_STEP), root, TAG=TAG, GITHUB_OUTPUT=str(output)) assert result.returncode != 0 assert "disagree on distribution name" in result.stdout + + +def _jvm_release_result(tmp_path: Path, mutate) -> subprocess.CompletedProcess: + root = _jvm_release_tree(tmp_path) + mutate(root / "dist") + return _run(_script("release-jvm.yml", VERIFY_STEP), root, TAG=TAG, GITHUB_OUTPUT=str(tmp_path / "github_output")) + + +def test_the_jvm_wheel_check_wants_one_wheel_for_each_platform_and_no_more(tmp_path): + # the layout of 26.11.1.dev0 and before: one cpXY wheel for each Python and platform, 16 files + def per_python(dist: Path) -> None: + for py in ("cp311", "cp312"): + for platform in ("manylinux_2_28_x86_64", "manylinux_2_28_aarch64", "macosx_11_0_arm64", "win_amd64"): + _wheel(dist, f"arcadedb_jvm-{TAG}-{py}-{py}-{platform}", "arcadedb-jvm", top_level="arcadedb") + + result = _jvm_release_result(tmp_path, per_python) + assert result.returncode != 0 + assert "expected 4 wheels" in result.stdout + + +def test_the_jvm_wheel_check_refuses_a_platform_wheel_tagged_for_one_python(tmp_path): + def retag(dist: Path) -> None: + (dist / f"arcadedb_jvm-{TAG}-py3-none-win_amd64.whl").unlink() + _wheel(dist, f"arcadedb_jvm-{TAG}-cp312-cp312-win_amd64", "arcadedb-jvm", "\r\n", top_level="arcadedb") + + result = _jvm_release_result(tmp_path, retag) + assert result.returncode != 0 + assert "tagged cp312-cp312, expected py3-none" in result.stderr + + +def test_the_jvm_wheel_check_refuses_a_wheel_for_no_platform(tmp_path): + def drop_a_platform(dist: Path) -> None: + (dist / f"arcadedb_jvm-{TAG}-py3-none-macosx_11_0_arm64.whl").unlink() + _wheel(dist, f"arcadedb_jvm-{TAG}-py3-none-any", "arcadedb-jvm", top_level="arcadedb") + + result = _jvm_release_result(tmp_path, drop_a_platform) + assert result.returncode != 0 + assert "darwin/arm64: expected 1 wheel, got 0" in result.stdout diff --git a/tests/test_select_matrix.py b/tests/test_select_matrix.py index cc95fb0..a8ddc28 100644 --- a/tests/test_select_matrix.py +++ b/tests/test_select_matrix.py @@ -61,22 +61,31 @@ def _pairs(matrix): return set(pairs) -@pytest.mark.parametrize("workflow", ["jvm", "examples"]) @pytest.mark.parametrize("event", NON_PULL_REQUEST_EVENTS) -def test_every_event_but_pull_request_runs_the_full_matrix(workflow, event): - matrix = sm.select(workflow, event) +def test_every_event_but_pull_request_runs_the_full_examples_matrix(event): + matrix = sm.select("examples", event) assert len(matrix["include"]) == 16 assert _pairs(matrix) == FULL -def test_pull_request_jvm_matrix(): - pairs = _pairs(sm.select("jvm", "pull_request")) +@pytest.mark.parametrize("event", ["pull_request", *NON_PULL_REQUEST_EVENTS]) +def test_jvm_tests_the_one_wheel_of_a_platform_on_the_same_ten_legs_on_every_event(event): + # The wheel is py3-none-: every Python of linux/amd64, the oldest and the newest elsewhere (as test-native.yml) + pairs = _pairs(sm.select("jvm", event)) assert {v for p, v in pairs if p == "linux/amd64"} == set(PYTHONS) for platform in ("linux/arm64", "darwin/arm64", "windows/amd64"): assert {v for p, v in pairs if p == platform} == {"3.11", "3.14"}, platform assert len(pairs) == 10 +@pytest.mark.parametrize("event", ["pull_request", *NON_PULL_REQUEST_EVENTS]) +def test_jvm_builds_one_wheel_for_each_platform_whatever_the_python(event): + # One build leg for each platform of the test legs, in the order of RUNNERS; no Python in it + assert sm.select_wheels("jvm", event) == { + "include": [{"platform": p, "runs-on": r} for p, r in PLATFORMS.items()] + } + + def test_pull_request_examples_matrix(): pairs = _pairs(sm.select("examples", "pull_request")) assert pairs == {(p, "3.14") for p in PLATFORMS} @@ -85,6 +94,7 @@ def test_pull_request_examples_matrix(): @pytest.mark.parametrize("workflow", ["jvm", "examples"]) def test_pull_request_matrix_is_part_of_the_full_matrix(workflow): assert _pairs(sm.select(workflow, "pull_request")) < FULL + assert _pairs(sm.select(workflow, "pull_request")) <= _pairs(sm.select(workflow, "push")) def test_unknown_workflow_is_refused(): @@ -93,8 +103,8 @@ def test_unknown_workflow_is_refused(): def test_pythons_match_the_classifiers_of_pyproject(): - # release-jvm.yml expects one wheel per classifier Python and platform from the full matrix; a - # Python added to one place and not the other would fail the release, not a pull request. + # The Pythons the CI covers and the Pythons the wheel claims are one list; a Python added to + # one place and not the other would be tested without a classifier, or claimed without a test. project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ "project" ] @@ -105,12 +115,16 @@ def test_pythons_match_the_classifiers_of_pyproject(): assert classifiers, "no Python version classifiers found in pyproject.toml" assert list(sm.PYTHONS) == classifiers # The oldest Python of the matrix is the floor of requires-python, so dropping or adding the - # oldest one cannot leave the install metadata saying something else. - assert project["requires-python"] == f">={sm.PYTHONS[0]}" + # oldest one cannot leave the install metadata saying something else. The ceiling is the minor after + # the newest: the wheel is py3-none, so its tag no longer keeps pip off a Python the matrix does not + # cover (3.15 has no JPype wheel yet), and requires-python has to. Remove it with the JPype cap + # in pyproject.toml. + newest_minor = int(sm.PYTHONS[-1].split(".")[1]) + assert project["requires-python"] == f">={sm.PYTHONS[0]},<3.{newest_minor + 1}" def test_command_line_prints_one_line_of_matrix_json(): - for event, count in (("pull_request", 10), ("push", 16)): + for event, count in (("pull_request", 10), ("push", 10)): out = subprocess.run( # nosec B603 - fixed argument list, no untrusted input [sys.executable, "-I", str(SCRIPT_PATH), "jvm", "--event", event], capture_output=True, @@ -127,6 +141,16 @@ def test_workflow_takes_its_matrix_from_the_script(workflow): assert f"scripts/select_matrix.py {workflow}" in text assert "matrix: ${{ fromJSON(needs.download-jars.outputs.matrix) }}" in text # the lists are in the script; a copy left in the workflow would say one thing and run another - assert ( - "python-version:\n" not in text.split("strategy:", 1)[1].split("steps:", 1)[0] - ) + for strategy in text.split("strategy:")[1:]: + assert "python-version:\n" not in strategy.split("steps:", 1)[0] + + +def test_the_jvm_workflow_builds_each_platform_once_and_tests_that_wheel(): + text = WORKFLOWS["jvm"].read_text(encoding="utf-8") + assert "scripts/select_matrix.py jvm --event \"$EVENT_NAME\" --wheels" in text + assert "matrix: ${{ fromJSON(needs.download-jars.outputs.wheel-matrix) }}" in text + build, test = text.split("\n build:\n")[1].split("\n test:\n") + # no Python in the build matrix, and the test job builds nothing: it installs the wheel the build job uploaded + assert "python-version: ${{ matrix.python-version }}" not in build + assert "scripts/build.sh" in build and "scripts/build.sh" not in test.split("\n test-summary:")[0] + assert "actions/download-artifact" in test and "name: wheel-" in test diff --git a/tests/test_wheel_tags.py b/tests/test_wheel_tags.py new file mode 100644 index 0000000..b739b63 --- /dev/null +++ b/tests/test_wheel_tags.py @@ -0,0 +1,107 @@ +"""The JVM wheel is tagged py3-none-: one wheel for each platform, not one for each Python version. + +The wheel bundles a platform-specific JRE, but nothing in it is compiled against CPython, so setup.py keeps the platform tag and +drops the interpreter and ABI tags (a cpXY-cpXY wheel would be 16 files of about 65 MB for a release where 4 do). Which Pythons it +installs on is `requires-python` and the `jpype1` dependency, not the tag; tests/test_select_matrix.py pins `requires-python`. + +In a source checkout the test builds the wheel with setuptools (no JARs, no JRE needed). In the wheel's test container there is no +``setup.py``, so it reads the WHEEL file of the installed distribution instead. +""" + +import importlib.metadata +import importlib.util +import shutil +import subprocess # nosec B404 - test-controlled child process +import sys +import zipfile +from pathlib import Path + +import pytest + +PYTHON_DIR = Path(__file__).resolve().parents[1] + + +def _verifier(): + spec = importlib.util.spec_from_file_location( + "verify_wheel_tag", PYTHON_DIR / "scripts" / "verify_wheel_tag.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _build(tmp_path: Path, *plat_args: str) -> Path: + """The wheel `setup.py bdist_wheel` makes from a copy of the sources, with a stand-in JRE file so it is a platform wheel.""" + project = tmp_path / "project" + project.mkdir() + shutil.copy(PYTHON_DIR / "pyproject.toml", project) + shutil.copy(PYTHON_DIR / "setup.py", project) + shutil.copytree( + PYTHON_DIR / "src" / "arcadedb", + project / "src" / "arcadedb", + ignore=shutil.ignore_patterns("__pycache__", "jars", "jre"), + ) + shutil.copy(PYTHON_DIR / "README.md", project) + dist = tmp_path / "dist" + subprocess.run( # nosec B603 - fixed argv (sys.executable setup.py bdist_wheel), no shell + [sys.executable, "setup.py", "-q", "bdist_wheel", "--dist-dir", str(dist), *plat_args], + cwd=project, + check=True, + capture_output=True, + ) + (wheel,) = dist.glob("*.whl") + return wheel + + +@pytest.mark.parametrize( + "plat", + ["manylinux_2_28_x86_64", "manylinux_2_28_aarch64", "win_amd64"], +) +def test_the_build_gives_a_py3_none_platform_wheel(tmp_path, plat): + if not (PYTHON_DIR / "setup.py").exists(): + return # the wheel's test container: test_installed_distribution_is_py3_none covers it + wheel = _build(tmp_path, "--plat-name", plat) + # not cpXY-cpXY-, whatever Python builds it + assert wheel.name.endswith(f"-py3-none-{plat}.whl"), wheel.name + assert _verifier().problems(wheel) == [] + with zipfile.ZipFile(wheel) as zf: + (name,) = [n for n in zf.namelist() if n.endswith(".dist-info/WHEEL")] + text = zf.read(name).decode() + assert f"Tag: py3-none-{plat}" in text + assert "Root-Is-Purelib: false" in text + + +def test_the_build_without_a_platform_name_keeps_the_hosts(tmp_path): + if not (PYTHON_DIR / "setup.py").exists(): + return + wheel = _build(tmp_path) + assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl"), wheel.name + + +def test_the_verifier_refuses_the_old_per_python_tag_and_the_any_tag(tmp_path): + verifier = _verifier() + for name, expected in [ + ("a-1-cp312-cp312-win_amd64.whl", "tagged cp312-cp312, expected py3-none"), + ("a-1-py3-none-any.whl", "platform tag 'any'"), + ]: + wheel = tmp_path / name + tag = "-".join(name[: -len(".whl")].split("-")[-3:]) + with zipfile.ZipFile(wheel, "w") as zf: + zf.writestr("a-1.dist-info/WHEEL", f"Root-Is-Purelib: false\nTag: {tag}\n") + assert any(expected in line for line in verifier.problems(wheel)), name + + +def test_installed_distribution_is_py3_none(): + try: + dist = importlib.metadata.distribution("arcadedb-jvm") + except importlib.metadata.PackageNotFoundError: + return # not installed as the arcadedb-jvm wheel (a source checkout run) + text = dist.read_text("WHEEL") + if text is None: + return # an editable install has no WHEEL tag of a built wheel + lines = [line.strip() for line in text.splitlines()] + tags = [line.split(":", 1)[1].strip() for line in lines if line.startswith("Tag:")] + assert len(tags) == 1 and tags[0].startswith("py3-none-"), tags + assert tags[0] != "py3-none-any", tags + assert "Root-Is-Purelib: false" in lines From 8e91598bb328d2c473de73ca7fca8b60a5e0dc60 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 16:26:24 +0900 Subject: [PATCH 2/2] tests: run test_log_redirect.py on the native build too (JVM-only JEP 498 trigger and probes) #88 made the child of the redirect tests call jpype.JClass("java.lang.System") for the Java stream probes and the SimpleMappedReader$Supplier trigger of the JEP 498 Unsafe warning. The native build has no JVM, so four tests of the file failed in the native parity run (job 121437057831). The redirect hygiene is shared behaviour, so the tests stay on both builds; only the Java-level probes, the trigger and the UNSAFE_WARNING assertions are gated on arcadedb.__backend__ == "jvm". Co-Authored-By: Claude Sonnet 5.5 --- tests/test_log_redirect.py | 82 +++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/tests/test_log_redirect.py b/tests/test_log_redirect.py index ee38b97..cbf7393 100644 --- a/tests/test_log_redirect.py +++ b/tests/test_log_redirect.py @@ -17,13 +17,18 @@ ``System.out`` and ``System.err`` alone left it on the cached handle. (A canary file that took over the stale handle's value after the host's ``dup2`` held exactly these four lines.) -The redirect tests run on every platform: on POSIX writes go to fd 1 and 2 by number, so a redirect is followed and the default -passes trivially there, and on Windows it is the real check. The console-off test also fails on POSIX without the second fix, because -the Java probes below then go to the redirect target. The child does the same work as ``test_lsm_persistence`` and -``test_lsm_vector_search_approximate_persistence`` in ``test_vector.py`` (create a vector index, build the graph, close, reopen, -query), ``ARCADEDB_REDIRECT_REPEATS`` times in one process (default 10), and after every step prints a probe line through Java's -``System.out`` and ``System.err``: a write through the cached handle that hits a file whenever a database file owns the handle -value, which no log line has to be around for. Every file under the data directory is then scanned for log text and probes. +The redirect tests run on every platform and on both builds (the first fix is shared; the second is the JVM build's, see below): +on POSIX writes go to fd 1 and 2 by number, so a redirect is followed and the default passes trivially there, and on Windows it is +the real check. The JVM build's console-off test also fails on POSIX without the second fix, because the Java probes below then go +to the redirect target. The child does the same work as ``test_lsm_persistence`` and ``test_lsm_vector_search_approximate_persistence`` +in ``test_vector.py`` (create a vector index, build the graph, close, reopen, query), ``ARCADEDB_REDIRECT_REPEATS`` times in one +process (default 10). On the JVM build it prints a probe line after every step through Java's ``System.out`` and ``System.err``: a +write through the cached handle that hits a file whenever a database file owns the handle value, which no log line has to be around +for. Every file under the data directory is then scanned for log text and probes. + +The probes and the JEP 498 trigger call Java through JPype, so they run on the JVM build only (``JVM`` below). The native build has no +JVM and no Java streams: its child runs the same steps with zero probes, and the log text in the redirect target, in the database +files and on the console is checked the same way. One text is not covered: the runtime prints ``WARNING: Using incubator modules: jdk.incubator.vector`` to stderr while the JVM is created, before the library can install its streams. In the child the JVM starts before the redirect, so that line goes to the @@ -41,8 +46,12 @@ import pytest +import arcadedb from arcadedb import _logdir +# The Java-level probes and the JEP 498 trigger call Java through JPype, so they need the JVM build (arcadedb-jvm), not the native one. +JVM = arcadedb.__backend__ == "jvm" + # One engine log line as the file and console formatters write it: date, time with milliseconds, level. LOG_LINE = re.compile( rb"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} " @@ -74,11 +83,11 @@ import sys import time -import jpype - import arcadedb mode, data_dir, sink, result, repeats, marker = sys.argv[1:7] +# the probes and the JEP 498 trigger call Java through JPype: only the JVM build has a JVM to call +jvm = arcadedb.__backend__ == "jvm" # start the library first: on Windows it caches the handles of fd 1 and 2 now warm = arcadedb.create_database(os.path.join(data_dir, "warm")) @@ -92,25 +101,30 @@ os.dup2(fd, 2) os.close(fd) -system = jpype.JClass("java.lang.System") probes = 0 - -# the runtime's own warning, now and not at some later reopen: jvector unmaps a file with sun.misc.Unsafe.invokeCleaner unsafe = False -try: - target = os.path.join(os.path.dirname(sink), "mapped.bin") - with open(target, "wb") as handle: - handle.write(b"x" * 64) - supplier = jpype.JClass("io.github.jbellis.jvector.disk.SimpleMappedReader$Supplier") - supplier(jpype.JClass("java.nio.file.Path").of(target, [])).close() - unsafe = True -except Exception: - pass # jvector changed: the reopen below still reaches the warning if the runtime prints it +if jvm: + import jpype + + system = jpype.JClass("java.lang.System") + + # the runtime's own warning, now and not at some later reopen: jvector unmaps a file with sun.misc.Unsafe.invokeCleaner + try: + target = os.path.join(os.path.dirname(sink), "mapped.bin") + with open(target, "wb") as handle: + handle.write(b"x" * 64) + supplier = jpype.JClass("io.github.jbellis.jvector.disk.SimpleMappedReader$Supplier") + supplier(jpype.JClass("java.nio.file.Path").of(target, [])).close() + unsafe = True + except Exception: + pass # jvector changed: the reopen below still reaches the warning if the runtime prints it def probe(step): - # a Java-level write to stdout and to stderr, as a runtime warning or a library that prints does + # a Java-level write to stdout and to stderr, as a runtime warning or a library that prints does (none on the native build) global probes + if not jvm: + return probes += 1 system.out.println(f"{marker} out {probes} {step}") system.err.println(f"{marker} err {probes} {step}") @@ -252,9 +266,9 @@ def _log_file_text(log_dir: Path) -> bytes: def test_redirecting_fd_1_and_2_leaves_the_database_files_clean(tmp_path): run = _run(tmp_path, "redirect") assert run.result == run.expected - assert ( - run.result["probes"] >= 6 * REPEATS - ) # the probes ran: a clean scan is not clean because nothing wrote + if JVM: + # the probes ran: a clean scan is not clean because nothing wrote + assert run.result["probes"] >= 6 * REPEATS assert _stray_text_in_files(run.data_dir) == [] # the work did log, to the file: the scan above is not clean because nothing was logged assert LOG_LINE.search(_log_file_text(run.log_dir)), list(run.log_dir.iterdir()) @@ -267,15 +281,16 @@ def test_redirect_with_the_console_off_sends_nothing_to_the_redirect_target(tmp_ assert run.result == run.expected assert LOG_LINE.search(run.sink) is None, run.sink[:500] assert PROBE not in run.sink, run.sink[:500] - assert UNSAFE_WARNING not in run.sink, run.sink[:500] assert _stray_text_in_files(run.data_dir) == [] assert LOG_LINE.search(_log_file_text(run.log_dir)) - # every probe, from both streams, is in the file the library owns (and nowhere else) + # every probe, from both streams, is in the file the library owns (and nowhere else); the native build has none assert run.stdio.count(PROBE) == 2 * run.result["probes"], run.stdio[:500] assert run.stdio.count(b" out ") == run.stdio.count(b" err ") - # the runtime's own warning is printed to System.initialErr, which setErr does not reach: it is there too - assert run.result["unsafe"], UNSAFE_TRIGGER_FAILED - assert UNSAFE_WARNING in run.stdio, run.stdio[:500] + if JVM: + assert UNSAFE_WARNING not in run.sink, run.sink[:500] + # the runtime's own warning is printed to System.initialErr, which setErr does not reach: it is there too + assert run.result["unsafe"], UNSAFE_TRIGGER_FAILED + assert UNSAFE_WARNING in run.stdio, run.stdio[:500] def test_console_log_can_be_turned_on(tmp_path): @@ -298,10 +313,11 @@ def test_console_log_can_be_turned_off(tmp_path): console = run.proc.stdout + run.proc.stderr assert LOG_LINE.search(console) is None, console[:500] assert PROBE not in console, console[:500] - assert UNSAFE_WARNING not in console, console[:500] assert run.stdio.count(PROBE) == 2 * run.result["probes"], run.stdio[:500] - assert run.result["unsafe"], UNSAFE_TRIGGER_FAILED - assert UNSAFE_WARNING in run.stdio, run.stdio[:500] + if JVM: + assert UNSAFE_WARNING not in console, console[:500] + assert run.result["unsafe"], UNSAFE_TRIGGER_FAILED + assert UNSAFE_WARNING in run.stdio, run.stdio[:500] assert LOG_LINE.search(_log_file_text(run.log_dir))