diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 8d33190..0c0f2d4 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -12,13 +12,13 @@ # This workflow never publishes: it has no id-token permission and no environment. # # Beside the full parity runs, the `pyver` job installs the already built linux-x86_64 wheel (py3-none: no rebuild) into the Pythons the full -# runs of the event leave out (a pull request: 3.11, 3.13, 3.14; main, nightly, dispatch: 3.11, 3.13; a release: none) and into the free-threaded -# 3.14t, and runs a small subset of the tests there (about 20 s of tests per Python). It is informational: release-native.yml does not need it. +# runs of the event leave out (a pull request: 3.11, 3.13, 3.14; main, nightly, dispatch: 3.11, 3.13; a release: none), into 3.15 (always: the full suite +# needs jpype1 and ladybug, which have no cp315 wheel yet, so 3.15 runs without the test files that need jpype1) and into the free-threaded 3.14t, and runs a small subset of the tests there (about 20 s of tests per Python). It is informational: release-native.yml does not need it. # scripts/select_native_matrix.py decides its legs. # # One wheel per platform, py3-none-: -# linux-x86_64 manylinux_2_34 container manylinux_2_34_x86_64 libarcadedb.so + libjvector-x86_64.so -# linux-aarch64 manylinux_2_34 aarch64 container manylinux_2_34_aarch64 libarcadedb.so +# linux-x86_64 manylinux_2_28 container manylinux_2_28_x86_64 libarcadedb.so + libjvector-x86_64.so +# linux-aarch64 manylinux_2_28 aarch64 container manylinux_2_28_aarch64 libarcadedb.so # macos-arm64 native, heap and threads capped macosx__arm64 libarcadedb.dylib # windows-x64 native, MSVC environment loaded win_amd64 libarcadedb.dll # Only free GitHub-hosted standard runners. GraalVM Community Edition only (pinned by sha256 per platform), never Oracle GraalVM. @@ -278,9 +278,9 @@ jobs: mv /usr/bin/link.exe /usr/bin/link-gnu.exe || true echo "link is now: $(command -v link || echo none)" - # -march=compatibility, -J-Xmx12g, -H:-VectorAPISupport, fastops.mode 1, inside the manylinux_2_34 container (glibc 2.34) so the + # -march=compatibility, -J-Xmx12g, -H:-VectorAPISupport, fastops.mode 1, inside the manylinux_2_28 container (glibc 2.28) so the # glibc floor of libarcadedb.so is a guarantee, not a property of the runner. - - name: Build the native image in manylinux_2_34 + - name: Build the native image in manylinux_2_28 if: ${{ matrix.kind == 'linux' }} env: ENGINE: ${{ needs.plan.outputs.engine }} @@ -296,11 +296,14 @@ jobs: "$MANYLINUX_IMAGE" bash -euo pipefail -c ' ldd --version | sed -n 1p # not `head -1`: under pipefail its early exit can SIGPIPE ldd uname -m - # zlib-devel, unzip and which are in the base repositories; EPEL is not needed and its mirrors fail now and then - for i in 1 2 3 4 5; do - dnf install -y --disablerepo="epel*" --setopt=install_weak_deps=False zlib-devel unzip which >/dev/null && break - [ "$i" -lt 5 ] && { echo "dnf failed, retry $i in $((i * 20))s" >&2; sleep $((i * 20)); } || exit 1 - done + # zlib-devel, unzip and which are in the manylinux_2_28 image already; dnf runs only for a missing one (the AlmaLinux 8 mirror + # metadata fails now and then). EPEL is not needed and its mirrors fail too. + if ! rpm -q zlib-devel unzip which >/dev/null; then + for i in 1 2 3 4 5; do + dnf install -y --disablerepo="epel*" --setopt=install_weak_deps=False zlib-devel unzip which >/dev/null && break + [ "$i" -lt 5 ] && { echo "dnf failed, retry $i in $((i * 20))s" >&2; sleep $((i * 20)); } || exit 1 + done + fi cd /work bash native/scripts/build-image.sh /out ' || { tail -n 80 "$T/lib/build.log" || true; exit 1; } @@ -334,7 +337,7 @@ jobs: for so in "$T"/lib/*.so; do top="$(objdump -T "$so" | grep -Eo 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n 1)" echo "$(basename "$so"): highest $top" - [ "$(printf '%s\nGLIBC_2.34\n' "$top" | sort -V | tail -n 1)" = "GLIBC_2.34" ] || { echo "$so needs $top, above the manylinux_2_34 floor" >&2; exit 1; } + [ "$(printf '%s\nGLIBC_2.28\n' "$top" | sort -V | tail -n 1)" = "GLIBC_2.28" ] || { echo "$so needs $top, above the manylinux_2_28 floor" >&2; exit 1; } done - name: Build the wheel @@ -369,12 +372,34 @@ jobs: # auditwheel wraps its report to the terminal width, so match on one line with the whitespace collapsed tr -s '[:space:]' ' ' < "$T/auditwheel.txt" > "$T/auditwheel.flat" grep -F 'consistent with the following platform tag' "$T/auditwheel.flat" - grep -F "\"$PLAT\"" "$T/auditwheel.flat" + # auditwheel names the LOWEST tag the wheel is consistent with. The wheel carries the tag of its build container (build_wheel.py + # --glibc-policy); the library may need less (its glibc symbols stop at 2.17, jvector's libstdc++ asks for more than manylinux2014), + # never more. + granted="$(grep -Eo 'consistent with the following platform tag: "manylinux_[0-9]+_[0-9]+' "$T/auditwheel.flat" | grep -Eo 'manylinux_[0-9]+_[0-9]+' || true)" + want="$(printf '%s' "$PLAT" | grep -Eo 'manylinux_[0-9]+_[0-9]+')" + echo "auditwheel grants ${granted:-an older policy}; the wheel is tagged $want" + [ "$(printf '%s\n%s\n' "${granted:-manylinux_2_5}" "$want" | sort -V | tail -n 1)" = "$want" ] || { echo "auditwheel needs a tag above $want" >&2; exit 1; } if grep -Eiq 'instruction set|x86_64_v[0-9]' "$T/auditwheel.flat"; then echo "auditwheel reports an ISA warning: the image is not baseline x86-64" >&2 exit 1 fi + # The tests run on the runner (glibc 2.39). This step runs the wheel where its tag says it must work: in the manylinux_2_28 container + # (glibc 2.28), on that image's Python 3.12, with nothing but the wheel installed. + - name: "Gate: the wheel installs and opens a database on glibc 2.28" + if: ${{ matrix.kind == 'linux' }} + env: + MANYLINUX_IMAGE: ${{ matrix.container }} + run: | + set -euo pipefail + docker run --rm -v "$T/wheel/dist:/wheel:ro" -v "$GITHUB_WORKSPACE/scripts/smoke_wheel.py:/smoke_wheel.py:ro" "$MANYLINUX_IMAGE" bash -euo pipefail -c ' + ldd --version | sed -n 1p + /opt/python/cp312-cp312/bin/python -m venv /tmp/venv + /tmp/venv/bin/python -m pip install --no-index /wheel/*.whl + cd /tmp + /tmp/venv/bin/python /smoke_wheel.py + ' + - name: "Gate: the macOS library links system libraries only" if: ${{ matrix.kind == 'macos' }} run: | @@ -597,6 +622,7 @@ jobs: KEY: linux-x86_64 PY: ${{ matrix.python }} WITH_POLARS: ${{ matrix.polars }} + WITH_JPYPE: ${{ matrix.jpype }} FREE_THREADED: ${{ matrix.freethreaded }} # The subset: ctypes layer and database lifecycle (test_core, test_native_udf), queries and results (test_resultset), parameters # (test_parameter_binding), type conversion (test_type_conversion), transactions (test_transaction_config), async (test_async_executor), @@ -635,10 +661,17 @@ jobs: files="$files,test_dataframe_ingest.py" with_polars=(--with polars) fi + # No jpype1 wheel for this Python (3.15): leave out test_parameter_binding.py (it imports jpype at module level) and test_vector.py + # (four tests build Java arrays with it). Everything else of the subset needs no JVM bridge. + with_jpype=(--with jpype1) + if [ "$WITH_JPYPE" != true ]; then + files="$(printf '%s' "$files" | tr ',' '\n' | grep -v -x -F -e test_parameter_binding.py -e test_vector.py | paste -sd, -)" + with_jpype=() + fi echo "python $PY, files: $files" cores="$(nproc)" - uv run --no-project --python "$PY" --with "$w" --with pytest --with numpy --with requests --with jpype1 --with pyarrow \ - --with pandas ${with_polars[@]+"${with_polars[@]}"} \ + uv run --no-project --python "$PY" --with "$w" --with pytest --with numpy --with requests --with pyarrow \ + --with pandas ${with_polars[@]+"${with_polars[@]}"} ${with_jpype[@]+"${with_jpype[@]}"} \ python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" --files "$files" \ --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" | tee "$T/parity/parity-wheel.run.log" @@ -647,16 +680,20 @@ jobs: run: | set -euo pipefail w="$(ls "$T"/wheel/*.whl)" - uv run --no-project --python "$PY" --with "$w" --with numpy --with pandas --with pyarrow --with jpype1 python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" + with_jpype=(--with jpype1) + if [ "$WITH_JPYPE" != true ]; then with_jpype=(); fi + uv run --no-project --python "$PY" --with "$w" --with numpy --with pandas --with pyarrow ${with_jpype[@]+"${with_jpype[@]}"} python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" import os import sys import arcadedb # noqa: F401 - import jpype # noqa: F401 import numpy # noqa: F401 import pandas # noqa: F401 import pyarrow # noqa: F401 + if os.environ["WITH_JPYPE"] == "true": + import jpype # noqa: F401 + gil = sys._is_gil_enabled() if hasattr(sys, "_is_gil_enabled") else True print(f"python {sys.version.split()[0]}, GIL enabled after loading the wheel and its dependencies: {gil}") if os.environ["FREE_THREADED"] == "true" and gil: diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index d1ba2ae..65cb003 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -55,7 +55,7 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package * A pull request runs it only when native paths change: `native/**`, `src/arcadedb/_native/**`, the native tests, or `test-native.yml` itself. It then runs on `linux-x86_64` with Python 3.12 only. * A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs. -* The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs. +* The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) into 3.15 and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs; 3.15 has no jpype1 wheel, so it leaves out the two files that need jpype1), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs. * A native image build takes about 1 to 2.5 hours for each platform. The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down. == Change the selection diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 07ddc28..9ce2825 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -81,8 +81,8 @@ Check which one you have with `python -c "import arcadedb; print(arcadedb.__back == Requirements -* Python 3.11 to 3.14. The wheels' CI runs all four versions on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). -* Linux (x86_64 and ARM64), macOS on Apple Silicon, or Windows (x86_64). The Windows wheel of the native build is the newest. +* Python 3.11 to 3.14, and 3.15 for `{native-pkg-name}`. The wheels' CI runs all four versions on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). +* Linux (x86_64 and ARM64, glibc: no Alpine), macOS on Apple Silicon (no Intel Mac), or Windows (x86_64). The Windows wheel of the native build is the newest. The glibc floors and the other platforms that are not built are in xref:installation.adoc#requirements[Requirements]. * No Java installation, for either package. == Links diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index be8f519..65dd281 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -33,30 +33,42 @@ NOTE: *Old names.* The JVM package was formerly `arcadedb-embedded`. It is now ` The JVM package is self-contained. It carries a Java 25 runtime for its platform, so you do not install Java. Its only Python dependency is `JPype1` 1.5.0 or later (the Java-Python bridge), which pip installs for you. The native package needs no Java and no JPype. Optional extras install the columnar helpers of the native package: `[numpy]`, `[arrow]`, `[pandas]` and `[polars]`. +[[requirements]] == Requirements -* Python 3.11 to 3.14. The wheels' CI runs all four versions on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). -* One of the four platforms below. The Windows wheel of the native build is the newest, and its test run is the least proven of the four. +* Python 3.11 to 3.14 for both packages. The native package also supports Python 3.15: it is one `py3-none` wheel that loads its library with `ctypes`, and the CI runs a subset of the tests on 3.15. `{jvm-pkg-name}` does not install on 3.15 yet, because its dependency JPype has no 3.15 wheel. The CI runs all four versions 3.11 to 3.14 on every supported platform for `main`, the nightly run and each release. A pull request runs a smaller selection (xref:development/ci.adoc[Continuous integration]). +* One of the four platforms below. The Windows wheel of the native build is the newest, and its test run is the least proven of the four. It needs the Microsoft Visual C++ runtime (`VCRUNTIME140.dll`), which every CPython for Windows already brings. Every wheel is platform-specific and built on a native runner of its platform, so pip needs no hint to pick the right one. The JVM wheel also carries a JRE for its platform. The native wheel is one file for each platform, and it serves every Python version because it loads its library with `ctypes`. -[cols="2,3"] +[cols="2,3,3"] |=== -| Platform | Wheel tag (JVM wheel, Python 3.12) +| Platform | Wheel tag, `{native-pkg-name}` | Wheel tag, `{jvm-pkg-name}` (Python 3.12) | Linux x86_64 -| `manylinux_2_34_x86_64` +| `manylinux_2_28_x86_64`: glibc 2.28 or later (RHEL 8 and Rocky 8, Debian 10, Ubuntu 20.04 and newer) +| `manylinux_2_34_x86_64`: glibc 2.34 or later (RHEL 9, Debian 12, Ubuntu 22.04 and newer) | Linux ARM64 -| `manylinux_2_34_aarch64` +| `manylinux_2_28_aarch64`: glibc 2.28 or later +| `manylinux_2_34_aarch64`: glibc 2.34 or later | macOS Apple Silicon | `macosx_11_0_arm64` +| `macosx_11_0_arm64` | Windows x86_64 | `win_amd64` +| `win_amd64` |=== +Each native wheel is about 38 MB. These platforms are not built, and pip finds no wheel for them: + +* *Intel Macs* (macOS x86_64). Apple has sold only Apple Silicon Macs since 2023, and macOS 26 is the last release for Intel Macs. On an Intel Mac, run the Linux wheel in Docker or in a Linux virtual machine. +* *Alpine Linux and other musl-based distributions.* The Linux wheels are for glibc. pip then says `No matching distribution found` (for a wheel file you downloaded: `is not a supported wheel on this platform`). Use a glibc image, for example `python:3.12-slim`. +* *Windows on ARM.* +* *Linux with an older glibc than the table*, for example Amazon Linux 2 and CentOS 7. + [[what-is-included]] == What is included diff --git a/docs/modules/ROOT/pages/which-package.adoc b/docs/modules/ROOT/pages/which-package.adoc index 4cb20af..03404ca 100644 --- a/docs/modules/ROOT/pages/which-package.adoc +++ b/docs/modules/ROOT/pages/which-package.adoc @@ -75,12 +75,12 @@ The API is the same. The runtime is not. These are the differences you can plan | Larger. It carries a Java 25 runtime for its platform. | Python versions -| 3.11 to 3.14 -| 3.11 to 3.14 +| 3.11 to 3.15. On 3.15 the CI runs a subset of the tests. +| 3.11 to 3.14. JPype has no 3.15 wheel yet. | Platforms -| Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. The Windows wheel is the newest: it is built in CI, and its test run is the least proven of the four. -| Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. +| Linux x86_64 and aarch64 (glibc 2.28 or later), macOS arm64, Windows x64. The Windows wheel is the newest: it is built in CI, and its test run is the least proven of the four. Not built: Intel Mac, Alpine (musl), Windows on ARM. See xref:installation.adoc#requirements[Requirements]. +| Linux x86_64 and aarch64 (glibc 2.34 or later), macOS arm64, Windows x64. Not built: Intel Mac, Alpine (musl), Windows on ARM. | Java-object APIs (`get_java_database`, `get_java_document`, `get_java_*`) | Not available. They raise `NotOnNativeYet`. diff --git a/native/wheel/NOTICE b/native/wheel/NOTICE index ea868f8..6d17094 100644 --- a/native/wheel/NOTICE +++ b/native/wheel/NOTICE @@ -3,7 +3,7 @@ Copyright 2021-present Arcade Data Ltd; Python bindings copyright Taewoon Kim (h Licensed under the Apache License, Version 2.0 (licenses/Apache-2.0.txt, LICENSE). This wheel contains the ArcadeDB engine (https://github.com/ArcadeData/arcadedb, release named in -arcadedb_native.ENGINE_VERSION) compiled ahead of time with GraalVM Community Edition 25.4.4.1.1 into libarcadedb.so. +arcadedb.ENGINE_VERSION) compiled ahead of time with GraalVM Community Edition 25.4.4.1.1 into libarcadedb.so. The NOTICE file of ArcadeDB (components developed for OrientDB, the openCypher grammar) and the NOTICE files of the bundled Apache libraries (Lucene, Commons Lang, Micrometer, Swagger) are reproduced verbatim in THIRD_PARTY_NOTICES.md. @@ -12,7 +12,7 @@ GraalVM native-image runtime and JDK classes in libarcadedb.so into the library. These parts are licensed under the GNU General Public License version 2 WITH the "Classpath" Exception (licenses/GraalVM-CE-LICENSE.txt, licenses/GraalVM-native-image-LICENSE.txt, and verbatim in THIRD_PARTY_NOTICES.md). The exception lets you link and distribute them with code under other licences, and it is why this wheel as a whole stays Apache-2.0 for ArcadeDB's own code. The corresponding source: - GraalVM: https://github.com/oracle/graal (release vm-25.4.4.1.1 line; the build used community-25.4.4.1.1+1.1) + GraalVM: https://github.com/oracle/graal (release vm-25.4.4.1.1 line; the build used community-25.4.4.1.1+1.1, whose release file names commit 95ce1499c8c96ab7d5a6697c5b4bf42160f3b68b of substratevm) OpenJDK: https://github.com/openjdk/jdk (jdk-25 line used by that GraalVM build) Other GraalVM modules carry the licences in licenses/GraalVM-THIRD_PARTY_LICENSE.txt (graal-sdk and polyglot: UPL-1.0). Oracle GraalVM (GFTC) is not used; only Community Edition. @@ -22,4 +22,8 @@ libjvector-x86_64.so Java libraries, the Highway/jvector kernels and JDK data inside libarcadedb.so: THIRD_PARTY_NOTICES.md (generated by gen_notices.py). -System libraries linked dynamically and NOT shipped: libz.so.1, libm.so.6, libc.so.6, libstdc++.so.6, libgcc_s.so.1. +System libraries linked dynamically and NOT shipped: + Linux: libz.so.1, libm.so.6, libc.so.6, libpthread.so.0, libdl.so.2, librt.so.1 (libarcadedb.so, built in a manylinux_2_28 container); + libstdc++.so.6, libgcc_s.so.1 (libjvector-x86_64.so only) + macOS: the system libraries of macOS + Windows: the Microsoft Visual C++ runtime (VCRUNTIME140.dll, VCRUNTIME140_1.dll) and the Windows C runtime (api-ms-win-crt-*.dll); CPython itself needs the same files diff --git a/native/wheel/README.md b/native/wheel/README.md index b147b00..9fa5dd7 100644 --- a/native/wheel/README.md +++ b/native/wheel/README.md @@ -45,22 +45,22 @@ with arcadedb.create_database("/tmp/demo") as db: ## Platforms -CPython 3.11 to 3.14. One wheel per platform serves every Python version, because the library is loaded with `ctypes` and is not compiled against CPython. The images are built for the baseline CPU of the platform (`-march=compatibility`), so they do not need a special instruction set. +CPython 3.11 to 3.15. One wheel per platform serves every Python version, because the library is loaded with `ctypes` and is not compiled against CPython. The images are built for the baseline CPU of the platform (`-march=compatibility`), so they do not need a special instruction set. On 3.15 the CI runs a subset of the tests, because jpype1 and ladybug, two dependencies of the full test suite, have no 3.15 wheel yet. | Platform | Wheel tag | Note | | --- | --- | --- | -| Linux x86-64, glibc 2.34+ | `manylinux_2_34_x86_64` | | -| Linux arm64, glibc 2.34+ | `manylinux_2_34_aarch64` | | +| Linux x86-64, glibc 2.28+ | `manylinux_2_28_x86_64` | RHEL 8 and Rocky 8, Debian 10, Ubuntu 20.04 and newer | +| Linux arm64, glibc 2.28+ | `manylinux_2_28_aarch64` | | | macOS arm64 | `macosx__arm64` | | | Windows x64 | `win_amd64` | The newest wheel. It is built in CI, and its test run is the least proven of the four | -Intel macOS and Windows arm64 are not built. +Intel macOS, Windows arm64 and musl-based Linux (Alpine) are not built and not supported. On an Intel Mac use Docker or a Linux virtual machine; on Alpine use a glibc image such as `python:3.12-slim`. ## Not available or not yet - **JVM-only APIs do not exist**: `get_java_database()` and the other `get_java_*` calls, `start_jvm()`, `jvm_kwargs` and JVM options. There is no JVM. Use `arcadedb-jvm` if you need them. - **A few API calls raise `NotOnNativeYet`** (a `NotImplementedError`) instead of misbehaving: `AsyncExecutor.transaction`, `find_nearest_approximate` over product quantization, `get_metadata` of a foreign index, and `get_stats`. -- Free-threaded Python builds are not in the CI matrix. +- Free-threaded Python (3.14t) runs a subset of the tests in CI; it is informational and does not gate a release. The bindings' own tests run against the installed wheel (the parity run). What does not pass is the list above, or JVM-only by nature. diff --git a/native/wheel/build_wheel.py b/native/wheel/build_wheel.py index 26f6f24..81624c5 100644 --- a/native/wheel/build_wheel.py +++ b/native/wheel/build_wheel.py @@ -7,7 +7,7 @@ rewrite `import jpype` to the bundled stub; flip the default backend to native; add the library of the build platform (libarcadedb.so on Linux, libarcadedb.dylib on macOS, libarcadedb.dll on Windows; on Linux x86-64 also libjvector-x86_64.so, jvector's Highway library: no other platform ships it); add LICENSE/NOTICE/GraalVM licence texts; check THIRD_PARTY_NOTICES.md against gen_notices.py (fails on drift); compute the platform tag from the library -(Linux: manylinux__ from the GLIBC symbol versions; macOS: macosx__arm64 from the load commands; Windows: win_amd64); +(Linux: manylinux__ from the GLIBC symbol versions, never below --glibc-policy; macOS: macosx__arm64 from the load commands; Windows: win_amd64); build with `uv build --wheel`. Run it on the platform the library was built on. """ @@ -42,7 +42,7 @@ def host_arch() -> str: }.get(m, m) -def glibc_floor(lib: Path) -> str: +def glibc_floor(lib: Path, policy: str) -> str: out = subprocess.run( ["objdump", "-T", str(lib)], capture_output=True, text=True, check=True ).stdout @@ -50,7 +50,9 @@ def glibc_floor(lib: Path) -> str: tuple(map(int, m.split("."))) for m in re.findall(r"GLIBC_(\d+\.\d+(?:\.\d+)?)", out) } - major, minor = max(v[:2] for v in vers) + # never below the manylinux policy of the container the image was built in: that is what the wheel promises (PEP 600), and the + # symbols alone say less (libarcadedb.so built in manylinux_2_28 needs glibc 2.17 at most) + major, minor = max(max(v[:2] for v in vers), tuple(map(int, policy.split(".")))) return f"manylinux_{major}_{minor}_{host_arch()}" @@ -73,9 +75,9 @@ def macos_tag(lib: Path) -> str: return f"macosx_{m.group(1)}_{m.group(2)}_arm64" -def platform_tag(lib: Path) -> str: +def platform_tag(lib: Path, policy: str) -> str: if sys.platform.startswith("linux"): - return glibc_floor(lib) + return glibc_floor(lib, policy) if sys.platform == "darwin": return macos_tag(lib) if sys.platform == "win32": @@ -132,6 +134,11 @@ def main(): action="store_true", help="do not compare THIRD_PARTY_NOTICES.md with the gen_notices.py output (needs /build.log)", ) + ap.add_argument( + "--glibc-policy", + default="2.28", + help="Linux: glibc of the manylinux container the image is built in; the tag is never below it (default 2.28, manylinux_2_28)", + ) ap.add_argument("--plat", default="", help="override the computed platform tag") a = ap.parse_args() lib_dir, out = Path(a.lib_dir), Path(a.out).resolve() @@ -272,7 +279,7 @@ def sub(m): ) shutil.copy(HERE / "setup.py", stage / "setup.py") - plat = a.plat or platform_tag(lib_dir / libname) + plat = a.plat or platform_tag(lib_dir / libname, a.glibc_policy) print("platform tag:", plat) import os diff --git a/native/wheel/pyproject.toml b/native/wheel/pyproject.toml index d63ebd1..5fa7c55 100644 --- a/native/wheel/pyproject.toml +++ b/native/wheel/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Topic :: Database", ] dependencies = [] diff --git a/scripts/select_native_matrix.py b/scripts/select_native_matrix.py index be4cb22..e4c1f05 100644 --- a/scripts/select_native_matrix.py +++ b/scripts/select_native_matrix.py @@ -28,10 +28,10 @@ "linux-x86_64": { "kind": "linux", "runner": "ubuntu-latest", - "container": "quay.io/pypa/manylinux_2_34_x86_64:latest", + "container": "quay.io/pypa/manylinux_2_28_x86_64:latest", "graalvm_asset": "graalvm-community-jdk-25i4-25.0.4.1.1_linux-x64_bin.tar.gz", "graalvm_sha256": "05ccbbe783210b6886ff7b08fcd0b061c5dce4852b05db87284fc0e24abb08e2", - "plat_re": "manylinux_2_34_x86_64", + "plat_re": "manylinux_2_28_x86_64", "xmx": "12g", "ni_extra": "", "macos_target": "", @@ -41,10 +41,10 @@ "linux-aarch64": { "kind": "linux", "runner": "ubuntu-24.04-arm", - "container": "quay.io/pypa/manylinux_2_34_aarch64:latest", + "container": "quay.io/pypa/manylinux_2_28_aarch64:latest", "graalvm_asset": "graalvm-community-jdk-25i4-25.0.4.1.1_linux-aarch64_bin.tar.gz", "graalvm_sha256": "e5f5e2f59643cf96765c741dc00b206f86c69c8c1bf843fe26050c871e0e2dbc", - "plat_re": "manylinux_2_34_aarch64", + "plat_re": "manylinux_2_28_aarch64", "xmx": "12g", "ni_extra": "", "macos_target": "", @@ -84,6 +84,13 @@ # Python versions the wheel supports (the classifiers of native/wheel/pyproject.toml). ALL_PYTHONS = ["3.11", "3.12", "3.13", "3.14"] +# Pythons the wheel supports (classifiers too) that only the `pyver` job covers: the full parity run installs test dependencies that have no wheel +# for them yet (jpype1 1.7.1 and ladybug have no cp315 wheel; test_parameter_binding.py imports jpype at module level). The wheel itself is +# py3-none: the library is loaded with ctypes, so only the pure-Python layer depends on the interpreter. +PYVER_ONLY_PYTHONS = ["3.15"] +# Pythons whose pyver leg runs without jpype1, and so without the test files that need it (test-native.yml drops them) +NO_JPYPE_PYTHONS = ["3.15"] + # Platforms whose parity run must pass before a pre-release (A.B.C.devN) is published. Windows is the newest wheel and is # informational for a pre-release; a FINAL release needs all four (see ALL_PLATFORMS and release-native.yml). PRERELEASE_REQUIRED = ["linux-x86_64", "linux-aarch64", "macos-arm64"] @@ -101,14 +108,15 @@ def pyver_legs(covered: list[str]) -> list[dict]: - """The legs of the cheap Python-version job: every supported Python that `covered` (the full parity run) leaves out.""" + """The legs of the cheap Python-version job: every supported Python that `covered` (the full parity run) leaves out, 3.15 and the free-threaded 3.14t.""" return [ { "python": py, "freethreaded": py in FREE_THREADED_PYTHONS, "polars": py not in FREE_THREADED_PYTHONS, + "jpype": py not in NO_JPYPE_PYTHONS, } - for py in ALL_PYTHONS + FREE_THREADED_PYTHONS + for py in ALL_PYTHONS + PYVER_ONLY_PYTHONS + FREE_THREADED_PYTHONS if py not in covered ] diff --git a/scripts/smoke_wheel.py b/scripts/smoke_wheel.py new file mode 100644 index 0000000..e53249b --- /dev/null +++ b/scripts/smoke_wheel.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Smoke test of an installed native `arcadedb` wheel: import it, open a database, run one SQL and one openCypher query. + +test-native.yml runs it inside the manylinux_2_28 container (glibc 2.28), the oldest system the Linux wheels are tagged for. +""" + +import os +import tempfile + +import arcadedb + +print(f"arcadedb {arcadedb.__version__}, backend {arcadedb.__backend__}, engine {arcadedb.ENGINE_VERSION}") +assert arcadedb.__backend__ == "native", arcadedb.__backend__ +with tempfile.TemporaryDirectory() as d: + with arcadedb.create_database(os.path.join(d, "smoke")) as db: + assert db.query("sql", "SELECT 1 AS one").first().get("one") == 1 + with db.transaction(): + db.command("sql", "CREATE VERTEX TYPE P") + db.command("sql", "CREATE VERTEX P SET name = 'Ada'") + assert db.query("opencypher", "MATCH (n:P) RETURN n.name AS name").to_list()[0]["name"] == "Ada" +print("smoke ok") diff --git a/tests/test_native_wheel_template.py b/tests/test_native_wheel_template.py index 0808993..1e4a277 100644 --- a/tests/test_native_wheel_template.py +++ b/tests/test_native_wheel_template.py @@ -28,7 +28,12 @@ def pythons(path): if c.startswith("Programming Language :: Python :: 3.") ] - assert pythons(WHEEL / "pyproject.toml") == pythons(ROOT / "pyproject.toml") + # The native wheel is py3-none (ctypes) and also runs on Python 3.15; the JVM wheel cannot claim 3.15 until + # JPype ships cp315 wheels. That is the only allowed difference: the JVM wheel's set is the native one minus 3.15. + native, jvm = pythons(WHEEL / "pyproject.toml"), pythons(ROOT / "pyproject.toml") + native_only = {"Programming Language :: Python :: 3.15"} + assert [c for c in native if c not in native_only] == jvm + assert set(native) - set(jvm) <= native_only def test_the_pypi_description_names_the_sibling_in_its_first_paragraph(): diff --git a/tests/test_select_native_matrix.py b/tests/test_select_native_matrix.py index d015daf..993e958 100644 --- a/tests/test_select_native_matrix.py +++ b/tests/test_select_native_matrix.py @@ -50,7 +50,7 @@ def test_pythons_are_the_classifiers_of_the_native_wheel(): for c in text["project"]["classifiers"] if c.startswith(prefix) ] - assert pythons == sel.ALL_PYTHONS + assert pythons == sel.ALL_PYTHONS + sel.PYVER_ONLY_PYTHONS def test_graalvm_is_community_edition_with_a_sha256_per_platform(): @@ -73,34 +73,37 @@ def test_the_script_prints_github_output_lines(): "3.11", "3.13", "3.14", + "3.15", "3.14t", ] def _pyver(event): return [ - (leg["python"], leg["freethreaded"], leg["polars"]) + (leg["python"], leg["freethreaded"], leg["polars"], leg["jpype"]) for leg in sel.select(event)["pyver"] ] def test_the_python_version_job_covers_what_the_full_parity_run_leaves_out(): - # a pull request tests 3.12 only: the other three Pythons and the free-threaded one run the cheap subset + # a pull request tests 3.12 only: the other three Pythons, 3.15 (no jpype1 wheel yet) and the free-threaded one run the cheap subset assert _pyver("pull_request") == [ - ("3.11", False, True), - ("3.13", False, True), - ("3.14", False, True), - ("3.14t", True, False), + ("3.11", False, True, True), + ("3.13", False, True, True), + ("3.14", False, True, True), + ("3.15", False, True, False), + ("3.14t", True, False, True), ] # main, nightly and dispatch test 3.12 and 3.14 in full for event in ("push", "schedule", "workflow_dispatch"): assert _pyver(event) == [ - ("3.11", False, True), - ("3.13", False, True), - ("3.14t", True, False), + ("3.11", False, True, True), + ("3.13", False, True, True), + ("3.15", False, True, False), + ("3.14t", True, False, True), ] - # a release tests 3.11 to 3.14 in full: only the free-threaded build is left - assert _pyver("release") == [("3.14t", True, False)] + # a release tests 3.11 to 3.14 in full: 3.15 and the free-threaded build are left + assert _pyver("release") == [("3.15", False, True, False), ("3.14t", True, False, True)] def test_the_python_version_job_never_repeats_a_full_parity_leg_and_has_no_polars_on_a_free_threaded_python(): @@ -127,3 +130,15 @@ def test_the_release_does_not_need_the_python_version_job(): assert patterns and all( p in ("native-passed-*", "native-wheel-*") for p in patterns ), patterns + + +def test_the_linux_legs_build_in_the_manylinux_container_of_the_tag_and_the_workflow_gates_that_floor(): + linux = {k: leg for k, leg in sel.PLATFORMS.items() if leg["kind"] == "linux"} + assert set(linux) == {"linux-x86_64", "linux-aarch64"} + for key, leg in linux.items(): + arch = key.split("-", 1)[1] + assert leg["container"] == f"quay.io/pypa/manylinux_2_28_{arch}:latest", key + assert leg["plat_re"] == f"manylinux_2_28_{arch}", key + workflow = (ROOT / ".github" / "workflows" / "test-native.yml").read_text() + assert "GLIBC_2.28" in workflow + assert "GLIBC_2.34" not in workflow