diff --git a/pom.xml b/pom.xml index ebb7fbd849..ab966051fa 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ 0.27ea1 3.6.2 3.6.4 + 1.7.0 3.11.4 1.20.0 @@ -725,6 +726,27 @@ + + apache-release + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus-staging-maven-plugin.version} + true + + https://repository.apache.org/ + apache.releases.https + false + true + 15 + + + + + + vector-plugins diff --git a/release/bin/prepare-rc.sh b/release/bin/prepare-rc.sh index 0c16a78012..5222ef8500 100755 --- a/release/bin/prepare-rc.sh +++ b/release/bin/prepare-rc.sh @@ -263,27 +263,35 @@ step_summary "### Nexus Deployment" settings_file=$(mktemp "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/parquet-release-settings.XXXXXX") generate_maven_settings "${settings_file}" -maven_deploy "${settings_file}" - -step_summary "Deployed artifacts to Apache Nexus staging" - -# Find and close the staging repo. Verify it actually contains the -# artifacts we just deployed before closing -- another concurrent run -# could have created the only open repo we see, and closing the wrong -# one would corrupt that release. -nexus_find_open_staging_repo "${NEXUS_PROFILE_NAME}" +staging_description="Apache Parquet ${version} RC${rc_number}" + +if ! maven_deploy "${settings_file}" "${staging_description}"; then + if maven_find_created_staging_repos; then + print_error "Deploy failed after creating staging repository ${created_staging_repo_ids[*]}; check whether it is still in Nexus and drop it before retrying." + step_summary "Deploy **FAILED** after creating staging repository \`${created_staging_repo_ids[*]}\`; check Nexus for leftovers" + else + step_summary "Deploy **FAILED** before a staging repository was created" + fi + exit 1 +fi -if [[ ${DRY_RUN:-1} -ne 1 ]]; then - if ! nexus_check_staging_artifact "${staging_repo_id}" "${version}"; then - print_error "Open staging repo ${staging_repo_id} does not contain ${version} artifacts." - print_error "Refusing to close it -- another release may be in flight. Investigate Nexus before retrying." - exit 1 +if ! maven_read_staging_repo_id; then + if maven_find_created_staging_repos; then + print_error "Deploy succeeded but the staging repository ID could not be read; the deploy log reports creating ${created_staging_repo_ids[*]}." + step_summary "Deploy succeeded but the staging repository ID could not be read; deploy log reports \`${created_staging_repo_ids[*]}\`" + else + print_error "Deploy succeeded but no staging repository was found; nexus-staging-maven-plugin may not have run, so check Nexus for an implicitly created repository." + step_summary "Deploy succeeded but no staging repository was found; check Nexus for an implicitly created repository" fi + exit 1 fi -nexus_close_staging_repo "${staging_repo_id}" "Apache Parquet ${version} RC${rc_number}" +if ! nexus_verify_staging_repo "${staging_repo_id}" "${version}" "${rc_number}"; then + step_summary "Verification **FAILED** for staging repository \`${staging_repo_id}\`" + exit 1 +fi -step_summary "Closed staging repository: \`${staging_repo_id}\`" +step_summary "Deployed, closed and verified staging repository: \`${staging_repo_id}\`" maven_cleanup_settings "${settings_file}" diff --git a/release/libs/_constants.sh b/release/libs/_constants.sh index 78c3978f26..108c21825d 100644 --- a/release/libs/_constants.sh +++ b/release/libs/_constants.sh @@ -33,6 +33,8 @@ NEXUS_CONTENT_BASE_URL=${NEXUS_CONTENT_BASE_URL:-"${NEXUS_BASE_URL}/repositories NEXUS_STAGING_GROUP_URL="https://repository.apache.org/content/groups/staging/org/apache/parquet/" NEXUS_PROFILE_NAME="org.apache.parquet" +NEXUS_LOCAL_STAGING_DIR="target/nexus-staging" +MAVEN_DEPLOY_LOG="target/nexus-deploy.log" NEXUS_VERIFY_GROUP_PATH="org/apache/parquet" NEXUS_VERIFY_ARTIFACT_ID="parquet-common" diff --git a/release/libs/_maven.sh b/release/libs/_maven.sh index ddb916adc4..dc89bdcb0b 100644 --- a/release/libs/_maven.sh +++ b/release/libs/_maven.sh @@ -62,20 +62,76 @@ SETTINGS_EOF print_info "Generated Maven settings at ${settings_file} (credentials resolved from env vars at build time)" } +# maven_deploy +# Deploys through nexus-staging-maven-plugin, which opens one staging +# repository, uploads every module into it, and closes it. Output is +# also written to MAVEN_DEPLOY_LOG. function maven_deploy { local settings_file="${1:-.release-settings.xml}" + local staging_description="$2" if [[ ! -f "${settings_file}" ]]; then print_info "Generating Maven settings..." generate_maven_settings "${settings_file}" fi + exec_process rm -rf "${NEXUS_LOCAL_STAGING_DIR}" + mkdir -p "$(dirname "${MAVEN_DEPLOY_LOG}")" + exec_process ./mvnw deploy \ -Papache-release \ -DskipTests \ -Darguments=-DskipTests \ + "-DstagingDescription=${staging_description}" \ --settings "${settings_file}" \ - --batch-mode + --batch-mode | tee "${MAVEN_DEPLOY_LOG}" + return "${PIPESTATUS[0]}" +} + +# maven_read_staging_repo_id [local-staging-dir] +# Sets staging_repo_id from the properties file nexus-staging-maven-plugin +# writes after uploading, before it attempts the close. +function maven_read_staging_repo_id { + local staging_dir="${1:-${NEXUS_LOCAL_STAGING_DIR}}/staging" + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_command "Dry-run, WOULD read staging repository ID from ${staging_dir}/*.properties" + staging_repo_id="DRY-RUN-REPO-ID" + return 0 + fi + + local properties_files=("${staging_dir}"/*.properties) + if [[ ${#properties_files[@]} -ne 1 || ! -f "${properties_files[0]}" ]]; then + print_error "Expected one staging properties file in ${staging_dir}, found: ${properties_files[*]}" + return 1 + fi + + staging_repo_id=$(sed -n 's/^stagingRepository\.id=//p' "${properties_files[0]}") + if [[ -z "${staging_repo_id}" ]]; then + print_error "No stagingRepository.id in ${properties_files[0]}" + return 1 + fi + + print_info "Staging repository: ${staging_repo_id}" + return 0 +} + +# maven_find_created_staging_repos [deploy-log] +# Sets created_staging_repo_ids to every staging repository the plugin +# reported creating. Used after a failed deploy, when the plugin may +# already have deleted its properties file. +function maven_find_created_staging_repos { + local deploy_log="${1:-${MAVEN_DEPLOY_LOG}}" + + created_staging_repo_ids=() + if [[ -f "${deploy_log}" ]]; then + local repo_id + while IFS= read -r repo_id; do + created_staging_repo_ids+=("${repo_id}") + done < <(sed -n 's/.*Created staging repository with ID "\([^"]*\)".*/\1/p' "${deploy_log}") + fi + + [[ ${#created_staging_repo_ids[@]} -gt 0 ]] } function maven_cleanup_settings { diff --git a/release/libs/_nexus.sh b/release/libs/_nexus.sh index d56adff7d4..60ed3e5044 100644 --- a/release/libs/_nexus.sh +++ b/release/libs/_nexus.sh @@ -50,12 +50,6 @@ function _nexus_bulk_action { fi } -function nexus_close_staging_repo { - local repo_id="$1" - local description="${2:-Closing staging repository}" - _nexus_bulk_action "close" "${repo_id}" "${description}" -} - function nexus_release_staging_repo { local repo_id="$1" local description="${2:-Releasing staging repository}" @@ -170,71 +164,3 @@ function nexus_verify_staging_repo { print_info "Staging repository ${repo_id} verified." return 0 } - -function nexus_find_open_staging_repo { - local profile_name="${1:-${NEXUS_PROFILE_NAME}}" - - print_info "Searching for open staging repository for ${profile_name}..." - - if [[ ${DRY_RUN:-1} -eq 1 ]]; then - print_command "Dry-run, WOULD search Nexus for open staging repo" - staging_repo_id="DRY-RUN-REPO-ID" - return 0 - fi - - local response - if ! response=$(curl --fail --silent --show-error \ - -K <(printf 'user = "%s:%s"\n' "${NEXUS_USERNAME}" "${NEXUS_PASSWORD}") \ - "${NEXUS_BASE_URL}/staging/profile_repositories"); then - print_error "Failed to query Nexus staging repositories" - return 1 - fi - - # Filter on exactly (not on a substring of ), - # and reject the result if multiple open repos exist for the profile — - # Nexus permits this but it indicates a leftover from a previous failed - # run that the operator must clean up before we can pick the right one. - local matches - matches=$(echo "${response}" | \ - NEXUS_PROFILE_NAME_VAR="${profile_name}" python3 -c " -import sys, os, xml.etree.ElementTree as ET -profile = os.environ['NEXUS_PROFILE_NAME_VAR'] -try: - tree = ET.parse(sys.stdin) -except ET.ParseError as e: - print('PARSE_ERROR:' + str(e), file=sys.stderr) - sys.exit(2) -for repo in tree.findall('.//stagingProfileRepository'): - repo_type = repo.find('type') - repo_profile = repo.find('profileName') - repo_id = repo.find('repositoryId') - if repo_type is None or repo_id is None or repo_profile is None: - continue - if repo_type.text == 'open' and repo_profile.text == profile: - print(repo_id.text) -" 2>&1) - local py_status=$? - - if [[ ${py_status} -ne 0 ]]; then - print_error "Failed to parse Nexus staging response: ${matches}" - return 1 - fi - - local match_count - match_count=$(echo -n "${matches}" | grep -c . || true) - - if [[ ${match_count} -eq 0 ]]; then - print_error "No open staging repository found for profile '${profile_name}'" - return 1 - fi - if [[ ${match_count} -gt 1 ]]; then - print_error "Multiple open staging repositories found for profile '${profile_name}':" - echo "${matches}" | while read -r line; do print_error " - ${line}"; done - print_error "Please drop or close the stale repos in Nexus and re-run." - return 1 - fi - - staging_repo_id=$(echo "${matches}" | head -n 1) - print_info "Found staging repository: ${staging_repo_id}" - return 0 -} diff --git a/release/tests/maven.bats b/release/tests/maven.bats index 9dff0e6a9a..b0a65096c3 100644 --- a/release/tests/maven.bats +++ b/release/tests/maven.bats @@ -24,6 +24,7 @@ setup() { export NEXUS_USERNAME="testuser" export NEXUS_PASSWORD="testpass" TEST_TMPDIR=$(mktemp -d) + MAVEN_DEPLOY_LOG="${TEST_TMPDIR}/target/nexus-deploy.log" } teardown() { @@ -82,7 +83,7 @@ teardown() { DRY_RUN=1 local settings_file="${TEST_TMPDIR}/settings.xml" generate_maven_settings "${settings_file}" - run maven_deploy "${settings_file}" + run maven_deploy "${settings_file}" "Apache Parquet 1.2.3 RC4" [ "$status" -eq 0 ] [[ "$output" == *"Dry-run"* ]] [[ "$output" == *"deploy"* ]] @@ -92,7 +93,7 @@ teardown() { DRY_RUN=1 local settings_file="${TEST_TMPDIR}/settings.xml" generate_maven_settings "${settings_file}" - run maven_deploy "${settings_file}" + run maven_deploy "${settings_file}" "Apache Parquet 1.2.3 RC4" [[ "$output" == *"apache-release"* ]] } @@ -100,10 +101,86 @@ teardown() { DRY_RUN=1 local settings_file="${TEST_TMPDIR}/settings.xml" generate_maven_settings "${settings_file}" - run maven_deploy "${settings_file}" + run maven_deploy "${settings_file}" "Apache Parquet 1.2.3 RC4" [[ "$output" == *"skipTests"* ]] } +@test "maven_deploy: passes staging description in dry-run output" { + DRY_RUN=1 + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + run maven_deploy "${settings_file}" "Apache Parquet 1.2.3 RC4" + [[ "$output" == *"-DstagingDescription=Apache Parquet 1.2.3 RC4"* ]] +} + +@test "maven_deploy: clears local staging directory in dry-run output" { + DRY_RUN=1 + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + run maven_deploy "${settings_file}" "Apache Parquet 1.2.3 RC4" + [[ "$output" == *"rm -rf target/nexus-staging"* ]] +} + +@test "maven_deploy: real mode passes description as one argument, logs output, and propagates failure" { + DRY_RUN=0 + cd "${TEST_TMPDIR}" + mkdir -p target/nexus-staging/staging + touch target/nexus-staging/staging/stale.properties + cat > mvnw <<'EOF' +#!/bin/bash +printf '%s\n' "$@" > mvnw-args.txt +echo ' * Created staging repository with ID "orgapacheparquet-1091".' +exit 3 +EOF + chmod +x mvnw + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + run maven_deploy "${settings_file}" "Apache Parquet 1.2.3 RC4" + [ "$status" -eq 3 ] + [ ! -e target/nexus-staging ] + grep -qx -- "-DstagingDescription=Apache Parquet 1.2.3 RC4" mvnw-args.txt + grep -q 'orgapacheparquet-1091' "${MAVEN_DEPLOY_LOG}" +} + +# ---- maven_read_staging_repo_id ---- + +@test "maven_read_staging_repo_id: dry-run returns placeholder" { + DRY_RUN=1 + maven_read_staging_repo_id "${TEST_TMPDIR}" + [ "${staging_repo_id}" = "DRY-RUN-REPO-ID" ] +} + +@test "maven_read_staging_repo_id: reads ID from plugin properties file" { + DRY_RUN=0 + mkdir -p "${TEST_TMPDIR}/staging" + cat > "${TEST_TMPDIR}/staging/1ab2c3d4e5f6.properties" <<'EOF' +#Generated by org.sonatype.plugins:nexus-staging-maven-plugin:1.7.0 +stagingRepository.managed=true +stagingRepository.id=orgapacheparquet-1090 +stagingRepository.profileId=1ab2c3d4e5f6 +stagingRepository.url=https\://repository.apache.org/content/repositories/orgapacheparquet-1090 +EOF + maven_read_staging_repo_id "${TEST_TMPDIR}" + [ "${staging_repo_id}" = "orgapacheparquet-1090" ] +} + +@test "maven_read_staging_repo_id: fails when no properties file exists" { + DRY_RUN=0 + mkdir -p "${TEST_TMPDIR}/staging" + run maven_read_staging_repo_id "${TEST_TMPDIR}" + [ "$status" -eq 1 ] + [[ "$output" == *"Expected one staging properties file"* ]] +} + +@test "maven_read_staging_repo_id: fails when properties file has no ID" { + DRY_RUN=0 + mkdir -p "${TEST_TMPDIR}/staging" + echo "stagingRepository.profileId=1ab2c3d4e5f6" > "${TEST_TMPDIR}/staging/1ab2c3d4e5f6.properties" + run maven_read_staging_repo_id "${TEST_TMPDIR}" + [ "$status" -eq 1 ] + [[ "$output" == *"No stagingRepository.id"* ]] +} + # ---- maven_cleanup_settings ---- @test "maven_cleanup_settings: removes settings file" { @@ -117,3 +194,49 @@ teardown() { run maven_cleanup_settings "${TEST_TMPDIR}/nonexistent.xml" [ "$status" -eq 0 ] } + +@test "maven_read_staging_repo_id: fails when more than one properties file exists" { + DRY_RUN=0 + mkdir -p "${TEST_TMPDIR}/staging" + echo "stagingRepository.id=orgapacheparquet-1" > "${TEST_TMPDIR}/staging/a.properties" + echo "stagingRepository.id=orgapacheparquet-2" > "${TEST_TMPDIR}/staging/b.properties" + run maven_read_staging_repo_id "${TEST_TMPDIR}" + [ "$status" -eq 1 ] + [[ "$output" == *"a.properties"* ]] + [[ "$output" == *"b.properties"* ]] +} + +@test "maven_read_staging_repo_id: defaults to the plugin's staging directory" { + DRY_RUN=0 + cd "${TEST_TMPDIR}" + mkdir -p target/nexus-staging/staging + echo "stagingRepository.id=orgapacheparquet-1092" > target/nexus-staging/staging/x.properties + maven_read_staging_repo_id + [ "${staging_repo_id}" = "orgapacheparquet-1092" ] +} + +# ---- maven_find_created_staging_repos ---- + +@test "maven_find_created_staging_repos: finds repository IDs in the deploy log" { + mkdir -p "$(dirname "${MAVEN_DEPLOY_LOG}")" + cat > "${MAVEN_DEPLOY_LOG}" <<'EOF' +[INFO] * Created staging repository with ID "orgapacheparquet-1093". +[ERROR] Rule failure while trying to close staging repository with ID "orgapacheparquet-1093". +[ERROR] * Not dropping failed staging repository with ID "orgapacheparquet-1093" (Rule failure). +EOF + maven_find_created_staging_repos + [ "${#created_staging_repo_ids[@]}" -eq 1 ] + [ "${created_staging_repo_ids[0]}" = "orgapacheparquet-1093" ] +} + +@test "maven_find_created_staging_repos: fails when no repository was created" { + mkdir -p "$(dirname "${MAVEN_DEPLOY_LOG}")" + echo "[ERROR] COMPILATION ERROR" > "${MAVEN_DEPLOY_LOG}" + run maven_find_created_staging_repos + [ "$status" -eq 1 ] +} + +@test "maven_find_created_staging_repos: fails when the deploy log is missing" { + run maven_find_created_staging_repos "${TEST_TMPDIR}/missing.log" + [ "$status" -eq 1 ] +} diff --git a/release/tests/nexus.bats b/release/tests/nexus.bats index 6ac108b513..be9c4e6806 100644 --- a/release/tests/nexus.bats +++ b/release/tests/nexus.bats @@ -25,22 +25,6 @@ setup() { export NEXUS_PASSWORD="testpass" } -# ---- nexus_close_staging_repo ---- - -@test "nexus_close_staging_repo: dry-run does not call curl" { - DRY_RUN=1 - run nexus_close_staging_repo "orgapacheparquet-1234" "test close" - [ "$status" -eq 0 ] - [[ "$output" == *"Dry-run"* ]] - [[ "$output" == *"close"* ]] -} - -@test "nexus_close_staging_repo: constructs correct URL in dry-run" { - DRY_RUN=1 - run nexus_close_staging_repo "orgapacheparquet-1234" - [[ "$output" == *"staging/bulk/close"* ]] -} - # ---- nexus_release_staging_repo ---- @test "nexus_release_staging_repo: dry-run includes promote URL" { @@ -72,129 +56,8 @@ setup() { [[ "$output" == *"example.com/nexus/staging/bulk/drop"* ]] } -# ---- nexus_find_open_staging_repo ---- - -@test "nexus_find_open_staging_repo: dry-run returns placeholder" { - DRY_RUN=1 - nexus_find_open_staging_repo "org.apache.parquet" - [ "$staging_repo_id" = "DRY-RUN-REPO-ID" ] -} - -@test "nexus_find_open_staging_repo: real mode picks the open repo for the profile" { - DRY_RUN=0 - curl() { - cat <<'XML' - - - - org.apache.iceberg - open - orgapacheiceberg-7777 - - - org.apache.parquet - closed - orgapacheparquet-1111 - - - org.apache.parquet - open - orgapacheparquet-2222 - - - -XML - return 0 - } - export -f curl - nexus_find_open_staging_repo "org.apache.parquet" - [ "${staging_repo_id}" = "orgapacheparquet-2222" ] -} - -@test "nexus_find_open_staging_repo: rejects when no open repo for profile" { - DRY_RUN=0 - curl() { - cat <<'XML' - - - - org.apache.parquet - closed - orgapacheparquet-1111 - - - -XML - return 0 - } - export -f curl - run nexus_find_open_staging_repo "org.apache.parquet" - [ "$status" -eq 1 ] - [[ "$output" == *"No open staging repository"* ]] -} - -@test "nexus_find_open_staging_repo: rejects when profile has multiple open repos" { - DRY_RUN=0 - curl() { - cat <<'XML' - - - - org.apache.parquet - open - orgapacheparquet-1111 - - - org.apache.parquet - open - orgapacheparquet-2222 - - - -XML - return 0 - } - export -f curl - run nexus_find_open_staging_repo "org.apache.parquet" - [ "$status" -eq 1 ] - [[ "$output" == *"Multiple open staging repositories"* ]] - [[ "$output" == *"orgapacheparquet-1111"* ]] - [[ "$output" == *"orgapacheparquet-2222"* ]] -} - -@test "nexus_find_open_staging_repo: rejects when curl fails" { - DRY_RUN=0 - curl() { return 22; } - export -f curl - run nexus_find_open_staging_repo "org.apache.parquet" - [ "$status" -eq 1 ] - [[ "$output" == *"Failed to query"* ]] -} - -@test "nexus_find_open_staging_repo: rejects malformed XML" { - DRY_RUN=0 - curl() { echo "&2 - return 0 - } - export -f curl - run nexus_close_staging_repo "orgapacheparquet-9999" "test desc" - [ "$status" -eq 0 ] - [[ "$output" == *"staging/bulk/close"* ]] - [[ "$output" == *"orgapacheparquet-9999"* ]] -} - @test "nexus_drop_staging_repo: real mode calls curl with auth" { DRY_RUN=0 curl() {