GH-3894: Prepare RC: stage through nexus-staging-maven-plugin

pom.xml

@@ -103,6 +103,7 @@
     <net.openhft.version>0.27ea1</net.openhft.version>
     <build-helper-maven-plugin.version>3.6.2</build-helper-maven-plugin.version>
     <exec-maven-plugin.version>3.6.4</exec-maven-plugin.version>
+    <nexus-staging-maven-plugin.version>1.7.0</nexus-staging-maven-plugin.version>
     <protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
     <jts.version>1.20.0</jts.version>
 
@@ -725,6 +726,27 @@
       </properties>
     </profile>
 
+    <profile>
+      <id>apache-release</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.sonatype.plugins</groupId>
+            <artifactId>nexus-staging-maven-plugin</artifactId>
+            <version>${nexus-staging-maven-plugin.version}</version>
+            <extensions>true</extensions>
+            <configuration>
+              <nexusUrl>https://repository.apache.org/</nexusUrl>
+              <serverId>apache.releases.https</serverId>
+              <autoReleaseAfterClose>false</autoReleaseAfterClose>
+              <keepStagingRepositoryOnCloseRuleFailure>true</keepStagingRepositoryOnCloseRuleFailure>
+              <stagingProgressTimeoutMinutes>15</stagingProgressTimeoutMinutes>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <profile>
       <id>vector-plugins</id>
       <modules>

release/bin/prepare-rc.sh

@@ -263,27 +263,35 @@ step_summary "### Nexus Deployment"
 settings_file=$(mktemp "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/parquet-release-settings.XXXXXX")
 generate_maven_settings "${settings_file}"
 
-maven_deploy "${settings_file}"
-
-step_summary "Deployed artifacts to Apache Nexus staging"
-
-# Find and close the staging repo. Verify it actually contains the
-# artifacts we just deployed before closing -- another concurrent run
-# could have created the only open repo we see, and closing the wrong
-# one would corrupt that release.
-nexus_find_open_staging_repo "${NEXUS_PROFILE_NAME}"
+staging_description="Apache Parquet ${version} RC${rc_number}"
+
+if ! maven_deploy "${settings_file}" "${staging_description}"; then
+  if maven_find_created_staging_repos; then
+    print_error "Deploy failed after creating staging repository ${created_staging_repo_ids[*]}; check whether it is still in Nexus and drop it before retrying."
+    step_summary "Deploy **FAILED** after creating staging repository \`${created_staging_repo_ids[*]}\`; check Nexus for leftovers"
+  else
+    step_summary "Deploy **FAILED** before a staging repository was created"
+  fi
+  exit 1
+fi
 
-if [[ ${DRY_RUN:-1} -ne 1 ]]; then
-  if ! nexus_check_staging_artifact "${staging_repo_id}" "${version}"; then
-    print_error "Open staging repo ${staging_repo_id} does not contain ${version} artifacts."
-    print_error "Refusing to close it -- another release may be in flight. Investigate Nexus before retrying."
-    exit 1
+if ! maven_read_staging_repo_id; then
+  if maven_find_created_staging_repos; then
+    print_error "Deploy succeeded but the staging repository ID could not be read; the deploy log reports creating ${created_staging_repo_ids[*]}."
+    step_summary "Deploy succeeded but the staging repository ID could not be read; deploy log reports \`${created_staging_repo_ids[*]}\`"
+  else
+    print_error "Deploy succeeded but no staging repository was found; nexus-staging-maven-plugin may not have run, so check Nexus for an implicitly created repository."
+    step_summary "Deploy succeeded but no staging repository was found; check Nexus for an implicitly created repository"
   fi
+  exit 1
 fi
 
-nexus_close_staging_repo "${staging_repo_id}" "Apache Parquet ${version} RC${rc_number}"
+if ! nexus_verify_staging_repo "${staging_repo_id}" "${version}" "${rc_number}"; then
+  step_summary "Verification **FAILED** for staging repository \`${staging_repo_id}\`"
+  exit 1
+fi
 
-step_summary "Closed staging repository: \`${staging_repo_id}\`"
+step_summary "Deployed, closed and verified staging repository: \`${staging_repo_id}\`"
 
 maven_cleanup_settings "${settings_file}"
 

release/libs/_constants.sh

@@ -33,6 +33,8 @@ NEXUS_CONTENT_BASE_URL=${NEXUS_CONTENT_BASE_URL:-"${NEXUS_BASE_URL}/repositories
 NEXUS_STAGING_GROUP_URL="https://repository.apache.org/content/groups/staging/org/apache/parquet/"
 
 NEXUS_PROFILE_NAME="org.apache.parquet"
+NEXUS_LOCAL_STAGING_DIR="target/nexus-staging"
+MAVEN_DEPLOY_LOG="target/nexus-deploy.log"
 NEXUS_VERIFY_GROUP_PATH="org/apache/parquet"
 NEXUS_VERIFY_ARTIFACT_ID="parquet-common"
 

release/libs/_maven.sh

@@ -62,20 +62,76 @@ SETTINGS_EOF
   print_info "Generated Maven settings at ${settings_file} (credentials resolved from env vars at build time)"
 }
 
+# maven_deploy <settings-file> <staging-description>
+#   Deploys through nexus-staging-maven-plugin, which opens one staging
+#   repository, uploads every module into it, and closes it. Output is
+#   also written to MAVEN_DEPLOY_LOG.
 function maven_deploy {
   local settings_file="${1:-.release-settings.xml}"
+  local staging_description="$2"
 
   if [[ ! -f "${settings_file}" ]]; then
     print_info "Generating Maven settings..."
     generate_maven_settings "${settings_file}"
   fi
 
+  exec_process rm -rf "${NEXUS_LOCAL_STAGING_DIR}"
+  mkdir -p "$(dirname "${MAVEN_DEPLOY_LOG}")"
+
   exec_process ./mvnw deploy \
     -Papache-release \
     -DskipTests \
     -Darguments=-DskipTests \
+    "-DstagingDescription=${staging_description}" \
     --settings "${settings_file}" \
-    --batch-mode
+    --batch-mode | tee "${MAVEN_DEPLOY_LOG}"
+  return "${PIPESTATUS[0]}"
+}
+
+# maven_read_staging_repo_id [local-staging-dir]
+#   Sets staging_repo_id from the properties file nexus-staging-maven-plugin
+#   writes after uploading, before it attempts the close.
+function maven_read_staging_repo_id {
+  local staging_dir="${1:-${NEXUS_LOCAL_STAGING_DIR}}/staging"
+
+  if [[ ${DRY_RUN:-1} -eq 1 ]]; then
+    print_command "Dry-run, WOULD read staging repository ID from ${staging_dir}/*.properties"
+    staging_repo_id="DRY-RUN-REPO-ID"
+    return 0
+  fi
+
+  local properties_files=("${staging_dir}"/*.properties)
+  if [[ ${#properties_files[@]} -ne 1 || ! -f "${properties_files[0]}" ]]; then
+    print_error "Expected one staging properties file in ${staging_dir}, found: ${properties_files[*]}"
+    return 1
+  fi
+
+  staging_repo_id=$(sed -n 's/^stagingRepository\.id=//p' "${properties_files[0]}")
+  if [[ -z "${staging_repo_id}" ]]; then
+    print_error "No stagingRepository.id in ${properties_files[0]}"
+    return 1
+  fi
+
+  print_info "Staging repository: ${staging_repo_id}"
+  return 0
+}
+
+# maven_find_created_staging_repos [deploy-log]
+#   Sets created_staging_repo_ids to every staging repository the plugin
+#   reported creating. Used after a failed deploy, when the plugin may
+#   already have deleted its properties file.
+function maven_find_created_staging_repos {
+  local deploy_log="${1:-${MAVEN_DEPLOY_LOG}}"
+
+  created_staging_repo_ids=()
+  if [[ -f "${deploy_log}" ]]; then
+    local repo_id
+    while IFS= read -r repo_id; do
+      created_staging_repo_ids+=("${repo_id}")
+    done < <(sed -n 's/.*Created staging repository with ID "\([^"]*\)".*/\1/p' "${deploy_log}")
+  fi
+
+  [[ ${#created_staging_repo_ids[@]} -gt 0 ]]
 }
 
 function maven_cleanup_settings {

release/libs/_nexus.sh

@@ -50,12 +50,6 @@ function _nexus_bulk_action {
   fi
 }
 
-function nexus_close_staging_repo {
-  local repo_id="$1"
-  local description="${2:-Closing staging repository}"
-  _nexus_bulk_action "close" "${repo_id}" "${description}"
-}
-
 function nexus_release_staging_repo {
   local repo_id="$1"
   local description="${2:-Releasing staging repository}"
@@ -170,71 +164,3 @@ function nexus_verify_staging_repo {
   print_info "Staging repository ${repo_id} verified."
   return 0
 }
-
-function nexus_find_open_staging_repo {
-  local profile_name="${1:-${NEXUS_PROFILE_NAME}}"
-
-  print_info "Searching for open staging repository for ${profile_name}..."
-
-  if [[ ${DRY_RUN:-1} -eq 1 ]]; then
-    print_command "Dry-run, WOULD search Nexus for open staging repo"
-    staging_repo_id="DRY-RUN-REPO-ID"
-    return 0
-  fi
-
-  local response
-  if ! response=$(curl --fail --silent --show-error \
-    -K <(printf 'user = "%s:%s"\n' "${NEXUS_USERNAME}" "${NEXUS_PASSWORD}") \
-    "${NEXUS_BASE_URL}/staging/profile_repositories"); then
-    print_error "Failed to query Nexus staging repositories"
-    return 1
-  fi
-
-  # Filter on <profileName> exactly (not on a substring of <repositoryId>),
-  # and reject the result if multiple open repos exist for the profile —
-  # Nexus permits this but it indicates a leftover from a previous failed
-  # run that the operator must clean up before we can pick the right one.
-  local matches
-  matches=$(echo "${response}" | \
-    NEXUS_PROFILE_NAME_VAR="${profile_name}" python3 -c "
-import sys, os, xml.etree.ElementTree as ET
-profile = os.environ['NEXUS_PROFILE_NAME_VAR']
-try:
-    tree = ET.parse(sys.stdin)
-except ET.ParseError as e:
-    print('PARSE_ERROR:' + str(e), file=sys.stderr)
-    sys.exit(2)
-for repo in tree.findall('.//stagingProfileRepository'):
-    repo_type = repo.find('type')
-    repo_profile = repo.find('profileName')
-    repo_id = repo.find('repositoryId')
-    if repo_type is None or repo_id is None or repo_profile is None:
-        continue
-    if repo_type.text == 'open' and repo_profile.text == profile:
-        print(repo_id.text)
-" 2>&1)
-  local py_status=$?
-
-  if [[ ${py_status} -ne 0 ]]; then
-    print_error "Failed to parse Nexus staging response: ${matches}"
-    return 1
-  fi
-
-  local match_count
-  match_count=$(echo -n "${matches}" | grep -c . || true)
-
-  if [[ ${match_count} -eq 0 ]]; then
-    print_error "No open staging repository found for profile '${profile_name}'"
-    return 1
-  fi
-  if [[ ${match_count} -gt 1 ]]; then
-    print_error "Multiple open staging repositories found for profile '${profile_name}':"
-    echo "${matches}" | while read -r line; do print_error "  - ${line}"; done
-    print_error "Please drop or close the stale repos in Nexus and re-run."
-    return 1
-  fi
-
-  staging_repo_id=$(echo "${matches}" | head -n 1)
-  print_info "Found staging repository: ${staging_repo_id}"
-  return 0
-}