Build/Test Tools: Harden CI setup against flaky downloads

.github/actions/download-with-retry/action.yml

@@ -0,0 +1,23 @@
+name: 'Download with retry'
+description: 'Downloads a file with curl, retrying on any error.'
+
+inputs:
+  url:
+    description: 'The URL to download.'
+    required: true
+  output:
+    description: 'The path to save the file to, relative to the workspace.'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Download the file
+      shell: bash
+      # A stalled attempt (connecting for over 30 s, or under 1 KB/s for 30 s) fails and is
+      # retried: curl has no transfer timeout, and `timeout-minutes` would end the step unretried.
+      # No retry starts after 10 minutes, so a long `Retry-After` can't stretch the waits.
+      run: curl --fail --no-progress-meter --location --retry 5 --retry-all-errors --retry-max-time 600 --connect-timeout 30 --speed-limit 1024 --speed-time 30 --output "$OUTPUT" "$URL"
+      env:
+        URL: ${{ inputs.url }}
+        OUTPUT: ${{ inputs.output }}

.github/actions/run-with-retry/action.yml

@@ -0,0 +1,28 @@
+name: 'Run with retry'
+description: 'Runs a Bash command, retrying it when it fails. For downloads curl can retry itself, use download-with-retry.'
+
+inputs:
+  command:
+    description: 'The Bash command to run. It runs as code: read values from the `env` of the calling step, not from expressions.'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Run the command
+      shell: bash
+      # Waits 10 s, then 20 s, between attempts, like `tools/local-env/scripts/utils.js`.
+      run: |
+        for (( attempt = 1; attempt <= MAX_ATTEMPTS; attempt++ )); do
+          if bash -eo pipefail -c "$COMMAND"; then
+            exit 0
+          fi
+          echo "::warning::Command failed on attempt $attempt of $MAX_ATTEMPTS."
+          if (( attempt < MAX_ATTEMPTS )); then
+            sleep $(( attempt * 10 ))
+          fi
+        done
+        exit 1
+      env:
+        COMMAND: ${{ inputs.command }}
+        MAX_ATTEMPTS: 3

.github/actions/setup-npm/action.yml

@@ -18,11 +18,22 @@ runs:
       id: npm
       env:
         WORKING_DIRECTORY: ${{ inputs.working-directory }}
+        NPM_INSTALL_MAX_ATTEMPTS: 3
       run: |
         NPM_VERSION="$(jq -r '.devEngines.packageManager.version // empty' package.json)"
         if [ -n "$NPM_VERSION" ]; then
           # Contributors need the floor; CI stays on that major's newest release.
-          npm install --global "npm@${NPM_VERSION/>=/^}"
+          # The registry can reset the connection mid-download (ECONNRESET), so retry.
+          for (( attempt = 1; attempt <= NPM_INSTALL_MAX_ATTEMPTS; attempt++ )); do
+            if npm install --global "npm@${NPM_VERSION/>=/^}"; then
+              break
+            fi
+            echo "::warning::npm install failed on attempt $attempt of $NPM_INSTALL_MAX_ATTEMPTS."
+            if (( attempt == NPM_INSTALL_MAX_ATTEMPTS )); then
+              exit 1
+            fi
+            sleep $(( attempt * 10 ))
+          done
           echo "Required npm version: $NPM_VERSION"
         else
           echo "No devEngines.packageManager.version, keeping the bundled npm."

.github/workflows/end-to-end-tests.yml

@@ -44,6 +44,7 @@ on:
       - 'tools/vendors/**'
       - 'tools/webpack/**'
       - '.github/actions/setup-npm/**'
+      - '.github/actions/download-with-retry/**'
       - '.github/workflows/end-to-end-tests.yml'
       - '.github/workflows/reusable-end-to-end-tests*.yml'
   workflow_dispatch:

.github/workflows/install-testing.yml

@@ -13,6 +13,7 @@ on:
     # Always test the workflow after it's updated.
     paths:
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.github/workflows/install-testing.yml'
       - '.version-support-*.json'
@@ -21,6 +22,7 @@ on:
     # Always test the workflow when changes are suggested.
     paths:
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.version-support-*.json'
       - '.github/workflows/install-testing.yml'
@@ -143,11 +145,15 @@ jobs:
       # WP-CLI 2.x extracts tarballs with PharData, which truncates paths longer than 100 characters.
       # Zip archives are not affected, so download the zip for every version.
       - name: Download WordPress
-        run: |
-          case "${WP_VERSION}" in
-            nightly|trunk) wp core download --version=nightly ;;
-            *) wp core download "https://wordpress.org/wordpress-${WP_VERSION}.zip" ;;
-          esac
+        # WP-CLI requests WordPress from WordPress.org once, so a network error fails the job.
+        # It extracts only after a complete download, so a retry starts from an empty directory.
+        uses: $/.github/actions/run-with-retry
+        with:
+          command: |
+            case "${WP_VERSION}" in
+              nightly|trunk) wp core download --version=nightly ;;
+              *) wp core download "https://wordpress.org/wordpress-${WP_VERSION}.zip" ;;
+            esac
         env:
           WP_VERSION: ${{ inputs.wp-version || 'nightly' }}
 

.github/workflows/performance.yml

@@ -44,7 +44,9 @@ on:
       - 'tools/vendors/**'
       - 'tools/webpack/**'
       - '.github/actions/setup-npm/**'
+      - '.github/actions/download-with-retry/**'
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/workflows/performance.yml'
       - '.github/workflows/reusable-performance-*.yml'
   workflow_dispatch:

.github/workflows/reusable-end-to-end-tests.yml

@@ -136,14 +136,22 @@ jobs:
           LOCAL_SCRIPT_DEBUG: ${{ inputs.LOCAL_SCRIPT_DEBUG }}
         run: npm run env:install
 
+      - name: Download Gutenberg
+        if: ${{ inputs.install-gutenberg }}
+        # Downloaded on the runner so failed downloads are retried: WP-CLI tries once.
+        # Release branches run this workflow from trunk: `$/` resolves to this workflow's commit, not the caller's.
+        uses: $/.github/actions/download-with-retry
+        with:
+          url: https://downloads.wordpress.org/plugin/gutenberg.${{ inputs.gutenberg-version || 'latest-stable' }}.zip
+          output: gutenberg.zip
+
       - name: Install Gutenberg
         if: ${{ inputs.install-gutenberg }}
         run: |
-          npm run env:cli -- plugin install gutenberg \
-          ${{ inputs.gutenberg-version && '--version="${GUTENBERG_VERSION}"' || '' }} \
-          --path="/var/www/${LOCAL_DIR}"
-        env:
-          GUTENBERG_VERSION: ${{ inputs.gutenberg-version }}
+          npm run env:cli -- plugin install gutenberg.zip --path="/var/www/${LOCAL_DIR}"
+          rm gutenberg.zip
+          # Installing from a zip doesn't log the version; unpinned branches get the latest.
+          npm run env:cli -- plugin get gutenberg --field=version --path="/var/www/${LOCAL_DIR}"
 
       - name: Install additional languages
         run: |

.github/workflows/reusable-performance-test-v2.yml

@@ -159,6 +159,9 @@ jobs:
         run: npm ci
 
       - name: Install Playwright browsers
+        # A hung install would otherwise run until the 6 hour job limit. The cap
+        # leaves room for slow downloads.
+        timeout-minutes: 15
         run: npm exec --no -- playwright install --with-deps chromium
 
       - name: Start Docker environment
@@ -202,9 +205,11 @@ jobs:
 
       - name: Set the environment to the baseline version
         if: ${{ inputs.subject == 'base' }}
-        run: |
-          VERSION="${BASE_TAG%.0}"
-          wp core download --version="$VERSION" --force --path="${LOCAL_DIR}"
+        # WP-CLI requests the release from WordPress.org once, so a network error fails the job.
+        # It extracts only after a complete download, and `--force` lets a retry overwrite files.
+        uses: $/.github/actions/run-with-retry
+        with:
+          command: wp core download --version="${BASE_TAG%.0}" --force --path="${LOCAL_DIR}"
 
       - name: Install object cache drop-in
         if: ${{ inputs.memcached }}
@@ -236,12 +241,29 @@ jobs:
           npm run env:cli -- theme enable twentytwentyfour --network --path="/var/www/${LOCAL_DIR}"
           npm run env:cli -- theme enable twentytwentyfive --network --path="/var/www/${LOCAL_DIR}"
 
+      - name: Download WordPress Importer plugin
+        # Downloaded on the runner so failed downloads are retried: WP-CLI tries once.
+        uses: $/.github/actions/download-with-retry
+        with:
+          url: https://downloads.wordpress.org/plugin/wordpress-importer.latest-stable.zip
+          output: wordpress-importer.zip
+
       - name: Install WordPress Importer plugin
-        run: npm run env:cli -- plugin install wordpress-importer --activate --path="/var/www/${LOCAL_DIR}"
+        run: |
+          npm run env:cli -- plugin install wordpress-importer.zip --activate --path="/var/www/${LOCAL_DIR}"
+          rm wordpress-importer.zip
+
+      - name: Download mock data
+        uses: $/.github/actions/download-with-retry
+        with:
+          url: https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml
+          output: themeunittestdata.wordpress.xml
 
       - name: Import mock data
+        # The importer waits up to 300 s per attachment download and skips the ones that
+        # fail: the cap leaves room for a stalled attachment, not a hung import.
+        timeout-minutes: 15
         run: |
-          curl -O https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml
           npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path="/var/www/${LOCAL_DIR}"
           rm themeunittestdata.wordpress.xml
 
@@ -271,13 +293,15 @@ jobs:
           cp ./tests/performance/wp-content/mu-plugins/server-timing.php "./${LOCAL_DIR}/wp-content/mu-plugins/server-timing.php"
 
       - name: Run performance tests
+        id: performance-tests
         run: npm run test:performance
         env:
           TEST_RESULTS_PREFIX: ${{ inputs.subject != 'current' && inputs.subject || '' }}
 
       - name: Archive artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        if: always()
+        # Without test results, a failed setup step would report a second failure here.
+        if: ${{ always() && steps.performance-tests.outcome != 'skipped' }}
         with:
           name: performance-${{ inputs.multisite && 'multisite' || 'single' }}-${{ inputs.memcached && 'memcached' || 'default' }}-${{ inputs.subject }}
           path: artifacts

.github/workflows/reusable-upgrade-testing.yml

@@ -95,7 +95,11 @@ jobs:
         uses: $/.github/actions/setup-wp-cli
 
       - name: Download WordPress ${{ inputs.wp }}
-        run: wp core download --version="${WP_VERSION}"
+        # WP-CLI requests the release from WordPress.org once, so a network error fails the job.
+        # It extracts only after a complete download, so a retry starts from an empty directory.
+        uses: $/.github/actions/run-with-retry
+        with:
+          command: wp core download --version="${WP_VERSION}"
         env:
           WP_VERSION: ${{ inputs.wp }}
 

.github/workflows/upgrade-develop-testing.yml

@@ -15,6 +15,7 @@ on:
       - '*.php' # PHP files in the root, like wp-config-sample.php.
       # Confirm any changes to relevant workflow files.
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.github/actions/setup-npm/**'
       - '.github/workflows/upgrade-develop-testing.yml'
@@ -31,6 +32,7 @@ on:
       # Confirm any changes to relevant workflow files.
       - '.github/actions/setup-npm/**'
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/workflows/upgrade-develop-testing.yml'
       - '.github/workflows/reusable-upgrade-testing.yml'
   schedule:

.github/workflows/upgrade-testing.yml

@@ -10,6 +10,7 @@ on:
     # Always test the workflow after it's updated.
     paths:
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.github/workflows/upgrade-testing.yml'
       - '.github/workflows/reusable-upgrade-testing.yml'
@@ -20,6 +21,7 @@ on:
     # Always test the workflow when changes are suggested.
     paths:
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/workflows/upgrade-testing.yml'
       - '.github/workflows/reusable-upgrade-testing.yml'
   workflow_dispatch: