From 6870d1f022f078dd57c4c8a477c87bb2865bfe47 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 01:34:05 -0700 Subject: [PATCH 1/5] Build/Test Tools: Add plugin compatibility testing workflow. Core CI covers core itself, but nothing checks that a new version of WordPress can still boot with popular plugins active. When a plugin's assumptions about core stop holding the result is a fatal error on every request, which is a white screen for real sites and is only discovered after release. Add a workflow that fetches the most popular plugins from the WordPress.org API at run time, then installs and activates each one on its own against the version of WordPress under test. A fatal is caught whether it happens on activation, while WP-CLI loads WordPress, on a front end or login request, or in the debug log, so a white screen with error display turned off is still detected. Failures are reported per plugin in the workflow summary and one broken plugin never stops the rest of the shard from being tested. Plugins that cannot be downloaded are reported as skipped rather than failed so that a network flake does not turn the run red. The run is manual or weekly rather than part of every commit, since a third party plugin breaking should be a signal to release leads, not a red check on unrelated work. --- .github/workflows/plugin-compatibility.yml | 205 ++++++++++++ .../reusable-plugin-compatibility.yml | 316 ++++++++++++++++++ 2 files changed, 521 insertions(+) create mode 100644 .github/workflows/plugin-compatibility.yml create mode 100644 .github/workflows/reusable-plugin-compatibility.yml diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..dc8d2f709c576 --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,205 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a small matrix. Each +# plugin is installed and activated on its own, so one broken plugin cannot hide another. +# +# This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on +# WordPress.org. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number.' + type: string + default: 'nightly' + plugin-count: + description: 'How many of the most popular plugins to test.' + type: string + default: '100' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Fetch the most popular plugins + id: plugin-shards + env: + # A pull request or scheduled run has no inputs, so fall back to the default count. + PLUGIN_COUNT: ${{ inputs.plugin-count || '100' }} + SHARD_COUNT: '5' + run: | + set -euo pipefail + + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer.\n' + exit 1 + fi + + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PLUGIN_COUNT}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins.json" + + SLUGS="$( jq -c '[ .plugins[].slug ] | map( select( . != null and . != "" ) )' "${RUNNER_TEMP}/plugins.json" )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shards.\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || 'nightly' }} + php-version: '8.3' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '8.4' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + SLACK_GHA_TIMEOUT_WEBHOOK: ${{ secrets.SLACK_GHA_TIMEOUT_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + needs: [ slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..8147b6b5ec4b1 --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,316 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + os: + description: 'Operating system to run tests on.' + required: false + type: 'string' + default: 'ubuntu-24.04' + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.3' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard.' + required: true + type: 'string' + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '8.4' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Sets up PHP. + # - Downloads the specified version of WordPress. + # - Creates a `wp-config.php` file with debugging and error logging enabled. + # - Installs WordPress. + # - Starts the PHP built-in web server so HTTP requests can be made against the site. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Writes a results table to the workflow summary and fails the job if any plugin fataled. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ${{ inputs.os }} + timeout-minutes: 30 + + services: + database: + image: ${{ inputs.db-type }}:${{ inputs.db-version }} + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + - name: Download WordPress ${{ inputs.wp-version }} + run: wp core download --version="${WP_VERSION}" + env: + WP_VERSION: ${{ inputs.wp-version }} + + - name: Create wp-config.php file + run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost="127.0.0.1:${DB_PORT}" + env: + DB_PORT: ${{ job.services.database.ports['3306'] }} + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by + # recovery mode, which would also deactivate the plugin mid-test. + - name: Enable debugging and error logging + run: | + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + + - name: Install WordPress + run: | + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email + env: + SITE_URL: http://127.0.0.1:8889 + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + - name: Start the PHP built-in web server + run: | + set -uo pipefail + + nohup php -S 127.0.0.1:8889 -t "$( pwd )" > "${RUNNER_TEMP}/php-server.log" 2>&1 & + + # Wait for the server to start answering before any plugin is installed. + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "http://127.0.0.1:8889/wp-login.php"; then + printf 'The PHP built-in server is ready.\n' + exit 0 + fi + sleep 1 + done + + printf 'The PHP built-in server did not start.\n' + cat "${RUNNER_TEMP}/php-server.log" + exit 1 + + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + PHP_VERSION: ${{ inputs.php-version }} + SITE_URL: http://127.0.0.1:8889 + WP_VERSION: ${{ inputs.wp-version }} + run: | + # `set -e` is deliberately not used here: a plugin that fatals must not stop the remaining + # plugins in the shard from being tested. + set -uo pipefail + + RESULTS="${RUNNER_TEMP}/plugin-results.tsv" + RESPONSE_BODY="${RUNNER_TEMP}/response.html" + : > "${RESULTS}" + + # record + # + # Appends one tab separated row to the results file. The reason is flattened so that it cannot + # break the markdown table that is generated from these rows later on. + record() { + SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + } + + # check_url + # + # Requests a path on the test site and prints a reason to stdout when the response looks broken. + # Prints nothing when the request looks healthy. + check_url() { + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" || printf '000' )" + HTTP_CODE="${HTTP_CODE:-000}" + + if [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete' "${1}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors + # back on for itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi + } + + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # cleanup_plugin + # + # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # command here is allowed to fail and the plugin directory is removed directly as a fallback. + # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. + cleanup_plugin() { + wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + + if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${2}" + fi + + rm -rf "wp-content/plugins/${1}" + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true + } + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + printf '::group::%s\n' "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + PLUGIN_DIR="" + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f wp-content/debug.log + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, + # since it generally means a network flake or a plugin that is no longer in the directory. + if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then + record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" + cleanup_plugin "${SLUG}" "" + printf '::endgroup::\n' + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. + if ! wp plugin activate "${SLUG}"; then + STATUS="FAIL" + REASON="The plugin could not be activated" + fi + + # Step 3: boot all of core plus the active plugin in a CLI context. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + EVAL_EXIT_CODE=$? + + if [ "${EVAL_EXIT_CODE}" -ne 0 ] || [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + STATUS="FAIL" + REASON="WordPress could not be loaded by WP-CLI with the plugin active" + fi + fi + + # Step 4: request the front page and the login screen through the PHP built-in server. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then + grep 'PHP Fatal' wp-content/debug.log + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" + fi + + # Step 6: record the outcome and put the site back the way it was found. + record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" + cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + printf '::endgroup::\n' + done < <( printf '%s' "${PLUGIN_SLUGS}" | jq -r '.[]' ) + + PASS_COUNT="$( awk -F '\t' '$3 == "PASS" { count++ } END { print count + 0 }' "${RESULTS}" )" + FAIL_COUNT="$( awk -F '\t' '$3 == "FAIL" { count++ } END { print count + 0 }' "${RESULTS}" )" + SKIP_COUNT="$( awk -F '\t' '$3 == "SKIPPED" { count++ } END { print count + 0 }' "${RESULTS}" )" + + { + printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" + printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" + printf '| Plugin | Version | Result | Details |\n' + printf '| --- | --- | --- | --- |\n' + + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + case "${ROW_STATUS}" in + PASS ) ICON=':white_check_mark:' ;; + FAIL ) ICON=':x:' ;; + * ) ICON=':warning:' ;; + esac + + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + done < "${RESULTS}" + + printf '\n' + } >> "${GITHUB_STEP_SUMMARY}" + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + exit 1 + fi + + printf 'No plugins fataled against WordPress %s.\n' "${WP_VERSION}" + + - name: Show the web server log + if: ${{ failure() }} + run: cat "${RUNNER_TEMP}/php-server.log" From 14340c7a4bde129331a33bfbbb54bd284f8c1925 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:02:36 -0700 Subject: [PATCH 2/5] Build/Test Tools: Correct plugin compatibility failure classification. The first run of this workflow reported four failures that were not fatals. Core refuses to activate a plugin whose `Requires Plugins` dependency is missing, which every WooCommerce extension hits when plugins are tested one at a time, and WP-CLI exits non-zero when a plugin redirects while loading. Both are correct behaviour, so record them as skipped and reserve a failure for an actual fatal. A front end request also reported the nonsense status "200000", because the curl fallback appended to output curl had already written. Capture the exit code separately so a stalled transfer is reported as what it is. The stall itself came from WordPress spawning WP-Cron as a loopback request that the single threaded built-in server could not answer while still serving the request that spawned it. Disable WP-Cron and give the server workers so plugin loopback requests cannot deadlock it. Ignore the zizmor unpinned image finding on the database service, which cannot be pinned to a digest while the version is an input. --- .../reusable-plugin-compatibility.yml | 77 +++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 8147b6b5ec4b1..4ca645f3c7aa9 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -64,7 +64,10 @@ jobs: services: database: - image: ${{ inputs.db-type }}:${{ inputs.db-version }} + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] ports: - 3306 options: >- @@ -100,12 +103,18 @@ jobs: # # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by # recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that spawned + # it, and the two deadlock until curl gives up. Catching fatals that only happen on a scheduled event is + # worth doing, but it needs to run through WP-CLI rather than a loopback, which is follow up work. - name: Enable debugging and error logging run: | wp config set WP_DEBUG true --raw wp config set WP_DEBUG_LOG true --raw wp config set WP_DEBUG_DISPLAY false --raw wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw - name: Install WordPress run: | @@ -116,7 +125,13 @@ jobs: # The site needs to answer real requests so that fatals which only happen on a front end or admin page # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. - name: Start the PHP built-in web server + env: + PHP_CLI_SERVER_WORKERS: '4' run: | set -uo pipefail @@ -164,11 +179,14 @@ jobs: # Requests a path on the test site and prints a reason to stdout when the response looks broken. # Prints nothing when the request looks healthy. check_url() { - HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" || printf '000' )" - HTTP_CODE="${HTTP_CODE:-000}" - - if [ "${HTTP_CODE}" = "000" ]; then - printf 'The request to %s did not complete' "${1}" + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense + # status like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" return fi @@ -177,6 +195,11 @@ jobs: return fi + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors # back on for itself. if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then @@ -232,20 +255,46 @@ jobs: PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. - if ! wp plugin activate "${SLUG}"; then - STATUS="FAIL" - REASON="The plugin could not be activated" + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met, most often a + # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means + # every WooCommerce extension lands here. That is core working as designed rather than a + # fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac fi # Step 3: boot all of core plus the active plugin in a CLI context. if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" - EVAL_EXIT_CODE=$? + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true - if [ "${EVAL_EXIT_CODE}" -ne 0 ] || [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then printf '%s\n' "${EVAL_OUTPUT}" - STATUS="FAIL" - REASON="WordPress could not be loaded by WP-CLI with the plugin active" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + STATUS="FAIL" + REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and + # debug log checks below, which see the same code in a real request. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac fi fi From 915017705dcd307e748b4e62a30762410f3d789c Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:02:36 -0700 Subject: [PATCH 3/5] Build/Test Tools: Support any plugin count in compatibility testing. The WordPress.org API caps `per_page` at 250 and quietly returns 250 for anything larger, so asking for more than that silently tested fewer plugins than requested. Page through the API instead and trim to the requested count, de-duplicating across pages because popularity ordering can shift between two requests. Reject a count above 1000 with a clear message rather than truncating without saying so. Size the shards to the plugin count rather than always splitting into five, so a run of 10 does not spin up five near empty jobs and a run of 250 is not squeezed into the same five. Point pull request and push runs at the latest stable release with a small count. Those runs exist to check that this workflow still works, and a genuine ecosystem fatal against nightly should not sit as a red check on every later change to these files. --- .github/workflows/plugin-compatibility.yml | 128 ++++++++++++++++----- 1 file changed, 100 insertions(+), 28 deletions(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index dc8d2f709c576..3ae64ce5abeaf 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -7,8 +7,13 @@ # every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so # that it can be found while there is still time to fix core or notify the plugin author. # -# The plugin list is fetched from the WordPress.org API at run time and is sharded across a small matrix. Each -# plugin is installed and activated on its own, so one broken plugin cannot hide another. +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. # # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. @@ -36,11 +41,11 @@ on: workflow_dispatch: inputs: wp-version: - description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number.' + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' type: string default: 'nightly' plugin-count: - description: 'How many of the most popular plugins to test.' + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250.' type: string default: '100' @@ -78,36 +83,84 @@ jobs: - name: Fetch the most popular plugins id: plugin-shards env: - # A pull request or scheduled run has no inputs, so fall back to the default count. - PLUGIN_COUNT: ${{ inputs.plugin-count || '100' }} - SHARD_COUNT: '5' + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # The API caps `per_page` at 250 and returns 250 without complaint for anything larger, so counts + # above that have to be paged. + PAGE_SIZE: '250' + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' run: | set -euo pipefail # Guard against a non-numeric value being passed to the API. if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then - printf 'The plugin-count input must be a positive integer.\n' + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" exit 1 fi - # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. - # The unneeded response fields are turned off to keep the payload small. - curl -sS --fail --retry 3 --retry-delay 5 \ - --get 'https://api.wordpress.org/plugins/info/1.2/' \ - --data-urlencode 'action=query_plugins' \ - --data-urlencode 'request[browse]=popular' \ - --data-urlencode "request[per_page]=${PLUGIN_COUNT}" \ - --data-urlencode 'request[fields][short_description]=0' \ - --data-urlencode 'request[fields][sections]=0' \ - --data-urlencode 'request[fields][icons]=0' \ - --data-urlencode 'request[fields][banners]=0' \ - --data-urlencode 'request[fields][ratings]=0' \ - --data-urlencode 'request[fields][tags]=0' \ - --data-urlencode 'request[fields][compatibility]=0' \ - --data-urlencode 'request[fields][screenshots]=0' \ - -o "${RUNNER_TEMP}/plugins.json" - - SLUGS="$( jq -c '[ .plugins[].slug ] | map( select( . != null and . != "" ) )' "${RUNNER_TEMP}/plugins.json" )" + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" if [ "${TOTAL}" -lt 1 ]; then @@ -115,6 +168,19 @@ jobs: exit 1 fi + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer # plugins were requested than there are shards. Each shard's slugs are passed to the reusable # workflow as a JSON string. @@ -135,8 +201,14 @@ jobs: printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. plugin-compatibility-tests: - name: WP ${{ inputs.wp-version || 'nightly' }} / Shard ${{ matrix.shard.index }} + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} uses: ./.github/workflows/reusable-plugin-compatibility.yml permissions: contents: read @@ -148,7 +220,7 @@ jobs: shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} with: os: 'ubuntu-24.04' - wp-version: ${{ inputs.wp-version || 'nightly' }} + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} php-version: '8.3' plugin-slugs: ${{ matrix.shard.slugs }} db-type: 'mysql' From cf18ee6356faabb66a7705db9daf7d300bb8d960 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:05:30 -0700 Subject: [PATCH 4/5] Build/Test Tools: Fix shard count wording in plugin compatibility log. A run that builds a single shard logged "across 1 shards". --- .github/workflows/plugin-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 3ae64ce5abeaf..b2a9c12d2d6a6 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -195,7 +195,7 @@ jobs: | map( { index: .index, slugs: ( .slugs | @json ) } ) ' )" - printf 'Testing %s plugins across %s shards.\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" From 67005f527ba98d30586aa98c1ad89f25fb1867f3 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 25 Aug 2026 11:58:18 -0700 Subject: [PATCH 5/5] Build/Test Tools: Install plugin dependencies before compatibility testing. Testing each plugin on its own meant core refused to activate anything with a `Requires Plugins` header, so every WooCommerce extension was recorded as skipped without ever being loaded. Those extensions are a large slice of the most popular plugins, which left a gap in exactly the part of the ecosystem the workflow exists to watch. Read the `Requires Plugins` header with WordPress' own parser, install and activate what it names, and only then activate the plugin under test. Chains are followed a level at a time so a dependency declaring its own is installed too, and anything already present is left alone, which terminates a circular declaration. Dependencies mean the plugin under test is no longer alone on the site, so the front page and login screen are checked with only the dependencies active first. A site already broken at that point is the dependency's doing, and the plugin is skipped rather than blamed for it. The debug log is cleared at the same point for the same reason. The results table gains a column naming what else was active so a failure can be read in context. Installing and activating a dependency the size of WooCommerce is not quick, so the job timeout goes from 30 to 45 minutes. --- .../reusable-plugin-compatibility.yml | 251 ++++++++++++++---- 1 file changed, 200 insertions(+), 51 deletions(-) diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 4ca645f3c7aa9..d32d2156b9b16 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -4,6 +4,11 @@ # # Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then # removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +# +# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until +# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the +# site is checked with only the dependencies active first so that a dependency's own breakage is not reported +# against the plugin under test. ## name: Plugin Compatibility Tests @@ -53,14 +58,17 @@ jobs: # - Creates a `wp-config.php` file with debugging and error logging enabled. # - Installs WordPress. # - Starts the PHP built-in web server so HTTP requests can be made against the site. - # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time, together with + # anything it declares in `Requires Plugins`. # - Writes a results table to the workflow summary and fails the job if any plugin fataled. plugin-compatibility-tests: name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} permissions: contents: read runs-on: ${{ inputs.os }} - timeout-minutes: 30 + # Installing a plugin's dependencies before testing it adds a download and an activation for each + # one, and a dependency the size of WooCommerce is not quick to activate. + timeout-minutes: 45 services: database: @@ -165,13 +173,24 @@ jobs: RESPONSE_BODY="${RUNNER_TEMP}/response.html" : > "${RESULTS}" - # record + # How many levels of `Requires Plugins` to follow. Chains in the directory are one level + # deep in practice, so the bound is only here to stop a circular declaration from looping + # forever. + MAX_DEPENDENCY_DEPTH=3 + + # Everything installed for the plugin currently under test, that plugin included. The + # directories are tracked alongside the slugs because a plugin does not always unpack into + # a directory named after its slug. + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + # record # # Appends one tab separated row to the results file. The reason is flattened so that it cannot # break the markdown table that is generated from these rows later on. record() { - SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" - printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" } # check_url @@ -210,20 +229,70 @@ jobs: # The plugins directory itself must never be removed, only directories inside it. PLUGINS_ROOT="$( wp plugin path )" - # cleanup_plugin + # install_plugin # - # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # Installs a plugin from WordPress.org and remembers it so that it is removed again once the + # plugin under test has been checked. The slug is remembered before the download is attempted + # so that a partial download is still cleaned up. Returns non zero when the download failed. + install_plugin() { + INSTALLED_SLUGS+=( "${1}" ) + INSTALLED_DIRS+=( "" ) + + wp plugin install "${1}" --skip-plugins --skip-themes || return 1 + + INSTALLED_DIRS[-1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + } + + # plugin_dependencies + # + # Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' + # own header parser is used so that the same rules apply here as when core decides whether a + # plugin's dependencies are met. + plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + # shellcheck disable=SC2016 + PLUGIN_FILE="${PLUGIN_FILE}" wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // Core only accepts a WordPress.org slug here, so anything else is ignored. + if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null + } + + # cleanup_plugins + # + # Returns the site to a clean slate by removing everything installed for the current plugin, + # its dependencies included. A plugin that fatals can take WP-CLI down with it, so every # command here is allowed to fail and the plugin directory is removed directly as a fallback. # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. - cleanup_plugin() { - wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + cleanup_plugins() { + for INDEX in "${!INSTALLED_SLUGS[@]}"; do + CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" + CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" - if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then - rm -rf "${2}" - fi + wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - rm -rf "wp-content/plugins/${1}" + if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${CLEANUP_DIR}" + fi + + rm -rf "wp-content/plugins/${CLEANUP_SLUG}" + done + + INSTALLED_SLUGS=() + INSTALLED_DIRS=() # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true @@ -237,46 +306,119 @@ jobs: STATUS="PASS" REASON="-" VERSION="unknown" - PLUGIN_DIR="" + DEPENDENCIES=() # Start every plugin with an empty log so that anything found in it belongs to this plugin. rm -f wp-content/debug.log # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, # since it generally means a network flake or a plugin that is no longer in the directory. - if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then - record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" - cleanup_plugin "${SLUG}" "" + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + cleanup_plugins printf '::endgroup::\n' continue fi VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" - PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" - - # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. - ACTIVATE_EXIT_CODE=0 - ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? - printf '%s\n' "${ACTIVATE_OUTPUT}" - - if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then - case "${ACTIVATE_OUTPUT}" in - # Core refuses to activate a plugin whose declared requirements are not met, most often a - # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means - # every WooCommerce extension lands here. That is core working as designed rather than a - # fatal, so it is recorded as skipped. - *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a + # plugin whose dependencies are missing, so without this every WooCommerce extension - a large + # slice of the most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is + # installed too. Anything already installed is left alone, which covers a plugin named twice + # in the same tree as well as a declaration pointing back at the plugin under test. + mapfile -t PENDING < <( plugin_dependencies "${SLUG}" ) + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + continue + fi + + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then STATUS="SKIPPED" - REASON="Core declined to activate the plugin because its declared requirements are not met" - ;; - * ) - STATUS="FAIL" - REASON="The plugin could not be activated" - ;; - esac + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + + DEPENDENCIES+=( "${DEPENDENCY}" ) + mapfile -t -O "${#NEXT[@]}" NEXT < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( "${NEXT[@]}" ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies, deepest first. That is the reverse of the order they + # were discovered in, and it matters because core will not activate a plugin ahead of its own + # requirements either. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do + DEPENDENCY="${DEPENDENCIES[${INDEX}]}" + DEPENDENCY_EXIT_CODE=0 + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} could not be activated" + break + fi + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so + # the baseline is checked before it is activated. A site that is already broken says something + # about the dependency rather than about the plugin being tested, and blaming the plugin for + # it would be the masking this workflow is built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # Anything the dependencies logged on their way up is not the responsibility of the plugin + # under test, so the log starts empty again here. + rm -f wp-content/debug.log fi - # Step 3: boot all of core plus the active plugin in a CLI context. + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. Dependencies + # are installed above, so what is left here is a plugin asking for a version of PHP or + # WordPress this job is not running, or for a dependency that is not on WordPress.org. + # That is core working as designed rather than a fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + fi + + # Step 6: boot all of core plus the active plugin in a CLI context. if [ "${STATUS}" = "PASS" ]; then EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true @@ -298,7 +440,7 @@ jobs: fi fi - # Step 4: request the front page and the login screen through the PHP built-in server. + # Step 7: request the front page and the login screen through the PHP built-in server. if [ "${STATUS}" = "PASS" ]; then for URL_PATH in "/" "/wp-login.php"; do HTTP_REASON="$( check_url "${URL_PATH}" )" @@ -311,7 +453,7 @@ jobs: done fi - # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown # hook, so the debug log is checked separately. if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then grep 'PHP Fatal' wp-content/debug.log @@ -319,9 +461,16 @@ jobs: REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" fi - # Step 6: record the outcome and put the site back the way it was found. - record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" - cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + cleanup_plugins printf '%s: %s\n' "${SLUG}" "${STATUS}" printf '::endgroup::\n' @@ -334,18 +483,18 @@ jobs: { printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" - printf '| Plugin | Version | Result | Details |\n' - printf '| --- | --- | --- | --- |\n' + printf '| Plugin | Version | Result | Also active | Details |\n' + printf '| --- | --- | --- | --- | --- |\n' - while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_DEPENDENCIES ROW_REASON; do case "${ROW_STATUS}" in PASS ) ICON=':white_check_mark:' ;; FAIL ) ICON=':x:' ;; * ) ICON=':warning:' ;; esac - printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ - "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_DEPENDENCIES}" "${ROW_REASON}" done < "${RESULTS}" printf '\n' @@ -354,7 +503,7 @@ jobs: # Plugins that could not be downloaded are reported but do not fail the run. if [ "${FAIL_COUNT}" -gt 0 ]; then printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" - awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" exit 1 fi