From 75678370118d88370fa324a0394f1bb2a80d30d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 08:12:50 +0000 Subject: [PATCH 1/4] Initial plan From 55e87f855a632712b1f3297b5aeec321775e6de8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 08:27:54 +0000 Subject: [PATCH 2/4] Parallelize repository cloning during development environment setup Agent-Logs-Url: https://github.com/wp-cli/wp-cli-dev/sessions/6509f703-3861-4c87-83a6-f0247ed1904b Co-authored-by: swissspidy <841956+swissspidy@users.noreply.github.com> --- .maintenance/clone-all-repositories.php | 13 +++++++++---- .maintenance/clone-repository.php | 11 +++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 .maintenance/clone-repository.php diff --git a/.maintenance/clone-all-repositories.php b/.maintenance/clone-all-repositories.php index 29c3656..beafb66 100644 --- a/.maintenance/clone-all-repositories.php +++ b/.maintenance/clone-all-repositories.php @@ -35,6 +35,7 @@ $pwd = getcwd(); $update_folders = []; +$clone_list = []; foreach ( $repositories as $repository ) { if ( in_array( $repository->name, $skip_list, true ) ) { @@ -44,13 +45,17 @@ $destination = isset( $clone_destination_map[ $repository->name ] ) ? $clone_destination_map[ $repository->name ] : $repository->name; if ( ! is_dir( $destination ) ) { - printf( "Fetching \033[32mwp-cli/{$repository->name}\033[0m...\n" ); - $clone_url = getenv( 'GITHUB_ACTION' ) ? $repository->clone_url : $repository->ssh_url; - system( "git clone {$clone_url} {$destination}" ); + $clone_url = getenv( 'GITHUB_ACTION' ) ? $repository->clone_url : $repository->ssh_url; + $clone_list[] = "{$destination} {$clone_url}"; } $update_folders[] = $destination; } -$updates = implode( '\n', $update_folders ); +if ( ! empty( $clone_list ) ) { + $clones = implode( "\n", $clone_list ); + system( "echo '$clones' | xargs -n2 -P8 php .maintenance/clone-repository.php" ); +} + +$updates = implode( "\n", $update_folders ); system( "echo '$updates' | xargs -n1 -P8 -I% php .maintenance/refresh-repository.php %" ); diff --git a/.maintenance/clone-repository.php b/.maintenance/clone-repository.php new file mode 100644 index 0000000..645eb90 --- /dev/null +++ b/.maintenance/clone-repository.php @@ -0,0 +1,11 @@ + \n" ); +} + +$destination = $argv[1]; +$clone_url = $argv[2]; + +printf( "Fetching \033[32m%s\033[0m...\n", $destination ); +system( 'git clone ' . escapeshellarg( $clone_url ) . ' ' . escapeshellarg( $destination ) ); From 94182ade04545e5675b0f8664ddb7e124645f97d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 12:55:42 +0000 Subject: [PATCH 3/4] Convert PHP cloning scripts to bash, detect CPU cores for parallelism Agent-Logs-Url: https://github.com/wp-cli/wp-cli-dev/sessions/28119b52-0b7a-445b-b43e-f3189c004e28 Co-authored-by: swissspidy <841956+swissspidy@users.noreply.github.com> --- .maintenance/clone-all-repositories.php | 61 ----------------- .maintenance/clone-all-repositories.sh | 89 +++++++++++++++++++++++++ .maintenance/clone-repository.php | 11 --- .maintenance/clone-repository.sh | 14 ++++ composer.json | 4 +- 5 files changed, 105 insertions(+), 74 deletions(-) delete mode 100644 .maintenance/clone-all-repositories.php create mode 100755 .maintenance/clone-all-repositories.sh delete mode 100644 .maintenance/clone-repository.php create mode 100755 .maintenance/clone-repository.sh diff --git a/.maintenance/clone-all-repositories.php b/.maintenance/clone-all-repositories.php deleted file mode 100644 index beafb66..0000000 --- a/.maintenance/clone-all-repositories.php +++ /dev/null @@ -1,61 +0,0 @@ - 'dot-github', -); - -$request = 'https://api.github.com/orgs/wp-cli/repos?per_page=100'; -$headers = ''; -$token = getenv( 'GITHUB_TOKEN' ); -if ( ! empty( $token ) ) { - $headers = "--header \"Authorization: Bearer $token\""; - $response = shell_exec( "curl -s {$headers} {$request}" ); -} else { - $response = shell_exec( "curl -s {$request}" ); -} -$repositories = json_decode( $response ); -if ( ! is_array( $repositories ) && property_exists( $repositories, 'message' ) ) { - echo 'GitHub responded with: ' . $repositories->message . "\n"; - echo "If you are running into a rate limiting issue during large events please set GITHUB_TOKEN environment variable.\n"; - echo "See https://github.com/settings/tokens\n"; - exit( 1 ); -} - -$pwd = getcwd(); -$update_folders = []; -$clone_list = []; - -foreach ( $repositories as $repository ) { - if ( in_array( $repository->name, $skip_list, true ) ) { - continue; - } - - $destination = isset( $clone_destination_map[ $repository->name ] ) ? $clone_destination_map[ $repository->name ] : $repository->name; - - if ( ! is_dir( $destination ) ) { - $clone_url = getenv( 'GITHUB_ACTION' ) ? $repository->clone_url : $repository->ssh_url; - $clone_list[] = "{$destination} {$clone_url}"; - } - - $update_folders[] = $destination; -} - -if ( ! empty( $clone_list ) ) { - $clones = implode( "\n", $clone_list ); - system( "echo '$clones' | xargs -n2 -P8 php .maintenance/clone-repository.php" ); -} - -$updates = implode( "\n", $update_folders ); -system( "echo '$updates' | xargs -n1 -P8 -I% php .maintenance/refresh-repository.php %" ); diff --git a/.maintenance/clone-all-repositories.sh b/.maintenance/clone-all-repositories.sh new file mode 100755 index 0000000..10d5b8b --- /dev/null +++ b/.maintenance/clone-all-repositories.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +SKIP_LIST=( + "autoload-splitter" + "composer-changelogs" + "dash-docset-generator" + "ideas" + "package-index" + "regenerate-readme" + "sample-plugin" + "wp-cli-dev" + "wp-cli-roadmap" +) + +# Detect number of CPU cores, defaulting to 4. +if command -v nproc &>/dev/null; then + CORES=$(nproc) +elif command -v sysctl &>/dev/null; then + CORES=$(sysctl -n hw.logicalcpu 2>/dev/null || echo 4) +else + CORES=4 +fi + +# Fetch repository list from the GitHub API. +CURL_OPTS=(-s) +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + CURL_OPTS+=(--header "Authorization: Bearer ${GITHUB_TOKEN}") +fi + +RESPONSE=$(curl "${CURL_OPTS[@]}" 'https://api.github.com/orgs/wp-cli/repos?per_page=100') + +# Detect API errors such as rate limiting. +if echo "${RESPONSE}" | jq -e '.message' &>/dev/null; then + MESSAGE=$(echo "${RESPONSE}" | jq -r '.message') + echo "GitHub responded with: ${MESSAGE}" + echo "If you are running into a rate limiting issue during large events please set GITHUB_TOKEN environment variable." + echo "See https://github.com/settings/tokens" + exit 1 +fi + +is_skipped() { + local name="$1" + for skip in "${SKIP_LIST[@]}"; do + [[ "${skip}" == "${name}" ]] && return 0 + done + return 1 +} + +get_destination() { + local name="$1" + if [[ "${name}" == ".github" ]]; then + echo "dot-github" + else + echo "${name}" + fi +} + +CLONE_LIST=() +UPDATE_FOLDERS=() + +while IFS=$'\t' read -r name clone_url ssh_url; do + if is_skipped "${name}"; then + continue + fi + + destination=$(get_destination "${name}") + + if [[ ! -d "${destination}" ]]; then + if [[ -n "${GITHUB_ACTION:-}" ]]; then + CLONE_LIST+=("${destination}"$'\t'"${clone_url}") + else + CLONE_LIST+=("${destination}"$'\t'"${ssh_url}") + fi + fi + + UPDATE_FOLDERS+=("${destination}") +done < <(echo "${RESPONSE}" | jq -r '.[] | [.name, .clone_url, .ssh_url] | @tsv') + +if [[ ${#CLONE_LIST[@]} -gt 0 ]]; then + printf '%s\n' "${CLONE_LIST[@]}" | xargs -n2 -P"${CORES}" bash "${SCRIPT_DIR}/clone-repository.sh" +fi + +if [[ ${#UPDATE_FOLDERS[@]} -gt 0 ]]; then + printf '%s\n' "${UPDATE_FOLDERS[@]}" | xargs -n1 -P"${CORES}" -I% php "${SCRIPT_DIR}/refresh-repository.php" % +fi diff --git a/.maintenance/clone-repository.php b/.maintenance/clone-repository.php deleted file mode 100644 index 645eb90..0000000 --- a/.maintenance/clone-repository.php +++ /dev/null @@ -1,11 +0,0 @@ - \n" ); -} - -$destination = $argv[1]; -$clone_url = $argv[2]; - -printf( "Fetching \033[32m%s\033[0m...\n", $destination ); -system( 'git clone ' . escapeshellarg( $clone_url ) . ' ' . escapeshellarg( $destination ) ); diff --git a/.maintenance/clone-repository.sh b/.maintenance/clone-repository.sh new file mode 100755 index 0000000..b3d2ea9 --- /dev/null +++ b/.maintenance/clone-repository.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: clone-repository.sh " >&2 + exit 1 +fi + +destination="$1" +clone_url="$2" + +printf "Fetching \033[32m%s\033[0m...\n" "${destination}" +git clone "${clone_url}" "${destination}" diff --git a/composer.json b/composer.json index f4ec007..7ecf94e 100644 --- a/composer.json +++ b/composer.json @@ -319,8 +319,8 @@ "minimum-stability": "dev", "prefer-stable": true, "scripts": { - "pre-install-cmd": "php .maintenance/clone-all-repositories.php", - "pre-update-cmd": "php .maintenance/clone-all-repositories.php", + "pre-install-cmd": "bash .maintenance/clone-all-repositories.sh", + "pre-update-cmd": "bash .maintenance/clone-all-repositories.sh", "post-install-cmd": [ "php .maintenance/symlink-vendor-folders.php", "php .maintenance/phpstorm.exclude-recursive-folders.php" From a7630d5363eb7befb81f71cb4428b38019bcb00a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 22:10:14 +0000 Subject: [PATCH 4/4] Apply reviewer suggestions: jq check, curl -fsS, JSON array validation, cap cores to 8 with env override, drop redundant -n1, exact arg count in clone helper Agent-Logs-Url: https://github.com/wp-cli/wp-cli-dev/sessions/aab74529-eccb-4e94-949e-b1062afb30b2 Co-authored-by: swissspidy <841956+swissspidy@users.noreply.github.com> --- .maintenance/clone-all-repositories.sh | 43 +++++++++++++++++++------- .maintenance/clone-repository.sh | 2 +- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/.maintenance/clone-all-repositories.sh b/.maintenance/clone-all-repositories.sh index 10d5b8b..33bf6b7 100755 --- a/.maintenance/clone-all-repositories.sh +++ b/.maintenance/clone-all-repositories.sh @@ -4,6 +4,11 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if ! command -v jq &>/dev/null; then + echo "Required command 'jq' is not installed or not available in PATH." >&2 + exit 1 +fi + SKIP_LIST=( "autoload-splitter" "composer-changelogs" @@ -18,27 +23,43 @@ SKIP_LIST=( # Detect number of CPU cores, defaulting to 4. if command -v nproc &>/dev/null; then - CORES=$(nproc) + DETECTED_CORES=$(nproc) elif command -v sysctl &>/dev/null; then - CORES=$(sysctl -n hw.logicalcpu 2>/dev/null || echo 4) + DETECTED_CORES=$(sysctl -n hw.logicalcpu 2>/dev/null || echo 4) else + DETECTED_CORES=4 +fi + +MAX_CORES=8 +CORES="${CLONE_JOBS:-${WPCLI_DEV_JOBS:-${DETECTED_CORES}}}" + +if ! [[ "${CORES}" =~ ^[1-9][0-9]*$ ]]; then CORES=4 +elif [[ -z "${CLONE_JOBS:-}" && -z "${WPCLI_DEV_JOBS:-}" && "${CORES}" -gt "${MAX_CORES}" ]]; then + CORES=${MAX_CORES} fi # Fetch repository list from the GitHub API. -CURL_OPTS=(-s) +CURL_OPTS=(-fsS) if [[ -n "${GITHUB_TOKEN:-}" ]]; then CURL_OPTS+=(--header "Authorization: Bearer ${GITHUB_TOKEN}") fi -RESPONSE=$(curl "${CURL_OPTS[@]}" 'https://api.github.com/orgs/wp-cli/repos?per_page=100') +if ! RESPONSE=$(curl "${CURL_OPTS[@]}" 'https://api.github.com/orgs/wp-cli/repos?per_page=100'); then + echo "Failed to fetch repository list from the GitHub API." >&2 + exit 1 +fi -# Detect API errors such as rate limiting. -if echo "${RESPONSE}" | jq -e '.message' &>/dev/null; then - MESSAGE=$(echo "${RESPONSE}" | jq -r '.message') - echo "GitHub responded with: ${MESSAGE}" - echo "If you are running into a rate limiting issue during large events please set GITHUB_TOKEN environment variable." - echo "See https://github.com/settings/tokens" +# Validate the response shape and detect API errors such as rate limiting. +if ! jq -e 'type == "array"' >/dev/null <<< "${RESPONSE}"; then + if jq -e '.message' >/dev/null <<< "${RESPONSE}"; then + MESSAGE=$(jq -r '.message' <<< "${RESPONSE}") + echo "GitHub responded with: ${MESSAGE}" >&2 + echo "If you are running into a rate limiting issue during large events please set GITHUB_TOKEN environment variable." >&2 + echo "See https://github.com/settings/tokens" >&2 + else + echo "GitHub API returned an unexpected response; expected a JSON array of repositories." >&2 + fi exit 1 fi @@ -85,5 +106,5 @@ if [[ ${#CLONE_LIST[@]} -gt 0 ]]; then fi if [[ ${#UPDATE_FOLDERS[@]} -gt 0 ]]; then - printf '%s\n' "${UPDATE_FOLDERS[@]}" | xargs -n1 -P"${CORES}" -I% php "${SCRIPT_DIR}/refresh-repository.php" % + printf '%s\n' "${UPDATE_FOLDERS[@]}" | xargs -P"${CORES}" -I% php "${SCRIPT_DIR}/refresh-repository.php" % fi diff --git a/.maintenance/clone-repository.sh b/.maintenance/clone-repository.sh index b3d2ea9..2b9567f 100755 --- a/.maintenance/clone-repository.sh +++ b/.maintenance/clone-repository.sh @@ -2,7 +2,7 @@ set -euo pipefail -if [[ $# -lt 2 ]]; then +if [[ $# -ne 2 ]]; then echo "Usage: clone-repository.sh " >&2 exit 1 fi