diff --git a/README.md b/README.md index 333bce0..32153e2 100644 --- a/README.md +++ b/README.md @@ -153,10 +153,37 @@ Whichever tool you use, the definitions are only found if ruby-build can actuall If a build fails in a way that looks nothing like the notes in this repo, check that the definition was actually picked up before debugging the compiler error. +### CI + +There's no cloud CI here. Run `bin/ci` before merging, and it signs off the commit for you +on success: + +```bash +bin/ci # lint + the full build matrix, then gh signoff +bin/ci --lint # lint only, no Docker — seconds, good for a quick check +bin/ci arch # lint + Arch only +bin/ci arch 2.7.8 # lint + a single build +``` + +Only a full run signs off. Anything narrower reports its results and explicitly declines to +sign, because a green tick that covered one platform is worse than no tick. + +The lint pass is cheap and catches the two mistakes that otherwise surface ten minutes into +a Docker build: a syntax error in a definition (ruby-build sources these, so a stray quote +is a build failure), and an `install_package` URL with no `#sha256` (ruby-build silently +skips verification when the checksum is missing). + +Sign-off needs the extension: + +```bash +gh extension install basecamp/gh-signoff +``` + ### Testing `test/build` builds definitions in throwaway Docker containers, so a clean-machine build -is checked without touching your own toolchain. +is checked without touching your own toolchain. `bin/ci` runs it for you; use it directly +when you want a specific slice. ```bash test/build arch 1.8.7-p374 # one version on one platform diff --git a/bin/ci b/bin/ci new file mode 100755 index 0000000..13b25c6 --- /dev/null +++ b/bin/ci @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# Local CI. There is no cloud CI for this repo — run this, then `gh signoff` +# marks the commit green so the PR can merge. +# +# Bash rather than the Ruby CI class the app repos use, deliberately: this repo +# exists because a working Ruby is the thing you don't have yet. Its own CI must +# not need one. +# +# bin/ci # lint + full build matrix, then sign off +# bin/ci arch # lint + Arch only (no signoff — partial run) +# bin/ci arch 2.7.8 # lint + one build (no signoff — partial run) +# bin/ci --lint # lint only, no Docker (no signoff — partial run) +# +# Only a full run signs off. A partial run deliberately won't: signing off on a +# subset is worse than not signing off at all. +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Before anything else, so --help doesn't sit through a lint pass first. +case "${1:-}" in + -h|--help) sed -n '2,15p' "$0" | sed 's/^#\{1,\} \{0,1\}//'; exit 0 ;; +esac + +BANNER=$'\033[1;32m'; TITLE=$'\033[1;35m'; SUBTITLE=$'\033[1;90m' +ERROR=$'\033[1;31m'; SUCCESS=$'\033[1;32m'; RESET=$'\033[0m' + +failures=() + +# A URL runs to the next whitespace or quote. Shell metacharacters are captured here and +# cut afterwards by trim_url, rather than excluded outright, so that an unquoted URL with +# a command attached (`…tar.gz;echo`) is seen whole before being trimmed — grep alone +# can't tell where the URL ends and the next command begins. +# +# Built here rather than inline because embedding a single quote inside a single-quoted +# grep pattern is unreadable. +SQ="'" +URL_RE="https?://[^[:space:]\"$SQ\`]+" + +# Cut an unquoted URL where the shell command continues. grep stops at whitespace, so +# `…tar.gz;echo done` arrives with `;echo` still attached — and trimming only trailing +# punctuation left it there, hiding the archive extension and skipping the download. +# +# Cutting at the *first* operator rather than the last is safe now that archive shape +# decides what gets checked: the savannah gitweb links are the only URLs here with +# mid-string semicolons, and they aren't archives either way. +trim_url() { + local u="$1" + printf '%s' "${u%%[;\)\(\&\|\<\>]*}" +} + +echo "${BANNER}🚀 Local CI for ruby-dev${RESET}" + +heading() { printf '\n%s%s%s\n' "$TITLE" "$1" "$RESET"; [ $# -gt 1 ] && printf '%s%s%s\n' "$SUBTITLE" "$2" "$RESET"; return 0; } +pass() { printf '%s ✓ %s%s\n' "$SUCCESS" "$1" "$RESET"; } +fail() { printf '%s ✗ %s%s\n' "$ERROR" "$1" "$RESET"; failures+=("$1"); } + +# Plain `sort`, not `sort -V`: macOS ships BSD sort, which rejects -V. That failure would +# not trip set -e here (it's a command substitution inside a `for` list), so the list would +# come back empty and every check below would silently pass over nothing — a green run that +# linted zero definitions. Ordering is cosmetic for linting, so portability wins. +definitions() { ls -1 [0-9]* 2>/dev/null | sort; } + +# And belt-and-braces: whatever the cause — wrong directory, failed glob, broken sort — an +# empty list must be a hard error, never a quiet pass. +require_definitions() { + if [ -z "$(definitions)" ]; then + printf '%serror: no definition files found in %s — refusing to report success%s\n' \ + "$ERROR" "$PWD" "$RESET" >&2 + exit 1 + fi +} + +# --- Shell syntax ----------------------------------------------------------- +# Definitions are sourced by ruby-build, so a syntax error in one is a build +# failure several minutes into a Docker run. Catch it in milliseconds instead. +lint_syntax() { + heading "Syntax" "bash -n over scripts and definitions" + local f + for f in bin/ci test/build $(definitions); do + if bash -n "$f" 2>/dev/null; then pass "$f"; else fail "$f has a syntax error"; bash -n "$f" || true; fi + done +} + +# --- Checksums -------------------------------------------------------------- +# Fold backslash-continued lines into one, so an install_package spelled across several +# lines is inspected whole. Matching raw lines would see only the first, and a missing +# checksum on a continuation would pass unnoticed — with another well-formed call in the +# file, even the "found nothing" guard below wouldn't fire. +# +# Pure bash rather than sed/awk: the usual line-joining one-liners differ between BSD and +# GNU, and macOS portability is the entire point of this section. +join_continuations() { + local line acc="" + while IFS= read -r line || [ -n "$line" ]; do + if [ "${line%\\}" != "$line" ]; then + # Join with nothing, matching shell: a backslash-newline is removed entirely, it + # does not become whitespace. Inserting a space would split tokens shell keeps + # together — `"https\` + `://host/x"` is one URL to bash but would arrive here as + # `https ://host/x` and match nothing, silently skipping that download. Ordinary + # continuations already carry their own whitespace around the backslash. + acc="${acc}${line%\\}" + else + printf '%s%s\n' "$acc" "$line" + acc="" + fi + done < "$1" + [ -n "$acc" ] && printf '%s\n' "$acc" + return 0 +} + +# ruby-build only verifies a download when the URL carries a #checksum. Without one it +# fetches and builds whatever it got, silently. A missing checksum is the kind of thing +# that survives review, so assert it here. +# +# Every URL in the file is considered, wherever it appears, and the ones that look like +# source archives must carry a digest. No attempt is made to work out which are arguments +# and which are prose — that question is what made earlier versions of this wrong. +# +# The cost is that a tarball URL written in a comment gets flagged too. That is rare, it +# says exactly what to do about it, and it errs loud rather than quiet. Reference links in +# these definitions point at issues and repos, not tarballs, so it doesn't arise today. +# +# Not checked, deliberately: the config.guess and config.sub fetches in 1.8.7 and 1.9.3. +# They come from GNU's git web view, which serves a moving HEAD with no release tarball +# and no digest to cite. They aren't archives, so they fall outside this rule naturally +# rather than needing an exemption list. The same is true of any future non-archive +# download, which is the honest limitation of matching on shape. +lint_checksums() { + heading "Checksums" "every source archive carries a #sha256" + local f raw url base path ok found + for f in $(definitions); do + ok=true + found=0 + while IFS= read -r raw; do + # Two questions, deliberately answered from different strings. + # + # Whether a digest is present is asked of the untrimmed URL, because `&` is both a + # shell operator and ordinary query syntax. Trimming first would cut + # `…tar.gz?a=1&b=2#` at the ampersand and report a missing digest that is + # right there. + # + # What kind of URL it is gets asked of the trimmed path, so an attached command + # (`…tar.gz;echo`) or a query string (`…tar.gz?download=1`) can't hide the extension. + url=$(trim_url "$raw") + base=${url%%#*} # up to the digest — what to report + path=${base%%\?*} # and without the query — what to classify on + # Only archives are checked, and that is what removes the need to understand the + # shell around them. install_package fetches release tarballs, so a URL ending in + # an archive extension is a download; anything else is a reference. + # + # The alternative was deciding by position — is this URL an argument, or inside a + # comment, or inside a heredoc — which means tokenising shell, and that is where + # every bug in this check came from. Quoting, continuations, `;#`, ANSI-C strings, + # heredocs: each one handled, each one exposing the next. Matching on what the URL + # *is* needs none of it. + case "$path" in + *.tar.gz|*.tar.bz2|*.tar.xz|*.tgz|*.tbz2|*.tar.Z|*.zip) ;; + *) continue ;; + esac + found=$(( found + 1 )) + # Unanchored, with a boundary, since the digest may be followed by a query + # remnant or an attached command rather than ending the string. + [[ $raw =~ \#[0-9a-f]{64}([^0-9a-f]|$) ]] || { fail "$f: no sha256 on $base"; ok=false; } + done < <(join_continuations "$f" | grep -oE "$URL_RE") + # Every definition fetches at least Ruby itself as a tarball. Finding none means the + # extractor stopped matching, not that the file is clean — don't call that a pass. + if [ "$found" -eq 0 ]; then + fail "$f: no source archives found — checksum lint could not inspect this file" + ok=false + fi + # Explicit if, not `$ok && pass`: that leaves the loop's exit status at 1 when + # the *last* definition fails, and set -e then kills the run before the later + # checks and the summary — failures reported, no verdict. + if $ok; then pass "$f"; fi + done +} + +# --- Shellcheck ------------------------------------------------------------- +# Optional: not everywhere, and not worth blocking a build matrix over. Report +# the skip out loud rather than passing silently, so nobody reads a green run as +# "shellcheck is clean" when it never ran. +lint_shellcheck() { + heading "Shellcheck" "optional static analysis" + if ! command -v shellcheck >/dev/null; then + printf '%s – skipped: shellcheck not installed%s\n' "$SUBTITLE" "$RESET" + return 0 + fi + # Scripts only. Definitions are ruby-build fragments, not standalone scripts — + # they call install_package et al from their sourcing shell, so shellcheck reads + # every one of those as an unknown command. + # + # --severity=warning on purpose: the info tier here is all intentional (ls over + # find on version-numbered filenames, deliberate word splitting). Gating on info + # would mean either noisy failures or a scattering of disable comments, and both + # train people to ignore the step. + if shellcheck -s bash --severity=warning bin/ci test/build; then + pass "scripts" + else + fail "shellcheck" + fi +} + +# --- Build matrix ----------------------------------------------------------- +build_matrix() { + heading "Builds" "test/build ${*:-all}" + if ! docker info >/dev/null 2>&1; then + fail "Docker isn't available — the build matrix can't run" + return 0 + fi + if test/build "${@:-all}"; then pass "build matrix"; else fail "build matrix"; fi +} + +# --- Signoff ---------------------------------------------------------------- +signoff() { + heading "📋 Signoff" "gh signoff" + if ! command -v gh >/dev/null || ! gh extension list 2>/dev/null | grep -q gh-signoff; then + printf '%s – skipped: gh signoff not installed (gh extension install basecamp/gh-signoff)%s\n' \ + "$SUBTITLE" "$RESET" + return 0 + fi + gh signoff +} + +started=$SECONDS + +require_definitions +lint_syntax +lint_checksums +lint_shellcheck + +partial=false +case "${1:-}" in + --lint) partial=true ;; + "") build_matrix ;; + *) partial=true; build_matrix "$@" ;; +esac + +elapsed=$(( SECONDS - started )) + +if [ ${#failures[@]} -eq 0 ]; then + printf '\n%s✅ CI passed in %ds%s\n' "$SUCCESS" "$elapsed" "$RESET" + if $partial; then + printf '%s📋 Partial run — not signing off. Run bin/ci with no arguments to sign off.%s\n' \ + "$SUBTITLE" "$RESET" + else + signoff + fi +else + printf '\n%s❌ CI failed in %ds%s\n' "$ERROR" "$elapsed" "$RESET" + for f in "${failures[@]}"; do printf '%s • %s%s\n' "$ERROR" "$f" "$RESET"; done + printf '%s📋 No sign-off. Fix the issues and try again.%s\n' "$SUBTITLE" "$RESET" + exit 1 +fi diff --git a/test/build b/test/build index 79545b2..7defbb6 100755 --- a/test/build +++ b/test/build @@ -4,15 +4,31 @@ set -euo pipefail cd "$(dirname "$0")/.." PLATFORMS="ubuntu-noble arch" -VERSIONS=$(ls -1 [0-9]* 2>/dev/null | sort -rV) + +# macOS ships BSD sort, which rejects -V. Fall back to plain sort there rather than dying +# on an unsupported flag — the order is cosmetic, it only sets which builds start first. +if printf '1\n' | sort -V >/dev/null 2>&1; then + VERSIONS=$(ls -1 [0-9]* 2>/dev/null | sort -rV) +else + VERSIONS=$(ls -1 [0-9]* 2>/dev/null | sort -r) +fi +[ -n "$VERSIONS" ] || { echo "error: no definition files found in $PWD" >&2; exit 1; } # Builds run concurrently. JOBS caps how many containers are in flight; MAKE_JOBS caps -# make parallelism inside each one. The product is what actually hits the CPU, so the -# defaults aim for a mild oversubscribe rather than JOBS x nproc meltdown. +# make parallelism inside each one. +# +# JOBS is deliberately generous relative to core count. Measuring a real run, each +# container averages ~1.0 core, not the MAKE_JOBS it's allowed: these old Ruby builds +# spend most of their wall time single-threaded — miniruby bootstrapping and generating +# exts.mk, the dependency-serialized tail of make, `gem install bundler` — with only brief +# bursts of parallel compilation. 1.9.3 is stricter still, forcing make -j1 in its own +# definition to dodge a parallel-make race. Budgeting MAKE_JOBS cores per container left +# the machine ~75% idle, and the long pole (1.9.3, ~190s vs ~110s for the rest) could sit +# queued behind short builds instead of starting immediately. CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) -JOBS=${JOBS:-$(( CORES / 4 ))} +JOBS=${JOBS:-$(( CORES / 2 ))} (( JOBS < 1 )) && JOBS=1 -(( JOBS > 12 )) && JOBS=12 +(( JOBS > 16 )) && JOBS=16 MAKE_JOBS=${MAKE_JOBS:-4} # Use multi-platform builder if available (faster cross-arch builds) @@ -58,7 +74,8 @@ build_image() { local platform=$1 local dockerfile="test/${platform}.dockerfile" local image="ruby-build-test:${platform}" - local target_platform=$(platform_for "$platform") + local target_platform + target_platform=$(platform_for "$platform") local build_args=(-f "$dockerfile" -t "$image" --load .) @@ -109,7 +126,8 @@ test_ruby() { local platform=$1 local version=$2 local image="ruby-build-test:${platform}" - local target_platform=$(platform_for "$platform") + local target_platform + target_platform=$(platform_for "$platform") local platform_flag="" [[ -n "$target_platform" ]] && platform_flag="--platform $target_platform" @@ -140,6 +158,11 @@ test_ruby() { " local started=$SECONDS output elapsed + # $platform_flag is deliberately unquoted: it holds two words ("--platform + # linux/amd64") and must split into two arguments. An array would be the tidier + # idiom, but expanding an empty one under `set -u` is an error on macOS's Bash + # 3.2, and this script has to keep working there. + # shellcheck disable=SC2086 if output=$(docker run --rm $platform_flag -e MAKE_OPTS="-j${MAKE_JOBS}" \ "$image" bash -c "$build_script" 2>&1); then elapsed=$(( SECONDS - started ))