diff --git a/exe/findingaid-cache-regen/fa-full-regen b/exe/findingaid-cache-regen/fa-full-regen index 5343ff8..140fd02 100755 --- a/exe/findingaid-cache-regen/fa-full-regen +++ b/exe/findingaid-cache-regen/fa-full-regen @@ -1,17 +1,88 @@ #!/bin/bash set -euo pipefail -# The environment should provide these variables -# SOLR_URL: e.g. https://solrhost.example.com/solr/select +usage() { + cat <<'EOF' +Usage: fa-full-regen [-v|--verbose] + +Regenerate the cached page for every finding aid Solr lists as a collection. + +Options: + -v, --verbose List every ARK that could not be regenerated. Without it only + the count is reported + -h, --help Show this help. + +Environment: + SOLR_URL Required, e.g. https://solrhost.example.com/solr/select +EOF +} + +VERBOSE=0 + +while [ $# -gt 0 ]; do + case "$1" in + -v|--verbose) VERBOSE=1 ;; + -h|--help) usage; exit 0 ;; + *) + echo "fa-full-regen: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done ARKS_FILE="$(mktemp)" export ARKS_FILE -trap 'rm -f "$ARKS_FILE"' EXIT +ERRORS_FILE="$(mktemp)" + +trap 'rm -f "$ARKS_FILE" "$ERRORS_FILE"' EXIT -echo "Attempting generation of a list of finding aids to reprocess" +echo "Attempting generation of a list of finding aids to regenerate" fetch-ead-arks -echo "List generated" -echo "Reprocessing finding aids" -xargs -n 1 -P 4 fa-regen < "$ARKS_FILE" +FOUND=$(wc -l < "$ARKS_FILE") + +PROCESSED=0 + +if [ "$FOUND" -le 0 ]; then + echo "$FOUND findingaid records found. Exiting..." + exit 0 +fi + +function print_progress { + PROCESSED=$((PROCESSED + 1)) + local PROGRESS_PERCENTAGE=$(( 100 * PROCESSED / FOUND )) + local PROGRESS_STR="[" + local HASH_COUNT=$(( PROGRESS_PERCENTAGE / 5 )) + local i + for i in {1..20}; do + if [ "$i" -le $HASH_COUNT ]; then + PROGRESS_STR+="#" + else + PROGRESS_STR+=" " + fi + done + PROGRESS_STR+="] " + PROGRESS_STR+="($PROGRESS_PERCENTAGE)%" + echo -ne "$PROGRESS_STR\r" +} + +xargs -n 1 -P 4 fa-regen < "$ARKS_FILE" 2> "$ERRORS_FILE" | while read -r _; do + print_progress +done +printf '\n' + +# Missing EADs are a data condition, not a failed run, so report and exit 0. +if [ -s "$ERRORS_FILE" ]; then + MISSING=$(wc -l < "$ERRORS_FILE") + if [ "$VERBOSE" -eq 1 ]; then + echo "$MISSING of $FOUND finding aids could not be regenerated:" >&2 + cat "$ERRORS_FILE" >&2 + else + echo "$MISSING of $FOUND finding aids could not be regenerated" \ + "(re-run with --verbose for the list)" >&2 + fi +fi + echo "Finding aids reprocessed" diff --git a/exe/findingaid-cache-regen/fa-regen b/exe/findingaid-cache-regen/fa-regen index 52c59d0..7afeb5e 100755 --- a/exe/findingaid-cache-regen/fa-regen +++ b/exe/findingaid-cache-regen/fa-regen @@ -6,4 +6,15 @@ TARGET_URL=http://web ID=$1 BASE="$TARGET_URL/fa/findingaid/?id=$ID&invalidate_cache=1" echo "$BASE" -curl "$BASE" >/dev/null + +# A finding aid whose EAD is missing from the pairtree redirects to / +STATUS=$(curl -sS -o /dev/null -w '%{http_code}' "$BASE") || STATUS=000 + +case "$STATUS" in + 2??) ;; + 302) echo "$ID: not found (redirected to /); no EAD in the pairtree" >&2 ;; + 000) echo "$ID: request to $TARGET_URL failed" >&2 ;; + *) echo "$ID: unexpected HTTP $STATUS" >&2 ;; +esac + +exit 0 diff --git a/exe/findingaid-cache-regen/fetch-ead-arks b/exe/findingaid-cache-regen/fetch-ead-arks index b43778b..1546be7 100755 --- a/exe/findingaid-cache-regen/fetch-ead-arks +++ b/exe/findingaid-cache-regen/fetch-ead-arks @@ -5,12 +5,16 @@ set -euo pipefail # SOLR_URL: e.g. https://solrhost.example.com/solr/select # ARKS_FILE: where ARKs should be written -COUNT_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections" -numFound=$(curl -s "$COUNT_URL" | jq -r .response.numFound) -echo "Found $numFound collections" +COUNT_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=0" +NUM_FOUND=$(curl -s "$COUNT_URL" | jq -r '.response.numFound') + +if ! [[ "$NUM_FOUND" =~ ^[0-9]+$ ]]; then + echo "fa-get-count: unexpected response from Solr: $NUM_FOUND" >&2 + exit 1 +fi echo "Fetching ids" -EADS_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=${numFound}" +EADS_URL="${SOLR_URL}?wt=json&fl=id&fq=format:collections&rows=${NUM_FOUND}" curl -s "$EADS_URL" | jq -r '.response.docs[] | .id' > "$ARKS_FILE" echo "IDs written to $ARKS_FILE"