diff --git a/render.txt b/render.txt index 081dc99..7368f36 100644 --- a/render.txt +++ b/render.txt @@ -20,7 +20,7 @@ esac test "$last" = 0 && ago="none on record" || ago="$(( (now - last) / 3600 )) h ago" echo "due: RECONCILE=$r RECONCILE_HOURS=$h, last reconcile $ago: $(test -f /work/.run/reconcile && echo reconciling || echo not this run)" -task: [list] rm -f /work/.run/publish.txt /work/.run/labeled.txt /work/.run/orphans.txt /work/.run/unlanded.txt /work/.run/fresh.txt +task: [list] rm -f /work/.run/publish.txt /work/.run/labeled.txt /work/.run/orphans.txt /work/.run/unlanded.txt /work/.run/fresh.txt /work/.run/index-same.txt task: [list] mkdir -p /work/.run /work/staging task: [retry] for i in 1 2 3 4 5; do rc=0; ( TZ=UTC rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://ftp.gnu.org/alpha/ > /work/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. @@ -285,7 +285,7 @@ esac task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete && : > /work/.run/index-real.txt task: [pages] { cat /work/.run/applied.txt; awk '{ print $0 "\t\t" }' /work/.run/upstream-dirs.txt; } | LC_ALL=C sort > /work/.run/page-state.txt -task: [pages] printf '%s\n' 2 katoptra.org.directory.index.html '' '' https://alpha.gnu.org/gnu/ '' gnu-alpha.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/.run/index-stamp.txt +task: [pages] printf '%s\n' 3 katoptra.org.directory.index.html '' '' https://alpha.gnu.org/gnu/ '' gnu-alpha.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/.run/index-stamp.txt task: [pages] awk 'NR == 1 && /^#pages / { next } { print }' /work/.run/indexed.txt > /work/.run/index-old.txt task: [pages] { LC_ALL=C comm -3 /work/.run/index-old.txt /work/.run/page-state.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'; head -1 /work/.run/indexed.txt | cmp -s - /work/.run/index-stamp.txt || cat /work/.run/page-state.txt; } | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/.run/index-dirty.txt @@ -308,6 +308,7 @@ task: [pages] test ! -s /work/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i + G = "\\.(sig|sign|asc|gpg|dsig|proof|md5|md5sum|sha|sha1|sha224|sha256|sha256sum|sha512|sum|checksums)$" if (C == "" && (j = index(H, ".")) > 1) C = substr(H, 1, j - 1) "@" substr(H, j + 1) css = ":root{color-scheme:light dark;--bg:#f6f5f2;--fg:#1c1c28;--mut:#6e6e7e;--line:#e2e0da;--hov:#edebe5;--a:#3b45d9}" css = css "@media (prefers-color-scheme:dark){:root{--bg:#121218;--fg:#f0f0f5;--mut:#8b8b9c;--line:#262630;--hov:#1a1a22;--a:#8b93ff}}" @@ -339,7 +340,7 @@ function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s function w(s) { if (real == "") printf("%s", s) > fa; if (cur != "") printf("%s", s) > fb } function shut( i, r, x, c, e, k, b) { split("", fold); split("", links) for (i = 1; i <= nfl; i++) { r = fname[i]; x = "" - while (match(r, /\.(sig|asc|sign|sha256|sha512)$/)) { r = substr(r, 1, RSTART - 1); if (r in has) x = r } + while (match(tolower(r), G)) { r = substr(r, 1, RSTART - 1); if (r in has) x = r } if (x != "") { fold[i] = 1 links[x] = links[x] "" esc(substr(fname[i], length(x) + 1)) "" } } c = (nd ? grp(nd) (nd == 1 ? " directory" : " directories") : "") @@ -353,7 +354,7 @@ function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s if (cur != "") w("../\n") for (i = 1; i <= nd; i++) w("" esc(dname[i]) "/\n") for (i = 1; i <= nfl; i++) if (!(i in fold)) - w("" esc(fname[i]) "" links[fname[i]] "" ymd(ftime[i]) "" iec(fsize[i]) "\n") + w("" esc(fname[i]) "" links[fname[i]] "" ymd(ftime[i]) "" iec(fsize[i]) "\n") w("" foot() "\n") if (real == "") close(fa); if (cur != "") close(fb) } !live || $1 != cur { if (live) shut(); open($1) } @@ -362,6 +363,16 @@ function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } { fname[++nfl] = $3; fsize[nfl] = $4; ftime[nfl] = $5; has[$3] = 1; tot += $4 } END { if (live) shut() }' +task: [index-same] aws s3api list-objects-v2 --bucket gnu-alpha --query 'Contents[].[Key,ETag]' --output text > /work/.run/bucket-etags.txt || { echo "index-same: the listing failed; index sends each page"; : > /work/.run/bucket-etags.txt; } +task: [index-same] { (cd /work/staging && find . -type f -exec md5sum {} +) | sed 's# \./#\tp\t#'; for d in /work/.run/slash/*/; do test -d "$d" || continue; (cd "$d" && find . -type f -exec md5sum {} +) | sed 's# \./#\ts\t#'; done; } > /work/.run/page-md5.txt +task: [index-same] awk -F'\t' -v E="/work/.run/bucket-etags.txt" -v H="/work/.run/page-md5.txt" -v I="katoptra.org.directory.index.html" -v S="/work/staging/" -v R="/work/.run/slash/" -v Y="/work/.run/index-same.txt" -v M="/work/.run/index-make.new" ' + FILENAME == E { gsub(/"/, "", $2); e[$1] = $2; next } + FILENAME == H { if ($2 == "s") d = $3 "/"; else if (substr($3, length($3) - length(I) + 1) == I) d = substr($3, 1, length($3) - length(I)); else next + seen[d] = 1; if (e[$3] != $1) diff[d] = 1; next } + !($0 in seen) || ($0 in diff) { print > M; next } + { print > Y; print S $0 I; if ($0 != "") { b = substr($0, 1, length($0) - 1); print R split(b, c, "/") "/" b } }' /work/.run/bucket-etags.txt /work/.run/page-md5.txt /work/.run/index-make.txt \ +| tr '\n' '\0' | xargs -0r rm -f +task: [index-same] touch /work/.run/index-same.txt /work/.run/index-make.new && mv /work/.run/index-make.new /work/.run/index-make.txt task: [index] test ! -s /work/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/staging/ s3://gnu-alpha/ task: [index] for d in /work/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://gnu-alpha/; done task: [index-copy] while IFS="$(printf '\t')" read -r s k; do aws s3api copy-object --bucket gnu-alpha --copy-source "gnu-alpha/$s" --key "$k" --metadata-directive COPY > /dev/null; echo "copy: $s to $k"; done < /work/.run/index-real.txt