feat(harvest): reforge-harvest — export a run's forge record for triage
Lifecycle step 6 had no tooling: a run's issues, reviews and charter deliverables stayed inside the root-owned sqlite archive. reforge-harvest walks every repo in the org and writes per-repo markdown (issues with comments, PRs with reviews and inline comments), raw API JSON, an INDEX, and a source-stripped tree of the charter repo (GAMEPLAN.md, CONTRACTS.md). Read-only on the forge; re-runnable at any point. Docs: step 6 now names the command and states that harvest is not optional — run #1 sat unharvested for three months with two Critical findings live in deployed forks. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
df0fd9a9ba
commit
109c5e2514
3 changed files with 214 additions and 6 deletions
193
scripts/reforge-harvest.sh
Executable file
193
scripts/reforge-harvest.sh
Executable file
|
|
@ -0,0 +1,193 @@
|
|||
#!/usr/bin/env bash
|
||||
# Reforge lifecycle step 6 (docs/reforge.md): export a run's full forge
|
||||
# record — every issue with its comments, every PR with its reviews and
|
||||
# review comments, per repo in the org, plus a tree snapshot of the charter
|
||||
# repo (GAMEPLAN.md, CONTRACTS.md, ...) — as plain markdown + raw JSON.
|
||||
#
|
||||
# The archive tarball (reforge-reset backup) preserves the run; this makes
|
||||
# it READABLE outside the forge, so findings can be triaged against the
|
||||
# real stack, committed next to the run config, and fed into the next run's
|
||||
# agenda. Without this step a run's value stays locked in a root-owned
|
||||
# sqlite snapshot.
|
||||
#
|
||||
# reforge-harvest [name] [outdir]
|
||||
#
|
||||
# Defaults: name "run"; outdir $REFORGE_HARVEST_DIR/<name>-<stamp>
|
||||
# (REFORGE_HARVEST_DIR defaults to ~/reforge-harvest). Read-only on the
|
||||
# forge; safe to re-run at any point in a run.
|
||||
#
|
||||
# Output layout:
|
||||
# INDEX.md counts per repo + what each file is
|
||||
# <repo>.md rendered issues (with comments) + PRs (with reviews)
|
||||
# raw/<repo>.issues.json API objects, one array per repo (issues incl. PRs)
|
||||
# raw/<repo>.pulls.json
|
||||
# raw/<repo>.comments.json {issue_number -> [comments]}
|
||||
# raw/<repo>.reviews.json {pr_number -> [reviews with .comments]}
|
||||
# charter-tree/ shallow working tree of the charter repo's main
|
||||
set -euo pipefail
|
||||
|
||||
FORGE_URL=${REFORGE_FORGE_URL:-http://localhost:3030}
|
||||
ORG=${REFORGE_ORG:-sandbox-team}
|
||||
ADMIN_USER=${REFORGE_ADMIN_USER:-sandbox-admin}
|
||||
TOKENS_DIR=${REFORGE_TOKENS_DIR:-/var/lib/forgejo-sandbox/tokens}
|
||||
TOKEN_FILE=${REFORGE_ADMIN_TOKEN_FILE:-$TOKENS_DIR/${ADMIN_USER}.token}
|
||||
CHARTER_REPO=${REFORGE_CHARTER_REPO:-charter}
|
||||
|
||||
NAME=${1:-run}
|
||||
STAMP=$(date +%Y%m%d-%H%M%S)
|
||||
OUT=${2:-${REFORGE_HARVEST_DIR:-$HOME/reforge-harvest}/$NAME-$STAMP}
|
||||
|
||||
API="$FORGE_URL/api/v1"
|
||||
TOKEN=$(cat "$TOKEN_FILE")
|
||||
PAGE=50
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
RESP="$WORK/resp"
|
||||
|
||||
api() { # path -> echoes HTTP code, body in $RESP
|
||||
curl -sS -o "$RESP" -w '%{http_code}' \
|
||||
-H "Authorization: token $TOKEN" "$API$1"
|
||||
}
|
||||
|
||||
# Follow ?page= until a page comes back empty; concatenate arrays.
|
||||
api_all() { # path-with-query (no page param) -> JSON array on stdout
|
||||
local path=$1 page=1 sep code
|
||||
case $path in *\?*) sep='&' ;; *) sep='?' ;; esac
|
||||
echo '['
|
||||
local first=1
|
||||
while :; do
|
||||
code=$(api "$path${sep}limit=$PAGE&page=$page")
|
||||
[ "$code" = 200 ] || { echo "harvest: GET $path page $page -> HTTP $code" >&2; cat "$RESP" >&2; exit 1; }
|
||||
local n
|
||||
n=$(jq 'length' <"$RESP")
|
||||
[ "$n" -gt 0 ] || break
|
||||
if [ $first = 1 ]; then first=0; else echo ','; fi
|
||||
jq '.[]' <"$RESP" | jq -s '.[]' | jq -c . | paste -sd, -
|
||||
[ "$n" -lt "$PAGE" ] && break
|
||||
page=$((page + 1))
|
||||
done
|
||||
echo ']'
|
||||
}
|
||||
|
||||
auth_url() { # name
|
||||
echo "http://$ADMIN_USER:$TOKEN@${FORGE_URL#http://}/$ORG/$1.git"
|
||||
}
|
||||
|
||||
mkdir -p "$OUT/raw"
|
||||
echo "→ reforge harvest '$NAME' → $OUT"
|
||||
echo " forge $FORGE_URL org $ORG"
|
||||
echo
|
||||
|
||||
api_all "/orgs/$ORG/repos" | jq -r '.[].name' | sort >"$WORK/repos"
|
||||
[ -s "$WORK/repos" ] || { echo "harvest: no repos in org $ORG" >&2; exit 1; }
|
||||
|
||||
{
|
||||
echo "# Harvest — $NAME"
|
||||
echo
|
||||
echo "Exported $(date -Iseconds) from $FORGE_URL (org \`$ORG\`)."
|
||||
echo "Read-only snapshot of the forge record; the authoritative copy is the"
|
||||
echo "run archive (reforge-reset backup). Raw API objects are under \`raw/\`."
|
||||
echo
|
||||
echo "| repo | issues | open | PRs | merged | file |"
|
||||
echo "|---|---|---|---|---|---|"
|
||||
} >"$OUT/INDEX.md"
|
||||
|
||||
while read -r repo; do
|
||||
printf -- '-- %-16s' "$repo"
|
||||
issues="$OUT/raw/$repo.issues.json"
|
||||
pulls="$OUT/raw/$repo.pulls.json"
|
||||
comments="$OUT/raw/$repo.comments.json"
|
||||
reviews="$OUT/raw/$repo.reviews.json"
|
||||
|
||||
api_all "/repos/$ORG/$repo/issues?state=all&type=issues" | jq . >"$issues"
|
||||
api_all "/repos/$ORG/$repo/pulls?state=all" | jq . >"$pulls"
|
||||
|
||||
# comments: issue-level comments cover both issues and PR conversation
|
||||
{
|
||||
echo '{'
|
||||
first=1
|
||||
for n in $(jq -r '.[].number' "$issues" "$pulls" | sort -nu); do
|
||||
c=$(api_all "/repos/$ORG/$repo/issues/$n/comments")
|
||||
[ "$(echo "$c" | jq 'length')" -gt 0 ] || continue
|
||||
if [ $first = 1 ]; then first=0; else echo ','; fi
|
||||
printf '"%s": %s' "$n" "$c"
|
||||
done
|
||||
echo '}'
|
||||
} | jq . >"$comments"
|
||||
|
||||
# reviews (+ their inline comments) per PR
|
||||
{
|
||||
echo '{'
|
||||
first=1
|
||||
for n in $(jq -r '.[].number' "$pulls" | sort -n); do
|
||||
r=$(api_all "/repos/$ORG/$repo/pulls/$n/reviews")
|
||||
[ "$(echo "$r" | jq 'length')" -gt 0 ] || continue
|
||||
# attach inline comments to each review
|
||||
r=$(echo "$r" | jq -c '.[]' | while read -r rev; do
|
||||
id=$(echo "$rev" | jq -r .id)
|
||||
rc=$(api_all "/repos/$ORG/$repo/pulls/$n/reviews/$id/comments")
|
||||
echo "$rev" | jq --argjson c "$rc" '. + {comments: $c}'
|
||||
done | jq -s .)
|
||||
if [ $first = 1 ]; then first=0; else echo ','; fi
|
||||
printf '"%s": %s' "$n" "$r"
|
||||
done
|
||||
echo '}'
|
||||
} | jq . >"$reviews"
|
||||
|
||||
# ── render ───────────────────────────────────────────────────────────
|
||||
jq -r --arg repo "$repo" --slurpfile comments "$comments" --slurpfile reviews "$reviews" \
|
||||
--slurpfile pulls "$pulls" '
|
||||
def md(s): (s // "" | gsub("\r"; ""));
|
||||
def who(u): (u.login // "?");
|
||||
def when(t): (t // "" | .[0:16] | gsub("T"; " "));
|
||||
def cmts(n): ($comments[0][n | tostring] // []);
|
||||
def revs(n): ($reviews[0][n | tostring] // []);
|
||||
def render_comments(n):
|
||||
(cmts(n) | if length == 0 then "" else
|
||||
"\n#### Comments\n" + (map("- **" + who(.user) + "** (" + when(.created_at) + "):\n\n " + (md(.body) | gsub("\n"; "\n ")) + "\n") | join("\n")) end);
|
||||
def render_reviews(n):
|
||||
(revs(n) | if length == 0 then "" else
|
||||
"\n#### Reviews\n" + (map(
|
||||
"- **" + who(.user) + "** — " + (.state // "?") + " (" + when(.submitted_at) + ")" +
|
||||
(if (md(.body) | length) > 0 then ":\n\n " + (md(.body) | gsub("\n"; "\n ")) else "" end) + "\n" +
|
||||
((.comments // []) | map(" - `" + (.path // "?") + "`: " + (md(.body) | gsub("\n"; " "))) | join("\n")) + "\n"
|
||||
) | join("\n")) end);
|
||||
def entry(kind):
|
||||
"### " + kind + " #" + (.number | tostring) + " " + .title + "\n\n" +
|
||||
"_" + .state + (if (.pull_request.merged // .merged // false) then ", merged" else "" end) +
|
||||
" · " + who(.user) + " · " + when(.created_at) + "_" +
|
||||
(if (.labels // []) | length > 0 then " · labels: " + ((.labels | map(.name)) | join(", ")) else "" end) + "\n\n" +
|
||||
md(.body) + "\n";
|
||||
"# " + $repo + "\n\n" +
|
||||
"## Issues (" + (length | tostring) + ")\n\n" +
|
||||
(map(entry("Issue") + render_comments(.number) + "\n---\n") | join("\n")) +
|
||||
"\n## Pull requests (" + ($pulls[0] | length | tostring) + ")\n\n" +
|
||||
($pulls[0] | map(entry("PR") + render_comments(.number) + render_reviews(.number) + "\n---\n") | join("\n"))
|
||||
' "$issues" >"$OUT/$repo.md"
|
||||
|
||||
ni=$(jq 'length' "$issues"); no=$(jq '[.[] | select(.state=="open")] | length' "$issues")
|
||||
np=$(jq 'length' "$pulls"); nm=$(jq '[.[] | select(.merged==true)] | length' "$pulls")
|
||||
echo "| $repo | $ni | $no | $np | $nm | [$repo.md]($repo.md) |" >>"$OUT/INDEX.md"
|
||||
echo " issues $ni (open $no) PRs $np (merged $nm)"
|
||||
done <"$WORK/repos"
|
||||
|
||||
# ── charter tree: the run's synthesized deliverables live as files there ──
|
||||
if grep -qx "$CHARTER_REPO" "$WORK/repos"; then
|
||||
if git clone --quiet --depth 1 "$(auth_url "$CHARTER_REPO")" "$OUT/charter-tree" 2>/dev/null; then
|
||||
rm -rf "$OUT/charter-tree/.git"
|
||||
{
|
||||
echo
|
||||
echo "Charter repo tree (main, source-stripped): \`charter-tree/\` —"
|
||||
find "$OUT/charter-tree" -type f | sed "s|$OUT/charter-tree/||" | sort | sed 's/^/ - /'
|
||||
} >>"$OUT/INDEX.md"
|
||||
echo "-- charter tree -> charter-tree/"
|
||||
else
|
||||
echo "-- charter tree: $CHARTER_REPO has no main yet, skipped"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "harvest: $OUT"
|
||||
echo "next: triage <repo>.md against the real stack; commit the harvest next to"
|
||||
echo " your run config; carry still-live findings into the next agenda."
|
||||
Loading…
Add table
Add a link
Reference in a new issue