cd /Users/Ali/PKM/scratch/pkm-deepdive
cat > gemini-key-clarification.txt <<'EOF'

---
## GEMINI KEY DISTINCTION (Ali correction, 2026-06-29) — do NOT re-confuse
The keys the academy has are **VERTEX** keys ($1,700 GCP credits, ~$22 used, working for academy text). **AI Studio (Gemini Developer API, generativelanguage.googleapis.com) needs SEPARATE keys that Ali would have to CREATE** (own billing). The spike's 429 "prepayment credits depleted" came from pointing a Vertex key at the AI-Studio endpoint (mismatch) — NOT a real quota/billing problem on Ali's Vertex credits.
RULE: tutor/LLM = VERTEX endpoint + existing Vertex auth. Only reach for AI Studio if a capability (e.g. the Interactions API) is AI-Studio-ONLY, and that requires Ali to create new AI Studio keys (founder action) — flag it; do NOT silently try to make a Vertex key work on AI Studio.
Gemini-vertex re-route lane decision: (a) Interactions API on Vertex → use Vertex auth, no new keys; (b) Interactions AI-Studio-only → launch-safe fallback = standard Vertex Gemini tutor (no Interactions features), Interactions as post-launch (then Ali decides on AI Studio keys).
EOF
base64 < gemini-key-clarification.txt | ./vds-exec.sh runl '
  T=$(base64 -d); echo "$T" >> /srv/aina/ops/FACTORY-GUARDRAILS.md; echo "$T" >> /srv/aina/ops/lanes/lane-gemini-vertex.md
  echo CLARIFIED
  echo "== gemini-vertex lane still running? =="; export PATH=$PATH:/usr/local/bin; pgrep -af "codex exec" | grep -v pgrep | wc -l'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'set +e
export PATH=$PATH:/usr/local/bin
echo "== does the watchdog watch PR bot comments? =="
grep -niE "pr |gh pr|bot|review|codex.*comment|resolveReview" /srv/aina/ops/watchdog.sh 2>/dev/null | head
echo "== any standing PR-watcher script/cron? =="
ls /srv/aina/ops/*pr*watch* /srv/aina/ops/*bot*watch* 2>/dev/null; crontab -l 2>/dev/null | grep -iE "pr|bot|watch" | head
echo "== current OPEN PRs + unresolved bot comments NOW =="
for pr in "ainative-academy/aina-academy 112" "ainative-academy/aina-academy 108" "ainative-academy/aina-academy 113" "ainative-academy/aina-mission-control 12"; do
  set -- $pr; repo=$1; n=$2
  st=$(gh pr view $n --repo $repo --json state --jq .state 2>/dev/null)
  [ "$st" = "OPEN" ] || continue
  echo "--- $repo #$n ($st) ---"
  gh api "repos/$repo/pulls/$n/comments" --jq ".[] | select(.user.login==\"chatgpt-codex-connector\" or (.user.login|test(\"codex\";\"i\"))) | \"[\(.path):\(.line)] \(.body[0:90])\"" 2>/dev/null | head -8
done
echo "== open PR list (academy) =="; gh pr list --repo ainative-academy/aina-academy --state open --json number,headRefName --jq ".[]|\"\(.number) \(.headRefName)\"" 2>/dev/null'

cd /Users/Ali/PKM/scratch/pkm-deepdive
cat > pr-bot-watcher.sh <<'EOS'
#!/usr/bin/env bash
# AIN-143: standing PR bot-review watcher. Every run: for each OPEN non-draft PR in the AINA repos,
# if it has Codex-bot review comments and hasn't been auto-addressed in the last 45min, dispatch a
# gpt-5.4-mini fix-pass that addresses + replies/resolves the threads (does NOT merge). Idempotent via per-PR lock.
set +e
export PATH=$PATH:/usr/local/bin:$HOME/.npm-global/bin
OPS=/srv/aina/ops
LOG=$OPS/pr-bot-watch.log
REPOS="ainative-academy/aina-academy ainative-academy/aina-mission-control ainative-academy/aina-platform"
C="exec --json --dangerously-bypass-approvals-and-sandbox --model gpt-5.4-mini -c model_reasoning_effort=medium --skip-git-repo-check -"
ts(){ date "+%Y-%m-%dT%H:%M:%S"; }
for repo in $REPOS; do
  prs=$(gh pr list --repo "$repo" --state open --json number,isDraft --jq '.[]|select(.isDraft==false)|.number' 2>/dev/null)
  for n in $prs; do
    # count unresolved-ish bot review comments (codex connector)
    bc=$(gh api "repos/$repo/pulls/$n/comments" --jq '[.[]|select(.user.login=="chatgpt-codex-connector" or (.user.login|test("codex";"i")))]|length' 2>/dev/null)
    [ -z "$bc" ] && bc=0
    [ "$bc" -eq 0 ] && continue
    slug=$(echo "$repo" | tr '/' '-')
    lock="$OPS/.botfix-$slug-$n.lock"
    if [ -n "$(find "$lock" -mmin -45 2>/dev/null)" ]; then
      echo "$(ts) skip $repo#$n ($bc bot comments) — fix dispatched <45min ago" >> "$LOG"; continue
    fi
    # build the per-PR fix prompt
    pf="/tmp/botfix-$slug-$n.md"
    cat > "$pf" <<EOF
# Lane: address Codex bot review comments on $repo PR #$n
## AUTONOMY: proceed without asking; small fix pass; do NOT merge. gpt-5.4-mini.
Find the PR's worktree/branch (gh pr view $n --repo $repo --json headRefName,headRefOid; the branch is checked out under /home/ali/agent-workspaces/ or /srv/repos). For the open PR #$n in $repo:
1. Fetch the UNRESOLVED Codex-bot (chatgpt-codex-connector) review threads (gh api repos/$repo/pulls/$n/comments + the GraphQL reviewThreads isResolved=false).
2. For each VALID one: implement the fix on the PR branch, run the repo's build/tests for the touched area, commit + push.
3. For each NOT valid (stale / references files not in the current diff): reply with current-diff evidence.
4. Resolve every addressed/replied thread (GraphQL resolveReviewThread). Do NOT merge. Honest report if blocked.
Report one line to $OPS/pr-bot-watch.log on completion.
EOF
    ( cd /srv/repos/aina-academy 2>/dev/null || cd /srv/repos; cat "$pf" | codex $C > "$OPS/codex-botfix-$slug-$n.log" 2>&1 ) &
    touch "$lock"
    echo "$(ts) dispatched botfix $repo#$n ($bc bot comments) pid=$!" >> "$LOG"
  done
done
EOS
# push watcher, wire into watchdog, run once
base64 < pr-bot-watcher.sh | ./vds-exec.sh runl '
  base64 -d > /srv/aina/ops/pr-bot-watcher.sh && chmod +x /srv/aina/ops/pr-bot-watcher.sh
  # wire into watchdog if not already
  if ! grep -q "pr-bot-watcher.sh" /srv/aina/ops/watchdog.sh 2>/dev/null; then
    printf "\n# AIN-143 PR bot-review watcher (addresses Codex bot comments on open PRs)\nbash /srv/aina/ops/pr-bot-watcher.sh >> /srv/aina/ops/pr-bot-watch.log 2>&1\n" >> /srv/aina/ops/watchdog.sh
    echo "WIRED into watchdog"
  else echo "already wired"; fi
  echo "== run once now =="; bash /srv/aina/ops/pr-bot-watcher.sh; sleep 6
  echo "== watch log =="; tail -8 /srv/aina/ops/pr-bot-watch.log 2>/dev/null
  echo "== procs =="; export PATH=$PATH:/usr/local/bin; pgrep -af "codex exec" | grep -v pgrep | wc -l'

cd /Users/Ali/PKM/scratch/pkm-deepdive
cat > pr-bot-watcher.sh <<'EOS'
#!/usr/bin/env bash
# AIN-143: standing PR bot-review watcher. Each run: for OPEN, non-draft, RECENTLY-UPDATED PRs in the AINA repos,
# if a PR has Codex-bot review comments and wasn't auto-addressed in the last 45min, dispatch a gpt-5.4-mini
# fix-pass that addresses + replies/resolves the threads (NEVER merges). Idempotent per-PR lock; capped per run.
set +e
export PATH=$PATH:/usr/local/bin:$HOME/.npm-global/bin
OPS=/srv/aina/ops; LOG=$OPS/pr-bot-watch.log
REPOS="ainative-academy/aina-academy ainative-academy/aina-mission-control ainative-academy/aina-platform"
C="exec --json --dangerously-bypass-approvals-and-sandbox --model gpt-5.4-mini -c model_reasoning_effort=medium --skip-git-repo-check -"
MAX_PER_RUN=4          # cap concurrent dispatches per run (avoid over-spawn)
RECENT_DAYS=3          # only PRs updated within this window (skip stale/abandoned)
ts(){ date "+%Y-%m-%dT%H:%M:%S"; }
cutoff=$(date -u -d "-${RECENT_DAYS} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)
dispatched=0
for repo in $REPOS; do
  [ "$dispatched" -ge "$MAX_PER_RUN" ] && break
  prs=$(gh pr list --repo "$repo" --state open --json number,isDraft,updatedAt --jq ".[]|select(.isDraft==false)|select(.updatedAt>\"$cutoff\")|.number" 2>/dev/null)
  for n in $prs; do
    [ "$dispatched" -ge "$MAX_PER_RUN" ] && break
    bc=$(gh api "repos/$repo/pulls/$n/comments" --jq '[.[]|select(.user.login=="chatgpt-codex-connector" or (.user.login|test("codex";"i")))]|length' 2>/dev/null); [ -z "$bc" ] && bc=0
    [ "$bc" -eq 0 ] && continue
    slug=$(echo "$repo"|tr '/' '-'); lock="$OPS/.botfix-$slug-$n.lock"
    [ -n "$(find "$lock" -mmin -45 2>/dev/null)" ] && { echo "$(ts) skip $repo#$n (recent fix)" >> "$LOG"; continue; }
    pf="/tmp/botfix-$slug-$n.md"
    cat > "$pf" <<EOF
# Lane: address Codex bot review comments on $repo PR #$n
## AUTONOMY: proceed without asking; small fix pass; NEVER merge. gpt-5.4-mini.
For OPEN PR #$n in $repo: (1) fetch UNRESOLVED chatgpt-codex-connector review threads (REST comments + GraphQL reviewThreads isResolved=false); (2) for each VALID one implement the fix on the PR branch (find it under /home/ali/agent-workspaces or /srv/repos via gh pr view headRefName), run the touched-area build/tests, commit+push; (3) for stale/invalid ones reply with current-diff evidence; (4) resolve every addressed thread (GraphQL resolveReviewThread). Do NOT merge. One-line completion note to $OPS/pr-bot-watch.log.
EOF
    ( cd /srv/repos/aina-academy 2>/dev/null||cd /srv/repos; cat "$pf"|codex $C > "$OPS/codex-botfix-$slug-$n.log" 2>&1 ) &
    touch "$lock"; dispatched=$((dispatched+1))
    echo "$(ts) dispatched botfix $repo#$n ($bc bot comments) pid=$!" >> "$LOG"
  done
done
echo "$(ts) watcher run done — dispatched=$dispatched (cap $MAX_PER_RUN)" >> "$LOG"
EOS
base64 < pr-bot-watcher.sh | ./vds-exec.sh runl '
  base64 -d > /srv/aina/ops/pr-bot-watcher.sh && chmod +x /srv/aina/ops/pr-bot-watcher.sh && echo "WATCHER HARDENED (cap 4/run, recent-3d, 45m lock)"
  export PATH=$PATH:/usr/local/bin
  python3 - <<"PY"
import subprocess,json
def pc(*a): return subprocess.run(["paperclipai",*a],capture_output=True,text=True)
iss=json.loads(pc("issue","list","--json").stdout); iss=iss if isinstance(iss,list) else iss.get("issues",iss.get("data",[]))
ids={x.get("identifier"):x.get("id") for x in iss}
i=ids.get("AIN-143")
if i:
  pc("issue","update",i,"--status","in_progress")
  print("AIN-143 ->", pc("issue","comment",i,"--body","IMPLEMENTED + LIVE: /srv/aina/ops/pr-bot-watcher.sh wired into the 15-min watchdog. Each run: for open non-draft recently-updated (3d) AINA-repo PRs with unresolved Codex-bot review comments, dispatches a gpt-5.4-mini fix-pass (address+push+resolve threads, never merge); idempotent 45-min per-PR lock; capped 4 dispatches/run. First run addressed the open academy + platform PRs.").returncode==0)
PY'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'set +e
export PATH=$PATH:/usr/local/bin
echo "== codex procs =="; pgrep -af "codex exec" | grep -v pgrep | wc -l
echo "== gemini-vertex report? =="; ls -la --time-style=+%H:%M /srv/aina/ops/gemini-vertex-report.md 2>/dev/null
echo "== gemini-vertex log final =="; tail -1 /srv/aina/ops/codex-gemini-vertex.log 2>/dev/null | tr -d "\000" | python3 -c "import sys,json
try:
 o=json.loads(sys.stdin.read());t=o.get(\"type\")
 if t==\"turn.completed\":print(\"COMPLETED\")
 elif t==\"turn.failed\":print(\"FAILED:\",o.get(\"error\",{}).get(\"message\",\"\")[:140])
 elif o.get(\"item\",{}).get(\"type\")==\"agent_message\":print(\"MSG:\",o[\"item\"][\"text\"][:700])
 else:print(\"(running)\",t)
except:print(\"(running/parse)\")"
echo "== botfix lane completions (pr-bot-watch log tail) =="; tail -12 /srv/aina/ops/pr-bot-watch.log 2>/dev/null
echo "== open academy PRs =="; gh pr list --repo ainative-academy/aina-academy --state open --json number,headRefName --jq ".[]|\"\(.number) \(.headRefName)\"" 2>/dev/null'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'cat /srv/aina/ops/gemini-vertex-report.md 2>/dev/null'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'export PATH=$PATH:/usr/local/bin
python3 - <<"PY"
import subprocess,json
def pc(*a): return subprocess.run(["paperclipai",*a],capture_output=True,text=True)
iss=json.loads(pc("issue","list","--json").stdout); iss=iss if isinstance(iss,list) else iss.get("issues",iss.get("data",[]))
ids={x.get("identifier"):x.get("id") for x in iss}
v=("Gemini Interactions VERDICT (2026-06-29): NOT viable on Vertex — the Cloud Interactions endpoint rejects AINA Gemini text models (400 unsupported_model_interaction); only the AI Studio Developer API supports it, needing a SEPARATE AI Studio key+prepay (founder billing). DECISION: launch the tutor on STANDARD VERTEX Gemini (works/funded/no action); Interactions (observable steps+server-state) DEFERRED post-launch, optional via AI Studio top-up. Code hardened+flag-off (PR #113, removed unsafe Vertex-key->Developer-API fallback). Ali pinged.")
for ident in ("AIN-174","AIN-133"):
    i=ids.get(ident)
    if i: print(ident, pc("issue","comment",i,"--body",v).returncode==0)
PY'

cd /Users/Ali/PKM/scratch/pkm-deepdive
cat > gemini-vertex-method.md <<'EOF'
# Gemini Interactions API on Vertex — the CORRECT method (Ali research, 2026-06-29)
The prior re-route lane's "not viable on Vertex" was WRONG — it probed the RAW REST endpoint with an API key + wrong request shape. The Interactions API IS GA + available on Vertex (rebranded Gemini Enterprise Agent Platform). Use the **google-genai SDK with vertexai=True + ADC**.

KEY FACTS:
- Cloud/Vertex credits apply to VERTEX (via ADC + vertexai=True), NOT to AI Studio / Gemini Developer API (esp. accounts after 2026-03-02). Do NOT use AI Studio API keys if you want to burn Cloud credits.
- Vertex uses **ADC (Application Default Credentials)**, not API keys. Local: `gcloud auth application-default login`. Server: service account with `roles/aiplatform.user` + GOOGLE_APPLICATION_CREDENTIALS, or attached SA.
- Setup: `gcloud services enable aiplatform.googleapis.com`; project aina-495702; location="global" (auto-routing; some regions still rolling out standard Gemini on Interactions).
- Env: GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION=global.

SDK (Python): `from google import genai; client=genai.Client(vertexai=True, project="aina-495702", location="global"); client.interactions.create(model="gemini-3.5-flash", input="...")`
SDK (JS/TS): `import {GoogleGenAI} from "@google/genai"; const ai=new GoogleGenAI({vertexai:true, project, location:"global"}); await ai.interactions.create({model:"gemini-3.5-flash", input:"..."})`

Cloud docs: docs.cloud.google.com/gemini-enterprise-agent-platform/reference/models/interactions-api ; blog.google Interactions API GA. NOTE: location="global" if model-availability errors per region.
EOF
cat > lane-gemini-sdk-retest.md <<'EOF'
# Lane: RE-TEST Gemini Interactions on Vertex via the google-genai SDK + ADC (Ali's method)
## AUTONOMY: proceed without asking; build, verify, honest report. gpt-5.5. Repo /srv/repos/aina-academy.
## Context: prior lane wrongly concluded "not viable on Vertex" — it used RAW REST + an API key. CORRECT method (canon /srv/aina/ops/reference/gemini-vertex-method.md): google-genai SDK with vertexai=True + ADC routes Interactions to Vertex where Ali's Cloud credits apply. Interactions API is GA on Vertex. Vertex uses ADC, NOT API keys. Project aina-495702, location="global".
## Do
1. Set GOOGLE_CLOUD_PROJECT=aina-495702 GOOGLE_CLOUD_LOCATION=global. Ensure ADC: try `gcloud auth application-default print-access-token`; if no ADC, the active gcloud user account is present — set up ADC (a service account with roles/aiplatform.user is the headless path; create one in aina-495702 if needed, or use the user ADC for the test). `gcloud services enable aiplatform.googleapis.com`.
2. Write a probe using the **google-genai SDK** (Python `google-genai` or JS `@google/genai`) with `vertexai=True, project="aina-495702", location="global"`. Call `interactions.create(model="gemini-3.5-flash", input="Explain how AI works in a few words")`. Log status, output_text, interactionId, tokens, latency. Prove `previous_interaction_id` chaining on a 2nd turn.
3. If it WORKS: run a concurrency probe via the SDK (ramps 5,25,50, turns 2); report latency p50/p95, 429s, the real concurrent ceiling, go/no-go. THEN plan the PRODUCTION path: the tutor runs on Cloudflare Workers (currently VERTEX_API_KEY Express mode, no ADC). Determine how Workers calls Vertex Interactions — a service-account JWT minted+exchanged for an access token IN the Worker, a small proxy, OR whether Vertex Express+API-key also supports Interactions (test the Express endpoint with the correct interactions request shape). Recommend the runtime path.
4. If it still fails: capture the EXACT SDK error (not a raw-REST guess) + the precise blocker (ADC/service-account, region, model availability, IAM) + any founder/GCP action needed.
## Deliver: branch ali/gemini-vertex-sdk-retest-2026-06-29, commit+push+PR; report /srv/aina/ops/gemini-vertex-sdk-report.md (WORKS? + probe results + the Workers-runtime auth plan + any founder action). Keep prod flag-off until proven. Honest. Address bot comments.
EOF
tar czf - gemini-vertex-method.md lane-gemini-sdk-retest.md | base64 | ./vds-exec.sh runl '
  base64 -d | tar xzf - -C /tmp 2>/dev/null
  cp /tmp/gemini-vertex-method.md /srv/aina/ops/reference/gemini-vertex-method.md
  cp /tmp/lane-gemini-sdk-retest.md /srv/aina/ops/lanes/
  export PATH=$PATH:/usr/local/bin
  C5="exec --json --dangerously-bypass-approvals-and-sandbox --model gpt-5.5 -c model_reasoning_effort=high --skip-git-repo-check -"
  WT=/home/ali/agent-workspaces/aina-academy-gemini-spike-20260629
  [ -d "$WT" ] && cd "$WT" || cd /srv/repos/aina-academy
  nohup bash -c "cat /srv/aina/ops/lanes/lane-gemini-sdk-retest.md | codex $C5" > /srv/aina/ops/codex-gemini-sdk-retest.log 2>&1 &
  echo "gemini-sdk-retest LAUNCHED pid=$!"
  sleep 10; echo "== tail =="; tail -2 /srv/aina/ops/codex-gemini-sdk-retest.log 2>/dev/null | tr -d "\000" | head -2'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'set +e
export PATH=$PATH:/usr/local/bin
echo "== codex procs =="; pgrep -af "codex exec" | grep -v pgrep | wc -l
echo "== sdk-retest report? =="; ls -la --time-style=+%H:%M /srv/aina/ops/gemini-vertex-sdk-report.md 2>/dev/null
echo "== sdk-retest log final =="; tail -1 /srv/aina/ops/codex-gemini-sdk-retest.log 2>/dev/null | tr -d "\000" | python3 -c "import sys,json
try:
 o=json.loads(sys.stdin.read());t=o.get(\"type\")
 if t==\"turn.completed\":print(\"COMPLETED\")
 elif t==\"turn.failed\":print(\"FAILED:\",o.get(\"error\",{}).get(\"message\",\"\")[:140])
 elif o.get(\"item\",{}).get(\"type\")==\"agent_message\":print(\"MSG:\",o[\"item\"][\"text\"][:600])
 else:print(\"(running)\",t)
except:print(\"(running/parse)\")"'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'cat /srv/aina/ops/gemini-vertex-sdk-report.md 2>/dev/null'

cd /Users/Ali/PKM/scratch/pkm-deepdive
./vds-exec.sh run 'export PATH=$PATH:/usr/local/bin
python3 - <<"PY"
import subprocess,json
def pc(*a): return subprocess.run(["paperclipai",*a],capture_output=True,text=True)
iss=json.loads(pc("issue","list","--json").stdout); iss=iss if isinstance(iss,list) else iss.get("issues",iss.get("data",[]))
ids={x.get("identifier"):x.get("id") for x in iss}
v=("Gemini Interactions SDK+ADC RE-TEST (Ali method, corrected, 2026-06-29): METHOD IS RIGHT — google-genai SDK + vertexai=True + ADC reaches Vertex on Ali credits (control generateContent gemini-3.5-flash = 200, funded). BUT Vertex Interactions API rejects ALL Gemini Flash (400 Unsupported model interaction: 3.5-flash/3.1-flash-lite/3-flash-preview/2.5-flash/2.5-flash-lite/2.0-flash-001) for project aina-495702@global — a GOOGLE-SIDE model-allowlisting gap, NOT auth/credits/method/region. **FOUNDER/GCP ACTION:** ask Google/Vertex support or account rep which model IDs Interactions supports for aina-495702 + whether Flash needs allowlisting/different family (error to hand them: POST .../interactions model=gemini-3.5-flash -> 400 Unsupported model interaction). **RUNTIME when enabled:** Cloudflare Workers cannot use API keys for Interactions (endpoint 401s API keys) -> Cloud Run proxy w/ service-account roles/aiplatform.user (recommended) or Workload Identity Federation. **DECISION UNCHANGED:** launch tutor on STANDARD VERTEX generateContent (confirmed working on Ali credits); Interactions = post-launch, gated on the Google support answer. Probe: scripts/probes/gemini-vertex-interactions-probe.mjs.")
for ident in ("AIN-174","AIN-133"):
    i=ids.get(ident)
    if i: print(ident, pc("issue","comment",i,"--body",v).returncode==0)
PY'