# Reads raw/items.jsonl, keeps exact config paths, one file per repo per kind, distinct blobs;
# fetches each blob (read-only GitHub API) to work/<id>/orig. The manifest carries no repo or path.
import json, subprocess, base64, os, sys, posixpath, random
from concurrent.futures import ThreadPoolExecutor
S=sys.argv[1]; CAP=int(sys.argv[2]) if len(sys.argv)>2 else 600
def kind_of(p):
    b=posixpath.basename(p); d=posixpath.basename(posixpath.dirname(p))
    if b==".mcp.json": return "root-mcp"
    if b=="mcp.json" and d==".cursor": return "cursor"
    if b=="mcp.json" and d==".vscode": return None
    if b=="mcp.json": return "plain-mcp"
    if b=="claude_desktop_config.json": return "claude-desktop"
    if b=="settings.json" and d==".claude": return "claude-settings"
    if b=="config.toml" and d==".codex": return "codex"
    if b=="settings.json" and d==".gemini": return "gemini"
    return None
items=[json.loads(l) for l in open(f"{S}/raw/items.jsonl")]
random.seed(20261026); random.shuffle(items)
byk={}; seen_repo=set(); seen_blob=set()
for it in items:
    k=kind_of(it["path"])
    if not k: continue
    if (it["repo"],k) in seen_repo or it["sha"] in seen_blob: continue
    if len(byk.setdefault(k,[]))>=CAP: continue
    seen_repo.add((it["repo"],k)); seen_blob.add(it["sha"]); byk[k].append(it)
jobs=[(k,it) for k,v in byk.items() for it in v]
print({k:len(v) for k,v in byk.items()}, file=sys.stderr)
os.makedirs(f"{S}/work",exist_ok=True)
def get(arg):
    i,(k,it)=arg
    r=subprocess.run(["gh","api",f"repos/{it['repo']}/git/blobs/{it['sha']}","--jq",".content"],capture_output=True,text=True)
    if r.returncode!=0: return None
    try: b=base64.b64decode(r.stdout.strip())
    except Exception: return None
    if len(b)>256*1024: return None
    d=f"{S}/work/{i:05d}"; os.makedirs(d,exist_ok=True)
    open(f"{d}/orig","wb").write(b)
    return {"id":f"{i:05d}","kind":k}
with ThreadPoolExecutor(8) as ex:
    res=[x for x in ex.map(get,enumerate(jobs)) if x]
json.dump(res,open(f"{S}/work/manifest.json","w"))
print(f"fetched {len(res)} of {len(jobs)}",file=sys.stderr)