Add literature review protocol, searches and first snowball round

Six seeds snowballed (Semantic Scholar) plus logged keyword searches
(OpenAlex, arXiv): 2,127 unique records, 480 title/abstract screened.
AI first-pass suggestions are in their own column; decisions are pending.

Claude-Session: https://claude.ai/code/session_01S8KN7NuUCm8nXLmy6eDuVv

02-literature/tools/ai_first_pass.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Record the AI first-pass screening suggestions in ../search/screening.csv.
+
+These are SUGGESTIONS from a title/abstract read by Claude (28/10/2026), not
+decisions. The `decision` column is left for Cormac. Codes are defined in
+protocol.md section 5. A '*' in the code marks a paper whose question looks
+close to the thesis RQ (needs a "how we differ" note in the extraction table).
+
+Matching is by arXiv id / DOI / title fragment. Re-runnable: it only writes
+the ai_proposal column (and reason, if empty).
+"""
+import csv
+import os
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SHEET = os.path.normpath(os.path.join(HERE, "..", "search", "screening.csv"))
+
+# (match, proposal, code, note)
+P = [
+    # --- seeds -------------------------------------------------------------
+    ("2010.04245", "include", "SEED", "origin of QK-norm (small NMT models)"),
+    ("findings-emnlp.379", "include", "SEED", "origin of QK-norm (ACL anthology copy of 2010.04245)"),
+    ("2302.05442", "include", "SEED", "QK-norm popularised to stop logit growth at 22B"),
+    ("2303.06296", "include", "SEED", "attention entropy collapse <-> instability"),
+    ("2309.14322", "include", "SEED*", "LR-sensitivity metric; qk-layernorm at small scale"),
+    ("2602.02522", "include", "SEED*", "QK-norm largest win at 70M, bundled with muP"),
+    ("2410.16682", "include", "IC*", "QK-norm + softcap -> 1.5x LR before divergence, 830M"),
+    # --- core: evaluates QK-norm / logit control on stability or LR ---------
+    ("2511.21377", "include", "IC*", "QK-norm vs alternatives for logit control; LR ablations"),
+    ("2607.18363", "include", "IC*", "QK-norm keeps 48-layer stacks trainable: depth evidence (H3)"),
+    ("2610.34582", "include", "IS*", "single-variable ablations at 130M incl. QK-norm (HobbyLM)"),
+    ("zenodo.20803082", "include", "IS*", "29-36M, QK-norm x depth x 3 seeds sweep; Zenodo preprint, check quality"),
+    ("2610.17969", "include", "IA*", "compares vs QK-Norm across 16x LR range at 124M"),
+    ("2606.16310", "include", "IC", "QK-norm permits higher LR; high-LR stress test vs QK-clip"),
+    ("2603.02170", "include", "IC", "QK-norm necessary for stable low-bit training at large tokens/step"),
+    ("2603.11611", "maybe", "IC", "NoPE loss spikes mitigated by QK-norm (side finding)"),
+    ("2602.05006", "include", "IC", "Lp-norm generalisation of QKNorm (variant)"),
+    ("2602.01212", "include", "IA", "SimpleGPT normalisation vs LLaMA2+QKNorm, LR ablations"),
+    ("2505.22014", "maybe", "IA", "approx. normalised transformer vs GPT+QK-norm convergence"),
+    ("2601.02031", "maybe", "IA", "output-logit divergence; z-loss / softcap context"),
+    ("2501.18795", "maybe", "IC", "RoPE/NoPE/QK-norm attention patterns, long context focus"),
+    ("2507.13338", "maybe", "IA", "Lipschitz transformers trained without QK-norm/softcap"),
+    ("Kimi K2", "include", "IA", "MuonClip/QK-clip: alternative fix for attention-logit explosion"),
+    ("2502.02732", "include", "IA", "Peri-LN: normalisation placement neighbour"),
+    ("2503.04598", "include", "IA", "HybridNorm: normalisation placement neighbour"),
+    ("2110.09456", "include", "IA", "NormFormer: extra normalisation"),
+    ("Spike No More", "include", "IA", "pretraining loss-spike mitigation"),
+    ("Stable and low-precision training for large-scale vision", "include", "IM", "Wortsman et al. follow-up on instability"),
+    ("On Layer Normalization in the Transformer Architecture", "include", "BG", "Pre-LN vs Post-LN, warmup and LR (2020)"),
+    ("Transformers without Normalization", "maybe", "IA", "DyT replaces norm layers"),
+    ("Qwen3.8-Next", "maybe", "IC", "industry architecture study incl. training stability"),
+    # --- mechanism: why logits grow / why QK-norm works ---------------------
+    ("2505.15548", "include", "IM", "dense local dependencies -> attention-logit explosion"),
+    ("2505.21910", "include", "IM", "W_q^T W_k spectral concentration -> entropy collapse, warmup"),
+    ("2501.12352", "maybe", "IM", "theory justifying QK-norm (test-time regression view)"),
+    ("2402.02098", "include", "IM", "QK eigenspectrum concentration and entropy collapse"),
+    ("2505.24333", "include", "IM", "rank vs entropy collapse in deep transformers (depth)"),
+    ("nht8-e622", "include", "IM", "variance sensitivity -> attention entropy collapse"),
+    ("2608.02091", "maybe", "IM", "low-precision attention collapse; QK-norm disrupts it"),
+    ("2609.34272", "maybe", "IM", "BF16 attention gradients; logit growth late in training"),
+    ("2608.14712", "include", "IM", "how to measure attention entropy properly (metric choice)"),
+    ("2502.00919", "maybe", "IM", "attention sinks persist with QK-norm"),
+    ("2410.10781", "include", "IM", "when attention sinks emerge (seed paper ref)"),
+    ("2402.17762", "include", "IM", "massive activations -> attention sinks"),
+    ("Efficient Streaming Language Models with Attention Sinks", "maybe", "IM", "attention sinks (inference paper, mechanism background)"),
+    ("The Curse of Depth", "include", "IM", "depth effects in LLMs (H3 context)"),
+    ("2601.22966", "maybe", "IM", "attention + residual sinks, outlier rescaling"),
+    ("Exact Attention Sensitivity", "maybe", "IM", "attention sensitivity and stability geometry"),
+    ("Normalization in Attention Dynamics", "maybe", "IM", "normalisation in attention dynamics"),
+    ("Active-Dormant Attention Heads", "maybe", "IM", "extreme-token phenomena mechanism"),
+    ("Mechanism-Aligned Observability", "maybe", "IM", "instability diagnostics; possible metrics"),
+    # --- method / scale / confounds ----------------------------------------
+    ("2203.03466", "include", "BG", "muP: the confound held OFF"),
+    ("PolyPythias", "include", "IS", "stability across 50 small-model seeds: seed-variance precedent"),
+    ("Weight Decay may matter more than muP", "maybe", "BG", "LR transfer without muP"),
+    ("CompleteP", "maybe", "BG", "depth-wise parametrisation, LR across depth"),
+    ("Effective Learning Rate Governs Loss Dynamics", "maybe", "BG", "effective LR view of loss dynamics"),
+    ("How to set AdamW's weight decay", "maybe", "BG", "AdamW hyperparameters vs scale"),
+    ("Deconstructing What Makes a Good Optimizer", "maybe", "BG", "LR stability across optimisers, small LMs"),
+    ("2609.14715", "maybe", "IS", "sub-150M depth study (QK-norm fixed, not varied)"),
+    ("zenodo.18924690", "maybe", "IS", "GPT-2-scale modernisation incl. QK-norm; Zenodo, check quality"),
+    ("Cerebras-GPT", "maybe", "BG", "muP in small open models"),
+    # --- duplicates / fragments --------------------------------------------
+    ("variancesensitivityinducesattentionentropycollapseintransformers", "exclude", "E4", "duplicate record"),
+    ("Intriguing", "exclude", "E4", "earlier title / duplicate of Wortsman et al. 2309.14322"),
+    ("2022resultstoprobe", "exclude", "E4", "citation-context fragment, not a paper"),
+    ("canrecoverfrom", "exclude", "E4", "citation-context fragment, not a paper"),
+    ("wealsoobserved", "exclude", "E4", "citation-context fragment, not a paper"),
+    ("Duality, Weight Decay, and Metrized", "exclude", "E4", "near-duplicate of 2507.13338"),
+    # --- LLMs for literature search (supervisor: 'find a paper that backs it up')
+    ("Preliminary Guidelines for Using and Evaluating GenAI", "include", "LLM*", "Kitchenham et al. GUEST guidelines for GenAI in SLRs (SE)"),
+    ("3674805.3686666", "include", "LLM", "ChatGPT for SLR selection in SE: accuracy evaluation"),
+    ("wsese66602.2025.00010", "include", "LLM", "difficulties replicating LLM-supported SLRs in SE"),
+    ("3539618.3591703", "include", "LLM", "ChatGPT writing Boolean search queries (SIGIR)"),
+    ("jrsm.1715", "include", "LLM", "GPT-4 vs humans across SR stages; pre-registered; caution"),
+    ("j.jclinepi.2025.111746", "include", "LLM", "scoping review: LLMs for SRs 'not yet ready'"),
+    ("s10515-026-00629-6", "maybe", "LLM", "GPT-4o on SLR sub-tasks (SE)"),
+    ("s13643-024-02575-4", "maybe", "LLM", "LLM title/abstract screening (biomedical)"),
+    ("2196/48996", "maybe", "LLM", "automated screening with LLMs (clinical)"),
+    ("jamanetworkopen.2024.20496", "maybe", "LLM", "LLM citation screening performance"),
+    ("ocaf063", "maybe", "LLM", "LLMs as literature-review tools: review"),
+    ("pnas.2411962122", "maybe", "LLM", "LLMs in literature screening (PNAS perspective)"),
+]
+
+DOMAIN = ("E1", "applies a transformer in another domain; QK-norm/logit control incidental")
+
+
+def main():
+    rows = list(csv.DictReader(open(SHEET)))
+    hits = {m: 0 for m, *_ in P}
+    for r in rows:
+        if r["auto_stage"] != "SCREEN":
+            continue
+        hay = f"{r['key']} {r['title']}".lower()
+        for m, prop, code, note in P:
+            if m.lower() in hay:
+                r["ai_proposal"] = f"{prop} [{code}] {note}"
+                hits[m] += 1
+                break
+        else:
+            lit = r["found_by"].startswith("L") and ";" not in r["found_by"]
+            if lit:
+                r["ai_proposal"] = "exclude [E6] uses LLMs as a research subject, not as a review tool"
+            else:
+                r["ai_proposal"] = "exclude [E5] title/abstract: not about attention-logit control or LM training stability"
+    with open(SHEET, "w", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=list(rows[0]))
+        w.writeheader()
+        w.writerows(rows)
+    missing = [m for m, n in hits.items() if n == 0]
+    from collections import Counter
+    c = Counter(r["ai_proposal"].split(" ")[0] for r in rows if r["auto_stage"] == "SCREEN")
+    print("proposals:", dict(c))
+    if missing:
+        print("patterns that matched nothing:", missing)
+
+
+if __name__ == "__main__":
+    main()

02-literature/tools/merge.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Merge all search/snowball results into one de-duplicated screening sheet.
+
+Usage: python3 merge.py            (reads ../search/results/*.csv)
+
+Writes ../search/screening.csv. Columns:
+  found_by        which searches / snowball steps returned the paper
+  auto_stage      what the automatic filters decided (see protocol.md section 5):
+                    EXCLUDE-pre2020 | EXCLUDE-offtopic | SCREEN
+  ai_proposal     filled later: first-pass include/exclude suggestion
+  decision        YOURS to fill: include / exclude / maybe
+  reason          short reason for the decision
+
+Existing decision/ai_proposal/reason values are preserved on re-runs, so you
+can re-merge after new searches without losing screening work.
+"""
+import csv
+import glob
+import os
+import re
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SEARCH = os.path.normpath(os.path.join(HERE, "..", "search"))
+OUT = os.path.join(SEARCH, "screening.csv")
+
+# Topic filter (protocol.md section 5, criterion I2): a paper goes to manual
+# screening only if its title/abstract touches attention-logit control or
+# training stability of transformers.
+TOPIC = re.compile(
+    r"qk[- ]?(norm|layer ?norm)|query[- ]key norm|attention logit|logit (growth|explosion|cap|soft)"
+    r"|entropy collapse|attention entropy|training (instabilit|stabilit)|loss spike|divergen"
+    r"|learning[- ]rate (sensitiv|stabil|transfer)|maximal update|\bmu-?p\b|μp|z-loss"
+    r"|massive activation|attention sink|layer ?norm|rms ?norm|normali[sz]ation",
+    re.I,
+)
+TRANSFORMER = re.compile(r"transformer|attention|language model|\bllm|\bvit\b|gpt", re.I)
+LLM_SEARCH = re.compile(r"systematic|literature|screening|review", re.I)
+# Seed set (protocol.md section 3): always screened, whatever the filters say.
+SEEDS = {"2010.04245", "2302.05442", "2303.06296", "2309.14322", "2602.02522", "2410.16682"}
+
+
+def key(r):
+    if r.get("arxiv", "").lower().startswith("10.48550/arxiv."):  # old rows: DOI in arxiv column
+        r["arxiv"] = r["arxiv"].lower().split("arxiv.")[-1]
+    if not r.get("arxiv") and "10.48550/arxiv." in r.get("doi", "").lower():
+        r["arxiv"] = r["doi"].lower().split("arxiv.")[-1]
+    if r.get("arxiv"):
+        return "arxiv:" + r["arxiv"].lower()
+    if r.get("doi"):
+        return "doi:" + r["doi"].lower()
+    return "title:" + re.sub(r"[^a-z0-9]", "", r["title"].lower())
+
+
+def main():
+    merged = {}
+    for path in sorted(glob.glob(os.path.join(SEARCH, "results", "*.csv"))):
+        name = os.path.basename(path)[:-4]
+        for r in csv.DictReader(open(path)):
+            if not r.get("title"):
+                continue
+            k = key(r)
+            m = merged.setdefault(k, {**r, "found_by": set()})
+            m["found_by"].add(name)
+            for f in ("arxiv", "doi", "abstract", "venue"):
+                if not m.get(f) and r.get(f):
+                    m[f] = r[f]
+            try:
+                m["citations"] = max(int(m.get("citations") or 0), int(r.get("citations") or 0))
+            except ValueError:
+                pass
+    # also merge title-duplicates that slipped through on different id types
+    by_title = {}
+    for k, m in list(merged.items()):
+        t = re.sub(r"[^a-z0-9]", "", m["title"].lower())
+        if t in by_title:
+            by_title[t]["found_by"] |= m["found_by"]
+            for f in ("arxiv", "doi", "abstract"):
+                by_title[t][f] = by_title[t].get(f) or m.get(f)
+            del merged[k]
+        else:
+            by_title[t] = m
+
+    old = {}
+    if os.path.exists(OUT):
+        for r in csv.DictReader(open(OUT)):
+            old[r["key"]] = r
+
+    rows = []
+    for k, m in merged.items():
+        text = f"{m['title']} {m.get('abstract', '')}"
+        lit_only = all(f.startswith("L") for f in m["found_by"])
+        try:
+            year = int(m.get("year") or 0)
+        except ValueError:
+            year = 0
+        if any(sd in k for sd in SEEDS) or "findings-emnlp.379" in k:
+            stage = "SCREEN"
+        elif lit_only:
+            stage = "SCREEN" if LLM_SEARCH.search(m["title"]) else "EXCLUDE-offtopic"
+        elif year and year < 2020:
+            stage = "EXCLUDE-pre2020"
+        elif TOPIC.search(text) and TRANSFORMER.search(text):
+            stage = "SCREEN"
+        else:
+            stage = "EXCLUDE-offtopic"
+        prev = old.get(k, {})
+        rows.append({
+            "key": k, "year": m.get("year", ""), "title": m["title"],
+            "first_author": m.get("first_author", ""), "venue": m.get("venue", ""),
+            "citations": m.get("citations", ""), "found_by": ";".join(sorted(m["found_by"])),
+            "n_sources": len(m["found_by"]), "auto_stage": stage,
+            "ai_proposal": prev.get("ai_proposal", ""), "decision": prev.get("decision", ""),
+            "reason": prev.get("reason", ""), "abstract": m.get("abstract", ""),
+        })
+    rows.sort(key=lambda r: (r["auto_stage"] != "SCREEN", -r["n_sources"], str(r["year"])))
+    with open(OUT, "w", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=list(rows[0]))
+        w.writeheader()
+        w.writerows(rows)
+    from collections import Counter
+    c = Counter(r["auto_stage"] for r in rows)
+    print(f"{len(rows)} unique records -> {os.path.relpath(OUT, SEARCH)}: {dict(c)}")
+
+
+if __name__ == "__main__":
+    main()

02-literature/tools/s2.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""Tiny Semantic Scholar Graph API client for the literature search (stdlib only).
+
+Every API response is cached under ../search/.cache/ so re-running a search is
+free and reproducible, and the raw JSON is kept as an audit trail.
+
+Usage:
+    python3 s2.py search "query string" [--year 2020-] [--limit 100] [--out NAME]
+    python3 s2.py refs   ARXIV_ID_OR_S2_ID [--out NAME]   # backward snowball
+    python3 s2.py cites  ARXIV_ID_OR_S2_ID [--out NAME]   # forward snowball
+
+Output: a CSV in ../search/results/ (one row per paper) and a line appended
+to ../search/search-log.csv recording what was run, when, and how many hits.
+
+Set S2_API_KEY in the environment if you get one (free, raises rate limits).
+"""
+import csv
+import datetime as dt
+import hashlib
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SEARCH_DIR = os.path.normpath(os.path.join(HERE, "..", "search"))
+CACHE = os.path.join(SEARCH_DIR, ".cache")
+RESULTS = os.path.join(SEARCH_DIR, "results")
+LOG = os.path.join(SEARCH_DIR, "search-log.csv")
+API = "https://api.semanticscholar.org/graph/v1"
+FIELDS = "paperId,externalIds,title,year,venue,authors,citationCount,abstract,publicationDate"
+
+
+def get(path, params):
+    url = f"{API}/{path}?{urllib.parse.urlencode(params)}"
+    key = hashlib.sha1(url.encode()).hexdigest()
+    cpath = os.path.join(CACHE, key + ".json")
+    if os.path.exists(cpath):
+        return json.load(open(cpath))
+    headers = {"User-Agent": "msc-thesis-lit-search/0.1"}
+    if os.environ.get("S2_API_KEY"):
+        headers["x-api-key"] = os.environ["S2_API_KEY"]
+    for attempt in range(int(os.environ.get("S2_RETRIES", 8))):
+        try:
+            with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as r:
+                data = json.load(r)
+            os.makedirs(CACHE, exist_ok=True)
+            json.dump(data, open(cpath, "w"))
+            time.sleep(1.2)  # stay under the shared rate limit
+            return data
+        except urllib.error.HTTPError as e:
+            if e.code == 404:
+                return None
+            if e.code not in (429, 500, 502, 503, 504):
+                raise
+        except (urllib.error.URLError, TimeoutError):
+            pass
+        wait = min(60, 3 * 2 ** attempt)
+        print(f"  rate-limited/transient error; retry in {wait}s", file=sys.stderr)
+        time.sleep(wait)
+    sys.exit(f"gave up on {url}")
+
+
+def norm_id(pid):
+    if pid[:1].isdigit() and "." in pid:  # bare arXiv id like 2309.14322
+        return "arXiv:" + pid
+    return pid
+
+
+def row(p):
+    ext = p.get("externalIds") or {}
+    return {
+        "s2_id": p.get("paperId"),
+        "arxiv": ext.get("ArXiv", ""),
+        "doi": ext.get("DOI", ""),
+        "year": p.get("year") or "",
+        "title": " ".join((p.get("title") or "").split()),
+        "first_author": ((p.get("authors") or [{}])[0] or {}).get("name", ""),
+        "n_authors": len(p.get("authors") or []),
+        "venue": p.get("venue") or "",
+        "citations": p.get("citationCount") or 0,
+        "abstract": " ".join((p.get("abstract") or "").split()),
+    }
+
+
+def search(query, year=None, limit=100):
+    """Relevance-ranked search (top `limit` hits)."""
+    out, offset, total = [], 0, 0
+    while len(out) < limit:
+        params = {"query": query, "fields": FIELDS, "offset": offset, "limit": min(100, limit - len(out))}
+        if year:
+            params["year"] = year
+        data = get("paper/search", params)
+        out += data.get("data") or []
+        total = data.get("total", len(out))
+        if "next" not in data:
+            break
+        offset = data["next"]
+    return out[:limit], total
+
+
+def snowball(pid, direction):
+    field = "citedPaper" if direction == "refs" else "citingPaper"
+    path = f"paper/{norm_id(pid)}/{'references' if direction == 'refs' else 'citations'}"
+    out, offset = [], 0
+    while True:
+        data = get(path, {"fields": FIELDS, "limit": 500, "offset": offset})
+        if data is None:
+            sys.exit(f"paper not found: {pid}")
+        out += [d[field] for d in data.get("data") or [] if d.get(field, {}).get("title")]
+        if "next" not in data:
+            return out, len(out)
+        offset = data["next"]
+
+
+def write(name, papers, what, total):
+    os.makedirs(RESULTS, exist_ok=True)
+    rows = [row(p) for p in papers]
+    path = os.path.join(RESULTS, name + ".csv")
+    with open(path, "w", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=list(rows[0]) if rows else ["s2_id"])
+        w.writeheader()
+        w.writerows(rows)
+    new = not os.path.exists(LOG)
+    with open(LOG, "a", newline="") as f:
+        w = csv.writer(f)
+        if new:
+            w.writerow(["date", "source", "action", "query_or_seed", "filters", "total_hits", "retrieved", "results_file"])
+        w.writerow([dt.date.today().isoformat(), "Semantic Scholar API", *what, total, len(rows), os.path.relpath(path, SEARCH_DIR)])
+    print(f"{what[0]} '{what[1]}': {total} total, {len(rows)} saved -> {os.path.relpath(path, SEARCH_DIR)}")
+
+
+def main(a):
+    def opt(flag, default=None):
+        return a[a.index(flag) + 1] if flag in a else default
+
+    cmd, arg = a[0], a[1]
+    out = opt("--out", hashlib.sha1(f"{cmd}{arg}".encode()).hexdigest()[:8])
+    if cmd == "search":
+        year = opt("--year")
+        papers, total = search(arg, year, int(opt("--limit", 100)))
+        write(out, papers, ("search", arg, f"year={year or 'any'}"), total)
+    elif cmd in ("refs", "cites"):
+        papers, total = snowball(arg, cmd)
+        label = "backward snowball (references)" if cmd == "refs" else "forward snowball (citations)"
+        write(out, papers, (label, arg, ""), total)
+    else:
+        sys.exit(__doc__)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

02-literature/tools/search.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Keyword search over OpenAlex and arXiv for the literature review (stdlib only).
+
+Usage:
+    python3 search.py openalex 'TERMS' --out NAME [--since 2020-01-01] [--limit 200]
+    python3 search.py arxiv    'ARXIV_QUERY' --out NAME [--since 2020-01-01] [--limit 200]
+
+OpenAlex TERMS: '|'-separated phrases (OR), ' AND ' between concept blocks, e.g.
+    '"qk-norm"|"query-key normalization"'
+arXiv ARXIV_QUERY: arXiv API syntax, e.g.
+    'abs:"qk norm" OR abs:"query-key normalization"'
+
+Writes ../search/results/NAME.csv and appends a row to ../search/search-log.csv
+(same format as s2.py). Raw responses are cached in ../search/.cache/.
+"""
+import csv
+import datetime as dt
+import hashlib
+import json
+import os
+import sys
+import time
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SEARCH_DIR = os.path.normpath(os.path.join(HERE, "..", "search"))
+CACHE = os.path.join(SEARCH_DIR, ".cache")
+RESULTS = os.path.join(SEARCH_DIR, "results")
+LOG = os.path.join(SEARCH_DIR, "search-log.csv")
+COLS = ["source_id", "arxiv", "doi", "year", "title", "first_author", "n_authors", "venue", "citations", "abstract"]
+NS = {"a": "http://www.w3.org/2005/Atom"}
+
+
+def fetch(url, raw=False):
+    key = os.path.join(CACHE, hashlib.sha1(url.encode()).hexdigest())
+    if os.path.exists(key):
+        body = open(key, "rb").read()
+    else:
+        for attempt in range(6):
+            try:
+                req = urllib.request.Request(url, headers={"User-Agent": "msc-thesis-lit-search/0.1"})
+                with urllib.request.urlopen(req, timeout=60) as r:
+                    body = r.read()
+                break
+            except Exception as e:
+                wait = 5 * 2 ** attempt
+                print(f"  {e}; retry in {wait}s", file=sys.stderr)
+                time.sleep(wait)
+        else:
+            sys.exit(f"gave up on {url}")
+        os.makedirs(CACHE, exist_ok=True)
+        open(key, "wb").write(body)
+        time.sleep(1)
+    return body if raw else json.loads(body)
+
+
+def clean(s):
+    return " ".join((s or "").split())
+
+
+def openalex_abstract(inv):
+    if not inv:
+        return ""
+    pos = sorted((i, w) for w, idx in inv.items() for i in idx)
+    return " ".join(w for _, w in pos)
+
+
+def openalex(terms, since, limit):
+    # ' AND ' separates concept blocks; each block is an OR-list of phrases.
+    blocks = [f"title_and_abstract.search:{b.strip()}" for b in terms.split(" AND ")]
+    filt = ",".join(blocks + ([f"from_publication_date:{since}"] if since else []))
+    rows, page, total = [], 1, 0
+    while len(rows) < limit:
+        url = "https://api.openalex.org/works?" + urllib.parse.urlencode(
+            {"filter": filt, "per_page": 100, "page": page}
+        )
+        d = fetch(url)
+        total = d["meta"]["count"]
+        for w in d["results"]:
+            ids = w.get("ids") or {}
+            doi = (w.get("doi") or "").replace("https://doi.org/", "")
+            arxiv = doi.lower().split("arxiv.")[-1] if "arxiv." in doi.lower() else ""
+            src = ((w.get("primary_location") or {}).get("source") or {}).get("display_name", "")
+            auth = w.get("authorships") or []
+            rows.append({
+                "source_id": ids.get("openalex", "").split("/")[-1],
+                "arxiv": arxiv, "doi": doi, "year": w.get("publication_year") or "",
+                "title": clean(w.get("title")),
+                "first_author": (auth[0]["author"]["display_name"] if auth else ""),
+                "n_authors": len(auth), "venue": src, "citations": w.get("cited_by_count") or 0,
+                "abstract": clean(openalex_abstract(w.get("abstract_inverted_index"))),
+            })
+        if len(d["results"]) < 100:
+            break
+        page += 1
+    return rows[:limit], total
+
+
+def arxiv(query, since, limit):
+    rows, start, total = [], 0, 0
+    while len(rows) < limit:
+        url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode(
+            {"search_query": query, "start": start, "max_results": 100, "sortBy": "relevance"}
+        )
+        root = ET.fromstring(fetch(url, raw=True))
+        total = int(root.find("{http://a9.com/-/spec/opensearch/1.1/}totalResults").text)
+        entries = root.findall("a:entry", NS)
+        for e in entries:
+            year = e.find("a:published", NS).text[:10]
+            if since and year < since:
+                continue
+            aid = e.find("a:id", NS).text.split("/abs/")[-1].rsplit("v", 1)[0]
+            au = e.findall("a:author", NS)
+            rows.append({
+                "source_id": aid, "arxiv": aid, "doi": "", "year": year[:4],
+                "title": clean(e.find("a:title", NS).text),
+                "first_author": au[0].find("a:name", NS).text if au else "",
+                "n_authors": len(au), "venue": "arXiv", "citations": "",
+                "abstract": clean(e.find("a:summary", NS).text),
+            })
+        if len(entries) < 100:
+            break
+        start += 100
+        time.sleep(3)  # arXiv asks for >=3s between calls
+    return rows[:limit], total
+
+
+def main(a):
+    def opt(flag, default=None):
+        return a[a.index(flag) + 1] if flag in a else default
+
+    src, query, out = a[0], a[1], opt("--out")
+    since, limit = opt("--since", "2020-01-01"), int(opt("--limit", 200))
+    rows, total = {"openalex": openalex, "arxiv": arxiv}[src](query, since, limit)
+    os.makedirs(RESULTS, exist_ok=True)
+    path = os.path.join(RESULTS, out + ".csv")
+    with open(path, "w", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=COLS)
+        w.writeheader()
+        w.writerows(rows)
+    new = not os.path.exists(LOG)
+    with open(LOG, "a", newline="") as f:
+        w = csv.writer(f)
+        if new:
+            w.writerow(["date", "source", "action", "query_or_seed", "filters", "total_hits", "retrieved", "results_file"])
+        w.writerow([dt.date.today().isoformat(), {"openalex": "OpenAlex API", "arxiv": "arXiv API"}[src],
+                    "search", query, f"since={since}", total, len(rows), os.path.relpath(path, SEARCH_DIR)])
+    print(f"{src} {out}: {total} total, {len(rows)} saved")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])