Research Assistants

# Citation + RAG faithfulness stub. Pair with Vol. 19 hallucination tests. # Educational product gate — not a claim that the corpus is world-true. import re def extract_bracket_cites(answer: str): return re.findall(r"\[(\d+)\]", answer) def chunk_supports(chunk: str, claim: str, min_hits: int = 2) -> bool: words = {w.lower() for w in re.findall(r"[A-Za-z0-9]+", claim) if len(w) > 3} if not words: return False blob = chunk.lower() return sum(1 for w in words if w in blob) >= min_hits def score_research_item(answer: str, chunks_by_id: dict, gold_must_retrieve=None): cites = extract_bracket_cites(answer) dangling = [c for c in cites if c not in chunks_by_id] claims = [p for p in re.split(r"(?<=[.!?])\s+", answer.strip()) if p] grounded = [] for claim in claims: ids = extract_bracket_cites(claim) or list(chunks_by_id) ok = any(chunk_supports(chunks_by_id.get(i, ""), claim) for i in ids) grounded.append(ok) retrieval_hit = None if gold_must_retrieve is not None: blob = " ".join(chunks_by_id.values()).lower() retrieval_hit = gold_must_retrieve.lower() in blob return { "n_claims": len(claims), "groundedness": sum(grounded) / max(1, len(claims)), "dangling_citations": dangling, "retrieval_hit": retrieval_hit, "abstained": any(s in answer.lower() for s in ("i don't know", "insufficient sources")), } # CI rule: fail if dangling_citations or (retrieval_hit and groundedness < threshold). # If retrieval_hit is False, fix Vol. 14 retrieval before blaming the generator.