[{"data":1,"prerenderedAt":1006},["ShallowReactive",2],{"navigation_docs":3,"-docs-intelligence-evaluation":356,"-docs-intelligence-evaluation-surround":1001},[4,8,34,64,90,139,157,171,193,227,241,266,312,334],{"title":5,"path":6,"stem":7},"Documentation","\u002Fdocs","docs\u002Findex",{"title":9,"path":10,"stem":11,"children":12},"Getting started","\u002Fdocs\u002Fgetting-started","docs\u002F01.getting-started\u002Findex",[13,14,18,22,26,30],{"title":9,"path":10,"stem":11},{"title":15,"path":16,"stem":17},"Introduction","\u002Fdocs\u002Fgetting-started\u002Fintroduction","docs\u002F01.getting-started\u002F01.introduction",{"title":19,"path":20,"stem":21},"Concepts","\u002Fdocs\u002Fgetting-started\u002Fconcepts","docs\u002F01.getting-started\u002F02.concepts",{"title":23,"path":24,"stem":25},"Quickstart (5 minutes)","\u002Fdocs\u002Fgetting-started\u002Fquickstart","docs\u002F01.getting-started\u002F03.quickstart",{"title":27,"path":28,"stem":29},"Local development","\u002Fdocs\u002Fgetting-started\u002Flocal-development","docs\u002F01.getting-started\u002F04.local-development",{"title":31,"path":32,"stem":33},"FAQ","\u002Fdocs\u002Fgetting-started\u002Ffaq","docs\u002F01.getting-started\u002F05.faq",{"title":35,"path":36,"stem":37,"children":38},"Architecture","\u002Fdocs\u002Farchitecture","docs\u002F02.architecture\u002Findex",[39,40,44,48,52,56,60],{"title":35,"path":36,"stem":37},{"title":41,"path":42,"stem":43},"Architecture overview","\u002Fdocs\u002Farchitecture\u002Foverview","docs\u002F02.architecture\u002F01.overview",{"title":45,"path":46,"stem":47},"Services","\u002Fdocs\u002Farchitecture\u002Fservices","docs\u002F02.architecture\u002F02.services",{"title":49,"path":50,"stem":51},"Request lifecycle","\u002Fdocs\u002Farchitecture\u002Frequest-lifecycle","docs\u002F02.architecture\u002F03.request-lifecycle",{"title":53,"path":54,"stem":55},"Data model","\u002Fdocs\u002Farchitecture\u002Fdata-model","docs\u002F02.architecture\u002F04.data-model",{"title":57,"path":58,"stem":59},"Status and evidence","\u002Fdocs\u002Farchitecture\u002Fstatus-and-evidence","docs\u002F02.architecture\u002F05.status-and-evidence",{"title":61,"path":62,"stem":63},"Target architecture and authority","\u002Fdocs\u002Farchitecture\u002Ftarget-architecture","docs\u002F02.architecture\u002F06.target-architecture",{"title":65,"path":66,"stem":67,"children":68},"Auth","\u002Fdocs\u002Fauth","docs\u002F03.auth\u002Findex",[69,70,74,78,82,86],{"title":65,"path":66,"stem":67},{"title":71,"path":72,"stem":73},"Overview","\u002Fdocs\u002Fauth\u002Foverview","docs\u002F03.auth\u002F01.overview",{"title":75,"path":76,"stem":77},"Organization and agent model","\u002Fdocs\u002Fauth\u002Forg-project-model","docs\u002F03.auth\u002F02.org-project-model",{"title":79,"path":80,"stem":81},"Issuing API keys","\u002Fdocs\u002Fauth\u002Fissuing-api-keys","docs\u002F03.auth\u002F03.issuing-api-keys",{"title":83,"path":84,"stem":85},"Multi-tenant RLS","\u002Fdocs\u002Fauth\u002Fmulti-tenant-rls","docs\u002F03.auth\u002F04.multi-tenant-rls",{"title":87,"path":88,"stem":89},"Operator sessions and step-up","\u002Fdocs\u002Fauth\u002Foperator-sessions","docs\u002F03.auth\u002F05.operator-sessions",{"title":91,"path":92,"stem":93,"children":94},"Proxy","\u002Fdocs\u002Fproxy","docs\u002F04.proxy\u002Findex",[95,96,99,103,107,111,115,119,123,127,131,135],{"title":91,"path":92,"stem":93},{"title":71,"path":97,"stem":98},"\u002Fdocs\u002Fproxy\u002Foverview","docs\u002F04.proxy\u002F01.overview",{"title":100,"path":101,"stem":102},"Authentication","\u002Fdocs\u002Fproxy\u002Fauthentication","docs\u002F04.proxy\u002F02.authentication",{"title":104,"path":105,"stem":106},"Request routing","\u002Fdocs\u002Fproxy\u002Frequest-routing","docs\u002F04.proxy\u002F03.request-routing",{"title":108,"path":109,"stem":110},"Sessions","\u002Fdocs\u002Fproxy\u002Fsessions","docs\u002F04.proxy\u002F04.sessions",{"title":112,"path":113,"stem":114},"Directives","\u002Fdocs\u002Fproxy\u002Fdirectives","docs\u002F04.proxy\u002F05.directives",{"title":116,"path":117,"stem":118},"Idempotency and budgets","\u002Fdocs\u002Fproxy\u002Fidempotency-and-budgets","docs\u002F04.proxy\u002F06.idempotency-and-budgets",{"title":120,"path":121,"stem":122},"Rate limiting","\u002Fdocs\u002Fproxy\u002Frate-limiting","docs\u002F04.proxy\u002F07.rate-limiting",{"title":124,"path":125,"stem":126},"Provider adapters","\u002Fdocs\u002Fproxy\u002Fprovider-adapters","docs\u002F04.proxy\u002F08.provider-adapters",{"title":128,"path":129,"stem":130},"Auth caching","\u002Fdocs\u002Fproxy\u002Fauth-caching","docs\u002F04.proxy\u002F09.auth-caching",{"title":132,"path":133,"stem":134},"Configuration","\u002Fdocs\u002Fproxy\u002Fconfiguration","docs\u002F04.proxy\u002F10.configuration",{"title":136,"path":137,"stem":138},"Provider routing and resilience","\u002Fdocs\u002Fproxy\u002Fprovider-routing-and-resilience","docs\u002F04.proxy\u002F11.provider-routing-and-resilience",{"title":140,"path":141,"stem":142,"children":143},"Memory","\u002Fdocs\u002Fmemory","docs\u002F05.memory\u002Findex",[144,145,149,153],{"title":140,"path":141,"stem":142},{"title":146,"path":147,"stem":148},"Memory overview","\u002Fdocs\u002Fmemory\u002Foverview","docs\u002F05.memory\u002F01.overview",{"title":150,"path":151,"stem":152},"Write path","\u002Fdocs\u002Fmemory\u002Fwrite-path","docs\u002F05.memory\u002F02.write-path",{"title":154,"path":155,"stem":156},"Search and lifecycle","\u002Fdocs\u002Fmemory\u002Fsearch-and-lifecycle","docs\u002F05.memory\u002F03.search-and-lifecycle",{"title":158,"path":159,"stem":160,"children":161},"Context","\u002Fdocs\u002Fcontext","docs\u002F06.context\u002Findex",[162,163,167],{"title":158,"path":159,"stem":160},{"title":164,"path":165,"stem":166},"Assembly pipeline","\u002Fdocs\u002Fcontext\u002Foverview","docs\u002F06.context\u002F01.overview",{"title":168,"path":169,"stem":170},"Ranking and packing","\u002Fdocs\u002Fcontext\u002Franking-and-packing","docs\u002F06.context\u002F02.ranking-and-packing",{"title":172,"path":173,"stem":174,"children":175},"Intelligence","\u002Fdocs\u002Fintelligence","docs\u002F07.intelligence\u002Findex",[176,177,181,185,189],{"title":172,"path":173,"stem":174},{"title":178,"path":179,"stem":180},"Workers","\u002Fdocs\u002Fintelligence\u002Fworkers","docs\u002F07.intelligence\u002F01.workers",{"title":182,"path":183,"stem":184},"Extraction","\u002Fdocs\u002Fintelligence\u002Fextraction","docs\u002F07.intelligence\u002F02.extraction",{"title":186,"path":187,"stem":188},"Embedder","\u002Fdocs\u002Fintelligence\u002Fembedder","docs\u002F07.intelligence\u002F03.embedder",{"title":190,"path":191,"stem":192},"Extraction evaluation","\u002Fdocs\u002Fintelligence\u002Fevaluation","docs\u002F07.intelligence\u002F04.evaluation",{"title":194,"path":195,"stem":196,"children":197},"API reference","\u002Fdocs\u002Fapi-reference","docs\u002F08.api-reference\u002Findex",[198,199,203,207,211,215,219,223],{"title":194,"path":195,"stem":196},{"title":200,"path":201,"stem":202},"Chat completions","\u002Fdocs\u002Fapi-reference\u002Fchat-completions","docs\u002F08.api-reference\u002F01.chat-completions",{"title":204,"path":205,"stem":206},"Responses","\u002Fdocs\u002Fapi-reference\u002Fresponses","docs\u002F08.api-reference\u002F02.responses",{"title":208,"path":209,"stem":210},"Errors","\u002Fdocs\u002Fapi-reference\u002Ferrors","docs\u002F08.api-reference\u002F03.errors",{"title":212,"path":213,"stem":214},"Health and metrics","\u002Fdocs\u002Fapi-reference\u002Fproxy-health","docs\u002F08.api-reference\u002F04.proxy-health",{"title":216,"path":217,"stem":218},"HTTP route inventory","\u002Fdocs\u002Fapi-reference\u002Fhttp-routes","docs\u002F08.api-reference\u002F05.http-routes",{"title":220,"path":221,"stem":222},"Management API","\u002Fdocs\u002Fapi-reference\u002Fmanagement-api","docs\u002F08.api-reference\u002F06.management-api",{"title":224,"path":225,"stem":226},"Auth gRPC (ValidateToken, ValidateAgent)","\u002Fdocs\u002Fapi-reference\u002Fauth-grpc","docs\u002F08.api-reference\u002F07.auth-grpc",{"title":228,"path":229,"stem":230,"children":231},"MCP","\u002Fdocs\u002Fmcp","docs\u002F09.mcp\u002Findex",[232,233,237],{"title":228,"path":229,"stem":230},{"title":234,"path":235,"stem":236},"Memory server","\u002Fdocs\u002Fmcp\u002Fmemory-server","docs\u002F09.mcp\u002F01.memory-server",{"title":238,"path":239,"stem":240},"Operator gateway","\u002Fdocs\u002Fmcp\u002Foperator-gateway","docs\u002F09.mcp\u002F02.operator-gateway",{"title":242,"path":243,"stem":244,"children":245},"Security","\u002Fdocs\u002Fsecurity","docs\u002F10.security\u002Findex",[246,247,251,254,258,262],{"title":242,"path":243,"stem":244},{"title":248,"path":249,"stem":250},"Security overview","\u002Fdocs\u002Fsecurity\u002Foverview","docs\u002F10.security\u002F01.overview",{"title":100,"path":252,"stem":253},"\u002Fdocs\u002Fsecurity\u002Fauthentication","docs\u002F10.security\u002F02.authentication",{"title":255,"path":256,"stem":257},"Tenant isolation","\u002Fdocs\u002Fsecurity\u002Ftenant-isolation","docs\u002F10.security\u002F03.tenant-isolation",{"title":259,"path":260,"stem":261},"Secrets and keys","\u002Fdocs\u002Fsecurity\u002Fsecrets-and-keys","docs\u002F10.security\u002F04.secrets-and-keys",{"title":263,"path":264,"stem":265},"Prompt injection","\u002Fdocs\u002Fsecurity\u002Fprompt-injection","docs\u002F10.security\u002F05.prompt-injection",{"title":267,"path":268,"stem":269,"children":270},"Operations","\u002Fdocs\u002Foperations","docs\u002F11.operations\u002Findex",[271,272,276,280,284,288,292,296,300,304,308],{"title":267,"path":268,"stem":269},{"title":273,"path":274,"stem":275},"Observability","\u002Fdocs\u002Foperations\u002Fobservability","docs\u002F11.operations\u002F01.observability",{"title":277,"path":278,"stem":279},"Traces and ClickHouse","\u002Fdocs\u002Foperations\u002Ftraces-and-clickhouse","docs\u002F11.operations\u002F02.traces-and-clickhouse",{"title":281,"path":282,"stem":283},"Health checks","\u002Fdocs\u002Foperations\u002Fhealth-checks","docs\u002F11.operations\u002F03.health-checks",{"title":285,"path":286,"stem":287},"Incident response","\u002Fdocs\u002Foperations\u002Fincident-response","docs\u002F11.operations\u002F04.incident-response",{"title":289,"path":290,"stem":291},"Troubleshooting","\u002Fdocs\u002Foperations\u002Ftroubleshooting","docs\u002F11.operations\u002F05.troubleshooting",{"title":293,"path":294,"stem":295},"Billing and budgets","\u002Fdocs\u002Foperations\u002Fbilling-and-budgets","docs\u002F11.operations\u002F06.billing-and-budgets",{"title":297,"path":298,"stem":299},"Evidence and relay","\u002Fdocs\u002Foperations\u002Fevidence-and-relay","docs\u002F11.operations\u002F07.evidence-and-relay",{"title":301,"path":302,"stem":303},"Console","\u002Fdocs\u002Foperations\u002Fconsole","docs\u002F11.operations\u002F08.console",{"title":305,"path":306,"stem":307},"Failure modes and recovery","\u002Fdocs\u002Foperations\u002Ffailure-and-recovery","docs\u002F11.operations\u002F09.failure-and-recovery",{"title":309,"path":310,"stem":311},"Data lifecycle and deletion","\u002Fdocs\u002Foperations\u002Fdata-lifecycle-and-deletion","docs\u002F11.operations\u002F10.data-lifecycle-and-deletion",{"title":313,"path":314,"stem":315,"children":316},"Deployment","\u002Fdocs\u002Fdeployment","docs\u002F12.deployment\u002Findex",[317,318,322,326,330],{"title":313,"path":314,"stem":315},{"title":319,"path":320,"stem":321},"Docker Compose (dev)","\u002Fdocs\u002Fdeployment\u002Fdocker-compose","docs\u002F12.deployment\u002F01.docker-compose",{"title":323,"path":324,"stem":325},"Environment variables","\u002Fdocs\u002Fdeployment\u002Fenvironment-variables","docs\u002F12.deployment\u002F02.environment-variables",{"title":327,"path":328,"stem":329},"Kubernetes","\u002Fdocs\u002Fdeployment\u002Fkubernetes","docs\u002F12.deployment\u002F03.kubernetes",{"title":331,"path":332,"stem":333},"Production readiness","\u002Fdocs\u002Fdeployment\u002Fproduction-readiness","docs\u002F12.deployment\u002F04.production-readiness",{"title":335,"path":336,"stem":337,"children":338},"Reference","\u002Fdocs\u002Freference","docs\u002F13.reference\u002Findex",[339,340,344,348,352],{"title":335,"path":336,"stem":337},{"title":341,"path":342,"stem":343},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","docs\u002F13.reference\u002F01.glossary",{"title":345,"path":346,"stem":347},"Roadmap and gates","\u002Fdocs\u002Freference\u002Froadmap","docs\u002F13.reference\u002F02.roadmap",{"title":349,"path":350,"stem":351},"Architecture decisions","\u002Fdocs\u002Freference\u002Fadr","docs\u002F13.reference\u002F03.adr",{"title":353,"path":354,"stem":355},"Compatibility and versioning","\u002Fdocs\u002Freference\u002Fcompatibility","docs\u002F13.reference\u002F04.compatibility",{"id":357,"title":190,"audience":358,"author":359,"authorUrl":359,"body":360,"coverIndex":359,"coverLabel":359,"coverTone":359,"date":359,"decisionStatus":359,"description":992,"excerpt":359,"extension":993,"featured":994,"historical":994,"meta":995,"navigation":996,"path":191,"readingTime":359,"scope":997,"seo":998,"sourceOfTruth":999,"status":367,"stem":192,"tags":359,"verifiedDate":368,"__hash__":1000},"docs\u002Fdocs\u002F07.intelligence\u002F04.evaluation.md","architect",null,{"type":361,"value":362,"toc":981},"minimark",[363,369,378,383,461,464,528,535,539,542,558,561,565,614,621,635,642,646,649,666,673,677,688,747,758,762,812,819,823,826,904,921,925,942,945,949,977],[364,365],"page-status",{"note":366,"status":367,"verified":368},"The harness and CI gate are implemented; live model quality is not established by the oracle-aligned baseline","PROVISIONAL","2026-10-21",[370,371,372,373,377],"p",{},"The worker has a versioned extraction-quality harness. It is valuable because it makes prompt, schema, and scoring regressions visible. It is dangerous to read as a model benchmark: the CI baseline currently replays ",[374,375,376],"strong",{},"oracle-aligned"," cassettes, so a perfect score proves the scoring and gate wiring, not that a live model extracts perfectly.",[379,380,382],"h2",{"id":381},"gold-set-v1","Gold set v1",[384,385,386,399],"table",{},[387,388,389],"thead",{},[390,391,392,396],"tr",{},[393,394,395],"th",{},"Property",[393,397,398],{},"Value",[400,401,402,414,422,430,438,446,453],"tbody",{},[390,403,404,408],{},[405,406,407],"td",{},"Version",[405,409,410],{},[411,412,413],"code",{},"v1",[390,415,416,419],{},[405,417,418],{},"Conversations",[405,420,421],{},"125",[390,423,424,427],{},[405,425,426],{},"Expected memory rows",[405,428,429],{},"130",[390,431,432,435],{},[405,433,434],{},"Categories",[405,436,437],{},"Factual, procedural, preference, behavioral, episodic",[390,439,440,443],{},[405,441,442],{},"Multi-label memories",[405,444,445],{},"10",[390,447,448,451],{},[405,449,450],{},"Supersession cases",[405,452,445],{},[390,454,455,458],{},[405,456,457],{},"Label review",[405,459,460],{},"Hand-labeled synthetic conversations, reviewed 2026-09-03",[370,462,463],{},"Files separate inputs, expectations, provider cassettes, and baselines:",[384,465,466,476],{},[387,467,468],{},[390,469,470,473],{},[393,471,472],{},"File",[393,474,475],{},"Role",[400,477,478,488,498,508,518],{},[390,479,480,485],{},[405,481,482],{},[411,483,484],{},"conversations.jsonl",[405,486,487],{},"Conversation turns",[390,489,490,495],{},[405,491,492],{},[411,493,494],{},"expected_memories.jsonl",[405,496,497],{},"Expected memories plus temporal kind",[390,499,500,505],{},[405,501,502],{},[411,503,504],{},"openai_cassettes.jsonl",[405,506,507],{},"Deterministic provider output per conversation",[390,509,510,515],{},[405,511,512],{},[411,513,514],{},"cassette_manifest.json",[405,516,517],{},"Prompt\u002Fschema hashes, cassette kind, file hashes, counts",[390,519,520,525],{},[405,521,522],{},[411,523,524],{},"baseline_results.json",[405,526,527],{},"Per-provider metrics and enforcement mode",[370,529,530,531,534],{},"A future ",[411,532,533],{},"v2"," can add cases without silently breaking comparability; comparisons must name the gold-set version.",[379,536,538],{"id":537},"scoring","Scoring",[370,540,541],{},"The harness computes, per category:",[543,544,545,549,552,555],"ul",{},[546,547,548],"li",{},"Precision and recall against expected memories",[546,550,551],{},"Category-assignment accuracy",[546,553,554],{},"Temporal-field accuracy",[546,556,557],{},"Macro precision and recall",[370,559,560],{},"It also tracks multi-label and supersession behavior. A memory that matches content but has the wrong category or temporal kind is not a full success.",[379,562,564],{"id":563},"modes-and-enforcement","Modes and enforcement",[384,566,567,580],{},[387,568,569],{},[390,570,571,574,577],{},[393,572,573],{},"Provider",[393,575,576],{},"Enforcement",[393,578,579],{},"How numbers are produced",[400,581,582,598],{},[390,583,584,590,595],{},[405,585,586,587],{},"OpenAI ",[411,588,589],{},"gpt-4o-mini",[405,591,592],{},[411,593,594],{},"ci",[405,596,597],{},"Cassette replay on PR\u002Fsmoke\u002Ffast; optional live weekly run with a key",[390,599,600,606,611],{},[405,601,602,603],{},"vLLM ",[411,604,605],{},"Qwen2.5-14B-Instruct",[405,607,608],{},[411,609,610],{},"manual",[405,612,613],{},"No GPU CI runner; null metrics until a manual runbook run is published",[370,615,616,617,620],{},"Cassette replay is free and deterministic. The ",[411,618,619],{},"cassette_kind"," is provenance, not cryptographic proof of model output:",[543,622,623,629],{},[546,624,625,628],{},[411,626,627],{},"oracle_aligned_expected_json"," — cassettes embed the expected JSON, producing a perfect baseline while testing scoring and gate wiring.",[546,630,631,634],{},[411,632,633],{},"live_openai_recorded"," — cassettes were recorded from a live OpenAI call.",[370,636,637,638,641],{},"The checked-in OpenAI baseline is oracle-aligned and reports 1.0 across gated metrics. The vLLM baseline has null metrics and ",[411,639,640],{},"enforcement: manual",". Treat both honestly: one tests plumbing, the other is not yet measured.",[379,643,645],{"id":644},"contract-and-oracle-gates","Contract and oracle gates",[370,647,648],{},"Prompt and schema changes are contract changes. The harness hashes:",[543,650,651,657],{},[546,652,653,656],{},[411,654,655],{},"sha256"," of the batch extraction system prompt",[546,658,659,661,662,665],{},[411,660,655],{}," of the canonical ",[411,663,664],{},"BatchExtractionResult"," JSON schema",[370,667,668,669,672],{},"Cassette, smoke, and fast runs fail on drift unless a local override is set; CI must never set that override. If cassettes are still oracle-aligned and a pull request touches the prompt or schema, CI fails unless the PR body carries the reviewer-visible ",[411,670,671],{},"EXTRACTION_EVAL_ORACLE_OK=1"," exception or the cassettes were re-recorded live. Bumping a hash alone does not satisfy the gate.",[379,674,676],{"id":675},"regression-gate","Regression gate",[370,678,679,680,683,684,687],{},"The gate compares the latest run to the CI-enforced provider block and fails on an absolute drop greater than ",[374,681,682],{},"3 percentage points"," in any gated metric. Only ",[411,685,686],{},"enforcement: ci"," providers can fail the build. An empty or unmatched provider set is a failure, not a pass.",[689,690,695],"pre",{"className":691,"code":692,"language":693,"meta":694,"style":694},"language-bash shiki shiki-themes material-theme-lighter vitesse-light vitesse-dark","cd services\u002Fworker\n.venv\u002Fbin\u002Fpytest -q eval\u002F\n.venv\u002Fbin\u002Fpython eval\u002Frun_eval.py --mode cassette\n.venv\u002Fbin\u002Fpython eval\u002Fregression_gate.py\n","bash","",[411,696,697,710,724,739],{"__ignoreMap":694},[698,699,702,706],"span",{"class":700,"line":701},"line",1,[698,703,705],{"class":704},"sJdAF","cd",[698,707,709],{"class":708},"sTbE_"," services\u002Fworker\n",[698,711,713,717,721],{"class":700,"line":712},2,[698,714,716],{"class":715},"sYn-s",".venv\u002Fbin\u002Fpytest",[698,718,720],{"class":719},"sz9Cv"," -q",[698,722,723],{"class":708}," eval\u002F\n",[698,725,727,730,733,736],{"class":700,"line":726},3,[698,728,729],{"class":715},".venv\u002Fbin\u002Fpython",[698,731,732],{"class":708}," eval\u002Frun_eval.py",[698,734,735],{"class":719}," --mode",[698,737,738],{"class":708}," cassette\n",[698,740,742,744],{"class":700,"line":741},4,[698,743,729],{"class":715},[698,745,746],{"class":708}," eval\u002Fregression_gate.py\n",[370,748,749,750,753,754,757],{},"Outputs are written under ",[411,751,752],{},"eval\u002Foutput\u002F","; the gate exits ",[411,755,756],{},"1"," on regression.",[379,759,761],{"id":760},"recording-live-cassettes","Recording live cassettes",[689,763,765],{"className":691,"code":764,"language":693,"meta":694,"style":694},"export OPENAI_API_KEY=…       # budgeted; never commit\nexport EXTRACTION_EVAL_MODE=record\n.venv\u002Fbin\u002Fpython eval\u002Frun_eval.py --mode record\n",[411,766,767,789,801],{"__ignoreMap":694},[698,768,769,773,777,781,785],{"class":700,"line":701},[698,770,772],{"class":771},"s5Kfy","export",[698,774,776],{"class":775},"sSC40"," OPENAI_API_KEY",[698,778,780],{"class":779},"soVBu","=",[698,782,784],{"class":783},"sftqT","…       ",[698,786,788],{"class":787},"s9Tkl","# budgeted; never commit\n",[698,790,791,793,796,798],{"class":700,"line":712},[698,792,772],{"class":771},[698,794,795],{"class":775}," EXTRACTION_EVAL_MODE",[698,797,780],{"class":779},[698,799,800],{"class":775},"record\n",[698,802,803,805,807,809],{"class":700,"line":726},[698,804,729],{"class":715},[698,806,732],{"class":708},[698,808,735],{"class":719},[698,810,811],{"class":708}," record\n",[370,813,814,815,818],{},"This refreshes the cassette file and manifest, sets ",[411,816,817],{},"cassette_kind=live_openai_recorded",", and updates prompt\u002Fschema hashes. Re-run cassette evaluation and update the baseline metrics and refresh date. A live recording is a new evidence event; it does not retroactively make old numbers live.",[379,820,822],{"id":821},"manual-vllm-comparison","Manual vLLM comparison",[370,824,825],{},"After the manual vLLM server is running:",[689,827,829],{"className":691,"code":828,"language":693,"meta":694,"style":694},"export IBEX_WORKER_EXTRACTION_PROVIDER=vllm\nexport IBEX_WORKER_EXTRACTION_BASE_URL=http:\u002F\u002F127.0.0.1:8000\u002Fv1\ncd services\u002Fworker\n.venv\u002Fbin\u002Fpython eval\u002Frun_eval.py --mode vllm\n",[411,830,831,843,887,893],{"__ignoreMap":694},[698,832,833,835,838,840],{"class":700,"line":701},[698,834,772],{"class":771},[698,836,837],{"class":775}," IBEX_WORKER_EXTRACTION_PROVIDER",[698,839,780],{"class":779},[698,841,842],{"class":775},"vllm\n",[698,844,845,847,850,852,855,858,861,864,867,869,871,873,875,878,881,884],{"class":700,"line":712},[698,846,772],{"class":771},[698,848,849],{"class":775}," IBEX_WORKER_EXTRACTION_BASE_URL",[698,851,780],{"class":779},[698,853,854],{"class":775},"http",[698,856,857],{"class":783},":\u002F\u002F",[698,859,860],{"class":775},"127",[698,862,863],{"class":783},".",[698,865,866],{"class":775},"0",[698,868,863],{"class":783},[698,870,866],{"class":775},[698,872,863],{"class":783},[698,874,756],{"class":775},[698,876,877],{"class":783},":",[698,879,880],{"class":775},"8000",[698,882,883],{"class":783},"\u002F",[698,885,886],{"class":775},"v1\n",[698,888,889,891],{"class":700,"line":726},[698,890,705],{"class":704},[698,892,709],{"class":708},[698,894,895,897,899,901],{"class":700,"line":741},[698,896,729],{"class":715},[698,898,732],{"class":708},[698,900,735],{"class":719},[698,902,903],{"class":708}," vllm\n",[370,905,906,907,910,911,914,915,917,918,920],{},"Copy metrics into ",[411,908,909],{},"providers.vllm.metrics",", set ",[411,912,913],{},"last_refreshed",", and keep ",[411,916,640],{},". Do not flip the mode to ",[411,919,594],{}," without a GPU-backed CI runner and accepted cost\u002Favailability evidence.",[379,922,924],{"id":923},"what-the-numbers-do-not-prove","What the numbers do not prove",[543,926,927,930,933,936,939],{},[546,928,929],{},"That live model precision or recall is 1.0.",[546,931,932],{},"That synthetic conversations represent customer traffic.",[546,934,935],{},"That extraction is safe against prompt injection or poisoned memory content.",[546,937,938],{},"That vLLM is benchmarked, GPU capacity exists, or latency fits the queue budget.",[546,940,941],{},"That a passing gate covers embedding quality, retrieval quality, or end-to-end answer quality.",[370,943,944],{},"Evaluation is one gate in the intelligence plane. Retrieval ranking, conflict handling, and packer quality need their own evidence.",[379,946,948],{"id":947},"related","Related",[543,950,951,957,962,967,972],{},[546,952,953,956],{},[954,955,182],"a",{"href":183}," — provider adapters and queue behavior",[546,958,959,961],{},[954,960,186],{"href":187}," — embedding profiles and readiness",[546,963,964,966],{},[954,965,178],{"href":179}," — task families and retries",[546,968,969,971],{},[954,970,168],{"href":169}," — context quality boundaries",[546,973,974,976],{},[954,975,345],{"href":346}," — where evaluation evidence fits",[978,979,980],"style",{},"html pre.shiki code .sJdAF, html code.shiki .sJdAF{--shiki-light:#6182B8;--shiki-default:#998418;--shiki-dark:#B8A965}html pre.shiki code .sTbE_, html code.shiki .sTbE_{--shiki-light:#91B859;--shiki-default:#B56959;--shiki-dark:#C98A7D}html pre.shiki code .sYn-s, html code.shiki .sYn-s{--shiki-light:#E2931D;--shiki-default:#59873A;--shiki-dark:#80A665}html pre.shiki code .sz9Cv, html code.shiki .sz9Cv{--shiki-light:#91B859;--shiki-default:#A65E2B;--shiki-dark:#C99076}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s5Kfy, html code.shiki .s5Kfy{--shiki-light:#9C3EDA;--shiki-default:#AB5959;--shiki-dark:#CB7676}html pre.shiki code .sSC40, html code.shiki .sSC40{--shiki-light:#90A4AE;--shiki-default:#B07D48;--shiki-dark:#BD976A}html pre.shiki code .soVBu, html code.shiki .soVBu{--shiki-light:#39ADB5;--shiki-default:#999999;--shiki-dark:#666666}html pre.shiki code .sftqT, html code.shiki .sftqT{--shiki-light:#90A4AE;--shiki-default:#393A34;--shiki-dark:#DBD7CAEE}html pre.shiki code .s9Tkl, html code.shiki .s9Tkl{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#A0ADA0;--shiki-default-font-style:inherit;--shiki-dark:#758575DD;--shiki-dark-font-style:inherit}",{"title":694,"searchDepth":712,"depth":712,"links":982},[983,984,985,986,987,988,989,990,991],{"id":381,"depth":712,"text":382},{"id":537,"depth":712,"text":538},{"id":563,"depth":712,"text":564},{"id":644,"depth":712,"text":645},{"id":675,"depth":712,"text":676},{"id":760,"depth":712,"text":761},{"id":821,"depth":712,"text":822},{"id":923,"depth":712,"text":924},{"id":947,"depth":712,"text":948},"The versioned gold set, cassette replay, regression gate, and why a perfect score is not a quality claim.","md",false,{},true,"provisional",{"title":190,"description":992},"services\u002Fworker\u002Feval; services\u002Fworker\u002FREADME.md; ADR-0066","UcsF_gp2vEUfcNqqiUNxFglL5XOLFVKxh7BwCtZkd4Q",[1002,1004],{"title":186,"path":187,"stem":188,"description":1003,"children":-1},"Embedding profiles, backend geometry checks, tenant-scoped caching, and readiness boundaries.",{"title":194,"path":195,"stem":196,"description":1005,"children":-1},"HTTP and gRPC contracts for the proxy, management plane, auth service, and health surfaces.",1792875076534]