dimagi-internal/ace / skills/demo-data-setup-qa
demo-data-setup-qa
VerifiedInternal /ace:run step — Pass/fail check that a demo dataset handoff is live.
checks.ts
View on GitHub/**
* Static QA checks for `demo-data-setup-qa`, promoted to importable TS.
*
* Why these two exist (dimagi-internal/ace#1161, absorbing #1037):
*
* The skill's stated job, per `lib/artifact-manifest.ts`, is that "a dead
* dashboard must not reach a stakeholder". Run against
* hh-poverty-targeting/20260730-2210 it returned **7/7 pass** on a demo whose
* review dashboard was analytically dead (#1160) and whose walkthrough scored
* concept 2.0/5, user 1.0/5, arc 1.0/5. Every check inspected the HANDOFF —
* realized.json's shape, a URL against a regex, plan↔handoff key agreement, an
* integer, a date. None fetched a `par_url` and looked at what it renders, and
* a regex cannot tell a real run from a fabricated id.
*
* The other half (#1037): check 2 demanded `&opportunity_id=` on every
* dashboard while `demo-data-setup` Step 4 requires `&program_id=` for
* program-owned rollups — so a correctly-built `program_admin_report` failed
* its own gate, and the only URL that passed was one verified to render
* "Workflow definition 5040 not found". Scope is now tied to OWNERSHIP, which
* turns a contradiction into a check that catches the real defect.
*
* Both are pure functions over already-fetched data, so they stay inside the
* QA contract (binary, no LLM, no scores) and are unit-tested directly.
*/
import type { QACheckResult } from '../../lib/qa-types';
import {
classifyLiveLoadEvidence,
describeLiveLoadEvidence,
LIVE_LOAD_PROBE_HINT,
} from '../../lib/interactive-live-load.js';
import type {
ConstraintReport,
DatasetSpec,
ScrubReport,
UnparsedExpression,
} from '../../lib/dataset-constraints';
import {
checkCascadeStoryPlan,
verifyCascadeStoryLanded,
type CascadeStoryPlan,
type GradedPeriod,
} from '../../lib/cascade-story';
import {
checkConstantDisplayedColumns,
checkDrillLevels,
checkReviewRouting,
checkWorkerRateDenominators,
type CascadeSnapshot,
} from '../../lib/cascade-build-qa';
import {
checkRegistryDisplayText,
describeDisplayTextFindings,
indicatorsDocOf,
} from '../../lib/registry-display-text';
/**
* Check 21 — the cascade story was planned AND landed (ace#2510).
*
* The plan is gated before generation; this re-runs that gate and then asks
* the only authority on what a viewer sees — the programme report's SAVED runs
* — whether each authored signal is actually there. A story that did not land
* films a partner table where nothing stands out, which no narration fixes.
*/
export function checkCascadeStory(
plan: CascadeStoryPlan | null | undefined,
registryIndicatorIds: readonly string[],
periods: GradedPeriod[],
/** Real people named in the opp's inputs; no invented worker may carry one's name (ace#2727). */
realPeople: readonly string[] = [],
): QACheckResult {
if (!plan) {
return {
pass: false,
detail: '7-synthetic/cascade-story.yaml is missing — the ace-run provider must author the story before generating',
auto_fix_hint: 'write the CascadeStoryPlan per skills/demo-data-setup § Process (ace-run) C3, then regenerate',
};
}
const planned = checkCascadeStoryPlan(plan, registryIndicatorIds, undefined, { realPeople });
if (planned.verdict === 'fail') {
return {
pass: false,
detail: planned.findings
.filter((f) => f.severity === 'fail')
.map((f) => `${f.signal ?? 'plan'}: ${f.detail}`)
.join('; '),
auto_fix_hint:
'fix the story plan before generating: all four signals on registry indicators with PDD citations; ' +
'realistic invented partner names marked "(example)"; a display roster naming every worker and entity; ' +
'headline indicators that differ across partners (demo-data-setup § C0, C3)',
};
}
const landed = verifyCascadeStoryLanded(plan, periods);
const lines = [
...landed.results.map((r) => `${r.kind} ${r.landed ? 'landed' : 'DID NOT LAND'} — ${r.observed}`),
...landed.headlines.map((h) => `headline ${h.spread_ok ? 'spread' : 'FLAT'} — ${h.observed}`),
];
if (landed.verdict === 'pass') return { pass: true, detail: lines.join('; ') };
return {
pass: false,
detail: lines.join('; '),
auto_fix_hint:
'change the transplant pool for the signal that did not land (widen the separation, fix the carrier), ' +
'regenerate that partner, rebuild history (demo-data-setup § C5) and re-verify. Never re-describe the ' +
'story to match what came out.',
};
}
/**
* Templates whose workflows are PROGRAM-owned (cross-opp rollups). The
* indicator programme report is created with `program_id` by
* demo-data-setup § C4 (ace#2510), so its run URL carries `&program_id=`.
*/
const PROGRAM_OWNED_TEMPLATES = ['program_admin_report', 'audit_par', 'indicator_programme_report'];
export interface DashboardRef {
key: string;
template: string;
par_url: string;
role?: string;
shape?: string;
}
const RUN_DEEPLINK =
/^https:\/\/labs\.connect\.dimagi\.com\/labs\/workflow\/\d+\/run\/\?run_id=\d+&(opportunity_id|program_id)=\d+$/;
/**
* Every `par_url` is a run deep-link AND carries the scope param its
* dashboard's ownership requires.
*
* Live proof both directions matter (workflow 5040 / run 5048 / program 10037):
* `&program_id=` renders the SOP grid; `&opportunity_id=` returns 200 with body
* "Workflow definition 5040 not found." A 200 is not evidence — which is why
* the scope, not just the shape, has to be checked.
*/
export function checkParUrlScope(dashboards: DashboardRef[]): QACheckResult {
const problems: string[] = [];
for (const d of dashboards) {
const m = RUN_DEEPLINK.exec(d.par_url ?? '');
if (!m) {
problems.push(
`${d.key}: not a run deep-link (need /workflow/<id>/run/?run_id=<id>&(opportunity_id|program_id)=<id>) — got ${d.par_url}`,
);
continue;
}
const scope = m[1];
const wantsProgram = PROGRAM_OWNED_TEMPLATES.includes(d.template);
if (wantsProgram && scope !== 'program_id') {
problems.push(
`${d.key}: template '${d.template}' is program-owned, so its par_url MUST carry &program_id= — ` +
`&opportunity_id= 404s it ("Workflow definition not found")`,
);
}
if (!wantsProgram && scope !== 'opportunity_id') {
problems.push(
`${d.key}: template '${d.template}' is opp-owned, so its par_url MUST carry &opportunity_id=`,
);
}
}
if (problems.length === 0) return { pass: true, detail: `${dashboards.length} par_url(s) correctly scoped` };
return {
pass: false,
detail: problems.join('; '),
auto_fix_hint:
'rebuild each par_url with the scope its ownership requires: program_admin_report / audit_par → ' +
'&program_id=<program_id>; every opp-owned dashboard → &opportunity_id=<labs_opp_id>. Both forms ' +
'return HTTP 200, so verify by fetching and confirming the page renders its grid rather than ' +
'"Workflow definition <id> not found" (dimagi-internal/ace#1037).',
};
}
/**
* Every `*worker_review_url` in realized.json is a run deep-link that carries
* `program_id=` — the page-scope param a program-owned worker review needs
* when it is opened COLD (a DDD scene `url:`), not after a click-through.
*
* `owning_program_id` is NOT a substitute. In labs it is only a data-access
* hint (`_run_program_hint`, connect_labs/workflow/views.py) and is
* deliberately not one of `labs.context.CONTEXT_PARAMS`
* (`organization_id`, `program_id`, `opportunity_id`). With no context param
* on the URL, labs appends the session's last-selected `opportunity_id` and
* the page reads "Workflow definition <id> not found under opportunity <n>."
* Observed spark-facilitator/20260926-1413 (ace#2521): the `owning_program_id`
* form 302s to `&opportunity_id=10087` and renders no `#workflow-data`; the
* `&program_id=10085` form renders the review.
*/
export function checkWorkerReviewUrlScope(realized: Record<string, unknown>): QACheckResult {
const keys = Object.keys(realized).filter((k) => /(^|_)worker_review_url$/.test(k));
const problems: string[] = [];
for (const key of keys) {
const raw = realized[key];
let url: URL;
try {
url = new URL(String(raw));
} catch {
problems.push(`${key}: not a URL — got ${String(raw)}`);
continue;
}
if (!/^\/labs\/workflow\/\d+\/run\/$/.test(url.pathname) || !/^\d+$/.test(url.searchParams.get('run_id') ?? '')) {
problems.push(`${key}: not a run deep-link (need /labs/workflow/<id>/run/?run_id=<id>&program_id=<id>) — got ${raw}`);
continue;
}
if (!/^\d+$/.test(url.searchParams.get('program_id') ?? '')) {
const hint = url.searchParams.has('owning_program_id')
? ' — owning_program_id is a data hint, not a page-scope param; opened cold, labs scopes the page to the ' +
"session's last opportunity and renders \"Workflow definition <id> not found\""
: '';
problems.push(`${key}: worker review is program-owned, so its URL MUST carry &program_id=<id>${hint}`);
}
}
if (problems.length === 0) return { pass: true, detail: `${keys.length} worker_review_url(s) correctly scoped` };
return {
pass: false,
detail: problems.join('; '),
auto_fix_hint:
'rebuild each worker_review_url with scripts/worker-review-url.ts (--fix <realized.json> repairs in place): ' +
'/labs/workflow/<review>/run/?run_id=<review run>&program_id=<program>' +
'&flw=<opp>%3A%3A<username>&source_run=<latest> (demo-data-setup § C7; dimagi-internal/ace#2521).',
};
}
export type PayloadFindingKind =
| 'snapshot-missing-pipelines'
| 'field-all-null'
| 'live-pipelines-unavailable';
export interface PayloadFinding {
kind: PayloadFindingKind;
alias?: string;
field?: string;
detail: string;
}
export interface PayloadReport {
pass: boolean;
findings: PayloadFinding[];
}
export interface SnapshotPipeline {
alias?: string;
rows?: Record<string, unknown>[];
}
export interface WorkflowPayload {
definition?: { pipeline_sources?: Record<string, unknown> | Array<{ alias?: string; [k: string]: unknown }> };
instance?: {
status?: string;
/**
* The run's half-open `[period_start, period_end)` visit-date window, both
* exposed on the `instance` prop (verified live 2026-08-14). Read only to
* EXPLAIN a cross-dashboard disagreement (ace#1683) — never to excuse one.
*/
period_start?: string;
period_end?: string;
/**
* `pipelines` arrives in EITHER shape (ace#1701):
* - a dict keyed by alias — what labs actually writes today
* (`connect_labs/workflow/templates/__init__.py`:
* `out["pipelines"] = {alias: pipelines[alias] ...}`)
* - an array of `{alias, rows}` — the shape this check was written for.
*/
snapshot?: { pipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline> };
};
}
/**
* Both snapshot shapes, as one array (ace#1701).
*
* Reading only the array shape made this check throw `pipelines is not
* iterable` on every real completed run, and made the
* `snapshot-missing-pipelines` branch unreachable besides — a dict has no
* `.length`, and `undefined === 0` is false. Check 7 exists so that a dead
* dashboard cannot reach a stakeholder (#1161); it could not see one.
*/
export function normalizeSnapshotPipelines(
pipelines: SnapshotPipeline[] | Record<string, SnapshotPipeline> | undefined,
): SnapshotPipeline[] {
if (!pipelines) return [];
if (Array.isArray(pipelines)) return pipelines;
return Object.entries(pipelines).map(([alias, value]) => ({ alias, ...(value ?? {}) }));
}
/** `pipeline_sources` is a dict of alias->id in the stored definition and an array of {alias,...} on the run page. */
function declaredAliases(sources: WorkflowPayload['definition'] extends infer D ? any : never): string[] {
if (!sources) return [];
if (Array.isArray(sources)) {
return sources.map((s: { alias?: string }) => String(s?.alias ?? '')).filter(Boolean);
}
return Object.keys(sources);
}
/** Null, undefined, empty string, or numeric zero — the shapes a dead binding produces. */
function isDeadValue(v: unknown): boolean {
return v === null || v === undefined || v === '' || v === 0;
}
/**
* labs' own row columns, which every pipeline row carries whether or not the
* terminal stage fills them (ace#1701).
*
* Whether one of these is populated is decided by `terminal_stage`, not by the
* schema: WORKFLOW_REFERENCE § "Built-in Row Fields" gives `visit_level` rows
* `username / visit_date / entity_id / entity_name` and `aggregated` rows the
* `*_visits` counters and `first/last_visit_date`, and the `entity` stage
* "drops the status/flagged counters because they're visit-level facts". So a
* built-in that is null for every row is NOT evidence of a wrong schema path —
* and it fires on every entity-stage and visit-level pipeline. Measured on
* run 5258 (healthy, `pipeline_preview` `fields_all_null: []`): 15 findings,
* all built-ins, none a schema path.
*
* #1160's real defect is not one of these: that was render code binding the
* denormalized `worker.visit_count`, not a pipeline column.
*/
const LABS_BUILTIN_ROW_COLUMNS = new Set([
'id', 'username', 'visit_date', 'entity_id', 'entity_name', 'status', 'flagged',
'opportunity_id', 'days_active', 'total_visits', 'approved_visits', 'pending_visits',
'rejected_visits', 'flagged_visits', 'first_visit_date', 'last_visit_date',
]);
/**
* Does the page actually render data? Operates on the parsed `#workflow-data`
* payload the run page embeds — so the fetch belongs to the caller and the
* judgement is pure and testable.
*
* Two unambiguous shapes, both taken from #1160:
* - `pipeline_sources` declared while the snapshot carries no pipelines at all
* (workflow 5069 declared `performance_data -> 5068` with
* `snapshot_inputs.pipelines: []`, so `pipeline_data` was `[]` live);
* - a bound field that is null/zero for EVERY row (pipeline 5068 kept the
* stock `performance_review` schema pointed at `form.meta.instanceID`, which
* the synthetic generator never writes → `visit_count` 0 for all 8 workers,
* beside a chip reading `visits: 835`).
*
* A field that is zero for SOME rows is data, not a dead binding — flagging it
* would make this the always-fires class.
*/
/**
* @param livePipelines Rows for a run that is deliberately still `in_progress`
* (the `review-action` dashboard check 8 requires to stay live, #1162). Such
* a run has no snapshot BY DESIGN, and the server-rendered `#workflow-data`
* carries `pipeline_data: {}` because the page fills it over SSE after
* mount — so the caller fetches
* `GET /labs/workflow/api/<def>/pipeline-data/?opportunity_id=<opp>` and
* passes the result here. Absence is a reported finding, never a silent pass.
*/
export function checkParUrlPayloadPopulated(
payload: WorkflowPayload,
livePipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline>,
): PayloadReport {
const findings: PayloadFinding[] = [];
const declared = declaredAliases(payload?.definition?.pipeline_sources);
const completed = payload?.instance?.status === 'completed';
const live = normalizeSnapshotPipelines(
livePipelines ?? (payload as { pipeline_data?: Record<string, SnapshotPipeline> })?.pipeline_data,
);
const pipelines = completed
? normalizeSnapshotPipelines(payload?.instance?.snapshot?.pipelines)
: live;
if (declared.length > 0 && pipelines.length === 0) {
findings.push({
kind: completed ? 'snapshot-missing-pipelines' : 'live-pipelines-unavailable',
detail: completed
? `definition declares pipeline_sources (${declared.join(', ')}) but the run snapshot holds no ` +
`pipelines — the page renders no pipeline data at all. Declare snapshot_inputs.pipelines ` +
`covering every alias in pipeline_sources before completing the run.`
: `this run is '${payload?.instance?.status ?? 'unknown'}', so it has no snapshot by design ` +
`(#1162) — and no live pipeline rows were supplied for ${declared.join(', ')}, so nothing ` +
`was judged. Fetch GET /labs/workflow/api/<definition_id>/pipeline-data/?opportunity_id=<opp> ` +
`and pass it as livePipelines.`,
});
}
for (const p of pipelines) {
const rows = p.rows ?? [];
if (rows.length === 0) continue;
const columns = new Set<string>();
for (const r of rows) for (const k of Object.keys(r)) columns.add(k);
for (const field of columns) {
if (LABS_BUILTIN_ROW_COLUMNS.has(field)) continue;
const allDead = rows.every((r) => isDeadValue(r[field]));
if (allDead) {
findings.push({
kind: 'field-all-null',
alias: p.alias,
field,
detail: `${p.alias ?? 'pipeline'}.${field} is null/zero for all ${rows.length} row(s) — the schema path likely never matches the generated fixtures`,
});
}
}
}
return { pass: findings.length === 0, findings };
}
export function formatPayloadReport(report: PayloadReport): string {
if (report.pass) return 'par_url payload: populated — pipelines present and no uniformly-dead bound field';
return [
'par_url payload: the dashboard renders but is analytically DEAD —',
...report.findings.map((f) => ` [${f.kind}] ${f.detail}`),
'A dead dashboard must not reach a stakeholder (lib/artifact-manifest.ts).',
'Re-point the pipeline schema at the REAL form paths the generator writes, and',
'declare snapshot_inputs.pipelines for every alias (dimagi-internal/ace#1160/#1161).',
].join('\n');
}
// ── #1162: the interactive run must still be interactive at render time ──
/**
* Roles whose dashboard exists so that someone can ACT on it on camera.
* Compared after lowercasing and folding `_` to `-`.
*/
const INTERACTIVE_ROLES = new Set(['review-action', 'review', 'decision']);
function isInteractiveRole(role: string | undefined): boolean {
return INTERACTIVE_ROLES.has((role ?? '').trim().toLowerCase().replace(/_/g, '-'));
}
export interface DashboardPayloadPair {
dashboard: DashboardRef;
payload: WorkflowPayload;
}
/**
* Exactly the review-action run stays `in_progress`; every other run is
* `completed`.
*
* Completing a run is how `par_url` becomes a stable, idempotent deep-link —
* correct, and what most dashboards want. But it also flips the page
* read-only: workflow 5069's render code carries a `completed` branch that
* prints "This run is completed… Decisions are read-only" and disables the
* status dropdown. On hh-poverty-targeting/20260730-2210 Phase 7 completed
* BOTH runs ~14 minutes before the render, so the narrative's payoff — a
* reviewer taking a decision — had nothing to click: all 10 spec actions were
* wait_for/hold, 7 scenes yielded 2 distinct images, arc 1.0/5 (#1162).
*
* Two-sided on purpose. Firing only on the completed-interactive case would
* accept the opposite failure — everything left in_progress, quietly trading
* away snapshot stability on links a stakeholder keeps.
*
* A payload with no `instance.status` is reported, not failed: an absent field
* is a fetch-shape question, and a QA gate that fails on what it cannot see is
* the always-fires class (ace#1026).
*
* **ace#2430 — the mandate is now verified, and it has an evidenced escape.**
* Leaving the run `in_progress` costs a LIVE dependency: that page has no
* snapshot, so every load re-runs the pipeline SSE stream, while a completed
* run returns early on its snapshot and never opens one
* (`workflow-runner.tsx`: `if (snapshotCarriesPipelines) return;`). On
* `poverty-graduation/20260915-1518` that stream died after its first event and
* the interactive dashboard could not load at all — and because this check was
* one-sided, the ONLY QA-passing configuration was the one that could not
* render. So: the producer must record `source.interactive_live_load` (the
* probe it ran), an unproven `in_progress` run fails, a run whose own probe
* says it cannot load fails, and a `completed` interactive run passes when — and
* only when — the failure is evidenced against a rendering control run. Same
* cite-your-source discipline as `below_programme_scale` and
* `declared_omissions[]`; the escape is reported in `detail`, never silent.
*/
export function checkInteractiveRunsLive(
pairs: DashboardPayloadPair[],
liveLoad?: unknown,
): QACheckResult {
const problems: string[] = [];
const unknown: string[] = [];
const notes: string[] = [];
// ace#2430: the mandate now carries its proof. An interactive run has NO
// snapshot, so its page re-runs the live pipeline stream on every load —
// including every render take — while a completed run returns early on its
// snapshot and never opens one. That asymmetry is why `in_progress` needs
// verifying and `completed` does not, and it is what made the old one-sided
// rule enforce a page that could not render.
const anyInteractive = pairs.some(({ dashboard }) => isInteractiveRole(dashboard.role));
const evidence = classifyLiveLoadEvidence(liveLoad);
const escapeGranted = anyInteractive && evidence.status === 'failed' && evidence.granted;
const provenLive = anyInteractive && evidence.status === 'ok' && evidence.granted;
if (anyInteractive && !evidence.granted) {
problems.push(
`the interactive dashboard's live-load is unproven: ${evidence.problems.join('; ')}. An in_progress ` +
`run has no snapshot, so its page depends on the live pipeline stream every time it is opened`,
);
} else if (escapeGranted) {
notes.push(
`live-load escape taken (evidenced): ${describeLiveLoadEvidence(evidence, liveLoad)} — the interactive ` +
`run ships completed because its live load was observed failing against a rendering control`,
);
}
for (const { dashboard, payload } of pairs) {
const status = payload?.instance?.status;
const interactive = isInteractiveRole(dashboard.role);
if (!status) {
unknown.push(`${dashboard.key} (role ${dashboard.role ?? 'unset'})`);
continue;
}
if (interactive && status === 'completed' && !escapeGranted) {
problems.push(
`${dashboard.key}: role '${dashboard.role}' is interactive but its run is completed — the page ` +
`renders "This run is completed… Decisions are read-only" with the status control disabled, so ` +
`the decision the narrative demonstrates cannot be performed on camera`,
);
}
if (interactive && status !== 'completed' && escapeGranted) {
problems.push(
`${dashboard.key}: its recorded live-load probe says this page cannot load ` +
`(${describeLiveLoadEvidence(evidence, liveLoad)}), yet the run shipped '${status}' — a dead ` +
`par_url must not reach a stakeholder; complete the run and say so on the page`,
);
}
if (!interactive && status !== 'completed') {
problems.push(
`${dashboard.key}: role '${dashboard.role ?? 'unset'}' is non-interactive but its run is ` +
`'${status}' — only a completed run carries a snapshot, so this par_url is not a stable ` +
`idempotent deep-link`,
);
}
}
const unknownNote = unknown.length ? ` (unknown run status, not judged: ${unknown.join(', ')})` : '';
const noteText = notes.length ? `; ${notes.join('; ')}` : '';
if (problems.length === 0) {
const proof = provenLive ? `, interactive live-load proven (${describeLiveLoadEvidence(evidence, liveLoad)})` : '';
return {
pass: true,
detail: `${pairs.length} dashboard run state(s) match their role${proof}${unknownNote}${noteText}`,
};
}
return {
pass: false,
detail: problems.join('; ') + unknownNote + noteText,
auto_fix_hint:
'Leave ONLY the review-action dashboard\'s run in_progress — skip workflow_save_snapshot for it — ' +
'and complete every other dashboard\'s run as usual. The interactive dashboard trades snapshot ' +
'stability for a page the reviewer can actually act on; the rest keep it (dimagi-internal/ace#1162). ' +
LIVE_LOAD_PROBE_HINT,
};
}
// ── #1658: check 9's spec is DERIVED, and its auto-fix is one that exists ──
/**
* Check 9 (`dataset_obeys_pdd_constraints`) as a pure function.
*
* Two defects it closes, both measured on `bednet-check-2-visit` (ace#1658):
*
* 1. **The spec was author-supplied**, so under-declaring it produced a false
* green. `20260817-1720` recorded `pass` with `conditionalFields: []` and
* the justification "no conditional blocks" — while
* `get_opportunity_apps(2214, 'deliver')` returned two `relevant`
* expressions verbatim. `20260825-1310`, same opp / app / generator,
* declared them and measured 18 of 276 off-branch on each. So the spec now
* comes from `specFromDeliverApp`, and this check REFUSES a run that has
* no derivation behind it: a `derivation` of `null` is only legal with a
* stated reason (the `denovo` provider has no deliver app to read).
*
* 2. **The old auto-fix was impossible on this path.** "Regenerate with the
* constraint applied at the manifest" reached for a knob that, at the time,
* did not exist. It exists now: `BeneficiaryCohort.relevance_groups`
* (`connect-labs#1331`, merged 2026-08-27). The remedy is unchanged anyway,
* because the primitive is INERT for the runs this check guards — relevance
* is applied only to questions present in the HQ `FormSchema`
* (`fixtures/fields.py:417-422`), and the trailing orphan-write loop is gated
* by a set computed ONCE at `fields.py:484`, before it runs. A labs-only opp
* has no Connect `app_structure`, so `FormSchema.questions` is empty
* (`schema_loader.parse_form_schema_from_app_json`), every declared path is an
* orphan, and the controller is not in the record when line 484 evaluates the
* gate. Measured on `bednet-check-2-visit/20260828-0629`: the producer
* DECLARED `relevance_groups` and the generator still emitted 36 off-branch
* values (ace#1833). So the hint points at `scrubOffBranchFields` — a
* declared, reproducible, idempotent generator post-step — and says to
* declare `relevance_groups` on a schema-backed opp, where it does work.
*
* An unparsed `relevant` is a FINDING, not a silent pass: an expression the
* derivation could not read is a gate this check cannot prove was audited.
*
* 3. **It had no evidenced escape** (ace#2225), so a dataset whose only
* residual violations were fields labs STRUCTURALLY cannot simulate could
* never pass. On `poverty-graduation/20260908-0510` all six residuals were
* a REPEAT-group roster (the generator emits one flat object, not an
* array — TRUE THEN, REFUTED SINCE: the manifest carries
* `repeat_groups` and `auditDataset` reads inside an array, so a repeat is
* built, not exempted, ace#2432), a `Trigger` read-aloud label (CommCare
* submits no value for one),
* or an image with no labs `ImageConfig` corpus — with zero off-branch,
* zero integrality and an empty `unparsed[]`. The only two routes to green
* were to narrow the spec, which is the behaviour defect 1 exists to
* prevent, or to accept a permanent `fail` — and a permanently-red check is
* one nobody reads, which is how the real misses get waved through
* (ace#1744). So it takes the shape `checkDetectionCohortFloor` already
* uses for `below_programme_scale`: an EVIDENCED escape, refused when
* unevidenced, reported rather than silent, and scoped to the two halves a
* structural absence actually shows up in — a `conditional-missing`
* violation and the scrub's `unresolvedFields`. An off-branch value, an
* integrality violation and an unparsed gate stay unexemptible: each names
* a value that IS present and wrong, which the scrub or an added spec entry
* can repair.
*/
export interface DatasetConstraintCheckInput {
/**
* `specFromDeliverApp(get_opportunity_apps(<opp>, 'deliver'))`, or `null`
* when the demo has no deliver app to derive from.
*/
derivation: { unparsed: UnparsedExpression[]; questionsSeen: number; gatesParsed: number } | null;
/** Required when `derivation` is null — why no app was read (e.g. provider `denovo`). */
noDeliverAppReason?: string;
/** The scrub actually applied to the fixture before the dashboards read it. */
scrub?: ScrubReport;
/** `auditDataset` over the records as they now stand. */
report: ConstraintReport;
/**
* `declared_omissions[]` from `7-synthetic/branch-scrub_report.yaml` — the
* fields the labs generator STRUCTURALLY cannot emit, each with the reason
* (ace#2225). Exempts that field's `conditional-missing` violation and its
* `unresolvedFields` entry, and nothing else. An entry with a blank reason
* exempts nothing and is named in the failure.
*/
declaredOmissions?: DeclaredOmission[];
/**
* `derivation.unparsed_resolutions[]` from `7-synthetic/branch-scrub_report.yaml`
* — how each gate `specFromDeliverApp` could not parse was actually audited
* (ace#2497). An unparsed entry is cleared only by a resolution matching it on
* `field` + `expression` with a known `resolution` kind and a non-empty
* `detail` stating the measurement. `derivation.unparsed` stays the RAW
* derivation output — never edit it to `[]`; that is indistinguishable from
* deleting the list.
*/
unparsedResolutions?: UnparsedResolution[];
/**
* The spec `auditDataset` actually ran over: `mergeDatasetSpecs(derived.spec, additions)`
* (ace#2245). A `spec-addition` resolution clears its unparsed gate only when this spec
* carries a rule on that field that audits the expression: a conditional entry for the
* field that does not still list the expression in `unparsedGates`, an integer entry with
* a bound, or a cross-field rule naming it. Without it, a `spec-addition` is an author
* assertion nothing checked, and it clears nothing.
*/
mergedSpec?: DatasetSpec;
/**
* `countFieldPresence(records, <fields the merged spec's rules read>)` over the records
* as they now stand. A spec-addition rule whose gate field is carried by 0 records is
* REPORTED as declared but never firing. It does not fail: the gate is declared, but it
* audits nothing (ace#2245, `enrol_gps_accuracy_band` in 0 of 182 records).
*/
fieldPresence?: Record<string, number>;
}
/** One entry of the producer's `derivation.unparsed_resolutions[]` (ace#2497). */
export interface UnparsedResolution {
field: string;
expression: string;
/**
* `spec-addition` (declared via mergeDatasetSpecs and audited),
* `direct-measured-assertion` (measured over the records; `detail` carries the count),
* `spec-correction` (a derived entry replaced by the app's own, tighter gate),
* `out-of-scope-form` (a question on a form the fixture does not generate; `detail` carries the count).
*/
resolution: string;
detail: string;
}
const RESOLUTION_KINDS = new Set(['spec-addition', 'direct-measured-assertion', 'spec-correction', 'out-of-scope-form']);
/** One `{field, reason}` entry of the producer's `declared_omissions[]`. */
export interface DeclaredOmission {
field: string;
/** Why labs cannot emit this field. Blank means the exemption is refused. */
reason: string;
}
const SCRUB_HINT =
'Re-run the branch scrub, do not narrow the spec: scrubOffBranchFields(records, spec.conditionalFields) ' +
'from lib/dataset-constraints.ts, then write the scrubbed user_visits.json back to the opp\'s fixture ' +
'folder BEFORE any dashboard run is minted, and record the per-field counts in the run summary. There is ' +
'no manifest-side remedy ON THIS PATH: BeneficiaryCohort.relevance_groups DOES exist ' +
'(dimagi-internal/connect-labs#1331, merged 2026-08-27), but relevance is only applied to questions ' +
'present in the HQ FormSchema. A labs-only opp has no Connect app_structure, so that schema is empty, ' +
'every declared path is orphan-written, and the orphan gate is computed once BEFORE the orphan loop ' +
'(fixtures/fields.py:484) from a record that does not yet hold the controller \u2014 so it can never fire. ' +
'Declaring relevance_groups here is inert: a run that declared it still emitted 36 off-branch values. ' +
'Do declare it on a schema-backed opp, where it works (dimagi-internal/ace#1658, #1833).';
/**
* The evidenced half of the declared-omission escape (ace#2225).
*
* A field only earns an exemption when its entry carries a non-empty reason —
* the same cite-your-source discipline `checkDetectionCohortFloor` imposes on
* `below_programme_scale`. An unevidenced entry is a silencer, and an escape
* is only worth having if turning the gate off costs the author the citation.
*/
function partitionOmissions(declared: DeclaredOmission[] | undefined): {
evidenced: Map<string, string>;
unevidenced: Set<string>;
} {
const evidenced = new Map<string, string>();
const unevidenced = new Set<string>();
for (const o of declared ?? []) {
const field = typeof o?.field === 'string' ? o.field.trim() : '';
if (!field) continue;
const reason = typeof o?.reason === 'string' ? o.reason.trim() : '';
if (reason) evidenced.set(field, reason);
else unevidenced.add(field);
}
return { evidenced, unevidenced };
}
const OMISSION_HINT =
'A residual the labs generator STRUCTURALLY cannot emit is declared, not narrowed away: add a ' +
'{field, reason} entry to declared_omissions[] in 7-synthetic/branch-scrub_report.yaml naming why the ' +
'value cannot exist (a Trigger read-aloud label CommCare submits no value for, an image with no labs ' +
'ImageConfig corpus). A REPEAT-group roster is NOT one of them any more: the manifest carries ' +
'BeneficiaryCohort.repeat_groups and auditDataset reads inside an array, so build the repeat rather than ' +
'exempting it (dimagi-internal/ace#2432). The reason is REQUIRED — an ' +
'unevidenced entry exempts nothing. This escape covers conditional-missing and an unresolved scrub field ' +
'only: an off-branch value, an integrality violation and an unparsed gate are never exempt, because each ' +
'of those is a dataset defect the scrub or an added spec entry can actually fix (dimagi-internal/ace#2225).';
/**
* The merged-spec rules that audit one unparsed expression, each with the fields it must
* read to fire (ace#2245). A `relevant` gate is audited by a conditional entry for the field
* that no longer lists the expression in `unparsedGates`, or a cross-field rule naming the
* field; a `constraint` by an integer entry with a bound or a cross-field rule.
*/
function additionRulesFor(spec: DatasetSpec | undefined, u: UnparsedExpression): Array<{ gateFields: string[] }> {
if (!spec) return [];
const expr = String(u.expression).trim();
const rules: Array<{ gateFields: string[] }> = [];
if (u.kind === 'relevant') {
for (const c of spec.conditionalFields ?? []) {
if (c.field === u.field && !(c.unparsedGates ?? []).some((g) => g.trim() === expr)) rules.push({ gateFields: [c.requiredWhen.field] });
}
} else {
for (const f of spec.integerFields ?? []) {
if (f.field === u.field && (f.min !== undefined || f.max !== undefined)) rules.push({ gateFields: [f.field] });
}
}
for (const r of spec.crossFieldRules ?? []) {
if (r.lhs === u.field || r.rhs === u.field) rules.push({ gateFields: [r.lhs, r.rhs] });
}
return rules;
}
export function checkDatasetObeysPddConstraints(input: DatasetConstraintCheckInput): QACheckResult {
const problems: string[] = [];
const hints: string[] = [];
const { evidenced, unevidenced } = partitionOmissions(input.declaredOmissions);
const exempted: string[] = [];
const noteExempt = (field: string) => {
if (!exempted.includes(field)) exempted.push(field);
};
if (!input.derivation) {
if (!input.noDeliverAppReason?.trim()) {
problems.push(
'no spec derivation and no stated reason — the spec would be hand-declared from prose, which is ' +
'the false-green shape this check exists to close',
);
hints.push(
'derive the spec with specFromDeliverApp(get_opportunity_apps(<connect opp>, \'deliver\')); if this ' +
'demo genuinely has no deliver app (provider denovo), state that as noDeliverAppReason.',
);
}
} else if (input.derivation.questionsSeen === 0) {
problems.push(
'the deliver app returned 0 questions, so conditionalFields / integerFields were derived from nothing',
);
hints.push(
're-fetch get_opportunity_apps(<connect opp>, \'deliver\') and confirm deliver_app is non-null before ' +
'auditing; an empty app derives an empty spec, and an empty spec measures zero violations.',
);
}
const rawUnparsed = input.derivation?.unparsed ?? [];
// ace#2497: an unparsed gate is cleared only by an EVIDENCED resolution that
// names it (field + expression) — same discipline as declared_omissions.
const resolutions = (input.unparsedResolutions ?? []).filter(
(r) =>
typeof r?.field === 'string' &&
typeof r?.expression === 'string' &&
RESOLUTION_KINDS.has(String(r?.resolution ?? '').trim()) &&
String(r?.detail ?? '').trim().length > 0,
);
const resolvedKey = new Set(resolutions.map((r) => `${r.field}\u0000${r.expression.trim()}`));
const resolvedNote: string[] = [];
// ace#2245: a spec-addition is checked against the merged spec, not taken on the author's word.
const unbacked: string[] = [];
const neverFires: string[] = [];
const unparsed = rawUnparsed.filter((u) => {
if (resolvedKey.has(`${u.field}\u0000${String(u.expression).trim()}`)) {
const r = resolutions.find((x) => x.field === u.field && x.expression.trim() === String(u.expression).trim());
if (r && r.resolution.trim() === 'spec-addition') {
const rules = additionRulesFor(input.mergedSpec, u);
if (rules.length === 0) {
unbacked.push(`${u.field} [${u.kind}] ${u.expression}`);
return true;
}
if (input.fieldPresence) {
const dead = rules.flatMap((g) => g.gateFields).filter((f) => input.fieldPresence![f] === 0);
if (dead.length && rules.every((g) => g.gateFields.some((f) => input.fieldPresence![f] === 0))) {
neverFires.push(`${u.field} (gate field ${[...new Set(dead)].join(', ')} carried by 0 of ${input.report.total} records)`);
}
}
}
if (r) resolvedNote.push(`${u.field}: ${r.resolution}`);
return false;
}
return true;
});
if (unbacked.length) {
problems.push(
`${unbacked.length} spec-addition resolution(s) name a gate the merged spec carries no rule for, so the ` +
`addition was asserted, not audited: ${unbacked.join('; ')}` +
(input.mergedSpec ? '' : ' (no mergedSpec supplied)'),
);
hints.push(
'pass mergedSpec = mergeDatasetSpecs(derived.spec, additions), the spec auditDataset ran over; a ' +
'spec-addition must put a rule on the unparsed field there (a conditional entry not still listing the ' +
'expression in unparsedGates, an integer bound, or a cross-field rule) (ace#2245).',
);
}
if (unparsed.length > 0) {
problems.push(
`${unparsed.length} expression(s) could not be derived into the spec, so those gates were NOT audited: ` +
unparsed.map((u) => `${u.field} [${u.kind}] ${u.expression}`).join('; '),
);
hints.push(
'hand-declare each unparsed gate as an ADDITION (mergeDatasetSpecs(derived, additions)) — never as a ' +
'replacement for the derived spec — then re-run the scrub and the audit, and record HOW each one was ' +
'audited as a {field, expression, resolution, detail} entry in derivation.unparsed_resolutions[] ' +
'(resolution: spec-addition | direct-measured-assertion | spec-correction | out-of-scope-form; detail ' +
'carries the measured count). Keep derivation.unparsed as the raw derivation output (ace#2497).',
);
}
// A field labs cannot emit is also a field the scrub cannot locate, so the
// escape has to cover BOTH halves or the check stays permanently red. On
// `poverty-graduation/20260908-0510` five of the six residuals appeared here
// as well as in the audit below (ace#2225).
const unresolved = (input.scrub?.unresolvedFields ?? []).filter((f) => {
if (evidenced.has(f)) {
noteExempt(f);
return false;
}
return true;
});
if (unresolved.length) {
problems.push(
`the branch scrub could not locate ${unresolved.join(', ')} in any record, so those ` +
'fields were never scrubbed',
);
hints.push(
'check the fixture nesting against the app XPath — the scrub resolves a field by XPath suffix, and a ' +
'field it cannot find is reported rather than silently counted as clean.',
);
}
// `conditional-missing` is the ONLY exemptible class: it is the one that can
// record an absence the generator was never able to produce. Every other
// class names a value that IS present and wrong, which the scrub or an added
// spec entry can actually repair — so none of them takes an escape.
const violations = input.report.violations.filter((v) => {
if (v.kind !== 'conditional-missing' || !v.field) return true;
if (evidenced.has(v.field)) {
noteExempt(v.field);
return false;
}
return true;
});
if (violations.length) {
problems.push(
`${violations.length} constraint class(es) violated across ${input.report.total} records: ` +
violations.map((v) => `[${v.kind}] ${v.count} of ${input.report.total} — ${v.field}`).join('; '),
);
hints.push(SCRUB_HINT);
}
// Name the refused declarations explicitly. Silently ignoring one would read
// to the author as "the escape does not work", and send them back to the
// spec-narrowing this check exists to prevent.
const refused = [...unevidenced].filter(
(f) =>
input.report.violations.some((v) => v.kind === 'conditional-missing' && v.field === f) ||
(input.scrub?.unresolvedFields ?? []).includes(f),
);
if (refused.length) {
problems.push(
`declared_omissions carries no reason for ${refused.join(', ')} — an unevidenced exemption is ` +
'rejected, so those residuals still count',
);
}
if (refused.length || violations.some((v) => v.kind === 'conditional-missing') || unresolved.length) {
hints.push(OMISSION_HINT);
}
const exemptNote = exempted.length
? `; ${exempted.length} declared omission(s) exempted — ` +
exempted.map((f) => `${f}: ${evidenced.get(f)}`).join('; ')
: '';
const resolvedTxt = resolvedNote.length
? `; ${resolvedNote.length} unparsed gate(s) resolved — ${resolvedNote.join('; ')}`
: '';
const neverFiresTxt = neverFires.length
? `; REPORTED, declared but never firing (ace#2245): ${neverFires.join('; ')}`
: '';
if (problems.length === 0) {
const derivedNote = input.derivation
? `spec derived from ${input.derivation.questionsSeen} question(s), ${input.derivation.gatesParsed} gate(s)`
: `no deliver app (${input.noDeliverAppReason})`;
const scrubNote = input.scrub
? `; branch scrub cleared ${input.scrub.totalCleared} off-branch value(s)`
: '';
return {
pass: true,
detail:
`0 unexempted violations across ${input.report.total} records (measured) — ` +
`${derivedNote}${scrubNote}${resolvedTxt}${neverFiresTxt}${exemptNote}`,
};
}
return {
pass: false,
detail: problems.join('; ') + resolvedTxt + neverFiresTxt + exemptNote,
auto_fix_hint: hints.join(' '),
};
}
// ── #2246: check 4's anchor clause names a value the gate can read ──
/**
* Check 4 (`timeline_pinned`) as a pure function (ace#2246).
*
* Two clauses: the manifest `timeline.start_date` is a fixed ISO Monday (a
* relative or sliding expression breaks idempotency), and it equals every
* minted run's `instance.period_start` — the `par_url` payloads check 7
* already fetched. The second clause used to read "equals the `par_url`
* opportunity's env anchor", which names no atom: `synthetic_env_get` is keyed
* by an ENV TEMPLATE, not an opportunity, so a gate could evaluate only the
* first clause and still report a full pass.
*
* A clause that cannot be judged is named (`anchor-not-judged`) and FAILS the
* check, because an applicable check that was not evaluated is a failure,
* never a skip. A payload whose run declares no `period_start` is listed as
* not judged; if no payload declares one, the clause was not judged.
*/
export function checkTimelinePinned(
startDate: unknown,
payloads: Array<{ key: string; payload: WorkflowPayload | null | undefined }>,
): QACheckResult {
const raw = typeof startDate === 'string' ? startDate.trim() : '';
const iso = /^\d{4}-\d{2}-\d{2}$/.test(raw) ? new Date(`${raw}T00:00:00Z`) : null;
if (!iso || Number.isNaN(iso.getTime()) || iso.toISOString().slice(0, 10) !== raw) {
return {
pass: false,
detail: `timeline.start_date ${JSON.stringify(startDate ?? null)} is not a fixed ISO date (YYYY-MM-DD)`,
auto_fix_hint: 'set --pin-monday to a fixed Monday; a relative or sliding window breaks idempotency',
};
}
if (iso.getUTCDay() !== 1) {
const day = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][iso.getUTCDay()];
return {
pass: false,
detail: `timeline.start_date ${raw} is a ${day}, not a Monday`,
auto_fix_hint: 'set --pin-monday to a fixed Monday; a relative or sliding window breaks idempotency',
};
}
const judged = payloads.filter((p) => typeof p.payload?.instance?.period_start === 'string' && p.payload.instance.period_start.trim());
const unjudged = payloads.filter((p) => !judged.includes(p)).map((p) => p.key);
if (judged.length === 0) {
return {
pass: false,
detail:
`timeline.start_date ${raw} is a fixed Monday; anchor-not-judged: ` +
(payloads.length
? `no par_url payload declares instance.period_start (${unjudged.join(', ')})`
: 'no par_url payload was supplied'),
auto_fix_hint:
'pass the par_url payloads check 7 fetched (#workflow-data); the anchor is each minted run\'s instance.period_start',
};
}
const off = judged.filter((p) => String(p.payload!.instance!.period_start).trim().slice(0, 10) !== raw);
const notJudgedTxt = unjudged.length ? `; not judged (no period_start): ${unjudged.join(', ')}` : '';
if (off.length) {
return {
pass: false,
detail:
`timeline.start_date ${raw} disagrees with the minted run(s): ` +
off.map((p) => `${p.key} period_start=${p.payload!.instance!.period_start}`).join('; ') +
notJudgedTxt,
auto_fix_hint:
're-mint the run(s) with period_start = timeline.start_date, or correct the manifest pin; the dashboards ' +
'and the manifest must describe one window',
};
}
return {
pass: true,
detail: `timeline.start_date ${raw} is a fixed Monday and equals period_start on ${judged.length} minted run(s)${notJudgedTxt}`,
};
}
// ── #1683: two dashboards over ONE dataset must report the same total ──
/**
* Check 11 (`cross_dashboard_totals_agree`) as a pure function.
*
* Nothing failed. Both runs reported `status` cleanly, both pages rendered,
* `fields_all_null` was empty, and every check 1–10 passed — because each of
* them inspects ONE dashboard at a time. The only thing that noticed was the
* DDD concept judge reading rendered numbers off the frames and finding they
* contradicted the narrative's own figures. That is the expensive way to catch
* an arithmetic disagreement, and it catches it only after the render.
*
* Measured on `hh-poverty-targeting/20260824-1404` (labs opp 10047), from each
* page's own `#workflow-data` payload:
*
* run 5245 snapshot (period_end 2026-08-30) total=2186 completed=1563
* run 5249 live total=2237 completed=1592
* fixture on Drive total=2237 completed=1592
* run 5250 snapshot (period_end 2026-08-31) total=2237 completed=1592
*
* Cause: a run window is half-open — `_date_window_where` in
* `connect_labs/labs/analysis/backends/sql/query_builder.py` emits
* `visit_date >= date_from AND visit_date < date_to` — so a `period_end` equal
* to the fixture's last `visit_date` silently drops that whole day. The live
* sibling has no snapshot, is therefore never period-scoped, and keeps it.
*
* The check is deliberately about AGREEMENT, not about any absolute number: it
* needs no fixture, no manifest and no expected total, only two payloads that
* claim to describe the same opportunity.
*/
/** A dashboard as check 11 sees it: the plan entry plus its fetched payload. */
export interface CrossDashboardInput {
dashboard: DashboardRef & {
/** Overrides the id parsed out of `par_url`. */
labs_opp_id?: number | string;
/**
* Set to `'partial'` to declare that this dashboard deliberately renders a
* SUB-WINDOW of the dataset (a single week of a multi-week timeline, say).
* Such a dashboard is excluded from the comparison and named in the detail,
* so an intentional design choice is stated rather than silently tolerated.
*/
period_scope?: string;
};
payload: WorkflowPayload;
/** Live rows for an `in_progress` run, exactly as check 7 takes them. */
livePipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline>;
}
export interface VisitTotal {
total: number;
/** How it was derived, for the failure message. */
basis: string;
}
/**
* Is this row's `visit_date` FILLED, or merely present as an unfilled built-in?
*
* The distinction is the whole fix for ace#2202. Every labs row carries every
* built-in column whether or not its terminal stage fills it (see
* `LABS_BUILTIN_ROW_COLUMNS` / ace#1701), so `'visit_date' in row` is true on
* an AGGREGATED pipeline as well and cannot separate the two grains. Measured
* live 2026-09-07 via `pipeline_preview` on opp 2232:
*
* 19166 `terminal_stage: aggregated` → {"username":"aisha_g","visit_date":null,"total_visits":250,...} x12
* 19169 `terminal_stage: visit_level` → {"username":"grace_a","visit_date":"2026-11-11T00:00:00","total_visits":0,...} x2742
*
* A filled `visit_date` is the visit_level stage's own output; a null one is a
* column the aggregated stage left alone.
*/
function hasFilledVisitDate(row: Record<string, unknown>): boolean {
const v = row.visit_date;
return v !== undefined && v !== null && v !== '';
}
/**
* The visit total for ONE pipeline, decided by its GRAIN (ace#2202).
*
* - VISIT-LEVEL — rows carry a filled `visit_date` and the `total_visits`
* counter is unfilled (absent, or zero on every row): one row IS one visit,
* so count the rows.
* - AGGREGATED — rows carry a meaningful `total_visits`: sum it.
*
* `null` when the rows are neither shape.
*
* Both signals are read, not just the first. Reading only `total_visits`
* presence is what #2202 was; reading only `visit_date` presence is the
* mirror-image bug, because an aggregated row carries `visit_date: null` (the
* measurement above) and would be counted as 12 visits instead of summed to
* 2,742. Requiring the counter to be UNFILLED as well also leaves a genuine
* per-day aggregate — filled `visit_date` AND a real `total_visits` — summed
* rather than miscounted as one visit per day.
*/
function pipelineVisitTotal(p: SnapshotPipeline): VisitTotal | null {
const rows = p.rows ?? [];
if (rows.length === 0) return null;
const alias = p.alias ?? 'pipeline';
const counters = rows
.map((r) => (typeof r.total_visits === 'number' ? r.total_visits : null))
.filter((n): n is number => n !== null);
if (rows.some(hasFilledVisitDate) && !counters.some((n) => n !== 0)) {
return { total: rows.length, basis: `row count of visit-level pipeline '${alias}'` };
}
if (counters.length > 0) {
return {
total: counters.reduce((acc, n) => acc + n, 0),
basis: `sum(total_visits) over ${rows.length} row(s) of '${alias}'`,
};
}
return null;
}
/**
* The one aggregate every visit-shaped dashboard can be asked for.
*
* Each pipeline is classified by its own GRAIN — a visit-level pipeline is one
* row per visit and is counted; an aggregated one carries labs' `total_visits`
* counter and is summed — and the first pipeline that yields a total wins.
*
* Returns `null` when no pipeline is visit-shaped. A dashboard that cannot state
* a visit total is reported as not-judged, never failed: a gate that fails on
* what it cannot see is the always-fires class (ace#1026).
*/
export function deriveVisitTotal(
payload: WorkflowPayload,
livePipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline>,
): VisitTotal | null {
const completed = payload?.instance?.status === 'completed';
const pipelines = completed
? normalizeSnapshotPipelines(payload?.instance?.snapshot?.pipelines)
: normalizeSnapshotPipelines(
livePipelines ?? (payload as { pipeline_data?: Record<string, SnapshotPipeline> })?.pipeline_data,
);
for (const p of pipelines) {
const total = pipelineVisitTotal(p);
if (total) return total;
}
return null;
}
/** `&opportunity_id=<id>` out of a par_url; null for a program-scoped rollup. */
function oppScopeOf(input: CrossDashboardInput): string | null {
if (input.dashboard.labs_opp_id !== undefined && input.dashboard.labs_opp_id !== null) {
return String(input.dashboard.labs_opp_id);
}
const m = /[?&]opportunity_id=(\d+)/.exec(input.dashboard.par_url ?? '');
return m ? m[1] : null;
}
function windowOf(payload: WorkflowPayload): string {
const s = payload?.instance?.period_start;
const e = payload?.instance?.period_end;
if (!s && !e) return 'no period declared';
return `[${s ?? '-inf'}, ${e ?? '+inf'})`;
}
/**
* Every dashboard reading one `labs_opp_id` must report the same visit total.
*
* Program-scoped rollups are excluded by construction: a cross-opp rollup
* aggregates a different population, so its total is not comparable to a
* single-opp dashboard's, and comparing them would be the always-fires class.
*
* @param opts.timelineEndDate the manifest's `timeline.end_date`. When supplied,
* a disagreeing dashboard whose `period_end` is at or before it gets the
* off-by-one named explicitly — that is the cause every time so far, and the
* fix is `period_end = timeline.end_date + 1 day` (ace#1683).
*/
export function checkCrossDashboardConsistency(
inputs: CrossDashboardInput[],
opts?: { timelineEndDate?: string },
): QACheckResult {
const groups = new Map<string, CrossDashboardInput[]>();
const excluded: string[] = [];
const notJudged: string[] = [];
for (const input of inputs) {
if ((input.dashboard.period_scope ?? '').trim().toLowerCase() === 'partial') {
excluded.push(`${input.dashboard.key} (declared period_scope: partial)`);
continue;
}
const opp = oppScopeOf(input);
if (opp === null) {
excluded.push(`${input.dashboard.key} (program-scoped rollup — different population)`);
continue;
}
const bucket = groups.get(opp);
if (bucket) bucket.push(input);
else groups.set(opp, [input]);
}
const problems: string[] = [];
let comparedGroups = 0;
for (const [opp, members] of groups) {
if (members.length < 2) continue;
const totals: Array<{ input: CrossDashboardInput; total: VisitTotal }> = [];
for (const m of members) {
const t = deriveVisitTotal(m.payload, m.livePipelines);
if (t === null) notJudged.push(`${m.dashboard.key} (no visit-shaped pipeline rows)`);
else totals.push({ input: m, total: t });
}
if (totals.length < 2) continue;
comparedGroups += 1;
const distinct = new Set(totals.map((t) => t.total.total));
if (distinct.size === 1) continue;
const lines = totals.map(
(t) =>
`${t.input.dashboard.key}=${t.total.total} (${t.total.basis}; window ${windowOf(t.input.payload)}, ` +
`run ${t.input.payload?.instance?.status ?? 'status unknown'})`,
);
const end = opts?.timelineEndDate;
const suspects = end
? totals
.filter((t) => {
const pe = t.input.payload?.instance?.period_end;
return !!pe && pe.slice(0, 10) <= end.slice(0, 10);
})
.map((t) => t.input.dashboard.key)
: [];
problems.push(
`opportunity ${opp}: dashboards over ONE dataset report different visit totals — ${lines.join(', ')}` +
(suspects.length
? `. ${suspects.join(', ')} carry a period_end at or before timeline.end_date ` +
`(${end}), and period_end is EXCLUSIVE — that drops the final day`
: ''),
);
}
const notes =
(excluded.length ? ` (excluded: ${excluded.join(', ')})` : '') +
(notJudged.length ? ` (not judged: ${notJudged.join(', ')})` : '');
if (problems.length === 0) {
return {
pass: true,
detail: comparedGroups
? `visit totals agree across every dashboard in ${comparedGroups} shared-opportunity group(s)${notes}`
: `no two comparable dashboards share an opportunity — nothing to cross-check${notes}`,
};
}
return {
pass: false,
detail: problems.join('; ') + notes,
auto_fix_hint:
'A run window is half-open: query_builder._date_window_where emits ' +
'`visit_date >= date_from AND visit_date < date_to`, so a period_end equal to the fixture\'s last ' +
'visit_date silently drops that whole day, while a live (never-snapshotted) sibling keeps it. ' +
'Re-mint the snapshotted run with period_end = timeline.end_date + 1 day and repoint the par_url — ' +
'on hh-poverty-targeting/20260824-1404 that turned 2186 into 2237, matching the fixture exactly. ' +
'If two dashboards are MEANT to show different windows, mark the narrower one ' +
'`period_scope: \'partial\'` in source.dashboards[] so the intent is declared rather than assumed ' +
'(dimagi-internal/ace#1683).',
};
}
// ---------------------------------------------------------------------------
// ace-run (cascade) provider — what the gate checks when there is no `source`
// ---------------------------------------------------------------------------
//
// Why this exists: the ace-run provider (ace#2510) writes `products.synthetic.
// cascade` + `workflows`, NOT the denovo `source` block that checks 2b–6 and
// 8–13 read. On spark-facilitator/20260926-1800 the gate was never run at all
// (no result file, no step in run_state), and nothing said which checks even
// apply to a cascade. `checksForProvider` names them, and the cascade checks
// below read what the provider actually writes, so a cascade run is checked
// rather than skipped.
export interface CascadeReportRef {
workflow_id?: number;
run_id?: number;
url?: string;
partner?: string;
opportunity_id?: number;
}
export interface CascadeBlock {
registry?: { registry_id?: number; indicators?: string[] };
program_id?: number;
/** `manifest`: where that partner's sent generator manifest is archived (ace#3251). */
partners?: Array<{ label?: string; opportunity_id?: number; manifest?: string }>;
programme_report?: CascadeReportRef;
worker_review?: CascadeReportRef;
opp_reports?: CascadeReportRef[];
history?: { run_ids?: number[]; first_period_end?: string; last_period_end?: string };
}
export interface SyntheticProducts {
provider?: string;
cascade?: CascadeBlock;
workflows?: Record<string, { workflow_id?: number; run_url?: string } | undefined>;
source?: unknown;
}
/** The cascade's dashboards as `checkParUrlScope` inputs (programme report is program-owned). */
export function cascadeDashboards(synthetic: SyntheticProducts | null | undefined): DashboardRef[] {
const c = synthetic?.cascade;
if (!c) return [];
const out: DashboardRef[] = [];
if (c.programme_report?.url) out.push({ key: 'programme_report', template: 'indicator_programme_report', par_url: c.programme_report.url });
for (const [i, r] of (c.opp_reports ?? []).entries()) {
out.push({ key: `opp_reports.${i}${r.partner ? ` (${r.partner})` : ''}`, template: 'indicator_opp_report', par_url: r.url ?? '' });
}
return out;
}
/**
* The cascade handoff is whole: a registry with indicators, ≥ 3 partners on
* labs-only opportunities (≥ 10000), a programme report whose run is the LAST
* history run, one opp report per partner, and a `workflows` map that mirrors
* every report (ace-web's run page renders `workflows`, so a report missing
* there is invisible to a reviewer).
*/
export function checkCascadeHandoff(synthetic: SyntheticProducts | null | undefined): QACheckResult {
const c = synthetic?.cascade;
if (!c) {
return {
pass: false,
detail: 'products.synthetic.cascade is missing on an ace-run provider',
auto_fix_hint: 'write products.synthetic.cascade as each id is minted (demo-data-setup § C1–C7)',
};
}
const problems: string[] = [];
const indicators = c.registry?.indicators ?? [];
if (!c.registry?.registry_id || indicators.length === 0) problems.push('registry has no id or no indicators');
const partners = c.partners ?? [];
if (partners.length < 3) problems.push(`${partners.length} partner(s) — the cascade needs ≥ 3`);
for (const p of partners) {
if (!(typeof p.opportunity_id === 'number' && p.opportunity_id >= 10000)) {
problems.push(`partner ${p.label ?? '?'}: opportunity ${p.opportunity_id} is not a labs-only opp (≥ 10000)`);
}
}
const runIds = c.history?.run_ids ?? [];
if (runIds.length === 0) problems.push('history.run_ids is empty — no saved weekly runs');
const pr = c.programme_report;
if (!pr?.workflow_id || !pr?.run_id || !pr?.url) problems.push('programme_report lacks workflow_id / run_id / url');
else if (runIds.length && pr.run_id !== runIds[runIds.length - 1]) {
problems.push(`programme_report.run_id ${pr.run_id} is not the latest history run ${runIds[runIds.length - 1]}`);
}
const opps = c.opp_reports ?? [];
const reportOpps = new Set(opps.map((r) => r.opportunity_id));
for (const p of partners) {
if (!reportOpps.has(p.opportunity_id)) problems.push(`partner ${p.label ?? p.opportunity_id} has no opp report`);
}
const wfIds = new Set(Object.values(synthetic?.workflows ?? {}).map((w) => w?.workflow_id));
for (const r of [pr, ...opps]) {
if (r?.workflow_id && !wfIds.has(r.workflow_id)) problems.push(`workflow ${r.workflow_id} is not mirrored in products.synthetic.workflows`);
}
if (problems.length === 0) {
return {
pass: true,
detail: `registry ${c.registry?.registry_id} (${indicators.length} indicators); ${partners.length} partners; programme report on run ${pr?.run_id} = latest of ${runIds.length} history runs; ${opps.length} opp reports, all mirrored in workflows`,
};
}
return {
pass: false,
detail: problems.join('; '),
auto_fix_hint: 'complete the cascade handoff (demo-data-setup § C4–C7): every report minted, its run the latest history run, mirrored into workflows{}',
};
}
/**
* `workflow_history_runs({include_snapshot: true})` → the periods check 21
* grades. Each COMPLETED run's `data.snapshot.state.snapshot` carries
* `{byLLO, byFLW, programInd}`; an in-progress run (the live one) has none and
* is skipped. Sorted by `period_end`, de-duplicated (the last run of a period wins).
*/
export function periodsFromHistoryRuns(response: unknown): GradedPeriod[] {
const runs = (response as { runs?: unknown[] } | null)?.runs ?? [];
const byEnd = new Map<string, GradedPeriod>();
for (const raw of runs) {
const run = raw as { status?: string; period_end?: string; data?: { snapshot?: { state?: { snapshot?: Record<string, unknown> } } } };
const snap = run.data?.snapshot?.state?.snapshot;
if (run.status !== 'completed' || !snap || !run.period_end) continue;
byEnd.set(run.period_end, {
period_end: run.period_end,
byLLO: (snap.byLLO as GradedPeriod['byLLO']) ?? [],
byFLW: (snap.byFLW as GradedPeriod['byFLW']) ?? [],
programInd: snap.programInd as GradedPeriod['programInd'],
});
}
return [...byEnd.values()].sort((a, b) => a.period_end.localeCompare(b.period_end));
}
/**
* Checks 24–27 (ace#2735): the cascade's BUILD output, judged before the DDD
* hand-off instead of by the demo judges after a render. One entry point so the
* script and the skill compute them identically. `visitRows` are every partner's
* visit rows (`pipeline_preview` of the visits pipeline); without them check 24
* fails as not-judged.
*/
export function cascadeBuildOutcomes(
plan: CascadeStoryPlan | null | undefined,
snapshot: CascadeSnapshot | null | undefined,
visitRows: Array<Record<string, unknown>>,
/** `visitSampleScope(<the preview responses>)` — appended to checks 24 and 26 so a sample never reads as every partner (ace#2973). */
visitScope: string | null = null,
): Record<
'cascade_review_routing_follows_pdd' | 'cascade_worker_rates_not_binary' | 'cascade_displayed_columns_vary' | 'cascade_drill_levels_branch',
QACheckResult
> {
const visitFlags = (snapshot?.display?.visit_flags ?? []).map((f) => f.column);
const columns = checkConstantDisplayedColumns(snapshot, visitRows);
const scoped = (r: QACheckResult): QACheckResult => (visitScope ? { ...r, detail: `${r.detail}; ${visitScope}` } : r);
return {
cascade_review_routing_follows_pdd: scoped(checkReviewRouting(visitRows, visitFlags, plan?.review_routing ?? [])),
cascade_worker_rates_not_binary: checkWorkerRateDenominators(snapshot),
cascade_displayed_columns_vary: scoped({ pass: columns.pass, detail: columns.detail, ...(columns.auto_fix_hint ? { auto_fix_hint: columns.auto_fix_hint } : {}) }),
cascade_drill_levels_branch: checkDrillLevels(snapshot, plan?.single_child_levels ?? []),
};
}
/**
* Check 28 — the registry text the cascade RENDERS reads plainly (ace#2748).
*
* `display.targets_note`, the nouns, case/visit field labels, visit-flag labels
* and descriptions, and each indicator's `label` / `plain` are what a programme
* manager reads on screen. spark-facilitator/20261004-1706 (registry 7748)
* shipped "Targets are the PDD's own (§8.1 P1 ≥ 80%, P3 ≥ 75%)" and the DDD
* user-artifact judge capped the demo at clarity 2 on that one line. This gate
* reads the same text before any render. `registry` is the authored registry
* (`7-synthetic/semantic-registry-author_registry.json`) or a
* `semantic_registry_get` response — after a registry edit, the live one.
*/
export function checkRegistryTextIsPlain(registry: unknown): QACheckResult {
const doc = indicatorsDocOf(registry);
if (!doc) {
return {
pass: false,
detail: 'no registry indicators_doc supplied, so the rendered registry text was not read',
auto_fix_hint: 'pass --registry <7-synthetic/semantic-registry-author_registry.json, or the semantic_registry_get response>',
};
}
const r = checkRegistryDisplayText(doc);
if (r.pass) return { pass: true, detail: `${r.checked} reader-facing registry strings read plainly` };
return {
pass: false,
detail: `${r.findings.length} problem(s) in reader-facing registry text: ${describeDisplayTextFindings(r.findings)}`,
auto_fix_hint:
'rewrite the text for a programme manager (skills/semantic-registry-author § 3), e.g. "Targets come from the pilot design: ' +
'a meeting in at least 80% of each community\'s weeks …"; apply it with semantic_registry_update, then rebuild history ' +
'(demo-data-setup § C5). PDD citations stay in scope_note',
};
}
/**
* Which check ids apply to which provider. The gate runs EVERY applicable
* check; a check that applies but cannot be evaluated is a FAILURE of that
* check, never a skip — and the result writer fails a run that evaluated none.
*/
export const CHECKS_BY_PROVIDER: Record<'ace-run' | 'denovo' | 'clone', readonly string[]> = {
'ace-run': [
'realized_json_parses',
'every_par_url_is_run_deeplink',
'worker_review_url_scoped',
'cascade_handoff_complete',
'cascade_story_landed',
// ace#2735 — build defects the DDD judges were finding after render.
'cascade_review_routing_follows_pdd',
'cascade_worker_rates_not_binary',
'cascade_displayed_columns_vary',
'cascade_drill_levels_branch',
// ace#2748 — reader-facing registry text, before the DDD judge reads it.
'cascade_registry_text_is_plain',
// ace#3251 — the two required build-half artifacts, before finalize finds them missing.
'cascade_build_artifacts_written',
],
denovo: [
'realized_json_parses',
'every_par_url_is_run_deeplink',
'dashboards_match_realized',
'opp_is_labs_only',
'timeline_pinned',
'flagged_worker_not_pre_seeded',
'deliver_units_present',
'par_url_payload_is_populated',
'interactive_run_is_live',
'dataset_obeys_pdd_constraints',
'cross_dashboard_totals_agree',
],
clone: [
'realized_json_parses',
'every_par_url_is_run_deeplink',
'dashboards_match_realized',
'opp_is_labs_only',
'deliver_units_present',
'par_url_payload_is_populated',
],
};
export function checksForProvider(provider: string | null | undefined): readonly string[] {
const p = (provider ?? '').trim();
if (p === 'ace-run' || p === 'denovo' || p === 'clone') return CHECKS_BY_PROVIDER[p];
return [];
}
/** Check 1 as a function: realized.json is a FLAT object (DDD substitutes `${var}` verbatim). */
export function checkRealizedFlat(realized: unknown): QACheckResult {
if (!realized || typeof realized !== 'object' || Array.isArray(realized)) {
return { pass: false, detail: 'realized.json is missing or not a JSON object', auto_fix_hint: 're-run demo-data-setup step 5 / C7 — write the flat handoff' };
}
const nested = Object.entries(realized as Record<string, unknown>).filter(([, v]) => v !== null && typeof v === 'object');
if (nested.length) {
return { pass: false, detail: `nested values under ${nested.map(([k]) => k).join(', ')}`, auto_fix_hint: 'flatten realized.json — DDD substitutes ${var} verbatim' };
}
return { pass: true, detail: `flat JSON object with ${Object.keys(realized as object).length} keys` };
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |