Skip to content

dimagi-internal/ace / skills/demo-data-setup-qa

demo-data-setup-qa

Verified

Internal /ace:run step — Pass/fail check that a demo dataset handoff is live.

FileKindSize
SKILL.mdskill md80.4 KB
checks.tsscript69.3 KB
/**
 * Static QA checks for `demo-data-setup-qa`, promoted to importable TS.
 *
 * Why these two exist (dimagi-internal/ace#1161, absorbing #1037):
 *
 * The skill's stated job, per `lib/artifact-manifest.ts`, is that "a dead
 * dashboard must not reach a stakeholder". Run against
 * hh-poverty-targeting/20260730-2210 it returned **7/7 pass** on a demo whose
 * review dashboard was analytically dead (#1160) and whose walkthrough scored
 * concept 2.0/5, user 1.0/5, arc 1.0/5. Every check inspected the HANDOFF —
 * realized.json's shape, a URL against a regex, plan↔handoff key agreement, an
 * integer, a date. None fetched a `par_url` and looked at what it renders, and
 * a regex cannot tell a real run from a fabricated id.
 *
 * The other half (#1037): check 2 demanded `&opportunity_id=` on every
 * dashboard while `demo-data-setup` Step 4 requires `&program_id=` for
 * program-owned rollups — so a correctly-built `program_admin_report` failed
 * its own gate, and the only URL that passed was one verified to render
 * "Workflow definition 5040 not found". Scope is now tied to OWNERSHIP, which
 * turns a contradiction into a check that catches the real defect.
 *
 * Both are pure functions over already-fetched data, so they stay inside the
 * QA contract (binary, no LLM, no scores) and are unit-tested directly.
 */

import type { QACheckResult } from '../../lib/qa-types';
import {
  classifyLiveLoadEvidence,
  describeLiveLoadEvidence,
  LIVE_LOAD_PROBE_HINT,
} from '../../lib/interactive-live-load.js';
import type {
  ConstraintReport,
  DatasetSpec,
  ScrubReport,
  UnparsedExpression,
} from '../../lib/dataset-constraints';
import {
  checkCascadeStoryPlan,
  verifyCascadeStoryLanded,
  type CascadeStoryPlan,
  type GradedPeriod,
} from '../../lib/cascade-story';
import {
  checkConstantDisplayedColumns,
  checkDrillLevels,
  checkReviewRouting,
  checkWorkerRateDenominators,
  type CascadeSnapshot,
} from '../../lib/cascade-build-qa';
import {
  checkRegistryDisplayText,
  describeDisplayTextFindings,
  indicatorsDocOf,
} from '../../lib/registry-display-text';

/**
 * Check 21 — the cascade story was planned AND landed (ace#2510).
 *
 * The plan is gated before generation; this re-runs that gate and then asks
 * the only authority on what a viewer sees — the programme report's SAVED runs
 * — whether each authored signal is actually there. A story that did not land
 * films a partner table where nothing stands out, which no narration fixes.
 */
export function checkCascadeStory(
  plan: CascadeStoryPlan | null | undefined,
  registryIndicatorIds: readonly string[],
  periods: GradedPeriod[],
  /** Real people named in the opp's inputs; no invented worker may carry one's name (ace#2727). */
  realPeople: readonly string[] = [],
): QACheckResult {
  if (!plan) {
    return {
      pass: false,
      detail: '7-synthetic/cascade-story.yaml is missing — the ace-run provider must author the story before generating',
      auto_fix_hint: 'write the CascadeStoryPlan per skills/demo-data-setup § Process (ace-run) C3, then regenerate',
    };
  }
  const planned = checkCascadeStoryPlan(plan, registryIndicatorIds, undefined, { realPeople });
  if (planned.verdict === 'fail') {
    return {
      pass: false,
      detail: planned.findings
        .filter((f) => f.severity === 'fail')
        .map((f) => `${f.signal ?? 'plan'}: ${f.detail}`)
        .join('; '),
      auto_fix_hint:
        'fix the story plan before generating: all four signals on registry indicators with PDD citations; ' +
        'realistic invented partner names marked "(example)"; a display roster naming every worker and entity; ' +
        'headline indicators that differ across partners (demo-data-setup § C0, C3)',
    };
  }
  const landed = verifyCascadeStoryLanded(plan, periods);
  const lines = [
    ...landed.results.map((r) => `${r.kind} ${r.landed ? 'landed' : 'DID NOT LAND'} — ${r.observed}`),
    ...landed.headlines.map((h) => `headline ${h.spread_ok ? 'spread' : 'FLAT'} — ${h.observed}`),
  ];
  if (landed.verdict === 'pass') return { pass: true, detail: lines.join('; ') };
  return {
    pass: false,
    detail: lines.join('; '),
    auto_fix_hint:
      'change the transplant pool for the signal that did not land (widen the separation, fix the carrier), ' +
      'regenerate that partner, rebuild history (demo-data-setup § C5) and re-verify. Never re-describe the ' +
      'story to match what came out.',
  };
}

/**
 * Templates whose workflows are PROGRAM-owned (cross-opp rollups). The
 * indicator programme report is created with `program_id` by
 * demo-data-setup § C4 (ace#2510), so its run URL carries `&program_id=`.
 */
const PROGRAM_OWNED_TEMPLATES = ['program_admin_report', 'audit_par', 'indicator_programme_report'];

export interface DashboardRef {
  key: string;
  template: string;
  par_url: string;
  role?: string;
  shape?: string;
}

const RUN_DEEPLINK =
  /^https:\/\/labs\.connect\.dimagi\.com\/labs\/workflow\/\d+\/run\/\?run_id=\d+&(opportunity_id|program_id)=\d+$/;

/**
 * Every `par_url` is a run deep-link AND carries the scope param its
 * dashboard's ownership requires.
 *
 * Live proof both directions matter (workflow 5040 / run 5048 / program 10037):
 * `&program_id=` renders the SOP grid; `&opportunity_id=` returns 200 with body
 * "Workflow definition 5040 not found." A 200 is not evidence — which is why
 * the scope, not just the shape, has to be checked.
 */
export function checkParUrlScope(dashboards: DashboardRef[]): QACheckResult {
  const problems: string[] = [];
  for (const d of dashboards) {
    const m = RUN_DEEPLINK.exec(d.par_url ?? '');
    if (!m) {
      problems.push(
        `${d.key}: not a run deep-link (need /workflow/<id>/run/?run_id=<id>&(opportunity_id|program_id)=<id>) — got ${d.par_url}`,
      );
      continue;
    }
    const scope = m[1];
    const wantsProgram = PROGRAM_OWNED_TEMPLATES.includes(d.template);
    if (wantsProgram && scope !== 'program_id') {
      problems.push(
        `${d.key}: template '${d.template}' is program-owned, so its par_url MUST carry &program_id= — ` +
          `&opportunity_id= 404s it ("Workflow definition not found")`,
      );
    }
    if (!wantsProgram && scope !== 'opportunity_id') {
      problems.push(
        `${d.key}: template '${d.template}' is opp-owned, so its par_url MUST carry &opportunity_id=`,
      );
    }
  }
  if (problems.length === 0) return { pass: true, detail: `${dashboards.length} par_url(s) correctly scoped` };
  return {
    pass: false,
    detail: problems.join('; '),
    auto_fix_hint:
      'rebuild each par_url with the scope its ownership requires: program_admin_report / audit_par → ' +
      '&program_id=<program_id>; every opp-owned dashboard → &opportunity_id=<labs_opp_id>. Both forms ' +
      'return HTTP 200, so verify by fetching and confirming the page renders its grid rather than ' +
      '"Workflow definition <id> not found" (dimagi-internal/ace#1037).',
  };
}

/**
 * Every `*worker_review_url` in realized.json is a run deep-link that carries
 * `program_id=` — the page-scope param a program-owned worker review needs
 * when it is opened COLD (a DDD scene `url:`), not after a click-through.
 *
 * `owning_program_id` is NOT a substitute. In labs it is only a data-access
 * hint (`_run_program_hint`, connect_labs/workflow/views.py) and is
 * deliberately not one of `labs.context.CONTEXT_PARAMS`
 * (`organization_id`, `program_id`, `opportunity_id`). With no context param
 * on the URL, labs appends the session's last-selected `opportunity_id` and
 * the page reads "Workflow definition <id> not found under opportunity <n>."
 * Observed spark-facilitator/20260926-1413 (ace#2521): the `owning_program_id`
 * form 302s to `&opportunity_id=10087` and renders no `#workflow-data`; the
 * `&program_id=10085` form renders the review.
 */
export function checkWorkerReviewUrlScope(realized: Record<string, unknown>): QACheckResult {
  const keys = Object.keys(realized).filter((k) => /(^|_)worker_review_url$/.test(k));
  const problems: string[] = [];
  for (const key of keys) {
    const raw = realized[key];
    let url: URL;
    try {
      url = new URL(String(raw));
    } catch {
      problems.push(`${key}: not a URL — got ${String(raw)}`);
      continue;
    }
    if (!/^\/labs\/workflow\/\d+\/run\/$/.test(url.pathname) || !/^\d+$/.test(url.searchParams.get('run_id') ?? '')) {
      problems.push(`${key}: not a run deep-link (need /labs/workflow/<id>/run/?run_id=<id>&program_id=<id>) — got ${raw}`);
      continue;
    }
    if (!/^\d+$/.test(url.searchParams.get('program_id') ?? '')) {
      const hint = url.searchParams.has('owning_program_id')
        ? ' — owning_program_id is a data hint, not a page-scope param; opened cold, labs scopes the page to the ' +
          "session's last opportunity and renders \"Workflow definition <id> not found\""
        : '';
      problems.push(`${key}: worker review is program-owned, so its URL MUST carry &program_id=<id>${hint}`);
    }
  }
  if (problems.length === 0) return { pass: true, detail: `${keys.length} worker_review_url(s) correctly scoped` };
  return {
    pass: false,
    detail: problems.join('; '),
    auto_fix_hint:
      'rebuild each worker_review_url with scripts/worker-review-url.ts (--fix <realized.json> repairs in place): ' +
      '/labs/workflow/<review>/run/?run_id=<review run>&program_id=<program>' +
      '&flw=<opp>%3A%3A<username>&source_run=<latest> (demo-data-setup § C7; dimagi-internal/ace#2521).',
  };
}

export type PayloadFindingKind =
  | 'snapshot-missing-pipelines'
  | 'field-all-null'
  | 'live-pipelines-unavailable';

export interface PayloadFinding {
  kind: PayloadFindingKind;
  alias?: string;
  field?: string;
  detail: string;
}

export interface PayloadReport {
  pass: boolean;
  findings: PayloadFinding[];
}

export interface SnapshotPipeline {
  alias?: string;
  rows?: Record<string, unknown>[];
}

export interface WorkflowPayload {
  definition?: { pipeline_sources?: Record<string, unknown> | Array<{ alias?: string; [k: string]: unknown }> };
  instance?: {
    status?: string;
    /**
     * The run's half-open `[period_start, period_end)` visit-date window, both
     * exposed on the `instance` prop (verified live 2026-08-14). Read only to
     * EXPLAIN a cross-dashboard disagreement (ace#1683) — never to excuse one.
     */
    period_start?: string;
    period_end?: string;
    /**
     * `pipelines` arrives in EITHER shape (ace#1701):
     *   - a dict keyed by alias — what labs actually writes today
     *     (`connect_labs/workflow/templates/__init__.py`:
     *      `out["pipelines"] = {alias: pipelines[alias] ...}`)
     *   - an array of `{alias, rows}` — the shape this check was written for.
     */
    snapshot?: { pipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline> };
  };
}

/**
 * Both snapshot shapes, as one array (ace#1701).
 *
 * Reading only the array shape made this check throw `pipelines is not
 * iterable` on every real completed run, and made the
 * `snapshot-missing-pipelines` branch unreachable besides — a dict has no
 * `.length`, and `undefined === 0` is false. Check 7 exists so that a dead
 * dashboard cannot reach a stakeholder (#1161); it could not see one.
 */
export function normalizeSnapshotPipelines(
  pipelines: SnapshotPipeline[] | Record<string, SnapshotPipeline> | undefined,
): SnapshotPipeline[] {
  if (!pipelines) return [];
  if (Array.isArray(pipelines)) return pipelines;
  return Object.entries(pipelines).map(([alias, value]) => ({ alias, ...(value ?? {}) }));
}

/** `pipeline_sources` is a dict of alias->id in the stored definition and an array of {alias,...} on the run page. */
function declaredAliases(sources: WorkflowPayload['definition'] extends infer D ? any : never): string[] {
  if (!sources) return [];
  if (Array.isArray(sources)) {
    return sources.map((s: { alias?: string }) => String(s?.alias ?? '')).filter(Boolean);
  }
  return Object.keys(sources);
}

/** Null, undefined, empty string, or numeric zero — the shapes a dead binding produces. */
function isDeadValue(v: unknown): boolean {
  return v === null || v === undefined || v === '' || v === 0;
}

/**
 * labs' own row columns, which every pipeline row carries whether or not the
 * terminal stage fills them (ace#1701).
 *
 * Whether one of these is populated is decided by `terminal_stage`, not by the
 * schema: WORKFLOW_REFERENCE § "Built-in Row Fields" gives `visit_level` rows
 * `username / visit_date / entity_id / entity_name` and `aggregated` rows the
 * `*_visits` counters and `first/last_visit_date`, and the `entity` stage
 * "drops the status/flagged counters because they're visit-level facts". So a
 * built-in that is null for every row is NOT evidence of a wrong schema path —
 * and it fires on every entity-stage and visit-level pipeline. Measured on
 * run 5258 (healthy, `pipeline_preview` `fields_all_null: []`): 15 findings,
 * all built-ins, none a schema path.
 *
 * #1160's real defect is not one of these: that was render code binding the
 * denormalized `worker.visit_count`, not a pipeline column.
 */
const LABS_BUILTIN_ROW_COLUMNS = new Set([
  'id', 'username', 'visit_date', 'entity_id', 'entity_name', 'status', 'flagged',
  'opportunity_id', 'days_active', 'total_visits', 'approved_visits', 'pending_visits',
  'rejected_visits', 'flagged_visits', 'first_visit_date', 'last_visit_date',
]);

/**
 * Does the page actually render data? Operates on the parsed `#workflow-data`
 * payload the run page embeds — so the fetch belongs to the caller and the
 * judgement is pure and testable.
 *
 * Two unambiguous shapes, both taken from #1160:
 *  - `pipeline_sources` declared while the snapshot carries no pipelines at all
 *    (workflow 5069 declared `performance_data -> 5068` with
 *    `snapshot_inputs.pipelines: []`, so `pipeline_data` was `[]` live);
 *  - a bound field that is null/zero for EVERY row (pipeline 5068 kept the
 *    stock `performance_review` schema pointed at `form.meta.instanceID`, which
 *    the synthetic generator never writes → `visit_count` 0 for all 8 workers,
 *    beside a chip reading `visits: 835`).
 *
 * A field that is zero for SOME rows is data, not a dead binding — flagging it
 * would make this the always-fires class.
 */
/**
 * @param livePipelines Rows for a run that is deliberately still `in_progress`
 *   (the `review-action` dashboard check 8 requires to stay live, #1162). Such
 *   a run has no snapshot BY DESIGN, and the server-rendered `#workflow-data`
 *   carries `pipeline_data: {}` because the page fills it over SSE after
 *   mount — so the caller fetches
 *   `GET /labs/workflow/api/<def>/pipeline-data/?opportunity_id=<opp>` and
 *   passes the result here. Absence is a reported finding, never a silent pass.
 */
export function checkParUrlPayloadPopulated(
  payload: WorkflowPayload,
  livePipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline>,
): PayloadReport {
  const findings: PayloadFinding[] = [];
  const declared = declaredAliases(payload?.definition?.pipeline_sources);
  const completed = payload?.instance?.status === 'completed';

  const live = normalizeSnapshotPipelines(
    livePipelines ?? (payload as { pipeline_data?: Record<string, SnapshotPipeline> })?.pipeline_data,
  );
  const pipelines = completed
    ? normalizeSnapshotPipelines(payload?.instance?.snapshot?.pipelines)
    : live;

  if (declared.length > 0 && pipelines.length === 0) {
    findings.push({
      kind: completed ? 'snapshot-missing-pipelines' : 'live-pipelines-unavailable',
      detail: completed
        ? `definition declares pipeline_sources (${declared.join(', ')}) but the run snapshot holds no ` +
          `pipelines — the page renders no pipeline data at all. Declare snapshot_inputs.pipelines ` +
          `covering every alias in pipeline_sources before completing the run.`
        : `this run is '${payload?.instance?.status ?? 'unknown'}', so it has no snapshot by design ` +
          `(#1162) — and no live pipeline rows were supplied for ${declared.join(', ')}, so nothing ` +
          `was judged. Fetch GET /labs/workflow/api/<definition_id>/pipeline-data/?opportunity_id=<opp> ` +
          `and pass it as livePipelines.`,
    });
  }

  for (const p of pipelines) {
    const rows = p.rows ?? [];
    if (rows.length === 0) continue;
    const columns = new Set<string>();
    for (const r of rows) for (const k of Object.keys(r)) columns.add(k);
    for (const field of columns) {
      if (LABS_BUILTIN_ROW_COLUMNS.has(field)) continue;
      const allDead = rows.every((r) => isDeadValue(r[field]));
      if (allDead) {
        findings.push({
          kind: 'field-all-null',
          alias: p.alias,
          field,
          detail: `${p.alias ?? 'pipeline'}.${field} is null/zero for all ${rows.length} row(s) — the schema path likely never matches the generated fixtures`,
        });
      }
    }
  }

  return { pass: findings.length === 0, findings };
}

export function formatPayloadReport(report: PayloadReport): string {
  if (report.pass) return 'par_url payload: populated — pipelines present and no uniformly-dead bound field';
  return [
    'par_url payload: the dashboard renders but is analytically DEAD —',
    ...report.findings.map((f) => `  [${f.kind}] ${f.detail}`),
    'A dead dashboard must not reach a stakeholder (lib/artifact-manifest.ts).',
    'Re-point the pipeline schema at the REAL form paths the generator writes, and',
    'declare snapshot_inputs.pipelines for every alias (dimagi-internal/ace#1160/#1161).',
  ].join('\n');
}

// ── #1162: the interactive run must still be interactive at render time ──

/**
 * Roles whose dashboard exists so that someone can ACT on it on camera.
 * Compared after lowercasing and folding `_` to `-`.
 */
const INTERACTIVE_ROLES = new Set(['review-action', 'review', 'decision']);

function isInteractiveRole(role: string | undefined): boolean {
  return INTERACTIVE_ROLES.has((role ?? '').trim().toLowerCase().replace(/_/g, '-'));
}

export interface DashboardPayloadPair {
  dashboard: DashboardRef;
  payload: WorkflowPayload;
}

/**
 * Exactly the review-action run stays `in_progress`; every other run is
 * `completed`.
 *
 * Completing a run is how `par_url` becomes a stable, idempotent deep-link —
 * correct, and what most dashboards want. But it also flips the page
 * read-only: workflow 5069's render code carries a `completed` branch that
 * prints "This run is completed… Decisions are read-only" and disables the
 * status dropdown. On hh-poverty-targeting/20260730-2210 Phase 7 completed
 * BOTH runs ~14 minutes before the render, so the narrative's payoff — a
 * reviewer taking a decision — had nothing to click: all 10 spec actions were
 * wait_for/hold, 7 scenes yielded 2 distinct images, arc 1.0/5 (#1162).
 *
 * Two-sided on purpose. Firing only on the completed-interactive case would
 * accept the opposite failure — everything left in_progress, quietly trading
 * away snapshot stability on links a stakeholder keeps.
 *
 * A payload with no `instance.status` is reported, not failed: an absent field
 * is a fetch-shape question, and a QA gate that fails on what it cannot see is
 * the always-fires class (ace#1026).
 *
 * **ace#2430 — the mandate is now verified, and it has an evidenced escape.**
 * Leaving the run `in_progress` costs a LIVE dependency: that page has no
 * snapshot, so every load re-runs the pipeline SSE stream, while a completed
 * run returns early on its snapshot and never opens one
 * (`workflow-runner.tsx`: `if (snapshotCarriesPipelines) return;`). On
 * `poverty-graduation/20260915-1518` that stream died after its first event and
 * the interactive dashboard could not load at all — and because this check was
 * one-sided, the ONLY QA-passing configuration was the one that could not
 * render. So: the producer must record `source.interactive_live_load` (the
 * probe it ran), an unproven `in_progress` run fails, a run whose own probe
 * says it cannot load fails, and a `completed` interactive run passes when — and
 * only when — the failure is evidenced against a rendering control run. Same
 * cite-your-source discipline as `below_programme_scale` and
 * `declared_omissions[]`; the escape is reported in `detail`, never silent.
 */
export function checkInteractiveRunsLive(
  pairs: DashboardPayloadPair[],
  liveLoad?: unknown,
): QACheckResult {
  const problems: string[] = [];
  const unknown: string[] = [];
  const notes: string[] = [];

  // ace#2430: the mandate now carries its proof. An interactive run has NO
  // snapshot, so its page re-runs the live pipeline stream on every load —
  // including every render take — while a completed run returns early on its
  // snapshot and never opens one. That asymmetry is why `in_progress` needs
  // verifying and `completed` does not, and it is what made the old one-sided
  // rule enforce a page that could not render.
  const anyInteractive = pairs.some(({ dashboard }) => isInteractiveRole(dashboard.role));
  const evidence = classifyLiveLoadEvidence(liveLoad);
  const escapeGranted = anyInteractive && evidence.status === 'failed' && evidence.granted;
  const provenLive = anyInteractive && evidence.status === 'ok' && evidence.granted;

  if (anyInteractive && !evidence.granted) {
    problems.push(
      `the interactive dashboard's live-load is unproven: ${evidence.problems.join('; ')}. An in_progress ` +
        `run has no snapshot, so its page depends on the live pipeline stream every time it is opened`,
    );
  } else if (escapeGranted) {
    notes.push(
      `live-load escape taken (evidenced): ${describeLiveLoadEvidence(evidence, liveLoad)} — the interactive ` +
        `run ships completed because its live load was observed failing against a rendering control`,
    );
  }

  for (const { dashboard, payload } of pairs) {
    const status = payload?.instance?.status;
    const interactive = isInteractiveRole(dashboard.role);
    if (!status) {
      unknown.push(`${dashboard.key} (role ${dashboard.role ?? 'unset'})`);
      continue;
    }
    if (interactive && status === 'completed' && !escapeGranted) {
      problems.push(
        `${dashboard.key}: role '${dashboard.role}' is interactive but its run is completed — the page ` +
          `renders "This run is completed… Decisions are read-only" with the status control disabled, so ` +
          `the decision the narrative demonstrates cannot be performed on camera`,
      );
    }
    if (interactive && status !== 'completed' && escapeGranted) {
      problems.push(
        `${dashboard.key}: its recorded live-load probe says this page cannot load ` +
          `(${describeLiveLoadEvidence(evidence, liveLoad)}), yet the run shipped '${status}' — a dead ` +
          `par_url must not reach a stakeholder; complete the run and say so on the page`,
      );
    }
    if (!interactive && status !== 'completed') {
      problems.push(
        `${dashboard.key}: role '${dashboard.role ?? 'unset'}' is non-interactive but its run is ` +
          `'${status}' — only a completed run carries a snapshot, so this par_url is not a stable ` +
          `idempotent deep-link`,
      );
    }
  }

  const unknownNote = unknown.length ? ` (unknown run status, not judged: ${unknown.join(', ')})` : '';
  const noteText = notes.length ? `; ${notes.join('; ')}` : '';
  if (problems.length === 0) {
    const proof = provenLive ? `, interactive live-load proven (${describeLiveLoadEvidence(evidence, liveLoad)})` : '';
    return {
      pass: true,
      detail: `${pairs.length} dashboard run state(s) match their role${proof}${unknownNote}${noteText}`,
    };
  }
  return {
    pass: false,
    detail: problems.join('; ') + unknownNote + noteText,
    auto_fix_hint:
      'Leave ONLY the review-action dashboard\'s run in_progress — skip workflow_save_snapshot for it — ' +
      'and complete every other dashboard\'s run as usual. The interactive dashboard trades snapshot ' +
      'stability for a page the reviewer can actually act on; the rest keep it (dimagi-internal/ace#1162). ' +
      LIVE_LOAD_PROBE_HINT,
  };
}

// ── #1658: check 9's spec is DERIVED, and its auto-fix is one that exists ──

/**
 * Check 9 (`dataset_obeys_pdd_constraints`) as a pure function.
 *
 * Two defects it closes, both measured on `bednet-check-2-visit` (ace#1658):
 *
 * 1. **The spec was author-supplied**, so under-declaring it produced a false
 *    green. `20260817-1720` recorded `pass` with `conditionalFields: []` and
 *    the justification "no conditional blocks" — while
 *    `get_opportunity_apps(2214, 'deliver')` returned two `relevant`
 *    expressions verbatim. `20260825-1310`, same opp / app / generator,
 *    declared them and measured 18 of 276 off-branch on each. So the spec now
 *    comes from `specFromDeliverApp`, and this check REFUSES a run that has
 *    no derivation behind it: a `derivation` of `null` is only legal with a
 *    stated reason (the `denovo` provider has no deliver app to read).
 *
 * 2. **The old auto-fix was impossible on this path.** "Regenerate with the
 *    constraint applied at the manifest" reached for a knob that, at the time,
 *    did not exist. It exists now: `BeneficiaryCohort.relevance_groups`
 *    (`connect-labs#1331`, merged 2026-08-27). The remedy is unchanged anyway,
 *    because the primitive is INERT for the runs this check guards — relevance
 *    is applied only to questions present in the HQ `FormSchema`
 *    (`fixtures/fields.py:417-422`), and the trailing orphan-write loop is gated
 *    by a set computed ONCE at `fields.py:484`, before it runs. A labs-only opp
 *    has no Connect `app_structure`, so `FormSchema.questions` is empty
 *    (`schema_loader.parse_form_schema_from_app_json`), every declared path is an
 *    orphan, and the controller is not in the record when line 484 evaluates the
 *    gate. Measured on `bednet-check-2-visit/20260828-0629`: the producer
 *    DECLARED `relevance_groups` and the generator still emitted 36 off-branch
 *    values (ace#1833). So the hint points at `scrubOffBranchFields` — a
 *    declared, reproducible, idempotent generator post-step — and says to
 *    declare `relevance_groups` on a schema-backed opp, where it does work.
 *
 * An unparsed `relevant` is a FINDING, not a silent pass: an expression the
 * derivation could not read is a gate this check cannot prove was audited.
 *
 * 3. **It had no evidenced escape** (ace#2225), so a dataset whose only
 *    residual violations were fields labs STRUCTURALLY cannot simulate could
 *    never pass. On `poverty-graduation/20260908-0510` all six residuals were
 *    a REPEAT-group roster (the generator emits one flat object, not an
 *    array — TRUE THEN, REFUTED SINCE: the manifest carries
 *    `repeat_groups` and `auditDataset` reads inside an array, so a repeat is
 *    built, not exempted, ace#2432), a `Trigger` read-aloud label (CommCare
 *    submits no value for one),
 *    or an image with no labs `ImageConfig` corpus — with zero off-branch,
 *    zero integrality and an empty `unparsed[]`. The only two routes to green
 *    were to narrow the spec, which is the behaviour defect 1 exists to
 *    prevent, or to accept a permanent `fail` — and a permanently-red check is
 *    one nobody reads, which is how the real misses get waved through
 *    (ace#1744). So it takes the shape `checkDetectionCohortFloor` already
 *    uses for `below_programme_scale`: an EVIDENCED escape, refused when
 *    unevidenced, reported rather than silent, and scoped to the two halves a
 *    structural absence actually shows up in — a `conditional-missing`
 *    violation and the scrub's `unresolvedFields`. An off-branch value, an
 *    integrality violation and an unparsed gate stay unexemptible: each names
 *    a value that IS present and wrong, which the scrub or an added spec entry
 *    can repair.
 */
export interface DatasetConstraintCheckInput {
  /**
   * `specFromDeliverApp(get_opportunity_apps(<opp>, 'deliver'))`, or `null`
   * when the demo has no deliver app to derive from.
   */
  derivation: { unparsed: UnparsedExpression[]; questionsSeen: number; gatesParsed: number } | null;
  /** Required when `derivation` is null — why no app was read (e.g. provider `denovo`). */
  noDeliverAppReason?: string;
  /** The scrub actually applied to the fixture before the dashboards read it. */
  scrub?: ScrubReport;
  /** `auditDataset` over the records as they now stand. */
  report: ConstraintReport;
  /**
   * `declared_omissions[]` from `7-synthetic/branch-scrub_report.yaml` — the
   * fields the labs generator STRUCTURALLY cannot emit, each with the reason
   * (ace#2225). Exempts that field's `conditional-missing` violation and its
   * `unresolvedFields` entry, and nothing else. An entry with a blank reason
   * exempts nothing and is named in the failure.
   */
  declaredOmissions?: DeclaredOmission[];
  /**
   * `derivation.unparsed_resolutions[]` from `7-synthetic/branch-scrub_report.yaml`
   * — how each gate `specFromDeliverApp` could not parse was actually audited
   * (ace#2497). An unparsed entry is cleared only by a resolution matching it on
   * `field` + `expression` with a known `resolution` kind and a non-empty
   * `detail` stating the measurement. `derivation.unparsed` stays the RAW
   * derivation output — never edit it to `[]`; that is indistinguishable from
   * deleting the list.
   */
  unparsedResolutions?: UnparsedResolution[];
  /**
   * The spec `auditDataset` actually ran over: `mergeDatasetSpecs(derived.spec, additions)`
   * (ace#2245). A `spec-addition` resolution clears its unparsed gate only when this spec
   * carries a rule on that field that audits the expression: a conditional entry for the
   * field that does not still list the expression in `unparsedGates`, an integer entry with
   * a bound, or a cross-field rule naming it. Without it, a `spec-addition` is an author
   * assertion nothing checked, and it clears nothing.
   */
  mergedSpec?: DatasetSpec;
  /**
   * `countFieldPresence(records, <fields the merged spec's rules read>)` over the records
   * as they now stand. A spec-addition rule whose gate field is carried by 0 records is
   * REPORTED as declared but never firing. It does not fail: the gate is declared, but it
   * audits nothing (ace#2245, `enrol_gps_accuracy_band` in 0 of 182 records).
   */
  fieldPresence?: Record<string, number>;
}

/** One entry of the producer's `derivation.unparsed_resolutions[]` (ace#2497). */
export interface UnparsedResolution {
  field: string;
  expression: string;
  /**
   * `spec-addition` (declared via mergeDatasetSpecs and audited),
   * `direct-measured-assertion` (measured over the records; `detail` carries the count),
   * `spec-correction` (a derived entry replaced by the app's own, tighter gate),
   * `out-of-scope-form` (a question on a form the fixture does not generate; `detail` carries the count).
   */
  resolution: string;
  detail: string;
}

const RESOLUTION_KINDS = new Set(['spec-addition', 'direct-measured-assertion', 'spec-correction', 'out-of-scope-form']);

/** One `{field, reason}` entry of the producer's `declared_omissions[]`. */
export interface DeclaredOmission {
  field: string;
  /** Why labs cannot emit this field. Blank means the exemption is refused. */
  reason: string;
}

const SCRUB_HINT =
  'Re-run the branch scrub, do not narrow the spec: scrubOffBranchFields(records, spec.conditionalFields) ' +
  'from lib/dataset-constraints.ts, then write the scrubbed user_visits.json back to the opp\'s fixture ' +
  'folder BEFORE any dashboard run is minted, and record the per-field counts in the run summary. There is ' +
  'no manifest-side remedy ON THIS PATH: BeneficiaryCohort.relevance_groups DOES exist ' +
  '(dimagi-internal/connect-labs#1331, merged 2026-08-27), but relevance is only applied to questions ' +
  'present in the HQ FormSchema. A labs-only opp has no Connect app_structure, so that schema is empty, ' +
  'every declared path is orphan-written, and the orphan gate is computed once BEFORE the orphan loop ' +
  '(fixtures/fields.py:484) from a record that does not yet hold the controller \u2014 so it can never fire. ' +
  'Declaring relevance_groups here is inert: a run that declared it still emitted 36 off-branch values. ' +
  'Do declare it on a schema-backed opp, where it works (dimagi-internal/ace#1658, #1833).';

/**
 * The evidenced half of the declared-omission escape (ace#2225).
 *
 * A field only earns an exemption when its entry carries a non-empty reason —
 * the same cite-your-source discipline `checkDetectionCohortFloor` imposes on
 * `below_programme_scale`. An unevidenced entry is a silencer, and an escape
 * is only worth having if turning the gate off costs the author the citation.
 */
function partitionOmissions(declared: DeclaredOmission[] | undefined): {
  evidenced: Map<string, string>;
  unevidenced: Set<string>;
} {
  const evidenced = new Map<string, string>();
  const unevidenced = new Set<string>();
  for (const o of declared ?? []) {
    const field = typeof o?.field === 'string' ? o.field.trim() : '';
    if (!field) continue;
    const reason = typeof o?.reason === 'string' ? o.reason.trim() : '';
    if (reason) evidenced.set(field, reason);
    else unevidenced.add(field);
  }
  return { evidenced, unevidenced };
}

const OMISSION_HINT =
  'A residual the labs generator STRUCTURALLY cannot emit is declared, not narrowed away: add a ' +
  '{field, reason} entry to declared_omissions[] in 7-synthetic/branch-scrub_report.yaml naming why the ' +
  'value cannot exist (a Trigger read-aloud label CommCare submits no value for, an image with no labs ' +
  'ImageConfig corpus). A REPEAT-group roster is NOT one of them any more: the manifest carries ' +
  'BeneficiaryCohort.repeat_groups and auditDataset reads inside an array, so build the repeat rather than ' +
  'exempting it (dimagi-internal/ace#2432). The reason is REQUIRED — an ' +
  'unevidenced entry exempts nothing. This escape covers conditional-missing and an unresolved scrub field ' +
  'only: an off-branch value, an integrality violation and an unparsed gate are never exempt, because each ' +
  'of those is a dataset defect the scrub or an added spec entry can actually fix (dimagi-internal/ace#2225).';

/**
 * The merged-spec rules that audit one unparsed expression, each with the fields it must
 * read to fire (ace#2245). A `relevant` gate is audited by a conditional entry for the field
 * that no longer lists the expression in `unparsedGates`, or a cross-field rule naming the
 * field; a `constraint` by an integer entry with a bound or a cross-field rule.
 */
function additionRulesFor(spec: DatasetSpec | undefined, u: UnparsedExpression): Array<{ gateFields: string[] }> {
  if (!spec) return [];
  const expr = String(u.expression).trim();
  const rules: Array<{ gateFields: string[] }> = [];
  if (u.kind === 'relevant') {
    for (const c of spec.conditionalFields ?? []) {
      if (c.field === u.field && !(c.unparsedGates ?? []).some((g) => g.trim() === expr)) rules.push({ gateFields: [c.requiredWhen.field] });
    }
  } else {
    for (const f of spec.integerFields ?? []) {
      if (f.field === u.field && (f.min !== undefined || f.max !== undefined)) rules.push({ gateFields: [f.field] });
    }
  }
  for (const r of spec.crossFieldRules ?? []) {
    if (r.lhs === u.field || r.rhs === u.field) rules.push({ gateFields: [r.lhs, r.rhs] });
  }
  return rules;
}

export function checkDatasetObeysPddConstraints(input: DatasetConstraintCheckInput): QACheckResult {
  const problems: string[] = [];
  const hints: string[] = [];
  const { evidenced, unevidenced } = partitionOmissions(input.declaredOmissions);
  const exempted: string[] = [];
  const noteExempt = (field: string) => {
    if (!exempted.includes(field)) exempted.push(field);
  };

  if (!input.derivation) {
    if (!input.noDeliverAppReason?.trim()) {
      problems.push(
        'no spec derivation and no stated reason — the spec would be hand-declared from prose, which is ' +
          'the false-green shape this check exists to close',
      );
      hints.push(
        'derive the spec with specFromDeliverApp(get_opportunity_apps(<connect opp>, \'deliver\')); if this ' +
          'demo genuinely has no deliver app (provider denovo), state that as noDeliverAppReason.',
      );
    }
  } else if (input.derivation.questionsSeen === 0) {
    problems.push(
      'the deliver app returned 0 questions, so conditionalFields / integerFields were derived from nothing',
    );
    hints.push(
      're-fetch get_opportunity_apps(<connect opp>, \'deliver\') and confirm deliver_app is non-null before ' +
        'auditing; an empty app derives an empty spec, and an empty spec measures zero violations.',
    );
  }

  const rawUnparsed = input.derivation?.unparsed ?? [];
  // ace#2497: an unparsed gate is cleared only by an EVIDENCED resolution that
  // names it (field + expression) — same discipline as declared_omissions.
  const resolutions = (input.unparsedResolutions ?? []).filter(
    (r) =>
      typeof r?.field === 'string' &&
      typeof r?.expression === 'string' &&
      RESOLUTION_KINDS.has(String(r?.resolution ?? '').trim()) &&
      String(r?.detail ?? '').trim().length > 0,
  );
  const resolvedKey = new Set(resolutions.map((r) => `${r.field}\u0000${r.expression.trim()}`));
  const resolvedNote: string[] = [];
  // ace#2245: a spec-addition is checked against the merged spec, not taken on the author's word.
  const unbacked: string[] = [];
  const neverFires: string[] = [];
  const unparsed = rawUnparsed.filter((u) => {
    if (resolvedKey.has(`${u.field}\u0000${String(u.expression).trim()}`)) {
      const r = resolutions.find((x) => x.field === u.field && x.expression.trim() === String(u.expression).trim());
      if (r && r.resolution.trim() === 'spec-addition') {
        const rules = additionRulesFor(input.mergedSpec, u);
        if (rules.length === 0) {
          unbacked.push(`${u.field} [${u.kind}] ${u.expression}`);
          return true;
        }
        if (input.fieldPresence) {
          const dead = rules.flatMap((g) => g.gateFields).filter((f) => input.fieldPresence![f] === 0);
          if (dead.length && rules.every((g) => g.gateFields.some((f) => input.fieldPresence![f] === 0))) {
            neverFires.push(`${u.field} (gate field ${[...new Set(dead)].join(', ')} carried by 0 of ${input.report.total} records)`);
          }
        }
      }
      if (r) resolvedNote.push(`${u.field}: ${r.resolution}`);
      return false;
    }
    return true;
  });
  if (unbacked.length) {
    problems.push(
      `${unbacked.length} spec-addition resolution(s) name a gate the merged spec carries no rule for, so the ` +
        `addition was asserted, not audited: ${unbacked.join('; ')}` +
        (input.mergedSpec ? '' : ' (no mergedSpec supplied)'),
    );
    hints.push(
      'pass mergedSpec = mergeDatasetSpecs(derived.spec, additions), the spec auditDataset ran over; a ' +
        'spec-addition must put a rule on the unparsed field there (a conditional entry not still listing the ' +
        'expression in unparsedGates, an integer bound, or a cross-field rule) (ace#2245).',
    );
  }
  if (unparsed.length > 0) {
    problems.push(
      `${unparsed.length} expression(s) could not be derived into the spec, so those gates were NOT audited: ` +
        unparsed.map((u) => `${u.field} [${u.kind}] ${u.expression}`).join('; '),
    );
    hints.push(
      'hand-declare each unparsed gate as an ADDITION (mergeDatasetSpecs(derived, additions)) — never as a ' +
        'replacement for the derived spec — then re-run the scrub and the audit, and record HOW each one was ' +
        'audited as a {field, expression, resolution, detail} entry in derivation.unparsed_resolutions[] ' +
        '(resolution: spec-addition | direct-measured-assertion | spec-correction | out-of-scope-form; detail ' +
        'carries the measured count). Keep derivation.unparsed as the raw derivation output (ace#2497).',
    );
  }

  // A field labs cannot emit is also a field the scrub cannot locate, so the
  // escape has to cover BOTH halves or the check stays permanently red. On
  // `poverty-graduation/20260908-0510` five of the six residuals appeared here
  // as well as in the audit below (ace#2225).
  const unresolved = (input.scrub?.unresolvedFields ?? []).filter((f) => {
    if (evidenced.has(f)) {
      noteExempt(f);
      return false;
    }
    return true;
  });
  if (unresolved.length) {
    problems.push(
      `the branch scrub could not locate ${unresolved.join(', ')} in any record, so those ` +
        'fields were never scrubbed',
    );
    hints.push(
      'check the fixture nesting against the app XPath — the scrub resolves a field by XPath suffix, and a ' +
        'field it cannot find is reported rather than silently counted as clean.',
    );
  }

  // `conditional-missing` is the ONLY exemptible class: it is the one that can
  // record an absence the generator was never able to produce. Every other
  // class names a value that IS present and wrong, which the scrub or an added
  // spec entry can actually repair — so none of them takes an escape.
  const violations = input.report.violations.filter((v) => {
    if (v.kind !== 'conditional-missing' || !v.field) return true;
    if (evidenced.has(v.field)) {
      noteExempt(v.field);
      return false;
    }
    return true;
  });
  if (violations.length) {
    problems.push(
      `${violations.length} constraint class(es) violated across ${input.report.total} records: ` +
        violations.map((v) => `[${v.kind}] ${v.count} of ${input.report.total} — ${v.field}`).join('; '),
    );
    hints.push(SCRUB_HINT);
  }

  // Name the refused declarations explicitly. Silently ignoring one would read
  // to the author as "the escape does not work", and send them back to the
  // spec-narrowing this check exists to prevent.
  const refused = [...unevidenced].filter(
    (f) =>
      input.report.violations.some((v) => v.kind === 'conditional-missing' && v.field === f) ||
      (input.scrub?.unresolvedFields ?? []).includes(f),
  );
  if (refused.length) {
    problems.push(
      `declared_omissions carries no reason for ${refused.join(', ')} — an unevidenced exemption is ` +
        'rejected, so those residuals still count',
    );
  }
  if (refused.length || violations.some((v) => v.kind === 'conditional-missing') || unresolved.length) {
    hints.push(OMISSION_HINT);
  }

  const exemptNote = exempted.length
    ? `; ${exempted.length} declared omission(s) exempted — ` +
      exempted.map((f) => `${f}: ${evidenced.get(f)}`).join('; ')
    : '';
  const resolvedTxt = resolvedNote.length
    ? `; ${resolvedNote.length} unparsed gate(s) resolved — ${resolvedNote.join('; ')}`
    : '';
  const neverFiresTxt = neverFires.length
    ? `; REPORTED, declared but never firing (ace#2245): ${neverFires.join('; ')}`
    : '';

  if (problems.length === 0) {
    const derivedNote = input.derivation
      ? `spec derived from ${input.derivation.questionsSeen} question(s), ${input.derivation.gatesParsed} gate(s)`
      : `no deliver app (${input.noDeliverAppReason})`;
    const scrubNote = input.scrub
      ? `; branch scrub cleared ${input.scrub.totalCleared} off-branch value(s)`
      : '';
    return {
      pass: true,
      detail:
        `0 unexempted violations across ${input.report.total} records (measured) — ` +
        `${derivedNote}${scrubNote}${resolvedTxt}${neverFiresTxt}${exemptNote}`,
    };
  }

  return {
    pass: false,
    detail: problems.join('; ') + resolvedTxt + neverFiresTxt + exemptNote,
    auto_fix_hint: hints.join(' '),
  };
}

// ── #2246: check 4's anchor clause names a value the gate can read ──

/**
 * Check 4 (`timeline_pinned`) as a pure function (ace#2246).
 *
 * Two clauses: the manifest `timeline.start_date` is a fixed ISO Monday (a
 * relative or sliding expression breaks idempotency), and it equals every
 * minted run's `instance.period_start` — the `par_url` payloads check 7
 * already fetched. The second clause used to read "equals the `par_url`
 * opportunity's env anchor", which names no atom: `synthetic_env_get` is keyed
 * by an ENV TEMPLATE, not an opportunity, so a gate could evaluate only the
 * first clause and still report a full pass.
 *
 * A clause that cannot be judged is named (`anchor-not-judged`) and FAILS the
 * check, because an applicable check that was not evaluated is a failure,
 * never a skip. A payload whose run declares no `period_start` is listed as
 * not judged; if no payload declares one, the clause was not judged.
 */
export function checkTimelinePinned(
  startDate: unknown,
  payloads: Array<{ key: string; payload: WorkflowPayload | null | undefined }>,
): QACheckResult {
  const raw = typeof startDate === 'string' ? startDate.trim() : '';
  const iso = /^\d{4}-\d{2}-\d{2}$/.test(raw) ? new Date(`${raw}T00:00:00Z`) : null;
  if (!iso || Number.isNaN(iso.getTime()) || iso.toISOString().slice(0, 10) !== raw) {
    return {
      pass: false,
      detail: `timeline.start_date ${JSON.stringify(startDate ?? null)} is not a fixed ISO date (YYYY-MM-DD)`,
      auto_fix_hint: 'set --pin-monday to a fixed Monday; a relative or sliding window breaks idempotency',
    };
  }
  if (iso.getUTCDay() !== 1) {
    const day = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][iso.getUTCDay()];
    return {
      pass: false,
      detail: `timeline.start_date ${raw} is a ${day}, not a Monday`,
      auto_fix_hint: 'set --pin-monday to a fixed Monday; a relative or sliding window breaks idempotency',
    };
  }
  const judged = payloads.filter((p) => typeof p.payload?.instance?.period_start === 'string' && p.payload.instance.period_start.trim());
  const unjudged = payloads.filter((p) => !judged.includes(p)).map((p) => p.key);
  if (judged.length === 0) {
    return {
      pass: false,
      detail:
        `timeline.start_date ${raw} is a fixed Monday; anchor-not-judged: ` +
        (payloads.length
          ? `no par_url payload declares instance.period_start (${unjudged.join(', ')})`
          : 'no par_url payload was supplied'),
      auto_fix_hint:
        'pass the par_url payloads check 7 fetched (#workflow-data); the anchor is each minted run\'s instance.period_start',
    };
  }
  const off = judged.filter((p) => String(p.payload!.instance!.period_start).trim().slice(0, 10) !== raw);
  const notJudgedTxt = unjudged.length ? `; not judged (no period_start): ${unjudged.join(', ')}` : '';
  if (off.length) {
    return {
      pass: false,
      detail:
        `timeline.start_date ${raw} disagrees with the minted run(s): ` +
        off.map((p) => `${p.key} period_start=${p.payload!.instance!.period_start}`).join('; ') +
        notJudgedTxt,
      auto_fix_hint:
        're-mint the run(s) with period_start = timeline.start_date, or correct the manifest pin; the dashboards ' +
        'and the manifest must describe one window',
    };
  }
  return {
    pass: true,
    detail: `timeline.start_date ${raw} is a fixed Monday and equals period_start on ${judged.length} minted run(s)${notJudgedTxt}`,
  };
}

// ── #1683: two dashboards over ONE dataset must report the same total ──

/**
 * Check 11 (`cross_dashboard_totals_agree`) as a pure function.
 *
 * Nothing failed. Both runs reported `status` cleanly, both pages rendered,
 * `fields_all_null` was empty, and every check 1–10 passed — because each of
 * them inspects ONE dashboard at a time. The only thing that noticed was the
 * DDD concept judge reading rendered numbers off the frames and finding they
 * contradicted the narrative's own figures. That is the expensive way to catch
 * an arithmetic disagreement, and it catches it only after the render.
 *
 * Measured on `hh-poverty-targeting/20260824-1404` (labs opp 10047), from each
 * page's own `#workflow-data` payload:
 *
 *     run 5245  snapshot  (period_end 2026-08-30)  total=2186  completed=1563
 *     run 5249  live                               total=2237  completed=1592
 *     fixture   on Drive                           total=2237  completed=1592
 *     run 5250  snapshot  (period_end 2026-08-31)  total=2237  completed=1592
 *
 * Cause: a run window is half-open — `_date_window_where` in
 * `connect_labs/labs/analysis/backends/sql/query_builder.py` emits
 * `visit_date >= date_from AND visit_date < date_to` — so a `period_end` equal
 * to the fixture's last `visit_date` silently drops that whole day. The live
 * sibling has no snapshot, is therefore never period-scoped, and keeps it.
 *
 * The check is deliberately about AGREEMENT, not about any absolute number: it
 * needs no fixture, no manifest and no expected total, only two payloads that
 * claim to describe the same opportunity.
 */

/** A dashboard as check 11 sees it: the plan entry plus its fetched payload. */
export interface CrossDashboardInput {
  dashboard: DashboardRef & {
    /** Overrides the id parsed out of `par_url`. */
    labs_opp_id?: number | string;
    /**
     * Set to `'partial'` to declare that this dashboard deliberately renders a
     * SUB-WINDOW of the dataset (a single week of a multi-week timeline, say).
     * Such a dashboard is excluded from the comparison and named in the detail,
     * so an intentional design choice is stated rather than silently tolerated.
     */
    period_scope?: string;
  };
  payload: WorkflowPayload;
  /** Live rows for an `in_progress` run, exactly as check 7 takes them. */
  livePipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline>;
}

export interface VisitTotal {
  total: number;
  /** How it was derived, for the failure message. */
  basis: string;
}

/**
 * Is this row's `visit_date` FILLED, or merely present as an unfilled built-in?
 *
 * The distinction is the whole fix for ace#2202. Every labs row carries every
 * built-in column whether or not its terminal stage fills it (see
 * `LABS_BUILTIN_ROW_COLUMNS` / ace#1701), so `'visit_date' in row` is true on
 * an AGGREGATED pipeline as well and cannot separate the two grains. Measured
 * live 2026-09-07 via `pipeline_preview` on opp 2232:
 *
 *   19166 `terminal_stage: aggregated`  → {"username":"aisha_g","visit_date":null,"total_visits":250,...}  x12
 *   19169 `terminal_stage: visit_level` → {"username":"grace_a","visit_date":"2026-11-11T00:00:00","total_visits":0,...} x2742
 *
 * A filled `visit_date` is the visit_level stage's own output; a null one is a
 * column the aggregated stage left alone.
 */
function hasFilledVisitDate(row: Record<string, unknown>): boolean {
  const v = row.visit_date;
  return v !== undefined && v !== null && v !== '';
}

/**
 * The visit total for ONE pipeline, decided by its GRAIN (ace#2202).
 *
 *  - VISIT-LEVEL — rows carry a filled `visit_date` and the `total_visits`
 *    counter is unfilled (absent, or zero on every row): one row IS one visit,
 *    so count the rows.
 *  - AGGREGATED — rows carry a meaningful `total_visits`: sum it.
 *
 * `null` when the rows are neither shape.
 *
 * Both signals are read, not just the first. Reading only `total_visits`
 * presence is what #2202 was; reading only `visit_date` presence is the
 * mirror-image bug, because an aggregated row carries `visit_date: null` (the
 * measurement above) and would be counted as 12 visits instead of summed to
 * 2,742. Requiring the counter to be UNFILLED as well also leaves a genuine
 * per-day aggregate — filled `visit_date` AND a real `total_visits` — summed
 * rather than miscounted as one visit per day.
 */
function pipelineVisitTotal(p: SnapshotPipeline): VisitTotal | null {
  const rows = p.rows ?? [];
  if (rows.length === 0) return null;
  const alias = p.alias ?? 'pipeline';
  const counters = rows
    .map((r) => (typeof r.total_visits === 'number' ? r.total_visits : null))
    .filter((n): n is number => n !== null);

  if (rows.some(hasFilledVisitDate) && !counters.some((n) => n !== 0)) {
    return { total: rows.length, basis: `row count of visit-level pipeline '${alias}'` };
  }
  if (counters.length > 0) {
    return {
      total: counters.reduce((acc, n) => acc + n, 0),
      basis: `sum(total_visits) over ${rows.length} row(s) of '${alias}'`,
    };
  }
  return null;
}

/**
 * The one aggregate every visit-shaped dashboard can be asked for.
 *
 * Each pipeline is classified by its own GRAIN — a visit-level pipeline is one
 * row per visit and is counted; an aggregated one carries labs' `total_visits`
 * counter and is summed — and the first pipeline that yields a total wins.
 *
 * Returns `null` when no pipeline is visit-shaped. A dashboard that cannot state
 * a visit total is reported as not-judged, never failed: a gate that fails on
 * what it cannot see is the always-fires class (ace#1026).
 */
export function deriveVisitTotal(
  payload: WorkflowPayload,
  livePipelines?: SnapshotPipeline[] | Record<string, SnapshotPipeline>,
): VisitTotal | null {
  const completed = payload?.instance?.status === 'completed';
  const pipelines = completed
    ? normalizeSnapshotPipelines(payload?.instance?.snapshot?.pipelines)
    : normalizeSnapshotPipelines(
        livePipelines ?? (payload as { pipeline_data?: Record<string, SnapshotPipeline> })?.pipeline_data,
      );

  for (const p of pipelines) {
    const total = pipelineVisitTotal(p);
    if (total) return total;
  }

  return null;
}

/** `&opportunity_id=<id>` out of a par_url; null for a program-scoped rollup. */
function oppScopeOf(input: CrossDashboardInput): string | null {
  if (input.dashboard.labs_opp_id !== undefined && input.dashboard.labs_opp_id !== null) {
    return String(input.dashboard.labs_opp_id);
  }
  const m = /[?&]opportunity_id=(\d+)/.exec(input.dashboard.par_url ?? '');
  return m ? m[1] : null;
}

function windowOf(payload: WorkflowPayload): string {
  const s = payload?.instance?.period_start;
  const e = payload?.instance?.period_end;
  if (!s && !e) return 'no period declared';
  return `[${s ?? '-inf'}, ${e ?? '+inf'})`;
}

/**
 * Every dashboard reading one `labs_opp_id` must report the same visit total.
 *
 * Program-scoped rollups are excluded by construction: a cross-opp rollup
 * aggregates a different population, so its total is not comparable to a
 * single-opp dashboard's, and comparing them would be the always-fires class.
 *
 * @param opts.timelineEndDate the manifest's `timeline.end_date`. When supplied,
 *   a disagreeing dashboard whose `period_end` is at or before it gets the
 *   off-by-one named explicitly — that is the cause every time so far, and the
 *   fix is `period_end = timeline.end_date + 1 day` (ace#1683).
 */
export function checkCrossDashboardConsistency(
  inputs: CrossDashboardInput[],
  opts?: { timelineEndDate?: string },
): QACheckResult {
  const groups = new Map<string, CrossDashboardInput[]>();
  const excluded: string[] = [];
  const notJudged: string[] = [];

  for (const input of inputs) {
    if ((input.dashboard.period_scope ?? '').trim().toLowerCase() === 'partial') {
      excluded.push(`${input.dashboard.key} (declared period_scope: partial)`);
      continue;
    }
    const opp = oppScopeOf(input);
    if (opp === null) {
      excluded.push(`${input.dashboard.key} (program-scoped rollup — different population)`);
      continue;
    }
    const bucket = groups.get(opp);
    if (bucket) bucket.push(input);
    else groups.set(opp, [input]);
  }

  const problems: string[] = [];
  let comparedGroups = 0;

  for (const [opp, members] of groups) {
    if (members.length < 2) continue;
    const totals: Array<{ input: CrossDashboardInput; total: VisitTotal }> = [];
    for (const m of members) {
      const t = deriveVisitTotal(m.payload, m.livePipelines);
      if (t === null) notJudged.push(`${m.dashboard.key} (no visit-shaped pipeline rows)`);
      else totals.push({ input: m, total: t });
    }
    if (totals.length < 2) continue;
    comparedGroups += 1;

    const distinct = new Set(totals.map((t) => t.total.total));
    if (distinct.size === 1) continue;

    const lines = totals.map(
      (t) =>
        `${t.input.dashboard.key}=${t.total.total} (${t.total.basis}; window ${windowOf(t.input.payload)}, ` +
        `run ${t.input.payload?.instance?.status ?? 'status unknown'})`,
    );

    const end = opts?.timelineEndDate;
    const suspects = end
      ? totals
          .filter((t) => {
            const pe = t.input.payload?.instance?.period_end;
            return !!pe && pe.slice(0, 10) <= end.slice(0, 10);
          })
          .map((t) => t.input.dashboard.key)
      : [];

    problems.push(
      `opportunity ${opp}: dashboards over ONE dataset report different visit totals — ${lines.join(', ')}` +
        (suspects.length
          ? `. ${suspects.join(', ')} carry a period_end at or before timeline.end_date ` +
            `(${end}), and period_end is EXCLUSIVE — that drops the final day`
          : ''),
    );
  }

  const notes =
    (excluded.length ? ` (excluded: ${excluded.join(', ')})` : '') +
    (notJudged.length ? ` (not judged: ${notJudged.join(', ')})` : '');

  if (problems.length === 0) {
    return {
      pass: true,
      detail: comparedGroups
        ? `visit totals agree across every dashboard in ${comparedGroups} shared-opportunity group(s)${notes}`
        : `no two comparable dashboards share an opportunity — nothing to cross-check${notes}`,
    };
  }

  return {
    pass: false,
    detail: problems.join('; ') + notes,
    auto_fix_hint:
      'A run window is half-open: query_builder._date_window_where emits ' +
      '`visit_date >= date_from AND visit_date < date_to`, so a period_end equal to the fixture\'s last ' +
      'visit_date silently drops that whole day, while a live (never-snapshotted) sibling keeps it. ' +
      'Re-mint the snapshotted run with period_end = timeline.end_date + 1 day and repoint the par_url — ' +
      'on hh-poverty-targeting/20260824-1404 that turned 2186 into 2237, matching the fixture exactly. ' +
      'If two dashboards are MEANT to show different windows, mark the narrower one ' +
      '`period_scope: \'partial\'` in source.dashboards[] so the intent is declared rather than assumed ' +
      '(dimagi-internal/ace#1683).',
  };
}

// ---------------------------------------------------------------------------
// ace-run (cascade) provider — what the gate checks when there is no `source`
// ---------------------------------------------------------------------------
//
// Why this exists: the ace-run provider (ace#2510) writes `products.synthetic.
// cascade` + `workflows`, NOT the denovo `source` block that checks 2b–6 and
// 8–13 read. On spark-facilitator/20260926-1800 the gate was never run at all
// (no result file, no step in run_state), and nothing said which checks even
// apply to a cascade. `checksForProvider` names them, and the cascade checks
// below read what the provider actually writes, so a cascade run is checked
// rather than skipped.

export interface CascadeReportRef {
  workflow_id?: number;
  run_id?: number;
  url?: string;
  partner?: string;
  opportunity_id?: number;
}

export interface CascadeBlock {
  registry?: { registry_id?: number; indicators?: string[] };
  program_id?: number;
  /** `manifest`: where that partner's sent generator manifest is archived (ace#3251). */
  partners?: Array<{ label?: string; opportunity_id?: number; manifest?: string }>;
  programme_report?: CascadeReportRef;
  worker_review?: CascadeReportRef;
  opp_reports?: CascadeReportRef[];
  history?: { run_ids?: number[]; first_period_end?: string; last_period_end?: string };
}

export interface SyntheticProducts {
  provider?: string;
  cascade?: CascadeBlock;
  workflows?: Record<string, { workflow_id?: number; run_url?: string } | undefined>;
  source?: unknown;
}

/** The cascade's dashboards as `checkParUrlScope` inputs (programme report is program-owned). */
export function cascadeDashboards(synthetic: SyntheticProducts | null | undefined): DashboardRef[] {
  const c = synthetic?.cascade;
  if (!c) return [];
  const out: DashboardRef[] = [];
  if (c.programme_report?.url) out.push({ key: 'programme_report', template: 'indicator_programme_report', par_url: c.programme_report.url });
  for (const [i, r] of (c.opp_reports ?? []).entries()) {
    out.push({ key: `opp_reports.${i}${r.partner ? ` (${r.partner})` : ''}`, template: 'indicator_opp_report', par_url: r.url ?? '' });
  }
  return out;
}

/**
 * The cascade handoff is whole: a registry with indicators, ≥ 3 partners on
 * labs-only opportunities (≥ 10000), a programme report whose run is the LAST
 * history run, one opp report per partner, and a `workflows` map that mirrors
 * every report (ace-web's run page renders `workflows`, so a report missing
 * there is invisible to a reviewer).
 */
export function checkCascadeHandoff(synthetic: SyntheticProducts | null | undefined): QACheckResult {
  const c = synthetic?.cascade;
  if (!c) {
    return {
      pass: false,
      detail: 'products.synthetic.cascade is missing on an ace-run provider',
      auto_fix_hint: 'write products.synthetic.cascade as each id is minted (demo-data-setup § C1–C7)',
    };
  }
  const problems: string[] = [];
  const indicators = c.registry?.indicators ?? [];
  if (!c.registry?.registry_id || indicators.length === 0) problems.push('registry has no id or no indicators');
  const partners = c.partners ?? [];
  if (partners.length < 3) problems.push(`${partners.length} partner(s) — the cascade needs ≥ 3`);
  for (const p of partners) {
    if (!(typeof p.opportunity_id === 'number' && p.opportunity_id >= 10000)) {
      problems.push(`partner ${p.label ?? '?'}: opportunity ${p.opportunity_id} is not a labs-only opp (≥ 10000)`);
    }
  }
  const runIds = c.history?.run_ids ?? [];
  if (runIds.length === 0) problems.push('history.run_ids is empty — no saved weekly runs');
  const pr = c.programme_report;
  if (!pr?.workflow_id || !pr?.run_id || !pr?.url) problems.push('programme_report lacks workflow_id / run_id / url');
  else if (runIds.length && pr.run_id !== runIds[runIds.length - 1]) {
    problems.push(`programme_report.run_id ${pr.run_id} is not the latest history run ${runIds[runIds.length - 1]}`);
  }
  const opps = c.opp_reports ?? [];
  const reportOpps = new Set(opps.map((r) => r.opportunity_id));
  for (const p of partners) {
    if (!reportOpps.has(p.opportunity_id)) problems.push(`partner ${p.label ?? p.opportunity_id} has no opp report`);
  }
  const wfIds = new Set(Object.values(synthetic?.workflows ?? {}).map((w) => w?.workflow_id));
  for (const r of [pr, ...opps]) {
    if (r?.workflow_id && !wfIds.has(r.workflow_id)) problems.push(`workflow ${r.workflow_id} is not mirrored in products.synthetic.workflows`);
  }
  if (problems.length === 0) {
    return {
      pass: true,
      detail: `registry ${c.registry?.registry_id} (${indicators.length} indicators); ${partners.length} partners; programme report on run ${pr?.run_id} = latest of ${runIds.length} history runs; ${opps.length} opp reports, all mirrored in workflows`,
    };
  }
  return {
    pass: false,
    detail: problems.join('; '),
    auto_fix_hint: 'complete the cascade handoff (demo-data-setup § C4–C7): every report minted, its run the latest history run, mirrored into workflows{}',
  };
}

/**
 * `workflow_history_runs({include_snapshot: true})` → the periods check 21
 * grades. Each COMPLETED run's `data.snapshot.state.snapshot` carries
 * `{byLLO, byFLW, programInd}`; an in-progress run (the live one) has none and
 * is skipped. Sorted by `period_end`, de-duplicated (the last run of a period wins).
 */
export function periodsFromHistoryRuns(response: unknown): GradedPeriod[] {
  const runs = (response as { runs?: unknown[] } | null)?.runs ?? [];
  const byEnd = new Map<string, GradedPeriod>();
  for (const raw of runs) {
    const run = raw as { status?: string; period_end?: string; data?: { snapshot?: { state?: { snapshot?: Record<string, unknown> } } } };
    const snap = run.data?.snapshot?.state?.snapshot;
    if (run.status !== 'completed' || !snap || !run.period_end) continue;
    byEnd.set(run.period_end, {
      period_end: run.period_end,
      byLLO: (snap.byLLO as GradedPeriod['byLLO']) ?? [],
      byFLW: (snap.byFLW as GradedPeriod['byFLW']) ?? [],
      programInd: snap.programInd as GradedPeriod['programInd'],
    });
  }
  return [...byEnd.values()].sort((a, b) => a.period_end.localeCompare(b.period_end));
}

/**
 * Checks 24–27 (ace#2735): the cascade's BUILD output, judged before the DDD
 * hand-off instead of by the demo judges after a render. One entry point so the
 * script and the skill compute them identically. `visitRows` are every partner's
 * visit rows (`pipeline_preview` of the visits pipeline); without them check 24
 * fails as not-judged.
 */
export function cascadeBuildOutcomes(
  plan: CascadeStoryPlan | null | undefined,
  snapshot: CascadeSnapshot | null | undefined,
  visitRows: Array<Record<string, unknown>>,
  /** `visitSampleScope(<the preview responses>)` — appended to checks 24 and 26 so a sample never reads as every partner (ace#2973). */
  visitScope: string | null = null,
): Record<
  'cascade_review_routing_follows_pdd' | 'cascade_worker_rates_not_binary' | 'cascade_displayed_columns_vary' | 'cascade_drill_levels_branch',
  QACheckResult
> {
  const visitFlags = (snapshot?.display?.visit_flags ?? []).map((f) => f.column);
  const columns = checkConstantDisplayedColumns(snapshot, visitRows);
  const scoped = (r: QACheckResult): QACheckResult => (visitScope ? { ...r, detail: `${r.detail}; ${visitScope}` } : r);
  return {
    cascade_review_routing_follows_pdd: scoped(checkReviewRouting(visitRows, visitFlags, plan?.review_routing ?? [])),
    cascade_worker_rates_not_binary: checkWorkerRateDenominators(snapshot),
    cascade_displayed_columns_vary: scoped({ pass: columns.pass, detail: columns.detail, ...(columns.auto_fix_hint ? { auto_fix_hint: columns.auto_fix_hint } : {}) }),
    cascade_drill_levels_branch: checkDrillLevels(snapshot, plan?.single_child_levels ?? []),
  };
}

/**
 * Check 28 — the registry text the cascade RENDERS reads plainly (ace#2748).
 *
 * `display.targets_note`, the nouns, case/visit field labels, visit-flag labels
 * and descriptions, and each indicator's `label` / `plain` are what a programme
 * manager reads on screen. spark-facilitator/20261004-1706 (registry 7748)
 * shipped "Targets are the PDD's own (§8.1 P1 ≥ 80%, P3 ≥ 75%)" and the DDD
 * user-artifact judge capped the demo at clarity 2 on that one line. This gate
 * reads the same text before any render. `registry` is the authored registry
 * (`7-synthetic/semantic-registry-author_registry.json`) or a
 * `semantic_registry_get` response — after a registry edit, the live one.
 */
export function checkRegistryTextIsPlain(registry: unknown): QACheckResult {
  const doc = indicatorsDocOf(registry);
  if (!doc) {
    return {
      pass: false,
      detail: 'no registry indicators_doc supplied, so the rendered registry text was not read',
      auto_fix_hint: 'pass --registry <7-synthetic/semantic-registry-author_registry.json, or the semantic_registry_get response>',
    };
  }
  const r = checkRegistryDisplayText(doc);
  if (r.pass) return { pass: true, detail: `${r.checked} reader-facing registry strings read plainly` };
  return {
    pass: false,
    detail: `${r.findings.length} problem(s) in reader-facing registry text: ${describeDisplayTextFindings(r.findings)}`,
    auto_fix_hint:
      'rewrite the text for a programme manager (skills/semantic-registry-author § 3), e.g. "Targets come from the pilot design: ' +
      'a meeting in at least 80% of each community\'s weeks …"; apply it with semantic_registry_update, then rebuild history ' +
      '(demo-data-setup § C5). PDD citations stay in scope_note',
  };
}

/**
 * Which check ids apply to which provider. The gate runs EVERY applicable
 * check; a check that applies but cannot be evaluated is a FAILURE of that
 * check, never a skip — and the result writer fails a run that evaluated none.
 */
export const CHECKS_BY_PROVIDER: Record<'ace-run' | 'denovo' | 'clone', readonly string[]> = {
  'ace-run': [
    'realized_json_parses',
    'every_par_url_is_run_deeplink',
    'worker_review_url_scoped',
    'cascade_handoff_complete',
    'cascade_story_landed',
    // ace#2735 — build defects the DDD judges were finding after render.
    'cascade_review_routing_follows_pdd',
    'cascade_worker_rates_not_binary',
    'cascade_displayed_columns_vary',
    'cascade_drill_levels_branch',
    // ace#2748 — reader-facing registry text, before the DDD judge reads it.
    'cascade_registry_text_is_plain',
    // ace#3251 — the two required build-half artifacts, before finalize finds them missing.
    'cascade_build_artifacts_written',
  ],
  denovo: [
    'realized_json_parses',
    'every_par_url_is_run_deeplink',
    'dashboards_match_realized',
    'opp_is_labs_only',
    'timeline_pinned',
    'flagged_worker_not_pre_seeded',
    'deliver_units_present',
    'par_url_payload_is_populated',
    'interactive_run_is_live',
    'dataset_obeys_pdd_constraints',
    'cross_dashboard_totals_agree',
  ],
  clone: [
    'realized_json_parses',
    'every_par_url_is_run_deeplink',
    'dashboards_match_realized',
    'opp_is_labs_only',
    'deliver_units_present',
    'par_url_payload_is_populated',
  ],
};

export function checksForProvider(provider: string | null | undefined): readonly string[] {
  const p = (provider ?? '').trim();
  if (p === 'ace-run' || p === 'denovo' || p === 'clone') return CHECKS_BY_PROVIDER[p];
  return [];
}

/** Check 1 as a function: realized.json is a FLAT object (DDD substitutes `${var}` verbatim). */
export function checkRealizedFlat(realized: unknown): QACheckResult {
  if (!realized || typeof realized !== 'object' || Array.isArray(realized)) {
    return { pass: false, detail: 'realized.json is missing or not a JSON object', auto_fix_hint: 're-run demo-data-setup step 5 / C7 — write the flat handoff' };
  }
  const nested = Object.entries(realized as Record<string, unknown>).filter(([, v]) => v !== null && typeof v === 'object');
  if (nested.length) {
    return { pass: false, detail: `nested values under ${nested.map(([k]) => k).join(', ')}`, auto_fix_hint: 'flatten realized.json — DDD substitutes ${var} verbatim' };
  }
  return { pass: true, detail: `flat JSON object with ${Object.keys(realized as object).length} keys` };
}
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall