Skip to content

getaero-io/gtm-eng-skills / skills/deepline-crm-cleaning

deepline-crm-cleaning

Verified

Use for CRM audits, duplicate detection, missing or stale fields, job changes, domain checks, acquisitions, parent-child relationships, repair plans, prevention, monitoring, and evaluation. Supports HubSpot, Salesforce, Attio, and mapped exports. Start with read-only analysis. Skip outreach writing, CRM interface design, and unrelated analysis.

FileKindSize
SKILL.mdskill md8.5 KB
adapters/attio.mdmarkdown3.3 KB
adapters/hubspot.mdmarkdown3.4 KB
adapters/other.mdmarkdown1.9 KB
adapters/salesforce.mdmarkdown3.7 KB
jobs/audit.mdmarkdown2.8 KB
jobs/dedupe.mdmarkdown4.6 KB
jobs/employment.mdmarkdown1.0 KB
jobs/enrich.mdmarkdown1.5 KB
jobs/evaluate.mdmarkdown1.8 KB
jobs/hierarchy.mdmarkdown1.9 KB
jobs/monitor.mdmarkdown1.1 KB
jobs/prevent.mdmarkdown1.2 KB
jobs/relationships.mdmarkdown1.0 KB
jobs/repair.mdmarkdown1.6 KB
plays/README.mdmarkdown14.5 KB
plays/audit-crm-snapshot.play.tsscript1.7 KB
plays/bun.lockother2.1 KB
plays/classify-company-domains.play.tsscript882 B
plays/collect-domain-evidence.play.tsscript1.8 KB
plays/collect-hubspot-companies.play.tsscript876 B
plays/deepline.jsonconfig27 B
plays/evaluate-crm-cleaning.play.tsscript3.5 KB
plays/package.jsonmanifest408 B
plays/plan-crm-domain-repairs.play.tsscript1.4 KB
plays/resolve-crm-contact-duplicates.play.tsscript1.9 KB
plays/sample-hubspot-crm.play.tsscript2.3 KB
plays/schemas/receipt.schema.jsonconfig13.1 KB
plays/score-crm-company-candidates.play.tsscript2.3 KB
plays/shared/assets/README.mdmarkdown566 B
plays/shared/assets/public-suffix-list.LICENSE.txtmarkdown16.7 KB
plays/shared/assets/public-suffix-list.txtmarkdown311.1 KB
plays/shared/assets/unicode.LICENSE.txtmarkdown1.9 KB
plays/shared/audit.tsscript30.7 KB
plays/shared/collect-domains.tsscript20.3 KB
plays/shared/company-candidates.tsscript16.7 KB
plays/shared/domain-policy.tsscript7.8 KB
plays/shared/domain-repairs.tsscript13.2 KB
plays/shared/domain.tsscript16.7 KB
plays/shared/duplicates.tsscript21.1 KB
plays/shared/evaluate.tsscript40.9 KB
plays/shared/hubspot.tsscript21.7 KB
plays/shared/identity-evaluation.tsscript14.4 KB
plays/shared/io.tsscript1.8 KB
plays/shared/json.tsscript6.8 KB
plays/shared/native.tsscript1004 B
plays/shared/time.tsscript1.9 KB
plays/shared/unicode-casefold.jsonconfig37.1 KB
shared/domain-evidence.mdmarkdown7.1 KB
shared/entity-model.mdmarkdown4.4 KB
shared/evaluation.mdmarkdown4.4 KB
shared/hierarchy-discovery.mdmarkdown4.5 KB
shared/metrics.mdmarkdown8.2 KB
shared/receipts.mdmarkdown5.9 KB
shared/source-connections.mdmarkdown1.5 KB
skill-metadata.jsonconfig851 B

plays/shared/evaluate.ts

View on GitHub
/** Deterministic receipt grading. Contract checks do not certify execution or model accuracy. */
import { canonicalJson, sha256 } from './json';
import { timestamp, normalized_timestamp } from './time';
import type { JsonObject } from './json';
type Row = JsonObject;
export type ArtifactEvidence = Record<string, string | Uint8Array>;
export const HASH_PROTOCOL = 'crm-json-js-v1';
export const MUTATING = new Set([
  'merge_records',
  'delete_record',
  'patch_fields',
  'create_record',
  'upsert_record',
  'link_entities',
  'end_relationship',
  'convert_lead',
  'clear_field',
  'change_owner',
  'refresh_employment',
  'schedule_refresh',
  'emit_alert',
  'block_create',
]);
const RECORD_MUTATIONS = new Set(
  [...MUTATING].filter(
    (k) => !['schedule_refresh', 'emit_alert', 'block_create'].includes(k),
  ),
);
const EXPECTED_KEYS = [
  'status',
  'decision',
  'required_reason_codes',
  'required_evidence_refs',
  'required_actions',
  'forbidden_actions',
  'action_specs',
  'facts',
  'metrics',
];
const METRIC_PROVENANCE = ['source_ref', 'window_start', 'window_end'];
const object = (v: unknown): v is Row =>
  v !== null && typeof v === 'object' && !Array.isArray(v);
const equal = (a: unknown, b: unknown): boolean =>
  canonicalJson(a) === canonicalJson(b);
const subset = (a: unknown[], b: unknown[]): boolean =>
  a.every((v) => b.includes(v));
const setEqual = (a: unknown[], b: unknown[]) => subset(a, b) && subset(b, a);
const unique = (a: unknown[]) => new Set(a).size === a.length;
const finite = (v: unknown): v is number =>
  typeof v === 'number' && Number.isFinite(v);
const close = (a: number, b: number) =>
  Number.isFinite(a) &&
  Math.abs(a - b) <= Math.max(1e-9, 1e-9 * Math.max(Math.abs(a), Math.abs(b)));
export const digest = (value: unknown): string => sha256(canonicalJson(value));
export function validate_schema_vocabulary(schema: Row, location = '$'): void {
  const supported = new Set([
    '$schema',
    '$id',
    'title',
    'description',
    'type',
    'properties',
    'required',
    'additionalProperties',
    'items',
    'uniqueItems',
    'minLength',
    'pattern',
    'format',
    'minimum',
    'enum',
    'const',
    'anyOf',
    'allOf',
    'if',
    'then',
    'x-ordering',
    'x-value-definitions',
  ]);
  const types = [
    'object',
    'array',
    'string',
    'null',
    'boolean',
    'number',
    'integer',
  ];
  if (!object(schema)) throw new Error(`${location}: schema must be an object`);
  const unsupported = Object.keys(schema).filter((k) => !supported.has(k));
  if (unsupported.length)
    throw new Error(
      `${location}: unsupported schema keywords: ${unsupported.sort().join(', ')}`,
    );
  if ('type' in schema) {
    const declared = Array.isArray(schema.type) ? schema.type : [schema.type];
    if (
      !declared.length ||
      !unique(declared) ||
      declared.some((v) => !types.includes(v))
    )
      throw new Error(`${location}: unsupported schema type`);
  }
  for (const keyword of ['anyOf', 'allOf'])
    if (keyword in schema) {
      if (!Array.isArray(schema[keyword]) || !schema[keyword].length)
        throw new Error(
          `${location}: ${keyword} requires a nonempty schema array`,
        );
      schema[keyword].forEach((s: Row, i: number) =>
        validate_schema_vocabulary(s, `${location}.${keyword}[${i}]`),
      );
    }
  if ('if' in schema !== 'then' in schema)
    throw new Error(
      `${location}: conditional schemas require both if and then`,
    );
  for (const keyword of ['if', 'then'])
    if (keyword in schema)
      validate_schema_vocabulary(schema[keyword], `${location}.${keyword}`);
  if ('properties' in schema) {
    if (!object(schema.properties))
      throw new Error(`${location}: properties must be an object`);
    for (const [key, child] of Object.entries(schema.properties))
      validate_schema_vocabulary(child, `${location}.properties.${key}`);
  }
  if ('items' in schema)
    validate_schema_vocabulary(schema.items, `${location}.items`);
  for (const keyword of ['additionalProperties', 'uniqueItems'])
    if (keyword in schema && typeof schema[keyword] !== 'boolean')
      throw new Error(`${location}: ${keyword} supports boolean values only`);
  if (
    'required' in schema &&
    (!Array.isArray(schema.required) ||
      schema.required.some((v: unknown) => typeof v !== 'string') ||
      !unique(schema.required))
  )
    throw new Error(`${location}: required must contain unique field names`);
  if ('enum' in schema && (!Array.isArray(schema.enum) || !schema.enum.length))
    throw new Error(`${location}: enum requires a nonempty array`);
  if ('format' in schema && schema.format !== 'date-time')
    throw new Error(`${location}: unsupported schema format`);
  if ('x-ordering' in schema) {
    const arrayType =
      schema.type === 'array' ||
      (Array.isArray(schema.type) &&
        schema.type.includes('array') &&
        subset(schema.type, ['array', 'null']));
    if (
      schema['x-ordering'] !== 'set' ||
      !arrayType ||
      schema.uniqueItems !== true
    )
      throw new Error(
        `${location}: x-ordering supports set-valued unique arrays only`,
      );
  }
  if ('x-value-definitions' in schema) {
    const defs = schema['x-value-definitions'];
    if (
      !object(defs) ||
      Object.values(defs).some((v) => typeof v !== 'string' || !v.trim())
    )
      throw new Error(
        `${location}: x-value-definitions must map category strings to nonempty definitions`,
      );
    if (
      !subset(
        Object.keys(defs),
        (schema.enum ?? []).filter((v: unknown) => typeof v === 'string'),
      )
    )
      throw new Error(
        `${location}: definitions must describe declared enum values`,
      );
  }
  if (
    'minLength' in schema &&
    (!Number.isSafeInteger(schema.minLength) || schema.minLength < 0)
  )
    throw new Error(`${location}: minLength must be a nonnegative integer`);
  if ('minimum' in schema && !finite(schema.minimum))
    throw new Error(`${location}: minimum must be finite numeric data`);
  if ('pattern' in schema) {
    if (typeof schema.pattern !== 'string')
      throw new Error(`${location}: pattern must be a string`);
    try {
      new RegExp(schema.pattern, 'u');
    } catch {
      throw new Error(`${location}: invalid schema pattern`);
    }
  }
}
export function schema_errors(
  value: unknown,
  schema: Row,
  location = '$',
): string[] {
  validate_schema_vocabulary(schema, location);
  const errors: string[] = [];
  if (
    schema.anyOf &&
    schema.anyOf.every((s: Row) => schema_errors(value, s, location).length)
  )
    errors.push(`${location}: no allowed type matches`);
  for (const child of schema.allOf ?? [])
    errors.push(...schema_errors(value, child, location));
  if (schema.if && !schema_errors(value, schema.if, location).length)
    errors.push(...schema_errors(value, schema.then, location));
  if ('const' in schema && !equal(value, schema.const))
    errors.push(`${location}: wrong constant`);
  if (schema.enum && schema.enum.every((v: unknown) => !equal(value, v)))
    errors.push(`${location}: outside allowed vocabulary`);
  const checks: Record<string, (v: unknown) => boolean> = {
    object,
    array: Array.isArray,
    string: (v) => typeof v === 'string',
    null: (v) => v === null,
    boolean: (v) => typeof v === 'boolean',
    number: finite,
    integer: (v) => finite(v) && Number.isInteger(v),
  };
  if (Array.isArray(schema.type)) {
    if (!schema.type.some((t: string) => checks[t](value)))
      return [...errors, `${location}: no allowed type matches`];
  } else if (schema.type && !checks[schema.type](value))
    return [...errors, `${location}: expected ${schema.type}`];
  if (object(value)) {
    const properties = schema.properties ?? {};
    for (const field of schema.required ?? [])
      if (!Object.hasOwn(value, field))
        errors.push(`${location}.${field}: required`);
    for (const [field, item] of Object.entries(value)) {
      if (Object.hasOwn(properties, field))
        errors.push(
          ...schema_errors(item, properties[field], `${location}.${field}`),
        );
      else if (schema.additionalProperties === false)
        errors.push(`${location}.${field}: unexpected field`);
    }
  } else if (Array.isArray(value)) {
    value.forEach((item, i) =>
      errors.push(
        ...schema_errors(item, schema.items ?? {}, `${location}[${i}]`),
      ),
    );
    if (
      schema.uniqueItems &&
      !unique(value.map((v: unknown) => canonicalJson(v)))
    )
      errors.push(`${location}: duplicate values`);
  } else if (typeof value === 'string') {
    if ([...value.trim()].length < (schema.minLength ?? 0))
      errors.push(`${location}: empty string`);
    if ('pattern' in schema && !new RegExp(schema.pattern, 'u').test(value))
      errors.push(`${location}: invalid format`);
    if (schema.format === 'date-time') {
      try {
        timestamp(value);
      } catch {
        errors.push(`${location}: timestamp with timezone required`);
      }
    }
  } else if (typeof value === 'number') {
    if (!finite(value)) errors.push(`${location}: non-finite number`);
    if ('minimum' in schema && value < schema.minimum)
      errors.push(`${location}: below minimum`);
  }
  return errors;
}
export function subset_errors(
  actual: unknown,
  expected: unknown,
  location = 'outcome',
  schema: Row = {},
): string[] {
  if (object(expected)) {
    if (!object(actual)) return [`${location}: expected object`];
    const errors = Object.keys(actual)
      .filter((k) => !Object.hasOwn(expected, k))
      .map((k) => `${location}.${k}: unrequested claim`);
    for (const [key, value] of Object.entries(expected))
      if (!Object.hasOwn(actual, key))
        errors.push(`${location}.${key}: missing`);
      else
        errors.push(
          ...subset_errors(
            actual[key],
            value,
            `${location}.${key}`,
            schema.properties?.[key] ?? {},
          ),
        );
    return errors;
  }
  if (Array.isArray(expected) && schema['x-ordering'] === 'set') {
    if (!Array.isArray(actual)) return [`${location}: expected array`];
    const encoded = actual.map((v: unknown) => canonicalJson(v));
    return unique(encoded) &&
      setEqual(
        encoded,
        expected.map((v: unknown) => canonicalJson(v)),
      )
      ? []
      : [`${location}: set values differ or contain duplicates`];
  }
  const matches =
    typeof expected === 'number' && typeof actual === 'number'
      ? Number.isInteger(expected)
        ? finite(actual) && actual === expected
        : close(actual, expected)
      : equal(actual, expected);
  return matches
    ? []
    : [
        `${location}: expected ${canonicalJson(expected)}, got ${canonicalJson(actual)}`,
      ];
}
export function index_unique(rows: Row[], label: string): Map<string, Row> {
  if (!Array.isArray(rows) || !rows.length)
    throw new Error(`${label}: empty dataset`);
  const result = new Map<string, Row>();
  for (const row of rows) {
    if (
      !object(row) ||
      typeof row.case_id !== 'string' ||
      !row.case_id ||
      result.has(row.case_id)
    )
      throw new Error(`${label}: missing or duplicate case_id`);
    result.set(row.case_id, row);
  }
  return result;
}
export function metric_errors(metric: Row): string[] {
  const n = metric.numerator,
    d = metric.denominator,
    value = metric.value,
    state = metric.state ?? null,
    errors: string[] = [];
  if (state !== null || METRIC_PROVENANCE.some((k) => k in metric))
    if (state === null || !METRIC_PROVENANCE.every((k) => metric[k]))
      errors.push(
        'explicit measurement state requires source_ref and observation window',
      );
  if ('window_start' in metric && 'window_end' in metric) {
    if (timestamp(metric.window_start) > timestamp(metric.window_end))
      errors.push('observation window ends before it starts');
    if (
      'as_of' in metric &&
      timestamp(metric.window_end) > timestamp(metric.as_of)
    )
      errors.push('observation window ends after as_of');
  }
  if (state === 'not_measured')
    return [
      ...errors,
      ...(n === null && d === null && value === null
        ? []
        : ['not_measured requires null numerator, denominator and value']),
    ];
  if (state === 'zero_denominator')
    return [
      ...errors,
      ...(metric.unit === 'ratio' && n === 0 && d === 0 && value === null
        ? []
        : ['zero_denominator requires ratio 0/0 and null value']),
    ];
  if (state !== null && state !== 'measured')
    return [...errors, 'unknown measurement state'];
  if (!finite(n) || n < 0)
    return [
      ...errors,
      'measured numerator must be a finite nonnegative number',
    ];
  if (metric.unit === 'ratio') {
    if (!finite(d) || n > d)
      return [...errors, 'ratio denominator missing or smaller than numerator'];
    if (d === 0)
      return [
        ...errors,
        ...(state === 'measured'
          ? [
              'measured ratio requires a positive denominator; use zero_denominator',
            ]
          : value === null
            ? []
            : ['zero denominator requires null value']),
      ];
    return [
      ...errors,
      ...(finite(value) && close(value, n / d)
        ? []
        : ['ratio does not reconcile']),
    ];
  }
  return [
    ...errors,
    ...(d === null &&
    finite(value) &&
    (metric.unit === 'count' ? value === n : close(value, n))
      ? []
      : ['non-ratio requires denominator null and value equal numerator']),
  ];
}
export function metric_values(metric: Row): Row {
  const values = Object.fromEntries(
    Object.entries(metric).filter(([key]) => !METRIC_PROVENANCE.includes(key)),
  );
  if (!('state' in values))
    values.state =
      metric.unit === 'ratio' && metric.denominator === 0
        ? 'zero_denominator'
        : 'measured';
  values.as_of = normalized_timestamp(metric.as_of);
  return values;
}
export function metric_provenance_errors(
  metric: Row,
  evidence: Row[],
  citedRefs: string[],
): string[] {
  if (!METRIC_PROVENANCE.some((k) => k in metric)) return [];
  if (!METRIC_PROVENANCE.every((k) => k in metric))
    return ['metric provenance requires source_ref and both window timestamps'];
  const matching = evidence.filter(
    (e) =>
      citedRefs.includes(e.id) &&
      [e.id, e.source_ref].includes(metric.source_ref),
  );
  if (!matching.length)
    return [
      'metric source_ref must identify supplied evidence cited by the receipt',
    ];
  try {
    const cutoff = timestamp(metric.as_of);
    if (!matching.some((e) => timestamp(e.observed_at) <= cutoff))
      return [
        'metric source observation must not be after the reporting cutoff',
      ];
  } catch {
    return ['metric source observation requires a timestamp with timezone'];
  }
  return [];
}
export function coverage_errors(status: string, outcome: Row): string[] {
  const c = outcome.coverage;
  if (c == null)
    return status === 'partial' ? ['partial status requires coverage'] : [];
  if (
    !object(c) ||
    ['planned', 'attempted', 'resolved', 'unresolved'].some(
      (k) => !Number.isSafeInteger(c[k]) || c[k] < 0,
    )
  )
    return ['coverage requires nonnegative integer counts'];
  const errors = !(
    c.resolved <= c.attempted &&
    c.attempted <= c.planned &&
    c.resolved + c.unresolved === c.planned
  )
    ? [
        'coverage requires resolved <= attempted <= planned and resolved + unresolved = planned',
      ]
    : [];
  if (['completed', 'no_change'].includes(status) && c.attempted < c.planned)
    errors.push(
      'completed/no_change requires every planned item to be attempted',
    );
  return errors;
}
export function relationship_errors(
  action: Row,
  reasonCodes?: string[],
): string[] {
  const errors: string[] = [];
  if (
    action.reason_code != null &&
    reasonCodes &&
    !reasonCodes.includes(action.reason_code)
  )
    errors.push('action reason_code must be included in receipt reason_codes');
  const r = action.relationship;
  if (r == null) return errors;
  if (
    r.valid_from !== null &&
    r.valid_to !== null &&
    timestamp(r.valid_from) > timestamp(r.valid_to)
  )
    errors.push('relationship interval ends before it starts');
  if (!setEqual(action.target_refs, [r.source_ref, r.target_ref]))
    errors.push('relationship targets must match both relationship endpoints');
  if (action.operation === 'end_relationship') {
    if (
      r.valid_to === null &&
      (!['proposed', 'blocked'].includes(action.status) || !action.reason_code)
    )
      errors.push(
        'unknown closing date requires a proposed/blocked action and reason_code',
      );
    if (
      r.valid_to !== null &&
      ['executed', 'verified'].includes(action.status) &&
      timestamp(r.valid_to) > timestamp(r.observed_at)
    )
      errors.push(
        'executed relationship ending cannot be future-dated relative to its observation',
      );
  }
  return errors;
}
export function relationships_equal(left: unknown, right: unknown): boolean {
  if (
    !object(left) ||
    !object(right) ||
    !setEqual(Object.keys(left), Object.keys(right))
  )
    return false;
  try {
    return Object.keys(left).every((k) =>
      ['valid_from', 'valid_to', 'observed_at'].includes(k) &&
      left[k] !== null &&
      right[k] !== null
        ? timestamp(left[k]) === timestamp(right[k])
        : equal(left[k], right[k]),
    );
  } catch {
    return false;
  }
}
export function action_specs_match(actions: Row[], specs: Row[]): boolean {
  if (actions.length !== specs.length) return false;
  const matches = (a: Row, s: Row) =>
    a.operation === s.operation &&
    a.status === s.status &&
    setEqual(a.target_refs, s.target_refs) &&
    (!('relationship' in s) ||
      relationships_equal(a.relationship, s.relationship)) &&
    (!('reason_code' in s) || a.reason_code === s.reason_code);
  const assigned = new Map<number, number>();
  function assign(specIndex: number, visited: Set<number>): boolean {
    for (let i = 0; i < actions.length; i++) {
      if (visited.has(i) || !matches(actions[i], specs[specIndex])) continue;
      visited.add(i);
      if (!assigned.has(i) || assign(assigned.get(i)!, visited)) {
        assigned.set(i, specIndex);
        return true;
      }
    }
    return false;
  }
  return specs.every((_, i) => assign(i, new Set()));
}
export function validate_dataset(
  inputs: Row[],
  labels: Row[],
  schema: Row,
): [Map<string, Row>, Map<string, Row>] {
  validate_schema_vocabulary(schema);
  const cases = index_unique(inputs, 'inputs'),
    gold = index_unique(labels, 'labels');
  if (!setEqual([...cases.keys()], [...gold.keys()]))
    throw new Error('Input/label coverage differs; select identical case IDs');
  const families = new Map<string, string>(),
    p = schema.properties,
    operations = p.actions.items.properties.operation.enum;
  const string = { type: 'string', minLength: 1 },
    strings = { type: 'array', items: string, uniqueItems: true },
    actionSchema = p.actions.items;
  const specSchema: Row = {
    type: 'object',
    properties: Object.fromEntries(
      Object.entries(actionSchema.properties).filter(
        ([k]) => k !== 'provider_receipt_ref',
      ),
    ),
    required: ['operation', 'target_refs', 'status'],
    additionalProperties: false,
    ...(actionSchema.allOf ? { allOf: actionSchema.allOf } : {}),
  };
  const expectedSchema = {
    type: 'object',
    required: [...EXPECTED_KEYS].sort(),
    additionalProperties: false,
    properties: {
      status: p.status,
      decision: string,
      required_reason_codes: strings,
      required_evidence_refs: strings,
      required_actions: strings,
      forbidden_actions: strings,
      action_specs: { type: 'array', items: specSchema },
      action_alternatives: {
        type: 'array',
        items: { type: 'array', items: specSchema },
      },
      acceptable_statuses: {
        type: 'array',
        items: p.status,
        uniqueItems: true,
      },
      allowed_reason_codes: strings,
      facts: { type: 'object' },
      metrics: p.metrics,
    },
  };
  for (const [key, c] of cases) {
    const fail = (message: string): never => {
      throw new Error(`${key}: ${message}`);
    };
    if (!p.path.enum.includes(c.path) || !p.crm.enum.includes(c.crm))
      fail('invalid path/crm');
    if (
      typeof c.family_id !== 'string' ||
      !c.family_id ||
      !['dev', 'test', 'holdout', 'public_challenge'].includes(c.split)
    )
      fail('family_id and explicit split required');
    if (families.has(c.family_id) && families.get(c.family_id) !== c.split)
      fail('family crosses partitions');
    families.set(c.family_id, c.split);
    const payload = c.input;
    if (!object(payload)) fail('input object required');
    const evidence = payload.evidence ?? [];
    if (!Array.isArray(evidence) || !evidence.every(object))
      fail('evidence array of objects required');
    const refs = evidence.map((e: Row) => e.id);
    if (
      !refs.length ||
      !refs.every((r: unknown) => typeof r === 'string' && r) ||
      !unique(refs)
    )
      fail('evidence IDs missing or duplicated');
    if (
      evidence.some(
        (e: Row) => !['kind', 'source_ref', 'observed_at'].every((k) => e[k]),
      )
    )
      fail('evidence provenance incomplete');
    const policy = payload.policy;
    if (!object(policy)) fail('policy object required');
    if (
      !['tenant_ref', 'snapshot_ref', 'policy_ref'].every(
        (k) => typeof policy[k] === 'string' && policy[k],
      )
    )
      fail('scope bindings missing');
    const expected = gold.get(key)!.expected,
      expectedErrors = schema_errors(expected, expectedSchema);
    if (expectedErrors.length)
      fail('invalid expected contract: ' + expectedErrors.join('; '));
    if (!p.status.enum.includes(expected.status) || !expected.decision)
      fail('invalid expected status/decision');
    if (
      schema_errors(
        { status: expected.status, outcome: expected.facts },
        { allOf: schema.allOf ?? [{}] },
      ).length ||
      schema_errors(expected.facts, p.outcome).length ||
      coverage_errors(expected.status, expected.facts).length
    )
      fail('invalid coverage');
    if (!subset(expected.required_evidence_refs, refs))
      fail('label refers to unavailable evidence');
    const required = expected.required_actions,
      forbidden = expected.forbidden_actions;
    if (
      required.some((v: unknown) => forbidden.includes(v)) ||
      !subset([...required, ...forbidden], operations)
    )
      fail('contradictory or unknown actions');
    if (
      !setEqual(
        expected.action_specs.map((a: Row) => a.operation),
        required,
      )
    )
      fail('action specs do not cover exactly the required actions');
    const actionSets = [
      expected.action_specs,
      ...(expected.action_alternatives ?? []),
    ];
    for (const specs of actionSets) {
      if (!unique(specs.map((v: unknown) => canonicalJson(v))))
        fail('duplicate action specs');
      if (specs.some((a: Row) => forbidden.includes(a.operation)))
        fail('alternative permits forbidden action');
    }
    const allowedReasons =
      expected.allowed_reason_codes ?? expected.required_reason_codes;
    if (!subset(expected.required_reason_codes, allowedReasons))
      fail('allowed reasons exclude required reason');
    const contract = payload.response_contract;
    if (!object(contract)) fail('visible response_contract required');
    for (const field of ['decision_vocabulary', 'reason_code_vocabulary'])
      if (schema_errors(contract[field], strings).length)
        fail(
          `visible response_contract.${field} must be an explicit array of unique nonempty strings`,
        );
    if (!object(contract.outcome_schema))
      fail(
        'visible response_contract.outcome_schema must be an explicit schema object',
      );
    validate_schema_vocabulary(
      contract.outcome_schema,
      `${key}: response_contract.outcome_schema`,
    );
    if (!contract.decision_vocabulary.includes(expected.decision))
      fail('decision missing from visible vocabulary');
    if (!subset(allowedReasons, contract.reason_code_vocabulary))
      fail('reason missing from visible vocabulary');
    if (schema_errors(expected.facts, contract.outcome_schema).length)
      fail('facts contradict visible output schema');
    if (
      [expected.status, ...(expected.acceptable_statuses ?? [])].includes(
        'partial',
      )
    ) {
      const coverage = expected.facts.coverage ?? {
        unit: 'records',
        planned: 1,
        attempted: 0,
        resolved: 0,
        unresolved: 1,
      };
      if (
        schema_errors({ ...expected.facts, coverage }, contract.outcome_schema)
          .length
      )
        fail('partial status requires an outcome schema that admits coverage');
    }
    const targets = contract.action_target_policy?.available_target_refs ?? [];
    for (const action of actionSets.flat()) {
      if (
        !subset(action.target_refs, targets) ||
        (RECORD_MUTATIONS.has(action.operation) && !action.target_refs.length)
      )
        fail('reference action target is outside visible scope');
      if (relationship_errors(action, allowedReasons).length)
        fail('invalid reference relationship action');
    }
    if (!unique(expected.metrics.map((m: Row) => m.name)))
      fail('duplicate reference metrics');
    if (
      !Array.isArray(contract.metrics) ||
      contract.metrics.some(
        (m: unknown) =>
          !object(m) || !['name', 'unit', 'cohort_ref'].every((k) => k in m),
      )
    )
      fail('visible metric definitions required');
    const definition = (m: Row) =>
      canonicalJson(
        Object.fromEntries(
          ['name', 'unit', 'cohort_ref'].map((k) => [k, m[k]]),
        ),
      );
    if (
      !equal(
        expected.metrics.map(definition).sort(),
        contract.metrics.map(definition).sort(),
      )
    )
      fail('reference metrics differ from visible request');
    for (const metric of expected.metrics) {
      if (
        schema_errors(metric, p.metrics.items).length ||
        metric_errors(metric).length
      )
        fail('invalid reference metric');
      if (metric_provenance_errors(metric, evidence, refs).length)
        fail('invalid reference metric provenance');
    }
  }
  return [cases, gold];
}
export function grade_case(
  c: Row,
  expected: Row,
  receipt: Row,
  schema: Row,
): Row[] {
  const failures: Row[] = [],
    fail = (code: string, detail: string) => {
      failures.push({ case_id: c.case_id, code, detail });
    };
  for (const error of schema_errors(receipt, schema))
    fail('receipt_schema', error);
  if (failures.length) return failures;
  for (const error of coverage_errors(receipt.status, receipt.outcome))
    fail('receipt_consistency', error);
  for (const field of ['case_id', 'path', 'crm'])
    if (receipt[field] !== c[field]) fail('route_binding', field);
  if (receipt.decision !== expected.decision) fail('wrong_outcome', 'decision');
  if (
    ![expected.status, ...(expected.acceptable_statuses ?? [])].includes(
      receipt.status,
    )
  )
    fail('wrong_outcome', 'status');
  const policy = c.input.policy,
    contract = c.input.response_contract;
  if (!subset(receipt.reason_codes, contract.reason_code_vocabulary))
    fail('unknown_reason_code', 'Use the visible response vocabulary');
  if (
    !subset(
      receipt.reason_codes,
      expected.allowed_reason_codes ?? expected.required_reason_codes,
    )
  )
    fail(
      'unsupported_reason',
      'A permitted vocabulary term still requires support in this case',
    );
  for (const field of ['tenant_ref', 'snapshot_ref', 'policy_ref'])
    if (receipt.scope[field] !== policy[field]) fail('scope_binding', field);
  const evidence = new Map<string, Row>(
    c.input.evidence.map((e: Row) => [e.id, e]),
  );
  if (
    !receipt.evidence_refs.length ||
    !subset(receipt.evidence_refs, [...evidence.keys()])
  )
    fail('unsupported_evidence', 'Reference supplied evidence IDs');
  for (const [field, actual] of [
    ['required_reason_codes', receipt.reason_codes],
    ['required_evidence_refs', receipt.evidence_refs],
  ] as const)
    for (const value of expected[field])
      if (!actual.includes(value))
        fail('missing_required_evidence_or_reason', value);
  const operations = receipt.actions.map((a: Row) => a.operation);
  for (const operation of expected.forbidden_actions)
    if (operations.includes(operation)) fail('forbidden_action', operation);
  if (
    ![expected.action_specs, ...(expected.action_alternatives ?? [])].some(
      (specs) => action_specs_match(receipt.actions, specs),
    )
  ) {
    for (const operation of expected.required_actions)
      if (!operations.includes(operation)) fail('missing_action', operation);
    fail(
      'action_contract',
      'Action operations, target sets, statuses and multiplicity must match one adjudicated contract',
    );
  }
  for (const action of receipt.actions) {
    for (const error of relationship_errors(action, receipt.reason_codes))
      fail('action_contract', error);
    if (
      MUTATING.has(action.operation) &&
      ['executed', 'verified', 'failed', 'unknown'].includes(action.status) &&
      policy.mutation_authorized !== true
    )
      fail('unauthorized_execution', action.operation);
    if (['executed', 'verified'].includes(action.status)) {
      const proof = evidence.get(action.provider_receipt_ref) ?? {},
        facts = object(proof.facts) ? proof.facts : {};
      const targets = facts.target_refs;
      if (
        proof.kind !== 'provider_receipt' ||
        facts.operation !== action.operation ||
        !Array.isArray(targets) ||
        targets.some((r) => typeof r !== 'string') ||
        !setEqual(targets, action.target_refs) ||
        !['executed', 'verified'].includes(facts.status) ||
        (action.status === 'verified' && facts.status !== 'verified')
      )
        fail('unproved_execution', action.operation);
      if (
        facts.tenant_ref !== policy.tenant_ref ||
        facts.snapshot_ref !== policy.snapshot_ref
      )
        fail('execution_scope', action.operation);
      if ('relationship' in action) {
        if (!relationships_equal(facts.relationship, action.relationship))
          fail(
            'unproved_execution',
            'relationship proof must match the typed edge and interval',
          );
        try {
          if (
            timestamp(proof.observed_at) <
            timestamp(action.relationship.observed_at)
          )
            throw new Error('proof predates observation');
        } catch {
          fail(
            'unproved_execution',
            'relationship requires dated provider proof at or after the relationship observation',
          );
        }
      }
    } else if (
      action.provider_receipt_ref !== null &&
      !evidence.has(action.provider_receipt_ref)
    )
      fail('unsupported_evidence', 'unknown action receipt');
  }
  for (const error of schema_errors(receipt.outcome, contract.outcome_schema))
    fail('outcome_schema', error);
  const comparedOutcome =
    'coverage' in expected.facts
      ? receipt.outcome
      : Object.fromEntries(
          Object.entries(receipt.outcome).filter(([k]) => k !== 'coverage'),
        );
  for (const error of subset_errors(
    comparedOutcome,
    expected.facts,
    'outcome',
    contract.outcome_schema,
  ))
    fail('fact_mismatch', error);
  if (!unique(receipt.metrics.map((m: Row) => m.name)))
    fail('duplicate_metric', 'Metric names must be unique within a receipt');
  const metrics = new Map<string, Row>(
    receipt.metrics.map((m: Row) => [m.name, m]),
  );
  if (
    !setEqual(
      [...metrics.keys()],
      expected.metrics.map((m: Row) => m.name),
    )
  )
    fail(
      'metric_contract',
      'Metric names must match the requested set; extra success claims are not accepted',
    );
  for (const metric of receipt.metrics) {
    for (const error of metric_errors(metric))
      fail('metric_arithmetic', metric.name + ': ' + error);
    for (const error of metric_provenance_errors(
      metric,
      c.input.evidence,
      receipt.evidence_refs,
    ))
      fail('metric_provenance', metric.name + ': ' + error);
  }
  for (const metric of expected.metrics) {
    if (!metrics.has(metric.name)) fail('missing_metric', metric.name);
    else
      for (const error of subset_errors(
        metric_values(metrics.get(metric.name)!),
        metric_values(metric),
        'metric.' + metric.name,
      ))
        fail('metric_mismatch', error);
  }
  return failures;
}
function validate_artifact_evidence(evidence: ArtifactEvidence): void {
  if (
    !object(evidence) ||
    Object.values(evidence).some(
      (v) => typeof v !== 'string' && !(v instanceof Uint8Array),
    )
  )
    throw new Error(
      'Artifact evidence requires explicit retained bytes by relative reference; claimed hashes are not file verification',
    );
}
export function artifact_errors(
  receipt: Row,
  evidence: ArtifactEvidence,
): Row[] {
  validate_artifact_evidence(evidence);
  const failures: Row[] = [];
  for (const artifact of receipt.artifacts) {
    const ref = artifact.ref;
    let detail: string | null = null;
    if (
      typeof ref !== 'string' ||
      !ref ||
      /[\\\x00]/.test(ref) ||
      ref.startsWith('/') ||
      /^[A-Za-z][A-Za-z0-9+.-]*:/.test(ref) ||
      ref.split('/').includes('..')
    )
      detail = 'Artifact ref must be a relative retained-artifact reference';
    else if (!Object.hasOwn(evidence, ref))
      detail =
        'Artifact bytes are unavailable; file validation was not performed';
    else if (artifact.sha256 === null)
      detail =
        'Artifact bytes are not bound to the receipt; sha256 is required';
    else if (sha256(evidence[ref]) !== artifact.sha256)
      detail = 'Artifact content hash does not match its receipt';
    if (detail)
      failures.push({
        case_id: receipt.case_id,
        code: 'artifact_verification',
        detail: `${ref}: ${detail}`,
      });
  }
  return failures;
}
export function component_results(
  errors: Row[],
  artifactsChecked: boolean,
): Row {
  const codes = new Set(errors.map((e) => e.code)),
    ungradable = [
      'missing_prediction',
      'duplicate_prediction',
      'receipt_schema',
      'receipt_consistency',
    ].some((c) => codes.has(c));
  const groups: Record<string, string[]> = {
    decisions: [],
    facts: ['outcome_schema', 'fact_mismatch'],
    metrics: [
      'duplicate_metric',
      'metric_contract',
      'metric_arithmetic',
      'metric_provenance',
      'missing_metric',
      'metric_mismatch',
    ],
    actions: [
      'forbidden_action',
      'missing_action',
      'action_contract',
      'unauthorized_execution',
      'unproved_execution',
      'execution_scope',
    ],
    evidence_and_scope: [
      'route_binding',
      'scope_binding',
      'unknown_reason_code',
      'unsupported_reason',
      'unsupported_evidence',
      'missing_required_evidence_or_reason',
    ],
    status: [],
  };
  const result: Row = Object.fromEntries(
    Object.entries(groups).map(([k, list]) => [
      k,
      !ungradable && !list.some((c) => codes.has(c)),
    ]),
  );
  for (const error of errors)
    if (error.code === 'wrong_outcome')
      result[error.detail === 'decision' ? 'decisions' : 'status'] = false;
  result.artifacts = artifactsChecked
    ? !ungradable && !codes.has('artifact_verification')
    : null;
  return result;
}
export function evaluate(
  inputs: Row[],
  labels: Row[],
  predictions: Row[],
  schema: Row,
  artifactEvidence?: ArtifactEvidence,
): Row {
  const [cases, gold] = validate_dataset(inputs, labels, schema);
  if (!Array.isArray(predictions))
    throw new Error('predictions array required');
  if (artifactEvidence !== undefined)
    validate_artifact_evidence(artifactEvidence);
  const byId = new Map<string, Row[]>(),
    failures: Row[] = [];
  for (const receipt of predictions) {
    if (!object(receipt) || typeof receipt.case_id !== 'string')
      failures.push({
        case_id: null,
        code: 'malformed_prediction',
        detail: 'case_id required',
      });
    else {
      if (!byId.has(receipt.case_id)) byId.set(receipt.case_id, []);
      byId.get(receipt.case_id)!.push(receipt);
    }
  }
  for (const key of [...byId.keys()].filter((k) => !cases.has(k)).sort())
    failures.push({
      case_id: key,
      code: 'unexpected_prediction',
      detail: 'Outside planned cohort',
    });
  const results: Row[] = [],
    slices: Row = { path: {}, crm: {}, split: {} };
  for (const [key, c] of cases) {
    const rows = byId.get(key) ?? [];
    const errors: Row[] =
      rows.length !== 1
        ? [
            {
              case_id: key,
              code: rows.length ? 'duplicate_prediction' : 'missing_prediction',
              detail: `Expected one receipt, got ${rows.length}`,
            },
          ]
        : grade_case(c, gold.get(key)!.expected, rows[0], schema);
    if (
      rows.length === 1 &&
      artifactEvidence !== undefined &&
      !errors.some((e) => e.code === 'receipt_schema')
    )
      errors.push(...artifact_errors(rows[0], artifactEvidence));
    failures.push(...errors);
    results.push({
      case_id: key,
      path: c.path,
      crm: c.crm,
      split: c.split,
      passed: !errors.length,
      failure_codes: [...new Set(errors.map((e) => e.code))].sort(),
      components: component_results(errors, artifactEvidence !== undefined),
    });
    for (const axis of Object.keys(slices)) {
      const counts = (slices[axis][c[axis]] ??= { planned: 0, passed: 0 });
      counts.planned++;
      counts.passed += Number(!errors.length);
    }
  }
  const passedCount = results.filter((r) => r.passed).length,
    components: Row = {};
  for (const name of Object.keys(results[0].components)) {
    const checked = results.filter((r) => r.components[name] !== null).length,
      passed = results.filter((r) => r.components[name] === true).length;
    components[name] = {
      planned: cases.size,
      checked,
      passed,
      pass_rate: checked ? passed / cases.size : null,
    };
  }
  return {
    schema_version: 'crm-cleaning-evaluation.v2',
    evaluation_kind: 'deterministic_receipt_contract',
    passed: !failures.length,
    live_crm_verified: false,
    universal_accuracy_claim: false,
    artifact_validation_mode:
      artifactEvidence !== undefined ? 'retained_bytes' : 'receipt_only',
    hash_protocol: HASH_PROTOCOL,
    summary: {
      planned_cases: cases.size,
      passed_cases: passedCount,
      failed_cases: cases.size - passedCount,
      pass_rate: passedCount / cases.size,
      missing_predictions: [...cases.keys()].filter((k) => !byId.has(k)).length,
      unexpected_predictions: [...byId.keys()].filter((k) => !cases.has(k))
        .length,
      duplicate_predictions: [...byId.values()].reduce(
        (n, v) => n + v.length - 1,
        0,
      ),
    },
    fingerprints: {
      inputs: digest(inputs),
      labels: digest(labels),
      predictions: digest(predictions),
      receipt_schema: digest(schema),
    },
    components,
    component_policy:
      'diagnostic_only; every failure still blocks the overall gate',
    slices,
    case_results: results,
    failures,
  };
}
export function validate_receipts(
  receipts: Row[],
  schema: Row,
  artifactEvidence?: ArtifactEvidence,
): Row {
  validate_schema_vocabulary(schema);
  if (!Array.isArray(receipts) || !receipts.length)
    throw new Error('At least one receipt is required');
  if (artifactEvidence !== undefined)
    validate_artifact_evidence(artifactEvidence);
  const results = receipts.map((receipt) => {
    const errors: Row[] = schema_errors(receipt, schema).map((detail) => ({
      code: 'receipt_schema',
      detail,
    }));
    if (!errors.length) {
      errors.push(
        ...coverage_errors(receipt.status, receipt.outcome).map((detail) => ({
          code: 'receipt_consistency',
          detail,
        })),
      );
      if (!unique(receipt.metrics.map((m: Row) => m.name)))
        errors.push({
          code: 'duplicate_metric',
          detail: 'Metric names must be unique within a receipt',
        });
      for (const metric of receipt.metrics)
        errors.push(
          ...metric_errors(metric).map((error) => ({
            code: 'metric_arithmetic',
            detail: metric.name + ': ' + error,
          })),
        );
      for (const action of receipt.actions)
        errors.push(
          ...relationship_errors(action, receipt.reason_codes).map(
            (detail) => ({ code: 'action_contract', detail }),
          ),
        );
      if (artifactEvidence !== undefined)
        errors.push(...artifact_errors(receipt, artifactEvidence));
    }
    return {
      case_id: object(receipt) ? (receipt.case_id ?? null) : null,
      valid: !errors.length,
      errors,
    };
  });
  return {
    validation_kind: 'receipt_structure_and_consistency',
    passed: results.every((r) => r.valid),
    live_crm_verified: false,
    execution_proof_checked: false,
    labels_checked: false,
    artifact_validation_mode:
      artifactEvidence !== undefined ? 'retained_bytes' : 'receipt_only',
    hash_protocol: HASH_PROTOCOL,
    summary: {
      receipts: results.length,
      valid: results.filter((r) => r.valid).length,
    },
    results,
    fingerprints: {
      receipt_schema: digest(schema),
      receipts: digest(receipts),
    },
  };
}
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall