Skip to content

getaero-io/gtm-eng-skills / engineering/warm-intro-scoring

warm-intro-scoring

Verified

FileKindSize
.gitignoreother121 B
GUIDE.mdmarkdown4.7 KB
LICENSEother1.1 KB
README.mdmarkdown8.0 KB
SKILL.mdskill md327 B
TESTING.mdmarkdown8.9 KB
UPSTREAM.mdmarkdown1.3 KB
assets/company-research.play.tsscript1021 B
assets/customer-db.sqlother2.7 KB
assets/evaluation-example.jsonconfig4.6 KB
assets/example.jsonconfig6.9 KB
assets/portfolio-focus.weights.jsonconfig32 B
assets/review.htmlother7.4 KB
assets/tuning-example.jsonconfig1.2 KB
assets/tuning.htmlother30.1 KB
check_all.pyscript1.4 KB
package-manifest.jsonconfig9.8 KB
plays/README.mdmarkdown18.2 KB
plays/check-report.pyscript1.8 KB
plays/check_all.pyscript1.6 KB
plays/check_parity.pyscript6.0 KB
plays/collect.play.tsscript4.0 KB
plays/collection.tsscript1.9 KB
plays/core.tsscript44.2 KB
plays/evaluation-contract.mdmarkdown3.4 KB
plays/evaluation.tsscript13.8 KB
plays/features.play.tsscript4.7 KB
plays/features.tsscript22.6 KB
plays/generatedtemplate.tsscript38.1 KB
plays/guardrails.tsscript948 B
plays/legacy.tsscript9.7 KB
plays/merge-payloads.pyscript8.6 KB
plays/parity-driver.tsscript995 B
plays/parity-evaluation.pyscript8.5 KB
plays/parity-reports.pyscript2.3 KB
plays/prepare-inputs.pyscript6.5 KB
plays/quality.tsscript3.8 KB
plays/read-sources.play.tsscript3.4 KB
plays/report-cli.tsscript590 B
plays/report.play.tsscript2.9 KB
plays/report.tsscript3.1 KB
plays/reports.tsscript5.7 KB
plays/research.play.tsscript1.6 KB
plays/review-uncertainty.play.tsscript5.5 KB
plays/score.play.tsscript2.3 KB
plays/snapshots.play.tsscript4.2 KB
plays/strict-json.tsscript1.8 KB
plays/sync-template.pyscript849 B
plays/uncertainty.tsscript4.9 KB
plays/validation.play.tsscript3.6 KB
references/adversarial-review.mdmarkdown7.0 KB
references/batch-collection.mdmarkdown5.1 KB
references/company-network.mdmarkdown5.8 KB
references/competitive-review.mdmarkdown8.8 KB
references/evaluation.mdmarkdown8.5 KB
references/evaluator-validation.mdmarkdown3.4 KB
references/evaluator.mdmarkdown7.9 KB
references/external-data.mdmarkdown4.0 KB
references/factor-test-matrix.mdmarkdown4.6 KB
references/factors.mdmarkdown6.1 KB
references/intro-drafts.mdmarkdown1.6 KB
references/pipeline.mdmarkdown7.6 KB
references/readable-output.mdmarkdown5.4 KB
references/research-brief.mdmarkdown2.4 KB
references/sources.mdmarkdown7.5 KB
references/tuning.mdmarkdown14.6 KB
scripts/audit_history.pyscript5.1 KB
scripts/check.pyscript5.1 KB
scripts/compare_versions.pyscript1.9 KB
scripts/evaluate.pyscript22.5 KB
scripts/package-skill.pyscript1.9 KB
scripts/portfolio.pyscript3.7 KB
scripts/quality.pyscript5.3 KB
scripts/recipe-demo.tsscript2.6 KB
scripts/render.pyscript1.6 KB
scripts/score.pyscript9.6 KB
scripts/temporal.pyscript2.2 KB
scripts/test_ask_handoff.pyscript2.1 KB
scripts/test_audit_history.pyscript2.7 KB
scripts/test_handoff_evaluate.pyscript9.1 KB
scripts/test_portfolio.pyscript3.6 KB
scripts/test_quality.pyscript3.1 KB
scripts/test_temporal.pyscript1.3 KB
scripts/test_tuning.pyscript14.1 KB
scripts/test_tuning_browser.cjsscript7.7 KB
scripts/test_version_parity.pyscript1.0 KB
scripts/tuning.pyscript10.9 KB
tests/adversarial.test.tsscript8.2 KB
tests/browser_e2e.cjsscript1.9 KB
tests/cloud_e2e.pyscript6.9 KB
tests/e2e-fixture.jsonconfig3.6 KB
tests/feature_parity.pyscript2.4 KB
tests/features.test.tsscript8.3 KB
tests/test_prepare_inputs.pyscript8.6 KB
tests/uncertainty.test.tsscript2.5 KB
vendor-manifest.jsonconfig748 B

plays/evaluation.ts

View on GitHub
/* eslint-disable @typescript-eslint/no-explicit-any -- Ported JSON contract/parity boundary; runtime validators reject malformed values. */
/** Offline evaluator. Explicit as_of is its clock; no sends or automatic promotion. */
import {compareText} from './core.ts';
type RecordValue = Record<string, any>;
const get = (o:RecordValue,k:string,d:any) => k in o ? o[k] : d;
const fail = (message: string): never => { throw new Error(message); };
const object = (v: any, required: string[], optional: string[] = []) => {
  if (!v || typeof v !== 'object' || Array.isArray(v)) fail('expected object');
  if (required.some(k => !(k in v)) || Object.keys(v).some(k => !required.includes(k) && !optional.includes(k))) fail('missing or unknown fields');
};
const string = (v: any) => { if (typeof v !== 'string' || !v.trim()) fail('nonempty string required'); return v; };
const strings = (v: any): Set<string> => { if (!Array.isArray(v)) fail('list required'); v.forEach(string); const s = new Set<string>(v); if (s.size !== v.length) fail('duplicate IDs'); return s; };
const same = (a: Set<any>, b: Set<any>) => a.size === b.size && [...a].every(k => b.has(k));
const choice = (v: any, values: string[]) => { if (!values.includes(v)) fail('invalid choice'); };
export const timestamp = (v: any): bigint => {
  string(v);
  const m = v.match(/^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:[.,](\d+))?(Z|[+-]\d{2}:\d{2})$/);
  if (!m) fail('ISO timestamp with timezone required');
  const [year,month,day,hour,minute,second] = m.slice(1,7).map(Number);
  const leap = year%4===0 && (year%100!==0 || year%400===0);
  const days = [31,leap?29:28,31,30,31,30,31,31,30,31,30,31];
  if (year<1 || month<1 || month>12 || day<1 || day>days[month-1] || hour>23 || minute>59 || second>59) fail('invalid timestamp');
  if (m[8] !== 'Z' && (Number(m[8].slice(1,3))>23 || Number(m[8].slice(4,6))>59)) fail('invalid timezone');
  const whole = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}${m[8]}`;
  const milliseconds = Date.parse(whole); if (!Number.isFinite(milliseconds)) fail('invalid timestamp');
  // Python datetime truncates excess fractional digits; retain exact microseconds.
  return BigInt(milliseconds)*1000n + BigInt((m[7] ?? '').padEnd(6,'0').slice(0,6));
};
const integer = (v: any, lo: number, hi: number) => { if (!Number.isSafeInteger(v) || v < lo || v > hi) fail('integer out of range'); };
function finite(v: any) { if (typeof v === 'number' && !Number.isFinite(v)) fail('nonfinite number'); if (v && typeof v === 'object') Object.values(v).forEach(finite); }

export function validateEvaluation(data: RecordValue) {
  finite(data);
  object(data, ['as_of','queries'], ['seed','bootstrap_samples','outcome_window_days','outcomes','split','baseline_version','candidate_version','fictional']);
  const cutoff = timestamp(data.as_of);
  // JS transports cannot exactly represent integer seeds above 2^53-1: reject, never round silently.
  integer(get(data,'seed',1729), 0, Number.MAX_SAFE_INTEGER); integer(get(data,'bootstrap_samples',2000), 100, 100000); integer(get(data,'outcome_window_days',14), 1, 3650);
  for (const key of ['baseline_version','candidate_version']) if (key in data) string(data[key]);
  if ('fictional' in data && typeof data.fictional !== 'boolean') fail('fictional must be boolean');
  if (!Array.isArray(data.queries)) fail('queries list required');
  let sets: Record<string, Set<string>> | undefined;
  if ('split' in data) {
    const keys = ['train_target_ids','test_target_ids','train_account_ids','test_account_ids']; object(data.split, keys);
    sets = Object.fromEntries(keys.map(k => [k, strings(data.split[k])]));
    for (const g of ['target','account']) if ([...sets[`train_${g}_ids`]].some(k => sets![`test_${g}_ids`].has(k))) fail('train/test leakage');
  }
  const queryIds = new Set();
  for (const q of data.queries) {
    object(q, ['id','scored_at','target_id','account_id','labels','baseline','candidate','paths']);
    ['id','target_id','account_id'].forEach(k => string(q[k]));
    if (queryIds.has(q.id)) fail('duplicate query ID'); queryIds.add(q.id);
    if (sets) for (const g of ['target','account']) if (!sets[`test_${g}_ids`].has(q[`${g}_id`])) fail('query outside test split');
    const scored = timestamp(q.scored_at); if (scored > cutoff) fail('scored_at after as_of');
    if (!q.labels || typeof q.labels !== 'object' || Array.isArray(q.labels)) fail('labels object required');
    for (const [k,v] of Object.entries(q.labels)) { string(k); if (v !== null && v !== 0 && v !== 1) fail('invalid label'); }
    const universe = new Set(Object.keys(q.labels));
    for (const ranker of ['baseline','candidate']) if (!same(strings(q[ranker]), universe)) fail('ranking universe mismatch');
    if (!Array.isArray(q.paths)) fail('paths list required'); const ids = new Set();
    for (const p of q.paths) {
      object(p, ['id','status','sender_edge','target_edge','willingness','feature_dates']); string(p.id);
      if (ids.has(p.id)) fail('duplicate path ID'); ids.add(p.id);
      choice(p.status, ['ready','research','blocked']); for (const e of ['sender_edge','target_edge']) choice(p[e], ['strong','medium','weak','unknown']); choice(p.willingness, ['yes','no','unknown']);
      if (!Array.isArray(p.feature_dates) || !p.feature_dates.length) fail('feature dates required');
      p.feature_dates.forEach((v: any) => { if (timestamp(v) > scored) fail('feature leakage'); });
      if (p.status === 'ready' && (!['strong','medium'].includes(p.sender_edge) || !['strong','medium'].includes(p.target_edge) || p.willingness !== 'yes')) fail('unsafe ready path');
    }
    if (!same(ids, universe)) fail('path universe mismatch');
  }
  if ('outcomes' in data && !Array.isArray(data.outcomes)) fail('outcomes list required'); const ids = new Set();
  for (const o of data.outcomes ?? []) {
    object(o, ['id','status','created_at','reply','meeting'], ['sent_at','observed_through','reply_at','meeting_at']); string(o.id);
    if (ids.has(o.id)) fail('duplicate outcome ID'); ids.add(o.id); choice(o.status, ['draft','sent']);
    const dates: Record<string,bigint> = {}; for (const k of Object.keys(o).filter(k => k.endsWith('_at') || k === 'observed_through')) { dates[k] = timestamp(o[k]); if (dates[k] > cutoff) fail('future outcome'); }
    for (const e of ['reply','meeting']) { if (o[e] !== null && typeof o[e] !== 'boolean') fail('invalid outcome'); if ((o[e] === true) !== (`${e}_at` in dates)) fail('event timestamp mismatch'); }
    if (o.status === 'draft') { if (Object.keys(dates).length !== 1 || o.reply !== null || o.meeting !== null) fail('draft has observed outcomes'); }
    else { if (!('sent_at' in dates) || !('observed_through' in dates) || dates.created_at > dates.sent_at || dates.sent_at > dates.observed_through) fail('invalid sent chronology');
      for (const e of ['reply_at','meeting_at']) if (e in dates && (dates[e] < dates.sent_at || dates[e] > dates.observed_through)) fail('event outside observation'); }
  }
  return cutoff;
}

/** CPython integer seeding (init_by_array), MT19937 and _randbelow_with_getrandbits. */
export class PythonRandom {
  private mt = new Uint32Array(624); private index = 624;
  constructor(seed: number) {
    const key: number[] = []; let n = BigInt(seed); do { key.push(Number(n & 0xffffffffn)); n >>= 32n; } while (n);
    this.mt[0] = 19650218;
    for (let i = 1; i < 624; i++) this.mt[i] = (Math.imul(1812433253, this.mt[i-1] ^ (this.mt[i-1] >>> 30)) + i) >>> 0;
    let i = 1, j = 0;
    for (let k = Math.max(624,key.length); k; k--) { this.mt[i] = ((this.mt[i] ^ Math.imul(this.mt[i-1] ^ (this.mt[i-1] >>> 30),1664525)) + key[j] + j) >>> 0; i++; j++; if (i >= 624) { this.mt[0] = this.mt[623]; i = 1; } if (j >= key.length) j = 0; }
    for (let k = 623; k; k--) { this.mt[i] = ((this.mt[i] ^ Math.imul(this.mt[i-1] ^ (this.mt[i-1] >>> 30),1566083941)) - i) >>> 0; i++; if (i >= 624) { this.mt[0] = this.mt[623]; i = 1; } }
    this.mt[0] = 0x80000000;
  }
  private word() {
    if (this.index >= 624) { for (let i = 0; i < 624; i++) { const y = (this.mt[i] & 0x80000000) | (this.mt[(i+1)%624] & 0x7fffffff); this.mt[i] = this.mt[(i+397)%624] ^ (y >>> 1) ^ ((y & 1) ? 0x9908b0df : 0); } this.index = 0; }
    let y = this.mt[this.index++]; y ^= y >>> 11; y ^= (y << 7) & 0x9d2c5680; y ^= (y << 15) & 0xefc60000; y ^= y >>> 18; return y >>> 0;
  }
  randrange(n: number) { const bits = Math.floor(Math.log2(n)) + 1; let r; do { r = this.word() >>> (32-bits); } while (r >= n); return r; }
}
const keys = ['recall@3','mrr','ndcg@3'];
function mean(xs: number[]) { // Compensated summation reduces differences from Python statistics.fmean.
  let sum = 0, c = 0; for (const x of xs) { const t = sum+x; c += Math.abs(sum) >= Math.abs(x) ? (sum-t)+x : (x-t)+sum; sum = t; } return (sum+c)/xs.length;
}
export function rankMetrics(ranking: string[], labels: Record<string,number>) {
  const gains = ranking.map(k => labels[k]), relevant = Object.values(labels).reduce((a,b) => a+b,0);
  const ideal = Array.from({length:Math.min(3,relevant)}, (_,i) => 1/Math.log2(i+2)).reduce((a,b) => a+b,0);
  return {'recall@3':gains.slice(0,3).reduce((a,b)=>a+b,0)/relevant,mrr:1/(gains.indexOf(1)+1),'ndcg@3':gains.slice(0,3).reduce((a,g,i)=>a+g/Math.log2(i+2),0)/ideal};
}
function percentile(xs: number[], fraction: number) { const a = [...xs].sort((a,b)=>a-b), p = (a.length-1)*fraction; return a[Math.floor(p)]+(a[Math.ceil(p)]-a[Math.floor(p)])*(p-Math.floor(p)); }
export function outcomeReport(data: RecordValue) {
  const r: RecordValue = {total_records:0,drafts:0,sent:0,sent_immature:0,sent_unobserved:0,matured_sent_denominator:0,replies_within_window:0,meetings_within_window:0,raw_reply_events:0,raw_meeting_events:0};
  const days = get(data,'outcome_window_days',14);
  for (const o of data.outcomes ?? []) { r.total_records++; for (const e of ['reply','meeting']) r[`raw_${e}_events`] += Number(o[e] === true);
    if (o.status === 'draft') { r.drafts++; continue; } r.sent++; const horizon = timestamp(o.sent_at)+BigInt(days)*86400000000n;
    if (timestamp(o.observed_through) < horizon) { r.sent_immature++; continue; }
    if (o.reply === null || o.meeting === null) { r.sent_unobserved++; continue; } r.matured_sent_denominator++;
    for (const [e,k] of [['reply','replies_within_window'],['meeting','meetings_within_window']]) if (o[e] && timestamp(o[`${e}_at`]) <= horizon) r[k]++;
  }
  return {...r,window_days:days,reply_rate:r.matured_sent_denominator ? r.replies_within_window/r.matured_sent_denominator : null,meeting_rate:r.matured_sent_denominator ? r.meetings_within_window/r.matured_sent_denominator : null,interpretation:'Observed selected sends only; no causal comparison between rankers. Null rate means unavailable.'};
}
export function evaluate(data: RecordValue) {
  validateEvaluation(data);
  const coverage: RecordValue = {total_queries:data.queries.length,eligible_queries:0,total_labels:0,judged_labels:0,exclusions:{incomplete_judgments:0,no_relevant:0}};
  const rows: RecordValue[] = data.queries.map((q: RecordValue) => { const values = Object.values(q.labels); coverage.total_labels += values.length; coverage.judged_labels += values.filter(v=>v!==null).length;
    const reason = values.includes(null) ? 'incomplete_judgments' : !values.includes(1) ? 'no_relevant' : null;
    const row: RecordValue = {id:q.id,scored_at:q.scored_at,target_id:q.target_id,account_id:q.account_id,exclusion:reason};
    if (reason) coverage.exclusions[reason]++; else { coverage.eligible_queries++; row.baseline=rankMetrics(q.baseline,q.labels); row.candidate=rankMetrics(q.candidate,q.labels); row.delta=Object.fromEntries(keys.map(k=>[k,row.candidate[k]-row.baseline[k]])); } return row;
  });
  const eligible = rows.filter(r=>r.exclusion===null), n=eligible.length, samples=get(data,'bootstrap_samples',2000), seed=get(data,'seed',1729), rng=new PythonRandom(seed);
  const clusters = new Map<string,RecordValue[]>(); for (const r of eligible) { if (!clusters.has(r.account_id)) clusters.set(r.account_id,[]); clusters.get(r.account_id)!.push(r); }
  const accounts=[...clusters.keys()].sort(compareText); coverage.independent_accounts=accounts.length;
  const bootstrap: Record<string,number[]> = Object.fromEntries(keys.map(k=>[k,[]]));
  if (n) for (let i=0;i<samples;i++) { const drawn=accounts.flatMap(()=>clusters.get(accounts[rng.randrange(accounts.length)])!); for (const k of keys) bootstrap[k].push(mean(drawn.map(r=>r.delta[k]))); }
  const metrics=Object.fromEntries(keys.map(k=>[k,{baseline:n?mean(eligible.map(r=>r.baseline[k])):null,candidate:n?mean(eligible.map(r=>r.candidate[k])):null,delta:n?mean(eligible.map(r=>r.delta[k])):null,delta_ci95:n?[percentile(bootstrap[k],.025),percentile(bootstrap[k],.975)]:null,paired_query_denominator:n}]));
  const warnings=['Exploratory diagnostics only; not proven uplift. No automatic promotion or causal inference.','Supplied metadata cannot verify source truth or independently adjudicated labels.'];
  let insufficient=accounts.length<30 || coverage.exclusions.incomplete_judgments>0 || !!data.fictional;
  if(accounts.length<30) warnings.push('Small sample: fewer than 30 eligible accounts; uncertainty is unstable and evidence is insufficient.');
  if(data.fictional) warnings.push('Fictional fixture: validates behavior only, not real-world accuracy.');
  if(!('split' in data)) { insufficient=true; warnings.push('Train/test group metadata not supplied; leakage separation is unverified.'); }
  warnings.push('Account-cluster bootstrap assumes independent accounts; shared connectors and other cross-account dependencies are not modeled.');
  return {schema_version:1,as_of:data.as_of,status:insufficient?'insufficient_evidence':'diagnostic_only',promotion_allowed:false,gates:{valid:true,split_checked:'split'in data},versions:{baseline_version:data.baseline_version??null,candidate_version:data.candidate_version??null},coverage,metrics,queries:rows,outcomes:outcomeReport(data),bootstrap:{seed,samples,unit:'account',method:'paired percentile 95%, linear interpolation'},warnings};
}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall