alexandervashurin/semaphore / .claude/skills/gstack
gstack
FlaggedFast headless browser for QA testing and site dogfooding. Navigate any URL, interact with elements, verify page state, diff before/after actions, take annotated screenshots, check responsive layouts, test forms and uploads, handle dialogs, and assert element states. ~100ms per command. Use when you need to test a feature, verify a deployment, dogfood a user flow, or file a bug with evidence. gstack also includes development workflow skills. When you notice the user is at these stages, suggest the appropriate skill: - Brainstorming a new idea → suggest /office-hours - Reviewing a plan (strategy) → suggest /plan-ceo-review - Reviewing a plan (architecture) → suggest /plan-eng-review - Reviewing a plan (design) → suggest /plan-design-review - Creating a design system → suggest /design-consultation - Debugging errors → suggest /investigate - Testing the app → suggest /qa - Code review before merge → suggest /review - Visual design audit → suggest /design-review - Ready to deploy / create PR → suggest /ship - Post-ship doc updates → suggest /document-release - Weekly retrospective → suggest /retro - Wanting a second opinion or adversarial code review → suggest /codex - Working with production or live systems → suggest /careful - Want to scope edits to one module/directory → suggest /freeze - Maximum safety mode (destructive warnings + edit restrictions) → suggest /guard - Removing edit restrictions → suggest /unfreeze - Upgrading gstack to latest version → suggest /gstack-upgrade If the user pushes back on skill suggestions ("stop suggesting things", "I don't need suggestions", "too aggressive"): 1. Stop suggesting for the rest of this session 2. Run: gstack-config set proactive false 3. Say: "Got it — I'll stop suggesting skills. Just tell me to be proactive again if you change your mind." If the user says "be proactive again" or "turn on suggestions": 1. Run: gstack-config set proactive true 2. Say: "Proactive suggestions are back on."
test/helpers/eval-store.test.ts
View on GitHubimport { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import {
EvalCollector,
extractToolSummary,
findPreviousRun,
compareEvalResults,
formatComparison,
generateCommentary,
judgePassed,
} from './eval-store';
import type { EvalResult, EvalTestEntry, ComparisonResult } from './eval-store';
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-store-test-'));
});
afterEach(() => {
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
});
// --- Helper to make a minimal test entry ---
function makeEntry(overrides?: Partial<EvalTestEntry>): EvalTestEntry {
return {
name: 'test-1',
suite: 'suite-1',
tier: 'e2e',
passed: true,
duration_ms: 1000,
cost_usd: 0.05,
...overrides,
};
}
// --- Helper to make a minimal EvalResult ---
function makeResult(overrides?: Partial<EvalResult>): EvalResult {
return {
schema_version: 1,
version: '0.3.6',
branch: 'main',
git_sha: 'abc1234',
timestamp: '2026-03-14T12:00:00.000Z',
hostname: 'test-host',
tier: 'e2e',
total_tests: 1,
passed: 1,
failed: 0,
total_cost_usd: 0.05,
total_duration_ms: 1000,
tests: [makeEntry()],
...overrides,
};
}
// --- EvalCollector tests ---
describe('EvalCollector', () => {
test('addTest accumulates entries', () => {
const collector = new EvalCollector('e2e', tmpDir);
collector.addTest(makeEntry({ name: 'a' }));
collector.addTest(makeEntry({ name: 'b' }));
collector.addTest(makeEntry({ name: 'c' }));
// We can't inspect tests directly, but finalize will write them
});
test('finalize writes JSON file to eval dir', async () => {
const collector = new EvalCollector('e2e', tmpDir);
collector.addTest(makeEntry());
const filepath = await collector.finalize();
expect(filepath).toBeTruthy();
expect(fs.existsSync(filepath)).toBe(true);
const data = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
expect(data.tests).toHaveLength(1);
expect(data.tests[0].name).toBe('test-1');
});
test('written JSON has correct schema fields', async () => {
const collector = new EvalCollector('e2e', tmpDir);
collector.addTest(makeEntry({ passed: true, cost_usd: 0.10, duration_ms: 2000 }));
collector.addTest(makeEntry({ name: 'test-2', passed: false, cost_usd: 0.05, duration_ms: 1000 }));
const filepath = await collector.finalize();
const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
expect(data.schema_version).toBe(1);
expect(data.tier).toBe('e2e');
expect(data.total_tests).toBe(2);
expect(data.passed).toBe(1);
expect(data.failed).toBe(1);
expect(data.total_cost_usd).toBe(0.15);
expect(data.total_duration_ms).toBe(3000);
expect(data.timestamp).toBeTruthy();
expect(data.hostname).toBeTruthy();
});
test('finalize creates directory if missing', async () => {
const nestedDir = path.join(tmpDir, 'nested', 'deep', 'evals');
const collector = new EvalCollector('e2e', nestedDir);
collector.addTest(makeEntry());
const filepath = await collector.finalize();
expect(fs.existsSync(filepath)).toBe(true);
});
test('double finalize does not write twice', async () => {
const collector = new EvalCollector('e2e', tmpDir);
collector.addTest(makeEntry());
const filepath1 = await collector.finalize();
const filepath2 = await collector.finalize();
expect(filepath1).toBeTruthy();
expect(filepath2).toBe(''); // second call returns empty
expect(fs.readdirSync(tmpDir).filter(f => f.endsWith('.json') && !f.startsWith('_partial'))).toHaveLength(1);
});
test('empty collector writes valid file', async () => {
const collector = new EvalCollector('llm-judge', tmpDir);
const filepath = await collector.finalize();
const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
expect(data.total_tests).toBe(0);
expect(data.passed).toBe(0);
expect(data.tests).toHaveLength(0);
expect(data.tier).toBe('llm-judge');
});
});
// --- judgePassed tests ---
describe('judgePassed', () => {
test('passes when all thresholds met', () => {
expect(judgePassed(
{ detection_rate: 3, false_positives: 1, evidence_quality: 3 },
{ minimum_detection: 2, max_false_positives: 2 },
)).toBe(true);
});
test('fails when detection rate below minimum', () => {
expect(judgePassed(
{ detection_rate: 1, false_positives: 0, evidence_quality: 3 },
{ minimum_detection: 2, max_false_positives: 2 },
)).toBe(false);
});
test('fails when too many false positives', () => {
expect(judgePassed(
{ detection_rate: 3, false_positives: 3, evidence_quality: 3 },
{ minimum_detection: 2, max_false_positives: 2 },
)).toBe(false);
});
test('fails when evidence quality below 2', () => {
expect(judgePassed(
{ detection_rate: 3, false_positives: 0, evidence_quality: 1 },
{ minimum_detection: 2, max_false_positives: 2 },
)).toBe(false);
});
test('passes at exact thresholds', () => {
expect(judgePassed(
{ detection_rate: 2, false_positives: 2, evidence_quality: 2 },
{ minimum_detection: 2, max_false_positives: 2 },
)).toBe(true);
});
});
// --- extractToolSummary tests ---
describe('extractToolSummary', () => {
test('counts tool types from transcript events', () => {
const transcript = [
{ type: 'system', subtype: 'init' },
{ type: 'assistant', message: { content: [
{ type: 'tool_use', name: 'Bash', input: {} },
] } },
{ type: 'user', tool_use_result: { stdout: '' } },
{ type: 'assistant', message: { content: [
{ type: 'text', text: 'ok' },
{ type: 'tool_use', name: 'Read', input: {} },
] } },
{ type: 'assistant', message: { content: [
{ type: 'tool_use', name: 'Bash', input: {} },
{ type: 'tool_use', name: 'Write', input: {} },
] } },
];
const summary = extractToolSummary(transcript);
expect(summary).toEqual({ Bash: 2, Read: 1, Write: 1 });
});
test('returns empty object for empty transcript', () => {
expect(extractToolSummary([])).toEqual({});
});
test('handles events with no content array', () => {
const transcript = [
{ type: 'assistant', message: {} },
{ type: 'assistant' },
];
expect(extractToolSummary(transcript)).toEqual({});
});
});
// --- findPreviousRun tests ---
describe('findPreviousRun', () => {
test('finds correct file — same branch preferred, most recent', () => {
// Write three eval files
const files = [
{ name: '0.3.5-main-e2e-20260312-100000.json', data: makeResult({ branch: 'main', timestamp: '2026-03-12T10:00:00Z' }) },
{ name: '0.3.5-feature-e2e-20260313-100000.json', data: makeResult({ branch: 'feature', timestamp: '2026-03-13T10:00:00Z' }) },
{ name: '0.3.6-feature-e2e-20260314-100000.json', data: makeResult({ branch: 'feature', timestamp: '2026-03-14T10:00:00Z' }) },
];
for (const f of files) {
fs.writeFileSync(path.join(tmpDir, f.name), JSON.stringify(f.data));
}
// Should prefer feature branch (most recent on same branch)
const result = findPreviousRun(tmpDir, 'e2e', 'feature', path.join(tmpDir, 'current.json'));
expect(result).toContain('0.3.6-feature-e2e-20260314');
});
test('falls back to different branch when no same-branch match', () => {
const files = [
{ name: '0.3.5-main-e2e-20260312-100000.json', data: makeResult({ branch: 'main', timestamp: '2026-03-12T10:00:00Z' }) },
];
for (const f of files) {
fs.writeFileSync(path.join(tmpDir, f.name), JSON.stringify(f.data));
}
const result = findPreviousRun(tmpDir, 'e2e', 'new-branch', path.join(tmpDir, 'current.json'));
expect(result).toContain('0.3.5-main-e2e');
});
test('returns null when no prior runs exist', () => {
const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json'));
expect(result).toBeNull();
});
test('returns null when directory does not exist', () => {
const result = findPreviousRun('/nonexistent/path', 'e2e', 'main', 'current.json');
expect(result).toBeNull();
});
test('excludes the current file from results', () => {
const filename = '0.3.6-main-e2e-20260314-100000.json';
fs.writeFileSync(
path.join(tmpDir, filename),
JSON.stringify(makeResult({ branch: 'main', timestamp: '2026-03-14T10:00:00Z' })),
);
const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, filename));
expect(result).toBeNull(); // only file is excluded
});
test('filters by tier', () => {
fs.writeFileSync(
path.join(tmpDir, '0.3.6-main-llm-judge-20260314-100000.json'),
JSON.stringify(makeResult({ tier: 'llm-judge', branch: 'main', timestamp: '2026-03-14T10:00:00Z' })),
);
const result = findPreviousRun(tmpDir, 'e2e', 'main', 'current.json');
expect(result).toBeNull(); // only llm-judge file, looking for e2e
});
});
// --- compareEvalResults tests ---
describe('compareEvalResults', () => {
test('detects improved/regressed/unchanged per test', () => {
const before = makeResult({
tests: [
makeEntry({ name: 'test-a', passed: false }),
makeEntry({ name: 'test-b', passed: true }),
makeEntry({ name: 'test-c', passed: true }),
],
total_tests: 3, passed: 2, failed: 1,
});
const after = makeResult({
tests: [
makeEntry({ name: 'test-a', passed: true }), // improved
makeEntry({ name: 'test-b', passed: false }), // regressed
makeEntry({ name: 'test-c', passed: true }), // unchanged
],
total_tests: 3, passed: 2, failed: 1,
});
const result = compareEvalResults(before, after, 'before.json', 'after.json');
expect(result.improved).toBe(1);
expect(result.regressed).toBe(1);
expect(result.unchanged).toBe(1);
expect(result.deltas.find(d => d.name === 'test-a')?.status_change).toBe('improved');
expect(result.deltas.find(d => d.name === 'test-b')?.status_change).toBe('regressed');
expect(result.deltas.find(d => d.name === 'test-c')?.status_change).toBe('unchanged');
});
test('handles tests present in one run but not the other', () => {
const before = makeResult({
tests: [
makeEntry({ name: 'old-test', passed: true }),
makeEntry({ name: 'shared', passed: true }),
],
});
const after = makeResult({
tests: [
makeEntry({ name: 'shared', passed: true }),
makeEntry({ name: 'new-test', passed: true }),
],
});
const result = compareEvalResults(before, after, 'before.json', 'after.json');
expect(result.deltas).toHaveLength(3); // shared + new-test + old-test (removed)
expect(result.deltas.find(d => d.name.includes('old-test'))?.name).toContain('removed');
});
test('computes cost and duration deltas', () => {
const before = makeResult({ total_cost_usd: 2.00, total_duration_ms: 60000 });
const after = makeResult({ total_cost_usd: 1.50, total_duration_ms: 45000 });
const result = compareEvalResults(before, after, 'a.json', 'b.json');
expect(result.total_cost_delta).toBe(-0.50);
expect(result.total_duration_delta).toBe(-15000);
});
});
// --- formatComparison tests ---
describe('formatComparison', () => {
test('produces readable output with status arrows', () => {
const comparison: ComparisonResult = {
before_file: 'before.json',
after_file: 'after.json',
before_branch: 'main',
after_branch: 'feature',
before_timestamp: '2026-03-13T14:30:00Z',
after_timestamp: '2026-03-14T14:30:00Z',
deltas: [
{
name: 'browse basic',
before: { passed: true, cost_usd: 0.07, turns_used: 6, duration_ms: 24000, tool_summary: { Bash: 3 } },
after: { passed: true, cost_usd: 0.06, turns_used: 5, duration_ms: 19000, tool_summary: { Bash: 4 } },
status_change: 'unchanged',
},
{
name: 'planted bugs static',
before: { passed: false, cost_usd: 1.00, detection_rate: 3, tool_summary: {} },
after: { passed: true, cost_usd: 0.95, detection_rate: 4, tool_summary: {} },
status_change: 'improved',
},
],
total_cost_delta: -0.06,
total_duration_delta: -5000,
improved: 1,
regressed: 0,
unchanged: 1,
tool_count_before: 3,
tool_count_after: 4,
};
const output = formatComparison(comparison);
expect(output).toContain('vs previous');
expect(output).toContain('main');
expect(output).toContain('1 improved');
expect(output).toContain('1 unchanged');
expect(output).toContain('↑'); // improved arrow
expect(output).toContain('='); // unchanged arrow
// Turns and duration deltas
expect(output).toContain('6→5t');
expect(output).toContain('24→19s');
});
test('includes commentary section', () => {
const comparison: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '2026-03-13T14:30:00Z',
after_timestamp: '2026-03-14T14:30:00Z',
deltas: [
{
name: 'test-a',
before: { passed: true, cost_usd: 0.50, turns_used: 20, duration_ms: 120000 },
after: { passed: true, cost_usd: 0.30, turns_used: 10, duration_ms: 60000 },
status_change: 'unchanged',
},
{
name: 'test-b',
before: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
after: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
status_change: 'unchanged',
},
{
name: 'test-c',
before: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
after: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
status_change: 'unchanged',
},
],
total_cost_delta: -0.20,
total_duration_delta: -60000,
improved: 0, regressed: 0, unchanged: 3,
tool_count_before: 30, tool_count_after: 20,
};
const output = formatComparison(comparison);
expect(output).toContain('Takeaway');
expect(output).toContain('fewer turns');
expect(output).toContain('faster');
});
});
// --- generateCommentary tests ---
describe('generateCommentary', () => {
test('flags regressions prominently', () => {
const c: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '', after_timestamp: '',
deltas: [{
name: 'critical-test',
before: { passed: true, cost_usd: 0.10 },
after: { passed: false, cost_usd: 0.10 },
status_change: 'regressed',
}],
total_cost_delta: 0, total_duration_delta: 0,
improved: 0, regressed: 1, unchanged: 0,
tool_count_before: 0, tool_count_after: 0,
};
const notes = generateCommentary(c);
expect(notes.some(n => n.includes('REGRESSION'))).toBe(true);
expect(notes.some(n => n.includes('critical-test'))).toBe(true);
});
test('notes improvements', () => {
const c: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '', after_timestamp: '',
deltas: [{
name: 'fixed-test',
before: { passed: false, cost_usd: 0.10 },
after: { passed: true, cost_usd: 0.10 },
status_change: 'improved',
}],
total_cost_delta: 0, total_duration_delta: 0,
improved: 1, regressed: 0, unchanged: 0,
tool_count_before: 0, tool_count_after: 0,
};
const notes = generateCommentary(c);
expect(notes.some(n => n.includes('Fixed'))).toBe(true);
expect(notes.some(n => n.includes('fixed-test'))).toBe(true);
});
test('reports efficiency gains for stable tests', () => {
const c: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '', after_timestamp: '',
deltas: [{
name: 'fast-test',
before: { passed: true, cost_usd: 0.50, turns_used: 20, duration_ms: 120000 },
after: { passed: true, cost_usd: 0.25, turns_used: 10, duration_ms: 60000 },
status_change: 'unchanged',
}],
total_cost_delta: -0.25, total_duration_delta: -60000,
improved: 0, regressed: 0, unchanged: 1,
tool_count_before: 0, tool_count_after: 0,
};
const notes = generateCommentary(c);
expect(notes.some(n => n.includes('fewer turns'))).toBe(true);
expect(notes.some(n => n.includes('faster'))).toBe(true);
expect(notes.some(n => n.includes('cheaper'))).toBe(true);
});
test('reports detection rate changes', () => {
const c: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '', after_timestamp: '',
deltas: [{
name: 'detection-test',
before: { passed: true, cost_usd: 0.50, detection_rate: 3 },
after: { passed: true, cost_usd: 0.50, detection_rate: 5 },
status_change: 'unchanged',
}],
total_cost_delta: 0, total_duration_delta: 0,
improved: 0, regressed: 0, unchanged: 1,
tool_count_before: 0, tool_count_after: 0,
};
const notes = generateCommentary(c);
expect(notes.some(n => n.includes('detecting 2 more bugs'))).toBe(true);
});
test('produces overall summary for 3+ tests with no regressions', () => {
const c: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '', after_timestamp: '',
deltas: [
{ name: 'a', before: { passed: true, cost_usd: 0.50, turns_used: 10, duration_ms: 60000 },
after: { passed: true, cost_usd: 0.30, turns_used: 6, duration_ms: 40000 }, status_change: 'unchanged' },
{ name: 'b', before: { passed: true, cost_usd: 0.20, turns_used: 5, duration_ms: 30000 },
after: { passed: true, cost_usd: 0.15, turns_used: 4, duration_ms: 25000 }, status_change: 'unchanged' },
{ name: 'c', before: { passed: true, cost_usd: 0.10, turns_used: 3, duration_ms: 20000 },
after: { passed: true, cost_usd: 0.08, turns_used: 3, duration_ms: 18000 }, status_change: 'unchanged' },
],
total_cost_delta: -0.27, total_duration_delta: -27000,
improved: 0, regressed: 0, unchanged: 3,
tool_count_before: 0, tool_count_after: 0,
};
const notes = generateCommentary(c);
expect(notes.some(n => n.includes('Overall'))).toBe(true);
expect(notes.some(n => n.includes('No regressions'))).toBe(true);
});
test('returns empty for stable run with no significant changes', () => {
const c: ComparisonResult = {
before_file: 'a.json', after_file: 'b.json',
before_branch: 'main', after_branch: 'main',
before_timestamp: '', after_timestamp: '',
deltas: [
{ name: 'a', before: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
after: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 21000 }, status_change: 'unchanged' },
{ name: 'b', before: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
after: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 }, status_change: 'unchanged' },
{ name: 'c', before: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 },
after: { passed: true, cost_usd: 0.10, turns_used: 5, duration_ms: 20000 }, status_change: 'unchanged' },
],
total_cost_delta: 0, total_duration_delta: 1000,
improved: 0, regressed: 0, unchanged: 3,
tool_count_before: 15, tool_count_after: 15,
};
const notes = generateCommentary(c);
expect(notes.some(n => n.includes('Stable run'))).toBe(true);
});
});
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 468k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 468k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 468k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |