Skip to content

ahstn/oceans-llm / .agents/skills/verify-oceans-admin

verify-oceans-admin

Verified

Verify the Oceans LLM admin control plane and backend gateway against the real local stack, seeded demo data, and bounded live providers when required. Use for user-path checks of sign-in, models, API keys, observability, usage costs, spend controls, batches, agent sessions, request logs, Anthropic Messages thinking and effort compatibility, MCP registry and tool access, OpenRouter routing, guardrails, and Skills storage.

FileKindSize
SKILL.mdskill md29.0 KB
examples/mcp-candidates.jsonconfig3.0 KB
features/README.mdmarkdown10.0 KB
features/account-onboarding.mdmarkdown4.3 KB
features/agent-harnesses.mdmarkdown4.3 KB
features/agent-plugins.mdmarkdown1005 B
features/agent-sessions.mdmarkdown3.9 KB
features/anthropic-messages.mdmarkdown20.7 KB
features/api-keys.mdmarkdown6.5 KB
features/attribution.mdmarkdown13.9 KB
features/audit-log.mdmarkdown4.9 KB
features/backend-gateway.mdmarkdown5.2 KB
features/batch-requests.mdmarkdown4.2 KB
features/design-system.mdmarkdown4.6 KB
features/guardrails.mdmarkdown7.0 KB
features/harness-integration.mdmarkdown7.8 KB
features/identity.mdmarkdown4.2 KB
features/image-generation.mdmarkdown7.3 KB
features/leaderboard.mdmarkdown3.3 KB
features/live-llm-requests.mdmarkdown12.6 KB
features/mcp-client-oauth.mdmarkdown9.9 KB
features/mcp-ownership.mdmarkdown7.4 KB
features/mcp.mdmarkdown19.6 KB
features/models.mdmarkdown8.2 KB
features/password-sign-in.mdmarkdown8.3 KB
features/profile.mdmarkdown2.5 KB
features/rate-limits.mdmarkdown6.0 KB
features/request-logs.mdmarkdown8.6 KB
features/review-agent.mdmarkdown3.7 KB
features/routing.jsonconfig33.6 KB
features/service-tiers.mdmarkdown6.3 KB
features/skills.mdmarkdown9.8 KB
features/spend-controls.mdmarkdown7.3 KB
features/usage-costs.mdmarkdown3.7 KB
features/workspace-connections.mdmarkdown6.1 KB
scripts/check-messages.mjsexecutablescript2.4 KB
scripts/check-messages.test.mjsscript3.0 KB
scripts/control-oceans-adminexecutablescript25.3 KB
scripts/control-oceans-admin.test.mjsscript7.6 KB
scripts/drive-backend-gateway.mjsexecutablescript12.5 KB
scripts/drive-harness-integration.mjsscript6.6 KB
scripts/drive-mcp.mjsscript13.0 KB
scripts/drive-models.mjsscript12.9 KB
scripts/drive-observability.mjsscript16.8 KB
scripts/drive-profile.mjsscript8.2 KB
scripts/drive-service-tiers.mjsexecutablescript20.1 KB
scripts/drive-skills.mjsscript27.2 KB
scripts/mcp-browser.mjsscript26.5 KB
scripts/mcp-canary.mjsscript18.8 KB
scripts/mcp-candidates.mjsscript3.5 KB
scripts/mcp-verification.test.mjsscript10.2 KB
scripts/messages-regressions.mjsscript15.8 KB
scripts/plan-verification.mjsscript5.2 KB
scripts/plan-verification.test.mjsscript6.1 KB
scripts/proof-cli.mjsscript11.1 KB
scripts/proof.mjsscript9.2 KB
scripts/proof.test.mjsscript10.4 KB

scripts/proof.test.mjs

View on GitHub
import assert from 'node:assert/strict'
import { spawnSync } from 'node:child_process'
import fs from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import test from 'node:test'
import { fileURLToPath } from 'node:url'

import { parseCheck, renderReport, reportRows } from './proof-cli.mjs'
import { assessProof, buildEnvelope, createProofRecorder, currentSources, sourceFingerprint, verdictFor } from './proof.mjs'

const cli = path.join(path.dirname(fileURLToPath(import.meta.url)), 'proof-cli.mjs')

test('a required failure beats a required block, and optional checks never decide', () => {
  assert.equal(verdictFor([{ id: 'a', status: 'pass' }]), 'pass')
  assert.equal(verdictFor([{ id: 'a', status: 'pass' }, { id: 'b', status: 'blocked' }]), 'blocked')
  assert.equal(verdictFor([{ id: 'a', status: 'blocked' }, { id: 'b', status: 'fail' }]), 'fail')
  assert.equal(verdictFor([{ id: 'a', status: 'pass' }, { id: 'b', status: 'fail', required: false }]), 'pass')
  assert.equal(verdictFor([]), 'fail')
  assert.equal(verdictFor([{ id: 'a', status: 'pass', required: false }]), 'fail')
})

test('check specs parse id, status, and an optional note', () => {
  assert.deepEqual(parseCheck('kpis-match-api=pass:4 of 4', true), {
    id: 'kpis-match-api',
    status: 'pass',
    required: true,
    note: '4 of 4',
  })
  assert.throws(() => parseCheck('kpis=passed', true), /id=pass\|fail\|blocked/)
})

test('envelopes reject unknown features and carry provenance', () => {
  assert.throws(() => buildEnvelope({ featureIds: ['nope'], checks: [] }), /Unknown feature IDs: nope/)
  const envelope = buildEnvelope({ featureIds: ['models'], checks: [{ id: 'a', status: 'pass' }] })
  assert.equal(envelope.schemaVersion, 1)
  assert.match(envelope.gitSha, /^[0-9a-f]{40}$/)
  assert.match(envelope.sourceHash, /^[0-9a-f]{64}$/)
  assert.ok(envelope.sourceFiles > 0)
  assert.equal(envelope.verdict, 'pass')
})

test('fingerprints are stable per feature and differ between features', () => {
  assert.equal(sourceFingerprint(['models']).sourceHash, sourceFingerprint(['models']).sourceHash)
  assert.notEqual(sourceFingerprint(['models']).sourceHash, sourceFingerprint(['mcp']).sourceHash)
})

test('assessment rejects legacy, failing, forged, foreign, and stale proofs', () => {
  const good = buildEnvelope({ featureIds: ['models'], checks: [{ id: 'a', status: 'pass' }] })
  good.runId = 'run-a'
  good.launchSourceHash = good.sourceHash
  assert.match(assessProof({ ...good, launchSourceHash: null }, { requireCurrent: true }).join(), /not tied to a launched stack/)
  assert.deepEqual(assessProof(good, { requireCurrent: true, runId: 'run-a' }), [])
  assert.match(assessProof({ feature: 'models' })[0], /schemaVersion missing/)
  const failing = { ...good, verdict: 'fail', checks: [{ id: 'a', status: 'fail', note: 'count 3 != 4' }] }
  assert.match(assessProof(failing).join(), /verdict=fail \(a: count 3 != 4\)/)
  assert.match(assessProof({ ...good, verdict: 'pass', checks: [{ id: 'a', status: 'fail' }] }).join(), /does not match/)
  assert.match(assessProof(good, { runId: 'run-b' }).join(), /belongs to run run-a/)
  assert.match(assessProof({ ...good, sourceHash: '0'.repeat(64) }, { requireCurrent: true }).join(), /^stale/)
})

test('fingerprints include each file mode, so a mode-only change goes stale', () => {
  const sources = currentSources()
  const [file, entry] = Object.entries(sources).find(([name]) => name.startsWith('crates/admin-ui/'))
  assert.match(entry, /^100(644|755) [0-9a-f]{40}$/)
  const flipped = { ...sources, [file]: entry.replace(/^100(644|755)/, (mode) => (mode === '100644' ? '100755' : '100644')) }
  assert.notEqual(sourceFingerprint(['models'], undefined, flipped).sourceHash, sourceFingerprint(['models'], undefined, sources).sourceHash)
})

test('assessment rejects missing artifacts and proofs whose stack ran older source', async () => {
  const evidenceDir = await fs.mkdtemp(path.join(os.tmpdir(), 'oceans-proof-'))
  try {
    const proof = buildEnvelope({ featureIds: ['models'], checks: [{ id: 'a', status: 'pass' }], artifacts: ['01.png'] })
    assert.match(assessProof(proof, { evidenceDir }).join(), /missing artifacts: 01.png/)
    await fs.writeFile(path.join(evidenceDir, '01.png'), '')
    proof.launchSourceHash = proof.sourceHash
    assert.deepEqual(assessProof(proof, { evidenceDir, requireCurrent: true }), [])
    assert.match(assessProof({ ...proof, artifacts: ['../01.png'] }, { evidenceDir }).join(), /missing artifacts/)
    const relaunchNeeded = { ...proof, launchSourceHash: '0'.repeat(64) }
    assert.match(assessProof(relaunchNeeded, { evidenceDir, requireCurrent: true }).join(), /^stale: .*changed after launch/)
  } finally {
    await fs.rm(evidenceDir, { recursive: true, force: true })
  }
})

test('a recorder replaces an earlier passing proof before the driver can fail', async () => {
  const evidenceDir = await fs.mkdtemp(path.join(os.tmpdir(), 'oceans-proof-'))
  try {
    const first = createProofRecorder({ evidenceDir, file: 'models-proof.json', featureIds: ['models'] })
    first.pass('a')
    await first.write()
    createProofRecorder({ evidenceDir, file: 'models-proof.json', featureIds: ['models'] })
    const saved = JSON.parse(await fs.readFile(path.join(evidenceDir, 'models-proof.json'), 'utf8'))
    assert.equal(saved.verdict, 'fail')
    assert.equal(saved.checks[0].note, 'driver started but did not finish')
  } finally {
    await fs.rm(evidenceDir, { recursive: true, force: true })
  }
})

test('report lists planned features that have no passing proof', () => {
  const proof = buildEnvelope({ featureIds: ['models'], checks: [{ id: 'a', status: 'pass' }] })
  const blocked = buildEnvelope({
    featureIds: ['mcp'],
    checks: [{ id: 'candidate-exa', status: 'blocked', note: 'EXA key missing' }],
  })
  const proofs = [
    { file: 'models-proof.json', proof },
    { file: 'mcp-proof.json', proof: blocked },
  ]
  const markdown = renderReport({
    runId: 'run-a',
    rows: reportRows(proofs, (entry) => (entry === blocked ? ['verdict=blocked'] : [])),
    plan: { proofs: [{ id: 'models' }, { id: 'mcp' }, { id: 'usage-costs' }], unmapped: [], skipped: [] },
  })
  assert.match(markdown, /\| models \| ✅ pass \| 1\/1 \| models-proof.json \|/)
  assert.match(markdown, /\| mcp \| ⛔ blocked \| 0\/1; candidate-exa: blocked, EXA key missing \|/)
  assert.match(markdown, /\*\*Planned but not verified:\*\* mcp, usage-costs/)
})

test('report uses the newest proof per feature and marks invalid proofs', () => {
  const older = { ...buildEnvelope({ featureIds: ['models'], checks: [{ id: 'a', status: 'pass' }] }), generatedAt: '2026-01-01T00:00:00Z' }
  const newer = { ...buildEnvelope({ featureIds: ['models'], checks: [{ id: 'a', status: 'fail' }] }), generatedAt: '2026-01-02T00:00:00Z' }
  const foreign = buildEnvelope({ featureIds: ['mcp'], checks: [{ id: 'a', status: 'pass' }] })
  const proofs = [
    { file: 'models-proof.json', proof: older },
    { file: 'models-rerun-proof.json', proof: newer },
    { file: 'mcp-proof.json', proof: foreign },
  ]
  const assess = (entry) => (entry === newer ? ['verdict=fail (a)'] : entry === foreign ? ['proof belongs to run other, not run-a'] : [])
  const markdown = renderReport({
    runId: 'run-a',
    rows: reportRows(proofs, assess),
    plan: { proofs: [{ id: 'models' }, { id: 'mcp' }], unmapped: [], skipped: [] },
  })
  assert.doesNotMatch(markdown, /models-proof.json/)
  assert.match(markdown, /\| models \| ❌ fail \| 0\/1; a: fail \| models-rerun-proof.json \|/)
  assert.match(markdown, /\| mcp \| ❌ invalid: proof belongs to run other, not run-a \|/)
  assert.match(markdown, /\*\*Planned but not verified:\*\* models, mcp/)
})

test('record writes a manual proof that evidence accepts, and evidence fails on a missing feature', async () => {
  const evidenceDir = await fs.mkdtemp(path.join(os.tmpdir(), 'oceans-proof-'))
  const launchSources = path.join(evidenceDir, 'launch-sources.json')
  const env = { ...process.env, OCEANS_VERIFY_RUN_ID: 'proof-test', OCEANS_VERIFY_EVIDENCE_DIR: evidenceDir }
  const run = (...args) => spawnSync(process.execPath, [cli, ...args], { env, encoding: 'utf8' })
  try {
    assert.equal(run('snapshot-sources', launchSources).status, 0)
    env.OCEANS_VERIFY_LAUNCH_SOURCES = launchSources
    assert.equal(run('record', 'usage-costs', '--check', 'kpis=pass').status, 2, 'artifacts are required')
    await fs.writeFile(path.join(evidenceDir, '01-usage.png'), '')
    const recorded = run('record', 'usage-costs', '--check', 'kpis-match-api=pass:4 of 4', '--artifact', '01-usage.png')
    assert.equal(recorded.status, 0, recorded.stderr)
    const saved = JSON.parse(await fs.readFile(path.join(evidenceDir, 'usage-costs-proof.json'), 'utf8'))
    assert.equal(saved.method, 'manual')
    assert.deepEqual(saved.artifacts, ['01-usage.png'])
    const current = run('evidence', 'usage-costs', '--require-current')
    assert.equal(current.status, 0, current.stdout)
    assert.match(current.stdout, /PASS usage-costs \(usage-costs-proof.json\)/)
    assert.equal(run('report', '--plna').status, 2)
    const misspelled = run('evidence', 'usage-costs', '--require-curent')
    assert.equal(misspelled.status, 2)
    assert.match(misspelled.stderr, /Unknown evidence option --require-curent/)
    const missing = run('evidence', 'usage-costs', 'models')
    assert.equal(missing.status, 1)
    assert.match(missing.stdout, /FAIL models: no proof recorded/)
    const driven = run('record', 'models', '--check', 'page-open=pass', '--artifact', '01-usage.png')
    assert.equal(driven.status, 2)
    assert.match(driven.stderr, /models has a driver/)
    const escaped = run('record', 'usage-costs', '--check', 'a=pass', '--artifact', '01-usage.png', '--file', '../x-proof.json')
    assert.equal(escaped.status, 2)
    assert.match(escaped.stderr, /--file must be a file name/)
    assert.equal(run('add-check', 'usage-costs-proof.json', 'storage-cleanup=fail:prefix retained').status, 0)
    const cleaned = JSON.parse(await fs.readFile(path.join(evidenceDir, 'usage-costs-proof.json'), 'utf8'))
    assert.equal(cleaned.verdict, 'fail')
    assert.equal(run('evidence', 'usage-costs').status, 1)
    await fs.rm(path.join(evidenceDir, '01-usage.png'))
    assert.equal(run('add-check', 'usage-costs-proof.json', 'storage-cleanup=pass').status, 0)
    assert.match(run('evidence', 'usage-costs').stdout, /FAIL usage-costs .*missing artifacts: 01-usage.png/)
  } finally {
    await fs.rm(evidenceDir, { recursive: true, force: true })
  }
})
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall