ahstn/oceans-llm / .agents/skills/verify-oceans-admin
verify-oceans-admin
VerifiedVerify the Oceans LLM admin control plane and backend gateway against the real local stack, seeded demo data, and bounded live providers when required. Use for user-path checks of sign-in, models, API keys, observability, usage costs, spend controls, batches, agent sessions, request logs, Anthropic Messages thinking and effort compatibility, MCP registry and tool access, OpenRouter routing, guardrails, and Skills storage.
scripts/drive-models.mjs
View on GitHubimport fs from 'node:fs/promises'
import path from 'node:path'
import { createRequire } from 'node:module'
import { fileURLToPath } from 'node:url'
import { createProofRecorder } from './proof.mjs'
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..')
const requireFromAdminUi = createRequire(path.join(repoRoot, 'crates/admin-ui/web/package.json'))
const { chromium } = requireFromAdminUi('playwright')
const baseURL = requiredEnv('OCEANS_VERIFY_BASE_URL')
const evidenceDir = requiredEnv('OCEANS_VERIFY_EVIDENCE_DIR')
requiredEnv('OCEANS_VERIFY_GATEWAY_VERSION') // recorded in the proof envelope
const email = requiredEnv('OCEANS_VERIFY_ADMIN_EMAIL')
const password = requiredEnv('OCEANS_VERIFY_ADMIN_PASSWORD')
const targetModelId = 'gpt-6-astra'
const actions = []
const proof = createProofRecorder({ evidenceDir, file: 'models-proof.json', featureIds: ['models'] })
await fs.mkdir(evidenceDir, { recursive: true })
const browser = await chromium.launch({ headless: true })
let page
try {
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
page = await context.newPage()
page.on('console', (message) => console.log(`browser console ${message.type()}: ${message.text()}`))
page.on('pageerror', (error) => console.log(`browser page error: ${error.message}`))
page.on('requestfailed', (request) =>
console.log(`browser request failed: ${request.method()} ${request.url()} ${request.failure()?.errorText ?? ''}`),
)
await page.goto(`${baseURL}/admin/api-keys`, { waitUntil: 'domcontentloaded' })
await page.getByRole('heading', { name: 'Sign in' }).waitFor()
const signInButton = page.getByRole('button', { name: 'Sign in' })
try {
await signInButton.waitFor({ state: 'visible', timeout: 60_000 })
await page.waitForFunction(() => {
const button = Array.from(document.querySelectorAll('button')).find(
(candidate) => candidate.textContent?.trim() === 'Sign in',
)
return button instanceof HTMLButtonElement && !button.disabled
}, null, { timeout: 60_000 })
} catch {
console.log('Sign-in did not hydrate after the first Vite load; reloading once.')
await page.reload({ waitUntil: 'domcontentloaded' })
await page.getByRole('heading', { name: 'Sign in' }).waitFor()
await page.waitForFunction(() => {
const button = Array.from(document.querySelectorAll('button')).find(
(candidate) => candidate.textContent?.trim() === 'Sign in',
)
return button instanceof HTMLButtonElement && !button.disabled
}, null, { timeout: 60_000 })
}
actions.push({ action: 'open protected admin UI', result: page.url() })
proof.pass('protected-route-redirects-to-sign-in', { observed: page.url() })
await capture(page, '01-login')
await page.getByLabel('Email').fill(email)
await page.getByLabel('Password', { exact: true }).fill(password)
await Promise.all([
page.waitForURL(/\/admin\/(?:api-keys|models)(?:\?|$)/),
signInButton.click(),
])
actions.push({ action: 'sign in with seeded platform admin', result: page.url() })
proof.pass('sign-in-returns-to-protected-route', { observed: page.url() })
const modelsLink = page.getByRole('link', { name: 'Models' }).first()
await modelsLink.waitFor()
await Promise.all([page.waitForURL(/\/admin\/models(?:\?|$)/), modelsLink.click()])
await page.getByRole('heading', { name: 'Models', exact: true }).waitFor()
await page.getByRole('heading', { name: 'Model list' }).waitFor()
await page.getByTestId('models-desktop-table').waitFor()
await page.getByTestId(`models-desktop-cell-${targetModelId}`).waitFor()
await page.getByText(/^Benchmark scores by Artificial Analysis, retrieved via OpenRouter\.$/).waitFor()
proof.pass('benchmark-attribution-visible')
const pagination = page.getByRole('navigation', { name: 'Model pagination' })
await pagination.getByRole('combobox', { name: 'Rows per page' }).waitFor()
const rangeStatus = pagination.getByRole('status')
await rangeStatus.filter({ hasText: /^\d+–\d+ of \d+$/ }).waitFor()
const showingText = (await rangeStatus.textContent())?.trim()
if (!showingText) throw new Error('The Models page did not report a visible model count.')
const match = /^(\d+)–(\d+) of (\d+)$/.exec(showingText)
if (!match) throw new Error(`Unexpected Models pager text: ${showingText}`)
const firstShown = Number(match[1])
const lastShown = Number(match[2])
const displayedCount = lastShown === 0 ? 0 : lastShown - firstShown + 1
const totalCount = Number(match[3])
const renderedCount = await page.locator('[data-testid^="models-desktop-cell-"]').count()
if (displayedCount !== renderedCount) {
throw new Error(`UI reported ${displayedCount} displayed models but rendered ${renderedCount} model rows.`)
}
proof.pass('displayed-count-matches-rendered-rows', { observed: renderedCount, expected: displayedCount })
actions.push({ action: 'follow Models sidebar link', result: showingText })
await capture(page, '02-models')
const adminModels = await page.evaluate(async () => {
const response = await fetch('/api/v1/admin/models?page=1&page_size=100&include_aliases=false')
if (!response.ok) throw new Error(`admin models request returned ${response.status}`)
return response.json()
})
const apiCount = adminModels.data.total
if (apiCount !== totalCount) {
throw new Error(`UI total model count ${totalCount} did not match admin API count ${apiCount}.`)
}
proof.pass('total-count-matches-admin-api', { observed: totalCount, expected: apiCount })
const apiModel = adminModels.data.items.find((model) => model.id === targetModelId)
if (!apiModel) throw new Error(`The admin models API omitted ${targetModelId}.`)
const modelCell = page.getByTestId(`models-desktop-cell-${targetModelId}`)
const modelRow = modelCell.locator('xpath=ancestor::tr')
const intelligenceHeader = page.getByRole('columnheader', { name: /^Intelligence Index/ })
await intelligenceHeader.waitFor()
const intelligenceColumn = await intelligenceHeader.evaluate((header) =>
Array.from(header.parentElement.children).indexOf(header),
)
const intelligenceScore = apiModel.benchmark_scores.find(
(score) => score.metric_key === 'artificial_analysis_intelligence_index',
)
const intelligenceValue = intelligenceScore
? new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 }).format(intelligenceScore.value)
: '—'
await modelRow.getByRole('cell').nth(intelligenceColumn).getByText(intelligenceValue, { exact: true }).waitFor()
actions.push({ action: 'inspect always-visible Intelligence Index', result: intelligenceValue })
proof.pass('intelligence-index-matches-admin-api', { observed: intelligenceValue })
await modelRow.getByRole('button', { name: 'Info' }).click()
const infoDialog = page.getByRole('dialog', { name: 'Model info' })
await infoDialog.getByText(targetModelId, { exact: true }).first().waitFor()
const infoSections = infoDialog.getByRole('navigation', { name: 'Model info sections' })
const infoSectionNames = ['Overview', 'Routing', 'Benchmarks', 'Access']
for (const name of infoSectionNames) {
await infoSections.getByRole('button', { name, exact: true }).click()
await infoDialog.getByRole('heading', { name, exact: true }).waitFor()
if (name === 'Overview') {
await infoDialog.getByRole('heading', { name: 'Pricing and limits', exact: true }).waitFor()
await infoDialog.getByTestId('model-pricing-costs').waitFor()
}
if (name === 'Benchmarks') {
if (apiModel.benchmark_scores.length === 0) {
await infoDialog.getByText('No benchmark data is available for this model.').waitFor()
} else {
for (const score of apiModel.benchmark_scores) {
const scoreLabel = infoDialog.getByText(score.label, { exact: true })
await scoreLabel.waitFor()
const sourceLink = scoreLabel.locator('xpath=..').locator(
`a[href="${score.source_url}"]`,
{ hasText: `${score.source_model_id} on OpenRouter` },
)
await sourceLink.waitFor()
}
await infoDialog.getByRole('link', { name: 'Artificial Analysis', exact: true }).waitFor()
}
await capture(page, '03-model-benchmarks')
}
}
actions.push({ action: `inspect ${targetModelId} model info`, result: 'All platform-admin sections visible' })
proof.pass('model-info-sections-visible', { observed: infoSectionNames.length })
proof.pass('benchmarks-match-admin-api', { observed: apiModel.benchmark_scores.length })
await capture(page, '03-model-info')
await page.keyboard.press('Escape')
await infoDialog.waitFor({ state: 'hidden' })
await page.getByRole('button', { name: 'Columns', exact: true }).click()
await page.getByRole('checkbox', { name: /^Context window/ }).check()
await page.getByRole('columnheader', { name: 'Context window', exact: true }).waitFor()
await page.getByRole('checkbox', { name: /^Capabilities/ }).check()
await page.getByRole('columnheader', { name: 'Capabilities', exact: true }).waitFor()
await page.keyboard.press('Escape')
await intelligenceHeader.waitFor()
actions.push({ action: 'enable optional model columns', result: 'Context window and Capabilities visible; Intelligence Index remains visible' })
proof.pass('optional-columns-render')
await capture(page, '04-model-columns')
const configRow = page
.getByTestId(`models-desktop-cell-${targetModelId}`)
.locator('xpath=ancestor::tr')
await configRow
.getByRole('button', { name: `Generate client config for ${targetModelId}`, exact: true })
.click()
const configDialog = page.getByRole('dialog', { name: 'Client config' })
await configDialog.waitFor()
await configDialog.getByText(`${targetModelId} via`, { exact: false }).waitFor()
const clientConfigs = await verifyClientConfigs(page, configDialog)
actions.push({ action: `generate ${targetModelId} client config`, result: 'Client config dialog visible' })
proof.pass('client-configs-match-api', { observed: clientConfigs.length })
await capture(page, '05-model-client-config')
await proof.write({ details: {
entryUrl: `${baseURL}/admin/api-keys`,
finalUrl: page.url(),
modelId: targetModelId,
displayedCount,
renderedCount,
totalCount,
apiCount,
benchmarkScoreCount: apiModel.benchmark_scores.length,
clientConfigs,
actions,
} })
console.log(`models proof passed: ${displayedCount} displayed and ${totalCount} total UI models matched the rendered rows and API total`)
console.log(`evidence: ${evidenceDir}`)
} catch (error) {
if (page && !page.isClosed()) {
await capture(page, '99-models-failure').catch(() => {})
}
await proof.write({ error, details: { actions } })
throw error
} finally {
await browser.close()
}
async function verifyClientConfigs(page, dialog) {
const response = await page.evaluate(async (modelId) => {
const result = await fetch('/api/v1/admin/models/client-configs', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model_keys: [modelId] }),
})
if (!result.ok) throw new Error(`Client configuration API returned ${result.status}`)
return result.json()
}, targetModelId)
const configs = response.data.client_configurations
if (configs.length === 0) throw new Error('Client configuration API returned no configurations.')
const choices = dialog.getByRole('radiogroup', { name: 'Client config', exact: true })
const results = []
for (const config of configs) {
await choices.getByRole('radio', { name: config.label, exact: true }).check()
for (const block of config.blocks) {
const rendered = dialog.locator('[data-slot="code-block"]').filter({
has: page.getByText(block.filename, { exact: true }),
})
await rendered.waitFor()
const content = await rendered.locator('[data-slot="code-block-line"]').evaluateAll((lines) =>
lines.map((line) => line.lastElementChild.textContent.replace(/\u200b/g, '')).join('\n'),
)
if (content !== block.content.replace(/\r\n?/g, '\n')) {
throw new Error(`Rendered ${config.label} ${block.filename} did not match the configuration API.`)
}
}
await capture(page, `05-model-client-config-${config.key}`)
results.push({ key: config.key, label: config.label, files: config.blocks.map((block) => block.filename), matchesApi: true })
}
await choices.getByRole('radio', { name: configs[0].label, exact: true }).check()
actions.push({ action: 'compare rendered client configurations with production API', result: `${results.length} configurations matched` })
return results
}
async function capture(page, name) {
await page.screenshot({ path: path.join(evidenceDir, `${name}.png`), fullPage: true })
const snapshot = await page.locator('body').ariaSnapshot()
await fs.writeFile(path.join(evidenceDir, `${name}.aria.txt`), `${snapshot}\n`)
proof.artifact(`${name}.png`)
}
function requiredEnv(name) {
const value = process.env[name]
if (!value) throw new Error(`Missing required environment variable ${name}`)
return value
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |