Skip to content

ahstn/oceans-llm / .agents/skills/verify-oceans-admin

verify-oceans-admin

Verified

Verify the Oceans LLM admin control plane and backend gateway against the real local stack, seeded demo data, and bounded live providers when required. Use for user-path checks of sign-in, models, API keys, observability, usage costs, spend controls, batches, agent sessions, request logs, Anthropic Messages thinking and effort compatibility, MCP registry and tool access, OpenRouter routing, guardrails, and Skills storage.

FileKindSize
SKILL.mdskill md29.0 KB
examples/mcp-candidates.jsonconfig3.0 KB
features/README.mdmarkdown10.0 KB
features/account-onboarding.mdmarkdown4.3 KB
features/agent-harnesses.mdmarkdown4.3 KB
features/agent-plugins.mdmarkdown1005 B
features/agent-sessions.mdmarkdown3.9 KB
features/anthropic-messages.mdmarkdown20.7 KB
features/api-keys.mdmarkdown6.5 KB
features/attribution.mdmarkdown13.9 KB
features/audit-log.mdmarkdown4.9 KB
features/backend-gateway.mdmarkdown5.2 KB
features/batch-requests.mdmarkdown4.2 KB
features/design-system.mdmarkdown4.6 KB
features/guardrails.mdmarkdown7.0 KB
features/harness-integration.mdmarkdown7.8 KB
features/identity.mdmarkdown4.2 KB
features/image-generation.mdmarkdown7.3 KB
features/leaderboard.mdmarkdown3.3 KB
features/live-llm-requests.mdmarkdown12.6 KB
features/mcp-client-oauth.mdmarkdown9.9 KB
features/mcp-ownership.mdmarkdown7.4 KB
features/mcp.mdmarkdown19.6 KB
features/models.mdmarkdown8.2 KB
features/password-sign-in.mdmarkdown8.3 KB
features/profile.mdmarkdown2.5 KB
features/rate-limits.mdmarkdown6.0 KB
features/request-logs.mdmarkdown8.6 KB
features/review-agent.mdmarkdown3.7 KB
features/routing.jsonconfig33.6 KB
features/service-tiers.mdmarkdown6.3 KB
features/skills.mdmarkdown9.8 KB
features/spend-controls.mdmarkdown7.3 KB
features/usage-costs.mdmarkdown3.7 KB
features/workspace-connections.mdmarkdown6.1 KB
scripts/check-messages.mjsexecutablescript2.4 KB
scripts/check-messages.test.mjsscript3.0 KB
scripts/control-oceans-adminexecutablescript25.3 KB
scripts/control-oceans-admin.test.mjsscript7.6 KB
scripts/drive-backend-gateway.mjsexecutablescript12.5 KB
scripts/drive-harness-integration.mjsscript6.6 KB
scripts/drive-mcp.mjsscript13.0 KB
scripts/drive-models.mjsscript12.9 KB
scripts/drive-observability.mjsscript16.8 KB
scripts/drive-profile.mjsscript8.2 KB
scripts/drive-service-tiers.mjsexecutablescript20.1 KB
scripts/drive-skills.mjsscript27.2 KB
scripts/mcp-browser.mjsscript26.5 KB
scripts/mcp-canary.mjsscript18.8 KB
scripts/mcp-candidates.mjsscript3.5 KB
scripts/mcp-verification.test.mjsscript10.2 KB
scripts/messages-regressions.mjsscript15.8 KB
scripts/plan-verification.mjsscript5.2 KB
scripts/plan-verification.test.mjsscript6.1 KB
scripts/proof-cli.mjsscript11.1 KB
scripts/proof.mjsscript9.2 KB
scripts/proof.test.mjsscript10.4 KB

scripts/drive-models.mjs

View on GitHub
import fs from 'node:fs/promises'
import path from 'node:path'
import { createRequire } from 'node:module'
import { fileURLToPath } from 'node:url'

import { createProofRecorder } from './proof.mjs'

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..')
const requireFromAdminUi = createRequire(path.join(repoRoot, 'crates/admin-ui/web/package.json'))
const { chromium } = requireFromAdminUi('playwright')

const baseURL = requiredEnv('OCEANS_VERIFY_BASE_URL')
const evidenceDir = requiredEnv('OCEANS_VERIFY_EVIDENCE_DIR')
requiredEnv('OCEANS_VERIFY_GATEWAY_VERSION') // recorded in the proof envelope
const email = requiredEnv('OCEANS_VERIFY_ADMIN_EMAIL')
const password = requiredEnv('OCEANS_VERIFY_ADMIN_PASSWORD')
const targetModelId = 'gpt-6-astra'
const actions = []
const proof = createProofRecorder({ evidenceDir, file: 'models-proof.json', featureIds: ['models'] })

await fs.mkdir(evidenceDir, { recursive: true })
const browser = await chromium.launch({ headless: true })
let page

try {
  const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
  page = await context.newPage()
  page.on('console', (message) => console.log(`browser console ${message.type()}: ${message.text()}`))
  page.on('pageerror', (error) => console.log(`browser page error: ${error.message}`))
  page.on('requestfailed', (request) =>
    console.log(`browser request failed: ${request.method()} ${request.url()} ${request.failure()?.errorText ?? ''}`),
  )

  await page.goto(`${baseURL}/admin/api-keys`, { waitUntil: 'domcontentloaded' })
  await page.getByRole('heading', { name: 'Sign in' }).waitFor()
  const signInButton = page.getByRole('button', { name: 'Sign in' })
  try {
    await signInButton.waitFor({ state: 'visible', timeout: 60_000 })
    await page.waitForFunction(() => {
      const button = Array.from(document.querySelectorAll('button')).find(
        (candidate) => candidate.textContent?.trim() === 'Sign in',
      )
      return button instanceof HTMLButtonElement && !button.disabled
    }, null, { timeout: 60_000 })
  } catch {
    console.log('Sign-in did not hydrate after the first Vite load; reloading once.')
    await page.reload({ waitUntil: 'domcontentloaded' })
    await page.getByRole('heading', { name: 'Sign in' }).waitFor()
    await page.waitForFunction(() => {
      const button = Array.from(document.querySelectorAll('button')).find(
        (candidate) => candidate.textContent?.trim() === 'Sign in',
      )
      return button instanceof HTMLButtonElement && !button.disabled
    }, null, { timeout: 60_000 })
  }
  actions.push({ action: 'open protected admin UI', result: page.url() })
  proof.pass('protected-route-redirects-to-sign-in', { observed: page.url() })
  await capture(page, '01-login')

  await page.getByLabel('Email').fill(email)
  await page.getByLabel('Password', { exact: true }).fill(password)
  await Promise.all([
    page.waitForURL(/\/admin\/(?:api-keys|models)(?:\?|$)/),
    signInButton.click(),
  ])
  actions.push({ action: 'sign in with seeded platform admin', result: page.url() })
  proof.pass('sign-in-returns-to-protected-route', { observed: page.url() })

  const modelsLink = page.getByRole('link', { name: 'Models' }).first()
  await modelsLink.waitFor()
  await Promise.all([page.waitForURL(/\/admin\/models(?:\?|$)/), modelsLink.click()])

  await page.getByRole('heading', { name: 'Models', exact: true }).waitFor()
  await page.getByRole('heading', { name: 'Model list' }).waitFor()
  await page.getByTestId('models-desktop-table').waitFor()
  await page.getByTestId(`models-desktop-cell-${targetModelId}`).waitFor()
  await page.getByText(/^Benchmark scores by Artificial Analysis, retrieved via OpenRouter\.$/).waitFor()
  proof.pass('benchmark-attribution-visible')
  const pagination = page.getByRole('navigation', { name: 'Model pagination' })
  await pagination.getByRole('combobox', { name: 'Rows per page' }).waitFor()
  const rangeStatus = pagination.getByRole('status')
  await rangeStatus.filter({ hasText: /^\d+–\d+ of \d+$/ }).waitFor()
  const showingText = (await rangeStatus.textContent())?.trim()
  if (!showingText) throw new Error('The Models page did not report a visible model count.')
  const match = /^(\d+)–(\d+) of (\d+)$/.exec(showingText)
  if (!match) throw new Error(`Unexpected Models pager text: ${showingText}`)
  const firstShown = Number(match[1])
  const lastShown = Number(match[2])
  const displayedCount = lastShown === 0 ? 0 : lastShown - firstShown + 1
  const totalCount = Number(match[3])
  const renderedCount = await page.locator('[data-testid^="models-desktop-cell-"]').count()
  if (displayedCount !== renderedCount) {
    throw new Error(`UI reported ${displayedCount} displayed models but rendered ${renderedCount} model rows.`)
  }
  proof.pass('displayed-count-matches-rendered-rows', { observed: renderedCount, expected: displayedCount })
  actions.push({ action: 'follow Models sidebar link', result: showingText })
  await capture(page, '02-models')

  const adminModels = await page.evaluate(async () => {
    const response = await fetch('/api/v1/admin/models?page=1&page_size=100&include_aliases=false')
    if (!response.ok) throw new Error(`admin models request returned ${response.status}`)
    return response.json()
  })
  const apiCount = adminModels.data.total
  if (apiCount !== totalCount) {
    throw new Error(`UI total model count ${totalCount} did not match admin API count ${apiCount}.`)
  }
  proof.pass('total-count-matches-admin-api', { observed: totalCount, expected: apiCount })
  const apiModel = adminModels.data.items.find((model) => model.id === targetModelId)
  if (!apiModel) throw new Error(`The admin models API omitted ${targetModelId}.`)

  const modelCell = page.getByTestId(`models-desktop-cell-${targetModelId}`)
  const modelRow = modelCell.locator('xpath=ancestor::tr')
  const intelligenceHeader = page.getByRole('columnheader', { name: /^Intelligence Index/ })
  await intelligenceHeader.waitFor()
  const intelligenceColumn = await intelligenceHeader.evaluate((header) =>
    Array.from(header.parentElement.children).indexOf(header),
  )
  const intelligenceScore = apiModel.benchmark_scores.find(
    (score) => score.metric_key === 'artificial_analysis_intelligence_index',
  )
  const intelligenceValue = intelligenceScore
    ? new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 }).format(intelligenceScore.value)
    : '—'
  await modelRow.getByRole('cell').nth(intelligenceColumn).getByText(intelligenceValue, { exact: true }).waitFor()
  actions.push({ action: 'inspect always-visible Intelligence Index', result: intelligenceValue })
  proof.pass('intelligence-index-matches-admin-api', { observed: intelligenceValue })
  await modelRow.getByRole('button', { name: 'Info' }).click()
  const infoDialog = page.getByRole('dialog', { name: 'Model info' })
  await infoDialog.getByText(targetModelId, { exact: true }).first().waitFor()
  const infoSections = infoDialog.getByRole('navigation', { name: 'Model info sections' })
  const infoSectionNames = ['Overview', 'Routing', 'Benchmarks', 'Access']
  for (const name of infoSectionNames) {
    await infoSections.getByRole('button', { name, exact: true }).click()
    await infoDialog.getByRole('heading', { name, exact: true }).waitFor()
    if (name === 'Overview') {
      await infoDialog.getByRole('heading', { name: 'Pricing and limits', exact: true }).waitFor()
      await infoDialog.getByTestId('model-pricing-costs').waitFor()
    }
    if (name === 'Benchmarks') {
      if (apiModel.benchmark_scores.length === 0) {
        await infoDialog.getByText('No benchmark data is available for this model.').waitFor()
      } else {
        for (const score of apiModel.benchmark_scores) {
          const scoreLabel = infoDialog.getByText(score.label, { exact: true })
          await scoreLabel.waitFor()
          const sourceLink = scoreLabel.locator('xpath=..').locator(
            `a[href="${score.source_url}"]`,
            { hasText: `${score.source_model_id} on OpenRouter` },
          )
          await sourceLink.waitFor()
        }
        await infoDialog.getByRole('link', { name: 'Artificial Analysis', exact: true }).waitFor()
      }
      await capture(page, '03-model-benchmarks')
    }
  }
  actions.push({ action: `inspect ${targetModelId} model info`, result: 'All platform-admin sections visible' })
  proof.pass('model-info-sections-visible', { observed: infoSectionNames.length })
  proof.pass('benchmarks-match-admin-api', { observed: apiModel.benchmark_scores.length })
  await capture(page, '03-model-info')
  await page.keyboard.press('Escape')
  await infoDialog.waitFor({ state: 'hidden' })

  await page.getByRole('button', { name: 'Columns', exact: true }).click()
  await page.getByRole('checkbox', { name: /^Context window/ }).check()
  await page.getByRole('columnheader', { name: 'Context window', exact: true }).waitFor()
  await page.getByRole('checkbox', { name: /^Capabilities/ }).check()
  await page.getByRole('columnheader', { name: 'Capabilities', exact: true }).waitFor()
  await page.keyboard.press('Escape')
  await intelligenceHeader.waitFor()
  actions.push({ action: 'enable optional model columns', result: 'Context window and Capabilities visible; Intelligence Index remains visible' })
  proof.pass('optional-columns-render')
  await capture(page, '04-model-columns')

  const configRow = page
    .getByTestId(`models-desktop-cell-${targetModelId}`)
    .locator('xpath=ancestor::tr')
  await configRow
    .getByRole('button', { name: `Generate client config for ${targetModelId}`, exact: true })
    .click()
  const configDialog = page.getByRole('dialog', { name: 'Client config' })
  await configDialog.waitFor()
  await configDialog.getByText(`${targetModelId} via`, { exact: false }).waitFor()
  const clientConfigs = await verifyClientConfigs(page, configDialog)
  actions.push({ action: `generate ${targetModelId} client config`, result: 'Client config dialog visible' })
  proof.pass('client-configs-match-api', { observed: clientConfigs.length })
  await capture(page, '05-model-client-config')

  await proof.write({ details: {
    entryUrl: `${baseURL}/admin/api-keys`,
    finalUrl: page.url(),
    modelId: targetModelId,
    displayedCount,
    renderedCount,
    totalCount,
    apiCount,
    benchmarkScoreCount: apiModel.benchmark_scores.length,
    clientConfigs,
    actions,
  } })
  console.log(`models proof passed: ${displayedCount} displayed and ${totalCount} total UI models matched the rendered rows and API total`)
  console.log(`evidence: ${evidenceDir}`)
} catch (error) {
  if (page && !page.isClosed()) {
    await capture(page, '99-models-failure').catch(() => {})
  }
  await proof.write({ error, details: { actions } })
  throw error
} finally {
  await browser.close()
}

async function verifyClientConfigs(page, dialog) {
  const response = await page.evaluate(async (modelId) => {
    const result = await fetch('/api/v1/admin/models/client-configs', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ model_keys: [modelId] }),
    })
    if (!result.ok) throw new Error(`Client configuration API returned ${result.status}`)
    return result.json()
  }, targetModelId)
  const configs = response.data.client_configurations
  if (configs.length === 0) throw new Error('Client configuration API returned no configurations.')
  const choices = dialog.getByRole('radiogroup', { name: 'Client config', exact: true })
  const results = []
  for (const config of configs) {
    await choices.getByRole('radio', { name: config.label, exact: true }).check()
    for (const block of config.blocks) {
      const rendered = dialog.locator('[data-slot="code-block"]').filter({
        has: page.getByText(block.filename, { exact: true }),
      })
      await rendered.waitFor()
      const content = await rendered.locator('[data-slot="code-block-line"]').evaluateAll((lines) =>
        lines.map((line) => line.lastElementChild.textContent.replace(/\u200b/g, '')).join('\n'),
      )
      if (content !== block.content.replace(/\r\n?/g, '\n')) {
        throw new Error(`Rendered ${config.label} ${block.filename} did not match the configuration API.`)
      }
    }
    await capture(page, `05-model-client-config-${config.key}`)
    results.push({ key: config.key, label: config.label, files: config.blocks.map((block) => block.filename), matchesApi: true })
  }
  await choices.getByRole('radio', { name: configs[0].label, exact: true }).check()
  actions.push({ action: 'compare rendered client configurations with production API', result: `${results.length} configurations matched` })
  return results
}

async function capture(page, name) {
  await page.screenshot({ path: path.join(evidenceDir, `${name}.png`), fullPage: true })
  const snapshot = await page.locator('body').ariaSnapshot()
  await fs.writeFile(path.join(evidenceDir, `${name}.aria.txt`), `${snapshot}\n`)
  proof.artifact(`${name}.png`)
}

function requiredEnv(name) {
  const value = process.env[name]
  if (!value) throw new Error(`Missing required environment variable ${name}`)
  return value
}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall