Skip to content

ahstn/oceans-llm / .agents/skills/verify-oceans-admin

verify-oceans-admin

Verified

Verify the Oceans LLM admin control plane and backend gateway against the real local stack, seeded demo data, and bounded live providers when required. Use for user-path checks of sign-in, models, API keys, observability, usage costs, spend controls, batches, agent sessions, request logs, Anthropic Messages thinking and effort compatibility, MCP registry and tool access, OpenRouter routing, guardrails, and Skills storage.

FileKindSize
SKILL.mdskill md29.0 KB
examples/mcp-candidates.jsonconfig3.0 KB
features/README.mdmarkdown10.0 KB
features/account-onboarding.mdmarkdown4.3 KB
features/agent-harnesses.mdmarkdown4.3 KB
features/agent-plugins.mdmarkdown1005 B
features/agent-sessions.mdmarkdown3.9 KB
features/anthropic-messages.mdmarkdown20.7 KB
features/api-keys.mdmarkdown6.5 KB
features/attribution.mdmarkdown13.9 KB
features/audit-log.mdmarkdown4.9 KB
features/backend-gateway.mdmarkdown5.2 KB
features/batch-requests.mdmarkdown4.2 KB
features/design-system.mdmarkdown4.6 KB
features/guardrails.mdmarkdown7.0 KB
features/harness-integration.mdmarkdown7.8 KB
features/identity.mdmarkdown4.2 KB
features/image-generation.mdmarkdown7.3 KB
features/leaderboard.mdmarkdown3.3 KB
features/live-llm-requests.mdmarkdown12.6 KB
features/mcp-client-oauth.mdmarkdown9.9 KB
features/mcp-ownership.mdmarkdown7.4 KB
features/mcp.mdmarkdown19.6 KB
features/models.mdmarkdown8.2 KB
features/password-sign-in.mdmarkdown8.3 KB
features/profile.mdmarkdown2.5 KB
features/rate-limits.mdmarkdown6.0 KB
features/request-logs.mdmarkdown8.6 KB
features/review-agent.mdmarkdown3.7 KB
features/routing.jsonconfig33.6 KB
features/service-tiers.mdmarkdown6.3 KB
features/skills.mdmarkdown9.8 KB
features/spend-controls.mdmarkdown7.3 KB
features/usage-costs.mdmarkdown3.7 KB
features/workspace-connections.mdmarkdown6.1 KB
scripts/check-messages.mjsexecutablescript2.4 KB
scripts/check-messages.test.mjsscript3.0 KB
scripts/control-oceans-adminexecutablescript25.3 KB
scripts/control-oceans-admin.test.mjsscript7.6 KB
scripts/drive-backend-gateway.mjsexecutablescript12.5 KB
scripts/drive-harness-integration.mjsscript6.6 KB
scripts/drive-mcp.mjsscript13.0 KB
scripts/drive-models.mjsscript12.9 KB
scripts/drive-observability.mjsscript16.8 KB
scripts/drive-profile.mjsscript8.2 KB
scripts/drive-service-tiers.mjsexecutablescript20.1 KB
scripts/drive-skills.mjsscript27.2 KB
scripts/mcp-browser.mjsscript26.5 KB
scripts/mcp-canary.mjsscript18.8 KB
scripts/mcp-candidates.mjsscript3.5 KB
scripts/mcp-verification.test.mjsscript10.2 KB
scripts/messages-regressions.mjsscript15.8 KB
scripts/plan-verification.mjsscript5.2 KB
scripts/plan-verification.test.mjsscript6.1 KB
scripts/proof-cli.mjsscript11.1 KB
scripts/proof.mjsscript9.2 KB
scripts/proof.test.mjsscript10.4 KB

scripts/drive-observability.mjs

View on GitHub
import fs from "node:fs/promises";
import path from "node:path";
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";

import { createProofRecorder } from "./proof.mjs";

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../..");
const requireFromAdminUi = createRequire(path.join(repoRoot, "crates/admin-ui/web/package.json"));
const { chromium } = requireFromAdminUi("playwright");

const baseURL = requiredEnv("OCEANS_VERIFY_BASE_URL");
const evidenceDir = requiredEnv("OCEANS_VERIFY_EVIDENCE_DIR");
requiredEnv("OCEANS_VERIFY_GATEWAY_VERSION"); // recorded in the proof envelope
const email = requiredEnv("OCEANS_VERIFY_ADMIN_EMAIL");
const password = requiredEnv("OCEANS_VERIFY_ADMIN_PASSWORD");
const actions = [];
const proof = createProofRecorder({
  evidenceDir,
  file: "observability-proof.json",
  featureIds: ["leaderboard", "agent-harnesses"],
});
const currencyFormatter = new Intl.NumberFormat("en-US", {
  style: "currency",
  currency: "USD",
});
const numberFormatter = new Intl.NumberFormat("en-US");

await fs.mkdir(evidenceDir, { recursive: true });
const browser = await chromium.launch({ headless: true });

try {
  const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
  const page = await context.newPage();
  page.on("console", (message) =>
    console.log(`browser console ${message.type()}: ${message.text()}`),
  );
  page.on("pageerror", (error) => console.log(`browser page error: ${error.message}`));
  page.on("requestfailed", (request) =>
    console.log(
      `browser request failed: ${request.method()} ${request.url()} ${request.failure()?.errorText ?? ""}`,
    ),
  );

  const entryUrl = `${baseURL}/admin/observability/leaderboard`;
  await page.goto(entryUrl, { waitUntil: "domcontentloaded" });
  await waitForSignIn(page);
  actions.push({ action: "open protected leaderboard", result: page.url() });
  await capture(page, "01-observability-login");

  await page.getByLabel("Email").fill(email);
  await page.getByLabel("Password", { exact: true }).fill(password);
  await Promise.all([
    page.waitForURL(/\/admin\/observability\/leaderboard(?:\?|$)/),
    page.getByRole("button", { name: "Sign in" }).click(),
  ]);
  actions.push({ action: "sign in with seeded platform admin", result: page.url() });
  proof.pass("sign-in-returns-to-leaderboard", { observed: page.url() });

  await page.getByRole("heading", { name: "Leaderboard", exact: true }).waitFor();
  await page.getByTestId("leaderboard-table").waitFor();
  const leaderboard7d = await fetchView(page, "/api/v1/admin/observability/leaderboard?range=7d");
  assertEqual(leaderboard7d.range, "7d", "leaderboard API range");
  const leaderboard7dTable = await assertLeaderboard(page, leaderboard7d);
  await assertChartSeries(page, leaderboard7d.chart_users.length);
  proof.pass("leaderboard-7d-matches-api", { observed: leaderboard7d.leaders.length });
  actions.push({
    action: "compare 7d leaderboard with production admin API",
    result: `${leaderboard7d.leaders.length} rendered leaders matched`,
  });
  await capture(page, "02-leaderboard-7d");

  const leaderboard31d = await fetchView(page, "/api/v1/admin/observability/leaderboard?range=31d");
  assertEqual(leaderboard31d.range, "31d", "31d leaderboard API range");
  await selectRange(page, "leaderboard-table", "User", leaderboard31d.leaders, (leader) => ({
    key: leader.user_name,
    requests: leader.total_requests,
  }));
  const leaderboard31dTable = await assertLeaderboard(page, leaderboard31d);
  await assertChartSeries(page, leaderboard31d.chart_users.length);
  proof.pass("leaderboard-31d-matches-api", { observed: leaderboard31d.leaders.length });
  actions.push({
    action: "select leaderboard Last 31 days",
    result: `${leaderboard31d.leaders.length} rendered leaders matched`,
  });
  await capture(page, "03-leaderboard-31d");
  await page.setViewportSize({ width: 600, height: 1000 });
  await page.getByTestId("leaderboard-mobile-list").waitFor({ state: "visible" });
  await page.getByTestId("leaderboard-table").waitFor({ state: "hidden" });
  proof.pass("leaderboard-mobile-list");
  actions.push({
    action: "verify responsive leaderboard presentation",
    result: "mobile list visible and desktop table hidden below md",
  });
  await capture(page, "03b-leaderboard-mobile");
  await page.setViewportSize({ width: 1440, height: 1000 });

  const harnessesLink = page.getByRole("link", { name: "Agent Harnesses" }).first();
  await Promise.all([
    page.waitForURL(/\/admin\/observability\/agent-harnesses(?:\?|$)/),
    harnessesLink.click(),
  ]);
  await page.getByRole("heading", { name: "Agent harnesses", exact: true }).waitFor();
  await page.getByTestId("harness-usage-table").waitFor();

  const harness7d = await fetchView(page, "/api/v1/admin/observability/harness-usage?range=7d");
  assertEqual(harness7d.range, "7d", "harness API range");
  const harness7dTable = await assertHarnessUsage(page, harness7d);
  await assertChartSeries(page, harness7d.chart_harnesses.length);
  const iconChecks = await assertHarnessIcons(page, harness7d);
  proof.pass("harnesses-7d-match-api", { observed: harness7d.leaders.length });
  proof.pass("harness-icons-render", { observed: "Mastra, Oh My Pi" });
  actions.push({
    action: "compare 7d agent harnesses with production admin API",
    result: `${harness7d.leaders.length} rendered harnesses matched with token values`,
  });
  actions.push({
    action: "inspect Mastra and Oh My Pi rendering",
    result: "Mastra and Oh My Pi have their harness icons; the Oh My Pi mark is white",
  });
  await capture(page, "04-agent-harnesses-7d");

  const harness31d = await fetchView(page, "/api/v1/admin/observability/harness-usage?range=31d");
  assertEqual(harness31d.range, "31d", "31d harness API range");
  await selectRange(page, "harness-usage-table", "Harness", harness31d.leaders, (leader) => ({
    key: leader.agent_harness_label,
    requests: leader.total_requests,
  }));
  const harness31dTable = await assertHarnessUsage(page, harness31d);
  await assertChartSeries(page, harness31d.chart_harnesses.length);
  proof.pass("harnesses-31d-match-api", { observed: harness31d.leaders.length });
  actions.push({
    action: "select agent harnesses Last 31 days",
    result: `${harness31d.leaders.length} rendered harnesses matched`,
  });
  await capture(page, "05-agent-harnesses-31d");

  await proof.write({ details: {
    entryUrl,
    finalUrl: page.url(),
    leaderboard: {
      sevenDays: proofProjection(leaderboard7d, leaderboard7dTable),
      thirtyOneDays: proofProjection(leaderboard31d, leaderboard31dTable),
    },
    harnessUsage: {
      sevenDays: proofProjection(harness7d, harness7dTable),
      thirtyOneDays: proofProjection(harness31d, harness31dTable),
    },
    iconChecks,
    actions,
  } });
  console.log(
    `observability proof passed: 7d and 31d leaderboard and harness rows matched the production API`,
  );
  console.log(`evidence: ${evidenceDir}`);
} catch (error) {
  await proof.write({ error, details: { actions } });
  throw error;
} finally {
  await browser.close();
}

async function waitForSignIn(page) {
  await page.getByRole("heading", { name: "Sign in" }).waitFor();
  try {
    await page.waitForFunction(
      () => {
        const button = Array.from(document.querySelectorAll("button")).find(
          (candidate) => candidate.textContent?.trim() === "Sign in",
        );
        return button instanceof HTMLButtonElement && !button.disabled;
      },
      null,
      { timeout: 60_000 },
    );
  } catch {
    console.log("Sign-in did not hydrate after the first Vite load; reloading once.");
    await page.reload({ waitUntil: "domcontentloaded" });
    await page.getByRole("heading", { name: "Sign in" }).waitFor();
    await page.waitForFunction(
      () => {
        const button = Array.from(document.querySelectorAll("button")).find(
          (candidate) => candidate.textContent?.trim() === "Sign in",
        );
        return button instanceof HTMLButtonElement && !button.disabled;
      },
      null,
      { timeout: 60_000 },
    );
  }
}

async function fetchView(page, requestPath) {
  return page.evaluate(async (pathName) => {
    const response = await fetch(pathName);
    if (!response.ok) throw new Error(`${pathName} returned ${response.status}`);
    const envelope = await response.json();
    return envelope.data;
  }, requestPath);
}

async function assertLeaderboard(page, view) {
  const table = await readTable(page, "leaderboard-table");
  const columns = requireHeaders(table, [
    "Rank",
    "User",
    "Total spend",
    "Most used model",
    "Most used harness",
    "Total requests",
    "Avg tools",
  ]);
  assertEqual(table.rows.length, view.leaders.length, "leaderboard rendered row count");

  for (const [index, leader] of view.leaders.entries()) {
    const cells = table.rows[index];
    assertEqual(cells[columns.Rank], String(leader.rank), `leaderboard rank ${index + 1}`);
    assertEqual(cells[columns.User], leader.user_name, `leaderboard user ${index + 1}`);
    assertEqual(
      cells[columns["Total spend"]],
      currencyFormatter.format(leader.total_spend_usd_10000 / 10_000),
      `leaderboard spend for ${leader.user_name}`,
    );
    assertEqual(
      cells[columns["Most used model"]],
      leader.most_used_model ?? "—",
      `leaderboard model for ${leader.user_name}`,
    );
    assertEqual(
      cells[columns["Most used harness"]],
      leader.most_used_harness?.label ?? "—",
      `leaderboard harness for ${leader.user_name}`,
    );
    assertEqual(
      cells[columns["Total requests"]],
      numberFormatter.format(leader.total_requests),
      `leaderboard requests for ${leader.user_name}`,
    );
    const averages = leader.tool_cardinality_averages;
    assertEqual(
      cells[columns["Avg tools"]],
      `MCP ${formatAverageCount(averages.referenced_mcp_server_count)}exposed ${formatAverageCount(averages.exposed_tool_count)}called ${formatAverageCount(averages.invoked_tool_count)}filtered ${formatAverageCount(averages.filtered_tool_count)}`,
      `leaderboard tool averages for ${leader.user_name}`,
    );
  }
  return table;
}

async function assertHarnessUsage(page, view) {
  const table = await readTable(page, "harness-usage-table");
  const columns = requireHeaders(table, [
    "Rank",
    "Harness",
    "Requests",
    "Input tokens",
    "Output tokens",
    "Total tokens",
    "Key",
  ]);
  assertEqual(table.rows.length, view.leaders.length, "harness rendered row count");

  for (const [index, leader] of view.leaders.entries()) {
    const cells = table.rows[index];
    assertEqual(cells[columns.Rank], String(index + 1), `harness rank ${index + 1}`);
    assertEqual(cells[columns.Harness], leader.agent_harness_label, `harness label ${index + 1}`);
    assertEqual(
      cells[columns.Requests],
      numberFormatter.format(leader.total_requests),
      `requests for ${leader.agent_harness_key}`,
    );
    assertEqual(
      cells[columns["Input tokens"]],
      formatTokenCount(leader.prompt_tokens),
      `input tokens for ${leader.agent_harness_key}`,
    );
    assertEqual(
      cells[columns["Output tokens"]],
      formatTokenCount(leader.completion_tokens),
      `output tokens for ${leader.agent_harness_key}`,
    );
    assertEqual(
      cells[columns["Total tokens"]],
      formatTokenCount(leader.total_tokens),
      `total tokens for ${leader.agent_harness_key}`,
    );
    assertEqual(cells[columns.Key], leader.agent_harness_key, `key for harness row ${index + 1}`);
  }
  return table;
}

async function assertHarnessIcons(page, view) {
  const mastraIndex = view.leaders.findIndex((leader) => leader.agent_harness_key === "mastra");
  const ompIndex = view.leaders.findIndex((leader) => leader.agent_harness_key === "oh_my_pi");
  if (mastraIndex < 0 || ompIndex < 0) {
    throw new Error("The 7d demo API must include both Mastra and Oh My Pi harness rows.");
  }

  const rows = page.getByTestId("harness-usage-table").locator("tbody tr");
  const mastraIconCount = await rows
    .nth(mastraIndex)
    .locator('[data-agent-harness-icon="mastra"]')
    .count();
  const ompIcon = rows.nth(ompIndex).locator('[data-agent-harness-icon="ohmypi"]');
  const ompIconCount = await ompIcon.count();
  const ompFill = await ompIcon.locator("path").getAttribute("fill");
  const ompGradientCount = await ompIcon.locator("linearGradient").count();
  assertEqual(mastraIconCount, 1, "Mastra icon count");
  assertEqual(ompIconCount, 1, "Oh My Pi icon count");
  if (!["#fff", "#ffffff", "white", "var(--color-white)"].includes(ompFill?.trim().toLowerCase())) {
    throw new Error(`Oh My Pi icon fill: expected white, got ${JSON.stringify(ompFill)}`);
  }
  const ompRenderedFill = await ompIcon
    .locator("path")
    .evaluate((path) => getComputedStyle(path).fill);
  assertEqual(ompRenderedFill, "rgb(255, 255, 255)", "Oh My Pi icon rendered fill");
  assertEqual(ompGradientCount, 0, "Oh My Pi gradient count");
  return { mastraIconCount, ompIconCount, ompFill, ompRenderedFill };
}

async function selectRange(page, tableTestId, keyHeader, leaders, projectLeader) {
  const rangeToggle = page.getByRole("radio", { name: "Last 31 days" });
  await rangeToggle.click();
  await page.waitForFunction(
    ({ expectedRows, expectedKeyHeader, expectedTableTestId }) => {
      const table = document.querySelector(`[data-testid="${expectedTableTestId}"]`);
      if (!table) return false;
      const headers = Array.from(table.querySelectorAll("thead th")).map((cell) =>
        (cell.textContent ?? "").replace(/\s+/g, " ").trim(),
      );
      const keyIndex = headers.indexOf(expectedKeyHeader);
      const requestIndex =
        headers.indexOf("Total requests") >= 0
          ? headers.indexOf("Total requests")
          : headers.indexOf("Requests");
      if (keyIndex < 0 || requestIndex < 0) return false;
      const rows = Array.from(table.querySelectorAll("tbody tr"));
      if (rows.length !== expectedRows.length) return false;
      return rows.every((row, index) => {
        const cells = Array.from(row.querySelectorAll("td")).map((cell) =>
          (cell.textContent ?? "").replace(/\s+/g, " ").trim(),
        );
        const requests = cells[requestIndex].replace(/,/g, "");
        return (
          cells[keyIndex] === expectedRows[index].key &&
          requests === String(expectedRows[index].requests)
        );
      });
    },
    {
      expectedRows: leaders.map(projectLeader),
      expectedKeyHeader: keyHeader,
      expectedTableTestId: tableTestId,
    },
    { timeout: 60_000 },
  );
  const checked = await rangeToggle.getAttribute("aria-checked");
  assertEqual(checked, "true", `${tableTestId} Last 31 days selection`);
}

async function assertChartSeries(page, expectedCount) {
  await page.waitForFunction(
    (expected) => document.querySelectorAll("g.recharts-area").length === expected,
    expectedCount,
    { timeout: 60_000 },
  );
}

async function readTable(page, testId) {
  return page.getByTestId(testId).evaluate((table) => ({
    headers: Array.from(table.querySelectorAll("thead th")).map((cell) =>
      (cell.textContent ?? "").replace(/\s+/g, " ").trim(),
    ),
    rows: Array.from(table.querySelectorAll("tbody tr")).map((row) =>
      Array.from(row.querySelectorAll("td")).map((cell) =>
        (cell.textContent ?? "").replace(/\s+/g, " ").trim(),
      ),
    ),
  }));
}

function requireHeaders(table, expectedHeaders) {
  const columns = {};
  for (const header of expectedHeaders) {
    const index = table.headers.indexOf(header);
    if (index < 0) throw new Error(`Required table header ${header} is missing.`);
    columns[header] = index;
  }
  return columns;
}

function proofProjection(view, renderedTable) {
  return {
    apiRange: view.range,
    windowStart: view.window_start,
    windowEnd: view.window_end,
    chartSeriesCount: view.chart_users?.length ?? view.chart_harnesses.length,
    seriesPointCount: view.series.length,
    apiLeaders: view.leaders,
    renderedHeaders: renderedTable.headers,
    renderedRows: renderedTable.rows,
  };
}

function formatTokenCount(value) {
  return value == null ? "n/a" : numberFormatter.format(value);
}

function formatAverageCount(value) {
  if (value == null) return "n/a";
  return Number.isInteger(value) ? String(value) : value.toFixed(1);
}

function compact(value) {
  return value.replace(/[\s,|·•—]/g, "");
}

function assertEqual(actual, expected, label) {
  if (actual !== expected) {
    throw new Error(
      `${label}: expected ${JSON.stringify(expected)}, received ${JSON.stringify(actual)}`,
    );
  }
}

async function capture(page, name) {
  await page.screenshot({ path: path.join(evidenceDir, `${name}.png`), fullPage: true });
  const snapshot = await page.locator("body").ariaSnapshot();
  await fs.writeFile(path.join(evidenceDir, `${name}.aria.txt`), `${snapshot}\n`);
  proof.artifact(`${name}.png`);
}

function requiredEnv(name) {
  const value = process.env[name];
  if (!value) throw new Error(`Missing required environment variable ${name}`);
  return value;
}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall