ahstn/oceans-llm / .agents/skills/verify-oceans-admin
verify-oceans-admin
VerifiedVerify the Oceans LLM admin control plane and backend gateway against the real local stack, seeded demo data, and bounded live providers when required. Use for user-path checks of sign-in, models, API keys, observability, usage costs, spend controls, batches, agent sessions, request logs, Anthropic Messages thinking and effort compatibility, MCP registry and tool access, OpenRouter routing, guardrails, and Skills storage.
scripts/drive-mcp.mjs
View on GitHub#!/usr/bin/env node
import fs from "node:fs/promises";
import path from "node:path";
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";
import { registerServer, verifyDiscoveryFailure, disableFromMenu, enableFromMenu, verifyWorkbench, createAccess, attemptScope, loadStackEnv, mcpEndpointFor } from "./mcp-browser.mjs";
import { runMcpCanaries, verifyNoMcpGrant, testRevokedKey } from "./mcp-canary.mjs";
import { validateCandidates } from "./mcp-candidates.mjs";
import { createProofRecorder } from "./proof.mjs";
const repoRoot = process.env.OCEANS_VERIFY_REPO_ROOT ?? path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../..");
const requireFromAdminUi = createRequire(path.join(repoRoot, "crates/admin-ui/web/package.json"));
const { chromium } = requireFromAdminUi("playwright");
const { expect } = requireFromAdminUi("playwright/test");
const baseURL = requiredEnv("OCEANS_VERIFY_BASE_URL");
const gateway = new URL(baseURL);
if (!["127.0.0.1", "localhost", "[::1]"].includes(gateway.hostname) ||
!["http:", "https:"].includes(gateway.protocol) || gateway.username || gateway.password ||
gateway.search || gateway.hash || gateway.pathname !== "/") {
throw new Error("MCP verification requires a local gateway origin");
}
const evidenceDir = requiredEnv("OCEANS_VERIFY_EVIDENCE_DIR");
const runId = requiredEnv("OCEANS_VERIFY_RUN_ID");
// Replaces any earlier proof with a failing placeholder before any fallible setup.
const recorder = createProofRecorder({ evidenceDir, file: "mcp-proof.json", featureIds: ["mcp"] });
const config = JSON.parse(await fs.readFile(requiredEnv("OCEANS_VERIFY_MCP_CANDIDATES_FILE"), "utf8"));
validateCandidates(config);
const actions = [];
const failures = [];
const owned = { servers: [], toolsets: [], grants: [], apiKeyId: null, apiKeyName: null, rawKey: null };
requiredEnv("OCEANS_VERIFY_GATEWAY_VERSION"); // recorded in the proof envelope
// The gateway inherited its environment at launch, so credential aliases come from that record.
const stackEnv = await loadStackEnv(process.env.OCEANS_VERIFY_STACK_ENV);
const expectedGatewayUrl = config.expected_gateway_url ?? mcpEndpointFor(stackEnv.clientConfigBaseUrl ?? baseURL);
const scope = attemptScope(runId);
const proof = {
entryPoint: `${baseURL}/admin/mcp`, candidates: [], actions, failures, attemptScope: scope.key, aliasSource: stackEnv.source,
authenticationScope: "Configured gateway static header or bearer token; OAuth consent is not exercised.",
};
await fs.mkdir(evidenceDir, { recursive: true });
const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
const page = await context.newPage();
page.setDefaultTimeout(20_000);
const ui = { page, baseURL, expect, adminJson, poll, capture, actions, owned, scope, expectedGatewayUrl };
try {
await page.goto(`${baseURL}/admin/mcp`, { waitUntil: "domcontentloaded" });
await page.getByRole("heading", { name: "Sign in", exact: true }).waitFor();
await capture(page, "01-mcp-login");
await page.getByLabel("Email").fill(requiredEnv("OCEANS_VERIFY_ADMIN_EMAIL"));
await page.getByLabel("Password", { exact: true }).fill(requiredEnv("OCEANS_VERIFY_ADMIN_PASSWORD"));
await expect(page.getByRole("button", { name: "Sign in", exact: true })).toBeEnabled();
await page.getByRole("button", { name: "Sign in", exact: true }).click();
await page.getByRole("region", { name: "MCP server registry" }).waitFor();
await capture(page, "02-mcp-registry-entry");
actions.push({ action: "Open protected MCP registry and sign in", result: "Browser session authenticated" });
// The unreachable server contacts no upstream, so these run whatever credentials are set.
const unreachable = await recordStep("discovery-failure-feedback", () => verifyDiscoveryFailure(ui));
const disabled = unreachable && await recordStep("disable-from-menu", () => disableFromMenu(ui, unreachable));
if (!unreachable) recorder.add({ id: "disable-from-menu", status: "fail", note: "Not run: no unreachable server to disable" });
if (disabled) await recordStep("enable-from-menu", () => enableFromMenu(ui, unreachable));
else recorder.add({ id: "enable-from-menu", status: "fail", note: "Not run: the unreachable server was not disabled" });
proof.discoveryFailure = unreachable ? { serverId: unreachable.server.id, serverUrl: "https://unreachable.invalid/mcp" } : null;
const ready = [];
for (const candidate of config.candidates) {
const result = { key: candidate.key, label: candidate.label, authMode: candidate.auth_mode, required: candidate.required !== false };
proof.candidates.push(result);
// A missing alias at launch means the running gateway cannot authenticate discovery.
const credential = candidate.auth_config?.secret_ref?.replace(/^env\//, "");
if (credential && !stackEnv.aliases.has(credential)) {
Object.assign(result, { discovery: "blocked", failure: `${credential} was not set for the launched gateway; export it before launch (for example from fnox get), then relaunch` });
recorder.add({ id: `candidate-${candidate.key}`, status: "blocked", required: result.required, note: result.failure });
continue;
}
try {
const registered = await registerServer(ui, candidate);
if (registered.discoveryRejected) {
Object.assign(result, { discovery: "rejected as expected", serverId: registered.server.id, toolCount: 0, upstreamHttpStatus: registered.upstreamHttpStatus });
recorder.pass(`candidate-${candidate.key}`, { required: result.required, note: `upstream rejected discovery with HTTP ${registered.upstreamHttpStatus}` });
continue;
}
ready.push(registered);
Object.assign(result, { discovery: "passed", serverId: registered.server.id, toolCount: registered.toolCount, selectedTool: registered.tool.upstream_name });
recorder.pass(`candidate-${candidate.key}`, { required: result.required, observed: registered.toolCount });
} catch (error) {
result.discovery = "failed";
result.failure = error.name === "TimeoutError" ? "UI or discovery timed out" : "Registration or discovery contract failed";
failures.push({ candidate: candidate.key, phase: "discovery", required: result.required, reason: result.failure });
recorder.add({ id: `candidate-${candidate.key}`, status: "fail", required: result.required, note: result.failure });
// Error strings and screenshots of failed provider toasts can contain upstream content.
console.error(`MCP candidate ${candidate.key}: ${result.failure}`);
result.failureLocation = safeLocation(error);
}
}
if (ready.length > 0) {
const workbench = await verifyWorkbench(ui, ready);
proof.workbench = workbench.proof;
recorder.pass("toolset-workbench");
const key = await createAccess(ui, workbench.primary, ready, verifyNoMcpGrant);
proof.access = key.proof;
recorder.pass("grant-enforcement");
const canaries = await runMcpCanaries({ page, baseURL, rawKey: owned.rawKey, apiKeyId: owned.apiKeyId, candidates: ready, adminJson, actions, capture });
proof.canaries = canaries;
for (const failed of canaries.failures ?? []) {
const required = config.candidates.find((candidate) => candidate.key === failed.candidate)?.required !== false;
failures.push({ ...failed, required });
recorder.add({ id: `tool-call-${failed.candidate}`, status: "fail", required, note: failed.reasonClass });
}
if (!canaries.failures?.length) recorder.pass("tool-calls-and-invocations", { observed: ready.length });
} else if (!proof.candidates.some((result) => result.discovery === "blocked")) {
throw new Error("No MCP candidate completed discovery");
}
} catch (error) {
const reason = error.name === "TimeoutError" ? "Browser action timed out" : "MCP verification contract failed";
failures.push({ phase: "workflow", required: true, reason, location: safeLocation(error) });
recorder.add({ id: "workflow", status: "fail", note: `${reason} at ${safeLocation(error).join(", ")}` });
console.error("MCP workflow failed; sanitized phase evidence will be retained.");
} finally {
proof.cleanup = await cleanup();
owned.rawKey = null;
await browser.close();
recorder.add({ id: "cleanup", status: proof.cleanup.passed ? "pass" : "fail" });
}
const envelope = await recorder.write({ details: proof });
console.log(`MCP verification ${envelope.verdict}; evidence: ${evidenceDir}`);
if (envelope.verdict !== "pass") process.exitCode = 1;
async function cleanup() {
const results = [];
async function attempt(label, operation) {
try { await operation(); results.push({ action: label, passed: true }); }
catch { results.push({ action: label, passed: false }); }
}
// Use only production APIs and IDs created by this run; cleanup must survive an open dialog.
for (const grant of owned.grants) await attempt("Revoke owned MCP grant", () => adminJson(page, "/api/v1/admin/mcp/grants", { method: "DELETE", headers: { "content-type": "application/json" }, body: JSON.stringify(grant) }));
if (!owned.apiKeyId && owned.apiKeyName) await attempt("Resolve pending temporary key creation", async () => {
const items = (await adminJson(page, "/api/v1/admin/api-keys")).items;
owned.apiKeyId = items.find((item) => item.name === owned.apiKeyName)?.id ?? null;
});
if (owned.apiKeyId) {
await attempt("Revoke temporary API key", () => adminJson(page, `/api/v1/admin/api-keys/${owned.apiKeyId}/revoke`, { method: "POST" }));
if (owned.rawKey) await attempt("Confirm revoked key receives HTTP 401", () => testRevokedKey({ baseURL, rawKey: owned.rawKey, candidates: owned.servers.map((server) => ({ server })) }));
}
for (const set of owned.toolsets) await attempt("Disable owned tool set", async () => {
if (!set.id) set.id = (await adminJson(page, "/api/v1/admin/mcp/toolsets?include_disabled=true")).items.find((item) => item.toolset_key === set.toolset_key)?.id;
if (!set.id) return;
await adminJson(page, `/api/v1/admin/mcp/toolsets/${set.id}/disable`, { method: "POST" });
const saved = (await adminJson(page, "/api/v1/admin/mcp/toolsets?include_disabled=true")).items.find((item) => item.id === set.id);
if (saved?.status !== "disabled") throw new Error("Tool set remains active");
});
for (const server of owned.servers) await attempt("Disable owned MCP server", async () => {
if (!server.id) server.id = (await adminJson(page, "/api/v1/admin/mcp/servers?include_disabled=true")).items.find((item) => item.server_key === server.server_key)?.id;
if (!server.id) return;
await adminJson(page, `/api/v1/admin/mcp/servers/${server.id}/disable`, { method: "POST" });
const saved = (await adminJson(page, "/api/v1/admin/mcp/servers?include_disabled=true")).items.find((item) => item.id === server.id);
if (saved?.status !== "disabled") throw new Error("Server remains active");
});
return { passed: results.every((result) => result.passed), recordsRetainedDisabled: true, actions: results };
}
// Records one UI step as a required check without aborting the rest of the run.
async function recordStep(id, step) {
try {
const result = await step();
recorder.pass(id);
return result ?? true;
} catch (error) {
const reason = error.name === "TimeoutError" ? "UI or discovery timed out" : "UI contract failed";
failures.push({ phase: id, required: true, reason, location: safeLocation(error) });
recorder.add({ id, status: "fail", note: `${reason} at ${safeLocation(error).join(", ")}` });
return null;
}
}
async function adminJson(currentPage, requestPath, options) {
return currentPage.evaluate(async ({ requestPath, options }) => {
const response = await fetch(requestPath, { ...options, signal: AbortSignal.timeout(45_000) });
if (!response.ok) throw new Error(`Admin API returned HTTP ${response.status}`);
const body = await response.json();
return body.data;
}, { requestPath, options });
}
async function capture(currentPage, name) {
// Key creation is never captured while its one-time secret is visible.
if (await currentPage.getByTestId("new-api-key-raw-key").count()) throw new Error("Refusing to capture a displayed API key");
await currentPage.screenshot({ path: path.join(evidenceDir, `${name}.png`), fullPage: true });
await fs.writeFile(path.join(evidenceDir, `${name}.aria.txt`), `${await currentPage.locator("body").ariaSnapshot()}\n`);
recorder.artifact(`${name}.png`);
}
async function poll(operation, label, timeout = 45_000) {
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
const result = await operation();
if (result) return result;
await new Promise((resolve) => setTimeout(resolve, 250));
}
throw new Error(`Timed out waiting for ${label}`);
}
function requiredEnv(name) {
const value = process.env[name];
if (!value) throw new Error(`Missing required environment variable ${name}`);
return value;
}
function safeLocation(error) {
return [...(error.stack ?? "").matchAll(/\b(?:drive-mcp|mcp-browser|mcp-canary)\.mjs:\d+:\d+/g)].slice(0, 3).map((match) => match[0]);
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |