Skip to content

ahstn/oceans-llm / .agents/skills/verify-oceans-admin

verify-oceans-admin

Verified

Verify the Oceans LLM admin control plane and backend gateway against the real local stack, seeded demo data, and bounded live providers when required. Use for user-path checks of sign-in, models, API keys, observability, usage costs, spend controls, batches, agent sessions, request logs, Anthropic Messages thinking and effort compatibility, MCP registry and tool access, OpenRouter routing, guardrails, and Skills storage.

FileKindSize
SKILL.mdskill md29.0 KB
examples/mcp-candidates.jsonconfig3.0 KB
features/README.mdmarkdown10.0 KB
features/account-onboarding.mdmarkdown4.3 KB
features/agent-harnesses.mdmarkdown4.3 KB
features/agent-plugins.mdmarkdown1005 B
features/agent-sessions.mdmarkdown3.9 KB
features/anthropic-messages.mdmarkdown20.7 KB
features/api-keys.mdmarkdown6.5 KB
features/attribution.mdmarkdown13.9 KB
features/audit-log.mdmarkdown4.9 KB
features/backend-gateway.mdmarkdown5.2 KB
features/batch-requests.mdmarkdown4.2 KB
features/design-system.mdmarkdown4.6 KB
features/guardrails.mdmarkdown7.0 KB
features/harness-integration.mdmarkdown7.8 KB
features/identity.mdmarkdown4.2 KB
features/image-generation.mdmarkdown7.3 KB
features/leaderboard.mdmarkdown3.3 KB
features/live-llm-requests.mdmarkdown12.6 KB
features/mcp-client-oauth.mdmarkdown9.9 KB
features/mcp-ownership.mdmarkdown7.4 KB
features/mcp.mdmarkdown19.6 KB
features/models.mdmarkdown8.2 KB
features/password-sign-in.mdmarkdown8.3 KB
features/profile.mdmarkdown2.5 KB
features/rate-limits.mdmarkdown6.0 KB
features/request-logs.mdmarkdown8.6 KB
features/review-agent.mdmarkdown3.7 KB
features/routing.jsonconfig33.6 KB
features/service-tiers.mdmarkdown6.3 KB
features/skills.mdmarkdown9.8 KB
features/spend-controls.mdmarkdown7.3 KB
features/usage-costs.mdmarkdown3.7 KB
features/workspace-connections.mdmarkdown6.1 KB
scripts/check-messages.mjsexecutablescript2.4 KB
scripts/check-messages.test.mjsscript3.0 KB
scripts/control-oceans-adminexecutablescript25.3 KB
scripts/control-oceans-admin.test.mjsscript7.6 KB
scripts/drive-backend-gateway.mjsexecutablescript12.5 KB
scripts/drive-harness-integration.mjsscript6.6 KB
scripts/drive-mcp.mjsscript13.0 KB
scripts/drive-models.mjsscript12.9 KB
scripts/drive-observability.mjsscript16.8 KB
scripts/drive-profile.mjsscript8.2 KB
scripts/drive-service-tiers.mjsexecutablescript20.1 KB
scripts/drive-skills.mjsscript27.2 KB
scripts/mcp-browser.mjsscript26.5 KB
scripts/mcp-canary.mjsscript18.8 KB
scripts/mcp-candidates.mjsscript3.5 KB
scripts/mcp-verification.test.mjsscript10.2 KB
scripts/messages-regressions.mjsscript15.8 KB
scripts/plan-verification.mjsscript5.2 KB
scripts/plan-verification.test.mjsscript6.1 KB
scripts/proof-cli.mjsscript11.1 KB
scripts/proof.mjsscript9.2 KB
scripts/proof.test.mjsscript10.4 KB

scripts/drive-backend-gateway.mjs

View on GitHub
#!/usr/bin/env node

import fs from "node:fs/promises";
import path from "node:path";
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";

import { createProofRecorder, safeMessage } from "./proof.mjs";

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../..");
const requireFromAdminUi = createRequire(path.join(repoRoot, "crates/admin-ui/web/package.json"));
const { chromium } = requireFromAdminUi("playwright");

const baseURL = requiredEnv("OCEANS_VERIFY_BASE_URL");
const evidenceDir = requiredEnv("OCEANS_VERIFY_EVIDENCE_DIR");
requiredEnv("OCEANS_VERIFY_GATEWAY_VERSION"); // recorded in the proof envelope
const email = requiredEnv("OCEANS_VERIFY_ADMIN_EMAIL");
const password = requiredEnv("OCEANS_VERIFY_ADMIN_PASSWORD");
const model = "deepseek-v4.1-flash";
const upstreamModel = "deepseek/deepseek-v4.1-flash";
const syntheticCommand = "rm -rf /tmp/oceans-verify";
const gatewayRequestTimeoutMs = 30_000;
const actions = [];

await fs.mkdir(evidenceDir, { recursive: true });
// Replaces any earlier proof with a failing placeholder before the browser starts.
const proof = createProofRecorder({
  evidenceDir,
  file: "backend-gateway-canary-proof.json",
  featureIds: ["backend-gateway"],
});
const browser = await chromium.launch({ headless: true });
let apiKeyId;
let rawKey;
let page;
let details;
let failure;

try {
  const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
  page = await context.newPage();
  await page.goto(`${baseURL}/admin/api-keys`, { waitUntil: "domcontentloaded" });
  await waitForSignIn(page);
  await page.getByLabel("Email").fill(email);
  await page.getByLabel("Password", { exact: true }).fill(password);
  await Promise.all([
    page.waitForURL(/\/admin\/api-keys(?:\?|$)/),
    page.getByRole("button", { name: "Sign in" }).click(),
  ]);
  await page.getByRole("heading", { name: "API keys", exact: true }).waitFor();
  await capture(page, "01-backend-api-keys");

  const keyCatalog = await adminJson(page, "/api/v1/admin/api-keys");
  const owner =
    keyCatalog.users.find((user) => user.email === "alice@platform.local") ?? keyCatalog.users[0];
  const configuredModel = keyCatalog.models.find((entry) => entry.key === model);
  assert(owner, "API key catalog has no allowed user owner");
  assert(configuredModel, `gateway model ${model} is missing from API key grants`);

  const created = await adminJson(page, "/api/v1/admin/api-keys", {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({
      name: `Backend gateway verification ${Date.now()}`,
      owner_kind: "user",
      owner_user_id: owner.id,
      owner_team_id: null,
      owner_service_account_id: null,
      model_grant_mode: "explicit",
      model_keys: [model],
    }),
  });
  apiKeyId = created.api_key.id;
  rawKey = created.raw_key;
  assert(typeof rawKey === "string" && rawKey.startsWith("gwk_"), "temporary raw key is missing");
  actions.push({ action: "create temporary model-scoped key", result: model });
  proof.pass("temporary-key-created", { observed: model });

  const evaluationResponse = await fetch(`${baseURL}/api/v1/guardrails/evaluate`, {
    method: "POST",
    signal: AbortSignal.timeout(gatewayRequestTimeoutMs),
    headers: gatewayHeaders(rawKey),
    body: JSON.stringify({ tool_name: "bash", command: syntheticCommand }),
  });
  const evaluation = await responseJson(evaluationResponse, "direct guardrail evaluation");
  assertEqual(evaluation.action, "audit", "direct evaluation action");
  assertEqual(evaluation.allowed, true, "direct evaluation allowed state");
  assertEqual(evaluation.matched_rule?.pack_id, "core.filesystem", "direct evaluation pack");
  assertEqual(
    evaluation.matched_rule?.rule_id,
    "recursive-force-remove",
    "direct evaluation rule",
  );
  assertEqual(
    evaluation.reason_code,
    "filesystem.recursive_force_remove",
    "direct evaluation reason",
  );
  proof.pass("direct-guardrail-audit", { observed: "core.filesystem/recursive-force-remove" });
  actions.push({
    action: "evaluate synthetic destructive command without execution",
    result: "core.filesystem/recursive-force-remove audited",
  });

  const liveResponse = await fetch(`${baseURL}/v1/chat/completions`, {
    method: "POST",
    signal: AbortSignal.timeout(gatewayRequestTimeoutMs),
    headers: {
      ...gatewayHeaders(rawKey),
      "x-oceans-service": "verification",
      "x-oceans-component": "backend-gateway",
      "x-oceans-env": "local",
    },
    body: JSON.stringify({
      model,
      messages: [
        {
          role: "user",
          content: `Call bash exactly once with command: ${syntheticCommand}`,
        },
      ],
      tools: [
        {
          type: "function",
          function: {
            name: "bash",
            description: "Return a shell command for review without executing it",
            parameters: {
              type: "object",
              properties: {
                command: { type: "string", enum: [syntheticCommand] },
              },
              required: ["command"],
              additionalProperties: false,
            },
          },
        },
      ],
      tool_choice: { type: "function", function: { name: "bash" } },
      // DeepSeek Flash reasons before the tool call; 32 tokens truncated the arguments to {}.
      // Canaries may use up to 4096; 1024 leaves room to reason without a long answer.
      max_tokens: 1024,
    }),
  });
  const requestId = liveResponse.headers.get("x-request-id");
  const liveBody = await responseJson(liveResponse, "OpenRouter Chat Completions canary");
  assert(requestId, "live response did not include x-request-id");
  const toolCalls = liveBody.choices?.[0]?.message?.tool_calls ?? [];
  const bashCall = toolCalls.find((call) => call.function?.name === "bash");
  assert(bashCall, "live response omitted bash tool call");
  let bashArguments;
  try {
    bashArguments = JSON.parse(bashCall.function.arguments);
  } catch {
    throw new Error("live response returned invalid bash tool arguments");
  }
  assertEqual(bashArguments.command, syntheticCommand, "generated bash command");
  actions.push({ action: "send bounded OpenRouter tool-call canary", result: "HTTP 200" });
  proof.pass("openrouter-tool-call", { observed: "bash arguments matched" });

  const decisionPage = await poll(async () => {
    const value = await adminJson(
      page,
      `/api/v1/admin/guardrails/decisions?request_id=${encodeURIComponent(requestId)}&phase=generated_tool_call`,
    );
    return value.items.some((item) => item.pack_id === "core.filesystem") ? value : null;
  }, "request-linked generated-tool decision");
  const generatedDecision = decisionPage.items.find((item) => item.pack_id === "core.filesystem");
  assertEqual(generatedDecision.rule_id, "recursive-force-remove", "generated-tool rule");
  assertEqual(generatedDecision.action, "audit", "generated-tool action");
  proof.pass("request-linked-guardrail-decision", { observed: generatedDecision.rule_id });

  const requestPage = await poll(async () => {
    const value = await adminJson(
      page,
      `/api/v1/admin/observability/request-logs?request_id=${encodeURIComponent(requestId)}`,
    );
    return value.items.length === 1 ? value : null;
  }, "request-log summary");
  const summary = requestPage.items[0];
  const detail = await adminJson(
    page,
    `/api/v1/admin/observability/request-logs/${encodeURIComponent(summary.request_log_id)}`,
  );
  assertEqual(detail.log.model_key, model, "request-log model");
  assertEqual(detail.log.resolved_model_key, model, "request-log resolved model");
  assertEqual(detail.log.provider_key, "openrouter", "request-log provider");
  assertEqual(detail.log.status_code, 200, "request-log status");
  assert(
    detail.log.tool_cardinality.invoked_tool_count >= 1,
    "request log did not count the generated tool call",
  );
  const attempt = detail.attempts.find((item) => item.provider_key === "openrouter");
  assert(attempt, "request log omitted the OpenRouter provider attempt");
  assertEqual(attempt.upstream_model, upstreamModel, "provider-attempt upstream model");
  assertEqual(attempt.status, "success", "provider-attempt status");
  const usageRecorded = detail.log.total_tokens == null || detail.log.total_tokens > 0;
  assert(usageRecorded, "provider supplied a non-positive total token count");
  proof.pass("request-log-attempt-and-usage", { observed: attempt.upstream_model });
  actions.push({
    action: "confirm request, attempt, usage, tool, and decision records",
    result: "sanitized backend evidence matched",
  });

  details = {
    gatewayModel: model,
    provider: "openrouter",
    upstreamModel,
    requestId,
    statusCode: detail.log.status_code,
    usageRecorded: detail.log.total_tokens != null,
    invokedToolCount: detail.log.tool_cardinality.invoked_tool_count,
    guardrail: {
      phase: generatedDecision.phase,
      action: generatedDecision.action,
      packId: generatedDecision.pack_id,
      ruleId: generatedDecision.rule_id,
      reasonCode: generatedDecision.reason_code,
    },
    payloadCaptureMode: detail.log.payload_policy.capture_mode,
    actions,
  };
} catch (error) {
  failure = error;
} finally {
  try {
    if (page && apiKeyId) {
      const revokeResponse = await page.evaluate(
        async ({ id, timeoutMs }) => {
          const response = await fetch(`/api/v1/admin/api-keys/${encodeURIComponent(id)}/revoke`, {
            signal: AbortSignal.timeout(timeoutMs),
            method: "POST",
          });
          return response.status;
        },
        { id: apiKeyId, timeoutMs: gatewayRequestTimeoutMs },
      );
      assertEqual(revokeResponse, 200, "temporary API key revocation");
      if (rawKey) {
        const rejected = await fetch(`${baseURL}/v1/models`, {
          headers: { authorization: `Bearer ${rawKey}` },
          signal: AbortSignal.timeout(gatewayRequestTimeoutMs),
        });
        assertEqual(rejected.status, 401, "revoked key authentication");
      }
      proof.pass("temporary-key-revoked-and-rejected");
    }
  } catch (error) {
    proof.add({ id: "temporary-key-revoked-and-rejected", status: "fail", note: safeMessage(error) });
    failure ??= error;
  } finally {
    rawKey = undefined;
    await browser.close();
  }
}
await proof.write({ details: details ?? { actions }, error: details ? undefined : failure });
if (failure) throw failure;
console.log(`backend gateway proof passed for ${model} through OpenRouter`);
console.log(`evidence: ${evidenceDir}`);


async function waitForSignIn(currentPage) {
  await currentPage.getByRole("heading", { name: "Sign in" }).waitFor();
  await currentPage.waitForFunction(() => {
    const button = Array.from(document.querySelectorAll("button")).find(
      (candidate) => candidate.textContent?.trim() === "Sign in",
    );
    return button instanceof HTMLButtonElement && !button.disabled;
  });
}

async function adminJson(currentPage, requestPath, options) {
  return currentPage.evaluate(
    async ({ pathName, init }) => {
      const response = await fetch(pathName, init);
      const body = await response.json();
      if (!response.ok) throw new Error(`${pathName} returned ${response.status}`);
      return body.data;
    },
    { pathName: requestPath, init: options },
  );
}

async function responseJson(response, label) {
  const body = await response.json();
  if (!response.ok) {
    throw new Error(`${label} returned ${response.status}: ${JSON.stringify(body)}`);
  }
  return body;
}

async function poll(operation, label) {
  const deadline = Date.now() + 30_000;
  while (Date.now() < deadline) {
    const result = await operation();
    if (result) return result;
    await new Promise((resolve) => setTimeout(resolve, 250));
  }
  throw new Error(`Timed out waiting for ${label}`);
}

async function capture(currentPage, name) {
  await currentPage.screenshot({ path: path.join(evidenceDir, `${name}.png`), fullPage: true });
  await fs.writeFile(
    path.join(evidenceDir, `${name}.aria.txt`),
    `${await currentPage.locator("body").ariaSnapshot()}\n`,
  );
  proof.artifact(`${name}.png`);
}

function gatewayHeaders(key) {
  return {
    authorization: `Bearer ${key}`,
    "content-type": "application/json",
  };
}

function requiredEnv(name) {
  const value = process.env[name];
  if (!value) throw new Error(`Missing required environment variable ${name}`);
  return value;
}

function assert(condition, message) {
  if (!condition) throw new Error(message);
}

function assertEqual(actual, expected, label) {
  if (actual !== expected) {
    throw new Error(`${label}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
  }
}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall