Skip to content

aidenlx/zotlit / skills/zotlit-query

zotlit-query

Verified

Query Zotero works, files, and reading marks through ZotLit in Obsidian. Use for literature reviews, reading plans, missing PDFs, broken linked files, duplicate PDFs, Annotation evidence, counts by paper or year, and complete exports.

evals/research.mjs

View on GitHub
// The `research` case family: evidence is checked against literals, independently of the agent.
import { cp, readFile } from "node:fs/promises";
import { join } from "node:path";
import { isDeepStrictEqual } from "node:util";

import { checkRows, commonChecks, resultRows } from "./result-rows.mjs";

export function resolveExpected(value, runRoot) {
  if (typeof value === "string") return value.replaceAll("$RUN", runRoot);
  if (Array.isArray(value))
    return value.map((v) => resolveExpected(v, runRoot));
  if (value && typeof value === "object")
    return Object.fromEntries(
      Object.entries(value).map(([k, v]) => [k, resolveExpected(v, runRoot)]),
    );
  return value;
}
function expectedGroups(expected, path) {
  const values = expected.groupValues?.[path];
  if (values)
    return expected.groups.map((group) => ({
      ...group,
      value: values[group.value],
    }));
  if (expected.group === "item.indexedKey" && path === "item.title")
    return expected.groups
      .map((group) => ({ ...group, value: group.rows[0].values["item.title"] }))
      .sort((a, b) => a.value.localeCompare(b.value, "und"));
  return expected.groups;
}

// Per-paper counts can start from papers or from grouped reading marks.
export function researchExpectation(spec, envelope) {
  if (spec.group !== "item.indexedKey" || envelope.request?.from !== "items")
    return spec;
  const { groups, group: _group, ...rest } = spec;
  return {
    ...rest,
    from: "items",
    count: groups.length,
    fields: ["title"],
    rows: groups.map((g) => ({
      indexedKey: g.value,
      values: { title: g.rows[0].values["item.title"] },
    })),
    paperGroups: groups,
  };
}

function checkPaperCounts(rows, groups, required = true) {
  const errors = [];
  for (const row of rows ?? []) {
    const values = row.values ?? {};
    const counts = [
      ...(Object.hasOwn(values, "annotations.length")
        ? [values["annotations.length"]]
        : []),
      ...["annotations[]", "annotations"]
        .filter((field) => Object.hasOwn(values, field))
        .map((field) =>
          Array.isArray(values[field]) ? values[field].length : null,
        ),
    ];
    const expected = groups.find((g) => g.value === row.indexedKey)?.count;
    if (
      (required && !counts.length) ||
      counts.some((count) => count !== expected)
    )
      errors.push(`wrong mark count for ${row.indexedKey}`);
  }
  return errors;
}

export function checkResearch(spec, envelope, runRoot) {
  const expected = resolveExpected(
    researchExpectation(spec, envelope),
    runRoot,
  );
  const errors = [];
  if (envelope.request?.from !== expected.from)
    errors.push(`wrong Query Dataset; expected ${expected.from}`);
  for (const field of expected.fields)
    if (!envelope.request?.fields?.includes(field))
      errors.push(`missing projected field ${field}`);
  if (expected.groups) {
    if (
      envelope.request?.group !== expected.group &&
      !Object.hasOwn(expected.groupValues ?? {}, envelope.request?.group) &&
      !(
        expected.group === "item.indexedKey" &&
        envelope.request?.group === "item.title"
      )
    )
      errors.push("wrong group path");
    if (envelope.totalCount !== expected.count) errors.push("wrong totalCount");
    if (
      !Array.isArray(envelope.groups) ||
      envelope.groups.length !== expected.groups.length
    )
      return [...errors, "wrong groups"];
    const groups = expectedGroups(expected, envelope.request?.group);
    for (let i = 0; i < groups.length; i++) {
      const actual = envelope.groups[i],
        wanted = groups[i];
      if (actual.value !== wanted.value || actual.count !== wanted.count)
        errors.push("wrong group value, order, or count");
      const keys = new Set((actual.rows ?? []).map((row) => row.indexedKey));
      const limit = envelope.request?.limit ?? wanted.count;
      if (actual.rows?.length !== Math.min(limit, wanted.count))
        errors.push("wrong per-group sample size");
      errors.push(
        ...checkRows(
          actual.rows,
          wanted.rows.filter((row) => keys.has(row.indexedKey)),
        ),
      );
    }
  } else errors.push(...checkRows(resultRows(envelope), expected.rows));
  if (expected.paperGroups)
    errors.push(
      ...checkPaperCounts(resultRows(envelope), expected.paperGroups),
    );
  return errors;
}
function schemaFor(value) {
  if (value === null) return { type: "null" };
  if (Array.isArray(value))
    return {
      type: "array",
      items: value.length
        ? unionSchemas(value.map(schemaFor))
        : { type: "string" },
    };
  if (typeof value === "object")
    return {
      type: "object",
      additionalProperties: false,
      properties: Object.fromEntries(
        Object.entries(value).map(([k, v]) => [k, schemaFor(v)]),
      ),
      required: Object.keys(value),
    };
  return { type: typeof value === "number" ? "integer" : typeof value };
}
// Merge shapes, not literal values, so the agent learns no expected answer.
function unionSchemas(schemas) {
  const unique = [
    ...new Map(schemas.map((s) => [JSON.stringify(s), s])).values(),
  ];
  return unique.length === 1 ? unique[0] : { anyOf: unique };
}
export function researchSchema(expected) {
  const rows = expected.rows ?? expected.groups.flatMap((g) => g.rows);
  const rowSchema = {
    type: "object",
    additionalProperties: false,
    properties: {
      indexedKey: { type: "string" },
      values: {
        type: "object",
        additionalProperties: false,
        properties: Object.fromEntries(
          expected.fields.map((field) => [
            field,
            unionSchemas(rows.map((r) => schemaFor(r.values[field]))),
          ]),
        ),
        required: expected.fields,
      },
    },
    required: ["indexedKey", "values"],
  };
  if (expected.group === "item.indexedKey") {
    const summary = schemaFor({
      indexedKey: "",
      type: "",
      text: "",
      comment: "",
      pageLabel: "",
      pageIndex: 0,
    });
    for (const field of ["text", "comment", "pageLabel", "pageIndex"])
      summary.properties[field].type = [summary.properties[field].type, "null"];
    rowSchema.properties.values = {
      anyOf: [
        rowSchema.properties.values,
        schemaFor({ title: "", "annotations.length": 0 }),
        {
          type: "object",
          additionalProperties: false,
          properties: {
            title: { type: "string" },
            "annotations[]": { type: "array", items: summary },
          },
          required: ["title", "annotations[]"],
        },
      ],
    };
  }
  const properties = {
    answer: { type: "string" },
    count: { type: "integer" },
    rows: { type: "array", items: rowSchema },
    groups: {
      type: "array",
      items: {
        type: "object",
        additionalProperties: false,
        properties: {
          value: { type: ["string", "integer", "null"] },
          count: { type: "integer" },
        },
        required: ["value", "count"],
      },
    },
    limitation: { type: ["string", "null"] },
    exportPath: { type: ["string", "null"] },
  };
  return {
    type: "object",
    additionalProperties: false,
    properties,
    required: Object.keys(properties),
  };
}
function groupCounts(groups) {
  return groups
    ?.map(({ value, count }) => ({ value, count }))
    .sort((a, b) =>
      JSON.stringify(a.value).localeCompare(JSON.stringify(b.value)),
    );
}

export function checkResearchAnswer(
  spec,
  answer,
  { runRoot, resultPath, envelope },
) {
  const expected = researchExpectation(spec, envelope);
  const errors = [];
  if (typeof answer.answer !== "string" || !answer.answer.trim())
    errors.push("answer text is empty");
  if (answer.count !== expected.count) errors.push("answer count is wrong");
  const want = resolveExpected(expected, runRoot);
  const groups =
    want.paperGroups ?? expectedGroups(want, envelope?.request?.group);
  const returned = new Set(
    (resultRows(envelope) ?? []).map((row) => row.indexedKey),
  );
  errors.push(
    ...checkRows(
      want.paperGroups
        ? answer.rows?.map((row) => ({
            ...row,
            values: {
              ...row.values,
              title: row.values?.title ?? row.values?.["item.title"],
            },
          }))
        : answer.rows,
      want.rows ??
        // Element groups overlap: a paper in two groups is one answer row.
        [
          ...new Map(
            groups
              .flatMap((g) => g.rows)
              .filter((row) => returned.has(row.indexedKey))
              .map((row) => [row.indexedKey, row]),
          ).values(),
        ],
    ).map((e) => `answer: ${e}`),
  );
  const countsInRows = want.paperGroups && answer.groups?.length === 0;
  if (want.paperGroups)
    errors.push(
      ...checkPaperCounts(answer.rows, want.paperGroups, countsInRows),
    );
  if (
    !countsInRows &&
    !isDeepStrictEqual(groupCounts(answer.groups), groupCounts(groups ?? [])) &&
    !(
      (want.group === "item.indexedKey" || want.paperGroups) &&
      isDeepStrictEqual(
        groupCounts(answer.groups),
        groupCounts(
          expectedGroups(
            { ...want, group: "item.indexedKey", groups },
            "item.title",
          ),
        ),
      )
    )
  )
    errors.push("answer has wrong group counts");
  if (answer.limitation !== (expected.limitation ?? null))
    errors.push("answer has wrong capability limitation");
  if (
    answer.exportPath !==
    (expected.csv ? resultPath.replace(/result\.json$/, "advisor.csv") : null)
  )
    errors.push("answer has wrong CSV export path");
  return errors;
}

// Receipts are written by the CLI wrapper, outside the agent's writable folder.
export function checkCollectionDiscovery(spec, calls) {
  if (!spec.discovery) return [];
  const { library, path } = spec.discovery;
  const discovery = calls.findIndex(
    (call) =>
      call.exitCode === 0 &&
      call.argv[1] === "zotlit:query-values" &&
      call.argv.includes("kind=collections") &&
      call.discovered?.some(
        (values) => values.library === library && values.values.includes(path),
      ),
  );
  const query = calls.findIndex(
    (call, index) =>
      index > discovery &&
      call.exitCode === 0 &&
      call.argv[1] === "zotlit:query" &&
      call.argv.some((arg) => arg.startsWith("filter=") && arg.includes(path)),
  );
  return discovery >= 0 && query > discovery
    ? []
    : [
        "Discover the Collection path with zotlit:query-values before filtering by it",
      ];
}

export function checkEnvelope(spec, envelope, context) {
  const expected = researchExpectation(spec, envelope);
  return [
    ...commonChecks(envelope, expected, context).errors,
    ...checkResearch(expected, envelope, context.runRoot),
  ];
}

export const checkAnswer = checkResearchAnswer;
export const answerSchema = researchSchema;

export function prompt(_spec, { preamble, agentRoot, result }) {
  return `${preamble}Save the complete successful query envelope at ${result} and read it before answering. Use a single complete query when the task can be answered by following relations or grouping. Your final JSON must contain count (the total matched rows), rows (each identity and requested values; flatten grouped rows), groups (value and count, or []), limitation (null unless a requested capability is unavailable), and exportPath (null unless you produced a CSV). For a CSV task, save advisor.csv at ${join(agentRoot, "advisor.csv")} and retain the query JSON as evidence. The runner copies advisor.csv to ${join(agentRoot, "..", "advisor.csv")} after the run. Report that retained path; do not write there. Express an unavailable fuzzy-search capability as fuzzy-search-unavailable. Use only this folder for files you create.`;
}

export async function retainEvidence(spec, { agentRoot, root }) {
  if (spec.csv)
    await cp(join(agentRoot, "advisor.csv"), join(root, "advisor.csv"));
}

export async function loadEvidence(spec, root) {
  return spec.csv
    ? { csv: await readFile(join(root, "advisor.csv"), "utf8") }
    : {};
}

export function parseCsv(text) {
  const rows = [];
  let row = [],
    value = "",
    quoted = false;
  for (let i = 0; i < text.length; i++) {
    const char = text[i];
    if (char === '"') {
      if (quoted && text[i + 1] === '"') {
        value += '"';
        i++;
      } else quoted = !quoted;
    } else if (char === "," && !quoted) {
      row.push(value);
      value = "";
    } else if (char === "\n" && !quoted) {
      row.push(value.replace(/\r$/, ""));
      rows.push(row);
      row = [];
      value = "";
    } else value += char;
  }
  if (quoted) throw new Error("CSV has an unclosed quoted value");
  if (value || row.length) {
    row.push(value);
    rows.push(row);
  }
  return rows;
}

function checkCsv(spec, csv) {
  const parsed = parseCsv(csv);
  const expected = spec.rows.map((row) => [
    row.values.title,
    String(row.values["date.year"] ?? ""),
  ]);
  return parsed.length !== expected.length + 1 ||
    expected.some(
      (row) => !parsed.slice(1).some((found) => isDeepStrictEqual(found, row)),
    )
    ? ["CSV has wrong titles, years, or row count"]
    : [];
}

export function checkRun(spec, { cliCalls, metrics, csv }) {
  return [
    ...checkCollectionDiscovery(spec, cliCalls),
    ...(metrics.queryExitZero < 1
      ? ["agent made no completed Query call"]
      : []),
    ...(spec.csv ? checkCsv(spec, csv) : []),
  ];
}
add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks.Stars: 115kUpdated Claude CodeCodexVerifiedInstall