Skip to content

rohitg00/pro-workflow / skills/llm-council

llm-council

Verified

Provider-agnostic multi-LLM deliberation. Three phases — independent responses, cross-model anonymized ranking, chairman synthesis. Credentials come from explicit plugin configuration or pro-workflow-specific CLI variables. Persists transcript to a wiki page when --wiki <slug> is passed. Use when the user wants multiple AI perspectives, consensus-building, or the "LLM Council" approach for high-stakes reviews, plan critique, or contested learning rules.

FileKindSize
SKILL.mdskill md4.4 KB
scripts/council.jsexecutablescript11.4 KB

scripts/council.js

View on GitHub
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');
const { getCredential } = require('../../../scripts/lib/credentials.js');

const PRO_WORKFLOW_ROOT = path.resolve(__dirname, '..', '..', '..');
const COUNCIL_ROOT = path.join(os.homedir(), '.pro-workflow', 'council');

const PROVIDERS = {
  anthropic: {
    envKey: 'ANTHROPIC_API_KEY',
    baseUrl: 'https://api.anthropic.com',
    defaultModels: ['claude-opus-5-5', 'claude-sonnet-5', 'claude-haiku-4-5'],
    defaultChairman: 'claude-opus-5-5',
    call: callAnthropic,
  },
  openai: {
    envKey: 'OPENAI_API_KEY',
    baseUrl: 'https://api.openai.com/v1',
    defaultModels: ['gpt-4o', 'gpt-4o-mini', 'o3-mini'],
    defaultChairman: 'gpt-4o',
    call: callOpenAICompat,
  },
  openrouter: {
    envKey: 'OPENROUTER_API_KEY',
    baseUrl: 'https://openrouter.ai/api/v1',
    defaultModels: ['anthropic/claude-opus-4', 'openai/gpt-4o', 'google/gemini-2.0-flash'],
    defaultChairman: 'anthropic/claude-opus-4',
    call: callOpenAICompat,
  },
  fireworks: {
    envKey: 'FIREWORKS_API_KEY',
    baseUrl: 'https://api.fireworks.ai/inference/v1',
    defaultModels: [
      'accounts/fireworks/models/glm-5',
      'accounts/fireworks/models/deepseek-v3p2',
      'accounts/fireworks/models/kimi-k2p5',
    ],
    defaultChairman: 'accounts/fireworks/models/glm-5',
    call: callOpenAICompat,
  },
  custom: {
    envKey: 'LLM_COUNCIL_API_KEY',
    baseUrl: process.env.LLM_COUNCIL_BASE_URL || '',
    defaultModels: (process.env.LLM_COUNCIL_MODELS || '').split(',').filter(Boolean),
    defaultChairman: process.env.LLM_COUNCIL_CHAIRMAN || '',
    call: callOpenAICompat,
  },
};

function pickProvider(arg) {
  if (arg) return Object.hasOwn(PROVIDERS, arg) && getCredential(PROVIDERS[arg].envKey) ? arg : null;
  for (const [name, p] of Object.entries(PROVIDERS)) {
    if (getCredential(p.envKey)) return name;
  }
  return null;
}

function postJSON(urlStr, body, headers, timeoutMs = 120000) {
  return new Promise((resolve, reject) => {
    const url = new URL(urlStr);
    const data = JSON.stringify(body);
    const req = https.request({
      hostname: url.hostname,
      path: url.pathname + url.search,
      method: 'POST',
      headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers },
    }, res => {
      let chunks = '';
      res.on('data', c => { chunks += c; });
      res.on('end', () => resolve({ status: res.statusCode, body: chunks }));
    });
    req.setTimeout(timeoutMs, () => req.destroy(new Error('council request timeout')));
    req.on('error', reject);
    req.write(data);
    req.end();
  });
}

async function callOpenAICompat(provider, model, system, user) {
  const start = Date.now();
  const url = `${provider.baseUrl}/chat/completions`;
  const res = await postJSON(url, {
    model,
    messages: [{ role: 'system', content: system }, { role: 'user', content: user }],
    max_tokens: 4000,
    temperature: 1,
  }, { Authorization: `Bearer ${getCredential(provider.envKey)}` });
  const elapsed = Date.now() - start;
  if (res.status >= 400) return { success: false, content: `[ERROR ${res.status}: ${res.body.slice(0, 300)}]`, model, latency_ms: elapsed };
  let data;
  try { data = JSON.parse(res.body); } catch (e) { return { success: false, content: `[parse-error]`, model, latency_ms: elapsed }; }
  const content = data.choices?.[0]?.message?.content || '';
  return { success: true, content, model, latency_ms: elapsed, tokens: data.usage || {} };
}

async function callAnthropic(provider, model, system, user) {
  const start = Date.now();
  const url = `${provider.baseUrl}/v1/messages`;
  const res = await postJSON(url, {
    model,
    max_tokens: 4000,
    system,
    messages: [{ role: 'user', content: user }],
  }, {
    'x-api-key': getCredential(provider.envKey),
    'anthropic-version': '2023-06-01',
  });
  const elapsed = Date.now() - start;
  if (res.status >= 400) return { success: false, content: `[ERROR ${res.status}: ${res.body.slice(0, 300)}]`, model, latency_ms: elapsed };
  let data;
  try { data = JSON.parse(res.body); } catch { return { success: false, content: '[parse-error]', model, latency_ms: elapsed }; }
  const content = (data.content || []).map(b => b.text || '').join('');
  return { success: true, content, model, latency_ms: elapsed, tokens: data.usage || {} };
}

function parseArgs(argv) {
  const out = { _: [] };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a.startsWith('--')) {
      const key = a.slice(2);
      const next = argv[i + 1];
      if (next && !next.startsWith('--')) { out[key] = next; i++; }
      else out[key] = true;
    } else out._.push(a);
  }
  return out;
}

function ts() { return new Date().toISOString().replace(/[:.]/g, '-'); }

function persistToWiki(slug, sessionId, output) {
  const distPath = path.join(PRO_WORKFLOW_ROOT, 'dist', 'db', 'store.js');
  if (!fs.existsSync(distPath)) return null;
  const { createStore } = require(distPath);
  const store = createStore();
  try {
    const wiki = store.getWiki(slug);
    if (!wiki) return null;
    const relPath = path.posix.join('derived', 'council', `${sessionId}.md`);
    const fileAbs = path.join(wiki.root_path, relPath);
    fs.mkdirSync(path.dirname(fileAbs), { recursive: true });
    fs.writeFileSync(fileAbs, output);
    store.upsertWikiPage({
      wiki_slug: slug,
      rel_path: relPath,
      title: `Council session ${sessionId}`,
      summary: output.slice(0, 500),
      content: output,
      page_type: 'council',
      content_hash: null,
    });
    return fileAbs;
  } finally { store.close(); }
}

async function cmdRun(args) {
  const query = args._[0];
  if (!query) { console.error('run: query required'); process.exit(1); }
  const providerName = pickProvider(args.provider);
  if (!providerName) { console.error('No configured provider. Set a key in plugin settings, or use a PRO_WORKFLOW_*_API_KEY variable for the standalone CLI.'); process.exit(2); }
  const provider = PROVIDERS[providerName];
  if (!provider.baseUrl) { console.error(`provider ${providerName} requires LLM_COUNCIL_BASE_URL`); process.exit(2); }

  const models = (args.models ? String(args.models).split(',') : provider.defaultModels).filter(Boolean);
  const chairman = args.chairman || provider.defaultChairman;
  if (!models.length) { console.error('no models — pass --models'); process.exit(2); }
  if (!chairman) { console.error('no chairman — pass --chairman'); process.exit(2); }

  const sessionId = ts();
  const sessionDir = path.join(COUNCIL_ROOT, sessionId);
  fs.mkdirSync(sessionDir, { recursive: true });

  fs.writeFileSync(path.join(sessionDir, 'config.json'), JSON.stringify({ query, models, chairman, provider: providerName }, null, 2));

  function settledToEntry(model, settled) {
    if (settled.status === 'fulfilled') return settled.value;
    return { success: false, content: `[ERROR: ${settled.reason?.message || settled.reason}]`, model, latency_ms: 0 };
  }

  // Phase 1
  const sysIndep = 'You are participating in an LLM council deliberation. Provide your best, most thoughtful response to the query. Be comprehensive but focused.';
  const phase1Settled = await Promise.allSettled(models.map(m => provider.call(provider, m, sysIndep, query)));
  const phase1Entries = phase1Settled.map((s, i) => settledToEntry(models[i], s));
  const phase1 = Object.fromEntries(models.map((m, i) => [m, phase1Entries[i]]));
  fs.writeFileSync(path.join(sessionDir, 'phase1_responses.json'), JSON.stringify(phase1, null, 2));

  // Phase 2
  const labels = ['A', 'B', 'C', 'D', 'E', 'F', 'G'].slice(0, models.length);
  const labelOf = Object.fromEntries(models.map((m, i) => [m, labels[i]]));
  const anon = models.map(m => `=== Response ${labelOf[m]} ===\n${phase1[m].content}`).join('\n\n');
  const sysRank = (own) => `You are ranking AI responses objectively. Your own response is labeled '${own}'.`;
  const userRank = `QUERY:\n${query}\n\nRESPONSES:\n${anon}\n\nRank from BEST to WORST. Format:\nRANKINGS:\n1. [Letter] - [reason]\n2. [Letter] - [reason]\n...`;
  const phase2Settled = await Promise.allSettled(models.map(m => provider.call(provider, m, sysRank(labelOf[m]), userRank)));
  const phase2Entries = phase2Settled.map((s, i) => settledToEntry(models[i], s));
  const phase2 = { label_of: labelOf, rankings: Object.fromEntries(models.map((m, i) => [m, phase2Entries[i]])) };
  fs.writeFileSync(path.join(sessionDir, 'phase2_rankings.json'), JSON.stringify(phase2, null, 2));

  // Phase 3
  const responsesText = models.map(m => `=== ${labelOf[m]}: ${m} ===\n${phase1[m].content}`).join('\n\n');
  const rankingsText = models.map(m => `[${m}'s Rankings]\n${phase2.rankings[m].content}`).join('\n\n');
  const sysSynth = 'You are the Chairman of an LLM Council. Synthesize multiple AI perspectives into a definitive, comprehensive response.';
  const userSynth = `ORIGINAL QUERY:\n${query}\n\nINDIVIDUAL RESPONSES:\n${responsesText}\n\nMODEL RANKINGS:\n${rankingsText}\n\nProduce the FINAL SYNTHESIS:`;
  const synth = await provider.call(provider, chairman, sysSynth, userSynth);
  fs.writeFileSync(path.join(sessionDir, 'phase3_synthesis.txt'), synth.content);

  // Render
  const out = [];
  out.push(`# LLM Council Deliberation`);
  out.push(`Session: ${sessionId} · Provider: ${providerName}`);
  out.push('');
  out.push(`**Query:** ${query}`);
  out.push(`**Council:** ${models.join(', ')}`);
  out.push(`**Chairman:** ${chairman}`);
  out.push('');
  out.push('## Phase 1 — Individual responses');
  for (const m of models) {
    out.push(`### [${labelOf[m]}] ${m} (${phase1[m].latency_ms}ms)`);
    out.push(phase1[m].content);
    out.push('');
  }
  out.push('## Phase 2 — Cross-model rankings');
  for (const m of models) {
    out.push(`### ${m}`);
    out.push(phase2.rankings[m].content);
    out.push('');
  }
  out.push('## Phase 3 — Chairman synthesis');
  out.push(`### ${chairman}`);
  out.push(synth.content);

  const md = out.join('\n');
  fs.writeFileSync(path.join(sessionDir, 'final_output.md'), md);

  if (args.wiki) {
    const wikiPath = persistToWiki(args.wiki, sessionId, md);
    if (wikiPath) console.error(`[council] persisted to ${wikiPath}`);
    else console.error(`[council] wiki ${args.wiki} not found, skipping persist`);
  }

  console.log(md);
}

function cmdProviders() {
  const rows = Object.entries(PROVIDERS).map(([name, p]) => ({
    name,
    env_var: `PRO_WORKFLOW_${p.envKey}`,
    has_key: !!getCredential(p.envKey),
    base_url: p.baseUrl || '(unset)',
    default_models: p.defaultModels,
    default_chairman: p.defaultChairman,
  }));
  console.log(JSON.stringify(rows, null, 2));
}

function cmdShow(args) {
  const id = args._[0];
  if (!id) { console.error('show: session-id required'); process.exit(1); }
  const dir = path.join(COUNCIL_ROOT, id);
  const file = path.join(dir, 'final_output.md');
  if (!fs.existsSync(file)) { console.error('session not found'); process.exit(1); }
  console.log(fs.readFileSync(file, 'utf8'));
}

function usage() {
  console.error(`Usage:
  council.js run "<query>" [--models id1,id2,id3] [--chairman id] [--provider name] [--wiki slug]
  council.js providers
  council.js show <session-id>`);
  process.exit(1);
}

async function main() {
  const [, , cmd, ...rest] = process.argv;
  const args = parseArgs(rest);
  switch (cmd) {
    case 'run': await cmdRun(args); break;
    case 'providers': cmdProviders(); break;
    case 'show': cmdShow(args); break;
    default: usage();
  }
}

main().catch(e => { console.error(e); process.exit(1); });
planning-and-task-breakdownaddyosmani/agent-skillsBreaks work into ordered tasks. Use when you have a spec or clear requirements and need to break work into implementable tasks. Use when a task feels too large to start, when you need to estimate scope, or when parallel work is possible.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
screenpipethedotmack/claude-memSearch screen and audio history recorded by Screenpipe and optionally save selected context into claude-mem. Use when the user asks to recall desktop activity, a meeting, or something seen in another app through Screenpipe.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
contact-widgetnexu-io/open-designSelf-contained floating chat widget with welcome screen, social links, meeting button, and message input. Single HTML file, zero dependencies.Stars: 108kUpdated Claude CodeCodexVerifiedInstall
meeting-notesnexu-io/open-designMeeting notes page — title bar with attendees, agenda checklist, decisions block, action items table with owners + dates, and a "next meeting" footer. Use when the brief mentions "meeting notes", "minutes", "1:1 notes", "all-hands recap", or "会议纪要".Stars: 108kUpdated Claude CodeCodexVerifiedInstall