cskwork/supergoal-skill / skills/supergoal
supergoal
UnverifiedRoute objective-driven development, verification, planning, "teach/explain", codebase learning, and skill evaluation through the Supergoal workflow. Use when Supergoal is requested or an autonomous delivery workflow is needed.
templates/harness-eval-result.json
View on GitHub{
"case_id": "case-001",
"runtime_adapter": "portable",
"same_repo_snapshot": true,
"isolated_worktrees": true,
"runtime_preflight": {
"host_os": "win32",
"preferred_adapter": "claude-p",
"chosen_adapter": "claude-p",
"preflights": {
"claude-p": { "available": true, "ok": true, "reason": "edit ok, tests pass" }
}
},
"role_source": "shipped_files",
"statistics": {
"seeds_per_arm": 6,
"winning_comparison": "harness-vs-baseline",
"bca_ci_95": [0.4, 2.6],
"permutation_p": 0.03125,
"mcnemar": {
"discordant_baseline_only": 1,
"discordant_harness_only": 5,
"p": 0.03125,
"significant": true
},
"snr_filter": {
"matched_removed": 8,
"discordant_kept": 6,
"rule": "remove no-signal matched pass/pass and fail/fail pairs before McNemar"
},
"significant": true,
"seed_outcomes": [
{ "arm": "baseline", "seed": 0, "crashed": false, "recorded_loss": false },
{ "arm": "baseline", "seed": 1, "crashed": false, "recorded_loss": false },
{ "arm": "baseline", "seed": 2, "crashed": false, "recorded_loss": false },
{ "arm": "baseline", "seed": 3, "crashed": false, "recorded_loss": false },
{ "arm": "baseline", "seed": 4, "crashed": false, "recorded_loss": false },
{ "arm": "baseline", "seed": 5, "crashed": false, "recorded_loss": false },
{ "arm": "harness", "seed": 0, "crashed": false, "recorded_loss": false },
{ "arm": "harness", "seed": 1, "crashed": false, "recorded_loss": false },
{ "arm": "harness", "seed": 2, "crashed": false, "recorded_loss": false },
{ "arm": "harness", "seed": 3, "crashed": false, "recorded_loss": false },
{ "arm": "harness", "seed": 4, "crashed": false, "recorded_loss": false },
{ "arm": "harness", "seed": 5, "crashed": true, "recorded_loss": true }
]
},
"default_coding_ab": {
"applies_when": "coding harness eval requested without named cases",
"cases": [
{
"fixture": "revfactory-case-002-async-race/",
"spec": "revfactory-case-002-bug-fix.yaml",
"mode": "DEBUG/concurrency",
"reason": "hard runnable fixture with visible-pass hidden-fail starter"
},
{
"fixture": "revfactory-case-003-refactoring/",
"spec": "revfactory-case-003-refactoring.yaml",
"mode": "LEGACY/brownfield preservation",
"reason": "runnable preservation fixture whose hidden suite catches behavior drift"
}
],
"excluded_from_default": [
"underspec-001-deepmerge/",
"underspec-002-csvline/",
"underspec-003-authz-cache/"
],
"tie_rule": "If both default cases tie, report Not proven and require authored expert runnable fixtures."
},
"external_task": {
"applies": false,
"benchmark": "DeepSWE",
"benchmark_url": "https://deepswe.datacurve.ai/",
"benchmark_repo": "https://github.com/datacurve-ai/deep-swe",
"benchmark_ref": "3cda4081fed96103a6395de39c85e9b20275e307",
"task_id": "etree-xml-diff-patch",
"task_url": "https://deepswe.datacurve.ai/data/v1/tasks/etree-xml-diff-patch",
"upstream_repo": "https://github.com/beevik/etree",
"base_commit": "4032e04c8f2e2f35e43ce5d772fcef14a5df4d74",
"language": "go",
"category": "feature_request",
"stop_policy": {
"timeout_seconds": 900,
"manual_interrupt": "invalid_paired_correctness",
"capture_patch_on_budget_timeout": true,
"process_outcome": "completed | budget_timeout | error | invalid_manual_interrupt"
},
"base_task_text_sha256": "sha256 of original DeepSWE instruction.md",
"harness_source_sha256": [
{ "path": "SKILL.md", "sha256": "sha256" }
],
"verifier_artifacts": [
"verifier/reward.json",
"verifier/ctrf.json",
"verifier/test-stdout.txt",
"verifier/run.log"
],
"pass_rate_lift": {
"baseline_pass_rate": 0,
"harness_pass_rate": 0,
"absolute_percentage_points": 0,
"relative_percent": 0,
"partial_reward_delta": 0,
"claim": "not_proven"
}
},
"axis_metrics": {
"correctness": {
"metric": "paired pass/fail plus quality score",
"baseline": 0.7,
"harness": 0.9,
"delta": 0.2
},
"token_cost": {
"metric": "total_tokens per invocation",
"baseline": 1000,
"harness": 1200,
"delta": 200,
"source": "adapter telemetry"
},
"wall_clock": {
"metric": "duration_ms per invocation",
"baseline": 1000,
"harness": 1400,
"delta": 400,
"source": "adapter telemetry"
},
"routing_accuracy": {
"metric": "held-out trigger accuracy",
"status": "measured",
"baseline": 0.65,
"harness": 0.85,
"delta": 0.2
}
},
"routing_accuracy": {
"applies": true,
"prompt_count": 20,
"trials_per_prompt": 3,
"train_test_split": "60/40",
"should_trigger_rate": 0.9,
"should_not_trigger_rate": 0.8,
"heldout_accuracy": 0.85,
"near_miss_failures": ["route-017"],
"artifact": "routing/routing-probe.json"
},
"eval_intent": {
"goal": "the user's objective in their own terms",
"constraints": ["constraints learned from the conversation or repo rules"],
"tradeoffs": ["decision or tradeoff that affects review judgment"],
"rejected_approaches": ["alternative considered and rejected"]
},
"command_manifest": [
{
"name": "test",
"command": "npm test",
"source": "frozen_repo",
"used_by": "both",
"verifies": "visible and hidden acceptance tests"
},
{
"name": "lint",
"command": "npm run lint",
"source": "frozen_repo",
"used_by": "both",
"verifies": "static syntax/style constraints"
},
{
"name": "build",
"command": "npm run build",
"source": "frozen_repo",
"used_by": "both",
"verifies": "package can compile or bundle"
}
],
"decision_gates": [
{
"id": "r1",
"action": "auto-fix",
"status": "resolved",
"description": "mechanical low-risk issue fixed by the pipeline",
"recheck": "npm test exit=0 after fix"
},
{
"id": "i1",
"action": "no-op",
"status": "accepted",
"description": "informational finding that required no code change"
}
],
"adapter_fixture_replay": {
"status": "not_required",
"adapter_event_schema": "portable-result-json-v1",
"fixtures": [],
"redaction": "not required for this portable template",
"replay_command": "not required"
},
"surface_sync": {
"changed_surfaces": [],
"proof_commands": []
},
"baseline": {
"condition": "without_harness",
"machine_checks": [
{
"name": "test",
"status": "pass",
"evidence": "command and artifact path",
"verifies": "acceptance behavior covered by this check",
"does_not_verify": "known uncovered edges",
"confidence": "high"
},
{
"name": "lint",
"status": "pass",
"evidence": "command and artifact path",
"verifies": "static syntax/style constraints",
"does_not_verify": "runtime behavior",
"confidence": "medium"
},
{
"name": "build",
"status": "pass",
"evidence": "command and artifact path",
"verifies": "package can compile or bundle",
"does_not_verify": "hidden acceptance behavior",
"confidence": "medium"
}
],
"cost": { "tokens": 1000, "duration_ms": 1000, "tool_calls": 10 },
"telemetry": {
"artifact_root": "docs/changelog/<YYYY-MM-DD-harness-eval>/baseline",
"logs": ["raw/baseline.log"],
"commands": ["npm test", "npm run lint", "npm run build"],
"edited_files": ["src/example.js"],
"permissions_or_approvals": [],
"turns_completed": 1,
"exit_code": 0,
"crashed": false,
"context_exhausted": false
}
},
"harness": {
"condition": "with_harness",
"machine_checks": [
{
"name": "test",
"status": "pass",
"evidence": "command and artifact path",
"verifies": "acceptance behavior covered by this check",
"does_not_verify": "known uncovered edges",
"confidence": "high"
},
{
"name": "lint",
"status": "pass",
"evidence": "command and artifact path",
"verifies": "static syntax/style constraints",
"does_not_verify": "runtime behavior",
"confidence": "medium"
},
{
"name": "build",
"status": "pass",
"evidence": "command and artifact path",
"verifies": "package can compile or bundle",
"does_not_verify": "hidden acceptance behavior",
"confidence": "medium"
}
],
"cost": { "tokens": 1200, "duration_ms": 1400, "tool_calls": 12 },
"telemetry": {
"artifact_root": "docs/changelog/<YYYY-MM-DD-harness-eval>/harness",
"logs": ["raw/harness.log"],
"commands": ["npm test", "npm run lint", "npm run build"],
"edited_files": ["src/example.js"],
"permissions_or_approvals": [],
"turns_completed": 1,
"exit_code": 0,
"crashed": false,
"context_exhausted": false
}
},
"quality": {
"method": "RevFactory-style 10 dimensions, each 0-10, total 100; anchored to observable code properties.",
"baseline": {
"average_total": 70,
"by_case": {
"case-001": {
"total": 70,
"dimensions": {
"feature_completeness": { "score": 8, "rationale": "Requirements mostly implemented." },
"test_coverage": { "score": 6, "rationale": "Relevant tests exist but miss edge cases." },
"code_quality": { "score": 7, "rationale": "Readable and consistent." },
"error_handling": { "score": 6, "rationale": "Basic validation and failures." },
"efficiency": { "score": 8, "rationale": "Appropriate algorithms and dependencies." },
"correctness": { "score": 7, "rationale": "Machine checks pass with remaining risks." },
"architecture": { "score": 7, "rationale": "Structure fits task size." },
"extensibility": { "score": 7, "rationale": "Can be extended safely." },
"documentation": { "score": 6, "rationale": "Some usage or changelog notes." },
"dev_environment": { "score": 8, "rationale": "Runnable scripts and checks." }
}
}
}
},
"harness": {
"average_total": 78,
"by_case": {
"case-001": {
"total": 78,
"dimensions": {
"feature_completeness": { "score": 9, "rationale": "Requirements fully implemented." },
"test_coverage": { "score": 8, "rationale": "Edge-case tests included." },
"code_quality": { "score": 8, "rationale": "Readable and consistent." },
"error_handling": { "score": 8, "rationale": "Explicit validation and failures." },
"efficiency": { "score": 8, "rationale": "Appropriate algorithms and dependencies." },
"correctness": { "score": 9, "rationale": "Machine and hidden checks pass." },
"architecture": { "score": 8, "rationale": "Structure fits task and future changes." },
"extensibility": { "score": 8, "rationale": "Clear helpers and boundaries." },
"documentation": { "score": 6, "rationale": "Some usage or changelog notes." },
"dev_environment": { "score": 6, "rationale": "Runnable scripts and checks." }
}
}
}
},
"winner": "harness"
},
"harness_mutation_contract": {
"status": "not_proven",
"intended_delta": "behavior this harness change is expected to improve",
"safety_envelope": "what the harness is allowed to change and what it must not change",
"rollback": "exact revert or disable path",
"proof_command": "node templates/harness-eval-gate.mjs <result.json>",
"rejected_alternatives": ["simpler or riskier alternative and why it was rejected"]
},
"blind_grading": true,
"winner": "harness",
"claim_status": "not_proven"
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |