Skip to content

odinfree/storm-ecdsa-harness / skills/calibrated-static-cost-proxy

calibrated-static-cost-proxy

Verified

Compute the judged cost inside the artefact generator from structure alone (charged ops weighted by the firing rate of their enclosing conditions), calibrate it against the official judge once per design, and use it for all inner-loop decisions. Use when a judge run costs minutes or more and individual levers are worth less than about 1% each, in any benchmark-optimization campaign where a generator builds the artefact (circuits, prover constraints, gas, proof size).

FileKindSize
SKILL.mdskill md6.2 KB
assets/calibration.tsvother280 B
scripts/proxy_counter.pyscript5.9 KB

scripts/proxy_counter.py

View on GitHub
#!/usr/bin/env python3
"""proxy_counter.py: static expected-cost counter for a generator, and a calibration check.

As a library (copy the StaticCost class into the generator, or port its 20 lines):
    cost = StaticCost()
    cost.charge(1, "adder")              # unconditional charged op
    cost.push()                          # enter a block conditioned on a measurement outcome (rate 1/2)
    cost.charge(1, "repair")             # counts 0.5
    cost.push(rate=0.03)                 # a condition measured to fire on 3% of samples
    cost.charge(4, "rare-fix")           # counts 4 * 0.5 * 0.03
    cost.pop(); cost.pop()
    cost.expected, cost.emitted, cost.by_section

As a tool:
  proxy_counter.py check <calibration.tsv> [--lever SIZE]
      Reads rows of artefact, proxy, judge, ... and prints the gap of each. With
      --lever (smallest lever being decided, in judge units) the exit code is 1 when
      the latest gap is not below that size, because the proxy cannot then decide it.
  proxy_counter.py --selftest
"""
import sys


class StaticCost:
    def __init__(self):
        self.expected = 0.0          # proxy for the judged (executed) cost
        self.emitted = 0             # charged ops in the artefact
        self.by_section = {}
        self._weights = [1.0]

    def push(self, rate=0.5):
        self._weights.append(self._weights[-1] * rate)

    def pop(self):
        if len(self._weights) == 1:
            raise ValueError("pop without push")
        self._weights.pop()

    def charge(self, n=1, section="-"):
        w = n * self._weights[-1]
        self.expected += w
        self.emitted += n
        self.by_section[section] = self.by_section.get(section, 0.0) + w


def check(path, lever):
    rows = []
    with open(path) as f:
        for line in f:
            p = line.rstrip("\n").split("\t")
            if len(p) < 3 or line.startswith("#"):
                continue
            try:
                rows.append((p[0], float(p[1]), float(p[2])))
            except ValueError:
                continue                 # header or template row
    if not rows:
        print("no calibration rows in %s" % path)
        return 2
    print("%-16s %16s %16s %12s %9s" % ("artefact", "proxy", "judge", "gap", "relative"))
    for art, proxy, judge in rows:
        print("%-16s %16.3f %16.3f %+12.3f %8.3f%%" % (art[:16], proxy, judge, proxy - judge, 100 * (proxy - judge) / judge))
    art, proxy, judge = rows[-1]
    gap = abs(proxy - judge)
    if lever is None:
        return 0
    if gap < lever:
        print("latest gap %.3f is below the smallest lever %.3f: the proxy can decide it" % (gap, lever))
        return 0
    print("latest gap %.3f is NOT below the smallest lever %.3f: decide with the judge or recalibrate" % (gap, lever))
    return 1


def selftest():
    import math
    import os
    import random
    import tempfile
    rng = random.Random(3)

    def build(depth=0):
        # random block tree: ("op", n, section) | ("block", rate, [children])
        out = []
        for _ in range(rng.randrange(3, 7)):
            if depth < 3 and rng.random() < 0.4:
                out.append(("block", 0.5, build(depth + 1)))
            else:
                out.append(("op", rng.randrange(1, 9), "sec%d" % rng.randrange(3)))
        return out

    def count(tree, cost, rates=None):
        for node in tree:
            if node[0] == "op":
                cost.charge(node[1], node[2])
            else:
                cost.push(node[1] if rates is None else rates)
                count(node[2], cost, rates)
                cost.pop()

    def execute(tree, r):
        total = 0
        for node in tree:
            if node[0] == "op":
                total += node[1]
            elif r.random() < node[1]:
                total += execute(node[2], r)
        return total

    def measure(tree, n=40000):
        r = random.Random(9)
        xs = [execute(tree, r) for _ in range(n)]
        mean = sum(xs) / n
        se = math.sqrt(sum((x - mean) ** 2 for x in xs) / (n - 1) / n)
        return mean, se

    tree = [("block", 0.5, build())] + build()
    cost = StaticCost()
    count(tree, cost)
    mean, se = measure(tree)
    print("[1/4] balanced conditions: proxy %.3f, simulated judge %.3f +- %.3f, emitted %d" % (cost.expected, mean, se, cost.emitted))
    assert abs(cost.expected - mean) < 4 * se
    assert abs(sum(cost.by_section.values()) - cost.expected) < 1e-9, "sections must sum to the total"
    print("[2/4] sections sum to the total: %s" % {k: round(v, 3) for k, v in sorted(cost.by_section.items())})

    rare = [("op", 100, "a"), ("block", 0.05, [("op", 400, "fix")])]
    naive, fitted = StaticCost(), StaticCost()
    count(rare, naive, rates=0.5)
    count(rare, fitted)
    mean, se = measure(rare)
    print("[3/4] negative case, unbalanced condition (fires 5%%): 1/2-weight proxy %.1f vs judge %.1f; measured-rate proxy %.1f" %
          (naive.expected, mean, fitted.expected))
    assert abs(naive.expected - mean) > 20 * se and abs(fitted.expected - mean) < 4 * se

    with tempfile.TemporaryDirectory() as d:
        p = os.path.join(d, "calibration.tsv")
        with open(p, "w") as f:
            f.write("artefact_sha256\tproxy\tjudge\tgap_rel\tdate\tevaluator_commit\tnote\n")
            f.write("aaaa\t5010346\t5007345.518\t\t\t\t\n")
            f.write("bbbb\t4929726.5\t4854904.726\t\t\t\tforeign module spliced\n")
        print("[4/4] check: a 1.5 percent gap cannot decide a 30,000-unit lever")
        assert check(p, 30000.0) == 1
        assert check(p, 100000.0) == 0
    print("proxy_counter selftest: PASS")
    return 0


def main(argv):
    if "--selftest" in argv:
        return selftest()
    if len(argv) >= 3 and argv[1] == "check":
        lever = float(argv[argv.index("--lever") + 1]) if "--lever" in argv else None
        return check(argv[2], lever)
    print(__doc__)
    return 2


if __name__ == "__main__":
    sys.exit(main(sys.argv))
brainstormingobra/superpowersYou MUST use this before any creative work - creating features, building components, adding functionality, modifying behavior, or planning anything new, in software or out of it (a talk, a business, a renovation).Stars: 322kUpdated Claude CodeCodexVerifiedInstall
brand-guidelinesanthropics/skillsApplies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
frontend-designanthropics/skillsGuidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
mcp-builderanthropics/skillsGuide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK).Stars: 198kUpdated Claude CodeCodexVerifiedInstall