Skip to content

brycewang-stanford/Auto-Empirical-Research-Skills / skills/35-bahayonghang-academic-writing-skills/skills/paper-audit

paper-audit

Verified

Deep-review-first audit for Chinese and English academic papers across LaTeX, Typst, and PDF formats. Use whenever the user wants reviewer-style paper critique, pre-submission readiness checks, pass/fail gate decisions, structured revision roadmaps, or re-audits of revised manuscripts. Trigger even if the user only says "review my paper", "check if this is ready to submit", "audit this PDF", "simulate peer review", "find the biggest problems in this manuscript", or "re-check whether I fixed the review issues". Do not use for direct source editing or compilation-heavy repair; route those to the format-specific writing skills instead.

FileKindSize
SKILL.mdskill md14.3 KB
agents/claims_evidence_reviewer_agent.mdmarkdown354 B
agents/committee_editor_agent.mdmarkdown1.9 KB
agents/committee_literature_agent.mdmarkdown1.9 KB
agents/committee_logic_agent.mdmarkdown1.5 KB
agents/committee_methodology_agent.mdmarkdown1.8 KB
agents/committee_theory_agent.mdmarkdown1.9 KB
agents/critical_reviewer_agent.mdmarkdown8.4 KB
agents/domain_reviewer_agent.mdmarkdown7.1 KB
agents/editor_in_chief_agent.mdmarkdown5.2 KB
agents/evaluation_fairness_reviewer_agent.mdmarkdown371 B
agents/literature_reviewer_agent.mdmarkdown3.6 KB
agents/methodology_reviewer_agent.mdmarkdown7.2 KB
agents/notation_consistency_reviewer_agent.mdmarkdown398 B
agents/prior_art_reviewer_agent.mdmarkdown2.9 KB
agents/section_reviewer_agent.mdmarkdown365 B
agents/self_consistency_reviewer_agent.mdmarkdown420 B
agents/synthesis_agent.mdmarkdown951 B
evals/evals.jsonconfig12.0 KB
evals/fixtures/deep_review_fixture.texother699 B
evals/fixtures/gate_ieee_fixture.texother617 B
evals/fixtures/polish_fixture.texother396 B
evals/fixtures/previous_final_issues.jsonconfig1.1 KB
evals/fixtures/previous_review_report.mdmarkdown205 B
evals/fixtures/quick_audit_fixture.texother538 B
examples/gate_example.mdmarkdown3.5 KB
examples/review_example.mdmarkdown2.5 KB
examples/self_check_example.mdmarkdown2.9 KB
references/AUDIT_GUIDE.mdmarkdown3.4 KB
references/CHANGELOG.mdmarkdown530 B
references/CHECKLIST.mdmarkdown6.0 KB
references/CONSOLIDATION_RULES.mdmarkdown907 B
references/DEEP_REVIEW_CRITERIA.mdmarkdown2.8 KB
references/FORBIDDEN_TERMS.mdmarkdown6.4 KB
references/ISSUE_SCHEMA.mdmarkdown928 B
references/LITERATURE_GROUNDING_GUIDE.mdmarkdown3.3 KB
references/POLISH_GUIDE.mdmarkdown2.3 KB
references/QUALITATIVE_STANDARDS.mdmarkdown5.5 KB
references/QUICK_REFERENCE.mdmarkdown1.1 KB
references/REVIEW_CRITERIA.mdmarkdown2.7 KB
references/REVIEW_LANE_GUIDE.mdmarkdown828 B
references/SCHOLAR_EVAL_GUIDE.mdmarkdown6.4 KB
references/SCORING_SYSTEMS.mdmarkdown1000 B
references/SUBAGENT_TEMPLATES.mdmarkdown987 B
references/TROUBLESHOOTING.mdmarkdown1.4 KB
references/VENUE_RULES.mdmarkdown764 B
references/editorial_decision_standards.mdmarkdown6.7 KB
references/quality_rubrics.mdmarkdown10.8 KB
scripts/audit.pyscript78.4 KB
scripts/build_claim_map.pyscript3.3 KB
scripts/check_citations.pyscript5.6 KB
scripts/check_references.pyscript19.9 KB
scripts/consolidate_review_findings.pyscript5.2 KB
scripts/detect_language.pyscript1.9 KB
scripts/diff_review_issues.pyscript2.0 KB
scripts/literature_compare.pyscript11.4 KB
scripts/literature_search.pyscript22.0 KB
scripts/models/scoring_model.jsonconfig737 B
scripts/parsers.pyscript3.3 KB
scripts/pdf_parser.pyscript11.4 KB
scripts/prepare_review_workspace.pyscript6.8 KB
scripts/render_deep_review_report.pyscript3.5 KB
scripts/report_generator.pyscript36.2 KB
scripts/scholar_eval.pyscript13.7 KB
scripts/scoring_model.pyscript6.2 KB
scripts/verify_quotes.pyscript1.6 KB
scripts/visual_check.pyscript10.7 KB
templates/audit_report_template.mdmarkdown2.8 KB
templates/final_issues_template.jsonconfig498 B
templates/review_report_template.mdmarkdown678 B
templates/revision_roadmap_template.mdmarkdown3.1 KB

scripts/scoring_model.py

View on GitHub
"""
Linear Regression Scoring Model for ScholarEval.

Provides a trained/trainable regression model for predicting overall paper
quality from 9 dimension scores + interaction features.
Falls back to weighted average when untrained.
"""

from __future__ import annotations

import json
from dataclasses import dataclass, field
from pathlib import Path


@dataclass
class ScoringPrediction:
    """Prediction result from the scoring model."""

    predicted_score: float
    confidence_interval: tuple[float, float]  # 95% CI
    feature_contributions: dict[str, float] = field(default_factory=dict)
    decision: str = ""  # readiness label
    model_type: str = ""  # "regression" or "weighted_average"


class RegressionScorer:
    """Ridge regression scorer for paper quality prediction."""

    FEATURE_NAMES: list[str] = [
        # 9 base dimensions
        "soundness",
        "clarity",
        "presentation",
        "novelty",
        "significance",
        "reproducibility",
        "ethics",
        "literature_grounding",
        "overall_base",
        # 3 interaction terms
        "soundness_x_novelty",
        "clarity_x_significance",
        "literature_grounding_x_novelty",
        # 2 meta features
        "critical_count",
        "dims_below_5",
    ]

    def __init__(
        self,
        coefficients: dict[str, float] | None = None,
        intercept: float = 0.0,
    ):
        self.coefficients: dict[str, float] = coefficients or {}
        self.intercept = intercept
        self._is_trained = bool(coefficients)

    def predict(
        self,
        dimension_scores: dict[str, float | None],
        critical_count: int = 0,
    ) -> ScoringPrediction:
        """Predict overall score from dimension scores.

        Falls back to weighted average if untrained.
        """
        if not self._is_trained:
            return self._fallback_predict(dimension_scores)

        features = self._extract_features(dimension_scores, critical_count)
        score = self.intercept + sum(
            self.coefficients.get(name, 0.0) * value for name, value in features.items()
        )
        score = max(1.0, min(10.0, score))

        # Simple CI estimate based on model uncertainty
        ci_width = 0.5
        ci = (max(1.0, score - ci_width), min(10.0, score + ci_width))

        # Feature contributions
        contributions = {
            name: round(self.coefficients.get(name, 0.0) * value, 3)
            for name, value in features.items()
            if abs(self.coefficients.get(name, 0.0) * value) > 0.01
        }

        return ScoringPrediction(
            predicted_score=round(score, 1),
            confidence_interval=(round(ci[0], 1), round(ci[1], 1)),
            feature_contributions=contributions,
            decision=self._score_to_decision(score),
            model_type="regression",
        )

    def _fallback_predict(self, dimension_scores: dict[str, float | None]) -> ScoringPrediction:
        """Weighted average fallback matching v2.0 behavior."""
        from scholar_eval import SCHOLAR_EVAL_DIMENSIONS, get_readiness_label

        total_weight = 0.0
        weighted_sum = 0.0
        for dim, cfg in SCHOLAR_EVAL_DIMENSIONS.items():
            if dim == "overall":
                continue
            score = dimension_scores.get(dim)
            if score is not None:
                weighted_sum += score * cfg["weight"]
                total_weight += cfg["weight"]

        avg = weighted_sum / total_weight if total_weight > 0 else 5.0

        return ScoringPrediction(
            predicted_score=round(avg, 1),
            confidence_interval=(
                round(max(1.0, avg - 1.5), 1),
                round(min(10.0, avg + 1.5), 1),
            ),
            feature_contributions={},
            decision=get_readiness_label(avg),
            model_type="weighted_average",
        )

    def _extract_features(
        self,
        scores: dict[str, float | None],
        critical_count: int = 0,
    ) -> dict[str, float]:
        """Extract 14 features from dimension scores."""

        def _g(key: str) -> float:
            v = scores.get(key)
            return v if v is not None else 5.0

        return {
            "soundness": _g("soundness"),
            "clarity": _g("clarity"),
            "presentation": _g("presentation"),
            "novelty": _g("novelty"),
            "significance": _g("significance"),
            "reproducibility": _g("reproducibility"),
            "ethics": _g("ethics"),
            "literature_grounding": _g("literature_grounding"),
            "overall_base": _g("overall"),
            # Interaction terms
            "soundness_x_novelty": _g("soundness") * _g("novelty") / 10.0,
            "clarity_x_significance": _g("clarity") * _g("significance") / 10.0,
            "literature_grounding_x_novelty": (_g("literature_grounding") * _g("novelty") / 10.0),
            # Meta features
            "critical_count": float(critical_count),
            "dims_below_5": float(sum(1 for v in scores.values() if v is not None and v < 5.0)),
        }

    @staticmethod
    def _score_to_decision(score: float) -> str:
        """Map score to readiness label."""
        from scholar_eval import get_readiness_label

        return get_readiness_label(score)

    @classmethod
    def load_model(cls, path: str | Path) -> RegressionScorer:
        """Load model from JSON file."""
        data = json.loads(Path(path).read_text(encoding="utf-8"))
        return cls(
            coefficients=data.get("coefficients", {}),
            intercept=data.get("intercept", 0.0),
        )

    def save_model(self, path: str | Path) -> None:
        """Save model to JSON file."""
        data = {
            "coefficients": self.coefficients,
            "intercept": self.intercept,
            "feature_names": self.FEATURE_NAMES,
        }
        Path(path).write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")

    @classmethod
    def train(cls, data_path: str | Path) -> RegressionScorer:
        """Train from labeled data (CSV with dimension scores + overall label).

        Requires numpy and scikit-learn.
        """
        raise NotImplementedError(
            "Training requires labeled data (OpenReview/PeerRead). "
            "Use default model or provide pre-trained coefficients."
        )
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall