{ "study_id": "agent-pr-precision-v3", "discipline_version": "0.25.3", "evaluated_commit": "2b0ad6d759a9d6e77fa6bd1277eb3b17efb27e47", "host": "Apple M1 macOS 26.4.1 arm64", "confidence_level": 0.95, "cohens_kappa": 0.0, "rater_kind": "llm-subagent", "synthetic_rater_disclosure": "Synthetic-rater disclosure: Rater 1, Rater 2, and the tie-breaker in this study were executed by isolated, source-tool-blinded LLM research subagents, not human engineers. Model-to-model agreement (Cohen's kappa) establishes rubric consistency across independent prompts, not external human peer review.", "pull_request_block_rates": { "total_pull_requests": 232, "discipline": { "blocked": 42, "n": 232, "point_estimate": 0.181034, "wilson_95_ci": [ 0.136814, 0.235646 ] }, "naive_sensitive_path_filter": { "blocked": 130, "n": 232, "point_estimate": 0.560345, "wilson_95_ci": [ 0.496008, 0.622716 ] } }, "class_results": [ { "class": "structural-ast-and-deterministic", "true_positives": 119, "false_positives": 4, "cannot_tell": 2, "k": 119, "n": 125, "required_n": 141, "precision_floor": 0.9, "point_estimate": 0.952, "wilson_95_ci": [ 0.899229, 0.977818 ], "verdict": "insufficient-n" }, { "class": "heuristic-and-prose", "true_positives": 3, "false_positives": 2, "cannot_tell": 1, "k": 3, "n": 6, "required_n": 245, "precision_floor": 0.8, "point_estimate": 0.5, "wilson_95_ci": [ 0.187616, 0.812384 ], "verdict": "insufficient-n" } ], "twin_baseline_result": { "id": "naive-sensitive-path-filter", "true_positives": 5, "false_positives": 144, "cannot_tell": 1, "k": 5, "n": 150, "required_n": 141, "precision_floor": 0.9, "point_estimate": 0.033333, "wilson_95_ci": [ 0.01432, 0.075652 ], "verdict": "REJECTED", "observed_mdd_vs_structural": 0.162077, "observed_precision_delta_structural_vs_twin": 0.918667 }, "per_gate_results": [ { "gate": "agent-scratch", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "agents-md", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "archive-contents", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "assertion-reduction", "class": "structural-ast-and-deterministic", "true_positives": 4, "false_positives": 1, "cannot_tell": 0, "k": 4, "n": 5, "required_n": 141, "precision_floor": 0.9, "point_estimate": 0.8, "wilson_95_ci": [ 0.375535, 0.963776 ], "verdict": "insufficient-n" }, { "gate": "build-hooks", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "ci-integrity", "class": "structural-ast-and-deterministic", "true_positives": 66, "false_positives": 0, "cannot_tell": 0, "k": 66, "n": 66, "required_n": 141, "precision_floor": 0.9, "point_estimate": 1.0, "wilson_95_ci": [ 0.944997, 1.0 ], "verdict": "insufficient-n" }, { "gate": "ci-skip-set", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "citation-anchors", "class": "heuristic-and-prose", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 245, "precision_floor": 0.8, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "citation-metadata", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "config-integrity", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "deletion-rationale", "class": "structural-ast-and-deterministic", "true_positives": 3, "false_positives": 2, "cannot_tell": 0, "k": 3, "n": 5, "required_n": 141, "precision_floor": 0.9, "point_estimate": 0.6, "wilson_95_ci": [ 0.230724, 0.882379 ], "verdict": "insufficient-n" }, { "gate": "dependency-delta", "class": "structural-ast-and-deterministic", "true_positives": 20, "false_positives": 0, "cannot_tell": 0, "k": 20, "n": 20, "required_n": 141, "precision_floor": 0.9, "point_estimate": 1.0, "wilson_95_ci": [ 0.838875, 1.0 ], "verdict": "insufficient-n" }, { "gate": "error-swallowing", "class": "structural-ast-and-deterministic", "true_positives": 11, "false_positives": 0, "cannot_tell": 0, "k": 11, "n": 11, "required_n": 141, "precision_floor": 0.9, "point_estimate": 1.0, "wilson_95_ci": [ 0.741167, 1.0 ], "verdict": "insufficient-n" }, { "gate": "gate-command-lint", "class": "heuristic-and-prose", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 245, "precision_floor": 0.8, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "golden-output", "class": "structural-ast-and-deterministic", "true_positives": 13, "false_positives": 0, "cannot_tell": 0, "k": 13, "n": 13, "required_n": 141, "precision_floor": 0.9, "point_estimate": 1.0, "wilson_95_ci": [ 0.771905, 1.0 ], "verdict": "insufficient-n" }, { "gate": "harness-tampering", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "ignored-tests", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "instruction-smuggling", "class": "heuristic-and-prose", "true_positives": 3, "false_positives": 1, "cannot_tell": 1, "k": 3, "n": 5, "required_n": 245, "precision_floor": 0.8, "point_estimate": 0.6, "wilson_95_ci": [ 0.230724, 0.882379 ], "verdict": "insufficient-n" }, { "gate": "issue-link", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "manifest-sync", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "mechanism-sections", "class": "heuristic-and-prose", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 245, "precision_floor": 0.8, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "pii", "class": "heuristic-and-prose", "true_positives": 0, "false_positives": 1, "cannot_tell": 0, "k": 0, "n": 1, "required_n": 245, "precision_floor": 0.8, "point_estimate": 0.0, "wilson_95_ci": [ 0.0, 0.793451 ], "verdict": "insufficient-n" }, { "gate": "pr-checklist", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "provenance-tags", "class": "heuristic-and-prose", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 245, "precision_floor": 0.8, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "sandbox-config", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "scope-confinement", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "shell-secrets", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "stub-bodies", "class": "structural-ast-and-deterministic", "true_positives": 1, "false_positives": 1, "cannot_tell": 0, "k": 1, "n": 2, "required_n": 141, "precision_floor": 0.9, "point_estimate": 0.5, "wilson_95_ci": [ 0.094531, 0.905469 ], "verdict": "insufficient-n" }, { "gate": "suppression-delta", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "test-budget", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "test-floor", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 2, "k": 0, "n": 2, "required_n": 141, "precision_floor": 0.9, "point_estimate": 0.0, "wilson_95_ci": [ 0.0, 0.65762 ], "verdict": "insufficient-n" }, { "gate": "time-estimates", "class": "heuristic-and-prose", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 245, "precision_floor": 0.8, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "toolchain-config", "class": "structural-ast-and-deterministic", "true_positives": 1, "false_positives": 0, "cannot_tell": 0, "k": 1, "n": 1, "required_n": 141, "precision_floor": 0.9, "point_estimate": 1.0, "wilson_95_ci": [ 0.206549, 1.0 ], "verdict": "insufficient-n" }, { "gate": "unsafe-safety-comment", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "vacuous-tests", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" }, { "gate": "version-lockstep", "class": "structural-ast-and-deterministic", "true_positives": 0, "false_positives": 0, "cannot_tell": 0, "k": 0, "n": 0, "required_n": 141, "precision_floor": 0.9, "point_estimate": null, "wilson_95_ci": null, "verdict": "insufficient-n" } ], "previous_study_id": "agent-pr-precision-v2", "preregistration": "docs/benchmarks/precision-preregistration-v3.toml", "corpus_prior_inspection": "The 232 pull requests and every 0.23.0 finding on them were inspected and labelled under agent-pr-precision-v2; v3 characterises the 0.25.3 binary on the same changes, not precision on unseen agent pull requests.", "reproduction_check": { "reference_binary": "cargo build --release at 82d02046e4d64012413885b354873cfa769be329", "reproduced": true, "differences": [], "v2_messages_recovered": 131 }, "label_reuse": { "inherited_from_v2": 280, "fresh": 1, "fresh_discipline": 1, "fresh_twin": 0, "twin_sample_identical_to_v2": true }, "findings_diff_vs_v2": { "unchanged": 130, "changed": 1, "new": 0, "gone": 0, "gone_v2_item_ids": [], "by_gate": { "assertion-reduction": { "unchanged": 5, "changed": 0, "new": 0, "gone": 0 }, "ci-integrity": { "unchanged": 66, "changed": 0, "new": 0, "gone": 0 }, "deletion-rationale": { "unchanged": 5, "changed": 0, "new": 0, "gone": 0 }, "dependency-delta": { "unchanged": 20, "changed": 0, "new": 0, "gone": 0 }, "error-swallowing": { "unchanged": 11, "changed": 0, "new": 0, "gone": 0 }, "golden-output": { "unchanged": 13, "changed": 0, "new": 0, "gone": 0 }, "instruction-smuggling": { "unchanged": 4, "changed": 1, "new": 0, "gone": 0 }, "pii": { "unchanged": 1, "changed": 0, "new": 0, "gone": 0 }, "stub-bodies": { "unchanged": 2, "changed": 0, "new": 0, "gone": 0 }, "test-floor": { "unchanged": 2, "changed": 0, "new": 0, "gone": 0 }, "toolchain-config": { "unchanged": 1, "changed": 0, "new": 0, "gone": 0 } } }, "anchor_check": { "anchors": 20, "rater_1_agrees_with_v2_final": 10, "rater_2_agrees_with_v2_final": 15, "resolved_agrees_with_v2_final": 15, "cohens_kappa_fresh_plus_anchor": 0.0, "items": [ { "packet_item_id": "PKT-0001", "v2_item_id": "ITEM-0268", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0002", "v2_item_id": "ITEM-0251", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0003", "v2_item_id": "ITEM-0085", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0004", "v2_item_id": "ITEM-0188", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0005", "v2_item_id": "ITEM-0070", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0006", "v2_item_id": "ITEM-0007", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0007", "v2_item_id": "ITEM-0185", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "true-positive", "tie_breaker_label": "true-positive" }, { "packet_item_id": "PKT-0008", "v2_item_id": "ITEM-0249", "source_tool": "discipline", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0009", "v2_item_id": "ITEM-0036", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0010", "v2_item_id": "ITEM-0101", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0011", "v2_item_id": "ITEM-0200", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "true-positive", "tie_breaker_label": "true-positive" }, { "packet_item_id": "PKT-0013", "v2_item_id": "ITEM-0269", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0014", "v2_item_id": "ITEM-0234", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0015", "v2_item_id": "ITEM-0073", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "true-positive", "tie_breaker_label": "true-positive" }, { "packet_item_id": "PKT-0016", "v2_item_id": "ITEM-0102", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0017", "v2_item_id": "ITEM-0209", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "false-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0018", "v2_item_id": "ITEM-0107", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0019", "v2_item_id": "ITEM-0024", "source_tool": "naive-sensitive-path-filter", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "false-positive", "tie_breaker_label": null }, { "packet_item_id": "PKT-0020", "v2_item_id": "ITEM-0255", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "true-positive", "tie_breaker_label": "true-positive" }, { "packet_item_id": "PKT-0021", "v2_item_id": "ITEM-0105", "source_tool": "discipline", "v2_final_label": "true-positive", "rater_1_label": "false-positive", "rater_2_label": "true-positive", "tie_breaker_label": "true-positive" } ], "use": "drift check only; anchor labels are never final labels" } }