{ "manifest_version": "0.1", "work_slug": "data-marketplace-slr-screening-analysis", "protocol": "protocol:secondary-analysis 0.1", "module": "agent/tools.py", "analysis_unit": "analysis.py", "published_results": "results.json", "source_dataset_doi": "10.4121/14673813.v2", "environment": { "language": "python", "tested_version": "3.13", "requirements": "requirements-reproduction.txt", "network": "deny-except-pinned-inputs" }, "execution": { "boundary": "docs/72-reproduction-contract.md", "live_invocation": "not connected; the sandboxed runner gates execution over MCP" }, "surface_note": "analysis.py names only its helpers (acquire_data, load_screening, cochran_armitage); every test lives inline in run_analysis() with its subset, scores, bin edges and null frozen. The tools re-implement those inline units over the named helpers, and each published acceptance call reproduces its results.json block whole, which is the evidence that the re-implementation is faithful. The pinned workbook is fetched through scripts/fetch_pinned.py against the same committed md5, because acquire_data falls back to a path on the authoring machine and fails with sys.exit.", "tools": [ { "name": "corpus_summary", "summary": "Size, span, sources and verified identity of the screening sheet.", "unit": "analysis.acquire_data constants and analysis.load_screening (the _meta block of run_analysis)", "parameters": [], "returns": "seed, source_md5 (digested from the fetched bytes), expected_md5, sheet, n_records, n_include, n_exclude, year_min, year_max, n_years, records_by_source", "supports_claims": [1, 2, 3], "deterministic": true }, { "name": "normality_check", "summary": "Shapiro-Wilk on publication year per decision: the case for rank-based tests.", "unit": "run_analysis normality_shapiro block (scipy.stats.shapiro)", "parameters": [], "returns": "test, W/p/n per decision, and the conclusion derived from the p-values", "supports_claims": [1], "deterministic": true }, { "name": "year_by_decision_test", "summary": "Mann-Whitney U on publication year, Include vs Exclude (H1), with rank-biserial effect size.", "unit": "run_analysis H1_mannwhitney_year_by_decision block, re-implemented; Holm via analysis.multipletests", "parameters": [ { "name": "alternative", "type": "string", "default": "two-sided", "range": "two-sided | greater (Include years stochastically later) | less", "published_values": ["two-sided"] }, { "name": "scopus_only", "type": "boolean", "default": false, "range": "true drops the 9 hand-seeded core articles (all Includes); Source has only these two values", "published_values": [false, true], "notes": "true reproduces the Mann-Whitney half of R1." }, { "name": "year_min", "type": "integer | null", "default": null, "range": "inclusive lower publication year; the data span 1980-2020 over 33 distinct years", "published_values": [null] }, { "name": "year_max", "type": "integer | null", "default": null, "range": "inclusive upper publication year; both decisions must remain in the window", "published_values": [null], "notes": "2009 closes the published R2 bin (2004, 2009], after which the binned inclusion rate goes from 0.09 to 0.60: 80 records up to 2009 (11 included), 425 from 2010 (269 included). The split is post hoc; the acceptance notes record how the within-era results depend on it." } ], "returns": "the published H1 block: test, statistic_U, p, effect_size_rank_biserial, auc_include_gt_exclude, medians, IQRs, both n, p_holm and significant_holm (Holm within the pre-registered family with this call substituted for H1)", "supports_claims": [1], "deterministic": true }, { "name": "inclusion_trend_test", "summary": "Cochran-Armitage trend in inclusion proportion across ordered publication years (H2), with Spearman rho.", "unit": "analysis.cochran_armitage over the per-year counts of run_analysis H2_cochran_armitage_trend; Holm via analysis.multipletests", "parameters": [ { "name": "scores", "type": "string", "default": "value", "range": "value (calendar year, as published) | rank (1..k over the observed years)", "published_values": ["value"], "notes": "Cochran-Armitage depends on its scores; rank scores remove the leverage of isolated early years." }, { "name": "scopus_only", "type": "boolean", "default": false, "range": "true drops the 9 hand-seeded core articles", "published_values": [false, true], "notes": "true reproduces the trend half of R1." }, { "name": "year_min", "type": "integer | null", "default": null, "range": "inclusive lower publication year, 1980-2020; at least two distinct years must remain", "published_values": [null] }, { "name": "year_max", "type": "integer | null", "default": null, "range": "inclusive upper publication year, 1980-2020", "published_values": [null] } ], "returns": "the published H2 block: test, statistic_Z, p, spearman_rho_year_include, spearman_p, direction, n, n_years, p_holm and significant_holm (Holm within the family with this call substituted for H2)", "supports_claims": [2], "deterministic": true }, { "name": "exclusion_reason_test", "summary": "Chi-square goodness of fit: are the exclusion-reason groups equiprobable? (H3), with Cohen's w.", "unit": "run_analysis H3_chisquare_exclusion_groups block, re-implemented with analysis.EXCL_LABELS; Holm via analysis.multipletests", "parameters": [ { "name": "groups", "type": "array of integer | null", "default": null, "range": "any two or more of the coded groups 1-5; null uses all five, as published", "published_values": [null], "notes": "Dropping group 1 (off-topic, 160 of 225) tests whether the remaining reasons are still non-uniform." } ], "returns": "the published H3 block: test, statistic_chi2, p, dof, effect_size_cohens_w, observed, expected_each, labels, n, p_holm and significant_holm (Holm within the family with this call substituted for H3)", "supports_claims": [3], "deterministic": true }, { "name": "holm_correction", "summary": "Holm-Bonferroni over a family of p-values, in input order.", "unit": "run_analysis holm_correction block (analysis.multipletests, method='holm')", "parameters": [ { "name": "p_values", "type": "array of number | null", "default": null, "range": "any non-empty list of p-values in [0, 1]; null recomputes the pre-registered family of H1, H2, H3", "published_values": [null] } ], "returns": "method, family, raw_p, corrected_p, reject_at_0.05", "supports_claims": [1, 2, 3], "deterministic": true }, { "name": "subset_robustness", "summary": "Re-run H1 and H2 at their published definitions on a subset of records (R1).", "unit": "run_analysis R1_robustness_scopus_only block, re-implemented over the same H1 and H2 units", "parameters": [ { "name": "scopus_only", "type": "boolean", "default": true, "range": "true drops the 9 hand-seeded core articles, as published; false keeps them", "published_values": [true], "notes": "The description counts the non-Scopus records dropped inside the year window. All 9 date from 2014-2020, so a window ending before 2014 drops none and says so." }, { "name": "year_min", "type": "integer | null", "default": null, "range": "inclusive lower publication year, 1980-2020", "published_values": [null] }, { "name": "year_max", "type": "integer | null", "default": null, "range": "inclusive upper publication year, 1980-2020", "published_values": [null] } ], "returns": "the published R1 block: description, n, n_include, n_exclude, mannwhitney {statistic_U, p, effect_size_rank_biserial, medians}, cochran_armitage {statistic_Z, p}", "supports_claims": [1, 2], "deterministic": true }, { "name": "binned_trend_test", "summary": "Cochran-Armitage trend on publication-year bins, scored at bin midpoints (R2).", "unit": "run_analysis R2_sensitivity_5yr_bins block (pd.cut + analysis.cochran_armitage), re-implemented", "parameters": [ { "name": "bin_edges", "type": "array of number", "default": [1979, 1984, 1989, 1994, 1999, 2004, 2009, 2014, 2020], "range": "three or more strictly increasing edges; bins are right-closed and must cover every analysed year (1980-2020), or the call fails rather than dropping records", "published_values": [[1979, 1984, 1989, 1994, 1999, 2004, 2009, 2014, 2020]], "notes": "The published last bin, (2014, 2020], is six years wide although the description says five." }, { "name": "scopus_only", "type": "boolean", "default": false, "range": "true drops the 9 hand-seeded core articles", "published_values": [false] } ], "returns": "the published R2 block: description, bins, n_per_bin, include_per_bin, inclusion_rate_per_bin, cochran_armitage {statistic_Z, p}", "supports_claims": [2], "deterministic": true } ] }