{ "dandiset": "000232", "page": "dandisets/000232/", "generated": "2026-10-19T16:23:39+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 19, "partially_reproduced": 7, "not_reproduced": 2, "blocked": 1 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000232-chang-2023-01", "dandiset": "000232", "dandiset_version": "0.240510.2038", "paper": "10.1101/2023.08.21.554194", "paper_title": "Preprint for: Rule-based modulation of a sensorimotor transformation across cortical areas", "date": "2026-10-17", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code (GitHub Actions)", "model": "claude-opus-5-5", "protocol": "pilot-7" }, "tier": "agent_verified", "entry": "000232-chang-2023", "entry_title": "Rule-based modulation of a sensorimotor transformation across cortical areas (Chang et al. 2023)", "description": "Tests the findings of Chang, Finkel, Xu and O'Connor (bioRxiv 2023, version 2) on dandiset 000232, which holds silicon-probe recordings from S1, S2, MM, ALM and AMM of mice performing a cross-modal sensory selection task, and the behaviour of optogenetic inhibition sessions in that task and in a tactile detection task. It covers behavioural performance, single-neuron rule and stimulus discrimination, population decoding of the rule, neural trajectories and subspaces, coding dimensions, rule transitions and the effects of optogenetic inhibition.", "notebooks": [ { "path": "reproduce_chang_2023.ipynb", "title": "Reproducing Chang et al. (2023) from dandiset 000232", "executed": "notebooks/000232-chang-2023/reproduce_chang_2023.html" } ], "outcomes": [ { "claim": "fig1e-recording-counts", "statement": "Single units were recorded with 64-channel probes in S1 (6 mice, 10 sessions, 177 neurons), S2 (5 mice, 8 sessions, 162 neurons), MM (7 mice, 9 sessions, 140 neurons) and ALM (8 mice, 13 sessions, 256 neurons), one area per session, and in AMM (110 neurons).", "source": "Results, first section; Figure 1E and 1G; Figure S1K", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Sessions, mice and units per area match exactly; S1 10, 6, 177; S2 8, 5, 162; MM 9, 7, 140; ALM 13, 8, 256; AMM 110 units in 9 sessions from 7 mice. The units are those left after the paper's quality filter.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig6-session-counts", "statement": "The optogenetic experiments in the cross-modal task comprise S1/S2 (4 mice, 10 sessions), MM (4, 11), ALM (4, 10), sham (7, 28), AMM (7, 21) and PPC (7, 21); in the tactile detection task sham (5, 9), S1/S2 (3, 3), MM (3, 5), ALM (3, 4), AMM (3, 4) and PPC (3, 4).", "source": "Figure 6 legend", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "All 24 counts of sessions and mice per optogenetic condition match the Figure 6 legend. The code's performance check, ported here, passes 27 of the 28 cross-modal sham sessions and all other cross-modal sessions.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig1d-performance", "statement": "Performance was about 75% correct under both rules (respond-to-touch 74 plus or minus 1%, respond-to-light 75 plus or minus 1%, mean and SEM over 12 mice), with no difference between them (paired t-test, p = 0.56).", "source": "Results, first section; Figure 1D", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Reading (a), trials as deposited, gives 73.6 plus or minus 0.8% (touch) and 74.8 plus or minus 1.5% (light) over 12 mice, paired t-test p = 0.561, matching the reported 74, 75 and p = 0.56. Reading (b), without the last 20 trials, gives 72.8 and 74.2% (p = 0.558), with the touch value outside tolerance.", "deviations": [ "The Methods remove the last 20 trials of each session and the code does not; the files may already lack them. Both readings are shown and reading (a) matches." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig1d-outcome-fractions", "statement": "The fractions of trial outcomes were similar under the two rules (touch against light; hit 36 against 35%, correct rejection 38 against 40%, miss 13 against 14%, false alarm 13 against 12%).", "source": "Figure 1D legend", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Under reading (a) all eight fractions are within the paper's SEM (touch against light; hit 35.6 and 35.2, correct rejection 38.1 and 39.6, miss 13.0 and 13.6, false alarm 13.3 and 11.6%). Reading (b) puts 6 of 8 within. Gaps between rows, which mark removed trials, separate 4.5% of consecutive rows within the last 20 rows of a session against 12.7% earlier; this is consistent with the last rows being the end of the session but does not show it.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig1h-thit-tcr-selective", "statement": "18 to 33% of neurons discriminate tactile hits from tactile correct rejections in the first 150 ms after the stimulus (S1 28.8%, S2 17.9%, MM 32.9%, ALM 23.4%), and most of these respond more in tactile hits (AUC above 0.5).", "source": "Results, first section; Figure 1H", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "S1 30.5, S2 16.7 and ALM 23.0% are within tolerance; MM is 36.4% against 32.9 plus or minus 3.29, and above the stated range widened by 10%. Over four other seeds MM gives 35.0 to 36.4%, S1 28.2 to 30.5, S2 17.9 to 19.1 and ALM 21.9 to 25.0%. MM is within tolerance in 3 of the 5 seeds, and the five-seed means (S1 29.7, S2 18.1, MM 35.6, ALM 23.1%) are all within tolerance, so the partial outcome depends on the bootstrap draw; it is judged on the main seed. 165 of 191 selective neurons (86.4%) have mean AUC above 0.5, as the claim states.", "deviations": [ "The bootstrap resamples are shared by all units and bins of a session; the code draws them separately. Each unit's interval depends only on its own resamples." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig2c-rule-prestim-units", "statement": "Before the stimulus (-100 to 0 ms), fewer than 5% of neurons in S1 and S2 discriminate the rule, while about 10 to 20% do in MM and ALM (S1 4.5%, S2 2.5%, MM 21.4%, ALM 10.2%).", "source": "Results, second section; Figure 2C", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "S2 2.5, MM 20.0 and ALM 9.8% are within tolerance; S1 is 5.6% (10 neurons) against 4.5% (8 neurons). Over the main seed and four others S1 gives 3.4 to 5.6% and is within tolerance in none (9 neurons, 5.1%, is just above it); ALM is within tolerance in 2 of 5. The five-seed means (S1 5.0, S2 2.8, MM 20.7, ALM 9.9%) are all within tolerance. The outcome is judged on the main seed. MM and ALM each exceed S1 and S2, as the claim states.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig2d-stimulus-prestim-units", "statement": "As a negative control, almost no neuron discriminates tactile from visual trials before the stimulus (S1 0%, S2 0%, MM 0%, ALM 0.8%).", "source": "Results, second section; Figure 2D", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "0% in S1, S2 and MM and 0.8% (2 neurons) in ALM, as in the paper.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig2e-stimulus-poststim-units", "statement": "After the stimulus (0 to 100 ms), more than 35% of neurons in S1 and S2 and about 15% in MM and ALM discriminate tactile from visual trials (S1 37.3%, S2 35.2%, MM 15%, ALM 14.1%).", "source": "Results, second section; Figure 2E", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "All four within tolerance of 37.3, 35.2, 15 and 14.1%.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig2f-prestim-predicts-touch-response", "statement": "Among neurons with significant rule discrimination before the stimulus, the pre-stimulus rule preference predicts the rule modulation of the early touch response (0 to 50 ms); Pearson r = 0.93 in MM (30 neurons) and 0.83 in ALM (26 neurons), both p < 0.001 (S1 r = 0.69, 8 neurons, p = 0.056; S2 r = 0.91, 4 neurons, p = 0.093).", "source": "Results, second section; Figure 2F", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "MM r = 0.958 over 28 neurons and ALM r = 0.841 over 25 neurons, both p < 0.001 (paper 30 and 26 neurons). S1 r = 0.681 (10 neurons, p = 0.030) and S2 r = 0.902 (4 neurons, p = 0.0985), against the paper's 0.69 (8 neurons, p = 0.056) and 0.91 (4, p = 0.093).", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig3b-rule-decoding", "statement": "The rule can be decoded from pre-stimulus population activity (-100 to 0 ms) in each area; median LDA accuracy over sessions, true against shuffled, S1 0.61 against 0.5, S2 0.62 against 0.53, MM 0.7 against 0.52, ALM 0.68 against 0.55.", "source": "Results, third section; Figure 3B", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Shuffled medians are 0.564, 0.532, 0.526 and 0.535; 37 of 40 sessions exceed their shuffled value. The S1 sessions differ from shuffled at Wilcoxon p = 0.055, the others at p below 0.01. The shuffled medians (0.526 to 0.564) are about equal to the larger block's share of correct trials (medians 0.529 to 0.556 by area).", "deviations": [ "The Methods name LDA; the code's diagonal LDA is used. The Methods' soft normalization and the code's z-score give the same diagonal LDA predictions." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig3c-rule-decoding-motor-stronger", "statement": "Rule decoding is robust in the motor areas, whose bootstrap 95% CIs of mean accuracy do not overlap those of shuffled data (MM 0.60 to 0.73 against 0.48 to 0.53; ALM 0.56 to 0.66 against 0.49 to 0.54), and limited in the sensory areas, whose true CIs are above 0.5 but overlap the shuffled ones (S1 0.52 to 0.61 against 0.49 to 0.55; S2 0.53 to 0.63 against 0.49 to 0.54).", "source": "Results, third section; Figure 3C", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Bootstrap 95% CIs, true against shuffled; MM 0.601 to 0.731 against 0.472 to 0.523; ALM 0.560 to 0.655 against 0.495 to 0.544 (no overlap); S1 0.530 to 0.614 against 0.493 to 0.545; S2 0.528 to 0.619 against 0.488 to 0.539 (overlap). These are within 0.012 of the paper's bounds.", "deviations": [ "200 resamples per area; the code's header sets 100 and its saved file names say 1000." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig3de-stimulus-decoding-controls", "statement": "Population activity discriminates tactile from visual trials after the stimulus (0 to 100 ms) but not before it (-100 to 0 ms).", "source": "Results, third section; Figure 3D and 3E; Figure S3C and S3D", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Before the stimulus true means 0.495 to 0.509 with CIs overlapping shuffled in all areas; after it 0.642 (MM) to 0.805 (S2), no overlap in any area.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "figs3-svm-rf-decoding", "statement": "Support vector machines and random forests decode the rule from pre-stimulus activity about as well as LDA (median accuracy true against shuffled; SVM S1 0.6 against 0.53, S2 0.61 against 0.51, MM 0.71 against 0.51, ALM 0.65 against 0.52; random forests S1 0.59 against 0.52, S2 0.6 against 0.52, MM 0.65 against 0.49, ALM 0.7 against 0.5).", "source": "Results, third section; Figure S3A and S3B", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "All eight within 10% and above their shuffled medians. With the code's z-score the SVM medians are 0.575, 0.603, 0.701 and 0.642.", "deviations": [ "scikit-learn's SVC and random forest stand in for MATLAB fitcsvm and TreeBagger with their defaults." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig4c-prestim-distance-predicts-divergence", "statement": "Sessions with a larger pre-stimulus distance between tactile hit and correct rejection trajectories (top 50% of 40 sessions) show a larger distance after the stimulus than the bottom 50% (permutation test, p < 0.001).", "source": "Results, fourth section; Figure 4B and 4C", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Top 20 sessions 1.171 against bottom 20 0.966 mean post-stimulus distance, permutation p = 0.004. With the code's normalization (range without the offset 5) p = 0.001. Pre- and post-stimulus mean distances correlate over sessions (Spearman 0.54, p = 0.0004).", "deviations": [ "The code's windows are offset by one 10 ms bin from the stated ones; the stated windows are used." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig4e-subspace-overlap", "statement": "In all four areas the subspace overlap of tactile correct rejections with tactile hits (0 to 150 ms) is lower than that of held-out tactile hits (tCR minus control tHit; S1 -0.23, 8 sessions, p = 0.0016; S2 -0.23, 7 sessions, p = 0.0086; MM -0.36, 5 sessions, p < 0.001; ALM -0.35, 11 sessions, p < 0.001; paired t-test).", "source": "Results, fifth section; Figure 4E", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Negative in all four areas with p = 0.0007, 0.025, 0.003 and below 0.0001 (8, 7, 5 and 12 sessions), but only S2 is within 10% of the paper. MM and ALM are larger in magnitude than the reported -0.36 and -0.35; the code's single-split procedure repeated 200 times gives 95% ranges of -0.497 to -0.376 (MM) and -0.473 to -0.359 (ALM), which exclude the reported values.", "deviations": [ "Sessions with at least 10 units (Methods), 12 ALM sessions; the code keeps more than 10, 11 ALM sessions, as the paper reports. Both are shown and give the same result.", "Overlaps average 100 random equal splits with offset 5 throughout; the code uses one split drawn separately for each neuron and offsets 5 and 0.01, shown as a variant." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig4f-prestim-overlap-predicts-poststim", "statement": "Across sessions, the tCR subspace overlap before the stimulus (-100 to 0 ms) correlates with that after it (Pearson r = 0.68, p < 0.001, 31 sessions).", "source": "Results, fifth section; Figure 4F", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "r = 0.688 over 32 sessions (p = 1.3e-05); 0.686 over the code's 31 sessions; the code's single-split procedure gives 0.573 to 0.734 over 200 runs.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "figs4ef-coding-dimensions", "statement": "Stimulus coding dimensions are aligned between the two rules in S1, S2 and MM but not in ALM, while choice coding dimensions (lick against no lick) are not aligned in any area (dot product magnitudes, true against shuffled, with 95% CIs).", "source": "Results, sixth section; Figure S4E and S4F", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Seven of eight judgements match. The MM stimulus CD is not aligned (0.366, CI 0.218 to 0.521, shuffled 0.150 to 0.441), where the paper reports 0.48 against 0.24 with non-overlapping CIs; with the code's sessions of more than 10 units it is 0.295 and also not aligned. S1, S2 and ALM values are close to the paper's (0.557, 0.620, 0.348 against 0.5, 0.62, 0.33).", "deviations": [ "All sessions are used, as the paper's session counts state; the code keeps sessions with more than 10 units, shown as a variant." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "figs4h-choice-right-left", "statement": "Choice coding dimensions computed from right against left lick trials are still not well aligned between the two rules (true against shuffled; S1 0.39 against 0.2, S2 0.32 against 0.2, MM 0.35 against 0.18, ALM 0.28 against 0.21).", "source": "Results, sixth section; Figure S4H", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "All four below 0.5; S1, MM and ALM within 0.1 of the paper, S2 0.422 against 0.32. All four CIs overlap the shuffled ones. Over the main seed and four others the S2 mean is 0.420 to 0.425, within 0.1 of the paper in 1 of 5 (0.420); the outcome is judged on the main seed.", "deviations": [ "The criterion's \"needed condition\" first read as each of the four rare categories left 1 of 10 S1 sessions, none in S2 and MM and 1 of 13 in ALM, so it could not decide; following the code's nanmean over stimuli, right- and left-lick trials are pooled over stimuli (at least 2 each, all 40 sessions). The first reading is shown as a variant." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig5b-first-hit-switches", "statement": "Mice switched before or immediately after the cue on the 9th trial; there were 88 switches into respond-to-touch blocks and 91 into respond-to-light blocks.", "source": "Results, seventh section; Figure 5B", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Non-first blocks with a hit, as the criterion and the code's GetFirstHitInfo define them, number 63 (touch) and 75 (light), against 88 and 91. Both counts fail, so two of the three conditions fail and the outcome is not reproduced (round 1 recorded partially reproduced, which the criterion does not allow). The third condition holds; first hits at or before the trial after the cue are 57.1% (touch) and 65.2% (light) of blocks with a cue row, counted in rows of the file, and 55.4% and 62.1% in trial numbers estimated from the gaps between rows. All blocks, including the first of each session, number exactly 88 and 91, and blocks with a hit including first blocks 87 and 91. The code's GetFirstHitInfo returns one entry per block, NaN for first blocks and blocks without a hit, so a count of its entries gives the number of blocks.", "deviations": [ "The code numbers trials by the behavioural system, including trials the files lack; first hits are placed relative to the cue row in the files, and as a variant in trial numbers estimated from the gaps between rows (xm.estimated_trial_numbers)." ], "disagreed": true, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig5de-motor-states-track-switch", "statement": "Over a rule transition, the fraction of trials classified as respond-to-touch from pre-stimulus activity changes progressively in MM and ALM (Kendall's tau, mean and 95% CI; touch to light MM -0.39 [-0.67, -0.11], ALM -0.29 [-0.54, -0.04]; light to touch MM 0.37 [0.07, 0.63], ALM 0.27 [0.03, 0.49]).", "source": "Results, seventh section; Figure 5D and 5E", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Touch to light MM -0.401 [-0.667, -0.111], ALM -0.302 [-0.538, -0.051]; light to touch MM 0.382 [0.074, 0.630], ALM 0.268 [0.026, 0.513]; all within 0.04 of the paper's values and bounds.", "deviations": [ "LDA with Ledoit-Wolf shrinkage stands in for the code's optimized LDA; 500 resamples (code 1000)." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "figs5-sensory-states-no-tracking", "statement": "Classification of trials in S1 and S2 shows no clear trend across block transitions (Kendall's tau touch to light S1 -0.16 [-0.43, 0.1], S2 -0.15 [-0.54, 0.21]; light to touch S1 0.21 [-0.07, 0.5], S2 0.25 [-0.17, 0.63]).", "source": "Results, seventh section; Figure S5", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "S1 -0.163 [-0.433, 0.117] and 0.219 [-0.067, 0.500]; S2 -0.140 [-0.500, 0.250] and 0.259 [-0.105, 0.620]; all include 0. The 10-fold estimates without resampling are as large in S1 and S2 as in MM and ALM (S2 to touch 0.583, MM 0.519), so the absence rests on wide CIs from 10 and 8 sessions.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig6c-pre-inhibition-tactile", "statement": "Inhibiting MM or ALM before the stimulus reduces tactile detection sensitivity in respond-to-touch blocks, while inhibiting S1/S2, AMM or the sham does not (95% CI of the change; S1/S2 [-0.47, 0.08], MM [-0.65, -0.12], ALM [-0.68, -0.02], sham [-0.12, 0.13], AMM [-0.19, 0.06]).", "source": "Results, eighth section; Figure 6C", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "MM -0.390 (CI -0.658 to -0.123) is below 0; ALM -0.373 has a CI of -0.707 to 0.005 when false-alarm trials are also resampled (Methods). S1/S2, sham and AMM include 0. With the code's resampling, which keeps the observed false-alarm rate, ALM is -0.682 to -0.018 and every CI is within 0.03 of the paper's, which would make the claim reproduced. The outcome is judged on the main seed; the ALM upper bound is at the edge of 0. Over ten other seeds it averages -0.001 (SD 0.003, range -0.006 to 0.001) and is below 0 in 7 of 10; with the code's resampling it is below 0 in all ten (mean -0.018). No other Methods-resampling bound of the optogenetic CIs checked over seeds changes side; the figs6b ALM interval with the code's resampling does (upper bound 0.002 on the main draw, below 0 in 9 of 10 other seeds), and the fig6ij laser-only calls keep their side in all ten seeds.", "deviations": [ "Hit and false-alarm trials are both resampled; the code resamples hits only, shown as a variant." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "figs6b-pre-inhibition-hit-rate", "statement": "The loss of tactile sensitivity with pre-stimulus inhibition is mainly a fall in the tactile hit rate (95% CI of the change; S1/S2 [-0.31, 0.01], MM [-0.49, -0.15], ALM [-0.46, -0.07]).", "source": "Results, eighth section; Figure S6B", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "MM -0.318 (-0.492 to -0.152), ALM -0.227 (-0.463 to -0.007), S1/S2 -0.147 (-0.303 to 0.010). The ALM upper bound is close to 0. Over ten other seeds it is below 0 in all ten (mean -0.004, SD 0.002). The code's resampling is the same procedure here, as there is no false-alarm group; its draw gives 0.002 and is below 0 in 9 of 10 other seeds.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig6d-post-inhibition-tactile", "statement": "Inhibiting S1/S2, MM or ALM from the stimulus onset reduces tactile sensitivity (95% CI S1/S2 [-0.67, -0.08], MM [-0.6, -0.34], ALM [-0.55, -0.27]).", "source": "Results, eighth section; Figure 6D", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "S1/S2 -0.389 (-0.699 to -0.080), MM -0.487 (-0.612 to -0.327), ALM -0.402 (-0.568 to -0.253).", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "figs7gh-visual-sensitivity", "statement": "Visual detection sensitivity in respond-to-light blocks falls with pre-stimulus inhibition of MM [-0.6, -0.18] and AMM [-0.35, -0.08] but not of ALM [-0.5, 0.04] or S1/S2 [-0.27, 0.05]; with inhibition from the stimulus onset it falls for MM, ALM and AMM but not for S1/S2 [-0.44, 0.04].", "source": "Results, eighth section; Figure S7G and S7H", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "All eight judgements match; pre MM -0.603 to -0.173, AMM -0.356 to -0.082, ALM -0.494 to 0.050, S1/S2 -0.282 to 0.070; post MM -0.839 to -0.518, ALM -0.870 to -0.385, AMM -0.725 to -0.353, S1/S2 -0.460 to 0.045.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig6-ppc-inhibition", "statement": "Inhibiting PPC before or after the stimulus onset reduces both tactile and visual detection sensitivity (tactile pre [-0.4, -0.06], post [-0.43, -0.04]; visual pre [-0.31, -0.05], post [-0.54, -0.07]).", "source": "Results, eighth section; Figure 6C and 6D; Figure S7G and S7H", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Tactile pre -0.408 to -0.052, post -0.466 to -0.009; visual pre -0.333 to -0.031, post -0.555 to -0.070. The tactile post upper bound is below 0 in all ten other seeds (mean -0.011, SD 0.003).", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig6fg-detection-task", "statement": "In a simple tactile detection task without rule switching, inhibiting S1/S2, MM or ALM after the stimulus onset reduces sensitivity, but inhibiting them before it does not (pre S1/S2 [-0.34, 0.18], MM [-0.23, 0.06], ALM [-0.28, 0.07]; post S1/S2 [-0.59, -0.31], MM [-0.59, -0.25], ALM [-0.45, -0.1]); AMM and PPC have no effect in either window.", "source": "Results, eighth section; Figure 6F and 6G", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "Pre-stimulus CIs include 0 for S1/S2, MM and ALM; post-stimulus S1/S2 -0.643 to -0.251, MM -0.628 to -0.233, ALM -0.471 to -0.065. AMM and PPC include 0 in both windows.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig6ij-laser-only-licks", "statement": "Lick probability on laser-only trials does not change during S1/S2, MM or ALM inhibition in either task, so the behavioural effects are not due to an inability to lick (cross-modal S1/S2 [-0.18, 0.24], MM [-0.31, 0.03], ALM [-0.24, 0.16]; detection S1/S2 [-0.13, 0.31], MM [-0.06, 0.45], ALM [-0.18, 0.34]).", "source": "Results, eighth section; Figure 6H, 6I and 6J", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_chang_2023.ipynb", "notes": "11 of 12 CIs include 0. On short laser-only trials of the detection task, MM inhibition raises the right-lick probability from 0.333 in the inter-trial window to 0.553 in the response window (change 0.258, CI 0.045 to 0.483; 141 trials, 5 sessions, 3 mice); over ten other seeds the lower bound is 0.040 to 0.053, and the other eleven CIs include 0 in every seed. All cross-modal CIs and all long- laser CIs include 0; the paper gives one CI per area and task (MM detection -0.06 to 0.45), without saying which laser duration. The windows are the code's (GetBehavInfo, GetBehavInfo_tDetection). In the cross-modal task, licks on short laser-only trials are scored 0 to 2 s after the laser offset, in the detection task -0.8 to 1.2 s from it, which covers the laser. The paper's cross-modal CIs match the long-laser rows (MM -0.312 to 0.030 against -0.31 to 0.03), so the criterion's twelve CIs test more than the paper reports. Its detection CIs match neither the short, the long nor the pooled rows (the code's \"all laser trials\"; pooled MM 0.020 to 0.274 also excludes 0). A rise in licking does not bear on the paper's point that inhibition does not prevent licking.", "deviations": [ "The paper reports one CI per area and task; short and long laser-only trials were tested apart, as the code computes them." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig1e-probe-locations", "statement": "The probe locations were reconstructed histologically and lie in S1, S2, MM and ALM.", "source": "Figure 1E; Methods, Electrophysiology", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The files give the recorded area only as a label; electrode x, y and z are NaN, electrode locations are \"unknown\", and no histology images or reconstructed coordinates are in the dandiset or the code repository.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null } ], "findings": [ { "kind": "selection_undocumented", "statement": "The ephys files hold only the trials and units the paper analysed. No trial has a lick between -1 and 0.1 s of the stimulus (the Methods remove about 25%; gaps of 8 to 64 per session remain in the trial times), and the units are those passing the quality filter and the 50 Hz cut (unit counts per area equal the paper's). The dandiset does not say that trials or units were removed." }, { "kind": "metadata_insufficient", "statement": "Units carry only spike times and observation intervals; there are no quality metrics, waveforms, channel or depth, and unit ids run from 0, so units the paper names by cluster number (for example unit 127 of YT084, 2020-02-01) cannot be identified." }, { "kind": "data_missing", "statement": "Electrode x, y and z are NaN in all 49 ephys files and electrode locations \"unknown\"; the probe locations of Figure 1E (histology) are not in the dandiset." }, { "kind": "metadata_insufficient", "statement": "The trials table columns have no descriptions. `response` (1 right lick, 2 left, 0 none, 3 on trials without a stimulus), `blockType` (\"Whisker\", \"Visual\", and \"Change\" for the cue trial), the opto trial types and the TTT to VVN indicator columns are explained only by the paper's code." }, { "kind": "metadata_error", "statement": "In the 130 behaviour files `start_time` equals the stimulus onset (or laser time on laser-only trials), so trial intervals overlap and do not delimit trials; licks before the stimulus are stored in the trial's lists." }, { "kind": "metadata_error", "statement": "In the tactile detection files, trials without a whisker stimulus are labelled \"Stim_Vis\" and \"Stim_Vis_NoCue\" with a visual stimulus type, though no visual onset is given." }, { "kind": "other", "statement": "Two laser trials start 1.6 and 2.7 s before the reference time and one lasts 9.5 s, outside the stated 0.8 s pre-stimulus and 2 s post-stimulus inhibition; the code counts them as pre-stimulus." }, { "kind": "paper_inconsistency", "statement": "Figure 5B gives 88 and 91 block switches; the 40 recording sessions hold exactly 88 touch and 91 light blocks when the first block of each session is counted, and 63 and 75 non-first blocks with a hit, so the reported numbers appear to count blocks, not switches. The code's GetFirstHitInfo returns one entry per block, NaN for first blocks and blocks without a hit, which a count of entries would give." }, { "kind": "code_mismatch", "statement": "The Methods describe soft normalization (x - mean) / (range + 5) for population analyses; the code's trajectory distance divides by the range alone, its subspace overlap uses offset 5 for the reference and 0.01 for compared data, the coding dimensions use 0.01, and decoding uses a z-score. The Methods exclude sessions with fewer than ten units from the subspace analysis; the code requires more than ten." }, { "kind": "code_mismatch", "statement": "The code's hierarchical bootstrap of optogenetic sensitivity draws resampled false-alarm trials but computes the false-alarm rate from the original trials, so only hits are resampled." }, { "kind": "code_mismatch", "statement": "The coding-dimension code keeps sessions with more than ten units (S1 8, S2 7, MM 5, ALM 11), while Figure S4 reports 10, 8, 9 and 13 sessions." }, { "kind": "other", "statement": "The shared clock of spike times and trial times could not be established by the stimulus-decoding check for three sessions, YT084 2020-02-02 and 2020-02-03 (AMM, 2 and 5 units) and YT086 2020-03-20 (MM, 5 units, accuracy 0.63 before and 0.65 after the onset). They are kept." }, { "kind": "other", "statement": "The cross-modal sham session set includes one session (of 28) that fails the code's own performance check as ported here; it is kept, as deposited." } ], "verifications": [ { "date": "2026-10-17", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The executed notebook of the first run with the conclusion cells and the summary removed, the helper module and the claims file. The attempt record was not among its materials. It checked the archive's file list and metadata for the blocked claim.", "agreed": 28, "of": 29, "disagreements": [ { "claim": "fig5b-first-hit-switches", "outcome": "not_reproduced", "reason": "Under the criterion (non-first blocks with a hit) the counts are touch 63 and light 75, against 88 and 91. Both counts fail; the before-or-at-cue+1 condition holds (0.57, 0.65). The all-block counts are exactly 88 and 91, and the paper says \"total number of block switches\", so the criterion's definition is probably not the paper's. Problem 2, that the count of blocks with a hit including first blocks is never printed, and that `rel` is counted in file rows, so `rel <= 1` can cover trials several positions after the cue, which biases the fraction upward." } ] }, { "date": "2026-10-17", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The full executed notebook of the second run (89 cells), the helper module, the claims file and the attempt record. Report in transcripts/000232/verifier-2-report.md. Points not tied to one claim were also answered. The introduction said 141 optogenetic behaviour sessions where the dandiset holds 130; corrected. The README said the run takes about 15 minutes, a quarter of it streaming, where streaming took 131 s of 14.7 min; it now gives the new run, 162 s of 8.7 min. The summary's \"no other optogenetic call changes\" and \"with other seeds\" were reworded (see fig6c, figs6b, fig6ij and fig1h). The clock part of round-1 problem B11 (three sessions, among them YT086 2020-03-20 in MM, where the stimulus-decoding check does not establish a shared clock), which was recorded only in the notebook, is now a finding. The re-run gives the same numbers as the second run in every cell that existed before.", "agreed": 29, "of": 29, "disagreements": [] } ] } ], "entries": [ { "name": "000232-chang-2023", "type": "reproduction", "title": "Rule-based modulation of a sensorimotor transformation across cortical areas (Chang et al. 2023)", "description": "Tests the findings of Chang, Finkel, Xu and O'Connor (bioRxiv 2023, version 2) on dandiset 000232, which holds silicon-probe recordings from S1, S2, MM, ALM and AMM of mice performing a cross-modal sensory selection task, and the behaviour of optogenetic inhibition sessions in that task and in a tactile detection task. It covers behavioural performance, single-neuron rule and stimulus discrimination, population decoding of the rule, neural trajectories and subspaces, coding dimensions, rule transitions and the effects of optogenetic inhibition.", "dandisets": [ "000232" ], "papers": [ "10.1101/2023.08.21.554194" ], "derived_from": [], "notebooks": [ { "path": "reproduce_chang_2023.ipynb", "title": "Reproducing Chang et al. (2023) from dandiset 000232", "executed": "notebooks/000232-chang-2023/reproduce_chang_2023.html" } ] } ] }