{ "dataset": { "archive": "dandi", "id": "000006", "label": "DANDI:000006" }, "page": "datasets/dandi/000006/", "generated": "2026-10-20T19:09:16+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_links": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000006-economo-2018-01", "dataset": { "archive": "dandi", "id": "000006", "version": "0.220126.1855", "doi": "10.48324/dandi.000006/0.220126.1855" }, "key": "dandi:000006", "label": "DANDI:000006", "dataset_version": "0.220126.1855", "paper": "10.1038/s41586-018-0642-9", "paper_title": "Distinct descending motor cortex pathways and their roles in movement", "date": "2026-10-14", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code (GitHub Actions)", "model": "claude-opus-5-5", "protocol": "pilot-6" }, "runner": { "platform": "GitHub Actions", "url": "https://github.com/catalystneuro/reproduction-registry/actions/runs/38593268430" }, "tier": "agent_verified", "deprecated": "Its claims were written from the abstract only. Read against the full text on 2026-10-20, pt-lower-late-and-motor and pt-types-differ-in-timing are tested with measures the paper does not use (it uses coding directions and early selectivity), and eleven of the paper's findings on this data have no claim.", "deprecated_on": "2026-10-20", "entry": "000006-economo-2018", "entry_title": "Two types of pyramidal tract neuron in mouse motor cortex (Economo et al. 2018)", "description": "Tests the findings in the abstract of Economo et al. (2018) that dandiset 000006 can test. The dandiset holds silicon probe recordings from the anterior lateral motor cortex of 12 mice in a delayed response task, with tagged PTupper and PTlower units. The notebook measures the selectivity for lick direction of the tagged units in the sample, delay and response epochs, and compares the two types. The transcriptomic and anatomical findings are not testable on it.", "notebooks": [ { "path": "reproduce_economo_2018.ipynb", "title": "Reproducing Economo et al. (2018) from dandiset 000006", "executed": "notebooks/000006-economo-2018/reproduce_economo_2018.html" } ], "outcomes": [ { "claim": "pt-types-transcriptomic", "statement": "In ALM scRNA-seq, glutamatergic neurons form 27 clusters and PT neurons map to three of them (Slco2a1, and the closely related Npsr1 and Hpgd), and with axonal reconstructions this suggests that PT neurons can be divided into two cell types.", "source": "Results, Two types of PT neuron in layer 5, paragraphs 1 and 5; Fig. 1", "quote": "ALM PT neurons mapped to three transcriptomic clusters ... Axonal reconstructions and scRNA-seq data suggest that PT neurons can be divided into two distinct cell types.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The dandiset holds electrophysiology only. The single-cell RNA-seq was described in an earlier paper, the bulk RNA-seq is at GEO (GSE119182), and Seq.zip and Axons.zip of the Figshare record hold sequencing and axon data; none is in the dandiset.", "deviations": [], "disagreed": false, "data": "elsewhere", "where": "scRNA-seq data of Tasic et al. 2018, doi:10.1038/s41586-018-0654-5 (Allen Institute); Figshare doi:10.25378/janelia.7007846 version 1, Seq.zip", "proposed": null }, { "claim": "pt-types-shared-targets", "statement": "Both PT types project to several shared targets; all reconstructed cells innervate the superior colliculus, and both types send collaterals to the subthalamic nucleus, zona incerta and midbrain reticular nucleus.", "source": "Results, Two types of PT neuron in layer 5, paragraph 2; Extended Data Fig. 2e legend", "quote": "All cells innervated the superior colliculus and subsets of both groups branched in other areas ... Both cell types extended axon collaterals locally within the same sublamina as their somata, the subthalamic nucleus, zona incerta, and the midbrain reticular nucleus.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "No anatomy or axonal reconstructions are in the dandiset; Anatomy.zip and Axons.zip of the Figshare record hold them.", "deviations": [], "disagreed": false, "data": "elsewhere", "where": "Figshare doi:10.25378/janelia.7007846 version 1, Axons.zip and Anatomy.zip", "proposed": null }, { "claim": "pt-upper-thalamus-projection", "statement": "Of 12 reconstructed PT neurons, 8 innervate the thalamus.", "source": "Results, Two types of PT neuron in layer 5, paragraph 2; Fig. 2a, b; Extended Data Fig. 2", "quote": "Patterns of axonal collaterals revealed two neuron types: one group innervated the thalamus (n = 8; Fig. 2a, b; green hues and Extended Data Fig. 2)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "No anatomy is in the dandiset. The files label tagged units PTupper without naming a projection target; the link to thalamus-projecting neurons comes from the paper's code.", "deviations": [], "disagreed": false, "data": "elsewhere", "where": "Figshare doi:10.25378/janelia.7007846 version 1, Axons.zip (MouseLight reconstructions)", "proposed": null }, { "claim": "pt-lower-medulla-projection", "statement": "The other 4 of 12 reconstructed PT neurons bypass the thalamus and branch in the medulla, including the reticular nuclei with premotor neurons for orofacial movements.", "source": "Results, Two types of PT neuron in layer 5, paragraph 2; Fig. 2a, b", "quote": "the other group bypassed the thalamus and branched in the medulla, including the reticular nuclei containing premotor neurons for orofacial movements23 (n = 4; magenta hues)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "No anatomy is in the dandiset. The files label tagged units PTlower without naming a projection target; the link to medulla-projecting neurons comes from the paper's code.", "deviations": [], "disagreed": false, "data": "elsewhere", "where": "Figshare doi:10.25378/janelia.7007846 version 1, Axons.zip (MouseLight reconstructions)", "proposed": null }, { "claim": "pt-preparatory-activity", "statement": "Along the late delay coding direction (CDlate) selectivity is substantial in both PT types, with 40 of 61 PTupper and 44 of 69 PTlower neurons selective.", "source": "Results, Preparatory activity, paragraph 2; Extended Data Fig. 7a, b", "quote": "along a different direction in activity space, which maximized selectivity at the end of the delay epoch, CDlate, selectivity was substantial in both cell types18 (Extended Data Fig. 7a, b; PTupper: 40 out of 61 neurons selective; PTlower: 44 out of 69)", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_economo_2018.ipynb", "notes": "72 of 100 included tagged units are selective for lick direction in the delay epoch (rank-sum p < 0.01); the label-shuffle null has a 99th percentile of 0.040 (p = 0.001). PTupper 29 of 44, PTlower 43 of 56; every animal 25% to 85%. 76 units at p < 0.05 and 67 at p < 0.001. The selective units lie at nominal depths of 430 to 1600 um; the files give no layer per unit, so the statement's \"layer 5\" is not checked. With one session per animal 26 of 36 units are selective (p = 0.001), and among the paper's 130 units from its processed data 89 are (p = 0.001).", "deviations": [ "Claims written from the abstract only; the criterion and the shuffle test of chance are this attempt's.", "Epoch rates are spike counts on the file's clock; the paper's code averaged a causally smoothed PSTH of spikes already assigned to trials. Leaving out the spikes that overlap the next trial, as far as the stored order shows them, gives 73 of 100.", "Units recorded on different days in one animal are counted separately, as in the code; they may be the same neurons (44 of 100 lie within 20 um of a unit of the same animal on another day, and the files hold no waveforms). A variant keeps one session per animal.", "The dandiset holds 44 of the paper's 61 PTupper and 56 of its 69 PTlower units; the code's inclusion rule gives the paper's counts on its processed data and the same decision for every unit the dandiset holds." ], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "pt-upper-early-preparatory", "statement": "Along the early coding direction (CDearly, first 400 ms of the sample epoch), selectivity is larger and more consistent across trials in the PTupper population than in the PTlower population.", "source": "Results, Preparatory activity, paragraph 2; Fig. 5a, b", "quote": "In this projection, selectivity was larger and more consistent across trials in the PTupper population compared to the PTlower population (Fig. 5b).", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_economo_2018.ipynb", "notes": "27 of 44 included PTupper units (0.614) are selective in the sample epoch (-2.5 to -2.1 s), against a null 99th percentile of 0.045 (p = 0.001); in the baseline epoch 2 of 44 (p = 0.079). All 8 PTupper animals have sample-selective units (25% to 100%). 0.795 at p < 0.05, 0.432 at p < 0.001; 26 of 44 without flagged spikes. One session per animal gives 8 of 20 (null 99th percentile 0.100, p = 0.001); the paper's 61 PTupper units give 40 of 61 (p = 0.001).", "deviations": [ "Claims written from the abstract only; \"early\" is read as the code's sample epoch, the first 400 ms after the pole comes in." ], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "pt-upper-persistent", "statement": "In the PTupper population, selectivity along CDearly remains constant through the sample and delay epochs until the go cue.", "source": "Results, Preparatory activity, paragraph 2; Fig. 5b", "quote": "Furthermore, selectivity in the PTupper population remained constant throughout the sample and delay epochs until the go cue, implying retention of decision-related information.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_economo_2018.ipynb", "notes": "(a) 27 of 44 PTupper units are selective in the late delay (null 99th percentile 0.046, p = 0.001). (b) The cosine similarity of sample and late delay d' is 0.495, against a shuffle null median of 0.182 and 95th percentile of 0.433 (p = 0.016). The code's Pearson correlation gives r = 0.496 (p = 0.0006). 17 of the 27 sample-selective units keep the sign of their d' in the late delay. Both parts hold without flagged spikes, without the trials marked not good, with one session per animal ((a) 12 of 20, p = 0.001; (b) cosine 0.670, p = 0.006) and on the paper's 61 PTupper units ((a) 38 of 61, p = 0.001; (b) cosine 0.514, p = 0.002).", "deviations": [ "The code compares coding directions with bootstraps and plots the correlation; the shuffle test of the similarity is this attempt's." ], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "pt-lower-late-and-motor", "statement": "PTlower neurons carry late preparatory activity (selectivity along CDlate) and a movement command; along the go cue coding direction (CDgo) selectivity is larger, emerges faster and persists longer in PTlower than in PTupper.", "source": "Results, Movement commands, paragraph 2; Fig. 6b, c; Extended Data Fig. 7b; Discussion, paragraphs 1 and 3", "quote": "Along CDgo, selectivity was larger, emerged faster, and persisted longer in the PTlower than in the PTupper population (Fig. 6b, c).", "statistics": [], "kind": "finding", "inferred": "", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_economo_2018.ipynb", "notes": "Among 56 PTlower units, 0.714 are selective in the late delay and 0.482 in the sample epoch (one-sided McNemar p = 0.0096), and 0.643 in the response epoch (p = 0.088 against the sample epoch), so part (a) holds and part (b) does not. The rise rests mostly on anm369962 (0.30 in the sample epoch, 0.90 in the late delay); the other three animals stay between 0.44 and 0.77 in all three epochs. At p < 0.001 both parts hold (p = 0.0033 and 0.018), at p < 0.05 neither (0.054 and 0.14). The response epoch rate differs from baseline in 53 of 56 PTlower units and 37 of 44 PTupper units (one-sided Fisher p = 0.080). The variants give the same split. With one session per animal (16 units) part (a) holds (p = 0.016) and part (b) does not (p = 0.23). On the paper's 69 PTlower units from its processed data, (a) holds (0.507 against 0.652, p = 0.049) and (b) does not (0.551, p = 0.37); there both parts hold at p < 0.001 and neither at p < 0.05.", "deviations": [ "Claims written from the abstract only; \"mainly late\" is read as more units selective late than in the sample epoch, and \"motor commands\" as selectivity in the first 400 ms after the go cue (a change from baseline is shown as a variant).", "The outcome depends on the test threshold, which the code sets at p < 0.01 for counting selective units." ], "disagreed": false, "data": "dataset", "where": null, "proposed": null }, { "claim": "pt-types-differ-in-timing", "statement": "The two PT types have different roles in time; PTupper populations carry early preparatory activity that persists until movement, while PTlower populations mainly carry late preparatory activity and movement commands.", "source": "Abstract; Discussion, paragraphs 1 and 3", "quote": "Preparatory activity appeared early and remained persistent in PTupper neurons, whereas movement commands were observed in PTlower neurons.", "statistics": [], "kind": "finding", "inferred": "", "outcome": "not_reproduced", "blocked_by": null, "notebook": "reproduce_economo_2018.ipynb", "notes": "(a) Sample epoch selectivity is 0.614 for PTupper and 0.482 for PTlower (one-sided Fisher p = 0.134). (b) Response epoch selectivity is 0.643 for PTlower and 0.841 for PTupper, the reverse of the claim (p = 0.99; p = 0.022 for the reverse). Over animals neither holds (Mann-Whitney p = 0.30 and 0.93). The code's coding-direction variant points the abstract's way but is not significant. The sample to late delay similarity is 0.473 +/- 0.214 for PTupper and 0.216 +/- 0.204 for PTlower, with PTupper higher in only 83 of 100 bootstraps. Without the bootstrap, PTupper's similarity is above its shuffle null (p = 0.007) and PTlower's is not (0.177, p = 0.43). On the paper's 130 units from its processed data, (a) is 0.656 against 0.507 (p = 0.062) and (b) is reversed (0.551 against 0.770, p = 0.007 for the reverse); with one session per animal, (a) p = 0.28 and (b) is reversed (0.438 against 0.800). The paper's measure of early information (Extended Data Fig. 9, single-unit decoding accuracy in the sample epoch, at least 70%) goes the abstract's way, with 7 of 44 PTupper against 1 of 56 PTlower units here (Fisher p = 0.012) and 12 of 61 against 1 of 69 on the paper's units (p = 0.0005), where the legend gives 15 of 61 against 3 of 69; the means match the legend (0.638 and 0.586 against 0.644 and 0.589), and the smaller counts are not explained. Part (a) is thus supported under the paper's measure but not under the criterion's. Part (b), selectivity for lick direction in the response epoch, is reversed in every variant. Under the variant reading of \"motor commands\" as a response epoch rate that differs from baseline, PTlower is ahead, 53 of 56 against 37 of 44, without reaching significance (one-sided Fisher p = 0.080). The reversal thus shows that PTlower units are less often direction-selective in the response epoch, not that they lack movement-related activity.", "deviations": [ "Claims written from the abstract only; the comparison of fractions between types is this attempt's reading of the contrast in the abstract, and the paper may compare the types differently.", "Each animal holds one type (8 PTupper and 4 PTlower animals), so type and animal are confounded; the PTlower recordings include sessions in both hemispheres.", "The dandiset lacks 17 of the paper's 61 PTupper and 13 of its 69 PTlower units; the variant on the paper's units gives the same outcome." ], "disagreed": false, "data": "dataset", "where": null, "proposed": null } ], "findings": [ { "kind": "data_missing", "statement": "11 sessions of the paper's processed data (Figshare doi:10.25378/janelia.7007846) that hold tagged units are not in the dandiset. They hold 17 of the 61 PTupper and 13 of the 69 PTlower units the paper analysed, so the dandiset holds 44 and 56. Every tagged unit the dandiset does hold is in the processed data with the same label and the same inclusion decision under the paper's code. The dandiset's description says some files were found to be broken and were left out, without naming them." }, { "kind": "data_missing", "statement": "The dandiset holds the electrophysiology only. The transcriptomic and anatomical data behind four of the abstract's findings (the two PT types and their projection targets) are not in it; they are at GEO (GSE119182) and on Figshare (doi:10.25378/janelia.7007846)." }, { "kind": "format_error", "statement": "Spike times are not sorted in 1120 of 1149 units. They were written trial by trial as trial start + go cue time + the spike time from the go cue. Each trial's spikes cover 6.4 s from about 0.97 s before the pole comes in, and trial starts are a median 6.4 s apart (5.4 to 7.9 s from the 1st to the 99th percentile), so consecutive trials overlap on the file's clock and some spikes of one trial lie in the next trial's early epochs (11.7% of the spikes counted in the baseline epoch, under 2% in later epochs, are detectably out of order). Trial membership of a spike cannot be recovered in full from the files." }, { "kind": "format_error", "statement": "The trial table's stop_time is NaN in every file, and pole_in_time, pole_out_time and cue_start_time are relative to the trial start without saying so. On the file's clock the go cue of a trial often falls after the next trial's start_time." }, { "kind": "metadata_insufficient", "statement": "The tagged unit labels PTupper and PTlower are not described in the files or the dandiset metadata, and the projection targets (thalamus and medulla) that the paper uses to name the types are stated only in the paper's code." }, { "kind": "metadata_insufficient", "statement": "Unit quality mixes two vocabularies (single, and Excellent, Good, Fair, Poor) and has undescribed empty values ([], 0, blank). Sex is U and genotype empty for every mouse, the electrode positions are all zero, and every electrode group gives layer 5 whatever the depth." }, { "kind": "other", "statement": "In the paper's processed data, in every one of the 53 sessions the dandiset holds, the spike windows of the first ten trials start later relative to the pole than those of the other trials (by about 2.7 s in the session examined in detail); the paper's code leaves out the first ten trials of each unit, which hides this. The files carry the same shift, as their spike times equal the processed ones in every unit." } ], "verifications": [ { "date": "2026-10-14", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The executed notebook of the first run with the conclusion cells and the summary removed, the helper module and the claims file. The header still listed the outcomes.", "agreed": 9, "of": 9, "disagreements": [] }, { "date": "2026-10-14", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "The full executed notebook of the second run with its conclusions and summary, the helper module, the claims file, the README and this attempt record, and the public Nature page of the paper with its Extended Data legends and Data availability statement.", "agreed": 9, "of": 9, "disagreements": [] } ] } ], "entries": [ { "name": "000006-economo-2018", "type": "reproduction", "title": "Two types of pyramidal tract neuron in mouse motor cortex (Economo et al. 2018)", "description": "Tests the findings in the abstract of Economo et al. (2018) that dandiset 000006 can test. The dandiset holds silicon probe recordings from the anterior lateral motor cortex of 12 mice in a delayed response task, with tagged PTupper and PTlower units. The notebook measures the selectivity for lick direction of the tagged units in the sample, delay and response epochs, and compares the two types. The transcriptomic and anatomical findings are not testable on it.", "datasets": [ "dandi:000006" ], "papers": [ "10.1038/s41586-018-0642-9" ], "derived_from": [], "notebooks": [ { "path": "reproduce_economo_2018.ipynb", "title": "Reproducing Economo et al. (2018) from dandiset 000006", "executed": "notebooks/000006-economo-2018/reproduce_economo_2018.html" } ] } ] }