{ "dandiset": "000691", "page": "dandisets/000691/", "generated": "2026-10-19T16:23:39+00:00", "caution": "A failure to reproduce is not a finding that the paper is wrong. It may be a failure of the attempt.", "outcome_counts": { "reproduced": 6, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 7 }, "outcome_counts_with_outside_data": { "reproduced": 0, "partially_reproduced": 0, "not_reproduced": 0, "blocked": 0 }, "outcome_counts_elsewhere": { "reproduced": 0, "partially_reproduced": 2, "not_reproduced": 0, "blocked": 0 }, "attempts": [ { "id": "000691-scherrer-2023-01", "dandiset": "000691", "dandiset_version": "0.260827.1358", "paper": "10.1038/s41592-023-01806-1", "paper_title": "An optical design enabling lightweight and large field-of-view head-mounted microscopes", "date": "2026-10-17", "kind": "agent", "accountable": "CatalystNeuro", "agent": { "harness": "Claude Code (GitHub Actions)", "model": "claude-opus-5-5", "protocol": "pilot-7" }, "tier": "agent_verified", "entry": "000691-scherrer-2023", "entry_title": "Imaging thousands of neurons in a freely moving mouse with a large-FOV head-mounted microscope (Scherrer et al. 2023)", "description": "Tests the findings of Scherrer, Lynch, Zhang and Fee (2023) that can be tested on dandiset 000691, which holds one session of one mouse imaged with the large-FOV head-mounted microscope while it explores a looped maze, with 2948 EXTRACT footprints and a behaviour video. It counts the footprints, tracks the mouse from the video, decodes its position and direction from the activity, maps local decoding across the field of view, looks for footprints with localized activity and for the spread of coactive populations, and tests the illumination profiles of Extended Data Fig. 2 on images in the paper's code repository. The other mice, the birds and the optical tests are not in the dandiset.", "notebooks": [ { "path": "reproduce_scherrer_2023.ipynb", "title": "Reproducing Scherrer et al. (2023) from dandiset 000691", "executed": "notebooks/000691-scherrer-2023/reproduce_scherrer_2023.html" } ], "outcomes": [ { "claim": "abstract-thousands-of-neurons", "statement": "The large-FOV microscope images thousands of neurons at once in a freely behaving mouse.", "source": "Abstract; Extended Data Fig. 6a", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "All 2948 footprints have a trace above zero at some time; no two have traces correlated above 0.77, so near-duplicate merging leaves 2948. The mouse visits all four arms and runs 48.5 loop lengths during the 1081 s of imaging that overlap the tracked video. Tracking has 2 steps above 40 px between video frames (one glitch at 1064.8 s) and 55 frames with a blob under 50 px; leaving them out changes the arm of 0.05% of imaging frames and the octant of 0.06%.", "deviations": [ "The count is of EXTRACT footprints before the paper's curation (2382 kept); the dandiset does not mark the curated set." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed6a-curated-footprints", "statement": "In the mouse with the 4 mm cranial window shown in the main figure, 2382 footprints remain after curation.", "source": "Extended Data Fig. 6a", "outcome": "blocked", "blocked_by": "metadata_insufficient", "notebook": null, "notes": "The dandiset holds 2948 footprints, the number the preprint gives for the extraction before curation, and its plane segmentation has no column that marks the 2382 kept. The selection is in the paper's Academic Torrents deposit (supporting_data/2021-07-26_isgood.mat and extract_curated.mat, infohash aa2a3d2f5ff0b3974871db47db57bc3dabf3c192); no peer connected in two attempts of 2.5 and 4 minutes from this environment, so it could not be fetched.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed6bcd-other-mice", "statement": "Three other mice (3 mm and 4 mm windows) gave 1029, 401 and 112 footprints.", "source": "Extended Data Fig. 6b to 6d", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The dandiset holds one mouse (4mm-mouse-1). The Academic Torrents deposit the paper names lists files of the same session only (2021-07-26), and no other dandiset of this paper was found.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed5-songbirds", "statement": "The weight-optimized microscope images hundreds of neurons in juvenile songbirds (46, 177 and 225 curated footprints in three birds).", "source": "Abstract; Extended Data Fig. 5", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "No bird data in the dandiset or in the Academic Torrents deposit, whose file list holds only the mouse session.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "abstract-kiloscope-fov", "statement": "The microscope optimized for field of view has a field of view of about 4 mm.", "source": "Abstract", "outcome": "blocked", "blocked_by": "metadata_insufficient", "notebook": null, "notes": "The NWB file gives no pixel size (no grid spacing in the imaging plane). Frames are 808 x 608 pixels; at the 5.94 um per pixel of the paper's code they would be 4.80 x 3.61 mm, and the footprint centroids span 3.09 x 3.31 mm. The pixel size comes from the code, so this does not test the claim on the dandiset.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "abstract-weights", "statement": "The two microscopes weigh 1.4 g (large FOV) and 1.0 g (weight-optimized, 1.0 mm FOV).", "source": "Abstract", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "Weights are hardware measurements; no data on them are in the dandiset or the paper's deposits.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed1-ed4-resolution", "statement": "Images of a USAF 1951 target resolve elements of 2.19 um line width with the Featherscope and 4.38 um with the Kiloscope, and 2.2 um with Kiloscope optics on a sensor with 1.4 um pixels.", "source": "Extended Data Fig. 1b, 1e and 4b", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The USAF target images are not in the dandiset, the Academic Torrents deposit or the code repository (which holds bead images for a point spread function analysis whose numbers are in the main text, not available here).", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed3-adc", "statement": "The onboard analog-to-digital converter records test tones with a root-mean-square residual of 0.2% of full scale.", "source": "Extended Data Fig. 3", "outcome": "blocked", "blocked_by": "data_missing", "notebook": null, "notes": "The test recordings of the analog-to-digital converter are not in the dandiset or the paper's deposits.", "deviations": [], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed8-coactive-distributed", "statement": "The footprints that take part in a large correlated event of each NMF component are spread across the field of view.", "source": "Extended Data Fig. 8", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "For all 8 components the event has 196 to 320 active footprints, more than half of the component's members, with a spread of 0.945 to 1.055 times that of all footprints (0.98 mm) and a 90th percentile of pairwise distance of 2.05 to 2.29 mm. Active thresholds of 0.3 and 0.7 also give 8 of 8. The best of 4 NMF replicates was kept (errors 97.55 to 97.58). Two events (80.4 and 122.9 s) fall in the opening 153 s the behaviour analysis leaves out; with event times from 153 s on, 8 of 8 pass (spread ratios 0.94 to 1.05). The test is weak. In 100 random 50 s windows, the median fraction of members active is 0.399 to 0.519, and the event's fraction (0.568 to 0.851) is above the 95th percentile of random windows for 6 of 8 components. Random sets of footprints of the same size have spread ratios of 0.951 to 1.042 (5th to 95th percentiles), so any set this large passes the criterion's half. The event's spread ratio lies within its own component's random range for 3 of 8 components, below it for 2 (0.945 and 0.956, against 5th percentiles of 0.962) and above it for 3 (1.042, 1.055 and 1.050, against 95th percentiles of 1.037, 1.042 and 1.036).", "deviations": [ "All 2948 footprints in place of the 2382 curated ones.", "Event times are each component's largest temporal weight; the code sets them by hand.", "Distances use the code's 5.94 um per pixel, which the dandiset does not state; the outcome uses ratios and does not depend on it." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "fig2-quadrant-direction-decoding", "statement": "The mouse's position in the maze (quadrant) together with its direction of movement can be decoded from the activity of the imaged population.", "source": "Figure 2 (legend not available); Extended Data Fig. 9a (\"in this case, direction is not coded\") and Fig. 10a; quadrant_direction_decoding.m", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "Median class accuracy 0.415 over 50 decoders (0.304 to 0.604), chance 0.125, 95th percentile of 50 shifted-label decoders 0.145; all 50 exceed it. 3.32 times chance; the preprint's text says at least four times chance, a number not used here as the version of record's text was not available. Six of eight classes reach 0.443 to 0.594, two only 0.236 and 0.239. On all frames, without the running selection, 0.348 against a null 95th percentile of 0.127. The SGD solver stops far from the optimum of its objective (2.04 to 3.21 on six splits, against 0.086 to 0.118 for an exact solver at its limit of 10,000 iterations). On the same splits the exact solver gives 0.262 to 0.318 for three real decoders and 0.131 to 0.164 for three with shifted labels, so the claim also holds with it.", "deviations": [ "Position tracked here from the video (the paper's tracking output is not in the dandiset), in the unrotated frame, with the loop's sides at the 2nd and 98th percentiles of position.", "All 2948 footprints in place of the 2382 curated ones.", "scikit-learn's SGD classifier (hinge loss, ridge alpha equal to the code's lambda, 10 passes) in place of MATLAB's fitcecoc with templateLinear, whose default for more than 100 predictors is 10 passes of dual SGD; 50 decoders in place of 200.", "The legend of Figure 2 was not available, so the claim is stated from the Extended Data legends and the code." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed9b-octant-decoding", "statement": "Position within the maze can be decoded with octant resolution from the activity.", "source": "Extended Data Fig. 9a and 9b", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "Median class accuracy 0.313 over 50 decoders (0.211 to 0.416), chance 0.125, null 95th percentile 0.144; all 50 exceed it, and each octant's mean accuracy (0.236 to 0.499) is above chance. On six splits fit by both solvers, SGD stops at objectives of 3.89 to 6.18 and the exact solver at 0.071 to 0.094 (at its limit of 10,000 iterations in all six); the exact solver gives 0.220 to 0.292 for three real decoders and 0.092 to 0.145 for three with shifted labels, so the claim also holds with it.", "deviations": [ "As for fig2-quadrant-direction-decoding." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed9c-local-decoding-map", "statement": "Decoders that use the 32 footprints closest to a point, within 270 um, decode octant position significantly (Benjamini-Hochberg false discovery rate 5%) in parts of the field of view, and accuracy differs across the field of view.", "source": "Extended Data Fig. 9c", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "With 32 footprints within 270 um (45 px), 150 grid points qualify and 18 (12%) are significant at Benjamini-Hochberg 5%, with scores of 0.112 to 0.187 (range 0.075, where the criterion asks for 0.05). Scores over all points 0.068 to 0.187, median 0.125, against null means of 0.065 to 0.123. The outcome depends on the solver. With 10 passes of SGD, as in round 1, on the same points and seeds, 1 point is significant and the criterion is not met; at four points SGD stops at objectives of 19.8 to 68.9 where the exact solver reaches 1.976 to 2.000, close to the 2.0 of predicting every class absent, so the scores rest on weak classifiers. liblinear reached its limit of 10,000 iterations in 2268 of 3000 fits. Null standard deviations per point 0.018 to 0.046 (5th to 95th percentile) from 10 shifted decoders; a point alone needs z above 3.40, and the smallest z among the significant points is 2.53. A Welch t-test of the 10 real against the 10 shifted decoders marks 80 points significant (range 0.079), and 5 with SGD (range 0.052). With the code's 64 footprints within 50 px, on a 60 px grid, 22 of 34 points are significant (scores 0.141 to 0.211, range 0.070), and 32 with the t-test. The exact fits stop at an iteration limit, so the counts can differ slightly between machines.", "deviations": [ "The legend says 32 footprints within 270 um; the code uses 64 within 50 pixels. The legend is followed for the outcome and the code's choice is shown as a variant.", "The paper does not describe its significance test. Here, 10 decoders and 10 shifted-label decoders per point and a one-sided z-test, chosen before the analysis.", "Local decoders are fit by liblinear (LinearSVC) on the same objective, as MATLAB's default for 100 or fewer predictors is BFGS run to convergence; liblinear also penalizes the intercept, weakly. Round 1 used 10 passes of SGD here, which is shown as a variant.", "A 30 pixel grid where the code uses 5 pixels; all 2948 footprints in place of 2382; position as for fig2-quadrant-direction-decoding.", "The code's variant is run on a 60 pixel grid where the criterion says 40; on the 40 pixel grid the notebook took 32 minutes on four cores, over the 30 minute limit." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed10-localized-activity", "statement": "Some footprints have activity localized to subregions of the maze.", "source": "Extended Data Fig. 10", "outcome": "reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "100 of 2948 footprints (3.4%) have an entropy over linear position below all 100 shifted values, against 29 expected (binomial p = 4.5e-25); 448 against 175 at a looser threshold. For the 100, the entropy is 0.20 to 0.90 bits below the shifted mean (5th to 95th percentile), out of a maximum of 6.64 bits. The binomial test treats footprints as independent, but the same 100 shifts serve every footprint; treating each shifted copy in turn as the observed data gives counts with a median of 28, a 95th percentile of 54 and a maximum of 62, and none of the 100 copies reaches 100.", "deviations": [ "The paper selected its examples by eye and shows two-dimensional maps; here a test over all footprints on linear position, with the entropy measure of place_cells.m.", "All 2948 footprints in place of 2382; position as for fig2-quadrant-direction-decoding." ], "disagreed": false, "data": "dandiset", "where": null, "proposed": null }, { "claim": "ed2b-featherscope-flat-illumination", "statement": "For the Featherscope, the axicon correction optics give a flat signal intensity profile across the field of view, where total vignetting (optical vignetting with Gaussian illumination) makes the signal fall toward the edges.", "source": "Extended Data Fig. 2b", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "At 0.4 mm the corrected profile is 0.742, total vignetting 0.342 and optical vignetting 0.628; the corrected profile is higher, but falls below 0.8 (minimum 0.742 over 0 to 0.4 mm). The minimum is 0.785 out to 0.38 mm and 0.698 out to 0.42 mm; with the centre moved by 100 um it is 0.626 to 0.867, and 0.766 about the image centre.", "deviations": [ "The radius of 0.4 mm and the level of 0.8 that define \"flat\" were chosen here; the legend gives no number." ], "disagreed": false, "data": "elsewhere", "where": "GitHub FeeLab/Feescope-Paper-Code at commit d596646b16a505907203828d8026feb7736ca69c, folder sf3/feather (images of a thin layer of fluorescent dye)", "proposed": null }, { "claim": "ed2c-kiloscope-flat-illumination", "statement": "For the Kiloscope, the axicon correction optics give a flat signal intensity profile across the field of view, where total vignetting makes the signal fall toward the edges.", "source": "Extended Data Fig. 2c", "outcome": "partially_reproduced", "blocked_by": null, "notebook": "reproduce_scherrer_2023.ipynb", "notes": "At 1.5 mm the corrected profile is 0.797, total vignetting 0.253 and optical vignetting 0.389. The corrected profile is higher, and its minimum over 0 to 1.5 mm, 0.797, is 0.003 below the criterion's 0.8; at 1.125 mm it is 0.922. The minimum is 0.846 out to 1.425 mm and 0.742 out to 1.575 mm; with the centre moved by 100 um it is 0.765 to 0.831 (above 0.8 for one of four moves), and 0.695 about the image centre.", "deviations": [ "The radius of 1.5 mm and the level of 0.8 were chosen here; the result sits at the level." ], "disagreed": false, "data": "elsewhere", "where": "GitHub FeeLab/Feescope-Paper-Code at commit d596646b16a505907203828d8026feb7736ca69c, folder sf3/widefield", "proposed": null } ], "findings": [ { "kind": "data_missing", "statement": "The dandiset holds one of the paper's four mice and none of the songbird recordings or optical test data. The Academic Torrents deposit the paper names for its imaging data holds files of the same mouse session only, so the other mice and the birds are not public." }, { "kind": "metadata_error", "statement": "The device description reads \"Head-mounted microscope (Kiloscope) optimized for weight (1.0 mm FOV; 1.0 g)\", and the dandiset description says the data come from the microscope optimized for weight. The data are from the large-FOV microscope (1.4 g, about 4 mm FOV) that the paper calls the Kiloscope; a 4 mm window, 2948 footprints and 808 x 608 pixel frames." }, { "kind": "metadata_error", "statement": "The imaging is stored as a TwoPhotonSeries and the dandiset lists two-photon microscopy among its techniques, but the microscope is a one-photon head-mounted epifluorescence microscope." }, { "kind": "metadata_error", "statement": "The behaviour video is described as about 30 fps and its AVI header says 30 frames/s, but its 15521 timestamps run at a median 12.5 frames/s over 1234 s; at 30 frames/s it would cover only 517 s of the 1234 s of imaging." }, { "kind": "metadata_insufficient", "statement": "The imaging plane gives no pixel size, and its indicator and location are \"unknown\" with no excitation wavelength, although general/virus says GCaMP7f. Distances in the paper's figures cannot be computed from the dandiset alone." }, { "kind": "selection_undocumented", "statement": "The 2948 footprints are the EXTRACT output before curation; the paper analyses the 2382 it kept (Extended Data Fig. 6a). Nothing in the dandiset marks the curated set or says that the footprints are uncurated." }, { "kind": "data_missing", "statement": "The mouse's position, which the paper's decoding and place analyses use, is not in the dandiset; it must be tracked from the behaviour video. The first 64 s of the video (by its nominal rate), which the paper's code drops because the experimenter is in view, are not marked." }, { "kind": "code_mismatch", "statement": "The Extended Data Fig. 9c legend describes decoders on the 32 closest footprints within 270 um; the code (subregion_map_octant.m) uses 64 footprints within 50 pixels, which at the code's own 5.94 um per pixel (sf3/widefield/analysis.m) is 297 um." }, { "kind": "paper_inconsistency", "statement": "The title of Extended Data Fig. 8 refers to the Kiloscope data \"shown in Figure 3\", and its legend takes the ridgeline plot from Fig. 2d, while the version of record has two figures, and Supplementary Video 1 and Extended Data Fig. 10 place the bird data in Fig. 2a-d and the mouse data in Fig. 2e-h." } ], "verifications": [ { "date": "2026-10-18", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 1. The first executed notebook with the conclusion cells and the summary removed, the helper module and the claims file, with the paper's abstract and Extended Data legends from the publisher page. The verifier read the archive's asset list and metadata, read the paper's code at d596646, and tested the decoder and tracking helpers on synthetic inputs. Its report had no attempt record and gave its own outcome for each claim; all 15 agree with the record as it then stood. The revision changed the outcome of ed9c-local-decoding-map, which round 2 reviews.", "agreed": 15, "of": 15, "disagreements": [] }, { "date": "2026-10-18", "kind": "agent", "by": "Independent agent run by CatalystNeuro (same harness, separate context, did not write the notebook)", "reviewed": "Round 2. The full executed notebook of the first revision, the helper module, the claims file, the README and the attempt record; no paper text was downloaded this round, and the verifier worked from its round-1 reading. It agreed with all 15 outcomes. It quoted sentences the printed numbers contradict. Most of them (18 significant points, 2268 fits at the limit, NMF errors 97.55 to 97.58, 634 s) are the numbers of a run on four cores, while the executed copy it read came from another run (19, 2264, 97.79 to 97.8, 358 s, 13.7 min); the exact fits stop at an iteration limit and the NMF uses float32, so these numbers differ slightly between machines. The entry was re-run on four cores, which again gives 18, 2268, 97.55 to 97.58 and about 640 s, and the text now matches that run. The run takes about 25 minutes, so the README and entry say about 25. The verifier also found two sentences no output showed. The AVI's byte size matching the Academic Torrents listing (cell 15) is removed, and the draft holding the same blobs (cell 0) is now printed in the data check (same assets, sizes and blob ids).", "agreed": 15, "of": 15, "disagreements": [] } ] } ], "entries": [ { "name": "000691-scherrer-2023", "type": "reproduction", "title": "Imaging thousands of neurons in a freely moving mouse with a large-FOV head-mounted microscope (Scherrer et al. 2023)", "description": "Tests the findings of Scherrer, Lynch, Zhang and Fee (2023) that can be tested on dandiset 000691, which holds one session of one mouse imaged with the large-FOV head-mounted microscope while it explores a looped maze, with 2948 EXTRACT footprints and a behaviour video. It counts the footprints, tracks the mouse from the video, decodes its position and direction from the activity, maps local decoding across the field of view, looks for footprints with localized activity and for the spread of coactive populations, and tests the illumination profiles of Extended Data Fig. 2 on images in the paper's code repository. The other mice, the birds and the optical tests are not in the dandiset.", "dandisets": [ "000691" ], "papers": [ "10.1038/s41592-023-01806-1" ], "derived_from": [], "notebooks": [ { "path": "reproduce_scherrer_2023.ipynb", "title": "Reproducing Scherrer et al. (2023) from dandiset 000691", "executed": "notebooks/000691-scherrer-2023/reproduce_scherrer_2023.html" } ] } ] }