Visual Learning session metadata#
The metadata for every Visual Learning session lives in the AIND document database (docDB). This page queries docDB and assembles a dataframe with one row per session, carrying the mouse, the session type, the acquisition date, the imaging planes and their depths, and the z-drift QC result.
That dataframe is the thing to take away. It is what the Visual Learning tutorials use to choose sessions, and the same query can be adapted to pull whatever other fields an analysis needs. Writing it to a CSV, at the end of this page, is a convenience rather than the goal.
Setup#
import re
import time
from datetime import datetime
import numpy as np
import pandas as pd
pd.set_option('display.width', 220)
pd.set_option('display.max_columns', 40)
from aind_data_access_api.document_db import MetadataDbClient
API_GATEWAY_HOST = "api.allenneuraldynamics.org"
OUTPUT_DIR = '/data/metadata'
DATABASE = 'metadata_index'
COLLECTION = 'data_assets'
docdb_api_client = MetadataDbClient(
host=API_GATEWAY_HOST,
version="v2",
database=DATABASE,
collection=COLLECTION,
)
print(docdb_api_client._base_url)
https://api.allenneuraldynamics.org/v2/metadata_index/data_assets
# The cohort is defined by subject: five different project_name values are
# interleaved across the same six mice.
VISUAL_LEARNING_MICE = ['782149', '790322', '788406', '800792', '800995', '804363']
# Assets are selected by modality rather than by a pattern on the asset name.
# 'pophys' is planar optical physiology, the modality for multiplane ophys.
MODALITY = 'pophys'
# Processed asset names end in _processed_<date>_<time>. Anchoring at end-of-string
# drops further-derived assets (behavior-nwb, cortical-zstack, coreg, ROICat) that
# carry _processed_ mid-name.
PROCESSED_PATTERN = (r'^multiplane-ophys_\d+_\d{4}-\d{2}-\d{2}_[\d-]+'
r'_processed_\d{4}-\d{2}-\d{2}_[\d-]+$')
# The QC request sends one asset name per document, so the whole cohort at once
# exceeds the gateway's header limit -- fetch it in batches.
BATCH = 40
Query#
aggregate = [
{
"$match": {
"data_description.subject_id": {"$in": VISUAL_LEARNING_MICE},
"data_description.modalities.abbreviation": MODALITY,
# exclude the post-training passive block
"acquisition.acquisition_type": {"$exists": True,
"$ne": "CENTER_MOUSEMOTION"},
},
},
{
"$project": {
"name": 1,
"subject_id": "$data_description.subject_id",
"project_name": "$data_description.project_name",
"modality": "$data_description.modalities.abbreviation",
"acquisition_type": "$acquisition.acquisition_type",
"session_start_time": "$acquisition.acquisition_start_time",
"session_end_time": "$acquisition.acquisition_end_time",
"rig": "$acquisition.instrument_id",
"genotype": "$subject.subject_details.genotype",
"sex": "$subject.subject_details.sex",
"date_of_birth": "$subject.subject_details.date_of_birth",
# flatten data_streams[] -> configurations[] -> images[] -> planes[]
"planes": {"$reduce": {
"input": {"$reduce": {
"input": "$acquisition.data_streams", "initialValue": [],
"in": {"$concatArrays": [
"$$value", {"$ifNull": ["$$this.configurations", []]}]}}},
"initialValue": [],
"in": {"$concatArrays": ["$$value",
{"$reduce": {
"input": {"$ifNull": ["$$this.images", []]}, "initialValue": [],
"in": {"$concatArrays": [
"$$value", {"$ifNull": ["$$this.planes", []]}]}}}]}}},
}
},
{
"$project": {
"name": 1, "subject_id": 1, "project_name": 1, "modality": 1,
"acquisition_type": 1,
"session_start_time": 1, "session_end_time": 1, "rig": 1,
"genotype": 1, "sex": 1, "date_of_birth": 1,
"n_planes": {"$size": "$planes"},
"plane_indices": "$planes.plane_index",
"imaging_depths": "$planes.depth",
"targeted_structures": "$planes.targeted_structure.acronym",
}
},
# drop the 2-plane test sessions (800792, 800995 -- 2 planes in every
# processing generation, so nothing is recovered by keeping them)
{"$match": {"n_planes": 8}},
]
records = docdb_api_client.aggregate_docdb_records(
pipeline = aggregate,
)
print(f'{len(records)} assets')
if len(records) == 0:
raise RuntimeError(
f'No assets matched. If the cohort and client are right, check that '
f'MODALITY={MODALITY!r} is the abbreviation docDB uses for these assets.')
# Confirm the modality filter selected what we expect, and nothing else.
print('modalities returned:', {m for r in records for m in (r.get('modality') or [])})
1432 assets
modalities returned: {'behavior-videos', 'pophys', 'behavior'}
Building the session table#
One row per session, keeping only the newest _processed_ generation: a session is
reprocessed whenever the pipeline changes, so it appears several times (3-8 deep) under
different stamps. The stamp format sorts lexicographically in chronological order, so
sort_values + keep='last' picks the newest.
sessions = pd.DataFrame(records)
sessions = sessions[sessions.name.str.match(PROCESSED_PATTERN)].copy()
sessions['session_id'] = sessions.name.str.extract(
r'^(multiplane-ophys_\d+_\d{4}-\d{2}-\d{2}_[\d-]+)_processed_')
sessions['processed_stamp'] = sessions.name.str.extract(
r'_processed_(\d{4}-\d{2}-\d{2}_[\d-]+)$')
sessions = (sessions.sort_values('processed_stamp')
.drop_duplicates('session_id', keep='last'))
print(f'{len(sessions)} unique sessions across {sessions.subject_id.nunique()} mice')
print(sessions.subject_id.value_counts().sort_index().to_string())
147 unique sessions across 6 mice
subject_id
782149 24
788406 32
790322 24
800792 25
800995 22
804363 20
# acquisition_date comes off the asset name; session_date/time off the timestamp.
# They agree on every row today -- kept separate because the name is what the mount
# and every derived asset are keyed by.
sessions['acquisition_date'] = sessions.session_id.str.extract(r'_(\d{4}-\d{2}-\d{2})_')
sessions['session_date'] = sessions.session_start_time.map(
lambda x: datetime.fromisoformat(x).date())
sessions['session_time'] = sessions.session_start_time.map(
lambda x: datetime.fromisoformat(x).time())
sessions['date_of_birth'] = sessions.date_of_birth.map(
lambda x: datetime.strptime(x, '%Y-%m-%d').date() if isinstance(x, str) else x)
sessions['age_days'] = [(a - b).days if pd.notnull(b) else np.nan
for a, b in zip(pd.to_datetime(sessions.acquisition_date).dt.date,
sessions.date_of_birth)]
sessions['session_type'] = sessions.acquisition_type
sessions['stage'] = sessions.session_type.str.extract(
r'^(TRAINING_\d|OPHYS_\d|STAGE_\d)')
sessions['image_set'] = sessions.session_type.str.extract(r'_images_([AB])')
sessions = sessions.sort_values(['subject_id', 'acquisition_date'])
sessions['session_number'] = sessions.groupby('subject_id').cumcount() + 1
# Plane columns, ordered by plane_index so depths line up with names
sessions['plane_names'] = [
[f'{s}_{i}' for i, s in sorted(zip(r.plane_indices, r.targeted_structures))]
for r in sessions.itertuples()]
sessions['imaging_depths'] = [
[d for _, d in sorted(zip(r.plane_indices, r.imaging_depths))]
for r in sessions.itertuples()]
sessions['targeted_structures'] = [
sorted(set(r.targeted_structures)) for r in sessions.itertuples()]
Z-drift QC#
QC lives in quality_control.metrics — a flat array of per-plane metrics, each with a
status_history whose last entry is current. Metric names carry the plane either
leading (VISp_0 Z-drift Analysis) or trailing
(VISp_0 Z-drift Analysis - VISp_0), so we check both ends.
Sessions whose processing generation predates the z-drift evaluation have no metric to
read; those stay NA rather than 0, so a session with no QC is not mistaken for a
session that passed.
zdrift = []
targets = sessions.name.tolist()
for i in range(0, len(targets), BATCH):
docs = docdb_api_client.retrieve_docdb_records(
filter_query={'name': {'$in': targets[i:i + BATCH]}},
projection={'name': 1, 'quality_control.metrics': 1},
limit=BATCH,
)
for doc in docs:
for metric in ((doc.get('quality_control') or {}).get('metrics') or []):
name = str(metric.get('name'))
if not re.search(r'z-?drift', name, re.I):
continue
history = metric.get('status_history') or []
zdrift.append({
'name': doc['name'],
'metric_name': name,
'status': history[-1].get('status') if history else None,
})
zdrift = pd.DataFrame(zdrift)
print(f'{len(zdrift)} z-drift metric rows from {zdrift.name.nunique()} assets')
# plane may lead or trail the metric name
zdrift['plane_name'] = zdrift.metric_name.str.extract(r'^(VISp_\d+)')[0].fillna(
zdrift.metric_name.str.extract(r'(VISp_\d+)\s*$')[0])
assert zdrift.plane_name.notna().all(), 'unparsed plane in a z-drift metric name'
zdrift = zdrift.drop_duplicates(['name', 'plane_name'])
print(zdrift.status.value_counts().to_string())
904 z-drift metric rows from 113 assets
status
Pass 788
Fail 116
# Failing plane names per session, plus the count. Sessions with no z-drift QC
# stay NA -- distinct from an empty list, which means QC ran and nothing failed.
failed = zdrift[zdrift.status == 'Fail'].copy()
failed['plane_index'] = failed.plane_name.str.extract(r'_(\d+)$').astype(int)
# sort by plane index, not lexically (VISp_10 would otherwise precede VISp_2)
failed_names = (failed.sort_values(['name', 'plane_index'])
.groupby('name').plane_name.apply(list))
have_qc = sessions.name.isin(zdrift.name)
sessions['planes_failing_zdrift'] = [
(failed_names.get(n, []) if has else pd.NA)
for n, has in zip(sessions.name, have_qc)]
sessions['n_planes_failing_zdrift'] = (
sessions.name.map(zdrift.status.eq('Fail').groupby(zdrift.name).sum())
.where(have_qc).astype('Int64'))
print(f'{int(have_qc.sum())} sessions with z-drift QC, '
f'{int((~have_qc).sum())} left NA')
print(sessions.n_planes_failing_zdrift.value_counts(dropna=False).sort_index().to_string())
113 sessions with z-drift QC, 34 left NA
n_planes_failing_zdrift
0 71
1 14
2 10
3 5
4 6
5 2
6 3
7 1
8 1
<NA> 34
The session table#
Ordered and reset, this is the finished table.
order = ['subject_id', 'session_id', 'name', 'session_type', 'acquisition_type',
'stage', 'image_set', 'session_number', 'acquisition_date', 'session_date',
'session_time', 'age_days', 'genotype', 'sex', 'date_of_birth', 'rig',
'project_name', 'modality', 'n_planes', 'plane_names', 'imaging_depths',
'targeted_structures', 'planes_failing_zdrift',
'n_planes_failing_zdrift', 'processed_stamp', '_id']
sessions = sessions[order].reset_index(drop=True)
sessions
| subject_id | session_id | name | session_type | acquisition_type | stage | image_set | session_number | acquisition_date | session_date | session_time | age_days | genotype | sex | date_of_birth | rig | project_name | modality | n_planes | plane_names | imaging_depths | targeted_structures | planes_failing_zdrift | n_planes_failing_zdrift | processed_stamp | _id | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 782149 | multiplane-ophys_782149_2025-03-25_09-46-08 | multiplane-ophys_782149_2025-03-25_09-46-08_pr... | TRAINING_0_gratings_autorewards_15min | TRAINING_0_gratings_autorewards_15min | TRAINING_0 | NaN | 1 | 2025-03-25 | 2025-03-25 | 09:46:08.591468 | 108 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Male | 2024-12-07 | 422_MESO2_20241017 | LearningmFISHTask1A | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [40, 320, 80, 280, 120, 240, 160, 200] | [VISp] | [] | 0 | 2026-08-19_00-32-51 | aca6e6d2-9f33-4ed6-8a66-86d69a282c30 |
| 1 | 782149 | multiplane-ophys_782149_2025-03-28_10-55-25 | multiplane-ophys_782149_2025-03-28_10-55-25_pr... | TRAINING_1_gratings | TRAINING_1_gratings | TRAINING_1 | NaN | 2 | 2025-03-28 | 2025-03-28 | 10:55:25.569080 | 111 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Male | 2024-12-07 | 429_MESO1_20241016 | LearningmFISHTask1A | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 200, 114, 244, 80, 280, 40, 310] | [VISp] | [] | 0 | 2026-08-19_00-34-09 | 2a3f8254-9f86-42fd-b4d7-fe1877ebf959 |
| 2 | 782149 | multiplane-ophys_782149_2025-03-29_10-10-29 | multiplane-ophys_782149_2025-03-29_10-10-29_pr... | TRAINING_1_gratings | TRAINING_1_gratings | TRAINING_1 | NaN | 3 | 2025-03-29 | 2025-03-29 | 10:10:29.493070 | 112 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Male | 2024-12-07 | 429_MESO1_20241016 | Learning mFISH-V1omFISH | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [158, 198, 114, 246, 80, 276, 43, 306] | [VISp] | [VISp_0, VISp_4, VISp_6] | 3 | 2026-08-19_00-33-54 | 7591b588-f6a1-474d-b00a-3dd10ffb4a60 |
| 3 | 782149 | multiplane-ophys_782149_2025-03-31_12-23-33 | multiplane-ophys_782149_2025-03-31_12-23-33_pr... | TRAINING_1_gratings | TRAINING_1_gratings | TRAINING_1 | NaN | 4 | 2025-03-31 | 2025-03-31 | 12:23:33.753970 | 114 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Male | 2024-12-07 | 429_MESO1_20241016 | LearningmFISHTask1A | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 200, 115, 245, 80, 280, 40, 310] | [VISp] | [] | 0 | 2026-08-19_00-34-28 | d00bf70e-41c7-4ec5-ac76-8aecb010fbe1 |
| 4 | 782149 | multiplane-ophys_782149_2025-04-01_09-42-11 | multiplane-ophys_782149_2025-04-01_09-42-11_pr... | TRAINING_2_gratings_flashed | TRAINING_2_gratings_flashed | TRAINING_2 | NaN | 5 | 2025-04-01 | 2025-04-01 | 09:42:11.814685 | 115 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Male | 2024-12-07 | 429_MESO1_20241016 | LearningmFISHTask1A | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 200, 115, 240, 85, 280, 40, 300] | [VISp] | [VISp_4, VISp_6] | 2 | 2026-08-19_00-33-58 | ac0f8e4d-cd12-453a-8d83-6b7951a6a957 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 142 | 804363 | multiplane-ophys_804363_2025-09-04_16-08-45 | multiplane-ophys_804363_2025-09-04_16-08-45_pr... | OPHYS_6_images_B | OPHYS_6_images_B | OPHYS_6 | B | 16 | 2025-09-04 | 2025-09-04 | 16:08:45.114554 | 130 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Female | 2025-04-27 | 429_MESO1_20241016 | Learning mFISH-V1omFISH | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 200, 120, 238, 80, 272, 48, 314] | [VISp] | [VISp_0, VISp_1, VISp_2, VISp_4, VISp_5, VISp_6] | 6 | 2026-08-19_01-05-08 | a7ae1f04-6d75-4f43-ada6-61055ef90896 |
| 143 | 804363 | multiplane-ophys_804363_2025-09-05_13-25-33 | multiplane-ophys_804363_2025-09-05_13-25-33_pr... | STAGE_0 | STAGE_0 | STAGE_0 | NaN | 17 | 2025-09-05 | 2025-09-05 | 13:25:33.172779 | 131 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Female | 2025-04-27 | 429_MESO1_20241016 | Learning mFISH-V1omFISH | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 196, 120, 240, 78, 276, 44, 320] | [VISp] | <NA> | <NA> | 2026-08-19_01-05-09 | 100f82f1-b73a-49c2-998d-dbcabf9362e7 |
| 144 | 804363 | multiplane-ophys_804363_2025-09-08_09-24-39 | multiplane-ophys_804363_2025-09-08_09-24-39_pr... | STAGE_1 | STAGE_1 | STAGE_1 | NaN | 18 | 2025-09-08 | 2025-09-08 | 09:24:39.253310 | 134 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Female | 2025-04-27 | 429_MESO1_20241016 | Learning mFISH-V1omFISH | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 198, 120, 240, 82, 276, 42, 320] | [VISp] | <NA> | <NA> | 2026-08-19_01-05-32 | 85a8228f-f0b4-480b-a402-95a0379c3c3b |
| 145 | 804363 | multiplane-ophys_804363_2025-09-09_11-44-14 | multiplane-ophys_804363_2025-09-09_11-44-14_pr... | STAGE_1 | STAGE_1 | STAGE_1 | NaN | 19 | 2025-09-09 | 2025-09-09 | 11:44:14.156559 | 135 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Female | 2025-04-27 | 429_MESO1_20241016 | LearningmFISHTask1A | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 200, 122, 240, 84, 280, 44, 322] | [VISp] | <NA> | <NA> | 2026-08-19_01-05-33 | 385b41e7-68c2-4c0f-8f5c-8771a9413498 |
| 146 | 804363 | multiplane-ophys_804363_2025-09-10_14-44-56 | multiplane-ophys_804363_2025-09-10_14-44-56_pr... | STAGE_1 | STAGE_1 | STAGE_1 | NaN | 20 | 2025-09-10 | 2025-09-10 | 14:44:56.562364 | 136 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-IC... | Female | 2025-04-27 | 429_MESO1_20241016 | Learning mFISH-V1omFISH | [pophys, behavior] | 8 | [VISp_0, VISp_1, VISp_2, VISp_3, VISp_4, VISp_... | [160, 200, 124, 240, 84, 280, 44, 322] | [VISp] | [] | 0 | 2026-08-19_01-05-12 | 5ed91986-57b6-43b1-8df1-5d0a39b085d2 |
147 rows × 26 columns
Views of the table#
With the table built, these views show what the dataset actually offers before picking sessions for an analysis.
# Column inventory: type, fill rate, and how much each column varies
rows = []
for col in sessions.columns:
s = sessions[col]
as_str = s.map(lambda v: str(v) if isinstance(v, list) else v)
rows.append({
'column': col,
'dtype': str(s.dtype),
'n_missing': int(s.isna().sum()),
'n_unique': int(as_str.nunique(dropna=True)),
'example': str(s.dropna().iloc[0])[:44] if s.notna().any() else '',
})
pd.DataFrame(rows)
| column | dtype | n_missing | n_unique | example | |
|---|---|---|---|---|---|
| 0 | subject_id | object | 0 | 6 | 782149 |
| 1 | session_id | object | 0 | 147 | multiplane-ophys_782149_2025-03-25_09-46-08 |
| 2 | name | object | 0 | 147 | multiplane-ophys_782149_2025-03-25_09-46-08_ |
| 3 | session_type | object | 0 | 13 | TRAINING_0_gratings_autorewards_15min |
| 4 | acquisition_type | object | 0 | 13 | TRAINING_0_gratings_autorewards_15min |
| 5 | stage | object | 0 | 11 | TRAINING_0 |
| 6 | image_set | object | 68 | 2 | A |
| 7 | session_number | int64 | 0 | 32 | 1 |
| 8 | acquisition_date | object | 0 | 96 | 2025-03-25 |
| 9 | session_date | object | 0 | 96 | 2025-03-25 |
| 10 | session_time | object | 0 | 147 | 09:46:08.591468 |
| 11 | age_days | int64 | 0 | 74 | 108 |
| 12 | genotype | object | 0 | 1 | Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE- |
| 13 | sex | object | 0 | 2 | Male |
| 14 | date_of_birth | object | 0 | 6 | 2024-12-07 |
| 15 | rig | object | 0 | 3 | 422_MESO2_20241017 |
| 16 | project_name | object | 0 | 4 | LearningmFISHTask1A |
| 17 | modality | object | 0 | 1 | ['pophys', 'behavior'] |
| 18 | n_planes | int64 | 0 | 1 | 8 |
| 19 | plane_names | object | 0 | 1 | ['VISp_0', 'VISp_1', 'VISp_2', 'VISp_3', 'VI |
| 20 | imaging_depths | object | 0 | 141 | [40, 320, 80, 280, 120, 240, 160, 200] |
| 21 | targeted_structures | object | 0 | 1 | ['VISp'] |
| 22 | planes_failing_zdrift | object | 34 | 22 | [] |
| 23 | n_planes_failing_zdrift | Int64 | 34 | 9 | 0 |
| 24 | processed_stamp | object | 0 | 143 | 2026-08-19_00-32-51 |
| 25 | _id | object | 0 | 147 | aca6e6d2-9f33-4ed6-8a66-86d69a282c30 |
# Which columns are constant across the cohort (no use as a selector)?
varying, constant = [], []
for col in sessions.columns:
as_str = sessions[col].map(lambda v: str(v) if isinstance(v, list) else v)
(constant if as_str.nunique(dropna=True) <= 1 else varying).append(col)
print(f'constant across all {len(sessions)} sessions:')
for c in constant:
print(f' {c} = {sessions[c].dropna().iloc[0] if sessions[c].notna().any() else "all NA"}')
print(f'\nvarying ({len(varying)}): {varying}')
constant across all 147 sessions:
genotype = Slc32a1-IRES-Cre/wt;Oi1(TIT2L-jGCaMP8s-WPRE-ICL-IRES-tTA2)/wt
modality = ['pophys', 'behavior']
n_planes = 8
plane_names = ['VISp_0', 'VISp_1', 'VISp_2', 'VISp_3', 'VISp_4', 'VISp_5', 'VISp_6', 'VISp_7']
targeted_structures = ['VISp']
varying (21): ['subject_id', 'session_id', 'name', 'session_type', 'acquisition_type', 'stage', 'image_set', 'session_number', 'acquisition_date', 'session_date', 'session_time', 'age_days', 'sex', 'date_of_birth', 'rig', 'project_name', 'imaging_depths', 'planes_failing_zdrift', 'n_planes_failing_zdrift', 'processed_stamp', '_id']
# Categorical columns worth filtering on
for col in ['session_type', 'image_set', 'rig', 'project_name', 'sex']:
counts = sessions[col].value_counts(dropna=False)
print(f'--- {col} ({counts.size} values)')
print(counts.to_string(), '\n')
--- session_type (13 values)
session_type
TRAINING_1_gratings 26
TRAINING_3_images_A_10uL_reward 19
STAGE_1 19
OPHYS_6_images_B 15
OPHYS_1_images_A 12
OPHYS_4_images_B 12
TRAINING_2_gratings_flashed 9
TRAINING_4_images_A_training 7
TRAINING_0_gratings_autorewards_15min 7
TRAINING_5_images_A_epilogue 7
STAGE_0 7
TRAINING_5_images_A_handoff_ready 6
TRAINING_5_images_A_handoff_lapsed 1
--- image_set (3 values)
image_set
NaN 68
A 52
B 27
--- rig (3 values)
rig
422_MESO2_20241017 75
429_MESO1_20241016 44
422_MESO2_20220218 28
--- project_name (4 values)
project_name
Learning mFISH-V1omFISH 80
LearningmFISHTask1A 56
U01BFCT 9
ISIx 2
--- sex (2 values)
sex
Male 80
Female 67
# Sessions per mouse per session_type -- where the usable data actually is
pd.crosstab(sessions.subject_id, sessions.session_type,
margins=True, margins_name='TOTAL')
| session_type | OPHYS_1_images_A | OPHYS_4_images_B | OPHYS_6_images_B | STAGE_0 | STAGE_1 | TRAINING_0_gratings_autorewards_15min | TRAINING_1_gratings | TRAINING_2_gratings_flashed | TRAINING_3_images_A_10uL_reward | TRAINING_4_images_A_training | TRAINING_5_images_A_epilogue | TRAINING_5_images_A_handoff_lapsed | TRAINING_5_images_A_handoff_ready | TOTAL |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| subject_id | ||||||||||||||
| 782149 | 2 | 2 | 2 | 1 | 3 | 1 | 3 | 1 | 3 | 2 | 2 | 1 | 1 | 24 |
| 788406 | 2 | 2 | 3 | 2 | 3 | 1 | 11 | 2 | 3 | 1 | 1 | 0 | 1 | 32 |
| 790322 | 2 | 2 | 2 | 1 | 4 | 2 | 3 | 2 | 3 | 1 | 1 | 0 | 1 | 24 |
| 800792 | 2 | 2 | 3 | 1 | 3 | 1 | 4 | 2 | 4 | 1 | 1 | 0 | 1 | 25 |
| 800995 | 2 | 2 | 3 | 1 | 3 | 1 | 3 | 1 | 3 | 1 | 1 | 0 | 1 | 22 |
| 804363 | 2 | 2 | 2 | 1 | 3 | 1 | 2 | 1 | 3 | 1 | 1 | 0 | 1 | 20 |
| TOTAL | 12 | 12 | 15 | 7 | 19 | 7 | 26 | 9 | 19 | 7 | 7 | 1 | 6 | 147 |
# Imaging geometry: are the 8 planes at consistent depths across sessions?
depths = sessions.explode('imaging_depths')
print('distinct imaging depths:', sorted(depths.imaging_depths.dropna().unique()))
print('\ndepth range per plane count')
print(sessions.groupby('n_planes').imaging_depths.apply(
lambda col: f'{min(min(d) for d in col)} - {max(max(d) for d in col)} um').to_string())
print('\ntargeted structures:',
sorted({s for lst in sessions.targeted_structures for s in lst}))
distinct imaging depths: [20, 22, 26, 28, 29, 30, 32, 34, 35, 36, 37, 38, 40, 41, 42, 43, 44, 45, 46, 48, 50, 64, 66, 68, 70, 72, 74, 75, 76, 78, 80, 81, 82, 84, 85, 86, 88, 90, 92, 94, 95, 96, 98, 100, 102, 104, 105, 108, 110, 112, 114, 115, 116, 118, 120, 122, 124, 126, 127, 128, 129, 130, 132, 139, 144, 146, 148, 150, 152, 154, 155, 156, 157, 158, 160, 162, 164, 166, 167, 168, 170, 172, 174, 175, 179, 182, 186, 188, 190, 192, 194, 195, 196, 198, 200, 202, 204, 206, 208, 210, 212, 216, 219, 220, 222, 223, 224, 225, 226, 228, 230, 232, 234, 235, 236, 238, 240, 242, 244, 245, 246, 248, 250, 252, 254, 256, 257, 258, 260, 262, 263, 264, 265, 266, 268, 270, 272, 274, 275, 276, 278, 280, 282, 284, 288, 289, 290, 292, 294, 296, 298, 300, 302, 304, 305, 306, 308, 310, 312, 314, 316, 318, 320, 322, 324, 328, 330, 344, 345, 347, 348, 354]
depth range per plane count
n_planes
8 20 - 354 um
targeted structures: ['VISp']
# Numeric spread, and how age and session count relate per mouse
print(sessions[['age_days', 'n_planes', 'session_number',
'n_planes_failing_zdrift']].describe().to_string())
print('\nper-mouse span')
print(sessions.groupby('subject_id').agg(
n_sessions=('session_id', 'size'),
first_date=('acquisition_date', 'min'),
last_date=('acquisition_date', 'max'),
age_first=('age_days', 'min'),
age_last=('age_days', 'max'),
).to_string())
age_days n_planes session_number n_planes_failing_zdrift
count 147.000000 147.0 147.000000 113.0
mean 141.020408 8.0 13.034014 1.026549
std 22.161138 0.0 7.610748 1.759972
min 107.000000 8.0 1.000000 0.0
25% 124.500000 8.0 7.000000 0.0
50% 137.000000 8.0 13.000000 0.0
75% 151.500000 8.0 19.000000 1.0
max 202.000000 8.0 32.000000 8.0
per-mouse span
n_sessions first_date last_date age_first age_last
subject_id
782149 24 2025-03-25 2025-05-07 108 151
788406 32 2025-05-29 2025-07-29 131 192
790322 24 2025-06-11 2025-08-21 131 202
800792 25 2025-07-22 2025-08-29 107 145
800995 22 2025-08-05 2025-09-18 120 164
804363 20 2025-08-12 2025-09-10 107 136
# Z-drift QC coverage -- and the caveat that NA is not a pass
qc_cov = sessions.n_planes_failing_zdrift.notna()
print(f'sessions with z-drift QC: {int(qc_cov.sum())} / {len(sessions)}')
print(f' clean (0 failing planes): {int((sessions.n_planes_failing_zdrift == 0).sum())}')
print(f' >=1 failing plane: {int((sessions.n_planes_failing_zdrift > 0).sum())}')
print(f' no QC (NA, NOT a pass): {int((~qc_cov).sum())}')
print('\nQC coverage by session_type')
print(sessions.assign(has_qc=qc_cov).groupby('session_type').has_qc.agg(
n='size', with_qc='sum').to_string())
sessions with z-drift QC: 113 / 147
clean (0 failing planes): 71
>=1 failing plane: 42
no QC (NA, NOT a pass): 34
QC coverage by session_type
n with_qc
session_type
OPHYS_1_images_A 12 12
OPHYS_4_images_B 12 12
OPHYS_6_images_B 15 15
STAGE_0 7 5
STAGE_1 19 14
TRAINING_0_gratings_autorewards_15min 7 3
TRAINING_1_gratings 26 17
TRAINING_2_gratings_flashed 9 5
TRAINING_3_images_A_10uL_reward 19 11
TRAINING_4_images_A_training 7 5
TRAINING_5_images_A_epilogue 7 7
TRAINING_5_images_A_handoff_lapsed 1 1
TRAINING_5_images_A_handoff_ready 6 6
# Candidate sessions for a problem set: QC present and nothing failing
usable = sessions[sessions.n_planes_failing_zdrift == 0]
print(f'{len(usable)} sessions with zero z-drift failures')
print(usable.groupby(['subject_id', 'session_type']).size().to_string())
# Which planes fail z-drift most often across the cohort?
exploded = sessions.planes_failing_zdrift.dropna().explode().dropna()
print('\nz-drift failures by plane')
print(exploded.value_counts().sort_index().to_string())
71 sessions with zero z-drift failures
subject_id session_type
782149 OPHYS_4_images_B 1
OPHYS_6_images_B 1
STAGE_0 1
TRAINING_0_gratings_autorewards_15min 1
TRAINING_1_gratings 2
TRAINING_3_images_A_10uL_reward 1
TRAINING_5_images_A_epilogue 1
788406 STAGE_1 1
TRAINING_0_gratings_autorewards_15min 1
TRAINING_1_gratings 6
TRAINING_2_gratings_flashed 2
TRAINING_3_images_A_10uL_reward 3
TRAINING_4_images_A_training 1
TRAINING_5_images_A_handoff_ready 1
790322 OPHYS_1_images_A 2
OPHYS_4_images_B 2
OPHYS_6_images_B 2
STAGE_1 3
TRAINING_0_gratings_autorewards_15min 1
TRAINING_1_gratings 3
TRAINING_2_gratings_flashed 2
TRAINING_3_images_A_10uL_reward 2
TRAINING_4_images_A_training 1
TRAINING_5_images_A_epilogue 1
TRAINING_5_images_A_handoff_ready 1
800792 OPHYS_1_images_A 2
OPHYS_4_images_B 2
OPHYS_6_images_B 3
TRAINING_5_images_A_epilogue 1
TRAINING_5_images_A_handoff_ready 1
800995 OPHYS_1_images_A 2
OPHYS_4_images_B 1
OPHYS_6_images_B 2
STAGE_1 3
TRAINING_3_images_A_10uL_reward 1
TRAINING_5_images_A_epilogue 1
TRAINING_5_images_A_handoff_ready 1
804363 OPHYS_1_images_A 1
OPHYS_4_images_B 2
STAGE_1 1
TRAINING_3_images_A_10uL_reward 1
TRAINING_4_images_A_training 1
TRAINING_5_images_A_epilogue 1
TRAINING_5_images_A_handoff_ready 1
z-drift failures by plane
planes_failing_zdrift
VISp_0 25
VISp_1 7
VISp_2 19
VISp_3 6
VISp_4 21
VISp_5 4
VISp_6 30
VISp_7 4
Sanity checks#
docDB drops rows silently — it returns no error when an asset simply is not indexed. Read these counts against what you expect from the processing batch; if a mouse is short, re-run rather than assuming the data is missing.
print('sessions per mouse')
print(sessions.subject_id.value_counts().sort_index().to_string())
print('\nplanes per session (8 for all -- enforced in the query)')
print(sessions.n_planes.value_counts().sort_index().to_string())
print('\nsession types')
print(sessions.session_type.value_counts().to_string())
missing = set(VISUAL_LEARNING_MICE) - set(sessions.subject_id)
if missing:
print(f'\nno sessions returned for: {sorted(missing)}')
sessions per mouse
subject_id
782149 24
788406 32
790322 24
800792 25
800995 22
804363 20
planes per session (8 for all -- enforced in the query)
n_planes
8 147
session types
session_type
TRAINING_1_gratings 26
TRAINING_3_images_A_10uL_reward 19
STAGE_1 19
OPHYS_6_images_B 15
OPHYS_1_images_A 12
OPHYS_4_images_B 12
TRAINING_2_gratings_flashed 9
TRAINING_4_images_A_training 7
TRAINING_0_gratings_autorewards_15min 7
TRAINING_5_images_A_epilogue 7
STAGE_0 7
TRAINING_5_images_A_handoff_ready 6
TRAINING_5_images_A_handoff_lapsed 1
Saving the table#
The tutorials read this table from a CSV in the mounted data asset, so it is written out here for that purpose. Nothing above depends on it.
session_csv = f'{OUTPUT_DIR}/visual_learning_session_metadata.csv'
sessions.to_csv(session_csv, index=False)
print(f'{session_csv} ({len(sessions)} rows, {sessions.shape[1]} columns)')
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
Cell In[19], line 2
1 session_csv = f'{OUTPUT_DIR}/visual_learning_session_metadata.csv'
----> 2 sessions.to_csv(session_csv, index=False)
3 print(f'{session_csv} ({len(sessions)} rows, {sessions.shape[1]} columns)')
File /opt/envs/query/lib/python3.12/site-packages/pandas/util/_decorators.py:333, in deprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper(*args, **kwargs)
327 if len(args) > num_allow_args:
328 warnings.warn(
329 msg.format(arguments=_format_argument_list(allow_args)),
330 FutureWarning,
331 stacklevel=find_stack_level(),
332 )
--> 333 return func(*args, **kwargs)
File /opt/envs/query/lib/python3.12/site-packages/pandas/core/generic.py:3967, in NDFrame.to_csv(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)
3963 float_format=float_format,
3964 decimal=decimal,
3965 )
3966
-> 3967 return DataFrameRenderer(formatter).to_csv(
3968 path_or_buf,
3969 lineterminator=lineterminator,
3970 sep=sep,
File /opt/envs/query/lib/python3.12/site-packages/pandas/io/formats/format.py:1014, in DataFrameRenderer.to_csv(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)
993 created_buffer = False
995 csv_formatter = CSVFormatter(
996 path_or_buf=path_or_buf,
997 lineterminator=lineterminator,
(...) 1012 formatter=self.fmt,
1013 )
-> 1014 csv_formatter.save()
1016 if created_buffer:
1017 assert isinstance(path_or_buf, StringIO)
File /opt/envs/query/lib/python3.12/site-packages/pandas/io/formats/csvs.py:251, in CSVFormatter.save(self)
247 """
248 Create the writer & save.
249 """
250 # apply compression and byte/text conversion
--> 251 with get_handle(
252 self.filepath_or_buffer,
253 self.mode,
254 encoding=self.encoding,
255 errors=self.errors,
256 compression=self.compression,
257 storage_options=self.storage_options,
258 ) as handles:
259 # Note: self.encoding is irrelevant here
260 self.writer = csvlib.writer(
261 handles.handle,
262 lineterminator=self.lineterminator,
(...) 267 quotechar=self.quotechar,
268 )
270 self._save()
File /opt/envs/query/lib/python3.12/site-packages/pandas/io/common.py:749, in get_handle(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)
747 # Only for write methods
748 if "r" not in mode and is_path:
--> 749 check_parent_directory(str(handle))
751 if compression:
752 if compression != "zstd":
753 # compression libraries do not like an explicit text-mode
File /opt/envs/query/lib/python3.12/site-packages/pandas/io/common.py:616, in check_parent_directory(path)
614 parent = Path(path).parent
615 if not parent.is_dir():
--> 616 raise OSError(rf"Cannot save file into a non-existent directory: '{parent}'")
OSError: Cannot save file into a non-existent directory: '/data/metadata'