Programmatic Access

Programmatic Access#

Important

Before using any programmatic access to the data, you first need to set up your CAVEclient token.

CAVEclient#

Most programmatic access to the CAVE services occurs through CAVEclient, a Python client to access various types of data from the online services.

Full documentation for CAVEclient is available here.

To initialize a caveclient, we give it a datastack, which is a name that defines a particular combination of imagery, segmentation, and annotation database. For the MICrONs public data, we use the datastack name minnie65_public.

import os
from caveclient import CAVEclient
datastack_name = 'minnie65_public'
client = CAVEclient(datastack_name)

# set version, for consistency across time
client.materialize.version = 1078 # Current as of Summer 2024

# Show the description of the datastack
client.info.get_datastack_info()['description']

---------------------------------------------------------------------------
RecursionError                            Traceback (most recent call last)
Cell In[2], line 7
client = CAVEclient(datastack_name)
# set version, for consistency across time
----> 7 client.materialize.version = 1078 # Current as of Summer 2024
# Show the description of the datastack
client.info.get_datastack_info()['description']

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/frameworkclient.py:449, in CAVEclientFull.materialize(self)
"""
A client for the materialization service. See [client.materialize](../client_api/materialize.md)
for more information.
"""
if self._materialize is None:
--> 449     self._materialize = MaterializationClient(
       server_address=self.local_server,
       auth_client=self.auth,
       datastack_name=self._datastack_name,
       synapse_table=self.info.get_datastack_info().get("synapse_table", None),
       max_retries=self._max_retries,
       pool_maxsize=self._pool_maxsize,
       pool_block=self._pool_block,
       over_client=self,
       desired_resolution=self.desired_resolution,
   )
return self._materialize

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:201, in MaterializationClient(server_address, datastack_name, auth_client, cg_client, synapse_table, api_version, version, verify, max_retries, pool_maxsize, pool_block, desired_resolution, over_client)
endpoints, api_version = _api_endpoints(
   api_version,
   SERVER_KEY,
   (...)
   verify=verify,
)
MatClient = client_mapping[api_version]
--> 201 return MatClient(
   server_address,
   auth_header,
   api_version,
   endpoints,
   SERVER_KEY,
   datastack_name,
   cg_client=cg_client,
   synapse_table=synapse_table,
   version=version,
   verify=verify,
   max_retries=max_retries,
   pool_maxsize=pool_maxsize,
   pool_block=pool_block,
   over_client=over_client,
   desired_resolution=desired_resolution,
)

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:1924, in MaterializationClientV3.__init__(self, *args, **kwargs)
if self.fc is not None:
   if metadata[0].result() is not None and metadata[1].result() is not None:
-> 1924         tables = TableManager(
           self.fc, metadata[0].result(), metadata[1].result()
       )
self._tables = tables
if self.fc is not None:

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:669, in TableManager.__init__(self, client, metadata, schema)
populate_table_cache(client, metadata=self._table_metadata)
for tn in self._tables:
--> 669     setattr(self, tn, make_query_filter(tn, self._table_metadata[tn], client))

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:623, in make_query_filter(table_name, meta, client)
def make_query_filter(table_name, meta, client):
   (
       pts,
       val_cols,
       all_unbd_pts,
       table_map,
       rename_map,
       table_list,
       desc,
--> 623     ) = get_table_info(table_name, meta, client)
   class_vals = make_class_vals(
       pts, val_cols, all_unbd_pts, table_map, rename_map, table_list
   )
   QueryFilter = attrs.make_class(
       table_name, class_vals, bases=(make_kwargs_mixin(client),)
   )

File /opt/envs/allensdk/lib/python3.10/site-packages/cachetools/__init__.py:741, in cached.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
except KeyError:
   pass  # key not found
--> 741 v = func(*args, **kwargs)
try:
   cache[k] = v

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:267, in get_table_info(tn, meta, client, allow_types, merge_schema, suffixes)
   name_ref = None
else:
--> 267     schema = table_metadata(ref_table, client).get("schema")
   ref_pts, ref_cols, ref_unbd_pts = get_col_info(
       meta["schema"], client, allow_types=allow_types, omit_fields=["target_id"]
   )
   name_base = ref_table

File /opt/envs/allensdk/lib/python3.10/site-packages/cachetools/__init__.py:741, in cached.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
except KeyError:
   pass  # key not found
--> 741 v = func(*args, **kwargs)
try:
   cache[k] = v

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:314, in table_metadata(table_name, client, meta)
   warnings.simplefilter(action="ignore")
   if meta is None:
--> 314         meta = client.materialize.get_table_metadata(table_name)
if "schema" not in meta:
   meta["schema"] = meta.get("schema_type")

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/frameworkclient.py:449, in CAVEclientFull.materialize(self)
"""
A client for the materialization service. See [client.materialize](../client_api/materialize.md)
for more information.
"""
if self._materialize is None:
--> 449     self._materialize = MaterializationClient(
       server_address=self.local_server,
       auth_client=self.auth,
       datastack_name=self._datastack_name,
       synapse_table=self.info.get_datastack_info().get("synapse_table", None),
       max_retries=self._max_retries,
       pool_maxsize=self._pool_maxsize,
       pool_block=self._pool_block,
       over_client=self,
       desired_resolution=self.desired_resolution,
   )
return self._materialize

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:201, in MaterializationClient(server_address, datastack_name, auth_client, cg_client, synapse_table, api_version, version, verify, max_retries, pool_maxsize, pool_block, desired_resolution, over_client)
endpoints, api_version = _api_endpoints(
   api_version,
   SERVER_KEY,
   (...)
   verify=verify,
)
MatClient = client_mapping[api_version]
--> 201 return MatClient(
   server_address,
   auth_header,
   api_version,
   endpoints,
   SERVER_KEY,
   datastack_name,
   cg_client=cg_client,
   synapse_table=synapse_table,
   version=version,
   verify=verify,
   max_retries=max_retries,
   pool_maxsize=pool_maxsize,
   pool_block=pool_block,
   over_client=over_client,
   desired_resolution=desired_resolution,
)

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:1924, in MaterializationClientV3.__init__(self, *args, **kwargs)
if self.fc is not None:
   if metadata[0].result() is not None and metadata[1].result() is not None:
-> 1924         tables = TableManager(
           self.fc, metadata[0].result(), metadata[1].result()
       )
self._tables = tables
if self.fc is not None:

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:669, in TableManager.__init__(self, client, metadata, schema)
populate_table_cache(client, metadata=self._table_metadata)
for tn in self._tables:
--> 669     setattr(self, tn, make_query_filter(tn, self._table_metadata[tn], client))

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:623, in make_query_filter(table_name, meta, client)
def make_query_filter(table_name, meta, client):
   (
       pts,
       val_cols,
       all_unbd_pts,
       table_map,
       rename_map,
       table_list,
       desc,
--> 623     ) = get_table_info(table_name, meta, client)
   class_vals = make_class_vals(
       pts, val_cols, all_unbd_pts, table_map, rename_map, table_list
   )
   QueryFilter = attrs.make_class(
       table_name, class_vals, bases=(make_kwargs_mixin(client),)
   )

    [... skipping similar frames: cached.<locals>.decorator.<locals>.wrapper at line 741 (1 times)]

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:267, in get_table_info(tn, meta, client, allow_types, merge_schema, suffixes)
   name_ref = None
else:
--> 267     schema = table_metadata(ref_table, client).get("schema")
   ref_pts, ref_cols, ref_unbd_pts = get_col_info(
       meta["schema"], client, allow_types=allow_types, omit_fields=["target_id"]
   )
   name_base = ref_table

    [... skipping similar frames: cached.<locals>.decorator.<locals>.wrapper at line 741 (1 times)]

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:314, in table_metadata(table_name, client, meta)
   warnings.simplefilter(action="ignore")
   if meta is None:
--> 314         meta = client.materialize.get_table_metadata(table_name)
if "schema" not in meta:
   meta["schema"] = meta.get("schema_type")

    [... skipping similar frames: cached.<locals>.decorator.<locals>.wrapper at line 741 (530 times), MaterializationClient at line 201 (265 times), MaterializationClientV3.__init__ at line 1924 (265 times), TableManager.__init__ at line 669 (265 times), get_table_info at line 267 (265 times), make_query_filter at line 623 (265 times), CAVEclientFull.materialize at line 449 (265 times), table_metadata at line 314 (265 times)]

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/frameworkclient.py:449, in CAVEclientFull.materialize(self)
"""
A client for the materialization service. See [client.materialize](../client_api/materialize.md)
for more information.
"""
if self._materialize is None:
--> 449     self._materialize = MaterializationClient(
       server_address=self.local_server,
       auth_client=self.auth,
       datastack_name=self._datastack_name,
       synapse_table=self.info.get_datastack_info().get("synapse_table", None),
       max_retries=self._max_retries,
       pool_maxsize=self._pool_maxsize,
       pool_block=self._pool_block,
       over_client=self,
       desired_resolution=self.desired_resolution,
   )
return self._materialize

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:201, in MaterializationClient(server_address, datastack_name, auth_client, cg_client, synapse_table, api_version, version, verify, max_retries, pool_maxsize, pool_block, desired_resolution, over_client)
endpoints, api_version = _api_endpoints(
   api_version,
   SERVER_KEY,
   (...)
   verify=verify,
)
MatClient = client_mapping[api_version]
--> 201 return MatClient(
   server_address,
   auth_header,
   api_version,
   endpoints,
   SERVER_KEY,
   datastack_name,
   cg_client=cg_client,
   synapse_table=synapse_table,
   version=version,
   verify=verify,
   max_retries=max_retries,
   pool_maxsize=pool_maxsize,
   pool_block=pool_block,
   over_client=over_client,
   desired_resolution=desired_resolution,
)

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:1924, in MaterializationClientV3.__init__(self, *args, **kwargs)
if self.fc is not None:
   if metadata[0].result() is not None and metadata[1].result() is not None:
-> 1924         tables = TableManager(
           self.fc, metadata[0].result(), metadata[1].result()
       )
self._tables = tables
if self.fc is not None:

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:669, in TableManager.__init__(self, client, metadata, schema)
populate_table_cache(client, metadata=self._table_metadata)
for tn in self._tables:
--> 669     setattr(self, tn, make_query_filter(tn, self._table_metadata[tn], client))

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:623, in make_query_filter(table_name, meta, client)
def make_query_filter(table_name, meta, client):
   (
       pts,
       val_cols,
       all_unbd_pts,
       table_map,
       rename_map,
       table_list,
       desc,
--> 623     ) = get_table_info(table_name, meta, client)
   class_vals = make_class_vals(
       pts, val_cols, all_unbd_pts, table_map, rename_map, table_list
   )
   QueryFilter = attrs.make_class(
       table_name, class_vals, bases=(make_kwargs_mixin(client),)
   )

    [... skipping similar frames: cached.<locals>.decorator.<locals>.wrapper at line 741 (1 times)]

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:267, in get_table_info(tn, meta, client, allow_types, merge_schema, suffixes)
   name_ref = None
else:
--> 267     schema = table_metadata(ref_table, client).get("schema")
   ref_pts, ref_cols, ref_unbd_pts = get_col_info(
       meta["schema"], client, allow_types=allow_types, omit_fields=["target_id"]
   )
   name_base = ref_table

    [... skipping similar frames: cached.<locals>.decorator.<locals>.wrapper at line 741 (1 times)]

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/tools/table_manager.py:314, in table_metadata(table_name, client, meta)
   warnings.simplefilter(action="ignore")
   if meta is None:
--> 314         meta = client.materialize.get_table_metadata(table_name)
if "schema" not in meta:
   meta["schema"] = meta.get("schema_type")

File /opt/envs/allensdk/lib/python3.10/site-packages/cachetools/__init__.py:741, in cached.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
except KeyError:
   pass  # key not found
--> 741 v = func(*args, **kwargs)
try:
   cache[k] = v

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:519, in MaterializationClientV2.get_table_metadata(self, table_name, datastack_name, version, log_warning)
   datastack_name = self.datastack_name
if version is None:
--> 519     version = self.version
endpoint_mapping = self.default_url_mapping
endpoint_mapping["datastack_name"] = datastack_name

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:280, in MaterializationClientV2.version(self)
"""The version of the materialization. Can be used to set up the
client to default to a specific version when timestamps or versions are not
specified in queries. If not set, defaults to the most recent version."""
if self._version is None:
--> 280     self._version = self.most_recent_version()
return self._version

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:335, in MaterializationClientV2.most_recent_version(self, datastack_name)
def most_recent_version(self, datastack_name=None) -> int:
   """
   Get the most recent version of materialization for this datastack name

   (...)
       Most recent version of materialization for this datastack name
   """
--> 335     versions = self.get_versions(datastack_name=datastack_name)
   return np.max(np.array(versions))

File /opt/envs/allensdk/lib/python3.10/site-packages/caveclient/materializationengine.py:360, in MaterializationClientV2.get_versions(self, datastack_name, expired)
url = self._endpoints["versions"].format_map(endpoint_mapping)
query_args = {"expired": expired}
--> 360 response = self.session.get(url, params=query_args)
self.raise_for_status(response)
return response.json()

File /opt/envs/allensdk/lib/python3.10/site-packages/requests/sessions.py:602, in Session.get(self, url, **kwargs)
r"""Sends a GET request. Returns :class:`Response` object.

:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", True)
--> 602 return self.request("GET", url, **kwargs)

File /opt/envs/allensdk/lib/python3.10/site-packages/requests/sessions.py:589, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
send_kwargs = {
   "timeout": timeout,
   "allow_redirects": allow_redirects,
}
send_kwargs.update(settings)
--> 589 resp = self.send(prep, **send_kwargs)
return resp

File /opt/envs/allensdk/lib/python3.10/site-packages/requests/sessions.py:703, in Session.send(self, request, **kwargs)
start = preferred_clock()
# Send the request
--> 703 r = adapter.send(request, **kwargs)
# Total elapsed time of the request (approximately)
elapsed = preferred_clock() - start

File /opt/envs/allensdk/lib/python3.10/site-packages/requests/adapters.py:667, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
   timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
--> 667     resp = conn.urlopen(
       method=request.method,
       url=url,
       body=request.body,
       headers=request.headers,
       redirect=False,
       assert_same_host=False,
       preload_content=False,
       decode_content=False,
       retries=self.max_retries,
       timeout=timeout,
       chunked=chunked,
   )
except (ProtocolError, OSError) as err:
   raise ConnectionError(err, request=request)

File /opt/envs/allensdk/lib/python3.10/site-packages/urllib3/connectionpool.py:789, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
--> 789 response = self._make_request(
   conn,
   method,
   url,
   timeout=timeout_obj,
   body=body,
   headers=headers,
   chunked=chunked,
   retries=retries,
   response_conn=response_conn,
   preload_content=preload_content,
   decode_content=decode_content,
   **response_kw,
)
# Everything went great!
clean_exit = True

File /opt/envs/allensdk/lib/python3.10/site-packages/urllib3/connectionpool.py:536, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
# Receive the response from the server
try:
--> 536     response = conn.getresponse()
except (BaseSSLError, OSError) as e:
   self._raise_timeout(err=e, url=url, timeout_value=read_timeout)

File /opt/envs/allensdk/lib/python3.10/site-packages/urllib3/connection.py:507, in HTTPConnection.getresponse(self)
from .response import HTTPResponse
# Get the response from http.client.HTTPConnection
--> 507 httplib_response = super().getresponse()
try:
   assert_header_parsing(httplib_response.msg)

File /opt/conda/lib/python3.10/http/client.py:1375, in HTTPConnection.getresponse(self)
try:
   try:
-> 1375         response.begin()
   except ConnectionError:
       self.close()

File /opt/conda/lib/python3.10/http/client.py:337, in HTTPResponse.begin(self)
else:
   raise UnknownProtocol(version)
--> 337 self.headers = self.msg = parse_headers(self.fp)
if self.debuglevel > 0:
   for hdr, val in self.headers.items():

File /opt/conda/lib/python3.10/http/client.py:236, in parse_headers(fp, _class)
headers = _read_headers(fp)
hstring = b''.join(headers).decode('iso-8859-1')
--> 236 return email.parser.Parser(_class=_class).parsestr(hstring)

File /opt/conda/lib/python3.10/email/parser.py:67, in Parser.parsestr(self, text, headersonly)
def parsestr(self, text, headersonly=False):
   """Create a message structure from a string.

   Returns the root of the message structure.  Optional headersonly is a
   (...)
   the file.
   """
---> 67     return self.parse(StringIO(text), headersonly=headersonly)

File /opt/conda/lib/python3.10/email/parser.py:56, in Parser.parse(self, fp, headersonly)
   if not data:
       break
---> 56     feedparser.feed(data)
return feedparser.close()

File /opt/conda/lib/python3.10/email/feedparser.py:176, in FeedParser.feed(self, data)
"""Push more data into the parser."""
self._input.push(data)
--> 176 self._call_parse()

File /opt/conda/lib/python3.10/email/feedparser.py:180, in FeedParser._call_parse(self)
def _call_parse(self):
   try:
--> 180         self._parse()
   except StopIteration:
       pass

File /opt/conda/lib/python3.10/email/feedparser.py:256, in FeedParser._parsegen(self)
   self._cur.set_payload(EMPTYSTRING.join(lines))
   return
--> 256 if self._cur.get_content_type() == 'message/delivery-status':
   # message/delivery-status contains blocks of headers separated by
   # a blank line.  We'll represent each header block as a separate
   # nested message object, but the processing is a bit different
   # than standard message/* types because there is no body for the
   # nested messages.  A blank line separates the subparts.
   while True:
       self._input.push_eof_matcher(NLCRE.match)

File /opt/conda/lib/python3.10/email/message.py:578, in Message.get_content_type(self)
"""Return the message's content type.

The returned string is coerced to lower case of the form
   (...)
message/rfc822.
"""
missing = object()
--> 578 value = self.get('content-type', missing)
if value is missing:
   # This should have no parameters
   return self.get_default_type()

File /opt/conda/lib/python3.10/email/message.py:471, in Message.get(self, name, failobj)
for k, v in self._headers:
   if k.lower() == name:
--> 471         return self.policy.header_fetch_parse(k, v)
return failobj

File /opt/conda/lib/python3.10/email/_policybase.py:316, in Compat32.header_fetch_parse(self, name, value)
def header_fetch_parse(self, name, value):
   """+
   If the value contains binary data, it is converted into a Header object
   using the unknown-8bit charset.  Otherwise it is returned unmodified.
   """
--> 316     return self._sanitize_header(name, value)

RecursionError: maximum recursion depth exceeded

CAVEclient Basics#

The most frequent use of the CAVEclient is to query the database for annotations like synapses. All database functions are under the client.materialize property. To see what tables are available, use the get_tables function:

client.materialize.get_tables()

For each table, you can see the metadata describing that table. For example, let’s look at the nucleus_detection_v0 table:

client.materialize.get_table_metadata('nucleus_detection_v0')

You get a dictionary of values. Two fields are particularly important: the description, which offers a text description of the contents of the table and voxel_resolution which defines how the coordinates in the table are defined, in nm/voxel.

Querying Tables#

To get the contents of a table, use the query_table function. This will return the whole contents of a table without any filtering, up to for a maximum limit of 200,000 rows. The table is returned as a Pandas DataFrame and you can immediately use standard Pandas function on it.

cell_type_df = client.materialize.query_table('nucleus_detection_v0')
cell_type_df.head()

Important

While most tables are small enough to be returned in full, the synapse table has hundreds of millions of rows and is too large to download this way

Tables have a collection of columns, some of which specify point in space (columns ending in _position), some a root id (ending in _root_id), and others that contain other information about the object at that point. Before describing some of the most important tables in the database, it’s useful to know about a few advanced options that apply when querying any table.

desired_resolution : This parameter allows you to convert the columns specifying spatial points to different resolutions. Many tables are stored at a resolution of 4x4x40 nm/voxel, for example, but you can convert to nanometers by setting desired_resolution=[1,1,1].
split_positions : This parameter allows you to split the columns specifying spatial points into separate columns for each dimension. The new column names will be the original column name with _x, _y, and _z appended.
select_columns : This parameter allows you to get only a subset of columns from the table. Once you know exactly what you want, this can save you some cleanup.
limit : This parameter allows you to limit the number of rows returned. If you are just testing out a query or trying to inspect the kind of data within a table, you can set this to a small number to make sure it works before downloading the whole table. Note that this will show a warning so that you don’t accidentally limit your query when you don’t mean to.

For example, using all of these together:

cell_type_df = client.materialize.query_table('nucleus_detection_v0', split_positions=True, desired_resolution=[1,1,1], select_columns=['pt_position', 'pt_root_id'], limit=10)
cell_type_df

Filtering Queries#

Filtering tables so that you only get data about certain rows back is a very common operation. While there are filtering options in the query_table function (see documentation for more details), a more unified filter interface is available through a “table manager” interface. Rather than passing a table name to the query_table function, client.materialize.tables has a subproperty for each table in the database that can be used to filter that table. The general pattern for usage is

client.materialize.tables.{table_name}({filter options}).query({format and timestamp options})

where {table_name} is the name of the table you want to filter, {filter options} is a collection of arguments for filtering the query, and {format and timestamp options} are those parameters controlling the format and timestamp of the query.

For example, let’s look at the table aibs_metamodel_celltypes_v661, which has cell type predictions across the dataset. We can get the whole table as a DataFrame:

cell_type_df = client.materialize.tables.aibs_metamodel_celltypes_v661().query()
cell_type_df.head()

and we can add similar formatting options as in the last section to the query function:

cell_type_df = client.materialize.tables.aibs_metamodel_celltypes_v661().query(split_positions=True, desired_resolution=[1,1,1], select_columns=['pt_position', 'pt_root_id', 'cell_type'], limit=10)
cell_type_df

However, now we can also filter the table to get only cells that are predicted to have cell type "BC" (for “basket cell”).

my_cell_type = "BC"
client.materialize.tables.aibs_metamodel_celltypes_v661(cell_type=my_cell_type).query()

or maybe we just want the cell types for a particular collection of root ids:

my_root_ids = [864691135771677771, 864691135560505569, 864691136723556861]
client.materialize.tables.aibs_metamodel_celltypes_v661(pt_root_id=my_root_ids).query()

You can get a list of all parameters than be used for querying with the standard IPython/Jupyter docstring functionality, e.g. client.materialize.tables.aibs_metamodel_celltypes_v661.

Note

Use of this functionality will show a brief warning that the interface is experimental. This is because the interface is still being developed and may change in the near future in response to user feedback.

Querying Synapses#

While synapses are stored as any other table in the database, in this case synapses_pni_2, this table is much larger than any other table at more than 337 million rows, and it works best when queried in a different way. The synapse_query function allows you to query the synapse table in a more convenient way than most other tables. In particular, the pre_ids and post_ids let you specify which root id (or collection of root ids) you want to query, with pre_ids indicating the collection of presynaptic neurons and post_ids the collection of postsynaptic neurons. Using both pre_ids and post_ids in one call is effectively a logical AND, returning only those synapses from neurons in the list of pre_ids that target neurons in the list of post_ids. Let’s look at one particular example.

my_root_id = 864691135808473885
syn_df = client.materialize.synapse_query(pre_ids=my_root_id)
print(f"Total number of output synapses for {my_root_id}: {len(syn_df)}")
syn_df.head()

Note that synapse queries always return the list of every synapse between the neurons in the query, even if there are multiple synapses between the same pair of neurons.

A common pattern to generate a list of connections between unique pairs of neurons is to group by the root ids of the presynaptic and postsynaptic neurons and then count the number of synapses between them. For example, to get the number of synapses from this neuron onto every other neuron, ordered

syn_df.groupby(
  ['pre_pt_root_id', 'post_pt_root_id']
).count()[['id']].rename(
  columns={'id': 'syn_count'}
).sort_values(
  by='syn_count',
  ascending=False,
)
# Note that the 'id' part here is just a way to quickly extract one column.
# This could be any of the remaining column names, but `id` is often convenient because it is common to all tables.