Skip to content

Instantly share code, notes, and snippets.

@thewtex

thewtex/I46_tiles_ngff_datatree.ipynb

Created April 21, 2022 21:23
Show Gist options
  • Save thewtex/bdf695d41f5d7a4dd90836442cbf266e to your computer and use it in GitHub Desktop.
Save thewtex/bdf695d41f5d7a4dd90836442cbf266e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
I46_tiles_ngff_datatree.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 158,
"id": "091ec92c-e6c3-4bfd-b113-2602c069809e",
"metadata": {},
"outputs": [],
"source": [
"import tifffile\n",
"from dandiio import RemoteDandiFileSystem\n",
"from dandiio.tiffio import get_ome_xml\n",
"import fsspec\n",
"from fsspec.implementations.memory import MemoryFileSystem\n",
"import imagecodecs.numcodecs\n",
"import xarray as xr\n",
"import re\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import itk\n",
"import json\n",
"import tifffile\n",
"import zarr\n",
"import os\n",
"from spatial_image import to_spatial_image\n",
"from spatial_image_multiscale import to_multiscale\n",
"from datatree import DataTree"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "6521784f-0c5f-4560-868b-fad8ffe448a6",
"metadata": {},
"outputs": [],
"source": [
"# Setup a DandiFS with root at dandiset/000026\n",
"dandifs = RemoteDandiFileSystem('000026')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "498c25e8-b244-4396-aa4d-0081566e54cf",
"metadata": {},
"outputs": [],
"source": [
"# Get all files\n",
"# Directly calling get_assets_with_path_prefix is way faster than\n",
"# using dandifs.glob (because glob uses walk, and therefore requests\n",
"# all files under the deepest directory)\n",
"files = dandifs.dandiset.get_assets_with_path_prefix(\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/'\n",
" 'sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "76699ab4-b1a3-412b-9886-209231468581",
"metadata": {},
"outputs": [],
"source": [
"# Split json/tif\n",
"files = list(files)\n",
"json_files = list(sorted(f.path for f in files if f.path.endswith('.json')))\n",
"files = list(sorted(f.path for f in files if f.path.endswith('.ome.tif')))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "b97b5a55-1d2e-42dc-8e97-20ec7db8f4ae",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-00_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-01_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-02_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-03_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-04_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-05_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-06_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-07_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-08_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-09_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-10_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-11_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-12_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-13_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-14_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-15_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-16_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-17_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-18_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-19_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-20_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-21_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-22_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-23_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-24_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-25_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-26_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-27_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-28_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-29_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-30_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-31_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-32_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-33_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-34_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-35_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-36_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-37_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-38_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-39_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-40_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-41_SPIM.ome.tif',\n",
" 'rawdata/sub-I48/ses-SPIM/microscopy/sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-42_SPIM.ome.tif']"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"files"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "b194d62b-da9f-466f-9490-77e110462fb6",
"metadata": {},
"outputs": [],
"source": [
"# Read metadata of first chunk\n",
"with dandifs.open(files[0]) as f:\n",
" ome = get_ome_xml(f)\n",
"shape = {\n",
" axis.lower(): ome['ome:Image'][0]['ome:Pixels']['@Size' + axis]\n",
" for axis in 'XYZ'\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "73117bd5-c144-416d-8bf7-296d63a1b0bd",
"metadata": {},
"outputs": [],
"source": [
"with dandifs.open(files[0]) as f:\n",
" with open('./test.tif', 'wb') as fw:\n",
" fw.write(f.read())\n",
"image = itk.imread('./test.tif')"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "ebb9fc40-56bf-40a5-9149-e38302adbf5b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'BodyPart': 'BRAIN',\n",
" 'BodyPartDetails': 'Brodmann area 44/45',\n",
" 'BodyPartDetailsOntology': 'FMA',\n",
" 'ChunkTransformMatrix': [[0.0, 0.0, 0.0, 0.0],\n",
" [0.0, 0.0, 0.0, 855.0],\n",
" [0.0, 0.0, 0.0, 0.0],\n",
" [0.0, 0.0, 0.0, 1.0]],\n",
" 'ChunkTransformMatrixAxis': ['X', 'Y', 'Z'],\n",
" 'Environment': 'exvivo',\n",
" 'InstitutionAddress': 'Via Nello Carrara 1, 50019 Sesto Fiorentino (FI), Italy',\n",
" 'InstitutionName': 'LENS - European Laboratory for NonLinear Spectroscopy - University of Florence',\n",
" 'InstitutionalDepartmentName': 'BioPhotonics Lab',\n",
" 'Pathology': 'no pathology',\n",
" 'PixelSize': [3.6, 3.6, 3.6],\n",
" 'PixelSizeUnits': 'um',\n",
" 'SampleExtractionInstitution': 'Massachusetts General Hospital',\n",
" 'SampleExtractionProtocol': 'autopsy',\n",
" 'SampleFixation': 'Formalin fixation for two years, stored in PLP 2% solution',\n",
" 'SampleStaining': 'Calretinin'}"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"with dandifs.open(json_files[3]) as f:\n",
" sidecar = json.load(f)\n",
"sidecar"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "169a8f05-1e08-4806-a57e-d4bdd62632c7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'@xmlns:ome': 'http://www.openmicroscopy.org/Schemas/OME/2016-06',\n",
" '@xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',\n",
" '@xsi:schemaLocation': 'http://www.openmicroscopy.org/Schemas/OME/2016-06 http://www.openmicroscopy.org/Schemas/OME/2016-06/ome.xsd',\n",
" 'ome:Image': [{'@ID': 'Image:0',\n",
" '@Name': 'sub-I48_ses-SPIM_sample-BrocaAreaS01_stain-Calretinin_chunk-00_SPIM.ome.tif',\n",
" 'ome:AcquisitionDate': '2021-07-26T00:00:00',\n",
" 'ome:Pixels': {'@BigEndian': False,\n",
" '@DimensionOrder': 'XYZCT',\n",
" '@ID': 'Pixels:0',\n",
" '@Interleaved': False,\n",
" '@SizeC': 1,\n",
" '@SizeT': 1,\n",
" '@SizeX': 12491,\n",
" '@SizeY': 342,\n",
" '@SizeZ': 242,\n",
" '@Type': 'uint16',\n",
" '@PhysicalSizeX': 3.6,\n",
" '@PhysicalSizeXUnit': 'µm',\n",
" '@PhysicalSizeY': 3.6,\n",
" '@PhysicalSizeYUnit': 'µm',\n",
" '@PhysicalSizeZ': 3.6,\n",
" '@PhysicalSizeZUnit': 'µm',\n",
" '@TimeIncrementUnit': 's',\n",
" 'ome:Channel': [{'@ID': 'Channel:00',\n",
" '@SamplesPerPixel': 1,\n",
" '@Name': 'Calretinin',\n",
" '@ExcitationWavelength': 488.0,\n",
" '@ExcitationWavelengthUnit': 'nm',\n",
" '@EmissionWavelength': 525.0,\n",
" '@EmissionWavelengthUnit': 'nm',\n",
" '@Fluor': 'Alexa Fluor 488',\n",
" '@PinholeSizeUnit': 'µm',\n",
" '@Color': -1,\n",
" 'ome:LightPath': None}],\n",
" 'ome:TiffData': [{'@IFD': 0,\n",
" '@PlaneCount': 242,\n",
" '@FirstZ': 0,\n",
" '@FirstT': 0,\n",
" '@FirstC': 0}]}}]}"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ome"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "30f565cb-b35c-4a20-960a-504f52868502",
"metadata": {},
"outputs": [],
"source": [
"endian = '>' if ome['ome:Image'][0]['ome:Pixels']['@BigEndian'] else '<'\n",
"dtype = np.dtype(ome['ome:Image'][0]['ome:Pixels']['@Type'])\n",
"if dtype.str[0] not in (endian, '|'):\n",
" dtype = endian + dtype.str[1:]\n",
"stain = re.search(r'stain-([^_\\.]+)', files[0]).group(1)"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "6abe3a28-2d50-4a0a-8440-dfefda773b27",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__enter__', '__eq__', '__exit__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_decoders', '_fh', '_files', '_lsm_fix_strip_bytecounts', '_lsm_fix_strip_offsets', '_lsm_load_pages', '_multifile', '_ndpi_load_pages', '_parent', '_philips_load_pages', '_series_bif', '_series_fluoview', '_series_generic', '_series_imagej', '_series_lsm', '_series_mdgel', '_series_ndpi', '_series_ome', '_series_qpi', '_series_scanimage', '_series_scn', '_series_shaped', '_series_sis', '_series_stk', '_series_svs', '_series_uniform', 'andor_metadata', 'asarray', 'aszarr', 'byteorder', 'close', 'eer_metadata', 'epics_metadata', 'fei_metadata', 'filehandle', 'filename', 'flags', 'fluoview_metadata', 'fstat', 'geotiff_metadata', 'imagej_metadata', 'is_appendable', 'is_bigtiff', 'is_lsm', 'is_mdgel', 'is_ndpi', 'is_philips', 'is_scanimage', 'is_uniform', 'lsm_metadata', 'mdgel_metadata', 'metaseries_metadata', 'micromanager_metadata', 'nih_metadata', 'ome_metadata', 'pages', 'philips_metadata', 'pilatus_metadata', 'scanimage_metadata', 'scn_metadata', 'sem_metadata', 'series', 'shaped_metadata', 'sis_metadata', 'stk_metadata', 'tiff', 'tvips_metadata']\n"
]
}
],
"source": [
"with dandifs.open(files[0]) as f:\n",
" tiff = tifffile.TiffFile(f)\n",
" print(dir(tiff))"
]
},
{
"cell_type": "code",
"execution_count": 102,
"id": "7cd6321a-91f7-4dda-992b-cc667e0e8a0e",
"metadata": {},
"outputs": [],
"source": [
"f = dandifs.open(files[0])\n",
"tiff = tifffile.TiffFile(f)"
]
},
{
"cell_type": "code",
"execution_count": 104,
"id": "77e81135-d25c-4f0b-96b7-3a669afaa281",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 7.4 s, sys: 1.69 s, total: 9.1 s\n",
"Wall time: 24.7 s\n"
]
}
],
"source": [
"%time z = tiff.aszarr()"
]
},
{
"cell_type": "code",
"execution_count": 106,
"id": "17d5f422-600a-44c3-b6e1-bc8c0c5e5293",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 522 ms, sys: 13.5 ms, total: 536 ms\n",
"Wall time: 1.28 s\n"
]
}
],
"source": [
"%time z.write_fsspec('./dandifs_fsspec.json', dandifs.s3_url(files[0]))"
]
},
{
"cell_type": "code",
"execution_count": 69,
"id": "865ac768-5ee6-4c64-95af-ff252c9c8a5f",
"metadata": {},
"outputs": [],
"source": [
"f.close()"
]
},
{
"cell_type": "code",
"execution_count": 70,
"id": "2c7fb5dd-a785-402d-8639-0b2f8d3028fd",
"metadata": {},
"outputs": [],
"source": [
"z = tifffile.imread('./test.tif', aszarr=True)"
]
},
{
"cell_type": "code",
"execution_count": 80,
"id": "ab906f63-2262-4016-af5c-1641f4311f42",
"metadata": {},
"outputs": [],
"source": [
"zg = zarr.open(z)"
]
},
{
"cell_type": "code",
"execution_count": 84,
"id": "d58f8df0-af6c-4eee-9fed-e7f036b5a680",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['__array__',\n",
" '__class__',\n",
" '__delattr__',\n",
" '__dict__',\n",
" '__dir__',\n",
" '__doc__',\n",
" '__eq__',\n",
" '__format__',\n",
" '__ge__',\n",
" '__getattribute__',\n",
" '__getitem__',\n",
" '__getstate__',\n",
" '__gt__',\n",
" '__hash__',\n",
" '__init__',\n",
" '__init_subclass__',\n",
" '__iter__',\n",
" '__le__',\n",
" '__len__',\n",
" '__lt__',\n",
" '__module__',\n",
" '__ne__',\n",
" '__new__',\n",
" '__reduce__',\n",
" '__reduce_ex__',\n",
" '__repr__',\n",
" '__setattr__',\n",
" '__setitem__',\n",
" '__setstate__',\n",
" '__sizeof__',\n",
" '__str__',\n",
" '__subclasshook__',\n",
" '__weakref__',\n",
" '_append_nosync',\n",
" '_attrs',\n",
" '_cache_metadata',\n",
" '_cdata_shape',\n",
" '_chunk_delitem',\n",
" '_chunk_delitems',\n",
" '_chunk_getitem',\n",
" '_chunk_getitems',\n",
" '_chunk_key',\n",
" '_chunk_setitem',\n",
" '_chunk_setitem_nosync',\n",
" '_chunk_setitems',\n",
" '_chunk_store',\n",
" '_chunks',\n",
" '_compressor',\n",
" '_decode_chunk',\n",
" '_dimension_separator',\n",
" '_dtype',\n",
" '_encode_chunk',\n",
" '_fill_value',\n",
" '_filters',\n",
" '_flush_metadata_nosync',\n",
" '_get_basic_selection_nd',\n",
" '_get_basic_selection_zd',\n",
" '_get_selection',\n",
" '_info_items_nosync',\n",
" '_info_reporter',\n",
" '_is_view',\n",
" '_key_prefix',\n",
" '_load_metadata',\n",
" '_load_metadata_nosync',\n",
" '_meta',\n",
" '_nbytes',\n",
" '_nchunks',\n",
" '_oindex',\n",
" '_order',\n",
" '_partial_decompress',\n",
" '_path',\n",
" '_process_chunk',\n",
" '_process_for_setitem',\n",
" '_read_only',\n",
" '_refresh_metadata',\n",
" '_refresh_metadata_nosync',\n",
" '_resize_nosync',\n",
" '_set_basic_selection_nd',\n",
" '_set_basic_selection_zd',\n",
" '_set_selection',\n",
" '_shape',\n",
" '_size',\n",
" '_store',\n",
" '_synchronized_op',\n",
" '_synchronizer',\n",
" '_vindex',\n",
" '_write_empty_chunks',\n",
" '_write_op',\n",
" 'append',\n",
" 'astype',\n",
" 'attrs',\n",
" 'basename',\n",
" 'cdata_shape',\n",
" 'chunk_store',\n",
" 'chunks',\n",
" 'compressor',\n",
" 'digest',\n",
" 'dtype',\n",
" 'fill_value',\n",
" 'filters',\n",
" 'get_basic_selection',\n",
" 'get_coordinate_selection',\n",
" 'get_mask_selection',\n",
" 'get_orthogonal_selection',\n",
" 'hexdigest',\n",
" 'info',\n",
" 'info_items',\n",
" 'initialized',\n",
" 'is_view',\n",
" 'islice',\n",
" 'itemsize',\n",
" 'name',\n",
" 'nbytes',\n",
" 'nbytes_stored',\n",
" 'nchunks',\n",
" 'nchunks_initialized',\n",
" 'ndim',\n",
" 'oindex',\n",
" 'order',\n",
" 'path',\n",
" 'read_only',\n",
" 'resize',\n",
" 'set_basic_selection',\n",
" 'set_coordinate_selection',\n",
" 'set_mask_selection',\n",
" 'set_orthogonal_selection',\n",
" 'shape',\n",
" 'size',\n",
" 'store',\n",
" 'synchronizer',\n",
" 'view',\n",
" 'vindex',\n",
" 'write_empty_chunks']"
]
},
"execution_count": 84,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir(zg)"
]
},
{
"cell_type": "code",
"execution_count": 95,
"id": "19198f04-3a64-4ddb-b54f-063d750b4bdf",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table class=\"zarr-info\"><tbody><tr><th style=\"text-align: left\">Type</th><td style=\"text-align: left\">zarr.core.Array</td></tr><tr><th style=\"text-align: left\">Data type</th><td style=\"text-align: left\">uint16</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(242, 342, 12491)</td></tr><tr><th style=\"text-align: left\">Chunk shape</th><td style=\"text-align: left\">(1, 2, 12491)</td></tr><tr><th style=\"text-align: left\">Order</th><td style=\"text-align: left\">C</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">False</td></tr><tr><th style=\"text-align: left\">Compressor</th><td style=\"text-align: left\">None</td></tr><tr><th style=\"text-align: left\">Store type</th><td style=\"text-align: left\">zarr.storage.KVStore</td></tr><tr><th style=\"text-align: left\">No. bytes</th><td style=\"text-align: left\">2067610248 (1.9G)</td></tr><tr><th style=\"text-align: left\">No. bytes stored</th><td style=\"text-align: left\">185</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">11176271.6</td></tr><tr><th style=\"text-align: left\">Chunks initialized</th><td style=\"text-align: left\">0/41382</td></tr></tbody></table>"
],
"text/plain": [
"Type : zarr.core.Array\n",
"Data type : uint16\n",
"Shape : (242, 342, 12491)\n",
"Chunk shape : (1, 2, 12491)\n",
"Order : C\n",
"Read-only : False\n",
"Compressor : None\n",
"Store type : zarr.storage.KVStore\n",
"No. bytes : 2067610248 (1.9G)\n",
"No. bytes stored : 185\n",
"Storage ratio : 11176271.6\n",
"Chunks initialized : 0/41382"
]
},
"execution_count": 95,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"zg.info\n"
]
},
{
"cell_type": "code",
"execution_count": 98,
"id": "7feaa997-35d7-4c9c-91a9-5faea54f0587",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'https://api.dandiarchive.org/api/dandisets/000026/'"
]
},
"execution_count": 98,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dandifs.dandiset.api_url"
]
},
{
"cell_type": "code",
"execution_count": 90,
"id": "2870fec0-cac0-4f28-95e4-6934f4ff86cc",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"262144"
]
},
"execution_count": 90,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"64*64*64"
]
},
{
"cell_type": "code",
"execution_count": 99,
"id": "8ae4375a-bead-4570-a4ce-06d28bcf20f8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'https://dandiarchive.s3.amazonaws.com/blobs/bea/4e8/bea4e8a7-4040-45bb-a666-032322066a72'"
]
},
"execution_count": 99,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dandifs.s3_url(files[0])"
]
},
{
"cell_type": "code",
"execution_count": 100,
"id": "dbe20857-0f4f-4421-9ac8-aa31ab2dc075",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 506 ms, sys: 17.5 ms, total: 524 ms\n",
"Wall time: 1.31 s\n"
]
}
],
"source": [
"%time z.write_fsspec('./fsspec.json', dandifs.s3_url(files[0]))"
]
},
{
"cell_type": "code",
"execution_count": 101,
"id": "3c029bb8-7480-499d-8f4c-518865025c6a",
"metadata": {},
"outputs": [],
"source": [
"z.close()"
]
},
{
"cell_type": "code",
"execution_count": 114,
"id": "9b98ac4a-eb40-4fb5-b7b0-dc2ebb9b91c0",
"metadata": {},
"outputs": [],
"source": [
"file_limit = 3"
]
},
{
"cell_type": "code",
"execution_count": 115,
"id": "94cc6843-f976-4801-9f72-43b45c8f58cb",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 6.32 s, sys: 4.58 s, total: 10.9 s\n",
"Wall time: 37.5 s\n"
]
}
],
"source": [
"downloaded_files = []\n",
"def download():\n",
" for file_index in range(file_limit):\n",
" with dandifs.open(files[file_index]) as f:\n",
" fn = os.path.basename(files[file_index])\n",
" with open(fn, 'wb') as fw:\n",
" fw.write(f.read())\n",
" downloaded_files.append(fn)\n",
" \n",
"%time download()"
]
},
{
"cell_type": "code",
"execution_count": 133,
"id": "2756e18e-9518-4337-9331-73ab398e6a45",
"metadata": {},
"outputs": [],
"source": [
"tiff_zstore = tifffile.imread(downloaded_files[2], aszarr=True)\n",
"tiff_zgroup = zarr.open(tiff_zstore)"
]
},
{
"cell_type": "code",
"execution_count": 135,
"id": "4e7cb200-128f-4edb-aa52-49b889979fcd",
"metadata": {},
"outputs": [],
"source": [
"si = to_spatial_image(tiff_zgroup)"
]
},
{
"cell_type": "code",
"execution_count": 136,
"id": "3884087f-da2f-4359-9928-8cec26993c69",
"metadata": {},
"outputs": [],
"source": [
"stain = re.search(r'stain-([^_\\.]+)', files[0]).group(1)\n",
"si.name = stain"
]
},
{
"cell_type": "code",
"execution_count": 137,
"id": "6dfea2c4-54b7-49d5-b0d7-12d22e6c3cc3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'BodyPart': 'BRAIN',\n",
" 'BodyPartDetails': 'Brodmann area 44/45',\n",
" 'BodyPartDetailsOntology': 'FMA',\n",
" 'ChunkTransformMatrix': [[0.0, 0.0, 0.0, 0.0],\n",
" [0.0, 0.0, 0.0, 570.0],\n",
" [0.0, 0.0, 0.0, 0.0],\n",
" [0.0, 0.0, 0.0, 1.0]],\n",
" 'ChunkTransformMatrixAxis': ['X', 'Y', 'Z'],\n",
" 'Environment': 'exvivo',\n",
" 'InstitutionAddress': 'Via Nello Carrara 1, 50019 Sesto Fiorentino (FI), Italy',\n",
" 'InstitutionName': 'LENS - European Laboratory for NonLinear Spectroscopy - University of Florence',\n",
" 'InstitutionalDepartmentName': 'BioPhotonics Lab',\n",
" 'Pathology': 'no pathology',\n",
" 'PixelSize': [3.6, 3.6, 3.6],\n",
" 'PixelSizeUnits': 'um',\n",
" 'SampleExtractionInstitution': 'Massachusetts General Hospital',\n",
" 'SampleExtractionProtocol': 'autopsy',\n",
" 'SampleFixation': 'Formalin fixation for two years, stored in PLP 2% solution',\n",
" 'SampleStaining': 'Calretinin'}"
]
},
"execution_count": 137,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"with dandifs.open(json_files[2]) as f:\n",
" sidecar = json.load(f)\n",
"sidecar"
]
},
{
"cell_type": "code",
"execution_count": 147,
"id": "a15d7285-d7a5-436d-8ba2-00a065fbb5f6",
"metadata": {},
"outputs": [],
"source": [
"pixel_size = sidecar['PixelSize']\n",
"scale = { 'z': pixel_size[2], 'y': pixel_size[1], 'x': pixel_size[0] }\n",
"matrix = sidecar['ChunkTransformMatrix']\n",
"translation = { 'z': matrix[2][2], 'y': matrix[1][3], 'x': matrix[0][3] }\n",
"dims = ('z', 'y', 'x')\n",
"# NGFF spec\n",
"units = 'micrometers'"
]
},
{
"cell_type": "code",
"execution_count": 148,
"id": "93b5d694-94f9-46bb-97e9-07f2b96a2a1d",
"metadata": {},
"outputs": [],
"source": [
"si = to_spatial_image(tiff_zgroup, dims=dims, name=stain, scale=scale, axis_units={ axis_name: units for axis_name in dims })"
]
},
{
"cell_type": "code",
"execution_count": 149,
"id": "cffe1f60-aa2b-4582-932a-624ad3db89d7",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2 {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.SpatialImage &#x27;Calretinin&#x27; (z: 242, y: 342, x: 12491)&gt;\n",
"array([[[ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0]],\n",
"\n",
" [[ 0, 111, 106, ..., 0, 0, 0],\n",
" [ 0, 92, 91, ..., 0, 0, 0],\n",
" [ 0, 101, 106, ..., 0, 0, 0],\n",
" ...,\n",
" [ 0, 104, 101, ..., 0, 0, 0],\n",
" [ 0, 111, 108, ..., 0, 0, 0],\n",
" [ 0, 116, 102, ..., 0, 0, 0]],\n",
"\n",
" [[ 0, 0, 103, ..., 0, 0, 0],\n",
" [ 0, 0, 97, ..., 0, 0, 0],\n",
" [ 0, 0, 103, ..., 0, 0, 0],\n",
" ...,\n",
"...\n",
" ...,\n",
" [ 0, 0, 0, ..., 101, 0, 0],\n",
" [ 0, 0, 0, ..., 112, 0, 0],\n",
" [ 0, 0, 0, ..., 99, 0, 0]],\n",
"\n",
" [[ 0, 0, 0, ..., 106, 106, 0],\n",
" [ 0, 0, 0, ..., 108, 108, 0],\n",
" [ 0, 0, 0, ..., 99, 99, 0],\n",
" ...,\n",
" [ 0, 0, 0, ..., 106, 106, 0],\n",
" [ 0, 0, 0, ..., 109, 109, 0],\n",
" [ 0, 0, 0, ..., 101, 101, 0]],\n",
"\n",
" [[ 0, 0, 0, ..., 106, 106, 106],\n",
" [ 0, 0, 0, ..., 99, 99, 99],\n",
" [ 0, 0, 0, ..., 104, 101, 101],\n",
" ...,\n",
" [ 0, 0, 0, ..., 101, 97, 97],\n",
" [ 0, 0, 0, ..., 103, 103, 103],\n",
" [ 0, 0, 0, ..., 103, 104, 104]]], dtype=uint16)\n",
"Coordinates:\n",
" * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 853.2 856.8 860.4 864.0 867.6\n",
" * y (y) float64 570.0 573.6 577.2 ... 1.79e+03 1.794e+03 1.798e+03\n",
" * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.SpatialImage</div><div class='xr-array-name'>'Calretinin'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>z</span>: 242</li><li><span class='xr-has-index'>y</span>: 342</li><li><span class='xr-has-index'>x</span>: 12491</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-cb712d93-f520-4e8c-893e-15285606b4ca' class='xr-array-in' type='checkbox' checked><label for='section-cb712d93-f520-4e8c-893e-15285606b4ca' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>0 0 0 0 0 0 0 0 0 0 0 ... 106 106 101 104 104 101 103 103 103 104 104</span></div><div class='xr-array-data'><pre>array([[[ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0]],\n",
"\n",
" [[ 0, 111, 106, ..., 0, 0, 0],\n",
" [ 0, 92, 91, ..., 0, 0, 0],\n",
" [ 0, 101, 106, ..., 0, 0, 0],\n",
" ...,\n",
" [ 0, 104, 101, ..., 0, 0, 0],\n",
" [ 0, 111, 108, ..., 0, 0, 0],\n",
" [ 0, 116, 102, ..., 0, 0, 0]],\n",
"\n",
" [[ 0, 0, 103, ..., 0, 0, 0],\n",
" [ 0, 0, 97, ..., 0, 0, 0],\n",
" [ 0, 0, 103, ..., 0, 0, 0],\n",
" ...,\n",
"...\n",
" ...,\n",
" [ 0, 0, 0, ..., 101, 0, 0],\n",
" [ 0, 0, 0, ..., 112, 0, 0],\n",
" [ 0, 0, 0, ..., 99, 0, 0]],\n",
"\n",
" [[ 0, 0, 0, ..., 106, 106, 0],\n",
" [ 0, 0, 0, ..., 108, 108, 0],\n",
" [ 0, 0, 0, ..., 99, 99, 0],\n",
" ...,\n",
" [ 0, 0, 0, ..., 106, 106, 0],\n",
" [ 0, 0, 0, ..., 109, 109, 0],\n",
" [ 0, 0, 0, ..., 101, 101, 0]],\n",
"\n",
" [[ 0, 0, 0, ..., 106, 106, 106],\n",
" [ 0, 0, 0, ..., 99, 99, 99],\n",
" [ 0, 0, 0, ..., 104, 101, 101],\n",
" ...,\n",
" [ 0, 0, 0, ..., 101, 97, 97],\n",
" [ 0, 0, 0, ..., 103, 103, 103],\n",
" [ 0, 0, 0, ..., 103, 104, 104]]], dtype=uint16)</pre></div></div></li><li class='xr-section-item'><input id='section-c67fe3a2-3087-4096-9f57-ae04fa150d54' class='xr-section-summary-in' type='checkbox' checked><label for='section-c67fe3a2-3087-4096-9f57-ae04fa150d54' class='xr-section-summary' >Coordinates: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>z</span></div><div class='xr-var-dims'>(z)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>0.0 3.6 7.2 ... 860.4 864.0 867.6</div><input id='attrs-98f3fd1e-f3a6-43df-9534-becf001ee99d' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-98f3fd1e-f3a6-43df-9534-becf001ee99d' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-362a9565-4c8e-464b-abc9-187e7c8a22e7' class='xr-var-data-in' type='checkbox'><label for='data-362a9565-4c8e-464b-abc9-187e7c8a22e7' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>z</dd><dt><span>units :</span></dt><dd>micrometers</dd></dl></div><div class='xr-var-data'><pre>array([ 0. , 3.6, 7.2, ..., 860.4, 864. , 867.6])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>y</span></div><div class='xr-var-dims'>(y)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>570.0 573.6 ... 1.794e+03 1.798e+03</div><input id='attrs-bc0e2ff7-8fbe-4e05-988c-103f78ff4b84' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-bc0e2ff7-8fbe-4e05-988c-103f78ff4b84' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-fb5debb3-9496-4e4d-a161-b95b19d8ef4a' class='xr-var-data-in' type='checkbox'><label for='data-fb5debb3-9496-4e4d-a161-b95b19d8ef4a' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>y</dd><dt><span>units :</span></dt><dd>micrometers</dd></dl></div><div class='xr-var-data'><pre>array([ 570. , 573.6, 577.2, ..., 1790.4, 1794. , 1797.6])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>x</span></div><div class='xr-var-dims'>(x)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>0.0 3.6 7.2 ... 4.496e+04 4.496e+04</div><input id='attrs-af11e906-979a-440d-b26e-8d398c601a75' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-af11e906-979a-440d-b26e-8d398c601a75' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-2a58b232-7ba7-48af-92ef-0234b13851d6' class='xr-var-data-in' type='checkbox'><label for='data-2a58b232-7ba7-48af-92ef-0234b13851d6' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>x</dd><dt><span>units :</span></dt><dd>micrometers</dd></dl></div><div class='xr-var-data'><pre>array([0.00000e+00, 3.60000e+00, 7.20000e+00, ..., 4.49568e+04, 4.49604e+04,\n",
" 4.49640e+04])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-b4edc6ad-cfae-4968-b57c-b6dbeb281182' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-b4edc6ad-cfae-4968-b57c-b6dbeb281182' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.SpatialImage 'Calretinin' (z: 242, y: 342, x: 12491)>\n",
"array([[[ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0],\n",
" [ 0, 0, 0, ..., 0, 0, 0]],\n",
"\n",
" [[ 0, 111, 106, ..., 0, 0, 0],\n",
" [ 0, 92, 91, ..., 0, 0, 0],\n",
" [ 0, 101, 106, ..., 0, 0, 0],\n",
" ...,\n",
" [ 0, 104, 101, ..., 0, 0, 0],\n",
" [ 0, 111, 108, ..., 0, 0, 0],\n",
" [ 0, 116, 102, ..., 0, 0, 0]],\n",
"\n",
" [[ 0, 0, 103, ..., 0, 0, 0],\n",
" [ 0, 0, 97, ..., 0, 0, 0],\n",
" [ 0, 0, 103, ..., 0, 0, 0],\n",
" ...,\n",
"...\n",
" ...,\n",
" [ 0, 0, 0, ..., 101, 0, 0],\n",
" [ 0, 0, 0, ..., 112, 0, 0],\n",
" [ 0, 0, 0, ..., 99, 0, 0]],\n",
"\n",
" [[ 0, 0, 0, ..., 106, 106, 0],\n",
" [ 0, 0, 0, ..., 108, 108, 0],\n",
" [ 0, 0, 0, ..., 99, 99, 0],\n",
" ...,\n",
" [ 0, 0, 0, ..., 106, 106, 0],\n",
" [ 0, 0, 0, ..., 109, 109, 0],\n",
" [ 0, 0, 0, ..., 101, 101, 0]],\n",
"\n",
" [[ 0, 0, 0, ..., 106, 106, 106],\n",
" [ 0, 0, 0, ..., 99, 99, 99],\n",
" [ 0, 0, 0, ..., 104, 101, 101],\n",
" ...,\n",
" [ 0, 0, 0, ..., 101, 97, 97],\n",
" [ 0, 0, 0, ..., 103, 103, 103],\n",
" [ 0, 0, 0, ..., 103, 104, 104]]], dtype=uint16)\n",
"Coordinates:\n",
" * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 853.2 856.8 860.4 864.0 867.6\n",
" * y (y) float64 570.0 573.6 577.2 ... 1.79e+03 1.794e+03 1.798e+03\n",
" * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04"
]
},
"execution_count": 149,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"si"
]
},
{
"cell_type": "code",
"execution_count": 150,
"id": "a2498743-8bb9-49cf-ba3d-15577957cac0",
"metadata": {},
"outputs": [],
"source": [
"multiscale = to_multiscale(si, [])"
]
},
{
"cell_type": "code",
"execution_count": 154,
"id": "9d7ff6cd-cac7-497d-b4f4-74524aba4275",
"metadata": {},
"outputs": [],
"source": [
"multiscale.ds.attrs['multiscales'][0]['coordinateTransformations'] = [{ \"type\": \"translation\", \"translation\": translation }]"
]
},
{
"cell_type": "code",
"execution_count": 155,
"id": "3b8a8509-9cc7-4abe-b915-2206c3d412b1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'multiscales': [{'version': '0.4',\n",
" 'name': 'Calretinin',\n",
" 'axes': [{'name': 'z', 'type': 'space', 'unit': 'micrometers'},\n",
" {'name': 'y', 'type': 'space', 'unit': 'micrometers'},\n",
" {'name': 'x', 'type': 'space', 'unit': 'micrometers'}],\n",
" 'datasets': [{'path': '0/Calretinin',\n",
" 'coordinateTransformations': [{'type': 'scale',\n",
" 'scale': [3.6, 3.6000000000000227, 3.6]},\n",
" {'type': 'translation', 'translation': [0.0, 570.0, 0.0]}]}],\n",
" 'coordinateTransformations': [{'type': 'translation',\n",
" 'translation': {'z': 0.0, 'y': 570.0, 'x': 0.0}}]}]}"
]
},
"execution_count": 155,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"multiscale.ds.attrs"
]
},
{
"cell_type": "code",
"execution_count": 170,
"id": "cdcc9227-2f4b-452e-8add-2de1fd688f2d",
"metadata": {},
"outputs": [],
"source": [
"tile_configuration = { '0/2/0': si }"
]
},
{
"cell_type": "code",
"execution_count": 171,
"id": "4e714328-b00f-465b-89b4-a128f083818a",
"metadata": {},
"outputs": [],
"source": [
"dt = DataTree.from_dict(name='tile_configuration', data_objects=tile_configuration)"
]
},
{
"cell_type": "code",
"execution_count": 175,
"id": "1293419c-c6f3-449c-a8d0-d49baf7bd481",
"metadata": {},
"outputs": [],
"source": [
"multiscale.parent = dt['0/2/0']"
]
},
{
"cell_type": "code",
"execution_count": 176,
"id": "e0e8b217-b1a9-45f8-ab05-b90970e27d26",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"DataTree('tile_configuration', parent=None)\n",
"└── DataTree('0')\n",
" └── DataTree('2')\n",
" └── DataTree('0')\n",
" │ Dimensions: (z: 242, y: 342, x: 12491)\n",
" │ Coordinates:\n",
" │ * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 856.8 860.4 864.0 867.6\n",
" │ * y (y) float64 570.0 573.6 577.2 ... 1.79e+03 1.794e+03 1.798e+03\n",
" │ * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04\n",
" │ Data variables:\n",
" │ Calretinin (z, y, x) uint16 0 0 0 0 0 0 0 0 ... 104 101 103 103 103 104 104\n",
" └── DataTree('ngff')\n",
" │ Dimensions: ()\n",
" │ Data variables:\n",
" │ *empty*\n",
" │ Attributes:\n",
" │ multiscales: [{'version': '0.4', 'name': 'Calretinin', 'axes': [{'name':...\n",
" └── DataTree('0')\n",
" Dimensions: (z: 242, y: 342, x: 12491)\n",
" Coordinates:\n",
" * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 856.8 860.4 864.0 867.6\n",
" * y (y) float64 570.0 573.6 577.2 ... 1.79e+03 1.794e+03 1.798e+03\n",
" * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04\n",
" Data variables:\n",
" Calretinin (z, y, x) uint16 0 0 0 0 0 0 0 0 ... 104 101 103 103 103 104 104\n"
]
}
],
"source": [
"print(dt)"
]
},
{
"cell_type": "code",
"execution_count": 179,
"id": "c2015518-381b-4799-98f0-b7cb0e040d4d",
"metadata": {},
"outputs": [],
"source": [
"tile_configuration = {}\n",
"multiscales = []\n",
"for file_index in range(file_limit):\n",
" tiff_zstore = tifffile.imread(downloaded_files[file_index], aszarr=True)\n",
" tiff_zgroup = zarr.open(tiff_zstore)\n",
" \n",
" with dandifs.open(json_files[file_index]) as f:\n",
" sidecar = json.load(f)\n",
" pixel_size = sidecar['PixelSize']\n",
" scale = { 'z': pixel_size[2], 'y': pixel_size[1], 'x': pixel_size[0] }\n",
" matrix = sidecar['ChunkTransformMatrix']\n",
" translation = { 'z': matrix[2][2], 'y': matrix[1][3], 'x': matrix[0][3] }\n",
" dims = ('z', 'y', 'x')\n",
" # NGFF spec\n",
" units = 'micrometers'\n",
" \n",
" si = to_spatial_image(tiff_zgroup, dims=dims, name=stain, scale=scale, axis_units={ axis_name: units for axis_name in dims })\n",
" multiscale = to_multiscale(si, [])\n",
" multiscale.ds.attrs['multiscales'][0]['coordinateTransformations'] = [{ \"type\": \"translation\", \"translation\": translation }]\n",
" multiscales.append(multiscale)\n",
" \n",
" # Placeholder\n",
" tile_configuration[f'0/{file_index}/0'] = None\n",
" \n",
"dt = DataTree.from_dict(name='tile_configuration', data_objects=tile_configuration)\n",
"for file_index in range(file_limit):\n",
" dt[f'0/{file_index}/0'] = multiscales[file_index]"
]
},
{
"cell_type": "code",
"execution_count": 180,
"id": "9a97fccc-cbf9-440d-baec-41b360f08ef4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"DataTree('tile_configuration', parent=None)\n",
"└── DataTree('0')\n",
" ├── DataTree('0')\n",
" │ └── DataTree('0')\n",
" │ └── DataTree('ngff')\n",
" │ │ Dimensions: ()\n",
" │ │ Data variables:\n",
" │ │ *empty*\n",
" │ │ Attributes:\n",
" │ │ multiscales: [{'version': '0.4', 'name': 'Calretinin', 'axes': [{'name':...\n",
" │ └── DataTree('0')\n",
" │ Dimensions: (z: 242, y: 342, x: 12491)\n",
" │ Coordinates:\n",
" │ * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 856.8 860.4 864.0 867.6\n",
" │ * y (y) float64 0.0 3.6 7.2 10.8 ... 1.22e+03 1.224e+03 1.228e+03\n",
" │ * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04\n",
" │ Data variables:\n",
" │ Calretinin (z, y, x) uint16 0 0 0 0 0 0 0 0 ... 106 103 93 93 103 104 104\n",
" ├── DataTree('1')\n",
" │ └── DataTree('0')\n",
" │ └── DataTree('ngff')\n",
" │ │ Dimensions: ()\n",
" │ │ Data variables:\n",
" │ │ *empty*\n",
" │ │ Attributes:\n",
" │ │ multiscales: [{'version': '0.4', 'name': 'Calretinin', 'axes': [{'name':...\n",
" │ └── DataTree('0')\n",
" │ Dimensions: (z: 242, y: 342, x: 12491)\n",
" │ Coordinates:\n",
" │ * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 856.8 860.4 864.0 867.6\n",
" │ * y (y) float64 0.0 3.6 7.2 10.8 ... 1.22e+03 1.224e+03 1.228e+03\n",
" │ * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04\n",
" │ Data variables:\n",
" │ Calretinin (z, y, x) uint16 0 0 0 0 0 0 0 0 ... 103 103 104 104 100 107 107\n",
" └── DataTree('2')\n",
" └── DataTree('0')\n",
" └── DataTree('ngff')\n",
" │ Dimensions: ()\n",
" │ Data variables:\n",
" │ *empty*\n",
" │ Attributes:\n",
" │ multiscales: [{'version': '0.4', 'name': 'Calretinin', 'axes': [{'name':...\n",
" └── DataTree('0')\n",
" Dimensions: (z: 242, y: 342, x: 12491)\n",
" Coordinates:\n",
" * z (z) float64 0.0 3.6 7.2 10.8 14.4 ... 856.8 860.4 864.0 867.6\n",
" * y (y) float64 0.0 3.6 7.2 10.8 ... 1.22e+03 1.224e+03 1.228e+03\n",
" * x (x) float64 0.0 3.6 7.2 10.8 ... 4.496e+04 4.496e+04 4.496e+04\n",
" Data variables:\n",
" Calretinin (z, y, x) uint16 0 0 0 0 0 0 0 0 ... 104 101 103 103 103 104 104\n"
]
}
],
"source": [
"print(dt)"
]
},
{
"cell_type": "code",
"execution_count": 181,
"id": "cc5be8d8-03b6-40f2-b5d2-6ce7fe432d4d",
"metadata": {},
"outputs": [],
"source": [
"dt.to_zarr('./dt.zarr')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e89f03a7-1fa1-407b-ac8f-807af70316c8",
"metadata": {},
"outputs": [],
"source": [
"# Todo: replace consolidated metadata with tiffile fsspec entries for image chunks"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment