Script to download cellmap ground truth data

download_cellmap.py

import os
import numpy as np
from fibsem_tools import io
from skimage.io import imsave
from scipy.ndimage import zoom

datasets = [
	'jrc_hela-2',
	'jrc_hela-3',
	'jrc_jurkat-1',
	'jrc_macrophage-2',
	'jrc_sum159-1'
]

dataset_raw_urls = [
	"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/raw/s0/",
	"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/raw/s0/",
	"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/raw/s0/",
	"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/raw/s0/",
	"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/raw/s0/"
]

dataset_gt_urls = [
	[
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop1/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop113/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop13/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop14/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop15/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop16/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop18/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop19/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop23/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop28/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop3/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop4/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop54/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop55/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop56/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop57/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop58/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop59/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop6/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop7/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop8/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop9/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop94/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-2/jrc_hela-2.n5/volumes/groundtruth/0003/crop95/labels/all/",
	],
	[
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop111/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop27/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop33/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop34/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop50/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop60/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop61/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop62/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop63/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop64/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop65/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop85/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop86/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_hela-3/jrc_hela-3.n5/volumes/groundtruth/0003/crop87/labels/all/",
	],
	[
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop112/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop37/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop43/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop47/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop66/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop67/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop68/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop69/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop70/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop71/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop91/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop92/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_jurkat-1/jrc_jurkat-1.n5/volumes/groundtruth/0003/crop93/labels/all/",
	],
	[
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop110/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop31/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop42/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop49/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop72/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop73/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop74/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop75/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop76/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop77/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop88/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop89/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_macrophage-2/jrc_macrophage-2.n5/volumes/groundtruth/0003/crop90/labels/all/",
	],
	[
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop20/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop21/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop22/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop25/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop26/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop81/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop82/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop83/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop84/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop97/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop98/labels/all/",
		"s3://janelia-cosem-publications/heinrich-2021a/jrc_sum159-1/jrc_sum159-1.n5/volumes/groundtruth/0003/crop99/labels/all/",
	]
]

if __name__ == '__main__':
    for name, raw_url, gt_urls in zip(datasets, dataset_raw_urls, dataset_gt_urls):
        # make directory with dataset name and image and mask
        # subdirectories to save the results
        os.makedirs(name, exist_ok=True)
        os.makedirs(f'{name}/images/', exist_ok=True)
        os.makedirs(f'{name}/masks/', exist_ok=True)

        # load the raw volume
        raw = io.read_xarray(raw_url, storage_options={'anon': True})
        print('Loaded raw of', raw.shape)

        for i,gt_url in enumerate(gt_urls):
            # load the gt crop
            gt = io.read_xarray(gt_url, storage_options={'anon': True})
            offset = gt.offset
            print(f'Loaded gt {i+1} of {len(gt_urls)} shape {gt.shape} with offset {offset}')

            # convert from offset to indices
            start_indices = [int(i // 4) for i in offset]
            start_indices = [start_indices[2], start_indices[1], start_indices[0]]
            end_indices = [int(i + s // 2) for i,s in zip(start_indices, gt.shape)]

            # move from x, y, z to z, y, x
            zs, ys, xs = start_indices
            ze, ye, xe = end_indices
            print(f'Slicing from {tuple([zs, ys, xs])}, {tuple([ze, ye, xe])}')

            raw_crop = raw[zs:ze, ys:ye, xs:xe]

            # resize the gt
            gt = zoom(gt, (0.5, 0.5, 0.5), order=0).astype(np.uint8)
            print(f'Raw of {raw_crop.shape} and GT of {gt.shape} and {gt.dtype}')

            crop_name = gt_url.split('/')[-4]

            imsave(f'{name}/images/{crop_name}.tif', raw_crop, check_contrast=False)
            imsave(f'{name}/masks/{crop_name}.tif', gt, check_contrast=False)

Downloads raw data images and labeled ground truth from https://open.quiltdata.com/b/janelia-cosem-publications/tree/heinrich-2021a/.

The only "exotic" dependency is fibsem_tools which can be installed from here: https://github.com/janelia-cosem/fibsem-tools

If you don't want to run the script, here's the zipped data from June 14th, 2022: https://drive.google.com/file/d/1zwWZnAJw-lCdWEi8D8ZF1Tzv1Q3JmXZQ/view?usp=sharing