Skip to content

Instantly share code, notes, and snippets.

View clbarnes's full-sized avatar

Chris Barnes clbarnes

View GitHub Profile
@clbarnes
clbarnes / rate_limited_httpx.py
Created July 8, 2023 01:52
httpx.AsyncClient subclass with semaphore-based rate limiting
import asyncio
import datetime as dt
from functools import wraps
from typing import Union
from httpx import AsyncClient
# unless you keep a strong reference to a running task, it can be dropped during execution
# https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
_background_tasks = set()
@clbarnes
clbarnes / nx_node_link_data_update.py
Created June 1, 2023 11:15
Update networkx v1 node_link_data to v2+
@clbarnes
clbarnes / nx_coalesce_nodes.py
Last active June 1, 2023 11:08
Group nodes together in a networkx.DiGraph
#!/usr/bin/env python3
"""
Group nodes together in a networkx DiGraph.
Requires networkx.
"""
from typing import Hashable, Any, Callable, Optional
import networkx as nx
@clbarnes
clbarnes / dask_open.py
Created February 3, 2023 10:44
Improve ergonomics for creating a dask array from a stack or chunking of files
#!/usr/bin/env python3
"""Load a list of images in dask."""
from pathlib import Path
import re
from typing import Callable, Iterable, Optional
import dask.array as da
from dask import delayed
import numpy as np
@clbarnes
clbarnes / copy_cache.py
Created January 18, 2023 16:53
functools.lru_cache wrapper which copies cached return values
from functools import lru_cache, wraps
from typing import Optional, Callable
from copy import copy, deepcopy
def copy_cache(deep: bool = True, maxsize: Optional[int] = 128, typed: bool = False):
"""Decorator factory wrapping functools.lru_cache, which copies return values.
Use this for functions with mutable return values.
@clbarnes
clbarnes / dfbuilder.py
Created October 10, 2022 14:26
Build dataframes row-wise in an ergonomic and reasonably efficient way
#!/usr/bin/env python3
"""Module for building pandas DataFrames by row."""
from collections.abc import Sequence, Collection
import typing as tp
import pandas as pd
class DataFrameBuilder:
def __init__(
@clbarnes
clbarnes / les.sh
Created May 17, 2022 10:21
les: ls if it's a directory, less if it's a file
#!/bin/sh
set -e
if [ $# -eq 0 ]; then
>&2 echo "No argument given, using ls"
exec ls
elif [ -d "$1" ]; then
>&2 echo "Directory detected, using ls"
exec ls "$1"
elif [ -f "$1" ]; then
>&2 echo "File detected, using less"
@clbarnes
clbarnes / makepool.py
Last active January 3, 2024 11:06
Make a zpool command for creating large pools
#!/usr/bin/env python3
"""
Make a `zpool create ...` command for a set of identical disks,
calculating vdev size etc. from the given configuration.
You will need to know the disk model name;
you should be able to identify it from the output of
`lsblk -d -e 7 -o NAME,SIZE,STATE,MODEL`.
"""
import getpass
from argparse import ArgumentParser
@clbarnes
clbarnes / neuron_n5_copy.py
Created February 7, 2022 13:53
Copy a minimum number of N5 chunks to visualise a navis neuron of interest
#%%
import os
import shutil
from pathlib import Path
import navis
import numpy as np
from scipy.ndimage import binary_fill_holes
# in nm, for converting world coords to voxel indices
resolution = np.array([3.8, 3.8, 50])
@clbarnes
clbarnes / add_downscales.py
Last active May 17, 2022 10:53
DEPRECATED, use https://github.com/clbarnes/bdv_meta/blob/main/add_downsamples.py | Script for (somewhat safely) adding metadata required by BigDataViewer (and optionally n5-viewer) to multiscale datasets stored in N5 containers.
#!/usr/bin/env python
"""
Script for (somewhat safely) adding metadata required by BigDataViewer
(and optionally n5-viewer) to multiscale datasets stored in N5 containers.
"""
from dataclasses import dataclass
import json
from pathlib import Path
from argparse import ArgumentParser
import re