martindurant

martindurant

martindurant

martindurant

from dask.layers import DataFrameIOLayer
import pandas as pd


def find_columns(df):
    io_layers = {k: lay for k, lay in df.dask.layers.items() if isinstance(lay, DataFrameIOLayer)}
    required = {k: set() for k in io_layers}

    for k, io_lay in io_layers.items():
        allcols = io_lay.collection_annotations["series_dtypes"]

martindurant

import astropy.io.fits._tiled_compression as tiled
from astropy.io import fits
import numcodecs
import fsspec
import zarr
import json
import numpy as np


class MY_RICE(tiled.codecs.Rice1):

martindurant

import kerchunk.hdf
import kerchunk.combine
import fsspec
import tarfile
import xarray as xr


ro = dict(
    s3={
        "anon": True,

martindurant

martindurant

import kerchunk.hdf
import kerchunk.combine
import fsspec
import tarfile
import xarray as xr


ro = dict(
    s3={
        "anon": True,

martindurant

name: pangeo-forge-recipes
channels:
  - conda-forge
  - defaults
dependencies:
  - _libgcc_mutex=0.1=conda_forge
  - _openmp_mutex=4.5=1_gnu
  - affine=2.3.0=py_0
  - aiohttp=3.7.4=py38h497a2fe_0
  - appdirs=1.4.4=pyh9f0ad1d_0

martindurant

url = "s3://prod-is-usgs-sb-prod-publish/609bf69ed34ea221ce39b261/breach_matanzas.nc"
from fsspec_reference_maker.hdf import *

so = dict(
    mode='rb', anon=True, requester_pays=True,
    default_fill_cache=False, default_cache_type='first'
    )
with fsspec.open(url, **so) as f:
    h5chunks = SingleHdf5ToZarr(f, url, xarray=True, inline_threshold=100)
    out = h5chunks.translate()