martindurant

url = "s3://prod-is-usgs-sb-prod-publish/609bf69ed34ea221ce39b261/breach_matanzas.nc"
from fsspec_reference_maker.hdf import *

so = dict(
    mode='rb', anon=True, requester_pays=True,
    default_fill_cache=False, default_cache_type='first'
    )
with fsspec.open(url, **so) as f:
    h5chunks = SingleHdf5ToZarr(f, url, xarray=True, inline_threshold=100)
    out = h5chunks.translate()

martindurant

cpdef void read_bitpacked8(NumpyIO file_obj, int count, NumpyIO o):
    # same as np.frombuffer(input, count, dtype=np.uint8)).astype(np.int32)
    cdef int i
    cdef int* outptr
    cdef char* inptr
    outptr = <int*>o.get_pointer()
    inptr = file_obj.get_pointer()
    for i in range(count):
        outptr[0] = <int>inptr[0] & 0xff
        outptr += 1

martindurant

2021-03-31 09:24:49,718 - fsspec.reference - DEBUG - _cat_file -- Reference: .zgroup, type str
2021-03-31 09:24:49,721 - fsspec.reference - DEBUG - _cat_file -- Reference: crs/.zarray, type str
2021-03-31 09:24:49,722 - fsspec.reference - DEBUG - _cat_file -- Reference: crs/.zarray, type str
2021-03-31 09:24:49,722 - fsspec.reference - DEBUG - _cat_file -- Reference: crs/.zattrs, type str
2021-03-31 09:24:49,723 - fsspec.reference - DEBUG - _cat_file -- Reference: elevation/.zarray, type str
2021-03-31 09:24:49,724 - fsspec.reference - DEBUG - _cat_file -- Reference: elevation/.zarray, type str
2021-03-31 09:24:49,725 - fsspec.reference - DEBUG - _cat_file -- Reference: elevation/.zattrs, type str
2021-03-31 09:24:49,726 - fsspec.reference - DEBUG - _cat_file -- Reference: feature_id/.zarray, type str
2021-03-31 09:24:49,726 - fsspec.reference - DEBUG - _cat_file -- Reference: feature_id/.zarray, type str
2021-03-31 09:24:49,727 - fsspec.reference - DEBUG - _cat_file -- Reference: feature_id/.zattrs, type str

martindurant

import mmap
import tempfile
import weakref
import uuid

import SharedArray as sa
import numpy as np
"""https://gitlab.com/tenzing/shared-array"""

martindurant

martindurant

from intake.catalog import Catalog
from intake.catalog.local import LocalCatalogEntry
import pandas as pd


class PangeoCSVSource(Catalog):
    _grid = [None]
    container = 'catalog'
    name = 'pangeo_csv'
    version = '0.0.1'

martindurant

import inspect
import panel
import hvplot.pandas
from hvplot import hvPlot
from hvplot.converter import HoloViewsConverter


plot_types = list(_ for _ in dir(hvPlot) if not _.startswith('_'))
plots_needs = {
    k: set(inspect.signature(getattr(hvPlot, k)).parameters) - {'self', 'kwds'}

martindurant

martindurant

sources:
  glob_source:
    description: glob of files
    driver: csv
    cache:
      - argkey: urlpath
        regex: 's3://bucket'
        sub: {{ CACHE_DIR }}
    args:

martindurant

import bokeh
from bokeh.io import output_notebook, push_notebook
output_notebook()

##

import numpy as np
from ipywidgets import interact, widgets

from bokeh.plotting import figure, show