This is a rough guide to using the OSN object storage service with Zarr data.
Open Storage Network (OSN) is a distributed data service to support active data sharing and transfer between academic institutions, leveraging existing NSF-funded resources. It is funded by NSF and the Schmidt Futures Foundation.
| import xarray as xr | |
| def detrend_dim(da, dim, deg=1): | |
| # detrend along a single dimension | |
| p = da.polyfit(dim=dim, deg=deg) | |
| fit = xr.polyval(dim, p.polyfit_coefficients) | |
| return da - fit | |
| def detrend(da, dims, deg=1): | |
| # detrend along multiple dimensions |
| r = rechunk(array, target_chunks, max_mem, | |
| store_target, temp_store=store_tmp, executor='prefect') | |
| flow = r.plan | |
| import datetime | |
| for task in flow.tasks: | |
| task.max_retries = 10 | |
| task.retry_delay = datetime.timedelta(seconds=5) | |
| from prefect.engine.executors import DaskExecutor |
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> |
| #!/bin/bash | |
| PERM="gke-2a8e5e49b2ab578131da:google-sudoers" | |
| NEWUID=$1 | |
| ORCID=$2 | |
| ARCHIVE=$3 | |
| SRC="mnt/nfs/home/ocean.pangeo.io/$ORCID" | |
| TAR="/mnt/nfs/uscentral1b/$NEWUID" |