Michael Sumner mdsumner

Here's some examples of using the new NASA experimental json Zarr

with xarray

import xarray 
## no
#import earthaccess
#earthaccess.login()  ## use env vars ??
#fs = earthaccess.get_s3_filesystem(daac="podaac")

longstrings <- c("analysis.\\\",\\\"title\\\":\\\"RSS CCMP V3.1 6-hourly surface winds (Level 4)\\\",\\\"coordinates\\\":\\\"latitude longitude t", 
                 "dsaoijdoijdd09d9de9e9e9e00\"RSS CCMPbalsljdojasdoiewwiow8us98hasdhasd")

findregions <- function(x, pattern, window = 10) {
  index <- gregexpr(pattern, x)
  outlist <- vector("list", length(index))
  for (i in seq_along(index)) {
    outlist[[i]] <- character(length(index[[i]]))
 for (j in seq_along(index[[i]])) {

https://bsky.app/profile/rsimmon.bsky.social/post/3lo57wojyns2m

following from https://developers.satellogic.com/imagery-products/data_samples.html#aws-cli

aws s3 ls s3://satellogic-sample-data/Baotou/0392e349-af81-4b9f-90ca-283070dc4dbb/0392e349-af81-4b9f-90ca-283070dc4dbb_L1C/ --no-sign-request

## Read Nuyina underway (1-minute interval data collection from the ocean)
get_underway <- function(x) {
  ## read the bulk
  d <- arrow::read_parquet("https://github.com/mdsumner/nuyina.underway/raw/main/data-raw/nuyina_underway.parquet")
  ## read the rest
  d1 <- tibble::as_tibble(vapour::vapour_read_fields("WFS:https://data.aad.gov.au/geoserver/ows?service=wfs&version=2.0.0&request=GetCapabilities",
                                                     sql = sprintf("SELECT * FROM \"underway:nuyina_underway\" WHERE datetime > '%s'", 
                                                                   format(max(d$datetime, "%Y-%m-%dT%H:%M:%SZ")))))

library(reticulate)
p <- c("polars", "h5netcdf", "fsspec", "aiohttp", "requests", "xarray", "dask")
py_require(p)
polars <- import("polars")
xarray <- import("xarray")

pp <-  arrow::read_parquet("https://projects.pawsey.org.au/idea-objects/idea-curated-objects.parquet")

## in R before
# p <- c("polars", "h5netcdf", "fsspec", "aiohttp", "requests", "xarray")
# reticulate::py_require(p)

import polars
pp = polars.read_parquet("https://projects.pawsey.org.au/idea-objects/idea-curated-objects.parquet")
d = pp.filter(polars.col("Dataset") == "oisst-avhrr-v02r01")
d = d.with_columns(source = polars.format("https://projects.pawsey.org.au/{}/{}", polars.col("Bucket"), polars.col("Key")))

dsn <- "https://raw.githubusercontent.com/OSGeo/gdal/refs/heads/master/frmts/wms/frmt_wms_bluemarble_s3_tms.xml"
library(terra)
r <- rast(dsn)



## first I did
m <- do.call(cbind, maps::map(plot = F)[1:2])
plot(project(m, to = "+proj=isea", from = "EPSG:4326"), pch = ".", asp = 1)

files <- readr::read_csv("https://ogd.swisstopo.admin.ch/resources/ch.swisstopo.swissalti3d-fOYMuina.csv", col_names = FALSE)

head(files$X1)
[1] "https://data.geo.admin.ch/ch.swisstopo.swissalti3d/swissalti3d_2019_2501-1120/swissalti3d_2019_2501-1120_0.5_2056_5728.tif"
[2] "https://data.geo.admin.ch/ch.swisstopo.swissalti3d/swissalti3d_2019_2501-1121/swissalti3d_2019_2501-1121_0.5_2056_5728.tif"
[3] "https://data.geo.admin.ch/ch.swisstopo.swissalti3d/swissalti3d_2019_2501-1122/swissalti3d_2019_2501-1122_0.5_2056_5728.tif"
[4] "https://data.geo.admin.ch/ch.swisstopo.swissalti3d/swissalti3d_2019_2502-1120/swissalti3d_2019_2502-1120_0.5_2056_5728.tif"

## a manifest array in a virtual Zarr store contains references like this (for this file it is >90000 references
##  because 28x51*1500x3600 in 1x1x300x300 blocks)

#'1.39.0.7': {'path': 'https://thredds.nci.org.au/thredds/fileServer/gb6/BRAN/BRAN2020/daily/ocean_salt_1993_01.nc', 
#  'offset': 302129687, 'length': 48186},

	"source"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20000101_20001231_epsg3031.tif"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20010101_20011231_epsg3031.tif"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20020101_20021231_epsg3031.tif"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20030101_20031231_epsg3031.tif"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20040101_20041231_epsg3031.tif"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20050101_20051231_epsg3031.tif"
	"https://projects.pawsey.org.au/ant-11-era5-evaluation/ps_ANT-11_ERA5_evaluation_r1i1p1f1_BAS_MetUM_v1-r1_day_20060101_20061231_epsg3031.tif"
	"https://projects.paw