Skip to content

Instantly share code, notes, and snippets.

Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@cboettig
cboettig / polygon-to-h3.R
Created February 4, 2025 23:28
polygon-to-h3
# get the h0 hexes covering a polygon.
library(duckdbfs)
library(dplyr)
library(sf)
library(spData)
duckdbfs::load_h3()
duckdbfs::load_spatial()
@cboettig
cboettig / pmtiles-filter.R
Created January 5, 2025 00:13
Dynamically filter pmtiles and plot with maplibre in R
library(mapgl)
library(dplyr)
library(duckdbfs)
# This test file demonstrates that given a SQL query on the parquet version
# of the data, we can identify all the features included in the response and
# construct a maplibre filter that filters the corresponding features in
# the pmtiles vesion.
pmtiles <- "https://data.source.coop/cboettig/social-vulnerability/svi2020_us_tract.pmtiles"
@cboettig
cboettig / ibis-duckdb-udfs.py
Created December 11, 2024 03:39
ibis pyarrow UDFs
import ibis
from ibis import _
import leafmap.maplibregl as leafmap
con = ibis.duckdb.connect(extensions=["spatial"])
states = con.read_geo("https://raw.githubusercontent.com/PublicaMundi/MappingAPI/master/data/geojson/us-states.json")
## Add a fill-color column
import matplotlib.cm as cm
import matplotlib.colors as mcolors
@cboettig
cboettig / h3_filter.py
Created November 26, 2024 21:30
filter data using h3 indices
park = (con
.read_geo("/vsicurl/https://huggingface.co/datasets/cboettig/biodiversity/resolve/main/data/NPS.gdb")
.filter(_.UNIT_NAME == "Yosemite National Park")
.select(_.SHAPE)
.mutate(SHAPE = _.SHAPE.convert('EPSG:3857', 'EPSG:4326'))
.mutate(lng = _.SHAPE.centroid().x(), lat = _.SHAPE.centroid().y())
.mutate(h1 = h3_latlng_to_cell(_.lat, _.lng, 1) )
.mutate(neighbors = h3_grid_disk(_.h1, 1))
#.mutate(cells = h3_polygon_wkt_to_cells_string(_.SHAPE.as_text(),1))
)
@cboettig
cboettig / llm-structured-response.py
Created November 23, 2024 05:04
langchain structured sql reply
import streamlit as st
import os
#os.environ["OPENAI_API_KEY"] = st.secrets['OPENAI_API_KEY'] # for gpt
os.environ["OPENAI_API_KEY"] = st.secrets['LITELLM_KEY'] # for litellm
from pydantic import BaseModel, Field
class SQLResponse(BaseModel):
"""Defines the structure for SQL response."""
@cboettig
cboettig / ibis-langchain-duckdb.py
Last active November 21, 2024 22:42
langchain + ibis share a connection
parquet = "https://espm-157-f24.github.io/spatial-carl-amanda-tyler/new_haven_stats.parquet"
# create sharable low-level connection, see: https://github.com/Mause/duckdb_engine
import sqlalchemy
eng = sqlalchemy.create_engine("duckdb:///:memory:")
# ibis can talk to this connection and create the VIEW
import ibis
con = ibis.duckdb.from_connection(eng.raw_connection())
@cboettig
cboettig / duckdb-s3.py
Created November 9, 2024 00:45
NRP use of S3 + duckdb
%%time
import ibis
from ibis import _
con = ibis.duckdb.connect()
endpoint="rook-ceph-rgw-nautiluss3.rook"
query= f'''
CREATE OR REPLACE SECRET secret2 (
@cboettig
cboettig / ibis_to_json.py
Created November 7, 2024 20:20
ibis / raw sql workarounds
import ibis
from ibis import _
con = ibis.duckdb.connect()
# example query, could be anything
subset = (gbif
.filter(_["class"] == "Aves")
.rename(hex = "h8")
.group_by(_.hex)
.agg(n = _.count())
@cboettig
cboettig / geoarrow-error.md
Created November 4, 2024 16:36
ibis geo error

To solve

ImportError: cannot import name 'types' from 'geoarrow' (unknown location), 

install geoarrow-types.

ibis may also need geoarrow-cto work with geoparquet now.