ianmcook

import tempfile
import pathlib
import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.parquet as pq
import pyarrow.dataset as ds


# create a small dataset for example purposes

ianmcook

#include <iostream>
#include <arrow/api.h>
#include <arrow/result.h>
#include <arrow/compute/api.h>
#include <arrow/compute/exec/exec_plan.h>

arrow::Status ExecutePlanAndCollectAsTable(
  std::shared_ptr<arrow::compute::ExecPlan> plan,
  std::shared_ptr<arrow::Schema> schema,
  arrow::AsyncGenerator<std::optional<arrow::compute::ExecBatch>> sink_gen) {

ianmcook

import google.auth
import ibis
from ibis import _

credentials, billing_project = google.auth.default()

conn = ibis.bigquery.connect(billing_project, 'bigquery-public-data.samples')
t = conn.table('github_nested')

expr = (

ianmcook

# before running:
# 1. install Ibis and its Snowflake backend: https://ibis-project.org/backends/Snowflake/
# 2. create and activate a Snowflake trial account
# 3. set environment variables SNOWSQL_USER, SNOWSQL_PWD, SNOWSQL_ACCOUNT

import os
import ibis
from ibis import _

ibis.options.interactive = True

ianmcook

# before running:
# 1. install Ibis and its Trino backend: https://ibis-project.org/backends/Trino/
# 2. pull and run the Trino docker container: https://trino.io/docs/current/installation/containers.html

import ibis
from ibis import _

# connect to Trino
conn = ibis.trino.connect(database='memory', schema='default')

ianmcook

# pip install 'ibis-framework[duckdb]'

import pandas as pd
import ibis
from ibis import _

# create a pandas DataFrame and write it to a Parquet file
df = pd.DataFrame(data={'repo': ['pandas', 'duckdb', 'ibis'],
                        'stars': [36622, 8074, 2336]})
df.to_parquet('repo_stars.parquet')

ianmcook

# run this script second

library(dplyr)

df <- read.csv("dirty.csv")

agg <- df %>%
  group_by(jira, github) %>%
  summarise(n = n(), .groups = "keep") %>%
  ungroup() %>%

ianmcook

#include <iostream>
#include <arrow/api.h>
#include <arrow/result.h>
#include <arrow/compute/api.h>
#include <arrow/compute/exec/exec_plan.h>

arrow::Status ExecutePlanAndCollectAsTable(
  std::shared_ptr<arrow::compute::ExecPlan> plan,
  std::shared_ptr<arrow::Schema> schema,
  arrow::AsyncGenerator<std::optional<arrow::compute::ExecBatch>> sink_gen) {

ianmcook

#include <iostream>
#include <arrow/api.h>
#include <arrow/result.h>
#include <arrow/compute/api.h>

arrow::Status Execute() {
  arrow::Int32Builder int_builder;
  ARROW_RETURN_NOT_OK(int_builder.Append(1));
  ARROW_RETURN_NOT_OK(int_builder.Append(2));
  ARROW_RETURN_NOT_OK(int_builder.Append(3));

ianmcook

# enquo() helpers for eager evaluation and idempotence

# wrap eager() around enquo() to evaluate the quosure immediately in the calling
# environment *if* it can do so without error, otherwise return the quosure
eager <- function(quo) {
  val <- try(eval_tidy(quo), silent = TRUE)
  if (inherits(val, "try-error")) {
    quo
  } else {
    val