This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import duckdb | |
import time | |
def gen_tpch(): | |
con = duckdb.connect() | |
con.execute("CALL dbgen(sf=20);") | |
con.execute("COPY lineitem to 'lineitem.csv'") | |
con.execute("COPY lineitem to 'lineitem.parquet'") | |
con.execute("COPY lineitem TO 'lineitem_zstd.parquet' (FORMAT 'parquet', CODEC 'zstd', COMPRESSION_LEVEL 1);") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import threading | |
import psutil | |
import resource | |
import duckdb | |
import pyarrow as pa | |
import os | |
import time | |
def create_db(): | |
con = duckdb.connect("duck.db") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import duckdb | |
import pyarrow as pa | |
import pandas as pd | |
import time | |
def time_function(function): | |
res = [] | |
for i in range (0,5): | |
start_time = time.monotonic() | |
ans = function() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import duckdb | |
import pyarrow as pa | |
import pandas as pd | |
import time | |
import pyarrow.compute as pc | |
def time_function(function): | |
res = [] | |
for i in range (0,5): | |
start_time = time.monotonic() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Files used in the example: | |
# !wget "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2016-01.parquet" | |
# !wget "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2016-02.parquet" | |
import duckdb | |
from duckdb.typing import * | |
import torch | |
import torch.nn as nn | |
import pyarrow as pa |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torch | |
import torch.nn as nn | |
import duckdb | |
import pyarrow as pa | |
import matplotlib.pyplot as plt | |
class LinearRegression(nn.Module): | |
def __init__(self, input_dim, output_dim): | |
super(LinearRegression, self).__init__() | |
self.linear = nn.Linear(input_dim, output_dim) |