Skip to content

Instantly share code, notes, and snippets.

View mehd-io's full-sized avatar
🎥
Coding as much as recording

Mehdi OUAZZA mehd-io

🎥
Coding as much as recording
View GitHub Profile
@mehd-io
mehd-io / ducky_catalog.sql
Created November 22, 2024 12:53
Duckdb as portable catalog
-- ataching the remote database (268KB)
ATTACH 's3://us-prd-motherduck-open-datasets/content/duckdb-as-catalog/ducky_catalog.ddb';
SHOW all tables;
-- queries the data
FROM ducky_catalog.customers limit 5;
-- Listing views
@mehd-io
mehd-io / hello_macro.sql
Created November 19, 2024 07:24
hello_macro.sql
CREATE MACRO add_one(value) AS (value + 1);
@mehd-io
mehd-io / timewrapper.py
Created August 22, 2024 12:31
Measure time wrapper
import logging
from datetime import datetime
import functools
# Setting up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def measure_time(func):
@functools.wraps(func)
@mehd-io
mehd-io / setup_duckdb.sh
Last active March 13, 2024 13:35
Script to install duckdb (Linux/Macos)
#!/usr/bin/env bash
set -e
CDN="https://github.com/duckdb/duckdb/releases/download"
INSTALL_DIR="$HOME/.local/bin"
# Function to initialize platform specifics
initPlatform() {
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
ARCH=$(uname -m)
@mehd-io
mehd-io / convert_file.sh
Created March 11, 2024 10:53
Convert csv<->parquet
#!/bin/bash
# A simple script for converting files between CSV and Parquet formats using DuckDB. Requires DuckDB installation.
convert_file() {
local input_file="$1"
local output_extension="$2"
# Extracting the filename without extension
local base_name=$(basename -- "$input_file")
local name="${base_name%.*}"
@mehd-io
mehd-io / duckdb_finance_data_example.sql
Created April 5, 2023 12:04
Query Yahoo Finance Data w/ DuckDB
-- Install httpfs extension
INSTALL httpfs;
LOAD httpfs;
-- Install Scrooge extension https://github.com/pdet/Scrooge-McDuck
-- NOTE : You need to start DuckDB with `-unsigned` flag to authorized to install & load 3rd party extension
SET custom_extension_repository='scrooge-duckdb.s3.us-west-2.amazonaws.com/scrooge/s3_deploy';
INSTALL scrooge;
LOAD scrooge;
-- Example of query
FROM yahoo_finance("^GSPC", "2023-02-01", "2023-02-04", "1d");
@mehd-io
mehd-io / csv_to_parquet.sh
Created March 21, 2023 16:23
Convert CSV to Parquet using DuckDB CLI
#!/bin/bash
# You can put this in your .bashrc or .zshrc
function csv_to_parquet() {
file_path="$1"
duckdb -c "COPY (SELECT * FROM read_csv_auto('$file_path')) TO '${file_path%.*}.parquet' (FORMAT PARQUET);"
}
@mehd-io
mehd-io / conftest.py
Last active May 24, 2022 18:32
medium-tftest-api-conftest
from pathlib import Path
import pytest
import tftest
from google.auth.transport.requests import AuthorizedSession
from .cloud_run_client import (
get_auth_session,
get_service_account_file_path,
get_token_credentials_from_service_account,
@mehd-io
mehd-io / test_hello.py
Created May 24, 2022 18:25
medium-tftest-api-hello
import json
import pytest
from google.auth.transport.requests import AuthorizedSession
from .cloud_run_client import request_wrapper
@pytest.mark.integration
def test_health_check(auth_session: AuthorizedSession):
@mehd-io
mehd-io / test_cloud_run.py
Created May 24, 2022 18:24
medium-tftest-infra
import pytest
import tftest
from pathlib import Path
@pytest.fixture
def plan():
file_path = Path(__file__).resolve()
base_dir = file_path.parent.parent.parent.absolute()
tf = tftest.TerraformTest(tfdir="terraform", basedir=base_dir)