Skip to content

Instantly share code, notes, and snippets.

@qpwo
qpwo / fastdl.sh
Last active January 26, 2025 23:56
fust download with r2 rclone and huggingface parallel
curl https://rclone.org/install.sh | sudo bash
mkdir ~/.config/rclone
echo "
[r2]
type = s3
provider = Cloudflare
access_key_id = $a
secret_access_key = $b
region = auto
endpoint = https://$c.r2.cloudflarestorage.com
@qpwo
qpwo / o1-fail.txt
Created January 22, 2025 02:35
o1 fail
Why does this run instantly after first trial?
import subprocess
import time
import shutil
import os
def run_rclone_command(command, num_trials=3, dest_path_base="/home/ubuntu/hff/405b-base"):
"""
# modal run issue.py
import os
import subprocess
import time
import modal
import modal.experimental
import modal.gpu
import modal.volume
@qpwo
qpwo / fast-huggingface-parallel-download.sh
Last active January 12, 2025 12:36
fastest huggingface parallel download ever
pip install huggingface_hub[hf_transfer]
export HF_HUB_ENABLE_HF_TRANSFER=1
model_name=meta-llama/Llama-3.1-405B
localdir=$(realpath ~/hff/405b)
huggingface-cli download --max-workers=8 --include="model-???[02468][02468]-of-?????.safetensors" --local-dir=$localdir $model_name & sleep 6
huggingface-cli download --max-workers=8 --include="model-???[02468][13579]-of-?????.safetensors" --local-dir=$localdir $model_name & sleep 6
huggingface-cli download --max-workers=8 --include="model-???[13579][02468]-of-?????.safetensors" --local-dir=$localdir $model_name & sleep 6
huggingface-cli download --max-workers=8 --include="model-???[13579][13579]-of-?????.safetensors" --local-dir=$localdir $model_name & sleep 0
@qpwo
qpwo / clippy.py
Created January 3, 2025 06:04
unquittable unminimizable clippy
import sys
from PyQt5.QtWidgets import QApplication, QVBoxLayout, QWidget, QLabel
from PyQt5.QtGui import QPixmap, QKeyEvent
from PyQt5.QtCore import QTimer, Qt, QPoint
import time
from datetime import datetime
def yyyy_mm_dd_hh_mm_ss():
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@qpwo
qpwo / pickle-append.py
Last active November 30, 2024 18:59
python pickle/pytorch append!
import os, pickle, time, numpy as np
def naive_append(filename, obj):
objs = []
if os.path.exists(filename):
with open(filename, 'rb') as f:
objs = pickle.load(f)
objs.append(obj)
with open(filename, 'wb') as f:
pickle.dump(objs, f)
@qpwo
qpwo / header-ag.py
Last active November 25, 2024 19:07
python header ag -- search for expression and get nearest headers
"""
Usage:
python my-ag.py <regex> <...paths>
Requirements/Features of my-ag.py:
* Takes regex pattern and file/directory paths as command-line arguments
* Recursively searches through Python files in given paths
* When finding a regex match, shows:
- All parent unindented lines ("headers")
@qpwo
qpwo / popen-selectors-repl.py
Created November 8, 2024 17:12
repl in python with popen and selectors
# unfortunately, the stdinready event fires all the time, not just when repl ready.
# Based on https://gist.github.com/andy0130tw/39472331530d1a0e25459a547ed2c9d5
import io
import selectors
from selectors import EVENT_READ, EVENT_WRITE
import subprocess
from typing import cast
sel = selectors.DefaultSelector()
@qpwo
qpwo / mullvad-ssh.sh
Created October 23, 2024 21:55
mullvad ssh setup
sudo apt install -y nftables
sudo systemctl restart nftables.service
# https://mullvad.net/en/help/split-tunneling-with-linux-advanced
echo '
table inet excludeTraffic {
chain allowIncoming {
type filter hook input priority -100; policy accept;
@qpwo
qpwo / myvagrant.py
Created October 3, 2024 00:59
vagrant wrapper thing to make it a one-off create/destroy thingy
import os
from pathlib import Path
import json
import shutil
# 192.168.122.#{i+150}
template = """
Vagrant.configure("2") do |config|
config.vm.box = "debian/bookworm64"
# provider config