Very common stuff that I forget all the time and I really shouldn't

common.md

DO this to some servers to make ssh connection over IDEs

sudo bash -c 'cp /etc/ssh/sshd_config /etc/ssh/sshd_config.backup && sed -i "/AllowTcpForwarding/c\AllowTcpForwarding yes" /etc/ssh/sshd_config && sed -i "/GatewayPorts/c\GatewayPorts yes" /etc/ssh/sshd_config && sed -i "/PermitTunnel/c\PermitTunnel yes" /etc/ssh/sshd_config' && sudo systemctl restart sshd

if that dont work,

sudo bash -c '
  cp /etc/ssh/sshd_config /etc/ssh/sshd_config.backup &&
  sed -i "/^#\?AllowTcpForwarding/c\AllowTcpForwarding yes" /etc/ssh/sshd_config &&
  sed -i "/^#\?GatewayPorts/c\GatewayPorts yes" /etc/ssh/sshd_config &&
  sed -i "/^#\?PermitTunnel/c\PermitTunnel yes" /etc/ssh/sshd_config
'

# … then restart (or reload) the correct service
sudo systemctl restart ssh 

clean clean

black .
autoflake --remove-all-unused-imports -i ./*
isort .

Kill cuda shit

lsof /dev/nvidia* | awk '{print $2}' | xargs -I {} kill {}

Check basic stuff

du -h | awk '$1 ~ /G/ && $1 > 1' | sort -h : over 1gb of current disk

Local git setup

git config --global init.defaultBranch main
git config --global user.name "Simo Ryu"
git config --global user.email "[email protected]"

common initial machine setups

Install python3.9, venv, and make venv

#!/bin/bash
cd

# Prevent interactive prompts
export DEBIAN_FRONTEND=noninteractive

# Update package lists and add repository without prompts
sudo apt update
sudo apt install -y software-properties-common
sudo add-apt-repository -y ppa:deadsnakes/ppa

# Install all required packages without prompts
sudo apt install -y \
    pdsh \
    tmux \
    htop \
    git-lfs \
    ffmpeg \
    python3.11 \
    python3.11-venv \
    python3.11-distutils \
    libpython3.11-dev \
    python3.11-dev

# Setup git-lfs
sudo git lfs install --skip-repo

# Create Python virtual environment
python3.11 -m ensurepip --upgrade
python3.11 -m venv ~/py311cuda

# Add environment activation to bashrc if not already present
if ! grep -q "source ~/py311cuda/bin/activate" ~/.bashrc; then
    echo "source ~/py311cuda/bin/activate" >> ~/.bashrc
fi

# Activate the environment for the current session
source ~/py311cuda/bin/activate

# Upgrade pip and install packages
pip install --no-cache-dir --upgrade pip
pip install --no-cache-dir \
    torch \
    tqdm \
    click \
    transformers \
    datasets \
    wandb \
    plotly \
    pandas \
    torchvision \
    openai \
    backoff

# Reload bash configuration
source ~/.bashrc

echo "Installation complete! Python environment is activated and will auto-activate in new shell sessions."
python -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}\nCUDA device count: {torch.cuda.device_count()}\nCurrent device: {torch.cuda.current_device()}')"

wandb login

Install basic stuff, git config, and cmd

git config --global user.name "Simo Ryu"
git config --global user.email "[email protected]"

ACTIVATE_CMD="source ~/py39cuda/bin/activate"

# Check if the command is already in .bashrc
if grep -Fxq "$ACTIVATE_CMD" ~/.bashrc
then
    echo "Virtual environment activation command already in .bashrc"
else
    # Add the command to .bashrc
    echo "$ACTIVATE_CMD" >> ~/.bashrc
    echo "Virtual environment activation command added to .bashrc"
fi

$ACTIVATE_CMD

ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC57w2K4gr9f+kpctuIZTGe1m+YC+5evJLc0igLs9MZ1zmBEHGqVmgtaWzuAunHqyRa2p64Wjv4OZ5i7qKb601ZU3uFtGkK9MxL1K6VvNC7HRr27chgs7nIc5/ZAslQoquPzbssOcRR/DTqKdfeb6l76ZkE7ZyXlVzX5JFRbPuUK4po7rtHPb0bE51dkFNyBqPLCidfY/+OuXMs1EXoH+n+TFfiXE/i5Ec5Zo9gvSuntiBewH+QFsozYItGEFKiodT40TuAZkHGqXydIzB+dc7NcL2ZAJsPebnuxolD3IE3tdticy/E/WlBfdCIeJsLS56lOUkSSzHf3dWzlw9cu2kH8XhfxfgE70z62NdFMNDih7AUoFovfHx42F5yEsr2kQSPZQXdwCvRauZ36Mu4u8t1yF8//GiY9ZNX+7SArE2eImo86nHPMGohn8ofVvWhGvhQP+tlUEdwdG2YMiCR5socBj8WLLFFITOo671IfXCcGMMeQJaom3+AoYo8g6irPvM= clone@BOOK-2UT1U9LBLL

Iterate over servers to put the key

SIMO_PUBKEY="ssh-rsa ..."

# Nodes
NODES=(
  "4a78144e-01.cloud.together.ai"
  "4a78144e-02.cloud.together.ai"
  "4a78144e-03.cloud.together.ai"
  "4a78144e-04.cloud.together.ai"
)

# Loop through each node and configure the SSH access for simo
for NODE in "${NODES[@]}"; do
  ssh root@$NODE <<EOF
    # Create .ssh directory and set permissions
    sudo -u simo mkdir -p /home/simo/.ssh
    sudo -u simo chmod 700 /home/simo/.ssh

    # Add the public key to authorized_keys
    echo "$SIMO_PUBKEY" | sudo -u simo tee -a /home/simo/.ssh/authorized_keys
    sudo -u simo chmod 600 /home/simo/.ssh/authorized_keys
EOF
done

addkey

#!/bin/bash

# Create the addkey script
cat << 'EOF' > ~/addkey
#!/bin/bash

# Check if an argument is provided
if [ -z "$1" ]; then
  echo "Usage: addkey \"ssh-rsa ...\""
  exit 1
fi

# Ensure the .ssh directory exists
mkdir -p ~/.ssh

# Add the provided key to the authorized_keys file
echo "$1" >> ~/.ssh/authorized_keys

# Set the correct permissions
chmod 700 ~/.ssh
chmod 600 ~/.ssh/authorized_keys

echo "Key added successfully."
EOF

# Make the addkey script executable
chmod +x ~/addkey

# Move the addkey script to /usr/local/bin to make it available system-wide
sudo mv ~/addkey /usr/local/bin/addkey

# Verify the installation
if command -v addkey &> /dev/null; then
  echo "The addkey command has been successfully installed."
else
  echo "There was an error installing the addkey command."
fi

Claude filesplit

#!/usr/bin/env python3
"""
filesplitter.py - Split a text file into multiple files based on special comment lines.

Usage:
    python filesplitter.py <input_file> [output_dir]

The script looks for lines that start with:
- "# " followed by a path
- "// File: " followed by a path

When found, all content following that line until the next file marker will be written
to the specified path.
"""

import os
import sys
import re


def ensure_directory_exists(filepath):
    """Create the directory for a file if it doesn't exist."""
    directory = os.path.dirname(filepath)
    if directory and not os.path.exists(directory):
        os.makedirs(directory)
        print(f"Created directory: {directory}")


def extract_filepath(line):
    """
    Extract the filepath from a marker line.
    Returns None if the line isn't a valid file marker.
    """
    # Match Python-style comments: "# path/to/file.py"
    python_match = re.match(r'^\s*#\s+(.+\.py)$', line)
    if python_match:
        return python_match.group(1).strip()
    
    # Match "//" style comments: "// File: path/to/file.ts"
    js_match = re.match(r'^\s*//\s+File:\s+(.+)$', line)
    if js_match:
        return js_match.group(1).strip()
    
    # Math "File: "
    file_match = re.match(r'^\s*#\s*File:\s+(.+)$', line)
    if file_match:
        midstr = file_match.group(1).strip()
        if midstr.startswith('```'):
            return "\n".join(midstr.split("\n")[1:-1])
        else:
            return midstr
    
    return None


def split_file(input_file, output_dir='.'):
    """
    Split the input file into multiple files based on marker comments.
    
    Args:
        input_file: Path to the input file
        output_dir: Base directory for output files (default: current directory)
    """
    current_file = None
    current_content = []
    file_count = 0
    
    print(f"Processing {input_file}...")
    
    with open(input_file, 'r', encoding='utf-8') as f:
        for line in f:
            filepath = extract_filepath(line)
            
            if filepath:
                # If we have a current file, write its content before moving to the next
                if current_file and current_content:
                    full_path = os.path.join(output_dir, current_file)
                    ensure_directory_exists(full_path)
                    
                    with open(full_path, 'w', encoding='utf-8') as outfile:
                        outfile.writelines(current_content)
                    
                    print(f"Created file: {full_path}")
                    file_count += 1
                
                # Set the new current file and reset content
                current_file = filepath
                current_content = []
            elif current_file is not None:
                # Add the line to the current file's content
                current_content.append(line)
    
    # Don't forget to write the last file
    if current_file and current_content:
        full_path = os.path.join(output_dir, current_file)
        ensure_directory_exists(full_path)
        
        with open(full_path, 'w', encoding='utf-8') as outfile:
            outfile.writelines(current_content)
        
        print(f"Created file: {full_path}")
        file_count += 1
    
    print(f"Completed! Split into {file_count} files.")


def main():
    if len(sys.argv) < 2:
        print(f"Usage: {sys.argv[0]} <input_file> [output_dir]")
        sys.exit(1)
    
    input_file = sys.argv[1]
    output_dir = sys.argv[2] if len(sys.argv) > 2 else '.'
    
    if not os.path.exists(input_file):
        print(f"Error: Input file '{input_file}' does not exist.")
        sys.exit(1)
    
    split_file(input_file, output_dir)


if __name__ == "__main__":
    main()