Notes for building flash attention on windows

build-flash-attention.ps1

## Building Flash Attention
# See https://huggingface.co/lldacing/flash-attention-windows-wheel for inspiration
# Especially https://huggingface.co/lldacing/flash-attention-windows-wheel/blob/main/WindowsWhlBuilder_cuda.bat

## Environment
# Set environment variables to trigger Windows specific build handling using MS toolchains
# See https://github.com/Dao-AILab/flash-attention/blob/b2476552432fd6ac991003db4564eb289dd77332/setup.py#L222-L224 

$env:BUILD_TARGET="cuda"
$env:FLASH_ATTENTION_FORCE_BUILD="True"
$env:DISKUTILS_USE_SDK="1"
$env:MAX_JOBS="1"
$env:FLASH_ATTN_CUDA_ARCHS="75"
$env:TORCH_CUDA_ARCH_LIST="7.5"

### Build Environment
# Set some flags that help with getting the build environment in good shape

# Visual Studio Build Tools
$vsVersion="2022"
$vsToolsPath= "C:\Program Files\Microsoft Visual Studio\$vsVersion\Community\Common7\Tools"
. "$vsToolsPath\Launch-VsDevShell.ps1" -Arch amd64 -HostArch amd64

# Cuda build tools
#"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\vXX.X"
$cudaPath=$env:CUDA_PATH
$env:PATH += ";$cudaPath\bin"
$gitRepository="https://github.com/Dao-AILab/flash-attention.git"
git clone --single-branch --depth 1 "$gitRepository"
(cd flash-attention && git fetch --tags)
(cd flash-attention && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)))