amd 6800 xt hip rocm and stable diffusion testing on linux

amd-hip-rocm-stable-diffusion.txt

## last tested nov. 2022
## fedora 37 (targetting rhel 9.1 amd repos) (using GUI based Fedora 37, which includes a working AMDGPU kernel driver by default)
## rocm 5.3.3

## note the amdgpu kernel driver in use before starting:
modinfo amdgpu |grep ver


# https://docs.amd.com/bundle/ROCm-Installation-Guide-v5.3.3/page/How_to_Install_ROCm.html#d23e7177

## update the os:
sudo dnf update

## install the ROCM installer package directly from amd's site. 
sudo yum install https://repo.radeon.com/amdgpu-install/5.3.3/rhel/9.0/amdgpu-install-5.3.50303-1.el9.noarch.rpm


## point the repo file to RHEL 9.1: (since we're not on RHEL, force the closest RHEL version to fedora, or just use ALMA linux 9.1 to ensure an exact match)
sudo nano /etc/yum.repos.d/amdgpu.repo

# from:
baseurl=https://repo.radeon.com/amdgpu/5.3.3/rhel/$amdgpudistro/main/x86_64

#to:
baseurl=https://repo.radeon.com/amdgpu/5.3.3/rhel/9.1/main/x86_64



## install the rocm bits: (wihtout the DKMS module, fedora already has an up-to-date kernel driver for amdgpu if running the gui)

sudo amdgpu-install --usecase=rocm --no-dkms

## completes successfully on FEDORA 37 - targetting RHEL 9.1 rocm repo!

## reboot

sudo systemctl reboot


## check status of install:
## check for dkms module: (dkms module NOT needed in a gui distro that already has the amdgpu driver module loaded)
dkms status

# check that rocminfo detects the gpu:
/opt/rocm-5.3.3/bin/rocminfo

# check that clinfo detects the gpu:
/opt/rocm-5.3.3/opencl/bin/clinfo

#############################################

## run some basic hip code to validate:

## install git:
sudo dnf install git cmake


sudo mkdir /opt/build

sudo chmod 777 /opt/build 

cd /opt/build
git clone https://github.com/ROCm-Developer-Tools/HIP-Examples.git


## run some test code: (see test_all.sh for some of these)
cd /opt/build/HIP-Examples

cd strided-access
make clean
make
./strided-access
cd ..

##
cd reduction
make clean
make
./run.sh
cd ..

##
cd cuda-stream
make clean
make
./stream
cd ..

##
cd vectorAdd
make clean
make
cd ..

##
cd rtm8
./build_hip.sh
./rtm8_hip
cd ..

## compile and run MIXBENCH

cd /opt/build
git clone https://github.com/ekondis/mixbench

cd mixbench

## create a build dir for mixbench:
mkdir build
cd /opt/build/mixbench/build

## add the set_property line to the end of the cmakelists file: (wasn't needed on alma 9.1, but is needed on fedora 37)
nano ../mixbench-hip/CMakeLists.txt

# gfx1030 is radeon 6800 xt (should also be same for 6800 and 6900 xt and 6950 xt) 
set_property(TARGET mixbench-hip PROPERTY HIP_ARCHITECTURES gfx1030)





#######################
## for cuda 

# install dev tools:
sudo yum groupinstall 'Development Tools'


## install home brew. (not sudo)
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"


#==> Next steps:
#- Run these three commands in your terminal to add Homebrew to your PATH:
#    echo '# Set PATH, MANPATH, etc., for Homebrew.' >> /home/user11/.bash_profile
#    echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"' >> /home/user11/.bash_profile
#    eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
#- Install Homebrew's dependencies if you have sudo access:
#    sudo yum groupinstall 'Development Tools'
#  For more information, see:
#    https://docs.brew.sh/Homebrew-on-Linux
#- We recommend that you install GCC:
#    brew install gcc
#- Run brew help to get started
##

## add homebrew envs to current shell to get 'brew' available:
## this also makes any apps installed by brew availabe to the current shell.
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

# install needed gcc 11 for cuda:
brew install 'gcc@11'

## link needed bins to the cuda area:

# skip this, we have gcc-11 already in the shell path from 'eval' from brew above.
#sudo su -
#cd /usr/local/bin
#for x in  /home/linuxbrew/.linuxbrew/bin/*-11; do  ln -s $x .; done


## link the brew installed gcc-11 bits directly into the CUDA folder:

#cd /usr/local/cuda/bin
#ln -s /usr/local/bin/gcc-11 gcc
#ln -s /usr/local/bin/g++-11 g++

sudo ln -s /home/linuxbrew/.linuxbrew/bin/gcc-11 /usr/local/cuda/bin/gcc
sudo ln -s /home/linuxbrew/.linuxbrew/bin/g++-11 /usr/local/cuda/bin/g++

## set the add the cuda/bin folder to the FRONT of the PATH variable for this session (to ensure 'gcc' is v11 from here)
PATH="/usr/local/cuda/bin/:$PATH"

## configure the cmake build for mixbench:
export CUDAARCHS='native'

# or:
#export CUDAARCHS='61' #for titanX

export CUDACXX='/usr/local/cuda/bin/nvcc'


## configure and generate make files:
cmake ../mixbench-cuda


#
####################  END CUDA SPECIFIC ###########
###################################################


## configure and generate make files:
cmake ../mixbench-hip

## compile the HIP mixbench binary:
make

## run the built bits, generates a CSV to std out with the results:

/opt/build/mixbench/build/mixbench-hip

/opt/build/mixbench/build/mixbench-cuda

##########################################################
#### compile and run GPU-STREAM benchmark: (BabelStream)

cd /opt/build
git clone https://github.com/UoB-HPC/BabelStream

## add the ROCM HIP path to the includes_directories variable in CMAKE:
cp /opt/build/BabelStream/CMakeLists.txt ~/CMakeLists.txt.bak

sed -i 's^include_directories(src)^include_directories(src /opt/rocm/hip/include)^' /opt/build/BabelStream/CMakeLists.txt

diff /opt/build/BabelStream/CMakeLists.txt ~/CMakeLists.txt.bak


# configure the build, build type defaults to Release (must manually specifcy the cxx compiler to use for HIP builds) from https://rocmdocs.amd.com/en/latest/Installation_Guide/Using-CMake-with-AMD-ROCm.html
cd /opt/build/BabelStream
cmake -Bbuild -H. -DCMAKE_CXX_COMPILER=/opt/rocm-5.3.3/bin/hipcc -DMODEL=hip

############## or for cuda:#####
cd /opt/build/BabelStream
cmake -Bbuild -H. -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCUDA_ARCH=native -DMODEL=cuda

############ end cuda ######

# compile
cd /opt/build/BabelStream
cmake --build build

# run executables in ./build
# hip:
/opt/build/BabelStream/build/hip-stream

# cuda:
/opt/build/BabelStream/build/cuda-stream



##############################################################
####################### end ROCM examples ####################


########################################################
##### testing stable diffusion:

### linux:
# instructions for stable diffusion on amd
# https://www.videogames.ai/2022/11/06/Stable-Diffusion-AMD-GPU-ROCm-Linux.html

# install conda:  (skipping prereqs for now)
# yum install libXcomposite libXcursor libXi libXtst libXrandr alsa-lib mesa-libEGL libXdamage mesa-libGL libXScrnSaver

# manually download the ~1GB installer .sh file:
# https://docs.anaconda.com/anaconda/install/linux/


cd ~/Downloads
chmod +x Anaconda3-2022.10-Linux-x86_64.sh

./Anaconda3-2022.10-Linux-x86_64.sh

# take defaults for the install, it will default to:
# /home/user11/anaconda3

######### end question: no #######
#Do you wish the installer to initialize Anaconda3
#by running conda init? [yes|no]
#[no] >>> 

#You have chosen to not have conda modify your shell scripts at all.
#To activate conda's base environment in your current shell session:

#eval "$(/home/user11/anaconda3/bin/conda shell.YOUR_SHELL_NAME hook)" 

#To install conda's shell functions for easier access, first activate, then:

#conda init

#If you'd prefer that conda's base environment not be activated on startup, 
#   set the auto_activate_base parameter to false: 

#conda config --set auto_activate_base false
###################

## setup conda into the path temporarily for this session:
source ~/anaconda3/bin/activate

## make sure your in (base) from the source command above first.

cd ~
git clone https://github.com/CompVis/stable-diffusion.git


cd ~/stable-diffusion/
conda env create -f environment.yaml


## create the model directory for sd v1:
cd ~/stable-diffusion
mkdir models/ldm/stable-diffusion-v1/

## link the model:
cd ~/stable-diffusion

## remove a previous sym link if needed:
rm models/ldm/stable-diffusion-v1/model.ckpt

#### pick a model to link as model.ckpt: (aquire the sd-v1-4.ckpt model elsewhere)
## SD 1.4 standard model:
ln -s /home/user11/Downloads/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt

#######


# now activate the new 'ldm' environment created above via 'conda env create...'
conda activate ldm

## install the version of pytorch for rocm: https://pytorch.org/  pick pytorch: stable, linux, pip,rocm to get the proper url/command line to use below:
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.2/

cp ~/stable-diffusion/scripts/txt2img.py ~/stable-diffusion/scripts/txt2img-safe.py


# use it! (first time running downloads about 1.5GB from github)
source ~/anaconda3/bin/activate
conda activate ldm

cd ~/stable-diffusion

# 6 image grid example:
python3 scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms

# single example:
python3 scripts/txt2img.py --prompt "astronaut riding a horse" --H 512 --W 512 --n_iter 1 --ddim_steps 50 --n_samples 1

# safe example:
python3 scripts/txt2img-safe.py --prompt "astronaut riding a horse" --H 512 --W 512 --n_iter 1 --ddim_steps 50 --n_samples 1