google colab VM initial

google_colab_VM_initial.ipynb.txt

#############################ipynb START###############################################

#################### the 1st step after factory reset VM runtime
!apt update ; apt install openssh-server
!echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config
!mkdir /root/.ssh
!echo 'ssh-rsa .....' >/root/.ssh/authorized_keys
!service ssh start

from google.colab import drive
drive.mount('/root/gdrive')
!ln -s /root/gdrive/MyDrive/ColabSingularity /root/sing

!cp /root/sing/tailscale_1.32.1_amd64.tgz ./
!tar zxvf tailscale_1.32.1_amd64.tgz
!cp tailscale_1.32.1_amd64/tailscale* /usr/bin/
!rm -rf tailscale_1.32.1_amd64*
!mkdir /var/lib/tailscale ; cp /root/sing/tailscaled.state /var/lib/tailscale/tailscaled.state
!nohup /usr/bin/tailscaled --tun=userspace-networking --state=/var/lib/tailscale/tailscaled.state --socket=/run/tailscale/tailscaled.sock --port 41641 >/dev/null 2>&1 &
#!cp /var/lib/tailscale/tailscaled.state /root/sing/    #first time, after tailscale inited
!tailscale up
!tailscale ip

##################### the 1st step after restart VM runtime
%cd /root/

#!JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '"' -f 4 | tr -d 'v')
#!cd /root/sing/ && wget "https://github.com/juicedata/juicefs/releases/download/v${JFS_LATEST_TAG}/juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz"
#!cd /root/sing/ && tar -zxf "juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz" juicefs && rm juicefs-${JFS_LATEST_TAG}-linux-amd64.tar.gz
!install /root/sing/juicefs /usr/local/bin

#!juicefs format --storage file --bucket /root/sing/gdrive_file_volume1 "sqlite3:///root/sing/gdrive_file_volume1.db" volume1
!while true; do nohup cp /root/gdrive_file_volume1.db /root/sing/ > /dev/null 2>&1; sleep 5; done &
!juicefs mount -d "sqlite3:///root/sing/gdrive_file_volume1.db" ./gdrive_file_volume1

#################### the 2nd step after restart VM runtime
#!wget https://github.com/OurGI-com/OurGI-src/blob/master/singularity/singularity_3.8.3_for_google_colab.deb?raw=true -O sing/singularity_3.8.3_for_google_colab.deb
!dpkg -i sing/singularity_3.8.3_for_google_colab.deb
!ln -s /usr/local/bin/singularity /usr/bin/sing
!apt install nano squashfs-tools uidmap
!mkdir -p /usr/local/var/singularity/mnt/session
!singularity config fakeroot --add root

#!sing build -s ./storj_demobucket_volume1/sing-debian-1/ docker://debian:buster-slim
#!cp sing/sing-debian-1.20211019.0416.tar.gz ./
#!pv sing-debian-1.20211019.0416.tar.gz| tar xzpf - -C ./gdrive__volume1
#extract debian image only cost 3.5 minutes
!sing shell --fakeroot --writable ./gdrive_volume1/sing-debian-1/

##################### misc console:
#show status:
!pwd
!tailscale ip
!ps axu | grep ssh
!ls
!nvidia-smi

#run single command:


#############################ipynb END###############################################


#############################run rootless docker in google colab:###############################################
useradd -md /opt/docker docker
apt-get -qq install iproute2 uidmap
sudo -Hu docker SKIP_IPTABLES=1 bash < <(curl -fsSL https://get.docker.com/rootless)
mkdir /run/docker/plugins
chown docker:docker -R /run/docker
su docker
cd /opt/docker
%%writefile docker-run.sh
#!/usr/bin/env bash
set -e
export DOCKER_SOCK=/opt/docker/.docker/run/docker.sock
export DOCKER_HOST=unix://$DOCKER_SOCK
export PATH=/opt/docker/bin:$PATH
export XDG_RUNTIME_DIR=/opt/docker/.docker/run
rootlesskit --debug --disable-host-loopback --copy-up=/etc --copy-up=/run /opt/docker/bin/dockerd -b none --experimental --iptables=false --storage-driver vfs &
for i in $(seq 5); do [ ! -S "$DOCKER_SOCK" ] && sleep 2 || break; done
docker $@
jobs -p
kill $(jobs -p)

chmod 777 docker-run.sh
./docker-run.sh run --cap-add SYS_ADMIN hello-world

#"join session keyring: create session key: operation not permitted": unknown.
#Which could be solved by !sysctl -w kernel.keys.maxkeys=500, however Colab doesn't allow it.


#############################procedure notes:###############################################

##################### juicefs mount file gdrive
#!juicefs format --storage file --bucket /root/sing/gdrive_file_volume1 "sqlite3:///root/sing/gdrive_file_volume1.db" volume1
!juicefs mount -d "sqlite3:///root/sing/gdrive_file_volume1.db" ./gdrive_file_volume1

###################### juicefs mount webdav
#enable google drive api: https://console.cloud.google.com/apis/library/drive.googleapis.com
#create oauth client credential: https://console.cloud.google.com/apis/credentials/oauthclient
!cd /root/sing && wget https://github.com/diyism/gdrive-webdav/releases/download/google_colab_bin/gdrave
!install /root/sing/gdrave /usr/local/bin/
!gdrave --client-id=... --client-secret=...
!nohup gdrave --client-id=... --client-secret=... >/dev/null 2>&1 &

#!juicefs format --storage webdav --bucket http://127.0.0.1:8765/ "sqlite3:///root/gdrive/MyDrive/ColabSingularity/gdrive_webdav_volume1.db" volume1
!juicefs mount -d "sqlite3:///root/gdrive_webdav_volume1.db" ./gdrive_webdav_volume1

###################### juicefs mount storj or minio
#!juicefs format --storage s3 --bucket https://gateway.us1.storjshare.io/demo-bucket --access-key ... --secret-key ... "sqlite3:///root/gdrive/MyDrive/ColabSingularity/storj_demobucket_volume1.db" volume1
#!juicefs format --storage s3 --bucket https://gateway.us1.storjshare.io/demo-bucket --access-key ... --secret-key ... "badger:///root/gdrive/MyDrive/ColabSingularity/storj_demobucket_volume1" volume1
#!juicefs format --storage minio --bucket http://<vps ip>:9000/bucket1 --access-key ... --secret-key ... "sqlite3:///root/gdrive/MyDrive/ColabSingularity/minio_bucket1_volume1.db" volume1
#the last param of juicefs "format" is the volume name, mapped to the storj storage's first level folder, the volume name will be saved into the storj_demobucket_volume1.db, so that while execing "juicefs mount" it can connect https://gateway.us1.storjshare.io/demo-bucket/volume1
!cp sing/storj_demobucket_volume1.db ./
!juicefs mount -d "sqlite3:///root/storj_demobucket_volume1.db" ./storj_demobucket_volume1
#!juicefs mount -d "badger:///root/storj_demobucket_volume1" ./storj_demobucket_volume1
#!juicefs umount ./storj_demobucket_volume1

###################### replace storj, self host s3 server(MinIO) on my vps
#single file, google colab->korea vps minio: 2.5MB/s  #china home->korea vps: 0.5MB/s, local to local 80MB/s
#sing file, google-colab->korea vps sshfs: 1.5MB/s    #sshfs support hardlink in mounted volume
#sing file, google colab or china home->storj crowd source network: 0.2MB/s
#sing file, google colab->google drive webdav: 7MB/s
#a folder, google colab->google drive webdav: 0MB/s   #gdrive-webdav has bug, leaking tcp  ESTAB while cp a folder
#sing file, google colab->google drive juicefs_folder: 200MB/s
#a folder, google colab->google drive juicefs_folder: 0.5MB/s(juicefs meta db out of gdrive),   0.05MB/s(juicefs meta db in gdrive)

wget https://dl.min.io/server/minio/release/linux-amd64/minio
sudo install minio /usr/local/bin/
rm minio
sudo mkdir /data
sudo chmod 777 /data
sudo iptables -I INPUT -p tcp -m state --state NEW -m tcp --dport 9091 -j ACCEPT
sudo iptables -I INPUT -p tcp -m state --state NEW -m tcp --dport 9000 -j ACCEPT
MINIO_ROOT_USER=admin MINIO_ROOT_PASSWORD=... minio server /data --console-address ":9091"
#visit http://<vps ip>:9091  with the MINIO_ROOT_USER and MINIO_ROOT_PASSWORD
#create bucket1 and user1

######################   storj official gateway is very slow
#try to run self hosted gateway on local PC to connect the mesh network:
./gateway setup --non-interactive --access <access grant>
./gateway run    #it will show the local access key and local secret key
juicefs format --storage s3 --bucket http://127.0.0.1:7777/demo-bucket --access-key <local access key> --secret-key <local secret key> "sqlite3://volume21.db" volume21
juicefs mount -d "sqlite3://volume21.db" ./volume21
pv ../b4gi.tar.gz | tar xzpf - -C ./volume21       #juicefs format and juicefs mount didn't create volume21 on storj website, unitl start to write file into volume21
ss -tuwnap | grep gateway   #I can see more than 100 nodes connected all over the world
#I can see the self hosted gateway is as slow as the official one: 10KB/s to 200KB/s

#####################不可行(关机时间不定性) run before restart/stop VM runtime
#run before restart/stop VM
!tar -czpf /content/sing/sing-debian-1.`date +%Y%m%d.%H%M`.tar.gz sing-debian-1
#sync to google drive:
from google.colab import drive
drive.flush_and_unmount()
drive.mount('/root/gdrive')

##################### (not work, maybe keyboard/mouse detecting) in browser console, run:
function ClickConnect() {console.log("Working"); document.querySelector('#top-toolbar > colab-connect-button').shadowRoot.querySelector('#connect').click()}
setInterval(ClickConnect, 60000)

##################### then login with ssh cloudflare proxy or normal ssh login through tailscale ip

==================apt install failed in sing shell:=======================================
#what a pity, mounted gdrive file system(like rclone: https://github.com/rclone/rclone/issues/3800) doesn't support hardlink,
#we can start a sing shell, but we can't apt install in this sing shell ("dpkg -i" using hardlink sys call)
#!cd /root && cp -R sing-debian-1 sing/
#before shutdown/leave, backup exec permissions:
#!cd /root/sing-debian-1 && find . -perm -100 -type f >/root/sing/sing-debian-1.acls
#after start, restore exec permissions:
#!cd /root/sing/sing-debian-1 && cat /root/sing/sing-debian-1.acls |xargs -I{} sh -c "test -e {} && chmod +x {}"
#!cd /root/sing && sing shell --fakeroot --writable sing-debian-1/

==================build writable sif file failed:===========================================
#!cd /root && sing build sing-debian-1.sif docker://debian:buster-slim
#!cd /root && dd if=/dev/zero of=overlay.img bs=1M count=1000 && mkfs.ext3 overlay.img
#!cd /root && singularity siftool add --datatype 4 --partfs 2 --parttype 4 --partarch 2 --groupid 1 sing-debian-1.sif overlay.img
#!cd /root && rm overlay.img && mv sing-debian-1.sif /root/sing/
#!cd /root/sing && sing shell --fakeroot --writable sing-debian-1.sif
#but failed with system setcap limitations

==================build singularity_3.8.3_for_google_colab.deb:===========================================

#build singularity debian package:
cd sing
#apt install acl
#backup acl permissions, because google drive won't keep it after unmount(to fix: drive.flush_and_unmount())
#getfacl -R ./ >../sing.acls
#setfacl --restore ../sing.acls
#if seems "setfacl" won't work with google drive mount
#the sing.acls file located at /content/gdrive/MyDrive/
#to use these 2 lines to replace getfacl and setfacl:
find . -perm -100 -type f >../sing.acls
cat ../sing.acls |xargs -I{} sh -c "test -e {} && chmod +x {}"

rm -rf /usr/local/go && wget -qO- https://golang.org/dl/go1.17.linux-amd64.tar.gz | sudo tar -xvz -C /usr/local
export PATH=$PATH:/usr/local/go/bin
export GOPATH=/root/sing/gopath
sed -i 's{/usr/games:/usr/local/games{/usr/local/go/bin{' /etc/environment
echo 'GOPATH=/root/sing/gopath' >> /etc/environment
cd sing
export VERSION=3.8.3
wget https://github.com/hpcng/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz
tar -xzf singularity-${VERSION}.tar.gz
cd singularity-${VERSION}
apt install cryptsetup libseccomp-dev
./mconfig
#need "make" first to prepare go pkgs for debmake
make -C builddir

#debmake >../debmake.log 2>&1
#create file debian/source/include-binaries with content in the attachment
#dpkg-source --commit -i'(^|/)(\.github|makeit-intermediate)($|/)'
#debuild -i'(^|/)(\.github|makeit-intermediate)($|/)'
#but debuild built a nearly empty deb file, to use "checkinstall" to build deb:

checkinstall -D make install
ln -s /usr/local/bin/singularity /usr/bin/sing

#the singularity_3.8.3_for_google_colab.deb is 28MB, uploading to github will show " Yowza, that’s a big file. Try again with a file smaller than 25MB.", to use git lfs, for example in Kali/debian:
#to check "Include Git LFS objects in archives" option in github project Settings
wget https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh
chmod 777 ./script.deb.sh
os=debian dist=bullseyes ./script.deb.sh
sudo apt install git-lfs
git clone --depth 1 https://github.com/OurGI-com/OurGI-src.git
cd OurGI-src/singularity
cp <builtdir>/singularity_3.8.3_for_google_colab.deb ./
git lfs install
git lfs track "singularity_3.8.3_for_google_colab.deb"
git add .gitattributes
git add .
git commit -m "Upload big file"
git push