bash shell cheats

bash-cheats.sh

#!/bin/bash

# exit on error and don't allow the use of unset variables
set -o errexit
set -o nounset

#NOTE: you can find a list of special variables at http://tldp.org/LDP/abs/html/internalvariables.html

# quietly execute GUI applications
function quietly() {
    nohup "$@" < /dev/null > /dev/null 2>&1 &
}

echo function firefox() {
    quietly firefox "$@"
}

function suquietly() {
    sudo -b "$@" < /dev/null > /dev/null 2>&1 &
}

# iterate over find results:
find . -name '*remove*' -print0 | while read -d $'\0' file; do echo "$file"; head "$file"; done

# run a command as the wwwrun user
sudo -u wwwrun /bin/bash -c 'ls /'

# recursive get without going into parent dirs
wget --recursive --no-parent 'http://www.matthias-baldauf.at/software/ndpi-converter/sourcecode/v1_5/'

# rsync with many excludes
rsync -a --progress --exclude='releases' --exclude='wh-env' --exclude='wormhole.db*' --exclude='non-canonical*.txt' --exclude='temp' wormhole/ 'donkey:/userdata/kss/wormhole'

# rsync with filtering and compression
rsync -avz --prune-empty-dirs --include='*.npz' --exclude='*.*' 'cadillac:/hpcdata/cgd/DO_Liver/analysis/alignment/8-way/' 8-way

# rsync with checksum confirmation and progress
rsync -a --checksum --progress --exclude='releases' wormhole/ 'donkey:/userdata/kss/wormhole'

# rsync recursively including files based on extension:
rsync -a '--include=*/' '--include=*.h5' '--exclude=*' --prune-empty-dirs 'winter:/fastscratch/sheppk/gait-mutants-2019-02-25' .

# syncing local dir to remote dir
rsync -az 2010-12-02-genotype-for-gary kss@truda:projects

# recursively copy text files from src to dest
rsync -r --include '*/' --include '*.txt' --exclude '*' src/ dest/

# using relative option to control which dirs are created for destination
rsync -avR "${share_root}/./NV9-CBAX2/2019-08-02" login.winter.jax.org:/fastscratch/sheppk/b6-btbr-2020-02-18
rsync -v '--include=*/' '--include=*.avi' '--exclude=*' -aR "${share_root}/./NV6-CBAX2/2019-10-11" login.winter.jax.org:/projects/sheppk/datasets/B6J+3F+strangers+4+days
rsync -v '--include=*/' '--include=*_pose_est_v3.h5' '--exclude=*' -aR login.winter.jax.org:/fastscratch/sheppk/B2B_MISSING_3M_stranger_4day-2020-11-12/./ /media/sheppk/TOSHIBA\ EXT/B2B_MISSING_3M_stranger_4day-2020-11-12-pose

# from a newline separated list of files
rsync -av --files-from=./networkPointNotFoundBXD.txt "${share_root}" login.winter.jax.org:/fastscratch/sheppk/networkPointNotFoundBXD

# rclone example
rclone copy --exclude Thumbs.db --transfers 8 --dropbox-chunk-size 150M --progress '/home/vivek/labshare/VideoData/MDS_Tests/exp_name' 'labdropbox:/VideoData/MDS_Tests/exp_name'
rclone check --exclude Thumbs.db --size-only '/home/vivek/labshare/VideoData/MDS_Tests/exp_name' 'labdropbox:/VideoData/MDS_Tests/exp_name'
rclone lsf --recursive --include '*_pose_est_v3.h5' --files-only "labdropbox:/VideoData/MDS_Tests/AD_4M_PS19_4day"
rclone copy --include '{AD_4B_PS19_4day,AD_4M_PS19_4day}/**.h5' --transfers 4 --progress "labdropbox:/VideoData/MDS_Tests" AD-models-PS19-poses
rclone copy --transfers 4 --progress --include-from ./missing-poses-2021-02-25.txt --ignore-checksum "labdropbox:/VideoData/MDS_Tests/AD_4B_PS19_4day" ~/sshfs/winterfastscratch/missing-poses-2021-02-25
# to synch back pose files
rclone copy --size-only --exclude '*.avi' --transfers 6 --dropbox-chunk-size 150M --progress "labdropbox:/VideoData" "${share_root}/VideoData"

# using curl to fetch multiple permutations of PNG file names
curl --fail 'http://geniverse.dev.concord.org/geniverse/cache/5/{HH,Hh,hH,hh},SS,{WW,Ww,wW,ww},{LL,Ll,lL,ll},{TT,Tt,tT,tt},pp,{FF,Ff,fF,ff},{AA,Aa,aA,aa},{BB,Bb,bB,bb}.png' -o '#1,SS,#2,#3,#4,pp,#5,#6,#7.png'

# port forwarding for postgres and mongo respectively
ssh -L 5432:localhost:5432 [email protected]
ssh -L 27017:localhost:27017 [email protected]

# compress and "scp" at the same time
tar cz dir | ssh name@host 'cat >/some-path/dir.tgz'

# cut the first column from a tab delimited text file
cut -f 1 -d $'\t' grouped-intens.txt  | uniq

# find and replace (-i '.tmp' means edit in-place)
sed -i '.tmp' 's/cuscus/triok/g' favorite-marsupial.txt

# a simple integer for loop
for((i=0; i<100; i++)); do echo "$i"; done

# looping and using printf to pad with zeros. See more examples of string manipulation at:
# http://tldp.org/LDP/abs/html/string-manipulation.html
for i in `"ls"`; do mv "$i" "$(printf %04d $ITERNUM)-$i"; export ITERNUM=$((ITERNUM + 1)); done
# removes all of the -3.3.3 before the .jar extension
for i in lucene-*.jar; do mv $i "${i:0:(${#i} - 10)}.jar"; done
# change "syntactic_inference" suffix to "_pose_est"
for i in $(find . -name '*syntactic_inference.h5'); do mv $i "${i:0:(${#i} - 22)}_pose_est.h5"; done
# which *.avi files don't have corresponding *.h5 pose est files
for i in $(find * -name '*.avi'); do if [[ ! -f "${i:0:(${#i} - 4)}_pose_est.h5" ]]; then echo $i; fi; done

# loop through tab delimited files and select one of the columns
for i in `tabtocsv goodstrain.txt | tssql -table input - 'select V1 from input' | sed '1d'`
do
    ls -l all-cel-files/$i
done

# file tests (from http://tldp.org/LDP/abs/html/fto.html)
if [ -e somefile.txt ]
then
    echo "somefile.txt exists\n"
fi

# find files and execute command
find zip_files/ -name '*.zip' -exec echo \{\} \;

# nohup with specified redirect
nohup ./run-musdiv.R 2>&1 > run-musdiv.out &

# for loops with index and calling functions
downloadInParanoid () {
    echo "Downloading InParanoid"
    inparSpecies=("C.elegans" "H.sapiens" "M.musculus" "S.cerevisiae")
    for (( i=0; i<${#inparSpecies[*]}; i++ ))
    do
        for (( j=i+1; j<${#inparSpecies[*]}; j++ ))
        do
            currFile="InParanoid.${inparSpecies[i]}-${inparSpecies[j]}.orthoXML"
            wget "http://inparanoid.sbc.su.se/download/current/orthoXML/$currFile"
        done
    done
}

downloadInParanoid

# grep-like functionality with perl one-liner
for i in *.txt; do perl -ne $'print if /canonical[12] = \\[[^\\]]+\\,/' $i > "mult-hits-only-${i}"; done

# replace regex group with perl. In this case everything to the right of and including '.' is removed
perl -pe 's/(.*)\..*/$1/g' CHARGE/ENSG.ids > CHARGE/ENSG-no-versions.ids

# (on osx) who is using port 25672?
lsof -i :25672
# (on linux) whois using port 80?
fuser -v -n tcp 80

# use imagemagick to resize a whole directory of images
cp -a Hqsegment Hqsegment_512
mogrify -resize 512x512 Hqsegment_512/*/*/*.png
mogrify -threshold 50% Hqsegment_bw/*/Seg/*.png

# use imagemagick to gamma adjust (improves contrast on mouse images)
mogrify -gamma 1.5 *.png

# use xmodmap to rebind capslock to ctrl
# enter: `xmodmap -pke > ~/.xmodmap`
# now change
keycode  66 = Caps_Lock NoSymbol Caps_Lock
# ... to ...
keycode  66 = Control_L NoSymbol Control_L
# `xev | grep key` will help you find a keycode if you don't know it

# OR you can do
setxkbmap -option ctrl:nocaps

# NOTE: the above didn't work for me with Gnome but GNOME Tweak Tool worked great

renice -n 15 -p 23312 23407 23422 23435 23409 23402
# 23312 (process ID) old priority 0, new priority 15
# 23407 (process ID) old priority 0, new priority 15
# 23422 (process ID) old priority 0, new priority 15
# 23435 (process ID) old priority 0, new priority 15
# 23409 (process ID) old priority 0, new priority 15
# 23402 (process ID) old priority 0, new priority 15

# here is some cool command substitution stuff
> Y=35
> echo "$Y"
35
> RAND=$(python -c "import random; print(random.random() * "$Y")")
> echo $RAND
19.4897899871

# Find ubuntu package that executable belongs to
apt-file search h5dump

# for ubuntu install from deb package
sudo apt install -f rstudio-1.2.1335-amd64.deb
sudo apt-get install -f

# extract an 8sec clip from video starting at the 188th second
ffmpeg -i LL5-1_M2FNLN_009_M0_F2-synth_out3.mp4 -ss 188 -t 8 LL5-1_M2FNLN_009_M0_F2-synth_out3_188_8.mp4

# conversions
ffmpeg -i 031309_A29_Block15_BCma1_s.seq -qscale 0 031309_A29_Block15_BCma1_s.avi
ffmpeg -i input.png -f mp4 -c:v libx264 -preset veryslow -profile:v main -pix_fmt yuv420p -g 30 output.mp4
for f in *.avi
do
    echo "converting ${f}"; [[ -f "${f:0:(${#f} - 4)}.mp4" ]] || ffmpeg -i "${f}" -vcodec libx264 "${f:0:(${#f} - 4)}.mp4"
done

# raspberry pi vid to mp4
ffmpeg -framerate 30 -i tempvid-hirez1.h264 -c copy tempvid-hirez1.mp4

# play videos side by side starting at 28 min:
ffplay \
    -ss 00:28:00 \
    -i LL5-4_SJL_ControlFemale-10-PSY_pose_est.avi \
    -vf 'pad=2*iw:ih [top]; movie=LL5-4_SJL_ControlFemale-10-PSY_pose_est_v3.avi [bottom]; [top][bottom] overlay=main_h:0'

# save side by side videos:
while read -r f
do
    ffmpeg \
        -i "${f:0:(${#f} - 4)}_pose_est.avi" \
        -vf "pad=2*iw:ih [top]; movie=${f:0:(${#f} - 4)}_pose_est_v2.avi [bottom]; [top][bottom] overlay=main_h:0" \
        -threads 12 -crf 18 "${f:0:(${#f} - 4)}_pose_est_combined.avi"
done <spot-check-batch.txt

# play videos stacked:
ffplay \
    -i LL5-4_SJL_ControlFemale-10-PSY_pose_est.avi \
    -vf 'pad=iw:2*ih [top]; movie=LL5-4_SJL_ControlFemale-10-PSY_pose_est_v3.avi [bottom]; [top][bottom] overlay=0:main_h/2'

for i in *t.seq; do echo "converting ${i} to AVI"; ffmpeg -i "${i}" -qscale 0 "${i:0:(${#i} - 4)}.avi"; done

# convert raspberry pi vids to mp4
ffmpeg -framerate 30 -i 2020-11-03_16-32-57_v2_000000005e8d2ebf.h264 -c copy 2020-11-03_16-32-57_v2_000000005e8d2ebf.mp4
for f in */*.h264; do echo "converting ${f}"; [[ -f "${f%.*}.mp4" ]] || ffmpeg -framerate 30 -i "${f}" -c copy "${f%.*}.mp4"; done

# transcode and pad non-standard aspect ratio video
ffmpeg -i VideoS2_vidplot-app.mp4 -f mp4 -c:v libx264 -preset veryslow -profile:v main -pix_fmt yuv420p -g 30 -vf 'pad=iw:862:(ow-iw)/2:(oh-ih)/2' Transcoded_VideoS2_vidplot-app.mp4

# bulk rename
find gait-mutants-2018-10-10 -name '*syntactic_inference.h5' -exec rename -v 's/syntactic_inference/_pose_est/' '{}' ';'

# splitting (chunking) a batch file
split --suffix-length=1 --additional-suffix=.txt -l 500 strain-survey-batch-2019-01-24.txt strain-survey-batch-2019-01-24-

# SLURM
scontrol update ArrayTaskThrottle=<count> JobId=<jobID>
sacct -a --format="Start,User,JobName,JobID,NodeList,AllocGRES" | less
sacct -S 2019-06-24 -a --format="JobName%25,JobID%20,User%15,Elapsed,start,ReqTres%40,ReqGRES"
squeue --partition=gpu --Format='UserName:10,Name,JobArrayID:22,State:10,TimeUsed:14,TimeLimit:14,tres-per-node:8,tres-alloc:20'
squeue --Format='UserName:10,Name:45,JobArrayID:22,State:10,TimeUsed:14,TimeLimit:14,tres-per-node:8,nodelist:11,tres-alloc:50'

# get filenames as a sorted array
IFS=$'\n'
xs=( $("ls" | "sort") )
unset IFS
echo "${xs[2]}"

# mounting an external USB drive in Ubuntu:
udisksctl mount --block-device /dev/disk/by-id/usb-TOSHIBA_External_USB_3.0_20180112013821F-0:0-part2

# mounting the lab share
sudo mount -t cifs -o user=sheppk,gid=1003,uid=1002 //bht2stor.jax.org/vkumar /home/sheppk/smb/labshare  # for personal workstation
sudo mount -t cifs -o user=sheppk,gid=1000,uid=1000,ro //bht2stor.jax.org/vkumar /home/vivek/labshare    # for lab workstation

# on newer ubuntu (see https://developer.gnome.org/gio/stable/gio.html)
gio mount smb://bht2stor.jax.org/vkumar
gio mount --unmount smb://bht2stor.jax.org/vkumar

sshfs -o volname=ldg-bh001 ldg-bh001.jax.org: ~/sshfs/ldg-bh001
sudo diskutil unmount force ~/sshfs/ldg-bh001

# stop warning message from appearing for unsigned apps like VNC in OS X
sudo xattr -d -r com.apple.quarantine /Applications/VLC.app

# my backup process
split -b 10G ../2020-10-29-archive.tgz
rclone copy --transfers 6 --progress 2020-10-29-archive_tgz "labdropbox:/People/KeithSheppard/archives/2020-10-29-archive_tgz"
rm -rf 2020-10-29-archive_tgz 2020-10-29-archive.tgz

# figuring out which files have corresponding metadata files:
find "${share_root}/${subfolder}" -name '*_pose_est_v3.h5' -printf "${subfolder}/%P\n" \
    | perl -pe 's/(.*)_pose_est_v3.h5/$1.avi/g' \
    | head

# which video files are missing pose?
comm -23 <(find "${share_root}/${subfolder}" -name '*.avi' -printf "${subfolder}/%P\n" | sort) \
         <(find "${share_root}/${subfolder}" -name '*_pose_est_v3.h5' -printf "${subfolder}/%P\n" | perl -pe 's/(.*)_pose_est_v3.h5/$1.avi/g' | sort)

# which pose files are missing video?
comm -13 <(find "${share_root}/${subfolder}" -name '*.avi' -printf "${subfolder}/%P\n" | sort) \
         <(find "${share_root}/${subfolder}" -name '*_pose_est_v3.h5' -printf "${subfolder}/%P\n" | perl -pe 's/(.*)_pose_est_v3.h5/$1.avi/g' | sort)

# Note that the -printf option requires GNU find. For OS X you can get this with:
brew install findutils
PATH="$(brew --prefix)/opt/findutils/libexec/gnubin:$PATH"

while read -r subfolder
do
    comm -23 <(find "${share_root}/${subfolder}" -name '*.avi' -printf "${subfolder}/%P\n" | sort) \
             <(find "${share_root}/${subfolder}" -name '*_pose_est_v3.h5' -printf "${subfolder}/%P\n" | perl -pe 's/(.*)_pose_est_v3.h5/$1.avi/g' | sort)
done < ~/temp/Metadatasheet_BXD_intraline-folders.txt

# fixing nvidia "Driver/library version mismatch" on ubuntu

sudo rmmod nvidia_drm
sudo rmmod nvidia_modeset
sudo rmmod nvidia_uvm
sudo rmmod nvidia

nvidia-smi

# turning youtube music into a desktop app that doesn't abuse the SSD
# --portable forces app to use it's own subdir as cache and then we
# remove write perms from that dir
nativefier --disable-old-build-warning-yesiknowitisinsecure --portable 'music.youtube.com'
chmod -R a-w /Applications/YouTube\ Music.app