unix snippets

awk example

find . -type f -exec cksum {} \; > cksum.txt

# change the order of the columns so that the filename is the first column. Use a comma to separate the fields.
awk '{ for(i=3; i<=NF; i++) { if (i==NF) {printf "%s", $i;} else {printf "%s ",$i;}} printf ", %d, %d\n", $1, $2;}' cksum.txt

# get the list of file extensions that exist in a directory and its children
for i in `find . -type f`; do basename $i | awk -F. '{print $NF}'; done | sort | uniq

# awk example, expanded program in shell script, prints filename and owner
#!/usr/bin/env bash
# Linux users have to change $8 to $9
awk '
BEGIN	{ print "File\tOwner" }
		{ print $9, "\t", \
		$3}
END		{ print "done"} 
'
# to test: ls -l | awk_example.sh

bash color example

# I have a project folder. Inside the project folder are individual projects named proj<nn>, where nn is a two digit project number.
# In each project folder there is a README.txt with a short description of the project.

GREEN='\033[0;32m'
NC='\033[0m' # No Color

CMD=$(ls -d proj* | sed 's/proj//' | sort -n); # sort by numerical order not alphabetic order

for i in $CMD; do
  PROJ_DIR="proj$i";
  echo -e "${GREEN}$PROJ_DIR${NC}"; # display in GREEN, then turn off colors
  cat "$PROJ_DIR/README.txt";
done

bash environment variables explanation

https://tldp.org/LDP/Bash-Beginners-Guide/html/sect_03_02.html

bash string functions

# advanced bash string manipulation
http://tldp.org/LDP/abs/html/string-manipulation.html

${string##substring}
Deletes longest match of $substring from front of $string.

${string%substring}
Deletes shortest match of $substring from back of $string.

# helper function to get ip address from host name
# hostname -i
function get_ip_address() {
  IP_ADDRESS=$(hostname)
  IP_ADDRESS=${IP_ADDRESS#ip-}
  IP_ADDRESS=${IP_ADDRESS//-/.}
  echo "$IP_ADDRESS";
}


# bash to copy a zip file into a directory that uses the filename for the directory name
for i in `ls *.zip`; do echo $i; export BASENAME=`echo "${i%.*}"`; echo $BASENAME; mkdir $BASENAME; mv $i $BASENAME; done

check if env variable is set

if [ -z "${MY_VAR+xxx}" ]; then
  # do something here
fi

clear disk space

docker system prune -a

conditional.sh

#!/usr/bin/env bash
# examples using if

A=0
B=1


# single bracket, with test -eq
if [ $A -eq 0 ]; then
  echo "A == 0"
else
  echo "A <> 0"
fi

# double bracket, with == comparison operator
if [[ $A == 0 ]]; then
  echo "A == 0"
else
  echo "A <> 0"
fi

# single bracket, -a conditional operator
if [ $A == 0 -a $B == 1 ]; then
  echo "A == 0 and B == 1"
else
  echo "A <> 0 or B <> 1"
fi

# single bracket, && conditional operator
if [ $A == 0 ] && [ $B == 1 ]; then
  echo "A == 0 and B == 1"
else
  echo "A <> 0 or B <> 1"
fi


# double bracket, && conditional operator
if [[ $A == 0 && $B == 1 ]]; then
  echo "A == 0 and B == 1"
else
  echo "A <> 0 or B <> 1"
fi


# C is undefined, gives an error. It doesn't matter like with a String if it is quoted or not.
if [ "$C" -eq 0 ]; then
  echo "C == 0"
else
  echo "C <> 0"
fi

cut example

# cut the first 10 chars from temp.txt. Output the rest of the line 
cut -c10- temp.txt

date example

date +"%Y.%m.%d.%H.%M"

Outputs: 2020.07.28.14.53

ec2 mount script

#!/usr/bin/env bash

# show command as it is run
set -x
# exit on error
set -e

USER="$(id -u)"

if [ "0" != "$USER" ]; then
  echo "This script should be run as root. Exiting."
  exit
fi

#lsblk
CMD_OUTPUT="$(file -s /dev/nvme1n1)"

echo "$CMD_OUTPUT"

if [ "/dev/nvme1n1: data" = "$CMD_OUTPUT" ]; then
  echo "No file system present on /dev/nvme1n1."
  echo "Creating file system..."
  mkfs -t xfs /dev/nvme1n1
fi
#sudo mkdir /data
mount /dev/nvme1n1 /data
mkdir /data/work
chown ubuntu:ubuntu /data/work

egrep example

egrep -e '\$[{]|\$[[:upper:]]' *.sh

# finds bash variables that start with $VAR_NAME or ${VAR_NAME}

# -A option specifies number of trailing lines to print after match is found

grep -A 2 'string to match' <file to grep>

find example

# find java and xml files, exclude .idea directory (-prune -o ...), copy to directory (.../Desktop/project/example), preserve sub-directories (--parent)
# change to the directory containing pom.xml

find . -name \.idea -prune -o \( -name "*.java" -o -name "*.xml" \) -exec cp --parents {} /c/Users/Dixson/Desktop/project/example \;


# find files that don't have an extension; useful when shell script files don't have an extension
find . -type f ! -name "*.*"

find . -type f ! -name "*.*" | grep bin | grep -v tools | xargs chmod +x

# find file in a jar file
for i in `find . -type f -name "*.jar"`; do
  echo $i;
  jar tvf $i | grep MyClassName;
done

# find file in a jar file, only print matches
for i in `find . -type f -name "*.jar"`; do
  GREP_RESULT="$(jar tvf $i | grep MyClassName)";
  if [ ! -z "$GREP_RESULT" ]; then
    echo $i;
    echo $GREP_RESULT;
  fi
done

find prune example

#!/usr/bin/env bash

# find -prune example with multiple directories
set -x

START_DIR="/home/dixson/work/src/16.2.1"

# this seems to be used as a string match to what is output by find . -print
# if a directory other that . is used, it needs to be pre-pended to the path
PATH1="./xap-premium-16.2.1/xap-tests"
PATH2="./xap-premium-16.2.1/xap-tools"
PATH3="./xap-premium-16.2.1/xap-extensions"
PATH4="./xap-16.2.1/xap-extensions"
PATH5="./xap-premium-16.2.1/xap-examples"
PATH6="./xap-premium-16.2.1/petclinic-jpa"
PATH7="./xap-premium-16.2.1/insightedge-extensions"
PATH8="./xap-16.2.1/xap-core/xap-datagrid/src/test"

PWD="`(pwd)`"

cd $START_DIR

find . \( -path $PATH1 -o \
          -path $PATH2 -o \
          -path $PATH3 -o \
          -path $PATH4 -o \
          -path $PATH5 -o \
          -path $PATH6 -o \
          -path $PATH7 -o \
          -path $PATH8    \
                \) -prune \
                  -o -type f -name "*.java" -exec egrep -e "\<100\>" {} \; -print > /home/dixson/tmp.txt

cd $PWD

find with timestamp example

touch -t 2401280000 after.txt
         ^ year
           ^ month
             ^ day
               ^ min
                 ^ sec

# find files after 1/28
find . -newer after.txt -print

touch -t 2401310000 before.txt

# find files after 1/28 and before 1/31
find / \( -newer after.txt -a ! -newer before.txt \) -ls

get list of filename extensions

for i in `find . -type f`; do filename=$(basename -- $i); extension="${filename##*.}"; echo $extension; done | sort | uniq

$(basename -- $filename) - remove path
${filename##*.} - use bash string manipulation to get extension

get the current directory

# get the current directory where the script being run is located

BIN_DIR="`dirname \"$0\"`"
BIN_DIR="`( cd \"$BIN_DIR\" && pwd )`"

gzip and tar old style

gunzip tar in one command

gunzip -c foo.tar.gz | tar xvf -
gunzip  < foo.tar.gz | tar xvf -


zipping:

tar cvf - foodir | gzip > foo.tar.gz

loop example

#!/bin/bash

PID=$1

LOG_DIR=/tmp
NUM_MINUTES=2
NUM_PER_MIN=12

NUM_TIMES=$(expr ${NUM_MINUTES} \* ${NUM_PER_MIN})

echo "This script will run ${NUM_TIMES} time(s).";

COUNTER=0
while [ ${COUNTER} -lt ${NUM_TIMES} ]
do
  TIMESTAMP=`date +'%Y%m%d-%H.%M.%S'`
  CMD="jstack -l ${PID} > ${LOG_DIR}/jstack_${PID}.${TIMESTAMP}.log"
  #echo "${CMD}"
  ${CMD}
  echo "about to sleep 5s...";
  sleep 5s
  COUNTER=$(expr $COUNTER + 1);
done

loop example preserves lines

The for loop is not designed to loop over "lines". Instead it loops over "words".

The idiomatic way to loop over lines is to use a while loop in combination with read.

Example
diff --brief -r dir1 dir2 | while read -r line; do
  echo "$line";
done


https://stackoverflow.com/questions/10748703/iterate-over-lines-instead-of-words-in-a-for-loop-of-shell-script

memory by process

ps -eo pmem,pcpu,vsize,pid,cmd | sort -k 1 -nr | head -5

poor mans multi-server ssh

# poor man's multi-server ssh

cat hosts.txt | xargs -i -P10 ssh -i <pem key name>.pem <username>@{} date

Or

#!/usr/bin/env bash

HOSTS_FILE="hosts.txt"

if [ -z "$HOSTS_FILE" ]; then
  echo "The file $HOSTS_FILE does not exist";
  exit -1;
fi

while read REMOTE_SERVER; do
  if [[ "$REMOTE_SERVER" == "#"* ]]; then
    # skip if commented out
    continue;
  fi
  echo "host is: $REMOTE_SERVER"
  scp -i <pem key name>.pem <local file to copy>.zip "<username>@$REMOTE_SERVER:/path/to/file"
  
  # needs < /dev/null because ssh reads from standard input, if missing it will consume all your remaining lines
  ssh -i <pem key name>.pem <username>@$REMOTE_SERVER "date" < /dev/null
done < "$HOSTS_FILE"

pstree example

  # let's say you have a script that starts a whole bunch of processes and you want to see all the child processes
  
  pstree -pT <pid>
    -p show pid
    -T hides threads and shows only processes

replace string in multiple files

Replace string in multiple files / global search and replace

find . -type f -exec grep -l Xmx6g {} \; | xargs sed -i~ 's/Xmx6g/Xmx1g/g'

  -type f, files only (no directories)
  -exec <command> {} \;, part of find, runs command on each matching file
  grep -l Xmx6g, the command, -l returns filename only when matches with string 'Xmx6g'
  | xargs, take this list of files and run
  sed -i~ 's/Xmx6g/Xmx1g/g',
    -i replace the string in file inline, copy old file to new filename with ~ appended to it as a backup
    's/Xmx6g/Xmx1g/g' - last g stands for global. If the match occurs more than once on a line replace it.
                        if it appears on 2 different lines, even w/o the g, the string will get replaced.

# sed alternatives - delete line with matching string
  sed -i~ '/my matching string/d'

# sed alternative - replace with matching group
  sed -i~ 's/export ADDRESS="\(.*\)"/export ADDRESS="\1:$PORT"/'

rsync example

rsync -avzru --progress /home/dixson/src /media/dixson/dest

-a archive, equivalent to -rlptgoD
-v verbose
-z compress
-r recursive
-u update (skip newer files on destination)

-rlptgoD
recursive, links, preserve permissions, preserve modification times, preserve group, preserve owner, preserve device files, preserve special files

sar examples

1. Real-time (samples and redirects to a file)
sar -u -r ALL -S -b -d -n ALL -B -w --human <time interval> <number of times> > <text file location>

Parameter meanings:
 -u all cpu
 -r ALL memory free
 -S swap space
 -b I/O activities
 -d individual block device
 -n ALL network statistics
 -B paging statistics
 -w context switch per second
 --human human readable

or
2.  Real-time using shortcut parameter that defines other parameters 
 -A equivalent to -bBdFHqSuvwWy -I SUM -I ALL -m ALL -n ALL -r ALL -u ALL -P ALL
sar -A --human <time interval> <number of times> > <text file location>

3. Historical report
 -f
 Extract  records from filename (created by the -o filename flag). The default value of the filename parameter is the current standard system activity daily data file.  If filename is a directory instead of a plain file then it is considered as the directory where the standard system activity daily data files are located. The -f option is exclusive of the -o option.

Example:
sar -A --human -f <filename>

sed example

sed example

sed 's/[0-9\.]\+\.jar/*.jar/g' jars.txt | sed 's/[0-9\.]\+GA\.jar/*.jar/g' | sed 's/[0-9\.]\+Final\.jar/*.jar/g'


 's/[0-9\.]\+\.jar/*.jar/g'
   s/XXX/YYY/g substitute globally
   [ ] character class
   0-9 any digit
   \. any period
   \+ one or more times (backslash required to escape, maybe because it is a bash special character)
   
 jars.txt will have a list of jar file names
 An example of a jar file name is log4j-1.2.17.jar.
 The sed string will replace the version so that the above will become log4j-*.jar
 Sometimes the jar file name will have the word GA or Final in the version, for example hibernate-core-5.2.18.Final.jar
 There is another sed expression that will convert the above to hibernate-core-*.jar

sed with backref

sed 's/Picture\([0-9]\+\)/newdir\/Picture\1/g'
              ^ escape
               ^ backref
                ^ any digit character class
                     ^ one or more times
                       ^ end backref
# For example replaces Picture1.png with newdir/Picture1.png

split example in bash

GS_OPTIONS_EXT="-Dcom.gs.work=/data/work"

function get_work_dir() {
  if [ ! -z "$GS_OPTIONS_EXT" ]; then

    GS_OPTIONS=$(echo $GS_OPTIONS_EXT | tr " " "\n")

    for OPTION in $GS_OPTIONS; do
      #echo "OPTION is: $OPTION"
      # find the key value pair with com.gs.work in it
      WORK_DIR_IS_SET="$(echo "$OPTION" | grep "com.gs.work")"
      if [ ! -z "$WORK_DIR_IS_SET" ]; then
        WORK_DIR=$(echo "$OPTION" | awk -F= '{print $2}');
        echo "$WORK_DIR";
        return
      fi
    done
  fi  
  echo "$GS_HOME/work"
}

WORK_DIR=$(get_work_dir)
echo "work directory is: $WORK_DIR"

split file example

split -b 50000000 log.txt log
  # -b bytes
  # log.txt file to split
  # log - the prefix to give the new file names
  
  
 # reconstitute
 cat prefix* > myfile.zip

ss --kill example

sudo ss -pn state listening src :6090 --kill
         ^ show pid
          ^ numeric (don't replace ip addresses, don't replace known ports with names)
            ^ listening ports
                            ^ listening on 6090
                                      ^ kill this port

ulimit bash built-in

Sets the hard and soft limits for -u (max number of processes) to 65536. Once the hard limit is set, a regular user can't increase beyond it. So running ulimit -Hu 70000 would now fail.

ulimit -Hu, ulimit -Su, ulimit -u

-H hard limit
-S soft limit
If -H or -S is not given soft limit is shown. The soft limit is the actual limit. The hard limit is what the soft limit may be increased to. The soft limit may not exceed the hard limit.

ulimit -a (shows all the limits, since -H nor -S is given, ulimit -Sa is shown)

ulimit -Ha (shows limit description, and the flag that is used when checking individually. For example -u is max user process, -n open files.)
core file size          (blocks, -c) unlimited
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 31175
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 32678
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) unlimited
cpu time               (seconds, -t) unlimited
max user processes              (-u) 65536
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

ulimit -n(ulimit -Sn) soft limit for open files
ulimit -u(ulimit -Su) soft limit for max user processes, for example ulimit -u 65536 (as non-root user)

Set ulimits in /etc/security/limits.conf

username soft    nofile          32768
username hard    nofile          32768

Set the system wide maximum number of open files on in /etc/sysctl.conf

fs.file-max = 300000

In ubuntu to enable core dumps, modify /etc/security/limits.conf, add following line (ubuntu is also the login id):
ubuntu - core unlimited

unzip and change destination directory

unzip zipfile.zip -d /location/where/contents/should/be/unzipped

xargs with mv

ls -rt | tail -4 | xargs -I {} mv {} /path/to/dest/

# take the 4 most recent files and mv to destination directory