unix snippets

awk example

find . -type f -exec cksum {} \; > cksum.txt

# change the order of the columns so that the filename is the first column. Use a comma to separate the fields.
awk '{ for(i=3; i<=NF; i++) { if (i==NF) {printf "%s", $i;} else {printf "%s ",$i;}} printf ", %d, %d\n", $1, $2;}' cksum.txt

# get the list of file extensions that exist in a directory and its children
for i in `find . -type f`; do basename $i | awk -F. '{print $NF}'; done | sort | uniq

# awk example, expanded program in shell script, prints filename and owner
#!/usr/bin/env bash
# Linux users have to change $8 to $9
awk '
BEGIN	{ print "File\tOwner" }
		{ print $9, "\t", \
		$3}
END		{ print "done"} 
'
# to test: ls -l | awk_example.sh

bash color example

# I have a project folder. Inside the project folder are individual projects named proj<nn>, where nn is a two digit project number.
# In each project folder there is a README.txt with a short description of the project.

GREEN='\033[0;32m'
NC='\033[0m' # No Color

CMD=$(ls -d proj* | sed 's/proj//' | sort -n); # sort by numerical order not alphabetic order

for i in $CMD; do
  PROJ_DIR="proj$i";
  echo -e "${GREEN}$PROJ_DIR${NC}"; # display in GREEN, then turn off colors
  cat "$PROJ_DIR/README.txt";
done

bash environment variables explanation

https://tldp.org/LDP/Bash-Beginners-Guide/html/sect_03_02.html

bash string functions

# advanced bash string manipulation
http://tldp.org/LDP/abs/html/string-manipulation.html

${string##substring}
Deletes longest match of $substring from front of $string.

${string%substring}
Deletes shortest match of $substring from back of $string.

# helper function to get ip address from host name
# hostname -i
function get_ip_address() {
  IP_ADDRESS=$(hostname)
  IP_ADDRESS=${IP_ADDRESS#ip-}
  IP_ADDRESS=${IP_ADDRESS//-/.}
  echo "$IP_ADDRESS";
}


# bash to copy a zip file into a directory that uses the filename for the directory name
for i in `ls *.zip`; do echo $i; export BASENAME=`echo "${i%.*}"`; echo $BASENAME; mkdir $BASENAME; mv $i $BASENAME; done

check if env variable is set

if [ -z "${MY_VAR+xxx}" ]; then
  # do something here
fi

clear disk space

docker system prune -a

conditional.sh

#!/usr/bin/env bash
# examples using if

A=0
B=1


# single bracket, with test -eq
if [ $A -eq 0 ]; then
  echo "A == 0"
else
  echo "A <> 0"
fi

# double bracket, with == comparison operator
if [[ $A == 0 ]]; then
  echo "A == 0"
else
  echo "A <> 0"
fi

# single bracket, -a conditional operator
if [ $A == 0 -a $B == 1 ]; then
  echo "A == 0 and B == 1"
else
  echo "A <> 0 or B <> 1"
fi

# single bracket, && conditional operator
if [ $A == 0 ] && [ $B == 1 ]; then
  echo "A == 0 and B == 1"
else
  echo "A <> 0 or B <> 1"
fi


# double bracket, && conditional operator
if [[ $A == 0 && $B == 1 ]]; then
  echo "A == 0 and B == 1"
else
  echo "A <> 0 or B <> 1"
fi


# C is undefined, gives an error. It doesn't matter like with a String if it is quoted or not.
if [ "$C" -eq 0 ]; then
  echo "C == 0"
else
  echo "C <> 0"
fi

cut example

# cut the first 10 chars from temp.txt. Output the rest of the line 
cut -c10- temp.txt

date example

date +"%Y.%m.%d.%H.%M"

Outputs: 2020.07.28.14.53

ec2 mount script

#!/usr/bin/env bash

# show command as it is run
set -x
# exit on error
set -e

USER="$(id -u)"

if [ "0" != "$USER" ]; then
  echo "This script should be run as root. Exiting."
  exit
fi

#lsblk
CMD_OUTPUT="$(file -s /dev/nvme1n1)"

echo "$CMD_OUTPUT"

if [ "/dev/nvme1n1: data" = "$CMD_OUTPUT" ]; then
  echo "No file system present on /dev/nvme1n1."
  echo "Creating file system..."
  mkfs -t xfs /dev/nvme1n1
fi
#sudo mkdir /data
mount /dev/nvme1n1 /data
mkdir /data/work
chown ubuntu:ubuntu /data/work

egrep example

egrep -e '\$[{]|\$[[:upper:]]' *.sh

# finds bash variables that start with $VAR_NAME or ${VAR_NAME}

# -A option specifies number of trailing lines to print after match is found

grep -A 2 'string to match' <file to grep>

find example

# find java and xml files, exclude .idea directory (-prune -o ...), copy to directory (.../Desktop/project/example), preserve sub-directories (--parent)
# change to the directory containing pom.xml

find . -name \.idea -prune -o \( -name "*.java" -o -name "*.xml" \) -exec cp --parents {} /c/Users/Dixson/Desktop/project/example \;


# find files that don't have an extension; useful when shell script files don't have an extension
find . -type f ! -name "*.*"

find . -type f ! -name "*.*" | grep bin | grep -v tools | xargs chmod +x

# find file in a jar file
for i in `find . -type f -name "*.jar"`; do
  echo $i;
  jar tvf $i | grep MyClassName;
done

# find file in a jar file, only print matches
for i in `find . -type f -name "*.jar"`; do
  GREP_RESULT="$(jar tvf $i | grep MyClassName)";
  if [ ! -z "$GREP_RESULT" ]; then
    echo $i;
    echo $GREP_RESULT;
  fi
done

find prune example

#!/usr/bin/env bash

# find -prune example with multiple directories
set -x

START_DIR="/home/dixson/work/src/16.2.1"

# this seems to be used as a string match to what is output by find . -print
# if a directory other that . is used, it needs to be pre-pended to the path
PATH1="./xap-premium-16.2.1/xap-tests"
PATH2="./xap-premium-16.2.1/xap-tools"
PATH3="./xap-premium-16.2.1/xap-extensions"
PATH4="./xap-16.2.1/xap-extensions"
PATH5="./xap-premium-16.2.1/xap-examples"
PATH6="./xap-premium-16.2.1/petclinic-jpa"
PATH7="./xap-premium-16.2.1/insightedge-extensions"
PATH8="./xap-16.2.1/xap-core/xap-datagrid/src/test"

PWD="`(pwd)`"

cd $START_DIR

find . \( -path $PATH1 -o \
          -path $PATH2 -o \
          -path $PATH3 -o \
          -path $PATH4 -o \
          -path $PATH5 -o \
          -path $PATH6 -o \
          -path $PATH7 -o \
          -path $PATH8    \
                \) -prune \
                  -o -type f -name "*.java" -exec egrep -e "\<100\>" {} \; -print > /home/dixson/tmp.txt

cd $PWD

find with timestamp example

touch -t 2401280000 after.txt
         ^ year
           ^ month
             ^ day
               ^ min
                 ^ sec

# find files after 1/28
find . -newer after.txt -print

touch -t 2401310000 before.txt

# find files after 1/28 and before 1/31
find / \( -newer after.txt -a ! -newer before.txt \) -ls

get list of filename extensions

for i in `find . -type f`; do filename=$(basename -- $i); extension="${filename##*.}"; echo $extension; done | sort | uniq

$(basename -- $filename) - remove path
${filename##*.} - use bash string manipulation to get extension

get the current directory

# get the current directory where the script being run is located

BIN_DIR="`dirname \"$0\"`"
BIN_DIR="`( cd \"$BIN_DIR\" && pwd )`"

gzip and tar old style

gunzip tar in one command

gunzip -c foo.tar.gz | tar xvf -
gunzip  < foo.tar.gz | tar xvf -


zipping:

tar cvf - foodir | gzip > foo.tar.gz

loop example

#!/bin/bash

PID=$1

LOG_DIR=/tmp
NUM_MINUTES=2
NUM_PER_MIN=12

NUM_TIMES=$(expr ${NUM_MINUTES} \* ${NUM_PER_MIN})

echo "This script will run ${NUM_TIMES} time(s).";

COUNTER=0
while [ ${COUNTER} -lt ${NUM_TIMES} ]
do
  TIMESTAMP=`date +'%Y%m%d-%H.%M.%S'`
  CMD="jstack -l ${PID} > ${LOG_DIR}/jstack_${PID}.${TIMESTAMP}.log"
  #echo "${CMD}"
  ${CMD}
  echo "about to sleep 5s...";
  sleep 5s
  COUNTER=$(expr $COUNTER + 1);
done

loop example preserves lines

The for loop is not designed to loop over "lines". Instead it loops over "words".

The idiomatic way to loop over lines is to use a while loop in combination with read.

Example
diff --brief -r dir1 dir2 | while read -r line; do
  echo "$line";
done


https://stackoverflow.com/questions/10748703/iterate-over-lines-instead-of-words-in-a-for-loop-of-shell-script

memory by process

ps -eo pmem,pcpu,vsize,pid,cmd | sort -k 1 -nr | head -5

poor mans multi-server ssh

# poor man's multi-server ssh

cat hosts.txt | xargs -i -P10 ssh -i <pem key name>.pem <username>@{} date

Or

#!/usr/bin/env bash

HOSTS_FILE="hosts.txt"

if [ -z "$HOSTS_FILE" ]; then
  echo "The file $HOSTS_FILE does not exist";
  exit -1;
fi

while read REMOTE_SERVER; do
  if [[ "$REMOTE_SERVER" == "#"* ]]; then
    # skip if commented out
    continue;
  fi
  echo "host is: $REMOTE_SERVER"
  scp -i <pem key name>.pem <local file to copy>.zip "<username>@$REMOTE_SERVER:/path/to/file"
  
  # needs < /dev/null because ssh reads from standard input, if missing it will consume all your remaining lines
  ssh -i <pem key name>.pem <username>@$REMOTE_SERVER "date" < /dev/null
done < "$HOSTS_FILE"

pstree example

  # let's say you have a script that starts a whole bunch of processes and you want to see all the child processes
  
  pstree -pT <pid>
    -p show pid
    -T hides threads and shows only processes

replace string in multiple files

Replace string in multiple files / global search and replace

find . -type f -exec grep -l Xmx6g {} \; | xargs sed -i~ 's/Xmx6g/Xmx1g/g'

  -type f, files only (no directories)
  -exec <command> {} \;, part of find, runs command on each matching file
  grep -l Xmx6g, the command, -l returns filename only when matches with string 'Xmx6g'
  | xargs, take this list of files and run
  sed -i~ 's/Xmx6g/Xmx1g/g',
    -i replace the string in file inline, copy old file to new filename with ~ appended to it as a backup
    's/Xmx6g/Xmx1g/g' - last g stands for global. If the match occurs more than once on a line replace it.
                        if it appears on 2 different lines, even w/o the g, the string will get replaced.

# sed alternatives - delete line with matching string
  sed -i~ '/my matching string/d'

# sed alternative - replace with matching group
  sed -i~ 's/export ADDRESS="\(.*\)"/export ADDRESS="\1:$PORT"/'

rsync example

rsync -avzru --progress /home/dixson/src /media/dixson/dest

-a archive, equivalent to -rlptgoD
-v verbose
-z compress
-r recursive
-u update (skip newer files on destination)

-rlptgoD
recursive, links, preserve permissions, preserve modification times, preserve group, preserve owner, preserve device files, preserve special files

sar examples

1. Real-time (samples and redirects to a file)
sar -u -r ALL -S -b -d -n ALL -B -w --human <time interval> <number of times> > <text file location>

Parameter meanings:
 -u all cpu
 -r ALL memory free
 -S swap space
 -b I/O activities
 -d individual block device
 -n ALL network statistics
 -B paging statistics
 -w context switch per second
 --human human readable

or
2.  Real-time using shortcut parameter that defines other parameters 
 -A equivalent to -bBdFHqSuvwWy -I SUM -I ALL -m ALL -n ALL -r ALL -u ALL -P ALL
sar -A --human <time interval> <number of times> > <text file location>

3. Historical report
 -f
 Extract  records from filename (created by the -o filename flag). The default value of the filename parameter is the current standard system activity daily data file.  If filename is a directory instead of a plain file then it is considered as the directory where the standard system activity daily data files are located. The -f option is exclusive of the -o option.

Example:
sar -A --human -f <filename>

sed example

sed example

sed 's/[0-9\.]\+\.jar/*.jar/g' jars.txt | sed 's/[0-9\.]\+GA\.jar/*.jar/g' | sed 's/[0-9\.]\+Final\.jar/*.jar/g'


 's/[0-9\.]\+\.jar/*.jar/g'
   s/XXX/YYY/g substitute globally
   [ ] character class
   0-9 any digit
   \. any period
   \+ one or more times (backslash required to escape, maybe because it is a bash special character)
   
 jars.txt will have a list of jar file names
 An example of a jar file name is log4j-1.2.17.jar.
 The sed string will replace the version so that the above will become log4j-*.jar
 Sometimes the jar file name will have the word GA or Final in the version, for example hibernate-core-5.2.18.Final.jar
 There is another sed expression that will convert the above to hibernate-core-*.jar

split example in bash

GS_OPTIONS_EXT="-Dcom.gs.work=/data/work"

function get_work_dir() {
  if [ ! -z "$GS_OPTIONS_EXT" ]; then

    GS_OPTIONS=$(echo $GS_OPTIONS_EXT | tr " " "\n")

    for OPTION in $GS_OPTIONS; do
      #echo "OPTION is: $OPTION"
      # find the key value pair with com.gs.work in it
      WORK_DIR_IS_SET="$(echo "$OPTION" | grep "com.gs.work")"
      if [ ! -z "$WORK_DIR_IS_SET" ]; then
        WORK_DIR=$(echo "$OPTION" | awk -F= '{print $2}');
        echo "$WORK_DIR";
        return
      fi
    done
  fi  
  echo "$GS_HOME/work"
}

WORK_DIR=$(get_work_dir)
echo "work directory is: $WORK_DIR"

split file example

split -b 50000000 log.txt log
  # -b bytes
  # log.txt file to split
  # log - the prefix to give the new file names

unzip and change destination directory

unzip zipfile.zip -d /location/where/contents/should/be/unzipped

xargs with mv

ls -rt | tail -4 | xargs -I {} mv {} /path/to/dest/

# take the 4 most recent files and mv to destination directory