Terminal General Reference

bash_cheatsheet.md

Terminal Cheat Sheet

• Bash Beginners Guide
• Linux Info - Commands
• bash args/opts
• colors
• print table utils

File System

filename expansion

FILE="example.tar.gz"
echo "${FILE%%.*}"
# example
echo "${FILE%.*}"
# example.tar
echo "${FILE#*.}"
# tar.gz
echo "${FILE##*.}"
# gz

dirname /path/to/dir/filename.txt
# /path/to/dir
basename /path/to/dir/filename.txt
# filename.txt
basename /path/to/dir/filename.txt .txt
# filename

find

Recursive walk - print all files

find * -type [f|d] -maxdepth 3 -mindepth 1 -name "foo.txt"

List all folders sorted by age

• find man page
• stat man page

ROOT=/workspaces/ServerData/cache/local;
find $ROOT/* -maxdepth 1 -mtime +30 -user $USER -exec stat --format "%x %n" '{}' \; | sort

Iterate over piped results

find * -type f | while read FILE; do
  echo $FILE;
  # do something with file...
done;

If

If variable is defined

see stackoverflow To test if a variable named var is defined:

if [ -z ${var+x} ]; then echo "var is unset"; else echo "var is set to '$var'"; fi

Value of $foo will default to "bar" if $1 is not already defined: foo=${1:-"bar"}

If path is a directory

if ! [ -d $HOME/foo ]; then echo "foo doesnt exist"; fi

If compare numbers

if [ $mod_file_count -eq 0 ]
then
    echo "mod_file_count = 0";
    return 0;
else
    echo "mod_file_count != 0";
    exit 1;
fi

Text Processing

yq Cookbook

cut examples

echo "col1,col2,col3" | cut -d',' -f2
# col2

awk

Column names

FILE=<filename>
head -n 1 $FILE | awk 'BEGIN { FS="|" } { for(i = 1; i <= NF; i++) { printf("%d  %s\n", i, $i) } } '

Column sample

FILE=path/to/pipedelimited.txt
COL_POS=1
head -n 10 $FILE | awk -v COL=$COL_POS 'BEGIN { FS="|" } { printf("%s\n", $COL) }'

Column value histogram

FILE=path/to/pipedelimited.txt
COL_POS=1
cat $FILE | awk -v COL=$COL_POS 'BEGIN { FS="|" } { printf("%s", $COL) }' | sort -n | uniq -c

Many Files (search)

COL_POS=1
find * -type f | while read FILE; do
  echo $FILE;
  cat $FILE | awk -v COL=$COL_POS 'BEGIN { FS="|" } { printf("%s\n", $COL) }'
done;

variables

reference

• FS: Field Seperator
• NF: Number of fields in current record
• NR: Current record number (overall line number)
• FNR: Current record number (file line number)
• $0: Entire current record
• $n: nth field in current record
• FILENAME: current filename
• ENVIRON: enviroment variables (eg ENVIRON["USER"])

sed

options

• -n|--quiet|--silent only print a line if explicitly requests with p
• -i|--inplace edit the file inplace

column pretty print

Pretty Print

cat somefile.txt | column -t -s,

Capture Results

see here

TASK_DESC=$(jira_desc $TASK_ID 2>&1)
: ${ACTIVATOR_BIN:="$(which activator)"}

Arrays

see here

Syntax	Result
arr=()	Create an empty array
arr=(1 2 3)	Initialize array
${arr[2]}	Retrieve third element
${arr[@]}	Retrieve all elements
${!arr[@]}	Retrieve array indices
${#arr[@]}	Calculate array size
arr[0]=3	Overwrite 1st element
unset arr[2]	Delete 3rd element
arr+=(4)	Append value(s)
str=$(ls)	Save ls output as a string
arr=( $(ls) )	Save ls output as an array of files
${arr[@]:s:n}	Retrieve n elements starting at index s

for ELEMENT in ${ARRAY[@]}; do
  printf "%s\n" $ELEMENT;
done;

Networking

List Used Ports

netstat -p tcp

Docker

mount current folder

docker run -it --mount "type=bind,source=$(pwd),destination=/foo" --user $(id -u):$(id -g) --workdir="/foo" ubuntu bash

commands.md

GIT

git-extras (cheatsheet])

also see: https://devhints.io/git-extras

• git pr 9546 upstream check out github PR-9546
• git back $num_commits un-commit changes from last $num_commits - files are still staged
• git back $num_commits remove/delete changes from last $num_commits - changes lost
• git ignore "*.log" add .gitignore entry
• git obliterate secret.yml remove all references to secret.yml

Inspection

• git summary repo age, commits, active days, etc
• git impact impact graph
• git effort commits per file

Branch

• git log --all --graph --decorate --oneline --simplify-by-decoration
• git branch -av get all branches (including from remotes)

tar

• compress tar cvzf new-gzipped-tar-file.tar.gz /path/to/some/input/folder
• list tar -tvf gzipped-tar-file.tar.gz
• extract tar -xcf gzipped-tar-file.tar.gz -C /path/to/some/output/folder

args

• c create
• v verbose
• z gzip (or j for bz2)
• f filename type of archive
• t list content
• x extract

csv-colnames.sh

#! /usr/bin/env bash

usage() { echo "Usage: $0 -d <\",\",\"|\",\"\\t\"> [-f <filename>]" 1>&2; exit 1; }

function column_list() {
  DELIMITER=$1
  FILE=$2
  head -n 1 $FILE | awk -v DELIMITER=$DELIMITER 'BEGIN { FS=DELIMITER } { for(i = 1; i <= NF; i++) { printf("%d  %s\n", i, $i) } } '
}

while getopts ":d:f:" ARGS; do
    case "${ARGS}" in
        d)
            d=${OPTARG}
            # ((d == "," || d == "\t")) || usage
            ;;
        f)
            f=${OPTARG}
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))

if [ -z "${d}" ]; then
    usage
fi

if [ -z "${f}" ]; then
    find * -type f | while read FILE; do
        printf "\n${FILE}\n";
        column_list $d $FILE;
    done;
else
    column_list $d $f;
fi
# echo "d = ${d}"
# echo "f = ${f}"

csv.md

tools

• SQL on csv with q
• csvkit
• xsv

columns

include columns 3,2,5

cut -d',' -f3,2,5

rows

inclusion

include first N rows

head -n N

include last N rows

tail -n N

include rows N to M

sed -n N,Mp somefile.txt

include rows containing

sed -n '/\(foo\|bar\)/!p'

exclusion

exclude first N rows

tail -n +<N+1>

exclude rows containing 'foo' or 'bar'

sed -n '/\(foo\|bar\)/!p'

complete example

cat some_file.csv | tail -n +2 | cut -d',' -f5,6,9,10 | sort | uniq | sed -n '/\(2020\|2021\)/p' | tail -n 1
#  file to stdout | skip line  | choose columns       |      |      | filter                     | last line

loop test

#!/bin/bash
# tolerate failure
set +e

while true; do
    # /app/bin/test
    echo "press enter to run tests or q to quit"
    ([[ "$0" == "/app/bin/loop" ]] && read -r -n 1 OUT && [[ ${OUT} != 'q' ]]) || exit
done

parallel.sh

#!/usr/bin/env bash

function foo() {
    echo "sleeping for $1 seconds"
    sleep $1
}

function main() {
    local PIDS=()
    local n=5
    # loop through the directories
    for i in {1..5}; do
        foo $(( n - i )) &
        PIDS+=($!)
    done
    echo "waiting for pids: ${PIDS[@]}"
    wait "${PIDS[@]}"
}

main

script-template.sh

#! /usr/bin/env bash

usage() { echo "Usage: $0 [-s <45|90>] [-p <string>]" 1>&2; exit 1; }

function handle_args() {
	# https://stackoverflow.com/a/16496491
	while getopts ":d:f:" ARGS; do
		case "${ARGS}" in
		d)
			d=${OPTARG}
			# ((d == "," || d == "\t")) || usage
			;;
		f)
			f=${OPTARG}
			;;
		*)
			usage
			;;
		esac
	done
	shift $((OPTIND - 1))

	if [ -z "${d}" ]; then
		usage
	fi

	if [ -z "${f}" ]; then
		find * -type f | while read FILE; do
			printf "\n${FILE}\n"
			column_list $d $FILE
		done
	else
		column_list $d $f
	fi

}
# check stdin for piped in data
if [ -p /dev/stdin ]; then
	echo "Data was piped to this script!"
	# If we want to read the input line by line
	while IFS= read line; do
		echo "Line: ${line}"
	done
	# Or if we want to simply grab all the data, we can simply use cat instead
	# cat
else
	echo "No input was found on stdin, skipping!"
	# Checking to ensure a filename was specified and that it exists
	if [ -f "$1" ]; then
		echo "Filename specified: ${1}"
		echo "Doing things now.."
	else
		echo "No input given!"
	fi
fi

search.sh

#!/usr/bin/env bash

# also try: https://stackoverflow.com/a/47420001
# grep -rl --include="*.js" "searchString" ${PWD}

function help_message {
    echo "Usage: $(basename $0) [FOLDER_PATH] QUERY"
    echo
    echo "-h | --help              display this message"
}

function search {
    root="$1"
    search="$2"
    declare -i count=0
    # see sub-shell https://stackoverflow.com/a/27175062/5154695
    find $root -type f | (
        while read FILE
        do
            result=$(grep -E "${search}" $FILE 2>&1)
            if [ $? -eq 0 ]; then
                echo $FILE
                echo $result
                count=${count}+1
            fi
        done &&
        if [ ${count} -eq 0 ]
        then
            echo "'${search}' not found";
        fi
    )
}

if [ $# -eq 1 ]; then
    ROOT=$PWD
    QUERY=$1
    search "$ROOT" "$QUERY"
elif [ $# -eq 2 ]; then
    ROOT=$1
    QUERY=$2
    search "$ROOT" "$QUERY"
else
    help_message
fi

space-seperated-args.sh

#!/bin/bash

# https://stackoverflow.com/a/14203146

# Usage: $0 -e conf -s /etc /etc/hosts

POSITIONAL_ARGS=()

while [[ $# -gt 0 ]]; do
  case $1 in
    -e|--extension)
      EXTENSION="$2"
      shift # past argument
      shift # past value
      ;;
    -s|--searchpath)
      SEARCHPATH="$2"
      shift # past argument
      shift # past value
      ;;
    --default)
      DEFAULT=YES
      shift # past argument
      ;;
    -*|--*)
      echo "Unknown option $1"
      exit 1
      ;;
    *)
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift # past argument
      ;;
  esac
done

set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters

echo "FILE EXTENSION  = ${EXTENSION}"
echo "SEARCH PATH     = ${SEARCHPATH}"
echo "DEFAULT         = ${DEFAULT}"
echo "Number files in SEARCH PATH with EXTENSION:" $(ls -1 "${SEARCHPATH}"/*."${EXTENSION}" | wc -l)

if [[ -n $1 ]]; then
    echo "Last line of file specified as non-opt/last argument:"
    tail -1 "$1"
fi

stopwatch.sh

#!/usr/bin/env bash

function stopwatch() {
    local BEGIN=$(date +%s)
    echo Starting Stopwatch...

    while true; do
        local NOW=$(date +%s)
        local DIFF=$(($NOW - $BEGIN))
        local MINS=$(($DIFF / 60))
        local SECS=$(($DIFF % 60))
        local HOURS=$(($DIFF / 3600))
        local DAYS=$(($DIFF / 86400))

        printf "\r%3d Days, %02d:%02d:%02d" $DAYS $HOURS $MINS $SECS
        sleep 0.5
    done
}

yes_or_no.sh

#!/usr/bin/env bash

# https://stackoverflow.com/questions/226703/how-do-i-prompt-for-yes-no-cancel-input-in-a-linux-shell-script

function yes() {
    echo "ok yes it is."
}

function no() {
    echo "ok no it's not."
}

function yes_or_no() {
    while true; do
        echo -n "Yes or No? "
        read yn
        case $yn in
            [Yy]* ) yes break;;
            [Nn]* ) no break;;
            * ) printf "Please answer Y|y or N|n.\n";;
        esac
    done
}

yes_or_no

function foo() {
  echo "foo";
}
 while sleep 3; do clear; foo; printf "current time: ${GREEN}$(date +%FT%H.%M.%S%Z)${RESET}"; done;