Basic unix

Basic unix.sh

# Resources:
# https://mally.stanford.edu/~sr/computing/basic-unix.html
# https://mally.stanford.edu/~sr/computing/more-unix.html
# https://devhints.io/bash


###############
# Console use #
###############

# Get help manual on a specific command
man commandname  # nice formatting, pager
commandname --help  # printed directly to console, no pager

# History - Re-run commands
history  # numbered list of recent commands 
history | tail -n 10
!12
!head  # re-run most recent head command
sudo !!  # re-run previous cmd with sudo added before (e.g. touch file.txt -> sudo touch file.txt)
!! -l  # re-run previous cmd with -l added after (e.g. ls -> ls -l)

# Writing to file > - Redirect console output to a file with >
head -n 5 file.csv > top_5_lines.txt

# Piping | - Redirect output to the next command with |
head -n 5 file.csv | tail -n 2  # lines 4 and 5
head -n 100 file.csv | sort
cut -d , -f 1 seasonal/spring.csv | grep -v Date | head -n 10  # col 1, remove line containing 'Date', top 10 lines

# Running commands on multiple input files
head -n 5 file1.csv file2.csv file3.csv
head -n 5 *.csv  # all files ending .csv
head -n 5 seasonal_data/*  # all files in seasonal_data directory
head -n 5 file?.csv  # file then any one char then .csv
head -n 5 file[123].csv  # file1.csv, file2.csv, file3.csv

# Save output of a command to a local variable
list_of_files=$(ls)

# Loops
for file in my/folder/path/*.py; do /user/bin/python3 $file; done  # Run each .py script in folder
for filetype in gif jpg png; do echo $filetype; done
for filename in seasonal/*.csv; do echo $filename; done
for file in seasonal/*.csv; do head -n 2 $file | tail -n 1; done
for f in seasonal/*.csv; do echo $f; head -n 2 $f | tail -n 1; done
for filename in seasonal_data/*
    do
        head -n 2 $filename | tail -n 1
        tail -n 1 $filename
    done
for file in $list_of_files; do name='"""Generate schema for '"${file%%.*}"'."""'; sed -i "1s/.*/$name/" $file; done


#########
# Files #
#########

# (Concatenate and) send output to console or file (uncompressed)
cat file1.txt file2.txt
cat file1.txt file2.txt >> file3.txt

# Display file content (even if compressed)
text

# Display top n lines
head file.txt -n 5

# Display tail n lines
tail file.txt -n 5

# Change file permissions
chgrp
chmod
chown

# List files/directories...
# -R recursive, -r reverse chronological, -l long (detailed), -a all (incl hidden), -h human readable file sizes
ls
ls -R

# Count files in folder
ls | wc -l

# View file content (space for more, q to exit pager view)
more
cd

# nano: ctr-o save, ctrl-x exit, ctrl-k delete line, ctrl-u undelete line
nano filename.txt  # Edit filename.txt, create it if does not exist

# Move and/or rename one file file (cut)
mv filename1 filename2  # Same directory, new name
mv filename1 ../filename1  # Up one level, same name

# Move and rename multiple files, replacing a prefix 'original' with 'new'
for x in enc_eng_payerti*; do
	echo \'"${x}"\' ' --> ' \'new"${x#'original'}"\'
	mv -- "${x}" new"${x#'original'}"
done

# Move multiple files to a directory
mv filename1 filename2 directory_name

# Move files in a folder up a level
mv -v /home/hawkem/projects/python_files/*.py /home/hawkem/projects
mv -v /home/hawkem/projects/python_files/*.py ..  # Not 100% sure this one works as expected

# Rename a directory
mv directory_name new_directory_name

# Move a directory (.../Documents --> .../backups/Documents)
mv documents backups/

# Copy one file (copy-paste)
cp filename1 filename2

# Copy multiple files to a directory
cp filename1 filename2 directory_name

# Remove... -i invokes confirmation dialogue
rm -i filename

# Diff
diff filename1 filename2

# Stats: lines (-l), words (-w), chars (-c)
wc filename
wc -c filename

# chmod... e.g. o+r for readable by all, o-r for unreadable by others
chmod options filename

# compress, decompress, view compressed
gzip filename  # zip
unzip filename
gunzip filename  # unzip
gunzip -c filename  # preview content

# Statistics info for a file (block size, number of blocks, file type, etc)
stat

# Check size of file(s)?
du filename

# Add new line to end of file
echo 'fs.inotify.max_user_watches=500000' >> /etc/sysctl.conf

# Add command output to end of file
ls>> files.txt

# Print file content (cat has no pager, less has a pager)
cat /etc/sysctl.conf
less /etc/sysctl.conf

# View the top of a file
head
head -n 5

# View the tail of a file
tail
tail -n 5

# View select columns from a file (cannot cope with commas in strings)
cut -f 2-5,8 -d , values.csv  # select cols 2-5 and 8, using comma as the separator

###############
# Directories #
###############

# Make directory
mkdir dirname

# Remove directory
rmdir

# Change directory (defaults to $HOME with nothing specified)
cd dirname
cd ..
cd

# Print working directory (current location)
pwd

# Check directory size?
du -s

# List directory content - R (recursive), l (long/details), t (chronologically), a (all, incl hidden)
ls
ls -latR

##########
# Search #
##########

# Find file... -p for partial name
# ff filename
find . -name "pattern" -print

# Find string in file (returns entire line containing the string)
# Flags: -c (count), -h (omit file names), -i (ignore case), -l (print file names only)
# -n (print line numbers), -v (show non-matches)
grep string filename

#######################
# About self, system #
######################

# Username
whoami

# Processes
ps -u username
kill PID

# Current date
date
cal
cal 2024

# system info
uname

# Storage use and availability
df
df -h

#############
# Variables #
#############

# Show value of an env variable
echo $ENVIRONMENT_VARIABLE_NAME
echo $TERM
echo $

# Set value of an env variable
setenv ENVIRONMENT_VARIABLE_NAME value

# Unset an env variable
unset ENVIRONMENT_VARIABLE_NAME


#############
# Functions #
#############

function hello() {
   echo "Hello, $1!"
}

function checktimes {
	echo "Checking recent start and end times, please wait..."
    powershell -Command "& { Invoke-Expression (Get-Content -Raw '//NDATA12/hawkem$/My Documents/WindowsPowerShell/checktimes.txt') }"
}

hello world  # "Hello, world!" - note that this only acknowledges one argument
hello one two  # "Hello, one!" - the 2nd parameter 'two' is not used

checktimes  # Runs PS script that checks recent 'Get-EventLog system' start and end times


#######
# ssh #
#######

# Activate ssh-agent
eval `ssh-agent -s`  # or just ssh-agent -s ?

# Create a new id_ed25519 key
ssh-add ~/.ssh/id_ed25519

# Remove a host from the known hosts file, such as if the key changes
ssh-keygen -R <host>


########
# Misc #
########

# Sort: -n numerically, -r reverse, -b ignore leading blanks, -f case insensitive
head -n 100 file.csv | sort -f

# Adjacent uniqueness (remove adjacent duplicates): -c (count)
head -n 100 file.csv | sort -f | uniq

# Create temporary local variable (NB: no spaces around equals signs)
path_csv_files=path/to/*.csv

# Create a global environment variable
export VAR=value
export MY_TEMP_NEW_ENV_VAR_123=3.14

# Download using curl (wget is an alternative)
curl -O -u <username> -X GET "https://bbc.co.uk/artifactory/LR_vscode/v1_63_2_Nov_2021/vscode-server-linux-x64.tar.gz"

# Backspace --- delete previous character
# CTRL-d --- delete next character
# CTRL-k --- delete rest of line
# CTRL-a --- go to start of line
# CTRL-e --- go to end of line
# CTRL-b --- go backwards without deleting
# CTRL-f --- go forward without deleting

# TAB --- complete filename or command up to the point of uniqueness
# CTRL-u --- cancel whole line
# CTRL-c --- cancel the processes after it has started
# CTRL-z --- suspend a running process (e.g. in order to do something else in between), you can then put the process in the background with bg
# CTRL-l --- redraws the screen
# | (piping) --- Lets you execute any number of commands in a sequence.
#   The second command will be executed once the first is done, and so forth, using the previous command's output as input. You can achieve the same effect by putting the output in a file and giving the filename as an argument to the second command, but that would be much more complicated, and you'd have to remember to remove all the junkfiles afterwards. Some examples that show the usefulness of this:
#   ls | more --- will show you one screenful at a time, which is useful with any command that will produce a lot of output, e.g. also ps -aux
#   man ls | grep time --- checks whether the man page for ls has something to say about listing files by time - very useful when you have a suspicion some command may be capable of doing what you want, but you aren't sure.
#   ls -lR | grep dvi --- will show you all your dvi files - useful to solve disk space problems, since they're large and usually can be deleted.

HDFS.sh

# HDFS file system accepts around 30 commands, including many Unix commands.
# May or may not need to prefix with "hdfs", depending on environment

# Copy file(s) from one place to another
# -f to force (overwrite if already present)
hdfs put  # HDFS
hdfs get  # HDFS
hdfs copyFromLocal  # HDFS
hdfs copyToLocal  # HDFS

# Move files
hdfs moveFromLocal  # HDFS
hdfs moveToLocal  # HDFS

# "dfs -" to interact with file system
hdfs dfs -ls
hdfs dfs -ls toy_data/toy_dataset_1
hdfs dfs- mkdir folder/path/here
hdfs dfs- rmdir folder/path/here
hdfs dfs -cp /hadoop/file1 /hadoop1
hdfs dfs -cp -f /hadoop/file1 /hadoop1
hdfs dfs -rm -r /user/the/path/to/your/dir
hadoop fs -rm -r -f /user/the/path/to/your/dir
hadoop fs -mv oldname newname

# View Hadoop(?) environment variables
hdfs envvars

#
hdfs getconf -confKey dfs.blocksize
hdfs fsck –locations –files -blocks

YARN.sh

# May or may not need to prefix with "yarn", depending on environment

# Command to configure and view information about the YARN cluster
yarn command

# List running applications
yarn application -list

# Kill a running application
yarn application -kill app-id

# View the logs of the specified application
yarn logs -applicationId app-id

# View the full list of command options
yarn -help

# View Yarn(?) environment variables
yarn envvars

<title>List of common Linux commands that can be used in the CDSW terminal</title> <style> </style>

Is	Displays information about files in the current directory.
pwd	Displays the current working directory.
mkdir	Creates a directory.
cd	To navigate between different folders.
rmdir	Removes empty directories from the directory lists.
cp	Moves files from one directory to another.
mv	Rename and Replace the files
rm	Delete files
uname	Command to get basic information about the OS
touch	Create empty files
cat	Display file contents on terminal
clear	Clear terminal
ps	Display the processes in terminal
man	Access manual for all Linux commands
grep	Search for a specific string in an output
echo	Display active processes on the terminal
whoami	Create or update passwords for existing users
sort	sort the file content
df	Check the details of the file system
wc	Check the lines, word count, and characters in a file using different options
uniq	Must be passed a sorted list to work, but removes duplicates
less	Display a file, scroll through using the arrow keys
time	Display time taken for process to execute

cat {file_name} | md5sum

Try graceful termination first

ps uxa | grep .vscode-server | awk '{print $2}' | xargs kill

Wait a few seconds

sleep 5

Force kill any remaining processes

ps uxa | grep .vscode-server | awk '{print $2}' | xargs kill -9