Skip to content

Instantly share code, notes, and snippets.

@mokanfar
Last active October 23, 2018 23:01
Show Gist options
  • Save mokanfar/117a040b8f1ccb52cae1bd5316a02d30 to your computer and use it in GitHub Desktop.
Save mokanfar/117a040b8f1ccb52cae1bd5316a02d30 to your computer and use it in GitHub Desktop.
Matched Rows, Unique Rows Merge Filter Bash Scripts for CSV files
#!/usr/bin/env bash
echo "filename 1 (file that has more rows):"
read file1
echo "filename 2:"
read file2
echo "common field name:"
read fieldname
#echo "file1: ${file1}.csv file2: ${file2}.csv common field name: ${fieldname}"
if [ -f $PWD/temp.db ]; then
rm $PWD/temp.db
fi
if [ -f $PWD/missing.csv ]; then
rm $PWD/missing.csv
fi
csvsql --db sqlite:///temp.db --insert ${file1}.csv ${file2}.csv
sql2csv --db sqlite:///temp.db --query "select ${file1}.${fieldname} from ${file1} LEFT JOIN ${file2} ON ${file1}.${fieldname}= ${file2}.${fieldname} WHERE ${file2}.${fieldname} is NULL" > $PWD/missing.csv
csvlook $PWD/missing.csv | less -S
exit 0
#!/usr/bin/env bash
echo "filename 1 (file that you want matched up against):"
read file1
echo "filename 2:"
read file2
echo "field name in file 1 to match:"
read fieldname1
echo "field name in file 2 to match:"
read fieldname2
#echo "file1: ${file1}.csv file2: ${file2}.csv common field name: ${fieldname}"
if [ -f $PWD/temp.db ]; then
rm $PWD/temp.db
fi
if [ -f $PWD/matched.csv ]; then
rm $PWD/matched.csv
fi
csvsql --db sqlite:///temp.db --insert ${file1}.csv ${file2}.csv
#merge ish
sql2csv --db sqlite:///temp.db --query "SELECT ${file1}.*, ${file2}.* FROM ${file1} LEFT JOIN ${file2} ON ${file1}.${fieldname1} = ${file2}.${fieldname2};" > $PWD/matched.csv
csvlook $PWD/matched.csv | less -S
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment