Last active
October 23, 2018 23:01
-
-
Save mokanfar/117a040b8f1ccb52cae1bd5316a02d30 to your computer and use it in GitHub Desktop.
Matched Rows, Unique Rows Merge Filter Bash Scripts for CSV files
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
echo "filename 1 (file that has more rows):" | |
read file1 | |
echo "filename 2:" | |
read file2 | |
echo "common field name:" | |
read fieldname | |
#echo "file1: ${file1}.csv file2: ${file2}.csv common field name: ${fieldname}" | |
if [ -f $PWD/temp.db ]; then | |
rm $PWD/temp.db | |
fi | |
if [ -f $PWD/missing.csv ]; then | |
rm $PWD/missing.csv | |
fi | |
csvsql --db sqlite:///temp.db --insert ${file1}.csv ${file2}.csv | |
sql2csv --db sqlite:///temp.db --query "select ${file1}.${fieldname} from ${file1} LEFT JOIN ${file2} ON ${file1}.${fieldname}= ${file2}.${fieldname} WHERE ${file2}.${fieldname} is NULL" > $PWD/missing.csv | |
csvlook $PWD/missing.csv | less -S | |
exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
echo "filename 1 (file that you want matched up against):" | |
read file1 | |
echo "filename 2:" | |
read file2 | |
echo "field name in file 1 to match:" | |
read fieldname1 | |
echo "field name in file 2 to match:" | |
read fieldname2 | |
#echo "file1: ${file1}.csv file2: ${file2}.csv common field name: ${fieldname}" | |
if [ -f $PWD/temp.db ]; then | |
rm $PWD/temp.db | |
fi | |
if [ -f $PWD/matched.csv ]; then | |
rm $PWD/matched.csv | |
fi | |
csvsql --db sqlite:///temp.db --insert ${file1}.csv ${file2}.csv | |
#merge ish | |
sql2csv --db sqlite:///temp.db --query "SELECT ${file1}.*, ${file2}.* FROM ${file1} LEFT JOIN ${file2} ON ${file1}.${fieldname1} = ${file2}.${fieldname2};" > $PWD/matched.csv | |
csvlook $PWD/matched.csv | less -S | |
exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment