linxlunx · July 13, 2020 12:33
diff --git a/add_column_and_remove_duplicate.sh b/add_column_and_remove_duplicate.sh
 # add to first column for matching purpose (lower char), $3 is key to be matched
 cat first.csv | awk -F"," 'BEGIN { OFS = "," }; {$1=tolower($3) OFS $1; print}' > firstWithLow.csv

 # remove match line by first column
 awk -F"," 'NR==FNR{a[$1]++;next} !(a[$1])' exclude_list.csv master_file.csv > outfile.csv

 # remove first column
 cat outfile.csv | cut -d"," -f2- > outfileclean.csv
	# add to first column for matching purpose (lower char), $3 is key to be matched
	cat first.csv \| awk -F"," 'BEGIN { OFS = "," }; {$1=tolower($3) OFS $1; print}' > firstWithLow.csv

	# remove match line by first column
	awk -F"," 'NR==FNR{a[$1]++;next} !(a[$1])' exclude_list.csv master_file.csv > outfile.csv

	# remove first column
	cat outfile.csv \| cut -d"," -f2- > outfileclean.csv