r2e-dedup.sh

gistfile1.txt

#!/bin/bash
# simple bash script to remove duplicated RSS items from rss2email

r2e list > r2e-list.txt
cat r2e-list.txt |cut -d'(' -f2 |awk '{print $1}' |sort |uniq -c | grep '2 http' > a
while read line; do cat r2e-list.txt |grep `echo $line |cut -d/ -f3` | tail -n1; done < a > b
cat b |cut -d: -f1|sort |uniq | xargs r2e delete
rm -i a b r2e-list.txt