print_url.awk

#!/usr/bin/awk -rf
BEGIN{
    url_list="url_list"
    while (getline < url_list > 0) {
        flvname=gensub(/^(http.+)\.flv.+/, "\\1", 1, $0)
        if(flvname in flv) true
            else {
                flv[flvname]++
            }
    }
    close(url_list)

}

/^http:.+\.flv/ {
    flvname=gensub(/^(http.+)\.flv.+/, "\\1", 1, $0)
    if(flvname in flv) next
        else {
            url[$1]
            flv[flvname]++
        }
}

END{
    print strftime("%Y/%m/%d %H:%M:%S", systime()) >> url_list
    for(i in url) print i >> url_list
}