scrape621

get_furs.py

import os
import sys
import pandas as pd
if not os.path.exists("floofs"):
  os.mkdir("floofs")

d = pd.read_csv("e621_"+sys.argv[1]+".csv")
d=d[d["rating"]=="s"]

import urllib.request
import time

opener = urllib.request.build_opener()
opener.addheaders = [('User-agent', 'FurGAN/1.0 (by guillefix on e621)')]
urllib.request.install_opener(opener)

t0 = time.time()
for j,(id,sample_url,file_ext) in d[["id","sample_url","file_ext"]].iterrows():
  # print(id,sample_url)
  print(j)
  try:
    urllib.request.urlretrieve(sample_url,"floofs/"+str(id)+"."+file_ext)
  except:
    continue
# d.columns
t1 = time.time()
print(t1-t0)

apt update
apt install python3-pip
wget https://gist.githubusercontent.com/guillefix/da2d848f8388014dfbc39cdfd5468204/raw/422f8e2de2428a5777b5fc080bb68a9baf0852e9/get_furs.py
pip3 install pandas