download all your twitter anime girls!

dedupe.py

import os
from collections import defaultdict
import re
import shutil

from PIL import Image
import imagehash

fmt = re.compile(r"(.*)\_(\d+)_(\d+)")

path = os.path.expanduser("/run/media/sanspapyrus683/PHILIPS/twitter")

os.chdir(path)

tweets = defaultdict(lambda: defaultdict(list))
for i in os.listdir():
    if i.endswith(".mp4"):
        continue

    match = fmt.match(i)
    author = match.group(1)
    id_ = int(match.group(2))
    pos = int(match.group(3))
    tweets[author][id_].append((pos, i, imagehash.average_hash(Image.open(i))))

to_del = []
duped = []
for author, g in tweets.items():
    ids = sorted(g.keys())
    for i in ids:
        g[i].sort()

    for i in range(len(ids)):
        g1 = g[ids[i]]
        for j in range(i + 1, len(ids)):
            g2 = g[ids[j]]

            if len(g1) != len(g2):
                continue

            for a, b in zip(g1, g2):
                if a[2] - b[2] >= 3:
                    break
            else:
                to_del.extend(g1)
                duped.append((g1, g2))
                break

"""
for v, (g1, g2) in enumerate(duped):
    for u, (a, b) in enumerate(zip(g1, g2)):
        ext1 = os.path.splitext(a[1])[1]
        ext2 = os.path.splitext(b[1])[1]
        shutil.copy(a[1], f"../asdf/{v}_{u}_0.{ext1}")
        shutil.copy(b[1], f"../asdf/{v}_{u}_1.{ext2}")
"""

for d in to_del:
    os.remove(d[1])

migrate.py

import sys
import os
import re

migrate_dir = "/run/media/sanspapyrus683/PHILIPS/twitter"
fmt = re.compile(r"(.*)\_(\d+)_(\d+)")

old_handle = sys.argv[1].lower()
new_handle = sys.argv[2]
print(f"{old_handle} -> {new_handle}. yes?")
if input().lower() not in ["y", "yes"]:
    sys.exit()

os.chdir(migrate_dir)
to_rename = []
for i in os.listdir():
    match = fmt.match(i)
    author = match.group(1)
    id_ = int(match.group(2))
    pos = int(match.group(3))
    ext = os.path.splitext(i)[1]

    if author.lower() == old_handle:
        to_rename.append((i, f"{new_handle}_{id_}_{pos}{ext}"))

for old, new in to_rename:
    os.rename(old, new)

validate.py

import sys
import os
from collections import defaultdict
from datetime import datetime, timedelta
import re

fmt = re.compile(r"(.*)\_(\d+)_(\d+)")

os.chdir(sys.argv[1])

groups = defaultdict(list)
order = []
for i in os.listdir():
    match = fmt.match(i)
    author, id_, num = fmt.match(i).groups()
    id_, num = int(id_), int(num)

    groups[id_].append((author, num, i))
    order.append((-id_, num, i))

for id_, tweets in sorted(groups.items(), reverse=True):
    tweets.sort(key=lambda t: t[1])
    all_authors = {t[0].lower() for t in tweets}
    all_nums = [t[1] for t in tweets]

    if len(all_authors) > 1:
        print("tweet {id_} has multiple authors: {all_authors}")
    if all_nums != list(range(1, len(all_nums) + 1)):
        print(f"tweet {id_} seems to be missing some photos- i only see {all_nums}")

order.sort()

now = datetime.now()
epoch = datetime(1970, 1, 1)
for v, t in enumerate(order):
    delta = (now - timedelta(seconds=3 * v) - epoch).total_seconds()
    os.utime(t[2], times=(delta, delta))