"Work Life" : One week after launch

itch_tag_page_all.csv

referrer_08_26_2024.tsv

referrer_09_06_2024.tsv

scrape.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import time

def filename(page: int) -> str:
    return f"itch_tag_page_{page}.html"

def scrape(start: int = 1, stop: int = 16):
    for page in range(start, stop):
        res = requests.get(f"https://itch.io/tags?page={page}")
        with open(filename(page), "+w", encoding='utf-8') as f:
            f.write(res.text)
        time.sleep(2)

class Filler(object):
    text = ""

def parse(start: int = 1, stop: int = 16):
    for page in range(start, stop):
        with open(filename(page), "r", encoding='utf-8') as f:
            source = f.read()
            soup = BeautifulSoup(source, 'html.parser')
            selectors = [
                ("title", '.browse_tags_page .tag_list .tag_cell .tag_title', r'(?P<ex>.*)'),
                ("total", '.browse_tags_page .tag_list .games_count strong', r'(?P<ex>.*) projects'),
                ("week", '.browse_tags_page .tag_list .games_count a', r'(?P<ex>.*) this week'),
            ]
            columns = []
            data = {}
            for column, selector, reg in selectors:
                columns.append(column)
                values = soup.select(selector)
                for i in range(0, 40-len(values)):
                    values.append(Filler())
                data[f"{column}_text"] = [v.text for v in values]
                data[column] = [ (re.match(reg, v.text) or {"ex": None})["ex"] for v in values]
                # print(values)
            for k, v in data.items():
                print(f"{k}: {len(v)}")
            df = pd.DataFrame.from_dict(data=data)
            df.to_csv(f"itch_tag_page_{page}.csv")


def concat(start: int = 1, stop: int = 16):
    all = pd.DataFrame()
    for page in range(start, stop):
        next = pd.read_csv(f"itch_tag_page_{page}.csv")
        all = pd.concat([all, next], axis = 0, ignore_index=True)
    all.reset_index()
    all.to_csv("itch_tag_page_all.csv")


# scrape(1,2)
# scrape(2,16)
# scrape(15, 16)
# parse(1,16)
concat(1,16)

test.ipynb