DeepWiki Extractor

extract.py

import requests
from bs4 import BeautifulSoup
import re
import sys

def extract_next_f_push_args(url: str) -> list[str]:
    # HTML取得
    res = requests.get(url)
    res.raise_for_status()
    html = res.text

    # BeautifulSoupでパース
    soup = BeautifulSoup(html, "html.parser")
    scripts = soup.body.find_all("script")

    # self.__next_f.push()呼び出しの引数をすべて抽出（括弧やクオートのネスト対応）
    all_args = []

    def extract_args_from_code(code):
        results = []
        idx = 0
        while True:
            start = code.find("self.__next_f.push(", idx)
            if start == -1:
                break
            i = start + len("self.__next_f.push(")
            depth = 1
            in_single = False
            in_double = False
            escape = False
            arg_start = i
            while i < len(code):
                c = code[i]
                if escape:
                    escape = False
                elif c == "\\":
                    escape = True
                elif in_single:
                    if c == "'" and not escape:
                        in_single = False
                elif in_double:
                    if c == '"' and not escape:
                        in_double = False
                else:
                    if c == "'":
                        in_single = True
                    elif c == '"':
                        in_double = True
                    elif c == "(":
                        depth += 1
                    elif c == ")":
                        depth -= 1
                        if depth == 0:
                            results.append(code[arg_start:i])
                            idx = i + 1
                            break
                i += 1
            else:
                # 対応する閉じ括弧が見つからなかった場合
                break
        return results

    for script in scripts:
        if script.string:
            code = script.string
        else:
            code = script.get_text()
        matches = extract_args_from_code(code)
        all_args.extend(matches)

    return all_args

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python extract.py <URL>")
        sys.exit(1)
    url = sys.argv[1]
    args_list = extract_next_f_push_args(url)
    with open("result.md", "w") as f:
        import ast
        for i, args in enumerate(args_list, 1):
            try:
                parsed = ast.literal_eval(args)
                if len(parsed) != 2:
                    continue
                text: str = parsed[1]
                if not text.startswith("# "):
                    continue
                print(parsed[1])
                f.write(parsed[1] + "\n")
            except Exception:
                print(args.strip())
            print()