download all text from UU

download_from_UU.py

import requests
from bs4 import BeautifulSoup
import sys
from opencc import OpenCC

def s2t(text):
    cc = OpenCC('s2t')
    converted = cc.convert(text)
    return converted

def get_html_source(link):
    r = requests.get(link)
    r.encoding = 'gb2312'
    return r.text

def extract_content(link):
    raw_html = get_html_source(link)
    s = BeautifulSoup(raw_html, 'html.parser')
    title = s.find(id='timu')
    content = s.find(id='contentbox')
    return title.text, content.text

def get_from_table(link):
    root = link[:link.rfind('/')]
    r = get_html_source(link)
    s = BeautifulSoup(r, 'html.parser')
    ss = s.find("ul", id="chapterList")
    sss = ss.find_all("a", href=True)
    sss.reverse()
    return sss

if __name__ == "__main__":
    start = sys.argv[1]
    chaplists = get_from_table(start)
    root = "https://www.uukanshu.com"
    links = [root + chap['href'] for chap in chaplists]
    all_text = ""
    for link in links:
        title, content = extract_content(link)
        print(title)
        all_text += title
        all_text += "\n\n"
        all_text += content
        all_text += "\n\n"
    all_text = s2t(all_text)
    with open("tmp.txt", "w") as f:
        f.write(all_text)