仓鼠症爆发

download_pdf.py

import requests
from bs4 import BeautifulSoup
from pathlib import Path

url = "https://cs186berkeley.net/"
folder = Path("/Users/xuzheng/Desktop/self/cs186-database/test/")
pdfs = []
r = requests.get(url)
html = r.text
bs = BeautifulSoup(html, 'html.parser')
print(bs.h1)

for link in bs.find_all('a'):
    cur_link = link.get('href')

    if cur_link is not None:
        if cur_link.endswith('pdf'):
            if cur_link[0] != "/":
                pdfs.append(url+cur_link)
            else:
                pdfs.append(url[:-1]+cur_link)


for url in pdfs:
    file_to_write = url.split("/")[-1]
    file_path = folder/file_to_write
    r = requests.get(url, stream=True)
    with open(file_path, 'wb') as f:
        f.write(r.content)