scrape seluruh paragraf berita yang ada di indeks.kompas.com

scrape_kompas.py

from bs4 import BeautifulSoup
import requests 


# url = 'https://indeks.kompas.com/'
# url='https://indeks.kompas.com/all/2019-04-01/2'
url = 'https://indeks.kompas.com/all/2019-04-01/3'
req = requests.get(url)

# print(req.text)

soup = BeautifulSoup(req.text, 'lxml')

a = soup.find_all('a',{'class':'article__link'})

kumpulan_link = []
kumpulan_paragraf = []


for link in a: 
    kumpulan_link.append(link['href'])

for link in kumpulan_link:
    halaman = requests.get(link)
    soup_baru = BeautifulSoup(halaman.text,'lxml')
    paragraf = soup_baru.find_all('p')
    for kalimat in paragraf: 
        kumpulan_paragraf.append(kalimat.text)

with open('paragraf.txt', 'a') as f: 
    for paragraf in kumpulan_paragraf: 
        print('penulisan berhasil')
        f.writelines(paragraf + '\n')

thanks