Tiktok Posts Scraper

tiktok_post_scraper.py

from curl_cffi import requests
from bs4 import BeautifulSoup


data = []


picuki = requests.get('https://www.picuki.com/profile/linustech', impersonate="chrome")
soup = BeautifulSoup(picuki.text, 'html.parser')
for post in soup.select('.posts-container a'):
    title = post.img.get('alt') if post.img else None
    item = post.find('div', {'class': 'posts__video-item-story-download popup-open'})
    music = item.get('data-music') if item else None
    url = item.get('data-source') if item else None
    data.append({'title': title, 'music': music, 'source': url})

load_more = soup.find('div', {'class': 'posts-load-more'})
is_load_more = True if load_more else False
is_first = True
next_url = None
while is_load_more:
    if not is_first:
        load_more = soup.find('input', {'class': 'pagination-next-page-input'})
    if not load_more:
        is_load_more = False
        continue
    next_url_path = load_more.get('data-next') if load_more.get('data-next') else load_more.get('value')
    next_url = 'https://www.picuki.com'+next_url_path
    picuki = requests.get(next_url, impersonate="chrome")
    soup = BeautifulSoup(picuki.text, 'html.parser')
    for post in soup.select('a'):
        title = post.img.get('alt') if post.img else None
        item = post.find('div', {'class': 'posts__video-item-story-download popup-open'})
        music = item.get('data-music') if item else None
        url = item.get('data-source') if item else None
        data.append({'title': title, 'music': music, 'source': url})
    is_first = False