Scrap youtube video page by going through the consent screen

scrap-yt-page.py

from requests import Session
from bs4 import BeautifulSoup

UA = (
    "Mozilla/5.0 (Linux; cli) pyrequests/0.1 "
    "(python, like Gecko, like KHTML, like wget, like CURL) myscrapper/1.0"
)
req = Session()
req.headers.update({"User-Agent": UA})

def get_page_source(url):
    r = req.get(url).text
    if "itemprop" in r:
        return r
    post_builder = {}
    soup = BeautifulSoup(r, 'html.parser')
    for i in soup.find_all("input"):
        try:
            post_builder.update({i["name"] : i["value"]})
        except KeyError:
            continue
    return req.post("https://consent.youtube.com/s", data=post_builder).text


print(get_page_source("https://www.youtube.com/watch?v=dQw4w9WgXcQ"))

So before you wanted to scrape a youtube page, setting appropriate User-Agent was enough but now it seems you are always greeted with page full of legal cookie nonsense, my thing here seems to go through that for now. The itemprop part is just an attribute that appears in video pages but doesn't on consent page.