Grab the visible text from a webpage

visible_webpage_text.py

"""Grab the visible text from a webpage
Based on the stackoverflow post https://stackoverflow.com/questions/1936466/beautifulsoup-grab-visible-webpage-text
"""
from bs4 import BeautifulSoup
from bs4.element import Comment
import requests


def tag_visible(element):
    if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
        return False
    if isinstance(element, Comment):
        return False
    return True


def text_from_html(body):
    soup = BeautifulSoup(body, 'html.parser')
    texts = soup.findAll(text=True)
    visible_texts = filter(tag_visible, texts)  
    return u" ".join(t.strip() for t in visible_texts)


def visible_webpage_text(uri):
    r = requests.get(uri)
    r = r.text
    return text_from_html(r)