github_scrapper.py

# coding: utf-8

"""
pip install requests beautifulsoup4
"""


import requests
from bs4 import BeautifulSoup

DOMAIN = 'https://github.com'


def filter_github_files(tag):
    return (
        tag.name == 'a' and
        tag.attrs.get('class', '') == ['js-navigation-open']
        and not tag.attrs.get('rel')
    )


def is_folder(link):
    return 'tree' in link


def find_files(url):
    response = requests.get(url)
    html_tree = BeautifulSoup(response.content, 'html.parser')

    for element in html_tree.find_all(filter_github_files):
        link = element.attrs['href']
        if is_folder(link):
            yield from find_files(DOMAIN + link)
        else:
            yield link


def main():
    for url in find_files(DOMAIN + '/IuryAlves/bastion/'):
        print(url)


main()