ravro_tool_dev.py

import requests
from lxml import html
from time import sleep

header = {}
status = {}
urls = []
visited_urls = []

entry = "https://microsoft.com/"
urls.append(entry)

counter = 0
while (True):
    if (counter >= 20):
        break
    target = urls[counter]
    if (target in visited_urls):
        counter += 1
        visited_urls.append(target)
        continue
    if (not "microsoft.com" in target):
        counter += 1
        continue
    visited_urls.append(target)
    try:
        sleep(2)
        res = requests.get(target)
    except requests.exceptions.MissingSchema:
        counter += 1
        visited_urls.append(target)
        continue
    header[target] = res.headers
    status[target] = res.status_code

    page = html.fromstring(res.content)
    urls.extend(page.xpath('//a/@href'))
    print("==={}===".format(target))
    print(status[target])
    print(header[target])
    print(urls)
    counter += 1






print(header)
print("===")
print(status)
print("===")
print(urls)
print("===")
print(visited_urls)

👍🏽👌