Scrape Gibson.com

scrape_gibson.py

import sys
import json
import logging
from typing import List, Tuple, Dict, Any

from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd

log = logging.getLogger(__name__)

BASE_URL = "https://www.gibson.com"


def setup_logger() -> None:
    formatter = (
        "[%(levelname)s] %(asctime)-15sZ [%(name)s %(funcName)s %(lineno)d] %(message)s"
    )
    logging.basicConfig(level="DEBUG", stream=sys.stdout, format=formatter)


def get_soup(driver, url: str) -> BeautifulSoup:
    _ = driver.get(url)
    html = driver.execute_script("return document.body.innerHTML")
    soup = BeautifulSoup(html, "html.parser")
    return soup


def get_all_les_pauls(driver, url: str) -> List[Tuple[str, str]]:
    soup = get_soup(driver, url)
    items = [
        (e.text, BASE_URL + e.attrs["href"])
        for e in soup.find_all(
            "a", attrs={"class": "col-sm-12 col-md-6 col-lg-4 px-2 mb-4"}
        )
    ]
    return items


def get_les_paul_details(driver, url: str) -> Dict:
    soup = get_soup(driver, url)
    price = soup.find("div", attrs={"class": "price"}).text
    price_numeric = float(price.replace("$", "").replace(",", ""))

    items = [("price", price_numeric), ("url", url)]

    for i in soup.find_all(
        "div", attrs={"class": "spec-item d-flex justify-content-between"}
    ):
        d = [j.text for j in i.find_all("div")]
        items.append(d)

    return dict(items)


def write_csv(xs: List[Dict], output: str) -> None:
    df = pd.DataFrame(xs)
    df.to_csv(output, index=False)
    log.info(f"Completed writing csv to {output}")


def write_json(xs: List[Dict], output: str) -> None:
    with open(output, "w") as f:
        json.dump(xs, f)
    log.info(f"Completed writing json to {output}")


def main() -> int:
    setup_logger()
    output = "scraped-gibson.json"
    output_csv = "scraped-gibson.csv"
    urls = (f"{BASE_URL}/Custom-Shop", f"{BASE_URL}/Guitars/Les-Paul")

    driver = webdriver.Firefox()

    results = []
    seen = set([])
    for url in urls:
        items = get_all_les_pauls(driver, url)
        for ix, gtr_url in items:
            if gtr_url not in seen:
                details = get_les_paul_details(driver, gtr_url)
                details["Model"] = ix.strip().replace('"', " ")
                log.info(f"scraped {ix} with {len(details)} metadata")
                results.append(details)
                seen.add(gtr_url)

    log.info(f"Completed scraping {len(results)} records")

    writers = [(write_json, output), (write_csv, output_csv)]
    for writer, output_file in writers:
        writer(results, output_file)

    driver.close()
    return 0


if __name__ == "__main__":
    sys.exit(main())

scraped-gibson.csv