Find the average rent (with confidence interval) for a room in Boston using Selenium

find_average_rent.py

import selenium
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.common.exceptions import ElementNotInteractableException
import numpy as np
import scipy.stats as st

try:
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--log-level=3")
    driver = Chrome(options=options)

    # rent period 3 is monthly
    # duplicates bundled
    driver.get(
        'https://boston.craigslist.org/search/roo?bundleDuplicates=1&availabilityMode=0&rent_period=3')

    prices = []
    while True:
        price_elements = driver.find_elements(
            by=By.CLASS_NAME, value="result-price")
        for price in price_elements:
            processed = price.text.replace("$", "").replace(",", "").strip()
            
            # exclude empty and extremely low prices as invalid
            if processed != '' and int(processed) > 500:
                prices.append(int(processed))
                print(f"${processed}")
        try:
            next_button = driver.find_element(
                by=By.CSS_SELECTOR, value="a.next")
            next_button.click()
        except ElementNotInteractableException as nsx:
            print("reached last page")
            break
    average = np.mean(prices)
    confidence_interval = st.t.interval(alpha=0.95, df=len(
        prices)-1, loc=np.mean(prices), scale=st.sem(prices))
    print(f"average: {average}")
    print(f"confidence_interval: {confidence_interval}")
except Exception as e:
    print(f"error with scraping: {e}")
finally:
    driver.quit()