Analyze Used Bikes from SG Bike Mart

used_bike_scrape_and_analyze.py

import calendar
import re
from datetime import datetime
from urllib.parse import urljoin, urlparse, parse_qs

import pandas as pd
import requests
from bs4 import BeautifulSoup

USED_BIKE_SEARCH_URL = "https://sgbikemart.com.sg/listing/usedbikes/listing/?bike_model=Honda+MSX125&bike_type=&price_from=&price_to=&license_class=2B&reg_year_from=1970&reg_year_to=2024&monthly_from=&monthly_to=&user=&status=10&category="


def calculate_depreciation(price, total_months_left):
    # If there are no remaining months, return 'N/A'
    if total_months_left == 'N/A' or total_months_left <= 0:
        return {'annual_depreciation': 'N/A', 'monthly_depreciation': 'N/A'}

    # Calculate depreciated value at the end of COE (10% of initial price)
    end_value = price * 0.10

    # Calculate total depreciation amount (price reduction needed)
    total_depreciation = price - end_value

    # Calculate monthly depreciation
    monthly_depreciation = total_depreciation / total_months_left

    # Calculate annual depreciation
    annual_depreciation = monthly_depreciation * 12

    return {
        'annual_depreciation': annual_depreciation,
        'monthly_depreciation': monthly_depreciation
    }


def split_currency_value(amount_str):
    # Identify where the numerical part starts
    for i, char in enumerate(amount_str):
        if char.isdigit():
            break

    # Split the string into currency and value
    currency = amount_str[:i].strip()
    currency = re.sub(r'[^A-Za-z]', '', currency)  # Remove any non-letter characters (like $)

    value = amount_str[i:].replace(",", "").strip()  # Removing commas if any

    return {"Currency": currency, "Price": float(value)}


def extract_bike_info(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')

    # Extracting information
    title = soup.find('h2', class_='card-title').text.strip()
    price = soup.find('h2', class_='text-center strong').text.strip()

    # Details Table
    details_table = soup.find('table', class_='table mb-0')
    details = {}
    for row in details_table.find_all('tr'):
        key = row.find('td', class_='name').text.strip()
        value = row.find('td', class_='value').text.strip()
        details[key] = value

    # Extracting specific details
    brand = details.get('Brand', 'N/A')
    model = details.get('Model', 'N/A')
    engine_capacity = details.get('Engine Capacity', 'N/A')
    classification = details.get('Classification', 'N/A')
    registration_date = details.get('Registration Date', 'N/A')
    coe_expiry_date = details.get('COE Expiry Date', 'N/A')
    vehicle_type = details.get('Type of Vehicle', 'N/A')

    # Calculating remaining COE
    if coe_expiry_date != 'N/A':
        coe_expiry_date_clean = coe_expiry_date.split()[0]
        coe_expiry_date_obj = datetime.strptime(coe_expiry_date_clean, '%d/%m/%Y')
        today = datetime.today()
        remaining_time = coe_expiry_date_obj - today
        remaining_years = remaining_time.days // 365
        remaining_months = (remaining_time.days % 365) // 30
        total_remaining_months = remaining_time.days / 30
    else:
        remaining_years = remaining_months = total_remaining_months = 'N/A'

    # Extracting description
    description_div = soup.find('div', class_='listing-details')
    description = description_div.text.strip() if description_div else 'N/A'

    # Creating the dictionary with the extracted information
    bike_info = {
        "Title": title,
        "Brand": brand,
        "Model": model,
        "Engine Capacity": engine_capacity,
        "Classification": classification,
        "Registration Date": registration_date,
        "COE Expiry Date": coe_expiry_date,
        "Total Months Left": total_remaining_months,
        "Years & Months Left": f"{remaining_years} years, {remaining_months} months",
        "Type of Vehicle": vehicle_type,
        "Description": description
    }
    bike_info.update(split_currency_value(price))

    # Returning the dictionary
    return bike_info


def project_vehicle_price(registration_date, coe_expiry_date, current_price, monthly_depreciation):
    # Convert registration and COE expiry dates to datetime objects
    registration_date = datetime.strptime(registration_date, '%d/%m/%Y')
    coe_expiry_date = datetime.strptime(coe_expiry_date.split()[0], '%d/%m/%Y')

    # Calculate the number of months the vehicle has already depreciated
    current_date = datetime.today()
    total_months_since_registration = (
                                              current_date.year - registration_date.year) * 12 + current_date.month - registration_date.month

    # Calculate the original price at the registration date
    original_price = current_price + (total_months_since_registration * monthly_depreciation)

    # Create a dictionary to store projected prices for each month
    monthly_prices = {}
    yearly_prices = {}

    # Iterate month by month from the registration date to the COE expiry date (inclusive)
    current_price = original_price
    current_date = registration_date
    while current_date <= coe_expiry_date:
        date_str = current_date.strftime('%d/%m/%Y')
        monthly_prices[date_str] = current_price

        # Add price to yearly_prices only if it's the first month of the year or the COE expiry date
        if current_date.month == registration_date.month or current_date == coe_expiry_date:
            yearly_prices[date_str] = current_price

        # Deduct the monthly depreciation
        current_price -= monthly_depreciation

        # Move to the next month
        next_month = current_date.month + 1 if current_date.month < 12 else 1
        next_year = current_date.year + 1 if current_date.month == 12 else current_date.year
        max_day_in_month = calendar.monthrange(next_year, next_month)[1]

        # Adjust the day if it is out of range
        day = min(registration_date.day, max_day_in_month)
        current_date = datetime(year=next_year, month=next_month, day=day)

    # Ensure the COE expiry date is included in both monthly and yearly prices
    expiry_date_str = coe_expiry_date.strftime('%d/%m/%Y')
    monthly_prices[expiry_date_str] = current_price
    yearly_prices[expiry_date_str] = current_price

    return monthly_prices, yearly_prices


def analyze_used_bike(url):
    bike_info = extract_bike_info(url)
    bike_depreciation = calculate_depreciation(bike_info["Price"], bike_info['Total Months Left'])
    bike_info.update(bike_depreciation)

    monthly_prices, yearly_prices = project_vehicle_price(bike_info["Registration Date"], bike_info["COE Expiry Date"],
                                                          bike_info["Price"], bike_depreciation["monthly_depreciation"])

    # Adds the starting price of the monthly_prices into the bike info as the dealer's assumed bike value
    bike_info["url"] = url
    bike_info["Dealer's Assumed Bike If new Value"] = list(monthly_prices.values())[0]
    bike_info["monthly_price_data"] = monthly_prices
    bike_info["yearly_price_data"] = yearly_prices

    return bike_info


def extract_coe_price(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')

    # Locate the card-title with "COE Results"
    coe_card_title = soup.find('div', class_='card-title', string='COE Results')

    if not coe_card_title:
        return None

    # Navigate to the parent div to get the whole COE card content
    coe_card = coe_card_title.find_parent('div', class_='card')

    # Locate the specific strong tag with the price information
    price_tag = coe_card.find('div', class_='col-4').find_all('strong')[1]  # This will be the second strong tag

    if price_tag:
        return price_tag.text.strip()

    return None


def get_current_coe_price():
    url = "https://sgbikemart.com.sg"
    response = requests.get(url)
    return split_currency_value(extract_coe_price(response.content))["Price"]


def extract_bike_listing_urls(base_url):
    """
    Extracts and returns a list of absolute URLs for bike listings from the given webpage.

    Parameters:
    base_url (str): The base URL of the page to scrape.

    Returns:
    list: A list of absolute URLs pointing to individual bike listings.
    """
    response = requests.get(base_url)
    html_content = response.text

    # Parse the HTML content using BeautifulSoup
    soup = BeautifulSoup(html_content, 'html.parser')

    # Find all the <a> tags that lead to the individual bike listings
    bike_links = soup.find_all('a', href=True)
    listing_urls = set()
    # Extract and print full URLs of the bike listings
    for link in bike_links:
        href = link['href']
        if '/listing/usedbike/' in href:
            full_url = urljoin(base_url, href)
            listing_urls.add(full_url)

    return list(listing_urls)


def parse_url(url):
    # Parse the URL
    parsed_url = urlparse(url)
    # Extract query parameters
    query_params = parse_qs(parsed_url.query)

    # Extract the relevant parameters
    bike_model = query_params.get('bike_model', [''])[0].lower()
    license_class = query_params.get('license_class', [''])[0].lower()
    reg_year_from = query_params.get('reg_year_from', [''])[0]
    reg_year_to = query_params.get('reg_year_to', [''])[0]

    # Format the output string
    result = f"{bike_model}_{license_class}_{reg_year_from}_{reg_year_to}"

    return result


print("Current COE Price:", get_current_coe_price())
# Provide the URL of the motorcycle listing page

bike_urls = extract_bike_listing_urls(USED_BIKE_SEARCH_URL)
bike_data = []
for url in bike_urls:
    try:
        bike_info = analyze_used_bike(url)
    except:
        continue
    print(bike_info)
    bike_data.append(bike_info)

df = pd.DataFrame(bike_data)
df.to_csv(f"{parse_url(USED_BIKE_SEARCH_URL)}.csv", index=False)

print(df)