GDOT_cam_finder.py

#!/usr/bin/python3

from bs4 import BeautifulSoup
import requests
import os

links = []

base = "http://www.511ga.org/mobile"
start = 'http://www.511ga.org/mobile/?action=view_events&idents=cctv_5201%2Ccctv_5212%2Ccctv_5229%2Ccctv_15317%2Ccctv_5220%2Ccctv_5211%2Ccctv_46425%2Ccctv_5188%2Ccctv_5190%2Ccctv_5209%2Ccctv_5219%2Ccctv_5244%2Ccctv_5187%2Ccctv_5210&template=cameras&trail=main_menu__view_list_cameras&title_suffix=%20on%20SR%20166'

def save_to_file(filename, text):
    with open(filename, 'w+') as f:
        f.write(text)


def remove_duplicates(original_list):
    cleaned_list = []
    for item in original_list:
        if item not in cleaned_list:
            cleaned_list.append(item)
    return cleaned_list


links.append(start)
soup = BeautifulSoup(requests.get(start).content)
try:
    pages = soup.findAll('div', {'class': 'page_jump'})[0].findAll('a')
    for p in pages:
        links.append(f"{base}{p['href'][1:]}")
except Exception as e:
    pass
print(links)

# pages = soup.findAll('div',{'class':'page_jump'})
# print([page.a for page in soup.findAll('div',{'class':'page_jump'})])
# for a in pages:
#    print(p.find(''))

image_links = []
for page in links:
    soup = BeautifulSoup(requests.get(page).content)
    # print(soup.prettify())
    for link in soup.find_all('a'):
        href = link.get('href')
        if href.startswith('./?action=view_single_camera'):
            # print(href)
            # print(base + href[1:])
            soup2 = BeautifulSoup(requests.get(base + href[1:]).content)
            # print(soup2)
            imgs = soup2.findAll('div', {'class': 'camera_container'})
            for i in imgs:
                if i.a['href'] not in image_links:
                    print("\"" + i.a['href'] + "\",")
                    image_links.append("\"" + i.a['href'] + "\",")
save_to_file('link_list.txt', '\n'.join(remove_duplicates(image_links)))
print("Links saved in link_list.txt")