Italian Geoportale WMS and WFS mining

Geoportale.py

import requests
from bs4 import BeautifulSoup
import json

def extract_dataset_info(html):
    soup = BeautifulSoup(html, 'html.parser')

    dataset_info_list = []

    # Find all occurrences of 'h3' and 'p' tags
    h3_elements = soup.find_all('h3')
    p_elements = soup.find_all('p')

    for h3, p in zip(h3_elements, p_elements):
        dataset_info = {
            'title': h3.get_text(strip=True),
            'description': p.get_text(strip=True)
        }

        links = soup.find_all('a', href=True)
        for link in links:
            if 'WMS' in link.get_text():
                dataset_info['WMS_link'] = link['href']
            elif 'WFS' in link.get_text():
                dataset_info['WFS_link'] = link['href']
            

        dataset_info_list.append(dataset_info)

    return dataset_info_list

def scrape_pages(url, num_pages):
    data_list = []

    for page in range(1, num_pages + 1):
        page_url = f"{url}&paged_e={page}"
        response = requests.get(page_url)

        if response.status_code == 200:
            page_data = extract_dataset_info(response.text)
            data_list.extend(page_data)
            print(f"Scraped page {page}")
        else:
            print(f"Failed to fetch page {page}. Status code: {response.status_code}")

    return data_list

# Specify the URL and the number of pages
url = "http://www.pcn.minambiente.it/mattm/visualizzazione-metadati/"
num_pages = 16

# Scrape data
scraped_data = scrape_pages(url, num_pages)

# Convert the data to JSON
json_data = json.dumps(scraped_data, indent=2)

# Save JSON data to a file or print it
with open('output.json', 'w') as file:
    file.write(json_data)