From NEA Rainfall data sensor location distance to DTL MRT Stations

downloadNEARainfallGeoMRT.py

import requests
import json
import os
from datetime import timedelta, date
from math import radians, cos, sin, asin, sqrt

def haversine(lon1, lat1, lon2, lat2):
    """
    Calculate the great circle distance between two points 
    on the earth (specified in decimal degrees)
    From https://stackoverflow.com/users/188595/michael-dunn
    """
    # convert decimal degrees to radians 
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])

    # haversine formula 
    dlon = lon2 - lon1 
    dlat = lat2 - lat1 
    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
    c = 2 * asin(sqrt(a)) 
    r = 6371 # Radius of earth in kilometers. Use 3956 for miles
    return c * r

def daterange(start_date, end_date):
    for n in range(int((end_date - start_date).days)):
        yield start_date + timedelta(n)

def putCSV(info_status, reading_type, meter_unit , initJSON, get_status_code, thisDate):

    stationGeoLoc = (("Bencoolen MRT Station",103.850454,1.298838,"BCL"),("Bendemeer MRT Station",103.863137,1.313295,"BDM"),("Bedok North MRT Station",103.918112,1.335058,"BDN"),("Bedok Reservoir MRT Station",103.932892,1.336312,"BDR"),("Bayfront MRT Station",103.858768,1.280974,"BFT"),("Bugis MRT Station",103.856134,1.300649,"BGS"),("Bukit Panjang MRT Station",103.761395,1.378413,"BKP"),("Botanic Gardens MRT Station",103.814876,1.3222,"BTN"),("Beauty World MRT Station",103.775712,1.341512,"BTW"),("Chinatown MRT Station",103.843828,1.284598,"CNT"),("Cashew MRT Station",103.764508,1.369789,"CSW"),("Downtown MRT Station",103.852813,1.279458,"DTN"),("Fort Canning MRT Station",103.844597,1.291514,"FCN"),("Geylang Bahru MRT Station",103.871466,1.321101,"GLB"),("Hume MRT Station",103.76904,1.35465,"HUME"),("Hillview MRT Station",103.767583,1.362261,"HVW"),("Jalan Besar MRT Station",103.855333,1.305279,"JLB"),("King Albert Park MRT Station",103.783265,1.335619,"KAP"),("Kaki Bukit MRT Station",103.908822,1.334939,"KKB"),("Little India MRT Station",103.849424,1.306539,"LTI"),("MacPherson MRT Station",103.88993,1.326627,"MPS"),("Mattar MRT Station",103.882822,1.327215,"MTR"),("Newton MRT Station",103.837627,1.313716,"NEW"),("Promenade MRT Station",103.86019,1.293844,"PMN"),("Rochor MRT Station",103.852548,1.303398,"RCR"),("Sixth Avenue MRT Station",103.796795,1.330724,"SAV"),("Stevens MRT Station",103.825999,1.319952,"STV"),("Tampines MRT Station",103.942894,1.355117,"TAM"),("Tan Kah Kee MRT Station",103.807546,1.325676,"TKK"),("Telok Ayer MRT Station",103.848539,1.281947,"TLA"),("Tampines East MRT Station",103.954626,1.35632,"TPE"),("Tampines West MRT Station",103.938233,1.345431,"TPW"),("Ubi MRT Station",103.899925,1.329983,"UBI"),("Upper Changi MRT Station",103.961309,1.34194,"UPC"),("IOCC Facilities",103.965311,1.333188,"XPI"),("Expo MRT Station",103.962759,1.335037,"XPO"))

    stationCode = []
    for eachItem in stationGeoLoc:
        stationCode.append(eachItem[3])

    #stationCode = ("BCL", "BDM", "BDN", "BDR", "BFT", "BGS", "BKP", "BTN", "BTW", "CNT", "CSW", "DTN", "FCN", "GLB", "HUME", "HVW", "JLB", "KAP", "KKB", "LTI", "MPS", "MTR", "NEW", "PMN", "RCR", "SAV", "STV", "TAM", "TKK", "TLA", "TPE", "TPW", "UBI", "UPC", "XPI", "XPO")    

    writeFile = open(f".\\rainFall_{thisDate}.csv", "a")
    writeFile.write(f"timestamp,latitude,longitude,value,station_id,name,meter_unit,reading_type,info_status,status_code,{','.join(stationCode)}\r" )   

    countFoundData = 0
    for eachDate in initJSON["items"]: #per timestamp items
        for eachReadings in eachDate["readings"]: #per readings in the timestamp
            if eachReadings["value"] != 0:
                countFoundData += 1
                for eachStation in initJSON["metadata"]["stations"]:
                    if eachReadings["station_id"] == eachStation["device_id"]:                        
                        stationDistance = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

                        for locStation in stationGeoLoc:
                            stationLon, stationLat = locStation[1], locStation[2]
                            rainLon, rainLat = eachStation["location"]["longitude"], eachStation["location"]["latitude"]
                            stationDistance[ stationCode.index(locStation[3]) ] = haversine(stationLon, stationLat, rainLon, rainLat)
                            #based on collected distance from the haversine code put inside the distance array list per rain sensor data

                        writeFile.write( f'{eachDate["timestamp"]},{eachStation["location"]["latitude"]},{eachStation["location"]["longitude"]},{eachReadings["value"]},{eachReadings["station_id"]},{eachStation["name"]},{meter_unit},{reading_type},{info_status},{get_status_code},{",".join([str(elem) for elem in stationDistance])}\r' )
                        #write print the data of the each row of rain fall sensor data with station distance from the rainfall sensor
    writeFile.close()
    if countFoundData == 0:
        os.remove(f".\\rainFall_{thisDate}.csv")


def jprint(obj, get_status_code, thisDate):
    print (f'Extracting data for date {thisDate}')
    # create a formatted string of the Python JSON object
    initJSON = json.dumps(obj) # in str format    
    initJSON = json.loads(initJSON) # switch to JSON format

    if len(initJSON["metadata"]["stations"]) > 0 and len(initJSON["items"]) > 0 and initJSON["api_info"]["status"] == 'healthy':
        info_status = initJSON["api_info"]["status"]  
        reading_type = initJSON["metadata"]["reading_type"]
        meter_unit = initJSON["metadata"]["reading_unit"]                          
        putCSV(info_status, reading_type, meter_unit, initJSON, get_status_code, thisDate)
    elif len(initJSON["metadata"]["stations"]) == 0 and len(initJSON["items"]) == 0 and initJSON["api_info"]["status"] == 'healthy':
        info_status = initJSON["api_info"]["status"]  
        print(f'{info_status}')

def main():
    start_date = date(2021, 2, 1)
    end_date = date(2021, 3, 10)
    for single_date in daterange(start_date, end_date):        
        thisDate = single_date.strftime("%Y-%m-%d")
        response = requests.get(f"https://api.data.gov.sg/v1/environment/rainfall?date={thisDate}")
        get_status_code = response.status_code
        jprint(response.json(), get_status_code, thisDate)

if __name__ == "__main__":
    main()