Get corona statistics with scrapy script (28.03.2020)

corona.py

#import json
from scrapy.selector import Selector
from requests import get
from time import sleep

def update():   
    r = get(url) 
    source_code = r.text 

    total_cases     =   Selector(text=source_code).xpath('//*[@id="maincounter-wrap"][1]/div/span/text()').get()
    total_recovered =   Selector(text=source_code).xpath('//*[@id="maincounter-wrap"][2]/div/span/text()').get()
    total_deaths    =   Selector(text=source_code).xpath('//*[@id="maincounter-wrap"][3]/div/span/text()').get()

    countries = []
    country_cases = []
    country_deaths = []
    country_recovered = []

    
    for i in range(1,92):
        temp_xpath = '//*[@id="main_table_countries_today"]/tbody[1]/tr['+str(i)+']/td[1]/a/text()'
        temp_country = Selector(text=source_code).xpath(temp_xpath).get()
        print(temp_country)
        countries.append(temp_country)

    for i in range(1,92):
        temp_xpath = '//*[@id="main_table_countries_today"]/tbody[1]/tr['+str(i)+']/td[2]/text()'
        temp_country_cases = Selector(text=source_code).xpath(temp_xpath).get()
        country_cases.append(temp_country_cases)

    for i in range(1,92):
        temp_xpath = '//*[@id="main_table_countries_today"]/tbody[1]/tr['+str(i)+']/td[4]/text()'
        temp_country_deaths = Selector(text=source_code).xpath(temp_xpath).get()
        country_deaths.append(temp_country_deaths)

    for i in range(1,92):
        temp_xpath = '//*[@id="main_table_countries_today"]/tbody[1]/tr['+str(i)+']/td[6]/text()'
        temp_country_recovered = Selector(text=source_code).xpath(temp_xpath).get()
        country_recovered.append(temp_country_recovered)

    print("Fetched!","Let's write to file")

    hepsi = open("All.txt",'a+')
    for i in range(1,len(countries)):
        file_name = str(countries[i])+".txt"
        f = open(file_name,'w+')
        prepare_text = str(country_cases[i]).strip()+" "+ str( country_deaths[i]).strip()+" "+str(country_recovered[i]).strip()
        f.write(prepare_text)
        hepsi.write(prepare_text+"\n")
        f.close()
    hepsi.close()

    # name1 = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[1]/td[1]/a/text()').get()
    # name2 = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[2]/td[1]/a/text()').get()
    # name3 = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[3]/td[1]/a/text()').get()
    # name4 = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[4]/td[1]/a/text()').get()
    # name5 = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[5]/td[1]/a/text()').get()
    # name6 = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[6]/td[1]/a/text()').get()
    

    # name1_cases   = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[1]/td[2]/text()').get()
    # name2_cases   = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[2]/td[2]/text()').get()
    # name3_cases     = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[3]/td[2]/text()').get()
    # name4_cases   = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[4]/td[2]/text()').get()
    # name5_cases = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[5]/td[2]/text()').get()
    # name6_cases    = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[6]/td[2]/text()').get()

    # name1_deaths    = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[1]/td[4]/text()').get()
    # name2_deaths    = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[2]/td[4]/text()').get()
    # name3_deaths      = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[3]/td[4]/text()').get()
    # name4_deaths    = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[4]/td[4]/text()').get()
    # name5_deaths  = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[5]/td[4]/text()').get()
    # name6_deaths     = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[6]/td[4]/text()').get()

    # name1_recovered     = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[1]/td[6]/text()').get()
    # name2_recovered     = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[2]/td[6]/text()').get()
    # name3_recovered       = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[3]/td[6]/text()').get()
    # name4_recovered     = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[4]/td[6]/text()').get()
    # name5_recovered   = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[5]/td[6]/text()').get()
    # name6_recovered      = Selector(text=source_code).xpath('//*[@id="main_table_countries_today"]/tbody[1]/tr[6]/td[6]/text()').get()


    # cities = ["total",name1,name2,name3,name4,name5,name6]
    # cases = ["total",name1_cases,name2_cases,name3_cases,name4_cases,name5_cases,name6_cases]
    # deaths = ["total",name1_deaths,name2_deaths,name3_deaths,name4_deaths,name5_deaths,name6_deaths]
    # recovered = ["total",name1_recovered,name2_recovered,name3_recovered,name4_recovered,name5_recovered,name6_recovered]

    # for i in range(0,len(cities)):
    #     file_name = str(cities[i])+".txt"
    #     f = open(file_name,'w+')
    #     prepare_text = str(cases[i]).strip()+" "+ str(deaths[i]).strip()+" "+str(recovered[i]).strip()
    #     f.write(prepare_text)
    #     f.close()

update_time = 60*60*0.4

while(1):
    print("Starting....")
    update()
    print("Updated....")
    print("Next Update",update_time/(60)," minutes...")
    sleep(update_time)
    print("Here we go again...")



# print("Total: ",total_cases.strip(),"Total Deaths: ",total_deaths.strip(),"Total Recovered: ",total_recovered)
# print("China: ",china.strip(),"\tDeaths: ",china_deaths.strip(),"\tRecovered: ",china_recovered)
# print("Italy: ",italy.strip(),"\tDeaths: ",italy_deaths.strip(),"\tRecovered: ",italy_recovered)
# print("USA: ",usa.strip(),"\tDeaths: ",usa_deaths.strip(),"\tRecovered: ",usa_recovered)
# print("Spain: ",spain.strip(),"\tDeaths: ",spain_deaths.strip(),"\tRecovered: ",spain_recovered)
# print("Germany: ",germany.strip(),"\tDeaths: ",germany_deaths.strip(),"\tRecovered: ",germany_recovered)
# print("Iran: ",iran.strip(),"\tDeaths: ",iran_deaths.strip(),"\tRecovered: ",iran_recovered)