nepse_floor_sheet.py

import json
import time



from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.firefox import GeckoDriverManager

user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " \
             "Chrome/85.0.4183.83 Safari/537.36 "

options = webdriver.FirefoxOptions()
prefs = {
    "translate_whitelists": {"np": "en"},
    "translate": {"enabled": "true"}
}
# options.headless = True
options.add_argument(f'user-agent={user_agent}')
options.add_argument("--window-size=1920,1080")
options.add_argument('--ignore-certificate-errors')
options.add_argument('--allow-running-insecure-content')
options.add_argument("--disable-extensions")
options.add_argument("--proxy-server='direct://'")
options.add_argument("--proxy-bypass-list=*")
options.add_argument("--start-maximized")
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--no-sandbox')

driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
driver.get(
    "https://merolagani.com/Floorsheet.aspx")

total_length = driver.find_element(By.XPATH, "//span[@id='ctl00_ContentPlaceHolder1_PagerControl2_litRecords']").text
floor_sheet = []
z = int((total_length.split()[-1]).replace(']', ''))
z = 1
for data in range(1, z + 1):
    driver.find_element(By.XPATH, "(//a[@title='Page {}'])[2]".format(data)).click()
    for value in driver.find_elements(By.XPATH, '//tbody/tr'):
        table_data = value.find_elements_by_tag_name('td')
        floor_sheet.append([td.text for td in table_data])

    time.sleep(1)
final_floor_sheet = []
for data in floor_sheet:
    daily_floorshet = {'transaction_no': data[1], 'symbol': data[2], 'buyer': data[3],'seller':data[4], 'quantity': data[5],
                       'rate': data[6], 'amount': data[7]}
    final_floor_sheet.append(daily_floorshet)

with open('new_floor_sheet.json', 'w', encoding='utf-8') as f:
    json.dump(final_floor_sheet, f, ensure_ascii=False, indent=4)