Scaraping from profiles.txt

scrape.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.keys import Keys
from parsel import Selector
from time import sleep
import requests, json
import time
import psutil
import re
import datetime
import os
import json 


chrome_options = webdriver.ChromeOptions()
chrome_options.accept_untrusted_certs = True
chrome_options.assume_untrusted_cert_issuer = True
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("log-level=3")
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-dev-shm-usage")
users=[]
works=[]

with open('profiles.txt') as f:
    profiles = f.read().splitlines()
for profile_url in profiles:
    user=[]
    print (profile_url)
    cmd = "get.bat "+profile_url
    os.system(cmd)
    #time.sleep(5)
    filelist = os.listdir("temp")[0]
    file = "c:\\scripts\\temp\\"+filelist
    #print (filelist)
    #print (file)
    driver = webdriver.Chrome(r'C:\Users\WhatUpTime.com\Downloads\chromedriver_win32\chromedriver',chrome_options=chrome_options)
    driver.get(file)
    #print(driver.title)
    #driver.find_element_by_xpath('//*[@id="app-container"]/div[3]').click()
    with open('out.txt', 'a') as f:
        #print("Name: ",driver.find_element_by_xpath('//*[@id="app-container"]/section[1]/dl/dt').text,file=f)
        user.append({"name":driver.find_element_by_xpath('//*[@id="app-container"]/section[1]/dl/dt').text})
        #print("Current Position: " , driver.find_element_by_xpath('/html/body/div[3]/section[1]/dl/dd[1]').text,file=f)
        user.append({"current":driver.find_element_by_xpath('/html/body/div[3]/section[1]/dl/dd[1]').text})

    #print(driver.find_element_by_xpath('//*[@id="app-container"]/div[7]/section[1]/ol/li[4]/div/ul/li/a/dl/dd[2]').get_attribute("outerHTML"))


    #print(driver.find_element_by_xpath('/html/body/div[3]/div[7]/section[1]/ol/li[1]').text)
    experience_section = driver.find_element_by_css_selector('section.experience-container.list-container.collapsable-list-container')

    experience = experience_section.find_elements_by_css_selector('dd.medium.time-period')
    years=0
    months=0

    for items in experience:
        duration=items.find_element_by_xpath('.//span[4]').get_attribute("textContent")
        #print(duration)
        
        try:        
            result = re.search(r'\d*\s[y]', duration)
            #print(result.group(0))
            years=years+int(result.group(0)[:2])
            #print(result.group(0))
            
        except:
            pass        
        try:         
            result1 = re.search(r'\d+.[m*]', duration)
            #print(result1.group(0))
            months=months+int(result1.group(0)[:2])
        except:
            pass
         
        #print("*****************")
    #print("years: ",years)
    #print("months: ", months) 
    
    work_all = experience_section.find_elements_by_class_name('detail-section')
    
    for items in work_all:
        work=[]
        work_details=[]
        print("=================================") 
        work_details.append(items.find_element_by_xpath('.//dl/dt').get_attribute("textContent")) 
        work_details.append(items.find_element_by_xpath('.//dl/dd[1]').get_attribute("textContent"))    
        work_details.append(items.find_element_by_xpath('.//dl/dd[2]').get_attribute("textContent"))
        work.append({"work details":work_details})    
        works.append({"work":work})    
    user.append({"works":works})    
        
    
    total_experience_years=years+(int(months/12))
    total_experience_string=" Year(s) "+ str(months%12) +" month(s)"
    user.append({"total exp":total_experience_string})
    with open('out.txt', 'a') as f:
        print("Total experience: ",total_experience_string,file=f )
    users.append(user)
    #driver.close()
with open('out.json', 'w+') as f:
    print(json.dumps(users),file=f)