RecordLinkFetcherSima.py

#!/usr/bin/env python3
import requests
import bs4
from urllib.parse import unquote
import json
import getpass

def getLoginData()->dict:
    global simaHost,session

    data = {}

    username = input("enter username: ")
    password = getpass.getpass("enter password: ") 
    
    data['username'] = username
    data['password'] = password

    resp = session.get(url=simaHost)

    soup = bs4.BeautifulSoup(resp.content,'lxml')
    try:
        logintoken = soup.find('input',attrs={'name':'logintoken','type':'hidden'})['value']
    except:
        print("Error: login token not found...")
        exit(0)


    data['logintoken'] = logintoken

    return data

def login()->list:
    loginPath='login/index.php'
    data = getLoginData()
    courseList = []
    
    resp = session.post(url=simaHost+loginPath,data=data)
    soup = bs4.BeautifulSoup(resp.content,'lxml')

    myCourse = soup.find('ul',attrs={'class':'unlist'})
    if myCourse == None:
        print("username or password is wrong can not login...")
        exit(0)
    for atag in myCourse.findAll('a'):
        courseList.append(atag['href'])
    
    return mapLinks(courseList)

def mapLinks(courselist:list)->list:
    global session
    trueLinks = []
    for courseLink in courselist:
        course_main_page = session.get(courseLink)
        soup = bs4.BeautifulSoup(course_main_page.content,'lxml')

        actualCoursePage = soup.findAll('div',attrs={'class':'activityinstance'})
        link = actualCoursePage[1].find('a')['href']
        trueLinks.append(link)
    return trueLinks

def getCourseInfo(courseLink:str)->dict:
    global session
    courseInfo ={}
    courseInfo['link'] = courseLink
    mainPage = session.get(url=courseLink)
    soup = bs4.BeautifulSoup(mainPage.content,'lxml')
    
    def extractTitle():
        cname = soup.find('title').text
        cpart = cname.split(':')
    
        courseInfo['code'] = cpart[1].strip()
        courseInfo['name'] = cpart[0].replace('کلاس آنلاین','').strip()
    extractTitle() 

    def fetchRecordTable():
        recordList=[]
        recordTable = soup.find('div',attrs={'id':'bigbluebuttonbn_recordings_table'})
        if recordTable == None:
            print("it's not available in this time =)")
            exit(0)
        for record in recordTable.findAll('tr'):
            record_detail = {}
            colRecList =  record.findAll('td')
            
            if len(colRecList) == 5:
                record_detail['time'] = colRecList[4].text
                record_detail['date'] = colRecList[3].text
            recordLink = record.find('a')
            
            if recordLink != None :
                longURL=unquote(recordLink['data-href'])
                record_detail['href'] = longURL.split("href=")[1]
            
            if len(record_detail) !=0:
                recordList.append(record_detail)
        courseInfo['records'] = recordList
    
    fetchRecordTable()
    print(courseInfo['name'],courseInfo['code'])
    return courseInfo



def main():
    courseList = login()
    mainList=[]
    for course in courseList:
        mainList.append(getCourseInfo(course))
    
    datajson = json.dumps(mainList,indent=4,ensure_ascii=False)
    with open('result.json','w') as file:
        file.write(datajson)

if __name__ == '__main__':
    simaHost = "http://sima.iaun.ac.ir/"
    session =  requests.Session()
    main()

requirements.txt

requests==2.24.0
beautifulsoup4==4.9.3