Downloading homeworks from moodle

homeworks.py

# -- coding: utf-8 --

# Koduylesannete alla laadimine
from __future__ import print_function
from pyquery import PyQuery as pq
import mechanize
import cookielib
import getpass
import os
import re

moodle_home = r"https://moodle.ut.ee/"
cookie_file = r"cookie.lwp"
course_id = 500 # Programmeerimine

def setup_browser(cookie_file):
    # Browser
    browser = mechanize.Browser()
    browser.addheaders = [('User-agent', 
        'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
    browser.set_handle_robots(False)
    browser.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    browser.set_handle_equiv(True)
    browser.set_handle_gzip(True)
    browser.set_handle_redirect(True)
    browser.set_handle_referer(True)

    # browser.set_debug_http(True)
    # browser.set_debug_redirects(True)
    # browser.set_debug_responses(True)
    # Cookie Jar
    cookie_jar = cookielib.LWPCookieJar(cookie_file)
    if os.access(cookie_file, os.F_OK):
        cookie_jar.load(ignore_discard=True)
    browser.set_cookiejar(cookie_jar)
    return browser, cookie_jar


def logged_in():
    return browser.geturl() != moodle_home


def login():
    browser.select_form(nr=0)
    browser.form["username"] = raw_input("Moodle username: ")
    browser.form["password"] = getpass.getpass("Password: ")
    browser.submit()
    cookie_jar.save(ignore_discard = True)


def list_assignments(html):
    " Returns a list of assignments - list of tuples (name, url) "

    def extract(assignment):
        q = pq(assignment)
        name = q(".instancename").text()
        url = q("a")[0].attrib["href"]
        return name, url
    q = pq(html)
    assignment_items = q(".course-content .modtype_assignment")
    return list(map(extract, assignment_items))


def choose_assignment(selected_index = None):
    assignments = list_assignments(browser.response().read())
    while selected_index is None:
        print("")
        for i, (desc, url) in enumerate(assignments):
            print(u"{i:2d}. {desc} ({url})".format(i=i, desc=desc, url=url))
        selected = raw_input("\nPlease enter task index: ")
        if selected.isdigit() and int(selected) < len(assignments):
            selected_index = int(selected)
    desc, url = assignments[selected_index]
    print("\nSelected assignment %d: %s at %s" % (selected_index, desc, url))
    return desc, url


def select_practice_group(selected_group = None):
    # open the table
    browser.follow_link(text_regex="Vaata esitatud")

    browser.select_form(nr=0)
    q = pq(browser.response().read())
    options = q("select[name=group] > option")
    group_names = map(lambda e: pq(e).text(), options)
    group_values = map(lambda e: pq(e).val(), options)

    while selected_group not in group_names:
        names = u", ".join(group_names)
        prompt = "\nPlease enter one of the following "+names+"\n> "
        selected_group = raw_input(prompt.encode("utf-8"))
    index = group_names.index(selected_group)
    browser.form["group"] = [group_values[index]]
    browser.submit() 


def get_solutions():
    q = pq(browser.response().read())
    rows = q("form table tbody").children("tr:not(.emptyrow)")

    get_name = lambda el: pq(el).children(".fullname").text()
    def get_solution_url(row):
        solution_link = pq(row).find(".files a")
        if solution_link:
            return solution_link[0].attrib['href']

    return [(get_name(r), get_solution_url(r)) for r in rows]


def convert_name(name):
    """ Converts a name with special characters to a  lowercase string 
        with words separated with underscores and special characters 
        replaced with similar ascii characters.
        
        > convert_name('Kärt-jaan jõemüür')
        'kart_jaan_joemuur'
        """
    name = unicode(name)
    import unicodedata
    name = unicodedata.normalize('NFKD', name).encode('ascii','ignore')
    return "_".join(re.split(" |-", name.lower()))


def extract_solution(url, target_folder):
    import shutil
    try: shutil.rmtree(target_folder)
    except: pass

    from pyunpack import Archive
    archive, _ = browser.retrieve(url)
    Archive(archive).extractall(target_folder, True)
    os.remove(archive)


def download_solutions(folder_prefix = None):
    if folder_prefix is None:
        folder_prefix = raw_input("\nPlease enter where to put the solutions (. for current folder): ")
        print("")
    for name, archive_url in get_solutions():
        if not archive_url:
            print(u"No solution uploaded from {}".format(name))
            continue
        target_folder = os.path.join(folder_prefix, convert_name(name))
        extract_solution(archive_url, target_folder)
        print(u"Extracted solution from {} to {}".format(name, target_folder))

if __name__ == "__main__":
    browser, cookie_jar = setup_browser(cookie_file)
    browser.open(moodle_home)
    if not logged_in():
        print("Not logged in. Please authenticate. At", browser.geturl())
        login()
    assert browser.geturl() == 'https://moodle.ut.ee/my/'
    print("\nLogged in successfully.")

    # Go to course page
    browser.open("https://moodle.ut.ee/course/view.php?id="+str(course_id))

    # Select assignment to download
    task_desc, task_url = choose_assignment()
    browser.open(task_url)

    # Choose group
    select_practice_group()

    download_solutions()