Grading script for COMP2710

COMP2710_GradingScript.md

This gist contains my homework grading scripts for Auburn COMP2017 in Spring semseter. grade1.py for homework 1, grade2.py for homework 2 and so on. Shoot me an email if you have questions.

grade1.py

import os
import re
from zipfile import ZipFile
from subprocess import run, Popen, PIPE
from math import isclose

import numpy as np


scores = {}
# http://stackoverflow.com/a/385597/1429714
re_float = r"[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?"

N = 100
x0 = np.random.random(N)
x1 = np.random.randint(1, 300, (N, 2))
X_test = np.hstack((x0[:, np.newaxis], x1))
y_test = x1[:,1] * 1. / x1[:,0] * x0 / 0.001
devnull = open(os.devnull, 'w')

scores = {}
# stat code: 0 - good, 1 - gccerr, 2 - other

with ZipFile('hw1.zip', 'r') as zf:
    files = zf.namelist()
    n = len(files)
    for i, f in enumerate(files):
        tmp = f.split('_')
        name = tmp[0]
        print('[{0:03d}/{1}] {2}'.format(i+1, n, name),
              end=' ', flush=True)

        if 'late' in tmp:
            pts = int(N * .9)
            print(' late ', end=' ', flush=True)
        else:
            pts = N

        _, ext = os.path.splitext(f)
        ext = ext.lower()
        if ext not in ['.cpp', '.cc', '.c', '.c++']:
            pts = 0
            print('{0} {1}'.format(name, pts))
            scores[name] = {'pts': pts, 'stat': 2}
            continue

        zf.extract(f, path='tmp')
        ret = run(['g++', '-o', 'hw1', 'tmp/'+f], stderr=devnull)
        if 0 != ret.returncode:
            pts = 0
            scores[name] = {'pts': pts, 'stat': 1}
            print(' Compile Error ', end=' ', flush=True)
        else:
            for x, y in zip(X_test, y_test):
                pipe = Popen('./hw1', stdin=PIPE, stdout=PIPE)
                inputs = str(x[0]) + ' ' +\
                         str(int(x[1])) + ' ' +\
                         str(int(x[2]))
                out = pipe.communicate(input=inputs.encode('utf8'))
                out = out[0].decode('utf8')
                matches = re.findall(re_float, out)
                if len(matches) != 1 or\
                   not (isclose(y, float(matches[0]), rel_tol=1e-02) or
                        int(y) == int(float(matches[0]))):
                    pts -= 1
            if pts < 0:
                pts = 0
            scores[name] = {'pts': pts, 'stat': 0 if N == pts else 2}

        print(pts)


stat = ['good', 'compile_error', 'other']
with open('hw1.csv', 'w') as w:
    for k, v in scores.items():
        w.write('{0},{1},{2}\n'.format(k, v['pts'], stat[v['stat']]))

grade2.py

import os
import re
from zipfile import ZipFile
from subprocess import run, Popen, PIPE, TimeoutExpired
import math
from enum import IntEnum

import numpy as np


# solution for hw2
def amortize(loan, rate_per_year, pay_per_month):
    balance = [loan]
    interest = [math.nan]
    principal = [math.nan]
    rate_per_month = rate_per_year * 0.01 / 12

    if loan*rate_per_month < pay_per_month:
        while balance[-1] > 0:
            i = balance[-1] * rate_per_month
            p = pay_per_month - i
            b = balance[-1] - p
            if b < 0:
                p = balance[-1]
                b = 0.
            balance += [b]
            interest += [i]
            principal += [p]

    return balance[1:], interest[1:], principal[1:]


# http://stackoverflow.com/a/385597/1429714
re_float = r'[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?'
re_line_bad = (r'\n\s*\d+' +
               r'\s+\$\s*(?P<balance>' + re_float +
               r')\s+\$\s*' + re_float +
               r'\s+\$?\s*' + re_float +
               r'\s+\$\s*(?P<interest>' + re_float +
               r')\s+\$\s*(?P<principal>' + re_float + ')')
re_line_good = (r'\n\s*\d+' +
                r'\s+\$\s*(?P<balance>' + re_float +
                r')\s+\$\s*' + re_float +
                r'\s+' + re_float +
                r'\s+\$\s*(?P<interest>' + re_float +
                r')\s+\$\s*(?P<principal>' + re_float + ')')
# re_line_bad = (r'\n\s*\d+' +
#                r'\s+(?P<balance>' + re_float +
#                r')\s+' + re_float +
#                r'\s+' + re_float +
#                r'\s+(?P<interest>' + re_float +
#                r')\s+(?P<principal>' + re_float + ')')

N = 10

class Stat(IntEnum):
    GOOD = 0                    # full credit
    LATE = 1                    # late submission
    COMPILE_ERR = 2             # compile error
    DOLLAR_SIGN = 4             # dollar sign before interest
    EDGE_CASE = 8               # edge cases
    OTHER = 16                  # other

CREDIT = 100
Penalty = [0] * 17
Penalty[Stat.GOOD] = 0
Penalty[Stat.LATE] = 10
Penalty[Stat.COMPILE_ERR] = 40
Penalty[Stat.DOLLAR_SIGN] = 5
Penalty[Stat.EDGE_CASE] = max(int(CREDIT*0.5/N), 1)
Penalty[Stat.OTHER] = max(int(CREDIT*0.5/N), 1)

loan = np.random.random(N) * 9000 + 1000
rate = np.random.random(N) * 100
pay = np.random.random(N) * 100 + loan*(rate*0.01/12)
X_test = np.stack([loan, rate, pay], axis=1)

X_edge = np.array([[1000., 10., 1.]])

devnull = open(os.devnull, 'w')
summary = {}

with ZipFile('test.zip', 'r') as zf:
    files = zf.namelist()
    n = len(files)
    for i, f in enumerate(files):
        tmp = f.split('_')
        name = tmp[0]
        print('[{0:03d}/{1}] {2}'.format(i+1, n, name),
              end=' ', flush=True)

        stat = Stat.GOOD
        penalty = 0

        # 10% penalty for late submission
        if 'late' in tmp:
            stat |= Stat.LATE
            penalty += Penalty[Stat.LATE]
            print(' late ', end=' ', flush=True)

        _, ext = os.path.splitext(f)
        ext = ext.lower()
        if ext not in ['.cpp', '.cc', '.c', '.c++']:
            stat |= Stat.COMPILE_ERR
            penalty += min(Penalty[Stat.COMPILE_ERR], CREDIT)
            print('{0} compile error'.format(CREDIT-penalty))
            summary[name] = {'pts': CREDIT-penalty, 'stat': stat}
            continue

        # extract to ./tmp/
        zf.extract(f, path='tmp')

        # compile, and ignore any error messages
        ret = run(['g++', '-o', 'hw2', 'tmp/'+f], stderr=devnull)

        # compilation error
        if 0 != ret.returncode:
            stat |= Stat.COMPILE_ERR
            penalty += Penalty[Stat.COMPILE_ERR]
            summary[name] = {'pts': CREDIT-penalty, 'stat': stat}
            print('{0} compile error'.format(CREDIT-penalty))
            continue

        # Only one edge case, i.e., monthly payment is not enough to
        # pay off debts.  Incorrect implementation never terminates.
        # Wait for 2 seconds.
        for x in X_edge:
            pipe = Popen('./hw2', stdin=PIPE, stdout=PIPE)
            inputs = ' '.join(np.char.mod('%f', x))
            try:
                pipe.communicate(input=inputs.encode('utf8'),
                                 timeout=2)
            except TimeoutExpired:
                pipe.kill()
                stat |= Stat.EDGE_CASE
                penalty += Penalty[Stat.EDGE_CASE]

        # Wow! it compiles!!
        for x in X_test:
            # feed input and grab output from the program
            pipe = Popen('./hw2', stdin=PIPE, stdout=PIPE)
            inputs = ' '.join(np.char.mod('%f', x))
            out = pipe.communicate(input=inputs.encode('utf8'))
            out = out[0].decode('utf8')

            # Extract balance, interest and principal.  Note that
            # there should be NO dollar sign ($) before interest, but
            # some students unintentionally added it.  5% penalty for
            # not following the format.
            matches = re.findall(re_line_good, out)

            if 0 == len(matches):
                matches = re.findall(re_line_bad, out)
                if not (stat & Stat.DOLLAR_SIGN):
                    stat |= Stat.DOLLAR_SIGN
                    penalty += Penalty[Stat.DOLLAR_SIGN]

            y0, y1, y2 = amortize(x[0], x[1], x[2])
            y_test = np.stack([y0, y1, y2], axis=1)
            y_pred = np.array(matches, dtype=np.float32)

            if y_test.shape[0] != y_pred.shape[0]:
                # That the length does not match means your algorithm
                # is totally wrong.
                stat |= Stat.OTHER
                penalty += Penalty[Stat.OTHER]
            else:
                # the values in two arrays should be roughly the same.
                # Due to rounding errors and conversion to/from
                # strings, a large tolerance is used.
                diff = np.max(y_test-y_pred)
                if diff > 0.1:
                    stat |= Stat.OTHER
                    penalty += Penalty[Stat.OTHER]

            if penalty >= CREDIT:
                penalty = CREDIT
                break


        summary[name] = {'pts': CREDIT-penalty, 'stat': stat}
        print(CREDIT-penalty)


def comment(stat):
    ret = []
    if Stat.GOOD == stat:
        return 'GOOD'

    for s in Stat:
        if (s & stat):
            ret += [s.name]
    return ' '.join(ret)


with open('hw2.csv', 'w') as w:
    for k, v in summary.items():
        w.write('{0},{1},{2}\n'.format(k, v['pts'],
                                       comment(v['stat'])))

grade3.py

import os
from zipfile import ZipFile
from subprocess import run, Popen, PIPE, TimeoutExpired


devnull = open(os.devnull, 'w')


with ZipFile('hw3.zip', 'r') as zf:
    files = zf.namelist()
    nb_files = len(files)

    for i, fn in enumerate(files):
        fn_parts = fn.split('_')
        name = fn_parts[0]
        outfn = name

        print('[{0:03d}/{1}] {2}'.format(i+1, nb_files, name),
              end=' ', flush=True)

        # 10% penalty for late submission
        if 'late' in fn_parts:
            outfn += '_late'
            print(' late ', end=' ', flush=True)

        _, ext = os.path.splitext(fn)
        ext = ext.lower()
        if ext not in ['.cpp', '.cc', '.c', '.c++']:
            print('unknown file extension {0}'.format(ext))
            continue

        # extract to ./tmp/
        zf.extract(fn, path='tmp')

        # compile, and ignore any error messages
        ret = run(['g++', '-o', 'hw3', 'tmp/'+fn], stderr=devnull)

        # compilation error
        if 0 != ret.returncode:
            outfn += '_compile'
            print(' compile error')
            continue

        outfn += '.txt'
        with open(outfn, 'w') as output:
        # feed input and grab output from the program
            pipe = Popen('./hw3', stdin=PIPE, stdout=output)
            inputs = '\n'.join(['\n']*20).encode('utf8')
            try:
                out, err = pipe.communicate(input=inputs, timeout=2)
            except TimeoutExpired:
                print('timeout')
                pipe.kill()
            else:
                print('done')

grade4.py

import os
import sys
from subprocess import run, Popen, PIPE, TimeoutExpired

import numpy as np


nb_tests = 100
Input_0 = 'input{0}-0.txt'
Input_1 = 'input{0}-1.txt'

def gen_test(nb_tests=1):
    N = 50
    M = 20
    y_test = []
    for i in range(nb_tests):
        out = np.array([], dtype=int)
        for fn in [Input_0, Input_1]:
            seqlen = np.random.choice(M)
            seq = np.random.choice(N, size=seqlen)
            seq.sort()
            out = np.append(out, seq)
            np.savetxt(fn.format(i), seq, fmt='%d', delimiter='\n')
        out.sort()
        y_test += [out]

    # Generate two canonical input just in case
    for fn in ['input1.txt', 'input2.txt']:
        seqlen = np.random.choice(M)
        seq = np.random.choice(N, size=seqlen)
        seq.sort()
        np.savetxt(fn, seq, fmt='%d', delimiter='\n')

    return y_test


devnull = open(os.devnull, 'w')

fn = 'hw4.cpp'

# compile, and ignore any error messages
ret = run(['g++', '-o', 'hw4', fn], stderr=devnull)

# compilation error
if 0 != ret.returncode:
    print('compile error')
    sys.exit(1)


# Wow! it compiles!!
y_test = gen_test(nb_tests)

num_err = 0
for i in range(nb_tests):
    # feed input and grab output from the program
    pipe = Popen('./hw4', stdin=PIPE, stdout=PIPE, stderr=devnull)
    inputs = '\n'.join(['\n'] * 5 + [Input_0.format(i),
                                     Input_1.format(i), 'tmp.txt'])
    try:
        out, err = pipe.communicate(
            input=inputs.encode('utf8'), timeout=2)
    except:
        pipe.kill()
    else:
        y_true = y_test[i]
        y_pred = np.genfromtxt('tmp.txt')
        if not np.all(y_true==y_pred):
            num_err += 1
            print('test case {0} error'.format(i+1))
        else:
            print('test case {0} pass'.format(i+1))

print('wrong cases: {0}'.format(num_err))