Grab TJNU Student Grades

grab_grades.go

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"golang.org/x/text/encoding/simplifiedchinese"
	"gopkg.in/xmlpath.v1"
)

type GradeRecord struct {
	Name     string
	Credit   float32
	Category string
	Score    float32
}

type StudentRecord struct {
	Dept      string
	Class     string
	StudentId string
	Name      string
	Grades    []GradeRecord
}

func getRemotePage(userCode string) (string, error) {
	const url = "http://202.113.110.23:8088/tjsfjw/student/xscj.stuckcj_data.jsp"

	form := make(map[string][]string)
	form["sjxz"] = []string{"sjxz1"}
	form["ysyx"] = []string{"yscj"}
	form["zx"] = []string{"1"}
	form["fx"] = []string{"1"}
	form["userCode"] = []string{userCode}
	form["xypjwchcnckcj"] = []string{"0"}
	form["pjwchckcjklpbcj"] = []string{"0"}
	form["xn1"] = []string{"2017"}
	form["ysyxS"] = []string{"on"}
	form["sjxzS"] = []string{"on"}
	form["zxC"] = []string{"on"}
	form["fxC"] = []string{"on"}
	form["menucode_current"] = []string{"JW1314"}

	if resp, err := http.PostForm(url, form); err == nil {
		if data, err := ioutil.ReadAll(resp.Body); err == nil {
			decoder := simplifiedchinese.GBK.NewDecoder()
			str, _ := decoder.String(string(data))
			return str, nil
		} else {
			return "", err
		}
	} else {
		return "", err
	}
}

func parsePage(html string) (*StudentRecord, error) {
	record := StudentRecord{}

	node, err := xmlpath.ParseHTML(strings.NewReader(html))
	if err != nil {
		return nil, err
	}

	// Block for parsing basic information:
	{
		path := xmlpath.MustCompile("//div[@group='group']/div")
		iter := path.Iter(node)
		for iter.Next() {
			line := iter.Node().String()
			cols := strings.Split(line, "：")
			switch cols[0] {
			case "院(系)/部":
				record.Dept = cols[1]
				break
			case "行政班级":
				record.Class = cols[1]
				break
			case "学号":
				record.StudentId = cols[1]
				break
			case "姓名":
				record.Name = cols[1]
			}
		}
	}

	// Block for parsing grades:
	{
		path := xmlpath.MustCompile("//tbody/tr")
		iter := path.Iter(node)
		for iter.Next() {
			tr := iter.Node()
			gradeRecord := GradeRecord{}

			tdPath := xmlpath.MustCompile("td")
			tdIter := tdPath.Iter(tr)
			currentCol := 0
			for tdIter.Next() {
				tdLine := tdIter.Node().String()
				switch currentCol {
				case 1:
					gradeRecord.Name = tdLine
					break
				case 2:
					credit, _ := strconv.ParseFloat(tdLine, 32)
					gradeRecord.Credit = float32(credit)
					break
				case 3:
					gradeRecord.Category = tdLine
					break
				case 7:
					score, _ := strconv.ParseFloat(tdLine, 32)
					gradeRecord.Score = float32(score)
					break
				}
				currentCol++
			}

			record.Grades = append(record.Grades, gradeRecord)
		}
	}

	return &record, nil
}

func fetchRecord(userCode string) *StudentRecord {
	if page, err := getRemotePage(userCode); err == nil {
		if record, err := parsePage(page); err == nil {
			return record
		}
	}

	return nil
}

func worker(requests <-chan string, results chan<- *StudentRecord) {
	for request := range requests {
		if record := fetchRecord(request); record != nil {
			results <- record
		}
	}
}

func consumer(results <-chan *StudentRecord, count *int32, done chan<- bool) {
	for result := range results {
		fmt.Printf("%+v\n", result)

		atomic.AddInt32(count, -1)
		if atomic.LoadInt32(count) == 0 {
			// Potential completion.
			time.Sleep(time.Second)
			if atomic.LoadInt32(count) == 0 {
				done <- true
				return
			}
		}
	}
}

func main() {
	startPtr := flag.Int("start", 280, "start of the fetching range")
	endPtr := flag.Int("end", 320, "end of the fetching range")

	flag.Parse()

	requests := make(chan string, 8)
	results := make(chan *StudentRecord, 8)
	done := make(chan bool)

	numToConsume := int32(*endPtr - *startPtr + 1)

	// Start up to 8 goroutines to make HTTP requests and parsings.
	for i := 0; i < 8; i++ {
		go worker(requests, results)
	}

	// Start a goroutine for printing.
	go consumer(results, &numToConsume, done)

	for j := *startPtr; j <= *endPtr; j++ {
		requests <- fmt.Sprintf("201616%d", j)
	}
	close(requests)

	<-done
}