NCNU Course Crawler

NCNUCourseCrawler.js

window.NCNUCourseCrawler = (_ => {
	const recperpage = 600;
	const COURSE_NUM = 1376;
	const PAGES = Math.ceil(COURSE_NUM / recperpage);
	const YEAR = 1042;
	let courseList = [];

	function start() {
		_getCourseList();
		courseList.forEach((val, i) => {
			_getCourseDetail(val.courseid, val.zclass);
		});
		// _getCourseDetail(courseList[0].courseid, courseList[0].zclass);
	}

	function _getCourseList(pageno = 1) {
		console.log(`Crawling page ${pageno}.`);
		$.ajax({
			url: 'https://ccweb.ncnu.edu.tw/student/aspmaker_course_opened_detail_viewlist.asp', 
			type: 'get', 
			dataType: 'html', 
			data: {
				pageno, 
				recperpage, 
				cmd: 'search', 
				t: 'aspmaker_course_opened_detail_view', 	
				z_zyear: 'BETWEEN', 
				x_zyear: YEAR, 
				y_zyear: YEAR
			}, 
			async: false, 
			success(data) {
				let $wrap = $('<div>');
				$wrap.append($(data));
				let courseTableRow = $wrap.find('#tbl_aspmaker_course_opened_detail_viewlist tbody tr');
				$.each(courseTableRow, (i, val) => {
					courseList = [...courseList, {
						zclass: $(val).find('td[data-name=zclass]').text().trim(), 
						courseid: $(val).find('td[data-name=courseid]').text().trim()
					}];
				});
				// has next page
				// let lastPage = $wrap.find('.ewPrevNext').first().find('div').last().find('a').first().hasClass('disabled');
				if(pageno < PAGES) {
					_getCourseList(+pageno + 1);
				} 
			}, 
			error(jqXHR) {
				console.log('GET_COURSE_LIST_ERR:');
				console.dir(jqXHR);
			}
		})
	}

	function _getCourseDetail(courseid, zclass) {
		$.ajax({
			url: 'https://ccweb.ncnu.edu.tw/student/aspmaker_course_opened_detail_viewview.asp', 
			type: 'get', 
			dataType: 'html', 
			data: {
				zyear: YEAR, 
				courseid, 
				zclass
			}, 
			success(data) {
				let $wrap = $('<div>');
				$wrap.html(data);
				let course = {
					courseid, 
					zclass, 
					cname: $wrap.find('#el_aspmaker_course_opened_detail_view_cname').text().trim(), 
					dept: $wrap.find('#el_aspmaker_course_opened_detail_view_sname').text().trim(), 
					grade: $wrap.find('#el_aspmaker_course_opened_detail_view_grade').text().trim(), 
					teacher: $wrap.find('#el_aspmaker_course_opened_detail_view_teachers').text().trim(), 
					location: $wrap.find('#el_aspmaker_course_opened_detail_view_place').text().trim(), 
					time: $wrap.find('#el_aspmaker_course_opened_detail_view_ztime').text().trim(), 
					summary: $wrap.find('#el_aspmaker_course_opened_detail_view_syllabus span').html().trim().replace(/<br>/g, '\r\n')
				};
				console.log(course);
				// Do whatever you want to do with each course data
			}, 
			error(jqXHR) {
				console.log('GET_COURSE_DETAIL_ERR: ');
				console.dir(jqXHR);
			}
		})
	}

	return {
		start
	};
})();