npm-trail-crawler-v1

crawler.js

const puppeteer = require("puppeteer");
const fs = require("fs");
const moment = require("moment");
const _ = require("lodash");

const baseId = "#ContentPlaceHolder1_";

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto(
    "https://npm.cpami.gov.tw/apply_1_2.aspx?unit=e6dd4652-2d37-4346-8f5d-6e538353e0c2"
  );
  await page.click("input[id='chk[]0']");
  await page.click("input[id='chk[]']");
  await page.click(`${baseId}btnagree`);
  await page.waitForNavigation();

  // get trails
  let trails = await page.evaluate(
    ({ baseId }) => {
      const options = [
        ...document.querySelectorAll(
          `${baseId}climblinemain > option:not([value=''])`
        )
      ].map(option => ({ value: option.value, name: option.text }));
      return Promise.resolve(options);
    },
    { baseId }
  );

  // get subTrails
  for (let i = 0; i < trails.length; i++) {
    await select(page, "#ContentPlaceHolder1_climblinemain", trails[i].value);
    await page.waitForSelector(`a[title='${trails[i].name}']`);
    const subTrails = await page.evaluate(() => {
      const options = [
        ...document.querySelectorAll(
          "select#ContentPlaceHolder1_climbline > option:not([value=''])"
        )
      ].map(option => ({ value: option.value, name: option.text }));
      return Promise.resolve(options);
    });
    trails[i].subTrails = subTrails;
  }

  // get totalDays
  for (let i = 0; i < trails.length; i++) {
    await select(page, `${baseId}climblinemain`, trails[i].value);
    await page.waitForSelector(`a[title='${trails[i].name}']`);
    const trail = trails[i];
    for (let j = 0; j < trail.subTrails.length; j++) {
      const subTrail = trail.subTrails[j];
      await select(page, `${baseId}climbline`, subTrail.value);
      await page.waitForSelector(
        `${baseId}sumday > option[selected='selected'][value='']`
      );
      const totalDays = await page.evaluate(
        ({ baseId }) => {
          const options = [
            ...document.querySelectorAll(
              `${baseId}sumday > option:not([value=''])`
            )
          ].map(option => ({ value: option.value, name: option.text }));
          return Promise.resolve(options);
        },
        { baseId }
      );
      trails[i].subTrails[j].totalDays = totalDays;
    }
  }

  fs.writeFile("trails.json", JSON.stringify(trails));
  browser.close();
})();

async function select(page, selector, value) {
  await page.evaluate(
    ({ selector, value }) => {
      const element = document.querySelector(selector);
      element.value = value;
      if ("createEvent" in document) {
        const evt = document.createEvent("HTMLEvents");
        evt.initEvent("change", false, true);
        element.dispatchEvent(evt);
      } else {
        element.fireEvent("onchange");
      }
      return Promise.resolve();
    },
    { selector, value }
  );
}