Data cleaning / processing for https://beta.observablehq.com/@lorenries/where-did-people-start-end-capital-bikeshare-trips-in-2017

index.js

// load libraries
const fs = require("fs");
const LineByLineReader = require("line-by-line");

// load our station location data into memory
const bikeshareLocations = JSON.parse(
  fs.readFileSync("./processed-locations.json")
);
const lr = new LineByLineReader("./data/csv/2017clean.csv");

// read the main file line by line
// by default, the line-by-line module will use a readable stream
// instead of loading the whole thing into memory
lr.on("error", function(err) {
  console.log(err);
});

lr.on("line", function(line) {
  // parse the comma separated line into an array of columns
  const columns = line.split(",");
  const startHour = new Date(columns[0]).getHours();
  const endHour = new Date(columns[1]).getHours();
  const startStationId = columns[2];
  const endStationId = columns[3];

  // for every bike trip in the big csv, increment a counter for the
  // number of rides at that hour at that bikeshare station
  if (bikeshareLocations.hasOwnProperty(startStationId)) {
    bikeshareLocations[startStationId].startHours[startHour].rides++;
  }
  if (bikeshareLocations.hasOwnProperty(endStationId)) {
    bikeshareLocations[endStationId].endHours[endHour].rides++;
  }
});

lr.on("end", function() {
  // create a geojson file for the number of rides per hour at each station
  const gj = { type: "FeatureCollection", features: [] };
  for (const prop in bikeshareLocations) {
    const val = bikeshareLocations[prop];
    gj.features.push({
      type: "Feature",
      geometry: {
        type: "Point",
        coordinates: [parseFloat(val.lon), parseFloat(val.lat)]
      },
      properties: {
        id: parseInt(val.id),
        address: val.address,
        startHours: val.startHours,
        endHours: val.endHours
      }
    });
  }
  fs.writeFileSync("./processed.json", JSON.stringify(gj));
});

locations.js

const fs = require("fs");

const csvFilePath = "./data/locations.json";

fs.readFile(csvFilePath, (err, data) => {
  if (err) throw err;
  const jsonObj = JSON.parse(data);
  const obj = {};
  jsonObj.forEach(val => {
    obj[val.id] = {
      ...val,
      startHours: hours(),
      endHours: hours()
    };
  });
  console.log(obj);
  fs.writeFile(
    "processed-locations.json",
    JSON.stringify(obj),
    "utf8",
    function(err) {
      // throws an error, you could also catch it here
      if (err) throw err;
      // success case, the file was saved
      console.log("done");
    }
  );
});

function hours() {
  const obj = {};
  for (let i = 0; i <= 23; i++) {
    obj[i] = { rides: 0 };
  }
  return obj;
}

processed.json