check_toronto_open_data.gs

// two google apps script functions to:
// - get email alerts when new datasets are published
// - get email alerts when specified datasets are updated
// update the variables below and set up a trigger to run either function daily

// update these two
const DATASETS_TO_WATCH_FOR_UPDATES = [
  'cycling-network', // example, replace with the ids for your dataset of interest (can find in the page URL)
];
const NOTIFY_EMAIL = 'youremail@example.com'; // REPLACE WITH YOUR EMAIL

const NEW_DATASETS_URL = 'https://ckan0.cf.opendata.inter.prod-toronto.ca/api/3/action/package_search?sort=date_published%20desc';
const SAFETY_BUFFER = 6; // overlap between checks, in hours

function checkForNewOpenData(newDatasetsURL=NEW_DATASETS_URL) {
  newDatasetsURL = NEW_DATASETS_URL; // param assignment not working when run from trigger
  // request datasets from Toronto CKAN, default return page size is 10
  const response = UrlFetchApp.fetch(newDatasetsURL);
  if (response.getResponseCode() !== 200) {
    throw new Error(
      `Request to ${newDatasetsURL} failed with response ${response.getResponseCode()}`
    );
  }
  const responseData = JSON.parse(response);
  const datasets = responseData['result']['results']

  // check to see if any datasets are new within the last 24 hours + safety buffer period
  let newDatasets = [];
  for (const dataset of datasets) {
    let publishDate = new Date(dataset['date_published']);
    if ((new Date() - publishDate)/1000/60/60 < (24 + SAFETY_BUFFER)) {
      newDatasets.push(dataset);
    }
  }

  // send an email if there are new datasets
  if (newDatasets.length > 0) {
    let datasetNamesList = newDatasets.map(dataset => dataset['title']).join(', ');
    let emailSubject = `${newDatasets.length} new Toronto ${
      newDatasets.length > 1 ? 'datasets' : 'dataset'
      }: ${datasetNamesList}`;
    let emailBody = newDatasets.map(dataset => [
      `<h2><a href="${
        'https://open.toronto.ca/dataset/' + dataset['name'] + '/'
        }">${dataset['title']}<a></h2>`,
      '<p>',
      `<b>Published:</b> ${dataset['date_published']}<br>`,
      `<b>Refresh rate:</b> ${dataset?.['refresh_rate'] ?? 'Not specified'}<br>`,
      `<b>Published by:</b> ${dataset?.['owner_division'] ?? 'Not specified'}<br>`,
      `<b>Data type:</b> ${dataset?.['dataset_category'] ?? 'Not specified'}<br>`,
      `<b>Formats:</b> ${(dataset?.['formats'] ?? ['Not specified']).join(', ')}<br>`,
      `<b>License:</b> ${dataset?.['license_title'] ?? 'Not specified'}<br>`,
      `<b>Topics:</b> ${(dataset?.['topics'] ?? ['Not specified']).join(', ')}<br>`,
      '</p>',
      `<p><b>Excerpt:</b> ${dataset?.['excerpt'] ?? 'None'}</p>`,
      `<p><b>Notes:</b> ${dataset?.['notes'] ?? 'None'}</p>`,
      `<p><b>Limitations:</b> ${dataset?.['limitations'] ?? 'None'}</p>`,
    ].join('\n')).join('\n');

    const RE_HTML = /<\/?([bp]|br)>/gi // to help generate plain text version of email
    GmailApp.sendEmail(
      NOTIFY_EMAIL, 
      emailSubject, 
      emailBody.replace(RE_HTML, ''), // plain text version
      { htmlBody: emailBody } // html version
    );
  }  
}

function checkWhenDatasetUpdated(datasetNames=DATASETS_TO_WATCH_FOR_UPDATES) {
  datasetNames = DATASETS_TO_WATCH_FOR_UPDATES; // param assignment not working when run from trigger
  let updatedDatasets = [];

  for (const datasetName of datasetNames) {
    // request datasets from Toronto CKAN, default return page size is 10
    const url = 'https://ckan0.cf.opendata.inter.prod-toronto.ca/api/3/action/package_show?id=' + datasetName;
    const response = UrlFetchApp.fetch(url);
    if (response.getResponseCode() !== 200) {
      throw new Error(
        `Request to ${url} failed with response ${response.getResponseCode()}`
      );
    }
    const responseData = JSON.parse(response);
    const metaData = responseData['result']

    // check to see if dataset was updated within the last 24 hours + safety buffer period
    let updateDate = new Date(metaData['last_refreshed']);
    if ((new Date() - updateDate)/1000/60/60 < (24 + SAFETY_BUFFER)) {
      updatedDatasets.push(metaData);
    }
  }

  // send an email if datasets have been updated
  if (updatedDatasets.length > 0) {
    let datasetNamesList = updatedDatasets.map(dataset => dataset['title']).join(', ');
    let emailSubject = `${updatedDatasets.length} updated Toronto ${
      updatedDatasets.length > 1 ? 'datasets' : 'dataset'
      }: ${datasetNamesList}`;
    let emailBody = updatedDatasets.map(dataset => [
      `<h2><a href="${
        'https://open.toronto.ca/dataset/' + dataset['name'] + '/'
        }">${dataset['title']}<a></h2>`,
      '<p>',
      `<b>Updated:</b> ${dataset['last_refreshed']}<br>`,
      `<b>Published:</b> ${dataset['date_published']}<br>`,
      `<b>Refresh rate:</b> ${dataset?.['refresh_rate'] ?? 'Not specified'}<br>`,
      `<b>Published by:</b> ${dataset?.['owner_division'] ?? 'Not specified'}<br>`,
      `<b>Data type:</b> ${dataset?.['dataset_category'] ?? 'Not specified'}<br>`,
      `<b>Formats:</b> ${(dataset?.['formats'] ?? ['Not specified']).join(', ')}<br>`,
      `<b>License:</b> ${dataset?.['license_title'] ?? 'Not specified'}<br>`,
      `<b>Topics:</b> ${(dataset?.['topics'] ?? ['Not specified']).join(', ')}<br>`,
      '</p>',
      `<p><b>Excerpt:</b> ${dataset?.['excerpt'] ?? 'None'}</p>`,
      `<p><b>Notes:</b> ${dataset?.['notes'] ?? 'None'}</p>`,
      `<p><b>Limitations:</b> ${dataset?.['limitations'] ?? 'None'}</p>`,
    ].join('\n')).join('\n');

    const RE_HTML = /<\/?([bp]|br)>/gi // to help generate plain text version of email
    GmailApp.sendEmail(
      NOTIFY_EMAIL, 
      emailSubject, 
      emailBody.replace(RE_HTML, ''), // plain text version
      { htmlBody: emailBody } // html version
    );
  }  
}