ManageBac Endpoint Requests

readme.md

ManageBac Endpoint Requests

Quickstart

Copy the Requests.gs file into your project. Use it:

function processEndpoint() {
  // Create the requests object, applying headers and query on every request
  var requests = Requests({
    headers: {"auth-token": "secret"},
    query: {"per_page": 20},
  });
                            
  // Make a get request with at the endpoint, and collect all the pages together
  var responses = requests.get('http://api.managebac.com/v2/students')
                          .paged('students');

  return Requests.utils.flatten(responses['students']);
}

The function processEndpoint connects with ManageBac's APIs and downloads student information. It does it by creating a requests object with get and paged methods. The former downloads the first page, and the latter will download the remaining pages asyncronously (using UrlFetchApp.fetchAll).

The responses object will be a concat result of all the above requests, including a students property with all students from every page. So then we use the Requests.utils.flatten function to turns all the student objects into Spreadsheet-friendly rows (with headers in the first row and corresponding values in the 2nd+ rows).

Write to the spreadsheet!

license.gs

/*
Copyright <YEAR> <COPYRIGHT HOLDER>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

MBRequests.gs

(function (global, Package) {
  global.Requests = (function wrapper (args) { 
    var wrapped = function () { return Package.apply(Package, arguments); } 
    for (i in args) { wrapped[i] = args[i]; }
    return wrapped;
   }({
    utils: {
    
      /*
        Merge keys from target into source
      */
      merge: function (source, target) {
        if (!source) { // TypeError if undefined or null
          return target;
        }
        
        var to = Object(source);
        
        if (target != null) { // Skip over if undefined or null
          for (var key in target) {
            if (Object.prototype.hasOwnProperty.call(target, key)) {
              to[key] = target[key];
            }
          }
        }
        
        return to;
      },
      
      /*
        Dotize: https://github.com/vardars/dotize/blob/master/src/dotize.js
      */
      dotize: function(obj, prefix) {
          var newObj = {};
      
          if ((!obj || typeof obj != "object") && !Array.isArray(obj)) {
              if (prefix) {
                  newObj[prefix] = obj;
                  return newObj;
              } else {
                  return obj;
              }
          }
      
          function isNumber(f) {
              return !isNaN(parseInt(f));
          }
      
          function isEmptyObj(obj) {
              for (var prop in obj) {
                  if (Object.hasOwnProperty.call(obj, prop))
                      return false;
              }
          }
      
          function getFieldName(field, prefix, isRoot, isArrayItem, isArray) {
              if (isArray)
                  return (prefix ? prefix : "") + (isNumber(field) ? "[" + field + "]" : (isRoot ? "" : ".") + field);
              else if (isArrayItem)
                  return (prefix ? prefix : "") + "[" + field + "]";
              else
                  return (prefix ? prefix + "." : "") + field;
          }
      
          return function recurse(o, p, isRoot) {
              var isArrayItem = Array.isArray(o);
              for (var f in o) {
                  var currentProp = o[f];
                  if (currentProp && typeof currentProp === "object") {
                      if (Array.isArray(currentProp)) {
                          newObj = recurse(currentProp, getFieldName(f, p, isRoot, false, true), isArrayItem); // array
                      } else {
                          if (isArrayItem && isEmptyObj(currentProp) == false) {
                              newObj = recurse(currentProp, getFieldName(f, p, isRoot, true)); // array item object
                          } else if (isEmptyObj(currentProp) == false) {
                              newObj = recurse(currentProp, getFieldName(f, p, isRoot)); // object
                          } else {
                              //
                          }
                      }
                  } else {
                      if (isArrayItem || isNumber(f)) {
                          newObj[getFieldName(f, p, isRoot, true)] = currentProp; // array item primitive
                      } else {
                          newObj[getFieldName(f, p, isRoot)] = currentProp; // primitive
                      }
                  }
              }
      
              return newObj;
          }(obj, prefix, true);
      },


      /*
        Flatten list into of rows with objects into list, first row being headers
      */
      flatten: function (rows, options) {
        options = options || {};
        options.pathDelimiter = options.pathDelimiter || '.';
        var headers;
        rows = rows.map(function (row) {
          return Requests.utils.dotize(row);
        });
        headers = rows.reduce(function (everyHeader, row) {
          var prop;
          for (prop in row) {
            everyHeader.push( prop );
          }
          return everyHeader;
        }, []);

        var mappedHeaders, finalHeaders;
        mappedHeaders = headers.map(function (el, i) {
          return { 
            index: i, 
            value: el === 'id' ? '' : el.toLowerCase()
          }
        });
        mappedHeaders.sort(function (a, b) {
          if (a.value > b.value) return 1;
          if (a.value < b.value) return -1;
          return 0;
        });
        finalHeaders = mappedHeaders.map(function (el) {
          return headers[el.index];
        }).filter(function (value, index, self) {
          return self.indexOf(value) === index;
        });

        return rows.reduce(function (acc, obj) {
          var row, value;
          row = [];
          for (var h=0; h < finalHeaders.length; h++) {
            value = obj[finalHeaders[h]];
            if (typeof value === 'undefined') value = "";
            row.push(value);
          }
          acc.push(row);
          return acc;
        }, [finalHeaders]);
  
       }

     },
   }
   ))
})(this, 

function Package_ (config) {

  var Response = function (_resp) {
  
    return {
    
      json: function () {
        try {
          return JSON.parse(this.text());
        } catch (err) {
          Logger.log(err);
          Logger.log(this.request.getUrl());
          Logger.log(this.text());
          throw Error("Response did not return a parsable json object");
        }
      },
      
      text: function () {
        return _resp.getContentText();
      },
      
      statusCode: function () {
        return _resp.getResponseCode();
      },
      
      getAllHeaders: function () {
        return _resp.getAllHeaders();
      },

      /*
        Return true if encounted rate limit
      */
      hitRateLimit: function () {
        if (this.statusCode() === 429) {
          var headers = this.getAllHeaders();
          var header_reset_at = headers['x-ratelimit-reset'];
          header_reset_at = header_reset_at.replace(" UTC", "+0000").replace(" ", "T");
          var reset_at = new Date(header_reset_at).getTime();
          var utf_now = new Date().getTime();
          var milliseconds = reset_at - utf_now + 10;
          if (milliseconds > 0) {
            Logger.log("Sleeping for " + (milliseconds/1000).toString() + " seconds.");
            Utilities.sleep(milliseconds);
          } 
          return true;
        }
        return false;
      },
      
      /*
      Take the response, detect paging and combine them into one new kind of response
      */
      paged: function (path) {
        if (typeof path === 'undefined') throw Error('Specify path');
        var json;
        json = this.json();
        if (!json.meta || !json.meta.total_pages) {
          throw Error("Paged called with incomplete or missing meta property")
        } else {
          var page, batch, req, rawRequest;
          batch = new BatchRequests();
          page = 2;
          while (json.meta.total_pages >= page) {
            req = this.request;
            req.setQuery('page', page);
            rawRequest = req.toRequestObject();
            batch.add( rawRequest );
            page++;
          }
          return batch.fetchAll().zip(path, json);
        }
      },
      
    }
  };
  
  var BatchResponses = function (_responses) {
    _responses = _responses || [];
    return {
      zip: function (path, options) {
        var json;
        options = options || {};
        json = options.initialValue || [];
        options.everyObj = options.everyObj || {};

        _responses.forEach(function (resp) {
          var j, req, match;
          j = resp.json();
          req = resp.request;
          if (j[path].length > 0) {
            j[path].forEach(function (o) {
              for (var key in options.everyObj) {
                if (options.everyObj[key] instanceof RegExp) {
                  match = req.getUrl().match(options.everyObj[key]);
                  o[key] = match[1];
                } else {
                  o[key] = options.everyObj[key];
                }
              }
            });
            Array.prototype.push.apply(json, j[path]);
          }
        });
        return json;
      },      
    
      getResponses: function () {
        return _responses;
      }
    }
  };

  var BatchRequests = function(_list) {
    _list = _list || [];
    return {
    
      fetchAll: function (expandForPages) {
        expandForPages = expandForPages || false;
        var responses, expandedRequests;
        expandedRequests = new BatchRequests();
        responses = UrlFetchApp.fetchAll(_list).reduce(function (acc, response, index) {
          var resp, origRequest ,url;
          origRequest = _list[index];
          resp = new Response(response);
          url = origRequest.url;
          delete origRequest.url;
          resp.request = new Request(url, origRequest);

          if (resp.hitRateLimit()) {
            var request, r;
            r = resp.request;
            resp = resp.request.fetch();
            resp.request = r;
          }
          acc.push(resp);
          if (expandForPages && (function (r, er) {
            var json, page, req, rawRequest;
            json = r.json();
            page = 2;
            while ((json.meta || {total_page: -1}).total_pages >= page) {
              req = r.request;
              req.setQuery('page', page);
              rawRequest = req.toRequestObject();
              er.add( rawRequest );
              page++;
            }
          })(resp, expandedRequests));
          return acc;
        }, []);
        
        if (expandForPages && (function (resps) {
          var newResponses;
          if (expandedRequests.length === 0) return;  // no need to continue
          newResponses = expandedRequests.fetchAll(false);
          Array.prototype.push.apply(resps, newResponses.getResponses());
        })(responses));
        
        return new BatchResponses(responses);
      },
      
      add: function (item) {
        _list.push(item);
      },
      
      length: function (item) {
        return _list.length;
      }
    };
  };

  var Request = function (_url, _fetchParams, _options) {
    _url = _url || config.baseUrl;
    _options = _options || {};
    _options.query = _options.query || {};
    _fetchParams['muteHttpExceptions'] = true;
    _fetchParams['payload'] = Requests.utils.merge(_options.body, config.body);
    _fetchParams['headers'] = Requests.utils.merge(_options.headers, config.headers);
    return {
    
      /*
        Returns a custom response object that includes a copy of me
      */
      build: function () {
        var url, req, resp, reply;
        
        reply = UrlFetchApp.fetch(this.getUrl(), _fetchParams);
        resp = new Response(reply);
        resp.request = this;
        return resp;
      },

      /*
        Fetches external resource, handling any API rate limitations
      */
      fetch: function () {
        var resp;
        resp = this.build();
        if (resp.hitRateLimit()) {
          resp = this.build();
        }
        return resp;
      },
      
      getUrl: function () {
        var obj = Requests.utils.merge(_options.query, config.query);
        if (_url.indexOf('?') !== -1) _url = _url.slice(0, _url.indexOf('?'));
        return _url + "?" + Object.keys(obj).reduce(function(a,k){a.push(k+'='+encodeURIComponent(obj[k]));return a},[]).join('&');
      },
      
      setQuery: function (key, value) {
        _options.query[key] = value;
      },
      
      toRequestObject: function () {
        return Requests.utils.merge({url: this.getUrl()}, _fetchParams);
      }
    };
  };

  return {
    get: function (url, options) {
      var req, resp;
      req = new Request(url, {method: 'get'}, options);
      resp = req.fetch();
      return resp;
    },
    
    batchGet: function (urlTemplate, items, path, options) {
      var requests, batchRequests;
      requests = items.reduce(function (acc, item) {
        var url, req, reqObj;
        if (typeof item[item.length-1] === 'object') {
          options = Requests.utils.merge(item[item.length-1], options);
          delete item[item.length-1];
        }
        item.splice(0, 0, urlTemplate);
        url = Utilities.formatString.apply(Utilities.formatString, item);
        req = new Request(url, {method: 'get'}, options);
        reqObj = req.toRequestObject();
        acc.push(reqObj);
        return acc;
      }, []);
      return new BatchRequests(requests).fetchAll(true);
    },
    
    post: function (url, options) {
      var r = new Request(url, {method: 'post'}, options);
    },
    put: function (url, options) {
      var r = new Request(url, {method: 'put'}, options);
    },
    delete_: function (url, options) {
      var r = new Request(url, {method: 'delete'}, options);
    },
    head: function (url, options) {
      var r = new Request(url, {method: 'head'}, options);
    },
    options: function (url, options) {
      var r = new Request(url, {method: 'options'}, options);
    },
  };
}

);