Link Headers in Rendr

readme.md

This is a set of changes used to add the ability to read link headers for pagination as is becoming the new standard.

https://developer.github.com/v3/#link-header

I'm leveraging the meta property off of the model/collection and putting them in there.

Code for link headers:

• app_collections_base.js, actually app/collections/base.js
• app_models_base.js, actually app/models/base.js
• app_shared_syncer.js, actually app/shared/syncer.js
• server_apiProxy.js, actually server/apiProxy.js

Sample code:

• app/templates/categories/index_sample.hbs
• app/controllers/categories_controller.js
• app/routes.js
• index.js

app_collections_base.js

var RendrBase = require('rendr/shared/base/collection'),
    _ = require('underscore'),
    syncer = require('../shared/syncer');

_.extend(RendrBase.prototype, syncer);

module.exports = RendrBase.extend({});

app_models_base.js

var RendrBase = require('rendr/shared/base/model'),
    _ = require('underscore'),
    syncer = require('../shared/syncer');

_.extend(RendrBase.prototype, syncer);

module.exports = RendrBase.extend({});
    

app_shared_syncer.js

/**
 * `syncer` is a collection of instance methods that are mixed into the prototypes
 * of `BaseModel` and `BaseCollection`. The purpose is to encapsulate shared logic
 * for fetching data from the API.
 */

var _ = require('underscore'),
    debug = require('debug')('rendr:Syncer'),
    Backbone = require('backbone'),

    // Pull out params in path, like '/users/:id'.
    extractParamNamesRe = /:([a-z_-]+)/ig,

    methodMap = {
        'create': 'POST',
        'update': 'PUT',
        'delete': 'DELETE',
        'read': 'GET'
    },

    isServer = (typeof window === 'undefined');

if (isServer) {
    // hide it from requirejs since it's server only
    var serverOnly_qs = 'qs2';
    var qs = require(serverOnly_qs);
} else {
    Backbone.$ = window.$ || require('jquery');
}

var syncer = module.exports;

function parseLinkHeader(linkHeader) {
    var PARAM_TRIM_RE = /[\s'"]/g;
    var URL_TRIM_RE = /[<>\s'"]/g;
    var links = {};
    if (linkHeader) {
        var relations = ["first", "prev", "next"];
        _.each(linkHeader.split(","), function(linkValue) {
            var linkParts = linkValue.split(";");
            var url = linkParts[0].replace(URL_TRIM_RE, '');
            var params = linkParts.slice(1);
            _.each(params, function(param) {
                var paramParts = param.split("=");
                var key = paramParts[0].replace(PARAM_TRIM_RE, '');
                var value = paramParts[1].replace(PARAM_TRIM_RE, '');
                if (key == "rel" && _.contains(relations, value)) links[value] = url;
            });
        });
    }
    return links;
}

function parseLinks(resp, options) {
    var linkHeader = options.xhr.getResponseHeader("Link");
    return parseLinkHeader(linkHeader);
}


function clientSync(method, model, options) {
  
    var error;
    options = _.clone(options);
    if (!_.isUndefined(options.data)) options.data = _.clone(options.data);
    options.url = this.getUrl(options.url, true, options.data);

    // Don't use the data if it's a read since it's been added already into the url
    if (method === 'read') {
        delete options.data;
    }

    error = options.error;
    if (error) {
        options.error = function(xhr) {
            var body = xhr.responseText,
                contentType = xhr.getResponseHeader('content-type'),
                resp;
            if (contentType && contentType.indexOf('application/json') !== -1) {
                try {
                    body = JSON.parse(body);
                } catch (e) {}
            }
            resp = {
                body: body,
                status: xhr.status
            };
            error(resp);
        };
    }

    var success = options.success;
    options.success = function(resp, status, xhr) {

        var newLinks = parseLinks(resp, _.extend({
            xhr: xhr
        }, options));
        if (newLinks) {
            // Add the links to the model.
            model.meta = newLinks;
            try {
                model.meta.total = parseInt(options.xhr.getResponseHeader('total'));
            } catch (error) {}
        }

        if (success) success(resp, status, xhr);
    };

    return Backbone.sync(method, model, options);
}
 
function serverSync(method, model, options) {
    var api, urlParts, verb, req, queryStr;

    options = _.clone(options);
    if (!_.isUndefined(options.data)) options.data = _.clone(options.data);
    options.url = this.getUrl(options.url, false, options.data);
    verb = methodMap[method];
    urlParts = options.url.split('?');
    req = this.app.req;
    queryStr = urlParts[1] || '';

    if (!_.isEmpty(options.data)) queryStr += '&' + qs.stringify(options.data);
    /**
     * if queryStr is initially an empty string, leading '&' will still get parsed correctly by qs.parse below.
     * e.g.  qs.parse('&baz=quux') => { baz: 'quux' }
     */

    api = {
        method: verb,
        path: urlParts[0],
        query: qs.parse(queryStr),
        headers: options.headers || {},
        api: _.result(this, 'api'),
        body: {}
    };

    if (verb === 'POST' || verb === 'PUT') {
        api.body = model.toJSON();
    }

    req.dataAdapter.request(req, api, function(err, response, body) {
        var resp;
        if (err) {
            resp = {
                body: body,
                // Pass through the statusCode, so lower-level code can handle i.e. 401 properly.
                status: err.status
            };

            if (options.error) {
                // This `error` has signature of $.ajax, not Backbone.sync.
                options.error(resp);
            } else {
                throw err;
            }
        } else {

            // This `success` has signature of $.ajax, not Backbone.sync.
            // Check for the link headers in the response and add them to the model.
            if (response.headers) {
                if (response.headers.link) {
                    var links = parseLinkHeader(response.headers.link);
                    model.meta = links;
                    if (response.headers.total) {
                        try {
                            model.meta.total = parseInt(response.headers.total);
                        } catch (error) {

                        }
                    }
                }
            }

            // This `success` has signature of $.ajax, not Backbone.sync.
            options.success(body);
        }
    });
}

syncer.clientSync = clientSync;
syncer.serverSync = serverSync;
syncer.sync = function sync() {
    var syncMethod = isServer ? serverSync : clientSync;
    return syncMethod.apply(this, arguments);
};

/**
 * 'model' is either a model or collection that
 * has a 'url' property, which can be a string or function.
 */
syncer.getUrl = function getUrl(url, clientPrefix, params) {
    if (clientPrefix == null) {
        clientPrefix = false;
    }
    params = params || {};
    url = url || _.result(this, 'url');
    if (clientPrefix && !~url.indexOf('://')) {
        url = this.formatClientUrl(url, _.result(this, 'api'));
    }
    return this.interpolateParams(this, url, params);
};


syncer.formatClientUrl = function(url, api) {
    var prefix = this.app.get('apiPath') || '/api';
    if (api) {
        prefix += '/' + api;
    }
    prefix += '/-';
    return prefix + url;
};

/**
 * Deeply-compare two objects to see if they differ.
 */
syncer.objectsDiffer = function objectsDiffer(data1, data2) {
    var changed = false,
        keys,
        key,
        value1,
        value2;

    keys = _.unique(_.keys(data1).concat(_.keys(data2)));
    for (var i = 0, len = keys.length; i < len; i++) {
        key = keys[i];
        value1 = data1[key];
        value2 = data2[key];

        // If attribute is an object recurse
        if (_.isObject(value1) && _.isObject(value2)) {
            changed = this.objectsDiffer(value1, value2);
            // Test for equality
        } else if (!_.isEqual(value1, value2)) {
            changed = true;
        }
    }
    return changed;
};


/**
 * This maps i.e. '/listings/:id' to '/listings/3' if
 * the model you supply has model.get('id') == 3.
 */
syncer.interpolateParams = function interpolateParams(model, url, params) {
    var matches = url.match(extractParamNamesRe);

    params = params || {};

    if (matches) {
        matches.forEach(function(param) {
            var property = param.slice(1),
                value;

            // Is collection? Then use options.
            if (model.length != null) {
                value = model.options[property];

                // Otherwise it's a model; use attrs.
            } else {
                value = model.get(property);
            }
            url = url.replace(param, value);

            /**
             * Delete the param from params hash, so we don't get urls like:
             * /v1/threads/1234?id=1234...
             */
            delete params[property];
        });
    }
    /**
     * Separate deletion of idAttribute from params hash necessary if using urlRoot in the model
     * so we don't get urls like: /v1/threads/1234?id=1234
     */
    delete params[model.idAttribute];

    return url;
};

categories_controller.js

module.exports = {
    index_sample: function(params, callback) {

        var spec = null;
        spec = {
            categories: {
                collection: 'Categories',
                params: params
            }
        };

        this.app.fetch(spec, function(err, result) {
            callback(err, result);
        });
    }
};

index.js

var express = require('express'),
    rendr = require('rendr'),
    mw = require('./server/middleware'),
    serveStatic = require('serve-static'),
    compress = require('compression'),
    bodyParser = require('body-parser'),
    customApiProxy = require('./server/middleware/apiProxy'),
    request = require('request'),
    app = express(),
    config = require('config');

/**
 * Initialize Express middleware stack.
 */
app.use(compress());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
  extended: true
})); 

app.use(serveStatic(__dirname + '/public'));

app.use(cookieParser());

app.use(session({
    secret: config.session.secret,
    resave: false,
    saveUninitialized: true
}));

/**
 * In this simple example, the DataAdapter config, which specifies host, port, etc. of the API
 * to hit, is written inline. In a real world example, you would probably move this out to a
 * config file. Also, if you want more control over the fetching of data, you can pass your own
 * `dataAdapter` object to the call to `rendr.createServer()`.
 */
var dataAdapterConfig = {
    'default': {
        host: config.api.default.host,
        protocol: config.api.default.protocol
    }
};

/**
 * Initialize our Rendr server.
 */ 
var server = rendr.createServer({
    apiProxy: customApiProxy,
    dataAdapterConfig: dataAdapterConfig
});

/**
 * To mount Rendr, which owns its own Express instance for better encapsulation,
 * simply add `server` as a middleware onto your Express app.
 * This will add all of the routes defined in your `app/routes.js`.
 * If you want to mount your Rendr app onto a path, you can do something like:
 *
 *     app.use('/my_cool_app', server);
 */
app.use('/', server.expressApp);

/**
 * Start the Express server.
 */

function start() {
    var port = process.env.PORT || config.server.port;
    app.listen(port);
    console.log("server pid %s listening on port %s in %s mode",
        process.pid,
        port,
        app.get('env')
    );
}

server.configure(function(rendrExpressApp) {
    // Set the config into the app
    rendrExpressApp.use(function(req, res, next) {
        var app = req.rendrApp;
        app.set('config', config);
        next();
    });
    rendrExpressApp.use(mw.initSession());
});

/**
 * Only start server if this script is executed, not if it's require()'d.
 * This makes it easier to run integration tests on ephemeral ports.
 */
if (require.main === module) {
    start();
}

exports.app = app;

index_sample.hbs

<div class="container">
    Total results: {{categories.meta.total}}
    {{#categories.meta.first}}
        <a href="/categories?page={{getParameterByName this 'page'}}">First</a>
    {{/categories.meta.first}}
    {{#categories.meta.prev}}
        <a href="/categories?page={{getParameterByName this 'page'}}">Prev</a>
    {{/categories.meta.prev}}
    {{#categories.meta.next}}
        <a href="/categories?page={{getParameterByName this 'page'}}">Next</a>
    {{/categories.meta.next}}
</div>

routes.js

module.exports = function(match) {
  match('categories',           'categories#index_sample');
  match('categories?*qs',       'categories#index_sample');
};

server_middleware_apiProxy.js

var _ = require('underscore');

/**
 * We have copied the base Rendr /server/middleware/apiProxy.js here and
 * have added in support for passing through our defined headers for
 * etag, cache-control, total, and links to support link headers.
 */

/**
 * The separator used in the path. Incoming paths can look like:
 *   /-/path/to/resource
 *   /api-name/-/path/to/resource
 */
var separator = '/-/';

/**
 * Middleware handler for intercepting API routes.
 */
module.exports = apiProxy;

function apiProxy(dataAdapter) {
    return function(req, res, next) {
        var api;

        api = _.pick(req, 'query', 'method', 'body');

        api.path = apiProxy.getApiPath(req.path);
        api.api = apiProxy.getApiName(req.path);
        api.headers = {
            'x-forwarded-for': apiProxy.getXForwardedForHeader(req.headers, req.ip)
        };

        dataAdapter.request(req, api, {
            convertErrorCode: false
        }, function(err, response, body) {
            if (err) return next(err);

            // Pass through statusCode.
            res.status(response.statusCode);

            if (response && response.headers) {
                if (response.headers.link) {
                    res.set('link', response.headers.link);
                }
                if (response.headers.etag) {
                    res.set('etag', response.headers.etag);
                }
                if (response.headers.total) {
                    res.set('total', response.headers.total);
                }
                if (response.headers['cache-control']) {
                    res.set('cache-control', response.headers['cache-control']);
                }
            }
            
            if (!response.jsonp){
                res.json(body);
            } else {
                res.jsonp(body);
            }            
        });
    };
}

apiProxy.getApiPath = function getApiPath(path) {
    var sepIndex = path.indexOf(separator),
        substrIndex = sepIndex === -1 ? 0 : sepIndex + separator.length - 1;
    return path.substr(substrIndex);
};

apiProxy.getApiName = function getApiName(path) {
    var sepIndex = path.indexOf(separator),
        apiName = null;
    if (sepIndex > 0) {
        apiName = path.substr(1, sepIndex - 1);
    }
    return apiName;
};

apiProxy.getXForwardedForHeader = function(headers, clientIp) {
    var existingHeader = headers['x-forwarded-for'],
        newHeaderValue = clientIp;

    if (existingHeader) {
        newHeaderValue = existingHeader + ', ' + clientIp;
    }

    return newHeaderValue;
};