crawler-cloud-fcn.js

/**Cloud Function for prerending
Pages that need indexing have been generated for:  account.html, contact.html, deletion.html, donate.html
faq.html, home.html, privacy.html,tos.html
All other pages require authentication and are not indexed by search engines
 * **/

var isnull = v => v === null || typeof v === 'undefined' 
var suf = (uri, suffix, delim) => uri.toLowerCase().endsWith(delim+suffix)
var m = (ua, piece) => ua.toLowerCase().indexOf(piece) >= 0
var is_suffix = (uri, list, delim) => {
    if (isnull(uri)) return false
    var ar = list.split('|')
    for (var i=0; i < ar.length; i++)
    {  if (suf(uri, ar[i], delim)) return true }
    return false
}
var is_match = (ua, list) => {
    if (isnull(ua)) return false
    var ar = list.split('|')
    for (var i=0; i < ar.length; i++)
    {  if (m(ua, ar[i])) return true }
    return false
}

function handler(event) {
                var request = event.request;
                var uri = request.uri;
                var headers = request.headers;
                var cookies = request.cookies;
                var ua = isnull(headers["user-agent"])?"":headers["user-agent"].value
                var host = isnull(headers["host"])?"":headers["host"].value
                var user_cookie = null
                if (!isnull(cookies))
                    user_cookie = isnull(cookies["user"])?null:cookies["user"].value
                var doPrerender = false
                if (uri==='/' || is_suffix(uri, "home|tos|privacy|account|faq|donate|contact|deletion|index.html", '/'))
                   doPrerender = true
                if (doPrerender && ua && host && isnull(user_cookie)) {
                  var prerender = is_match(ua, "googlebot|adsbot-google|Feedfetcher-Google|bingbot|yandex|baiduspider|Facebot|facebookexternalhit|twitterbot|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator")
                      if (prerender || (!isnull(request.querystring) && !isnull(request.querystring._escaped_fragment_) && !isnull(request.querystring._escaped_fragment_.value)))
                         prerender = true
                  if (prerender)
                  {   if (is_suffix(uri, "js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot", '.'))
                         prerender = false
                  }
                  if (prerender) {
                      if (isnull(uri) || uri === '/' || uri === '/home' || uri === '/index.html')
                          request.uri = '/crawlers/home.html'
                      else if (uri.endsWith('/deletion'))
                          request.uri = '/crawlers/deletion.html'
                      else
                          request.uri = "/crawlers" + request.uri + ".html"
                  }
               }
               return request
            }