Ubuntu 16.04 - Build Elasticsearch Site with Google authentication and authorization by Openresty-Nginx Lua Script

access.lua

-- Copyright 2015-2016 CloudFlare
-- Copyright 2014-2015 Aaron Westendorf

local json = require("cjson")
local http = require("resty.http")

local uri         = ngx.var.uri
local uri_args    = ngx.req.get_uri_args()
local scheme      = ngx.var.scheme

local client_id         = ngx.var.ngo_client_id
local client_secret     = ngx.var.ngo_client_secret
local token_secret      = ngx.var.ngo_token_secret
local domain            = ngx.var.ngo_domain
local cb_scheme         = ngx.var.ngo_callback_scheme or scheme
local cb_server_name    = ngx.var.ngo_callback_host or ngx.var.server_name
local cb_uri            = ngx.var.ngo_callback_uri or "/_oauth"
local cb_url            = cb_scheme .. "://" .. cb_server_name .. cb_uri
local redirect_url      = cb_scheme .. "://" .. cb_server_name .. ngx.var.request_uri
local signout_uri       = ngx.var.ngo_signout_uri or "/_signout"
local extra_validity    = tonumber(ngx.var.ngo_extra_validity or "0")
local whitelist         = ngx.var.ngo_whitelist or ""
local blacklist         = ngx.var.ngo_blacklist or ""
local secure_cookies    = ngx.var.ngo_secure_cookies == "true" or false
local http_only_cookies = ngx.var.ngo_http_only_cookies == "true" or false
local set_user          = ngx.var.ngo_user or false
local email_as_user     = ngx.var.ngo_email_as_user == "true" or false

if whitelist:len() == 0 then
  whitelist = nil
end

if blacklist:len() == 0 then
  blacklist = nil
end

local function handle_token_uris(email, token, expires)
  if uri == "/_token.json" then
    ngx.header["Content-type"] = "application/json"
    ngx.say(json.encode({
      email   = email,
      token   = token,
      expires = expires,
    }))
    ngx.exit(ngx.OK)
  end

  if uri == "/_token.txt" then
    ngx.header["Content-type"] = "text/plain"
    ngx.say("email: " .. email .. "\n" .. "token: " .. token .. "\n" .. "expires: " .. expires .. "\n")
    ngx.exit(ngx.OK)
  end

  if uri == "/_token.curl" then
    ngx.header["Content-type"] = "text/plain"
    ngx.say("-H \"OauthEmail: " .. email .. "\" -H \"OauthAccessToken: " .. token .. "\" -H \"OauthExpires: " .. expires .. "\"\n")
    ngx.exit(ngx.OK)
  end
end


local function on_auth(email, token, expires)
  local oauth_domain = email:match("[^@]+@(.+)")

  ngx.log(ngx.ERR, "email: " .. email .. "; token: " .. token .. "; expires: " .. expires)
  ngx.log(ngx.ERR, "domain: " .. domain .. "; oauth_domain: " .. oauth_domain)
  if not (whitelist or blacklist) then
    if domain:len() ~= 0 then
      if not string.find(" " .. domain .. " ", " " .. oauth_domain .. " ") then
        ngx.log(ngx.ERR, email .. " is not on " .. domain)
        return ngx.exit(ngx.HTTP_FORBIDDEN)
      end
    end
  end

  if whitelist then
    if not string.find(" " .. whitelist .. " ", " " .. email .. " ") then
      ngx.log(ngx.ERR, email .. " is not in whitelist")
      return ngx.exit(ngx.HTTP_FORBIDDEN)
    end
  end

  if blacklist then
    if string.find(" " .. blacklist .. " ", " " .. email .. " ") then
      ngx.log(ngx.ERR, email .. " is in blacklist")
      return ngx.exit(ngx.HTTP_FORBIDDEN)
    end
  end

  if set_user then
    if email_as_user then
      ngx.var.ngo_user = email
    else
      ngx.var.ngo_user = email:match("([^@]+)@.+")
    end
  end

  handle_token_uris(email, token, expires)
end

local function request_access_token(code)
  local request = http.new()

  request:set_timeout(7000)

  local res, err = request:request_uri("https://accounts.google.com/o/oauth2/token", {
    method = "POST",
    body = ngx.encode_args({
      code          = code,
      client_id     = client_id,
      client_secret = client_secret,
      redirect_uri  = cb_url,
      grant_type    = "authorization_code",
    }),
    headers = {
      ["Content-type"] = "application/x-www-form-urlencoded"
    },
    ssl_verify = true,
  })
  if not res then
    return nil, (err or "auth token request failed: " .. (err or "unknown reason"))
  end

  if res.status ~= 200 then
    return nil, "received " .. res.status .. " from https://accounts.google.com/o/oauth2/token: " .. res.body
  end

  return json.decode(res.body)
end

local function request_profile(token)
  local request = http.new()

  request:set_timeout(7000)

  local res, err = request:request_uri("https://www.googleapis.com/oauth2/v2/userinfo", {
    headers = {
      ["Authorization"] = "Bearer " .. token,
    },
    ssl_verify = true,
  })
  if not res then
    return nil, "auth info request failed: " .. (err or "unknown reason")
  end

  if res.status ~= 200 then
    return nil, "received " .. res.status .. " from https://www.googleapis.com/oauth2/v2/userinfo"
  end
  ngx.log(ngx.ERR, "profile: " .. res.body)
  return json.decode(res.body)
end

local function is_authorized()
  local headers = ngx.req.get_headers()

  local expires = tonumber(ngx.var.cookie_OauthExpires) or 0
  local email   = ngx.unescape_uri(ngx.var.cookie_OauthEmail or "")
  local token   = ngx.unescape_uri(ngx.var.cookie_OauthAccessToken or "")

  if expires == 0 and headers["oauthexpires"] then
    expires = tonumber(headers["oauthexpires"])
  end

  if email:len() == 0 and headers["oauthemail"] then
    email = headers["oauthemail"]
  end

  if token:len() == 0 and headers["oauthaccesstoken"] then
    token = headers["oauthaccesstoken"]
  end

  local expected_token = ngx.encode_base64(ngx.hmac_sha1(token_secret, cb_server_name .. email .. expires))

  if token == expected_token and expires and expires > ngx.time() - extra_validity then
    on_auth(email, expected_token, expires)
    return true
  else
    return false
  end
end

local function redirect_to_auth()
  -- google seems to accept space separated domain list in the login_hint, so use this undocumented feature.
  return ngx.redirect("https://accounts.google.com/o/oauth2/auth?" .. ngx.encode_args({
    client_id     = client_id,
    scope         = "email",
    response_type = "code",
    redirect_uri  = cb_url,
    state         = redirect_url,
    login_hint    = domain,
  }))
end

local function authorize()
  if uri ~= cb_uri then
    return redirect_to_auth()
  end

  if uri_args["error"] then
    ngx.log(ngx.ERR, "received " .. uri_args["error"] .. " from https://accounts.google.com/o/oauth2/auth")
    return ngx.exit(ngx.HTTP_FORBIDDEN)
  end

  local token, token_err = request_access_token(uri_args["code"])
  if not token then
    ngx.log(ngx.ERR, "got error during access token request: " .. token_err)
    return ngx.exit(ngx.HTTP_FORBIDDEN)
  end

  local profile, profile_err = request_profile(token["access_token"])
  if not profile then
    ngx.log(ngx.ERR, "got error during profile request: " .. profile_err)
    return ngx.exit(ngx.HTTP_FORBIDDEN)
  end

  local expires      = ngx.time() + token["expires_in"]
  local cookie_tail  = ";version=1;path=/;Max-Age=" .. expires
  if secure_cookies then
    cookie_tail = cookie_tail .. ";secure"
  end
  if http_only_cookies then
    cookie_tail = cookie_tail .. ";httponly"
  end

  local email      = profile["email"]
  local user_token = ngx.encode_base64(ngx.hmac_sha1(token_secret, cb_server_name .. email .. expires))

  on_auth(email, user_token, expires)

  ngx.header["Set-Cookie"] = {
    "OauthEmail="       .. ngx.escape_uri(email) .. cookie_tail,
    "OauthAccessToken=" .. ngx.escape_uri(user_token) .. cookie_tail,
    "OauthExpires="     .. expires .. cookie_tail,
  }

  return ngx.redirect(uri_args["state"])
end

local function handle_signout()
  if uri == signout_uri then
    ngx.header["Set-Cookie"] = "OauthAccessToken==deleted; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT"
    return ngx.redirect("/")
  end
end

handle_signout()

if not is_authorized() then
  authorize()
end

--[[

Provides Elasticserach endpoint authorization based on rules in Lua and authenticated user

See the `nginx_authorize_by_lua.conf` for the Nginx config.

Synopsis:

$ /usr/local/openresty/nginx/sbin/nginx -p $PWD/nginx/ -c $PWD/nginx_authorize_by_lua.conf

$ curl -i -X HEAD 'http://localhost:8080'
HTTP/1.1 401 Unauthorized

curl -i -X HEAD 'http://all:all@localhost:8080'
HTTP/1.1 200 OK

curl -i -X GET 'http://all:all@localhost:8080'
HTTP/1.1 403 Forbidden

curl -i -X GET 'http://user:user@localhost:8080'
HTTP/1.1 200 OK

curl -i -X GET 'http://user:user@localhost:8080/_search'
HTTP/1.1 200 OK

curl -i -X POST 'http://user:user@localhost:8080/_search'
HTTP/1.1 200 OK

curl -i -X GET 'http://user:user@localhost:8080/_aliases'
HTTP/1.1 200 OK

curl -i -X POST 'http://user:user@localhost:8080/_aliases'
HTTP/1.1 403 Forbidden

curl -i -X POST 'http://user:user@localhost:8080/myindex/mytype/1' -d '{"title" : "Test"}'
HTTP/1.1 403 Forbidden

curl -i -X DELETE 'http://user:user@localhost:8080/myindex/'
HTTP/1.1 403 Forbidden

curl -i -X POST 'http://admin:admin@localhost:8080/myindex/mytype/1' -d '{"title" : "Test"}'
HTTP/1.1 200 OK

curl -i -X DELETE 'http://admin:admin@localhost:8080/myindex/mytype/1'
HTTP/1.1 200 OK

curl -i -X DELETE 'http://admin:admin@localhost:8080/myindex/'
HTTP/1.1 200 OK

]]--

-- authorization rules

local restrictions = {
  all  = {
    ["^/$"]                             = { "HEAD" }
  },

  user = {
    ["^/$"]                             = { "GET" },
    ["^/?[^/]*/?[^/]*/_search"]         = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/_msearch"]        = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/_validate/query"] = { "GET", "POST" },
    ["/_aliases"]                       = { "GET" },
    ["/_cluster.*"]                     = { "GET" }
  },

  admin = {
    ["^/?[^/]*/?[^/]*/_bulk"]          = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/_refresh"]       = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/?[^/]*/_create"] = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/?[^/]*/_update"] = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/?.*"]            = { "GET", "POST", "PUT", "DELETE" },
    ["^/?[^/]*/?[^/]*$"]               = { "GET", "POST", "PUT", "DELETE" },
    ["/_aliases"]                      = { "GET", "POST" }
  },

  ["username"] = {
    ["^/?[^/]*/?[^/]*/_bulk"]          = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/_refresh"]       = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/?[^/]*/_create"] = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/?[^/]*/_update"] = { "GET", "POST" },
    ["^/?[^/]*/?[^/]*/?.*"]            = { "GET", "POST", "PUT", "DELETE" },
    ["^/?[^/]*/?[^/]*$"]               = { "GET", "POST", "PUT", "DELETE" },
    ["/_aliases"]                      = { "GET", "POST" }
  }
}

-- get authenticated user as role
local role = ngx.var.ngo_user
ngx.log(ngx.DEBUG, role)

-- exit 403 when no matching role has been found
if restrictions[role] == nil then
  ngx.header.content_type = 'text/plain'
  ngx.log(ngx.WARN, "Unknown role ["..role.."]")
  ngx.status = 403
  ngx.say("403 Forbidden: You don\'t have access to this resource.")
  return ngx.exit(403)
end

-- get URL
local uri = ngx.var.uri
ngx.log(ngx.DEBUG, uri)

-- get method
local method = ngx.req.get_method()
ngx.log(ngx.DEBUG, method)

local allowed  = false

for path, methods in pairs(restrictions[role]) do

  -- path matched rules?
  local p = string.match(uri, path)

  local m = nil

  -- method matched rules?
  for _, _method in pairs(methods) do
    m = m and m or string.match(method, _method)
  end

  if p and m then
    allowed = true
    ngx.log(ngx.NOTICE, method.." "..uri.." matched: "..tostring(m).." "..tostring(path).." for "..role)
    break
  end
end

if not allowed then
  ngx.header.content_type = 'text/plain'
  ngx.log(ngx.WARN, "Role ["..role.."] not allowed to access the resource ["..method.." "..uri.."]")
  ngx.status = 403
  ngx.say("403 Forbidden: You don\'t have access to this resource.")
  return ngx.exit(403)
end

elasticsearch.conf

# Nginx Configuration
upstream elasticsearch {
  server ElasticSearchIPAddress:9200;
  keepalive 15;
}

server {
  listen 80;

  # These configurations referenced from https://github.com/cloudflare/nginx-google-oauth
  # More details about them will be find there
  resolver 8.8.8.8 ipv6=off;
  set $ngo_client_id         "Google_Client_Id";
  set $ngo_client_secret     "Google_Client_Secret";
  set $ngo_token_secret      "a very long randomish string";
  set $ngo_callback_host     "secure.elasticsearch.com";
  set $ngo_domain            "my_google_private_domain.com"; 
  set $ngo_user              "true";
  set $ngo_secure_cookies    "false";

  access_by_lua_file '/usr/local/openresty/nginx/conf/elasticsearch/access.lua';

  location / {
    # Always deny the request to shutdown server, even from the admin user
    if ($request_filename ~ _shutdown) {
      return 403;
      break;
    }
    proxy_pass http://elasticsearch;
    proxy_redirect off;
    proxy_http_version 1.1;
    proxy_set_header Connection "Keep-Alive";
    proxy_set_header Proxy-Connection "Keep-Alive";
  }
}

install-openresty.sh

# This solution is the combination of:
# 1. Lua module to add Google OAuth to nginx (https://github.com/cloudflare/nginx-google-oauth)
# 2. Elastic Guide: Access Control List with Lua (https://www.elastic.co/blog/playing-http-tricks-nginx) (https://gist.github.com/karmi/b0a9b4c111ed3023a52d#file-authorize-lua)

# The nginx-google-oauth need openresty, therefore we should install openresty instead of the common nginx
sudo service nginx stop
wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
sudo apt-get -y install software-properties-common
sudo add-apt-repository -y "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main"
sudo apt-get update
sudo apt-get install openresty
sudo service openresty start
# Install lua-resty-http package for Google Oauth lua script
sudo opm get pintsized/lua-resty-http
# Openresty will be install into
cd /usr/local/openresty/
# Nginx configuration with Openresty will be in
cd /usr/local/openresty/nginx/
# Define nginx configuration with openresty
sudo mkdir -p /usr/local/openresty/nginx/conf/elasticsearch/
sudo vi /usr/local/openresty/nginx/conf/elasticsearch/elasticsearch.conf 
# Create access.lua 
sudo vi /usr/local/openresty/nginx/conf/elasticsearch/access.lua

# Test the nginx configuration
sudo service openresty configtest
sudo service openresty reload

IssuesAndSolutions.txt

# Issue 1 - lua script variable convention
2017/09/16 03:37:40 [error] 28233#28233: *3524 failed to load external Lua file "/usr/local/openresty/nginx/conf/elasticsearch/authorize.lua": /usr/local/openresty/nginx/conf/elasticsearch/authorize.lua:78: '}' expected (to close '{' at line 54) near 'tankhuu', client: 123.21.100.186, server: , request: "GET / HTTP/1.1", host: "secure.elasticsearch.com"
--> Solution:
In lua script, if a variable contain the special characters such as [email protected], we have to cover it with ["[email protected]"]

# Issue 2 - missing library
2017/09/16 04:34:32 [error] 28996#28996: *6 lua entry thread aborted: runtime error: /usr/local/openresty/nginx/conf/elasticsearch/access.lua:5: module 'resty.http' not found:
	no field package.preload['resty.http']
	no file '/usr/local/openresty/site/lualib/resty/http.ljbc'
	no file '/usr/local/openresty/site/lualib/resty/http/init.ljbc'
	no file '/usr/local/openresty/lualib/resty/http.ljbc'
	no file '/usr/local/openresty/lualib/resty/http/init.ljbc'
	no file '/usr/local/openresty/site/lualib/resty/http.lua'
	no file '/usr/local/openresty/site/lualib/resty/http/init.lua'
	no file '/usr/local/openresty/lualib/resty/http.lua'
	no file '/usr/local/openresty/lualib/resty/http/init.lua'
	no file './resty/http.lua'
	no file '/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/resty/http.lua'
	no file '/usr/local/share/lua/5.1/resty/http.lua'
	no file '/usr/local/share/lua/5.1/resty/http/init.lua'
	no file '/usr/local/openresty/luajit/share/lua/5.1/resty/http.lua'
	no file '/usr/local/openresty/luajit/share/lua/5.1/resty/http/init.lua'
	no file '/usr/local/openresty/site/lualib/resty/http.so'
	no file '/usr/local/openresty/lualib/resty/http.so'
	no file './resty/http.so'
	no file '/usr/local/lib/lua/5.1/resty/http.so'
	no file '/usr/local/openresty/luajit/lib/lua/5.1/resty/http.so'
	no file '/usr/local/lib/lua/5.1/loadall.so'
	no file '/usr/local/openresty/site/lualib/resty.so'
	no file '/usr/local/openresty/lualib/resty.so'
	no file './resty.so'
	no file '/usr/local/lib/lua/5.1/resty.so'
	no file '/usr/local/openresty/luajit/lib/lua/5.1/resty.so'
	no file '/usr/local/lib/lua/5.1/loadall.so'
stack traceback:
coroutine 0:
	[C]: in function 'require'
	/usr/local/openresty/nginx/conf/elasticsearch/access.lua:5: in function </usr/local/openresty/nginx/conf/elasticsearch/access.lua:1>, client: 123.21.100.186, server: , request: "GET / HTTP/1.1", host: "secure.elasticsearch.com"
--> Solution:
just install the missing library with openresty package manager: `sudo opm get pintsized/lua-resty-http`

# Issue 3 - Your web server didn't use SSL (HTTPS) - lua script cant verify the certificate which should be added in 
# Nginx configuration, server location: 
#    ssl_certificate      /etc/nginx/certs/supersecret.net.pem;
#    ssl_certificate_key  /etc/nginx/certs/supersecret.net.key;   
2017/09/16 05:18:48 [error] 2960#2960: *99 lua ssl certificate verify error: (20: unable to get local issuer certificate), client: 123.21.100.186, server: , request: "GET /_oauth?state=http://secure.elasticsearch.com/&code=4/7HzWpxrO-5c3mcxiOLbx6JOFM HTTP/1.1", host: "secure.elasticsearch.com"
--> Solution:
HTTPS is always recommended for your site, but if you can't implement it, then we can by pass this error by changing the option ssl_verify from "true" to "false", which are defined in 2 function of access.lua: request_access_token & request_profile

# Issue 4 - Combine access.lua (from Nginx-Google-Oauth) with authorize.lua (from Elasticsearch)
# script from Elasticsearch need the role to do authorization, which role will be received from access.lua by variable ngx.var.ngo_user
2017/09/16 12:38:04 [error] 5739#5739: *1541 lua entry thread aborted: runtime error: /usr/local/openresty/nginx/conf/elasticsearch/access.lua:343: attempt to concatenate local 'role' (a nil value)
stack traceback:
coroutine 0:
	/usr/local/openresty/nginx/conf/elasticsearch/access.lua: in function </usr/local/openresty/nginx/conf/elasticsearch/access.lua:1>, client: 123.21.100.186, server: , request: "GET /_oauth?state=http://secure.elasticsearch.com/&code=4/KUkAX2fDPHBP-cUPrQE7lVLvcpsCiA HTTP/1.1", host: "secure.elasticsearch.com"
--> Solution:
# Change the line 340 in access.lua from:
-- get authenticated user as role
local role = ngx.var.remote_user
# to:
-- get authenticated user as role
local role = ngx.var.ngo_user