arbiter.diff

diff --git a/vendor/arbiter/arbiter.cpp b/vendor/arbiter/arbiter.cpp
index 797358fdc..6cffbf9ec 100644
--- a/vendor/arbiter/arbiter.cpp
+++ b/vendor/arbiter/arbiter.cpp
@@ -86,13 +86,22 @@ namespace
 
         if (!b.isNull())
         {
-            for (const auto& key : b.getMemberNames())
+            if (b.isObject())
             {
-                // If A doesn't have this key, then set it to B's value.
-                // If A has the key but it's an object, then recursively merge.
-                // Otherwise A already has a value here that we won't overwrite.
-                if (!out.isMember(key)) out[key] = b[key];
-                else if (out[key].isObject()) merge(out[key], b[key]);
+                for (const auto& key : b.getMemberNames())
+                {
+                    // If A doesn't have this key, then set it to B's value.
+                    // If A has the key but it's an object, then recursively
+                    // merge.
+                    // Otherwise A already has a value here that we won't
+                    // overwrite.
+                    if (!out.isMember(key)) out[key] = b[key];
+                    else if (out[key].isObject()) merge(out[key], b[key]);
+                }
+            }
+            else
+            {
+                out = b;
             }
         }
 
@@ -142,24 +151,19 @@ Arbiter::Arbiter(const Json::Value& in)
     auto https(Https::create(*m_pool, json["http"]));
     if (https) m_drivers[https->type()] = std::move(https);
 
-    if (json["s3"].isArray())
-    {
-        for (const auto& sub : json["s3"])
-        {
-            auto s3(S3::create(*m_pool, sub));
-            m_drivers[s3->type()] = std::move(s3);
-        }
-    }
-    else
-    {
-        auto s3(S3::create(*m_pool, json["s3"]));
-        if (s3) m_drivers[s3->type()] = std::move(s3);
-    }
+    auto s3(S3::create(*m_pool, json["s3"]));
+    for (auto& s : s3) m_drivers[s->type()] = std::move(s);
 
     // Credential-based drivers should probably all do something similar to the
     // S3 driver to support multiple profiles.
     auto dropbox(Dropbox::create(*m_pool, json["dropbox"]));
     if (dropbox) m_drivers[dropbox->type()] = std::move(dropbox);
+
+#ifdef ARBITER_OPENSSL
+    auto google(Google::create(*m_pool, json["gs"]));
+    if (google) m_drivers[google->type()] = std::move(google);
+#endif
+
 #endif
 }
 
@@ -308,7 +312,7 @@ void Arbiter::copy(
             {
                 std::cout <<
                     ++i << " / " << paths.size() << ": " <<
-                    path << " -> " << dstEndpoint.fullPath(subpath) <<
+                    path << " -> " << dstEndpoint.prefixedFullPath(subpath) <<
                     std::endl;
             }
 
@@ -775,6 +779,41 @@ void Endpoint::put(
     getHttpDriver().put(path, data, headers, query);
 }
 
+http::Response Endpoint::httpGet(
+        std::string path,
+        http::Headers headers,
+        http::Query query,
+        const std::size_t reserve) const
+{
+    return getHttpDriver().internalGet(fullPath(path), headers, query, reserve);
+}
+
+http::Response Endpoint::httpPut(
+        std::string path,
+        const std::vector<char>& data,
+        http::Headers headers,
+        http::Query query) const
+{
+    return getHttpDriver().internalPut(fullPath(path), data, headers, query);
+}
+
+http::Response Endpoint::httpHead(
+        std::string path,
+        http::Headers headers,
+        http::Query query) const
+{
+    return getHttpDriver().internalHead(fullPath(path), headers, query);
+}
+
+http::Response Endpoint::httpPost(
+        std::string path,
+        const std::vector<char>& data,
+        http::Headers headers,
+        http::Query query) const
+{
+    return getHttpDriver().internalPost(fullPath(path), data, headers, query);
+}
+
 std::string Endpoint::fullPath(const std::string& subpath) const
 {
     return m_root + subpath;
@@ -1347,13 +1386,38 @@ void Http::put(
     }
 }
 
+void Http::post(
+        const std::string path,
+        const std::string& data,
+        const Headers h,
+        const Query q) const
+{
+    return post(path, std::vector<char>(data.begin(), data.end()), h, q);
+}
+
+void Http::post(
+        const std::string path,
+        const std::vector<char>& data,
+        const Headers headers,
+        const Query query) const
+{
+    auto http(m_pool.acquire());
+    auto res(http.post(typedPath(path), data, headers, query));
+
+    if (!res.ok())
+    {
+        std::cout << res.str() << std::endl;
+        throw ArbiterError("Couldn't HTTP POST to " + path);
+    }
+}
+
 Response Http::internalGet(
         const std::string path,
         const Headers headers,
         const Query query,
         const std::size_t reserve) const
 {
-    return m_pool.acquire().get(path, headers, query, reserve);
+    return m_pool.acquire().get(typedPath(path), headers, query, reserve);
 }
 
 Response Http::internalPut(
@@ -1362,7 +1426,7 @@ Response Http::internalPut(
         const Headers headers,
         const Query query) const
 {
-    return m_pool.acquire().put(path, data, headers, query);
+    return m_pool.acquire().put(typedPath(path), data, headers, query);
 }
 
 Response Http::internalHead(
@@ -1370,16 +1434,26 @@ Response Http::internalHead(
         const Headers headers,
         const Query query) const
 {
-    return m_pool.acquire().head(path, headers, query);
+    return m_pool.acquire().head(typedPath(path), headers, query);
 }
 
 Response Http::internalPost(
         const std::string path,
         const std::vector<char>& data,
-        const Headers headers,
+        Headers headers,
         const Query query) const
 {
-    return m_pool.acquire().post(path, data, headers, query);
+    if (!headers.count("Content-Length"))
+    {
+        headers["Content-Length"] = std::to_string(data.size());
+    }
+    return m_pool.acquire().post(typedPath(path), data, headers, query);
+}
+
+std::string Http::typedPath(const std::string& p) const
+{
+    if (Arbiter::getType(p) != "file") return p;
+    else return type() + "://" + p;
 }
 
 } // namespace drivers
@@ -1405,7 +1479,6 @@ Response Http::internalPost(
 // //////////////////////////////////////////////////////////////////////
 
 #ifndef ARBITER_IS_AMALGAMATION
-#include <arbiter/arbiter.hpp>
 #include <arbiter/drivers/s3.hpp>
 #endif
 
@@ -1449,7 +1522,7 @@ namespace
 
     // See:
     // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
-    const std::string credIp("http://169.254.169.254/");
+    const std::string credIp("169.254.169.254/");
     const std::string credBase(
             credIp + "latest/meta-data/iam/security-credentials/");
 
@@ -1516,30 +1589,51 @@ S3::S3(
     , m_config(std::move(config))
 { }
 
-std::unique_ptr<S3> S3::create(Pool& pool, const Json::Value& json)
+std::vector<std::unique_ptr<S3>> S3::create(Pool& pool, const Json::Value& json)
+{
+    std::vector<std::unique_ptr<S3>> result;
+
+    if (json.isArray())
+    {
+        for (const auto& curr : json)
+        {
+            if (auto s = createOne(pool, curr))
+            {
+                result.push_back(std::move(s));
+            }
+        }
+    }
+    else if (auto s = createOne(pool, json))
+    {
+        result.push_back(std::move(s));
+    }
+
+    return result;
+}
+
+std::unique_ptr<S3> S3::createOne(Pool& pool, const Json::Value& json)
 {
     const std::string profile(extractProfile(json));
 
     auto auth(Auth::create(json, profile));
     if (!auth) return std::unique_ptr<S3>();
 
-    auto config(Config::create(json, profile));
-    if (!config) return std::unique_ptr<S3>();
-
+    std::unique_ptr<Config> config(new Config(json, profile));
     return makeUnique<S3>(pool, profile, std::move(auth), std::move(config));
 }
 
 std::string S3::extractProfile(const Json::Value& json)
 {
-    if (auto p = util::env("AWS_PROFILE")) return *p;
-    else if (auto p = util::env("AWS_DEFAULT_PROFILE")) return *p;
-    else if (
+    if (
             !json.isNull() &&
             json.isMember("profile") &&
             json["profile"].asString().size())
     {
         return json["profile"].asString();
     }
+
+    if (auto p = util::env("AWS_PROFILE")) return *p;
+    if (auto p = util::env("AWS_DEFAULT_PROFILE")) return *p;
     else return "default";
 }
 
@@ -1547,38 +1641,41 @@ std::unique_ptr<S3::Auth> S3::Auth::create(
         const Json::Value& json,
         const std::string profile)
 {
-    // Try environment settings first.
+    // Try explicit JSON configuration first.
+    if (
+            !json.isNull() &&
+            json.isMember("access") &&
+            (json.isMember("secret") || json.isMember("hidden")))
+    {
+        return makeUnique<Auth>(
+                json["access"].asString(),
+                json.isMember("secret") ?
+                    json["secret"].asString() :
+                    json["hidden"].asString(),
+                json["token"].asString());
+    }
+
+    // Try environment settings next.
     {
         auto access(util::env("AWS_ACCESS_KEY_ID"));
         auto hidden(util::env("AWS_SECRET_ACCESS_KEY"));
+        auto token(util::env("AWS_SESSION_TOKEN"));
 
         if (access && hidden)
         {
-            return makeUnique<Auth>(*access, *hidden);
+            return makeUnique<Auth>(*access, *hidden, token ? *token : "");
         }
 
         access = util::env("AMAZON_ACCESS_KEY_ID");
         hidden = util::env("AMAZON_SECRET_ACCESS_KEY");
+        token = util::env("AMAZON_SESSION_TOKEN");
 
         if (access && hidden)
         {
-            return makeUnique<Auth>(*access, *hidden);
+            return makeUnique<Auth>(*access, *hidden, token ? *token : "");
         }
     }
 
-    // Try explicit JSON configuration next.
-    if (
-            !json.isNull() &&
-            json.isMember("access") &&
-            (json.isMember("secret") || json.isMember("hidden")))
-    {
-        return makeUnique<Auth>(
-                json["access"].asString(),
-                json.isMember("secret") ?
-                    json["secret"].asString() :
-                    json["hidden"].asString());
-    }
-
     const std::string credPath(
             util::env("AWS_CREDENTIAL_FILE") ?
                 *util::env("AWS_CREDENTIAL_FILE") : "~/.aws/credentials");
@@ -1627,31 +1724,21 @@ std::unique_ptr<S3::Auth> S3::Auth::create(
 }
 
 S3::Config::Config(
-        const std::string region,
-        const std::string baseUrl,
-        const bool sse,
-        const bool precheck)
-    : m_region(region)
-    , m_baseUrl(baseUrl)
-    , m_precheck(precheck)
-{
-    if (sse)
-    {
-        // This could grow to support other SSE schemes, like KMS and customer-
-        // supplied keys.
-        m_baseHeaders["x-amz-server-side-encryption"] = "AES256";
-    }
-}
-
-std::unique_ptr<S3::Config> S3::Config::create(
         const Json::Value& json,
         const std::string profile)
+    : m_region(extractRegion(json, profile))
+    , m_baseUrl(extractBaseUrl(json, m_region))
+    , m_precheck(json["precheck"].asBool())
 {
-    const auto region(extractRegion(json, profile));
-    const auto baseUrl(extractBaseUrl(json, region));
-    const bool sse(json["sse"].asBool());
-    const bool precheck(json["precheck"].asBool());
-    return makeUnique<Config>(region, baseUrl, sse, precheck);
+    if (json["sse"].asBool())
+    {
+        m_baseHeaders["x-amz-server-side-encryption"] = "AES256";
+    }
+
+    if (json["requesterPays"].asBool())
+    {
+        m_baseHeaders["x-amz-request-payer"] = "requester";
+    }
 }
 
 std::string S3::Config::extractRegion(
@@ -1769,7 +1856,10 @@ S3::AuthFields S3::Auth::fields() const
             m_access = creds["AccessKeyId"].asString();
             m_hidden = creds["SecretAccessKey"].asString();
             m_token = creds["Token"].asString();
-            m_expiration.reset(new Time(creds["Expiration"].asString(), arbiter::Time::iso8601));
+            m_expiration.reset(
+                    new Time(
+                        creds["Expiration"].asString(),
+                        arbiter::Time::iso8601));
 
             if (*m_expiration - now < reauthSeconds)
             {
@@ -1806,7 +1896,8 @@ std::unique_ptr<std::size_t> S3::tryGetSize(std::string rawPath) const
             Headers(),
             empty);
 
-    Response res(Http::internalHead(resource.url(), apiV4.headers()));
+    drivers::Http http(m_pool);
+    Response res(http.internalHead(resource.url(), apiV4.headers()));
 
     if (res.ok() && res.headers().count("Content-Length"))
     {
@@ -1820,9 +1911,12 @@ std::unique_ptr<std::size_t> S3::tryGetSize(std::string rawPath) const
 bool S3::get(
         const std::string rawPath,
         std::vector<char>& data,
-        const Headers headers,
+        const Headers userHeaders,
         const Query query) const
 {
+    Headers headers(m_config->baseHeaders());
+    headers.insert(userHeaders.begin(), userHeaders.end());
+
     std::unique_ptr<std::size_t> size(
             m_config->precheck() && !headers.count("Range") ?
                 tryGetSize(rawPath) : nullptr);
@@ -1837,8 +1931,9 @@ bool S3::get(
             headers,
             empty);
 
+    drivers::Http http(m_pool);
     Response res(
-            Http::internalGet(
+            http.internalGet(
                 resource.url(),
                 apiV4.headers(),
                 apiV4.query(),
@@ -1851,8 +1946,7 @@ bool S3::get(
     }
     else
     {
-        std::cout << std::string(res.data().data(), res.data().size()) <<
-            std::endl;
+        std::cout << res.code() << ": " << res.str() << std::endl;
         return false;
     }
 }
@@ -1877,8 +1971,9 @@ void S3::put(
             headers,
             data);
 
+    drivers::Http http(m_pool);
     Response res(
-            Http::internalPut(
+            http.internalPut(
                 resource.url(),
                 data,
                 apiV4.headers(),
@@ -1968,7 +2063,8 @@ std::vector<std::string> S3::glob(std::string path, bool verbose) const
                         // beyond the prefix if recursive is true.
                         if (recursive || !isSubdir)
                         {
-                            results.push_back("s3://" + bucket + "/" + key);
+                            results.push_back(
+                                    type() + "://" + bucket + "/" + key);
                         }
 
                         if (more)
@@ -2147,8 +2243,8 @@ std::string S3::ApiV4::getAuthHeader(
         "Signature=" + signature;
 }
 
-S3::Resource::Resource(std::string baseUrl, std::string fullPath)
-    : m_baseUrl(baseUrl)
+S3::Resource::Resource(std::string base, std::string fullPath)
+    : m_baseUrl(base)
     , m_bucket()
     , m_object()
     , m_virtualHosted(true)
@@ -2159,15 +2255,29 @@ S3::Resource::Resource(std::string baseUrl, std::string fullPath)
     m_bucket = fullPath.substr(0, split);
     if (split != std::string::npos) m_object = fullPath.substr(split + 1);
 
-    m_virtualHosted = m_bucket.find_first_of('.') == std::string::npos;
+    // Always use virtual-host style paths.  We'll use HTTP for our back-end
+    // calls to allow this.  If we were to use HTTPS on the back-end, then we
+    // would have to use non-virtual-hosted paths if the bucket name contained
+    // '.' characters.
+    //
+    // m_virtualHosted = m_bucket.find_first_of('.') == std::string::npos;
+}
+
+std::string S3::Resource::baseUrl() const
+{
+    return m_baseUrl;
+}
+
+std::string S3::Resource::bucket() const
+{
+    return m_virtualHosted ? m_bucket : "";
 }
 
 std::string S3::Resource::url() const
 {
-    // We can't use virtual-host style paths if the bucket contains dots.
     if (m_virtualHosted)
     {
-        return "https://" + m_bucket + "." + m_baseUrl + m_object;
+        return "http://" + m_bucket + "." + m_baseUrl + m_object;
     }
     else
     {
@@ -2212,6 +2322,389 @@ std::string S3::Resource::host() const
 
 
 
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: arbiter/drivers/google.cpp
+// //////////////////////////////////////////////////////////////////////
+
+#ifndef ARBITER_IS_AMALGAMATION
+#include <arbiter/drivers/google.hpp>
+#endif
+
+#include <vector>
+
+#ifdef ARBITER_OPENSSL
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/rsa.h>
+#include <openssl/sha.h>
+#endif
+
+#ifndef ARBITER_IS_AMALGAMATION
+#include <arbiter/arbiter.hpp>
+#include <arbiter/drivers/fs.hpp>
+#include <arbiter/util/transforms.hpp>
+#endif
+
+#ifdef ARBITER_CUSTOM_NAMESPACE
+namespace ARBITER_CUSTOM_NAMESPACE
+{
+#endif
+
+namespace arbiter
+{
+
+namespace
+{
+    std::mutex sslMutex;
+
+    const std::string baseGoogleUrl("www.googleapis.com/storage/v1/");
+    const std::string uploadUrl("www.googleapis.com/upload/storage/v1/");
+    const http::Query altMediaQuery{ { "alt", "media" } };
+
+    class GResource
+    {
+    public:
+        GResource(std::string path)
+        {
+            const std::size_t split(path.find("/"));
+            m_bucket = path.substr(0, split) + "/";
+            if (split != std::string::npos) m_object = path.substr(split + 1);
+        }
+
+        const std::string& bucket() const { return m_bucket; }
+        const std::string& object() const { return m_object; }
+        std::string endpoint() const
+        {
+            // https://cloud.google.com/storage/docs/json_api/#encoding
+            static const std::string exclusions("!$&'()*+,;=:@");
+
+            // https://cloud.google.com/storage/docs/json_api/v1/
+            return
+                baseGoogleUrl + "b/" + bucket() +
+                "o/" + http::sanitize(object(), exclusions);
+        }
+
+        std::string uploadEndpoint() const
+        {
+            return uploadUrl + "b/" + bucket() + "o";
+        }
+
+        std::string listEndpoint() const
+        {
+            return baseGoogleUrl + "b/" + bucket() + "o";
+        }
+
+    private:
+        std::string m_bucket;
+        std::string m_object;
+
+    };
+} // unnamed namespace
+
+namespace drivers
+{
+
+Google::Google(http::Pool& pool, std::unique_ptr<Auth> auth)
+    : Https(pool)
+    , m_auth(std::move(auth))
+{ }
+
+std::unique_ptr<Google> Google::create(
+        http::Pool& pool,
+        const Json::Value& json)
+{
+    if (auto auth = Auth::create(json))
+    {
+        return util::makeUnique<Google>(pool, std::move(auth));
+    }
+
+    return std::unique_ptr<Google>();
+}
+
+std::unique_ptr<std::size_t> Google::tryGetSize(const std::string path) const
+{
+    http::Headers headers(m_auth->headers());
+    const GResource resource(path);
+
+    drivers::Https https(m_pool);
+    const auto res(
+            https.internalHead(resource.endpoint(), headers, altMediaQuery));
+
+    if (res.ok() && res.headers().count("Content-Length"))
+    {
+        const auto& s(res.headers().at("Content-Length"));
+        return util::makeUnique<std::size_t>(std::stoul(s));
+    }
+
+    return std::unique_ptr<std::size_t>();
+}
+
+bool Google::get(
+        const std::string path,
+        std::vector<char>& data,
+        const http::Headers userHeaders,
+        const http::Query query) const
+{
+    http::Headers headers(m_auth->headers());
+    headers.insert(userHeaders.begin(), userHeaders.end());
+    const GResource resource(path);
+
+    drivers::Https https(m_pool);
+    const auto res(
+            https.internalGet(resource.endpoint(), headers, altMediaQuery));
+
+    if (res.ok())
+    {
+        data = res.data();
+        return true;
+    }
+    else
+    {
+        std::cout <<
+            "Failed get - " << res.code() << ": " << res.str() << std::endl;
+        return false;
+    }
+}
+
+void Google::put(
+        const std::string path,
+        const std::vector<char>& data,
+        const http::Headers userHeaders,
+        const http::Query userQuery) const
+{
+    const GResource resource(path);
+    const std::string url(resource.uploadEndpoint());
+
+    http::Headers headers(m_auth->headers());
+    headers["Expect"] = "";
+    headers.insert(userHeaders.begin(), userHeaders.end());
+
+    http::Query query(userQuery);
+    query["uploadType"] = "media";
+    query["name"] = resource.object();
+
+    drivers::Https https(m_pool);
+    const auto res(https.internalPost(url, data, headers, query));
+}
+
+std::vector<std::string> Google::glob(std::string path, bool verbose) const
+{
+    std::vector<std::string> results;
+
+    path.pop_back();
+    const bool recursive(path.back() == '*');
+    if (recursive) path.pop_back();
+
+    const GResource resource(path);
+    const std::string url(resource.listEndpoint());
+    std::string pageToken;
+
+    drivers::Https https(m_pool);
+    http::Query query;
+
+    // When the delimiter is set to "/", then the response will contain a
+    // "prefixes" key in addition to the "items" key.  The "prefixes" key will
+    // contain the directories found, which we will ignore.
+    if (!recursive) query["delimiter"] = "/";
+    if (resource.object().size()) query["prefix"] = resource.object();
+
+    do
+    {
+        if (pageToken.size()) query["pageToken"] = pageToken;
+
+        const auto res(https.internalGet(url, m_auth->headers(), query));
+
+        if (!res.ok())
+        {
+            throw ArbiterError(std::to_string(res.code()) + ": " + res.str());
+        }
+
+        const Json::Value json(util::parse(res.str()));
+        for (const auto& item : json["items"])
+        {
+            results.push_back(
+                    type() + "://" +
+                    resource.bucket() + item["name"].asString());
+        }
+
+        pageToken = json["nextPageToken"].asString();
+    } while (pageToken.size());
+
+    return results;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+std::unique_ptr<Google::Auth> Google::Auth::create(const Json::Value& json)
+{
+    if (auto path = util::env("GOOGLE_APPLICATION_CREDENTIALS"))
+    {
+        if (const auto file = drivers::Fs().tryGet(*path))
+        {
+            return util::makeUnique<Auth>(util::parse(*file));
+        }
+    }
+    else if (json.isString())
+    {
+        const auto path = json.asString();
+        if (const auto file = drivers::Fs().tryGet(path))
+        {
+            return util::makeUnique<Auth>(util::parse(*file));
+        }
+    }
+    else if (json.isObject())
+    {
+        return util::makeUnique<Auth>(json);
+    }
+
+    return std::unique_ptr<Auth>();
+}
+
+Google::Auth::Auth(const Json::Value& creds)
+    : m_creds(creds)
+{
+    maybeRefresh();
+}
+
+http::Headers Google::Auth::headers() const
+{
+    std::lock_guard<std::mutex> lock(m_mutex);
+    maybeRefresh();
+    return m_headers;
+}
+
+void Google::Auth::maybeRefresh() const
+{
+    using namespace crypto;
+
+    const auto now(Time().asUnix());
+    if (m_expiration - now > 120) return;   // Refresh when under 2 mins left.
+
+    // https://developers.google.com/identity/protocols/OAuth2ServiceAccount
+    Json::Value h;
+    h["alg"] = "RS256";
+    h["typ"] = "JWT";
+
+    Json::Value c;
+    c["iss"] = m_creds["client_email"].asString();
+    c["scope"] = "https://www.googleapis.com/auth/devstorage.read_write";
+    c["aud"] = "https://www.googleapis.com/oauth2/v4/token";
+    c["iat"] = Json::Int64(now);
+    c["exp"] = Json::Int64(now + 3600);
+
+    const std::string header(encodeBase64(util::toFastString(h)));
+    const std::string claims(encodeBase64(util::toFastString(c)));
+
+    const std::string key(m_creds["private_key"].asString());
+    const std::string signature(
+            http::sanitize(encodeBase64(sign(header + '.' + claims, key))));
+
+    const std::string assertion(header + '.' + claims + '.' + signature);
+    const std::string sbody =
+        "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&"
+        "assertion=" + assertion;
+    const std::vector<char> body(sbody.begin(), sbody.end());
+
+    const http::Headers headers { { "Expect", "" } };
+    const std::string tokenRequestUrl("www.googleapis.com/oauth2/v4/token");
+
+    http::Pool pool;
+    drivers::Https https(pool);
+    const auto res(https.internalPost(tokenRequestUrl, body, headers));
+
+    if (!res.ok()) throw ArbiterError("Failed to get token: " + res.str());
+
+    const Json::Value token(util::parse(res.str()));
+    m_headers["Authorization"] = "Bearer " + token["access_token"].asString();
+    m_expiration = now + token["expires_in"].asInt64();
+}
+
+std::string Google::Auth::sign(
+        const std::string data,
+        const std::string pkey) const
+{
+    std::string signature;
+
+#ifdef ARBITER_OPENSSL
+    std::lock_guard<std::mutex> lock(sslMutex);
+
+    auto loadKey([](std::string s, bool isPublic)->EVP_PKEY*
+    {
+        // BIO_new_mem_buf needs non-const char*, so use a vector.
+        std::vector<char> vec(s.data(), s.data() + s.size());
+
+        EVP_PKEY* key(nullptr);
+        if (BIO* bio = BIO_new_mem_buf(vec.data(), vec.size()))
+        {
+            if (isPublic)
+            {
+                key = PEM_read_bio_PUBKEY(bio, &key, nullptr, nullptr);
+            }
+            else
+            {
+                key = PEM_read_bio_PrivateKey(bio, &key, nullptr, nullptr);
+            }
+
+            BIO_free(bio);
+
+            if (!key)
+            {
+                std::vector<char> err(256, 0);
+                ERR_error_string(ERR_get_error(), err.data());
+                throw ArbiterError(
+                        std::string("Could not load key: ") + err.data());
+            }
+        }
+
+        return key;
+    });
+
+
+    EVP_PKEY* key(loadKey(pkey, false));
+
+    EVP_MD_CTX ctx;
+    EVP_MD_CTX_init(&ctx);
+    EVP_DigestSignInit(&ctx, nullptr, EVP_sha256(), nullptr, key);
+
+    if (EVP_DigestSignUpdate(&ctx, data.data(), data.size()) == 1)
+    {
+        std::size_t size(0);
+        if (EVP_DigestSignFinal(&ctx, nullptr, &size) == 1)
+        {
+            std::vector<unsigned char> v(size, 0);
+            if (EVP_DigestSignFinal(&ctx, v.data(), &size) == 1)
+            {
+                signature.assign(reinterpret_cast<const char*>(v.data()), size);
+            }
+        }
+    }
+
+    EVP_MD_CTX_cleanup(&ctx);
+    if (signature.empty()) throw ArbiterError("Could not sign JWT");
+    return signature;
+#else
+    throw ArbiterError("Cannot use google driver without OpenSSL");
+#endif
+}
+
+} // namespace drivers
+} // namespace arbiter
+
+#ifdef ARBITER_CUSTOM_NAMESPACE
+}
+#endif
+
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: arbiter/drivers/google.cpp
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+
 // //////////////////////////////////////////////////////////////////////
 // Beginning of content of file: arbiter/drivers/dropbox.cpp
 // //////////////////////////////////////////////////////////////////////
@@ -2253,8 +2746,9 @@ namespace arbiter
 
 namespace
 {
-    const std::string baseGetUrl("https://content.dropboxapi.com/");
-    const std::string getUrl(baseGetUrl + "2/files/download");
+    const std::string baseDropboxUrl("https://content.dropboxapi.com/");
+    const std::string getUrl(baseDropboxUrl + "2/files/download");
+    const std::string putUrl(baseDropboxUrl + "2/files/upload");
 
     const std::string listUrl("https://api.dropboxapi.com/2/files/list_folder");
     const std::string metaUrl("https://api.dropboxapi.com/2/files/get_metadata");
@@ -2281,6 +2775,7 @@ namespace drivers
 {
 
 using namespace http;
+using namespace util;
 
 Dropbox::Dropbox(Pool& pool, const Dropbox::Auth& auth)
     : Http(pool)
@@ -2289,14 +2784,19 @@ Dropbox::Dropbox(Pool& pool, const Dropbox::Auth& auth)
 
 std::unique_ptr<Dropbox> Dropbox::create(Pool& pool, const Json::Value& json)
 {
-    std::unique_ptr<Dropbox> dropbox;
-
-    if (!json.isNull() && json.isMember("token"))
+    if (!json.isNull())
     {
-        dropbox.reset(new Dropbox(pool, Auth(json["token"].asString())));
+        if (json.isObject() && json.isMember("token"))
+        {
+            return makeUnique<Dropbox>(pool, Auth(json["token"].asString()));
+        }
+        else if (json.isString())
+        {
+            return makeUnique<Dropbox>(pool, Auth(json.asString()));
+        }
     }
 
-    return dropbox;
+    return std::unique_ptr<Dropbox>();
 }
 
 Headers Dropbox::httpGetHeaders() const
@@ -2432,10 +2932,23 @@ bool Dropbox::get(
 void Dropbox::put(
         const std::string rawPath,
         const std::vector<char>& data,
-        const Headers headers,
+        const Headers userHeaders,
         const Query query) const
 {
-    throw ArbiterError("PUT not yet supported for " + type());
+    const std::string path(sanitize(rawPath));
+
+    Headers headers(httpGetHeaders());
+
+    Json::Value json;
+    json["path"] = std::string("/" + path);
+    headers["Dropbox-API-Arg"] = toSanitizedString(json);
+    headers["Content-Type"] = "application/octet-stream";
+
+    headers.insert(userHeaders.begin(), userHeaders.end());
+
+    const Response res(Http::internalPost(putUrl, data, headers, query));
+
+    if (!res.ok()) throw ArbiterError(res.str());
 }
 
 std::string Dropbox::continueFileInfo(std::string cursor) const
@@ -2464,19 +2977,22 @@ std::string Dropbox::continueFileInfo(std::string cursor) const
     return std::string("");
 }
 
-std::vector<std::string> Dropbox::glob(std::string rawPath, bool verbose) const
+std::vector<std::string> Dropbox::glob(std::string path, bool verbose) const
 {
     std::vector<std::string> results;
 
-    const std::string path(sanitize(rawPath.substr(0, rawPath.size() - 2)));
+    path.pop_back();
+    const bool recursive(path.back() == '*');
+    if (recursive) path.pop_back();
+    if (path.back() == '/') path.pop_back();
 
-    auto listPath = [this](std::string path)->std::string
+    auto listPath = [this, recursive](std::string path)->std::string
     {
         Headers headers(httpPostHeaders());
 
         Json::Value request;
         request["path"] = std::string("/" + path);
-        request["recursive"] = false;
+        request["recursive"] = recursive;
         request["include_media_info"] = false;
         request["include_deleted"] = false;
 
@@ -2507,7 +3023,8 @@ std::vector<std::string> Dropbox::glob(std::string rawPath, bool verbose) const
     bool more(false);
     std::string cursor("");
 
-    auto processPath = [verbose, &results, &more, &cursor](std::string data)
+    auto processPath =
+        [this, verbose, &results, &more, &cursor](std::string data)
     {
         if (data.empty()) return;
 
@@ -2540,7 +3057,7 @@ std::vector<std::string> Dropbox::glob(std::string rawPath, bool verbose) const
             if (std::equal(tag.begin(), tag.end(), fileTag.begin(), ins))
             {
                 // Results already begin with a slash.
-                results.push_back("dropbox:/" + v["path_lower"].asString());
+                results.push_back(type() + ":/" + v["path_lower"].asString());
             }
         }
     };
@@ -3816,7 +4333,7 @@ namespace
     const std::string hexVals("0123456789abcdef");
 } // unnamed namespace
 
-std::string encodeBase64(const std::vector<char>& data)
+std::string encodeBase64(const std::vector<char>& data, const bool pad)
 {
     std::vector<uint8_t> input;
     for (std::size_t i(0); i < data.size(); ++i)
@@ -3825,17 +4342,18 @@ std::string encodeBase64(const std::vector<char>& data)
         input.push_back(*reinterpret_cast<uint8_t*>(&c));
     }
 
-    std::size_t fullSteps(input.size() / 3);
+    const std::size_t fullSteps(data.size() / 3);
+    const std::size_t remainder(data.size() % 3);
+
     while (input.size() % 3) input.push_back(0);
     uint8_t* pos(input.data());
-    uint8_t* end(input.data() + fullSteps * 3);
 
     std::string output(fullSteps * 4, '_');
     std::size_t outIndex(0);
 
     const uint32_t mask(0x3F);
 
-    while (pos != end)
+    for (std::size_t i(0); i < fullSteps; ++i)
     {
         uint32_t chunk((*pos) << 16 | *(pos + 1) << 8 | *(pos + 2));
 
@@ -3847,24 +4365,26 @@ std::string encodeBase64(const std::vector<char>& data)
         pos += 3;
     }
 
-    if (end != input.data() + input.size())
+    if (remainder)
     {
-        const std::size_t num(pos - end == 1 ? 2 : 3);
         uint32_t chunk(*(pos) << 16 | *(pos + 1) << 8 | *(pos + 2));
 
         output.push_back(base64Vals[(chunk >> 18) & mask]);
         output.push_back(base64Vals[(chunk >> 12) & mask]);
-        if (num == 3) output.push_back(base64Vals[(chunk >> 6) & mask]);
-    }
+        if (remainder == 2) output.push_back(base64Vals[(chunk >> 6) & mask]);
 
-    while (output.size() % 4) output.push_back('=');
+        if (pad)
+        {
+            while (output.size() % 4) output.push_back('=');
+        }
+    }
 
     return output;
 }
 
-std::string encodeBase64(const std::string& input)
+std::string encodeBase64(const std::string& input, const bool pad)
 {
-    return encodeBase64(std::vector<char>(input.begin(), input.end()));
+    return encodeBase64(std::vector<char>(input.begin(), input.end()), pad);
 }
 
 std::string encodeAsHex(const std::vector<char>& input)
@@ -4017,6 +4537,12 @@ int64_t Time::operator-(const Time& other) const
     return std::difftime(m_time, other.m_time);
 }
 
+int64_t Time::asUnix() const
+{
+    static const Time epoch("1970-01-01T00:00:00Z");
+    return *this - epoch;
+}
+
 } // namespace arbiter
 
 #ifdef ARBITER_CUSTOM_NAMESPACE
diff --git a/vendor/arbiter/arbiter.hpp b/vendor/arbiter/arbiter.hpp
index a67730dae..a555b4371 100644
--- a/vendor/arbiter/arbiter.hpp
+++ b/vendor/arbiter/arbiter.hpp
@@ -1,7 +1,7 @@
 /// Arbiter amalgamated header (https://github.com/connormanning/arbiter).
 /// It is intended to be used with #include "arbiter.hpp"
 
-// Git SHA: 311b81e3dcb5e8c55b2148f7eba46a12cff66914
+// Git SHA: c3b439d09d272818e221d3c57c16de14b89a6512
 
 // //////////////////////////////////////////////////////////////////////
 // Beginning of content of file: LICENSE
@@ -153,6 +153,10 @@ public:
 
     std::vector<char> data() const { return m_data; }
     const Headers& headers() const { return m_headers; }
+    std::string str() const
+    {
+        return std::string(data().data(), data().size());
+    }
 
 private:
     int m_code;
@@ -523,6 +527,7 @@ public:
 
     // Return value is in seconds.
     int64_t operator-(const Time& other) const;
+    int64_t asUnix() const;
 
 private:
     std::time_t m_time;
@@ -803,6 +808,8 @@ class ARBITER_DLL Fs : public Driver
 public:
     Fs() { }
 
+    using Driver::get;
+
     static std::unique_ptr<Fs> create(const Json::Value& json);
 
     virtual std::string type() const override { return "file"; }
@@ -950,13 +957,14 @@ public:
             http::Headers headers,
             http::Query query) const;
 
-protected:
-    /** HTTP-derived Drivers should override this version of GET to allow for
-     * custom headers and query parameters.
-     */
-    virtual bool get(
+    void post(
             std::string path,
-            std::vector<char>& data,
+            const std::string& data,
+            http::Headers headers,
+            http::Query query) const;
+    void post(
+            std::string path,
+            const std::vector<char>& data,
             http::Headers headers,
             http::Query query) const;
 
@@ -986,6 +994,18 @@ protected:
             http::Headers headers = http::Headers(),
             http::Query query = http::Query()) const;
 
+protected:
+    /** HTTP-derived Drivers should override this version of GET to allow for
+     * custom headers and query parameters.
+     */
+    virtual bool get(
+            std::string path,
+            std::vector<char>& data,
+            http::Headers headers,
+            http::Query query) const;
+
+    http::Pool& m_pool;
+
 private:
     virtual bool get(
             std::string path,
@@ -994,12 +1014,7 @@ private:
         return get(path, data, http::Headers(), http::Query());
     }
 
-    std::string typedPath(const std::string& p) const
-    {
-        return type() + "://" + p;
-    }
-
-    http::Pool& m_pool;
+    std::string typedPath(const std::string& p) const;
 };
 
 /** @brief HTTPS driver.  Identical to the HTTP driver except for its type
@@ -3850,8 +3865,8 @@ namespace arbiter
 namespace crypto
 {
 
-std::string encodeBase64(const std::vector<char>& data);
-std::string encodeBase64(const std::string& data);
+std::string encodeBase64(const std::vector<char>& data, bool pad = true);
+std::string encodeBase64(const std::string& data, bool pad = true);
 
 std::string encodeAsHex(const std::vector<char>& data);
 std::string encodeAsHex(const std::string& data);
@@ -3887,6 +3902,11 @@ std::string encodeAsHex(const std::string& data);
 
 #ifndef ARBITER_IS_AMALGAMATION
 #include <arbiter/util/exports.hpp>
+
+#ifndef ARBITER_EXTERNAL_JSON
+#include <arbiter/third/json/json.hpp>
+#endif
+
 #endif
 
 
@@ -4056,6 +4076,26 @@ namespace util
         if (t) return makeUnique<T>(*t);
         else return std::unique_ptr<T>();
     }
+
+    inline Json::Value parse(const std::string& s)
+    {
+        Json::Reader reader;
+        Json::Value json;
+        if (!reader.parse(s, json))
+        {
+            throw std::runtime_error(
+                    "Parse failure: " + reader.getFormattedErrorMessages());
+        }
+        return json;
+    }
+
+    inline std::string toFastString(const Json::Value& json)
+    {
+        std::string s = Json::FastWriter().write(json);
+        s.pop_back();   // Strip trailing newline.
+        return s;
+    }
+
 } // namespace util
 
 } // namespace arbiter
@@ -4124,7 +4164,11 @@ public:
      *          `~/.aws/credentials` or the file at AWS_CREDENTIAL_FILE.
      *      - EC2 instance profile.
      */
-    static std::unique_ptr<S3> create(
+    static std::vector<std::unique_ptr<S3>> create(
+            http::Pool& pool,
+            const Json::Value& json);
+
+    static std::unique_ptr<S3> createOne(
             http::Pool& pool,
             const Json::Value& json);
 
@@ -4189,9 +4233,10 @@ private:
 class S3::Auth
 {
 public:
-    Auth(std::string access, std::string hidden)
+    Auth(std::string access, std::string hidden, std::string token = "")
         : m_access(access)
         , m_hidden(hidden)
+        , m_token(token)
     { }
 
     Auth(std::string iamRole)
@@ -4217,11 +4262,7 @@ private:
 class S3::Config
 {
 public:
-    Config(std::string region, std::string baseUrl, bool sse, bool precheck);
-
-    static std::unique_ptr<Config> create(
-            const Json::Value& json,
-            std::string profile);
+    Config(const Json::Value& json, std::string profile);
 
     const std::string& region() const { return m_region; }
     const std::string& baseUrl() const { return m_baseUrl; }
@@ -4252,8 +4293,8 @@ public:
 
     std::string url() const;
     std::string host() const;
-    std::string baseUrl() const { return m_baseUrl; }
-    std::string bucket() const { return m_bucket; }
+    std::string baseUrl() const;
+    std::string bucket() const;
     std::string object() const;
 
 private:
@@ -4327,6 +4368,103 @@ private:
 
 
 
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: arbiter/drivers/google.hpp
+// //////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+#ifndef ARBITER_IS_AMALGAMATION
+#include <arbiter/drivers/http.hpp>
+#endif
+
+#include <mutex>
+
+#ifdef ARBITER_CUSTOM_NAMESPACE
+namespace ARBITER_CUSTOM_NAMESPACE
+{
+#endif
+
+namespace arbiter
+{
+
+namespace drivers
+{
+
+class Google : public Https
+{
+    class Auth;
+public:
+    Google(http::Pool& pool, std::unique_ptr<Auth> auth);
+
+    static std::unique_ptr<Google> create(
+            http::Pool& pool,
+            const Json::Value& json);
+
+    // Overrides.
+    virtual std::string type() const override { return "gs"; }  // Match gsutil.
+
+    virtual std::unique_ptr<std::size_t> tryGetSize(
+            std::string path) const override;
+
+    /** Inherited from Drivers::Http. */
+    virtual void put(
+            std::string path,
+            const std::vector<char>& data,
+            http::Headers headers,
+            http::Query query) const override;
+
+private:
+    /** Inherited from Drivers::Http. */
+    virtual bool get(
+            std::string path,
+            std::vector<char>& data,
+            http::Headers headers,
+            http::Query query) const override;
+
+    virtual std::vector<std::string> glob(
+            std::string path,
+            bool verbose) const override;
+
+    std::unique_ptr<Auth> m_auth;
+};
+
+class Google::Auth
+{
+public:
+    Auth(const Json::Value& creds);
+    static std::unique_ptr<Auth> create(const Json::Value& json);
+
+    http::Headers headers() const;
+
+private:
+    void maybeRefresh() const;
+    std::string sign(std::string data, std::string privateKey) const;
+
+    const Json::Value m_creds;
+    mutable int64_t m_expiration = 0;   // Unix time.
+    mutable http::Headers m_headers;
+
+    mutable std::mutex m_mutex;
+};
+
+} // namespace drivers
+} // namespace arbiter
+
+#ifdef ARBITER_CUSTOM_NAMESPACE
+}
+#endif
+
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: arbiter/drivers/google.hpp
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+
 // //////////////////////////////////////////////////////////////////////
 // Beginning of content of file: arbiter/drivers/dropbox.hpp
 // //////////////////////////////////////////////////////////////////////
@@ -4652,6 +4790,29 @@ public:
     /** Get a further nested subpath relative to this Endpoint's root. */
     Endpoint getSubEndpoint(std::string subpath) const;
 
+    http::Response httpGet(
+            std::string path,
+            http::Headers headers = http::Headers(),
+            http::Query query = http::Query(),
+            std::size_t reserve = 0) const;
+
+    http::Response httpPut(
+            std::string path,
+            const std::vector<char>& data,
+            http::Headers headers = http::Headers(),
+            http::Query query = http::Query()) const;
+
+    http::Response httpHead(
+            std::string path,
+            http::Headers headers = http::Headers(),
+            http::Query query = http::Query()) const;
+
+    http::Response httpPost(
+            std::string path,
+            const std::vector<char>& data,
+            http::Headers headers = http::Headers(),
+            http::Query query = http::Query()) const;
+
 private:
     Endpoint(const Driver& driver, std::string root);
 
@@ -4696,15 +4857,17 @@ private:
 #endif
 
 #ifndef ARBITER_IS_AMALGAMATION
-#include <arbiter/util/exports.hpp>
-#include <arbiter/driver.hpp>
 #include <arbiter/endpoint.hpp>
+#include <arbiter/driver.hpp>
+#include <arbiter/drivers/dropbox.hpp>
 #include <arbiter/drivers/fs.hpp>
-#include <arbiter/drivers/test.hpp>
+#include <arbiter/drivers/google.hpp>
 #include <arbiter/drivers/http.hpp>
 #include <arbiter/drivers/s3.hpp>
-#include <arbiter/drivers/dropbox.hpp>
+#include <arbiter/drivers/test.hpp>
+#include <arbiter/util/exports.hpp>
 #include <arbiter/util/types.hpp>
+#include <arbiter/util/util.hpp>
 
 #ifndef ARBITER_EXTERNAL_JSON
 #include <arbiter/third/json/json.hpp>
@@ -4849,8 +5012,8 @@ public:
 
     /** Copy the single file @p file to the destination @p to.  If @p to ends
      * with a `/` or '\' character, then @p file will be copied into the
-     * directory @p to with the basename of @p file.  If @p does not end with a
-     * slash character, then @p to will be interpreted as a file path.
+     * directory @p to with the basename of @p file.  If @p to does not end
+     * with a slash character, then @p to will be interpreted as a file path.
      *
      * If @p to is a local filesystem path, then `fs::mkdirp` will be called
      * prior to copying.