gistfile1.txt

[2018-02-14T22:47:19.646Z]  WARN: muskie/27560 on 56564894-a7c2-470e-a218-3d859e7e1687: mako: connection failed (req_id=95747e03-ec42-4075-9721-317822b609c3, method=GET, path=/Joyent_Dev/public/builds/reg
istrar/master-latest, shark="[object SharkClient<17.stor.us-east.joyent.us>]")
    PoolFailedError: Pool 15b224b9-30d0-4214-8cf1-3fb68d8caa55 (17.stor.us-east.joyent.us) has failed and cannot take new requests.
        at Object._onImmediate (/opt/smartdc/muskie/node_modules/cueball/lib/pool.js:752:8)
        at processImmediate [as _immediateCallback] (timers.js:363:15)
[2018-02-14T22:47:19.648Z]  WARN: muskie/27560 on 56564894-a7c2-470e-a218-3d859e7e1687: request failed: internal error (req_id=95747e03-ec42-4075-9721-317822b609c3, method=GET, path=/Joyent_Dev/public/bui
lds/registrar/master-latest)
    InternalError: an unexpected error occurred
        at translateError (/opt/smartdc/muskie/lib/errors.js:681:13)
        at ServerResponse.formatJSON (/opt/smartdc/muskie/lib/server.js:51:16)
        at ServerResponse.format (/opt/smartdc/muskie/node_modules/restify/lib/response.js:102:23)
        at ServerResponse.send (/opt/smartdc/muskie/node_modules/restify/lib/response.js:208:24)
        at next (/opt/smartdc/muskie/node_modules/restify/lib/server.js:658:21)
        at f (/opt/smartdc/muskie/node_modules/once/once.js:16:25)
        at Queue.<anonymous> (/opt/smartdc/muskie/lib/obj.js:877:13)
        at Queue.g (events.js:180:16)
        at Queue.emit (events.js:92:17)
        at Queue.end (/opt/smartdc/muskie/node_modules/libmanta/lib/queue.js:91:18)
[2018-02-14T22:47:19.651Z]  INFO: muskie/HttpServer/27560 on 56564894-a7c2-470e-a218-3d859e7e1687: handled: 500 (audit=true, _audit=true, operation=getpublicstorage, billable_operation=GET, logicalRemoteA
ddress=24.80.208.239, remoteAddress=127.0.0.1, remotePort=47967, reqHeaderLength=740, resHeaderLength=348, latency=754, objectId=843522fc-9fd1-6178-ce02-91aabb71d888, entryShard=tcp://2.moray.us-east.joye
nt.us:2020, route=getpublicstorage, req.owner=7b315468-c6be-46dc-b99b-9c1f59224693)
    GET /Joyent_Dev/public/builds/registrar/master-latest HTTP/1.1
    accept: */*
    x-request-id: 95747e03-ec42-4075-9721-317822b609c3
    date: Wed, 14 Feb 2018 22:47:18 GMT
    authorization: Signature keyId="/jgilli/keys/78:66:e5:fb:87:4e:f2:99:18:c2:df:9e:06:78:fb:40",algorithm="rsa-sha256",signature="LdTdTRUoriscmYfRk8sJX8Bgj14vQEF+YmPDiyUS7ND/ycRWBI0g2+l91e6iZ/F90m9E5Oo/uvhboTcCM/6ZnDugXjN3MW3mFQtn06rpfxBGSjdJziZFCPWPd7Wged5WaW9xnCQKNiYgfBB1/BUSpyCMfWChM6RYBCTVPdS97V7SdHi/2pCaWEBpc5rPVXZFPXm1GYTgG80EqXFQ4E8Ado5B33pR68DUYKckJ7PHAIH5oyGL5N2LcugyV/DVdhzFOf5vso3BedaNGft2pbrYPGFU1VfwIvSxBnjLFcqKeBpXhG+JUQUFWvxPIeDdijZOERv46IknV3siIpTWRVe9RQ=="
    user-agent: restify/2.8.1 (x64-darwin; v8/4.5.103.37; OpenSSL/1.0.2j) node/4.6.1
    accept-version: ~1.0
    host: us-east.manta.joyent.com
    connection: keep-alive
    x-forwarded-for: ::ffff:24.80.208.239
    --
    HTTP/1.1 500 Internal Server Error
    etag: 843522fc-9fd1-6178-ce02-91aabb71d888
    last-modified: Thu, 11 Jan 2018 00:11:41 GMT
    access-control-allow-origin: *
    content-type: application/json
    content-length: 65
    content-md5: d6SoJabjG5BOFuLYgMFtgQ==
date: Wed, 14 Feb 2018 22:47:19 GMT
    server: Manta
    x-request-id: 95747e03-ec42-4075-9721-317822b609c3
    x-response-time: 754
    x-server-name: 56564894-a7c2-470e-a218-3d859e7e1687
    --
    PoolFailedError: Pool 15b224b9-30d0-4214-8cf1-3fb68d8caa55 (17.stor.us-east.joyent.us) has failed and cannot take new requests.
        at Object._onImmediate (/opt/smartdc/muskie/node_modules/cueball/lib/pool.js:752:8)
        at processImmediate [as _immediateCallback] (timers.js:363:15)
    --
    sharksContacted: [
      {
        "shark": "15.stor.us-east.joyent.us",
        "result": "fail",
        "timeToFirstByte": null,
        "timeTotal": 302,
        "_startTime": 1518648439041
      },
      {
        "shark": "17.stor.us-east.joyent.us",
        "result": "fail",
        "timeToFirstByte": null,
        "timeTotal": 301,
        "_startTime": 1518648439345
      }
    ]
    --
    req.caller: {
      "login": "jgilli",
      "uuid": "fac5151a-8a12-ec04-a990-dc0245937a4e",
      "groups": [
        "operators"
      ],
      "user": null
    }
    --
    req.timers: {
      "earlySetup": 71,
      "parseDate": 17,
      "parseQueryString": 31,
      "handler-3": 245,
      "checkIfPresigned": 9,
      "enforceSSL": 7,
      "ensureDependencies": 12,
      "_authSetup": 9,
      "preSignedUrl": 6,
      "checkAuthzScheme": 10,
      "parseAuthTokenHandler": 74,
      "signatureHandler": 149,
      "parseKeyId": 157,
      "loadCaller": 4699,
      "verifySignature": 1368,
"parseHttpAuthToken": 13,
      "loadOwner": 7050,
      "getActiveRoles": 179,
      "gatherContext": 31,
      "setup": 373,
      "getMetadata": 130169,
      "storageContext": 24,
      "authorize": 1096,
      "ensureEntryExists": 9,
      "assertMetadata": 6,
      "getDirectoryCount": 6,
      "getDirectory": 6,
      "negotiateContent": 85,
      "checkIfMatch": 19,
      "checkIfNoneMatch": 8,
      "checkIfModified": 15,
      "checkIfUnmodified": 9,
      "verifyRange": 7,
      "streamFromSharks": 608403
    }