download_appengine_logs.py

#!/usr/bin/env python
# This script is free software written by Warren Runk ([email protected])

"""
This is a (hopefully) simple example of how to download your appengine logs to a normal
linux (non google or appengine) computer using remote api and logservice.

You can use appcfg.py to "request_logs" but that is not practical for real workloads

From here you can download smaller time slices of logs as necessary depending on your scale.

Remember to check out google.appengine.api.logservice.logservice.RequestLog as there are a
TON of fields for each log entry.

You will need have the google_appengine main directory from the SDK in your PYTHONPATH for this to work
"""

from google.appengine.api.logservice import logservice
from google.appengine.ext.remote_api import remote_api_stub
from pprint import pprint
import time


def print_logs(admin_email, password, app_id, module, version, seconds_ago=3600):
    """
    Print the last hour of appengine logs for the specified environment. Note you can pass
    'default' for module, but passing 'default' for the version doesn't seem to work properly.
    """

    def auth_stupid_func():
        return admin_email, password

    remote_api_stub.ConfigureRemoteApi(
        # ** NOTE ** Specifying the app_id here like evite-services-test DOESN'T work!!
        # Causes errors like:
        # "google.appengine.api.logservice.logservice.InvalidArgumentError: Attempt to read a different
        #  application's logs without permission."
        # Annoying, so just use servername below
        #
        app_id=None,
        # This is the default, normal path. To make this work, enable remote api in app.yaml and deploy
        #  only
        path='/_ah/remote_api',
        auth_func=auth_stupid_func,
        servername='%s.appspot.com' % app_id
    )
    # Note this fetch script takes a lot of other useful params like request_ids
    for log in logservice.fetch(
        start_time=time.time() - seconds_ago,
        end_time=time.time(),
        minimum_log_level=logservice.LOG_LEVEL_INFO,
        module_versions=[(module, version)],
        include_app_logs=True,
        include_incomplete=True,
    ):
        # Note log here will be a google.appengine.api.logservice.logservice.RequestLog
        # Please review that class in the code included with the GAE SDK

        # Print things in dict format so you can see the keys. The you *SHOULD* access the members of the
        # RequestLog class like log.request_id etc
        pprint(vars(log._RequestLog__pb))

    print "Successfully finished printing all logs"


def main():
    print_logs('[email protected]', '<pw>', 'evite-test', 'default', 'bam')


if __name__ == '__main__':
    main()