Run a Jupyter Notebook on BinderHub non-interactively from the command line

README.md

Run a Jupyter Notebook on BinderHub non-interactively from the command line

The notebook should contain a cell starting with # Parameters:

Example:

./binderhub-exec.py \
    gist/manics/421f2927bb1dbdbc00754a7669eb3f69/master \
    idr-image.ipynb \
    IMAGE_ID=4495402

Output:

{'phase': 'built', 'imageName': 'gcr.io/binder-prod/r2d-f18835fd-421f2927bb1dbdbc00754a7669eb3f69-c1fe22:461fff755bb79bf68f05ed8b67a27ebd661850ed', 'message': 'Found built image, launching...\n'}
{'phase': 'launching', 'message': 'Launching server...\n'}
{'phase': 'ready', 'message': 'server running at https://hub.gke.mybinder.org/user/manics-421f2927-0754a7669eb3f69-cqkpdq8l/\n', 'image': 'gcr.io/binder-prod/r2d-f18835fd-421f2927bb1dbdbc00754a7669eb3f69-c1fe22:461fff755bb79bf68f05ed8b67a27ebd661850ed', 'repo_url': 'https://gist.github.com/manics/421f2927bb1dbdbc00754a7669eb3f69.git', 'token': 'Hi17tnUHSg6SbZzt9zdvQw', 'url': 'https://hub.gke.mybinder.org/user/manics-421f2927-0754a7669eb3f69-cqkpdq8l/'}
ERROR: {"id": "0bc9db9f-3e3b-4ac0-8956-acefcf867f8f", "name": "python3", "last_activity": "2019-09-10T16:33:11.017579Z", "execution_state": "starting", "connections": 0}
kernel {'id': '0bc9db9f-3e3b-4ac0-8956-acefcf867f8f', 'name': 'python3', 'last_activity': '2019-09-10T16:33:11.017579Z', 'execution_state': 'starting', 'connections': 0}

********** Cell 0 **********

Replacing parameters
Sending code:
# Parameters:
IMAGE_ID = 4495402
status
execute_input
status
execute_reply
{'status': 'ok', 'execution_count': 1, 'user_expressions': {}, 'payload': []}

********** Cell 1 **********

Sending code:
import requests
r = requests.get(f'https://idr.openmicroscopy.org/api/v0/m/images/{IMAGE_ID}/').json()
print(f'Name: {r["data"]["Name"]}')
p = r["data"]["Pixels"]
print(f'Size: {p["SizeX"], p["SizeY"], p["SizeZ"], p["SizeC"], p["SizeT"]}')
status
execute_input
stream
Name: Sagittal section
Size: (921600, 380928, 1, 1, 1)

status
execute_reply
{'status': 'ok', 'execution_count': 2, 'user_expressions': {}, 'payload': []}

binderhub-exec.py

#!/usr/bin/env python3

from argparse import ArgumentParser
import datetime
import json
import requests
import uuid
from websocket import create_connection


# https://stackoverflow.com/questions/54475896/interact-with-jupyter-notebooks-via-api/54551221#54551221
# https://jupyter-client.readthedocs.io/en/stable/messaging.html
# http://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyter/notebook/master/notebook/services/api/api.yaml

PARAMETERS_MARKER = '# Parameters:'


def create_kernel(base, headers, kernelname):
    """
    Create a new kernel for the notebook
    """
    r = requests.post(base + 'api/kernels', headers=headers,
                      json={'name': kernelname})
    if r.status_code != requests.codes.ok:
        print('ERROR: {}'.format(r.text))
    r.raise_for_status()
    return r.json()


def send_execute_request(code):
    msg_type = 'execute_request'
    content = {
        'code': code,
        'silent': False
    }
    hdr = {
        'msg_id': uuid.uuid4().hex,
        'username': 'test',
        'session': uuid.uuid4().hex,
        'data': datetime.datetime.now().isoformat(),
        'msg_type': msg_type,
        'version': '5.0',
    }
    msg = {
        'header': hdr,
        'parent_header': hdr,
        'metadata': {},
        'content': content,
    }
    return msg


def main(repo, notebook_path, params):
    binderbuild = 'https://mybinder.org/build/{}'.format(repo)
    r = requests.get(binderbuild, stream=True)
    for line in r.iter_lines():
        s = line.decode()
        if s.startswith('data: '):
            d = json.loads(s[6:])
            print(d)
    if d['phase'] != 'ready':
        raise Exception('Failed to start repo')

    base = d['url']
    headers = {'Authorization': 'Token {token}'.format(**d)}

    # Load the notebook and get the code of each cell
    r = requests.get(base + 'api/contents/' + notebook_path, headers=headers)
    r.raise_for_status()
    nb = r.json()
    code = [c['source'] for c in nb['content']['cells']
            if c['cell_type'] == 'code' and len(c['source']) > 0]

    # Create the kernel named in the notebook
    kernel = create_kernel(
        base, headers, nb['content']['metadata']['kernelspec']['name'])
    print('kernel', kernel)

    # Execution request/reply is done on websockets channels
    ws = create_connection(
        'ws' + base[4:] + 'api/kernels/{id}/channels'.format(**kernel),
        header=headers)

    # Note we don't actually need a notebook in the repo, since we're just
    # sending code chunks!
    for i, c in enumerate(code):
        print('\n********** Cell {} **********\n'.format(i))
        if c.startswith(PARAMETERS_MARKER):
            print('Replacing parameters')
            c = '{}\n{}'.format(
                PARAMETERS_MARKER,
                '\n'.join('{} = {}'.format(k, v) for (k, v) in params.items()))
        print('Sending code:\n{}'.format(c))
        r = ws.send(json.dumps(send_execute_request(c)))
        # print(r)

        while True:
            try:
                rsp = json.loads(ws.recv())
                msg_type = rsp['msg_type']
                print(msg_type)
                if msg_type in ('error', 'execute_reply'):
                    break
                if msg_type == 'stream':
                    print(rsp['content']['text'])
                elif msg_type not in ('execute_input', 'status'):
                    print(rsp)
                # Ignore all the other messages, we just get the code execution
                # output.
                # TODO: take into account large cell output, images, etc.
            except json.JSONDecodeError as e:
                print('Error decoding JSON: {}'.format(e))
                raise
        if msg_type == 'execute_reply':
            print(rsp['content'])
        if msg_type == 'error':
            raise Exception('Failed to execute: {content}'.format(**rsp))

    ws.close()


if __name__ == '__main__':
    p = ArgumentParser(description='Launch a mybinder repo notebook')
    p.add_argument('repospec', help=(
        'repository spec type/<username/repository/branch> e.g. '
        'gh/github-user/github-repo/master '
        'gist/github-user/master'))
    p.add_argument('notebook', help='path to notebook e.g. /example.ipynb')
    p.add_argument('parameters', nargs='*', help=(
        'Parameters in form name=value'))
    args = p.parse_args()
    params = {}
    for p in args.parameters:
        k, v = p.split('=', 1)
        params[k] = v

    main(args.repospec, args.notebook, params)

requirements.txt

requests
websockets