Skip to content

Instantly share code, notes, and snippets.

@manics
Last active September 10, 2019 18:56
Show Gist options
  • Save manics/2b5e3b51e80525d940b2c8f2be578a79 to your computer and use it in GitHub Desktop.
Save manics/2b5e3b51e80525d940b2c8f2be578a79 to your computer and use it in GitHub Desktop.
Run a Jupyter Notebook on BinderHub non-interactively from the command line

Run a Jupyter Notebook on BinderHub non-interactively from the command line

The notebook should contain a cell starting with # Parameters:

Example:

./binderhub-exec.py \
    gist/manics/421f2927bb1dbdbc00754a7669eb3f69/master \
    idr-image.ipynb \
    IMAGE_ID=4495402

Output:

{'phase': 'built', 'imageName': 'gcr.io/binder-prod/r2d-f18835fd-421f2927bb1dbdbc00754a7669eb3f69-c1fe22:461fff755bb79bf68f05ed8b67a27ebd661850ed', 'message': 'Found built image, launching...\n'}
{'phase': 'launching', 'message': 'Launching server...\n'}
{'phase': 'ready', 'message': 'server running at https://hub.gke.mybinder.org/user/manics-421f2927-0754a7669eb3f69-cqkpdq8l/\n', 'image': 'gcr.io/binder-prod/r2d-f18835fd-421f2927bb1dbdbc00754a7669eb3f69-c1fe22:461fff755bb79bf68f05ed8b67a27ebd661850ed', 'repo_url': 'https://gist.github.com/manics/421f2927bb1dbdbc00754a7669eb3f69.git', 'token': 'Hi17tnUHSg6SbZzt9zdvQw', 'url': 'https://hub.gke.mybinder.org/user/manics-421f2927-0754a7669eb3f69-cqkpdq8l/'}
ERROR: {"id": "0bc9db9f-3e3b-4ac0-8956-acefcf867f8f", "name": "python3", "last_activity": "2019-09-10T16:33:11.017579Z", "execution_state": "starting", "connections": 0}
kernel {'id': '0bc9db9f-3e3b-4ac0-8956-acefcf867f8f', 'name': 'python3', 'last_activity': '2019-09-10T16:33:11.017579Z', 'execution_state': 'starting', 'connections': 0}

********** Cell 0 **********

Replacing parameters
Sending code:
# Parameters:
IMAGE_ID = 4495402
status
execute_input
status
execute_reply
{'status': 'ok', 'execution_count': 1, 'user_expressions': {}, 'payload': []}

********** Cell 1 **********

Sending code:
import requests
r = requests.get(f'https://idr.openmicroscopy.org/api/v0/m/images/{IMAGE_ID}/').json()
print(f'Name: {r["data"]["Name"]}')
p = r["data"]["Pixels"]
print(f'Size: {p["SizeX"], p["SizeY"], p["SizeZ"], p["SizeC"], p["SizeT"]}')
status
execute_input
stream
Name: Sagittal section
Size: (921600, 380928, 1, 1, 1)

status
execute_reply
{'status': 'ok', 'execution_count': 2, 'user_expressions': {}, 'payload': []}
#!/usr/bin/env python3
from argparse import ArgumentParser
import datetime
import json
import requests
import uuid
from websocket import create_connection
# https://stackoverflow.com/questions/54475896/interact-with-jupyter-notebooks-via-api/54551221#54551221
# https://jupyter-client.readthedocs.io/en/stable/messaging.html
# http://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyter/notebook/master/notebook/services/api/api.yaml
PARAMETERS_MARKER = '# Parameters:'
def create_kernel(base, headers, kernelname):
"""
Create a new kernel for the notebook
"""
r = requests.post(base + 'api/kernels', headers=headers,
json={'name': kernelname})
if r.status_code != requests.codes.ok:
print('ERROR: {}'.format(r.text))
r.raise_for_status()
return r.json()
def send_execute_request(code):
msg_type = 'execute_request'
content = {
'code': code,
'silent': False
}
hdr = {
'msg_id': uuid.uuid4().hex,
'username': 'test',
'session': uuid.uuid4().hex,
'data': datetime.datetime.now().isoformat(),
'msg_type': msg_type,
'version': '5.0',
}
msg = {
'header': hdr,
'parent_header': hdr,
'metadata': {},
'content': content,
}
return msg
def main(repo, notebook_path, params):
binderbuild = 'https://mybinder.org/build/{}'.format(repo)
r = requests.get(binderbuild, stream=True)
for line in r.iter_lines():
s = line.decode()
if s.startswith('data: '):
d = json.loads(s[6:])
print(d)
if d['phase'] != 'ready':
raise Exception('Failed to start repo')
base = d['url']
headers = {'Authorization': 'Token {token}'.format(**d)}
# Load the notebook and get the code of each cell
r = requests.get(base + 'api/contents/' + notebook_path, headers=headers)
r.raise_for_status()
nb = r.json()
code = [c['source'] for c in nb['content']['cells']
if c['cell_type'] == 'code' and len(c['source']) > 0]
# Create the kernel named in the notebook
kernel = create_kernel(
base, headers, nb['content']['metadata']['kernelspec']['name'])
print('kernel', kernel)
# Execution request/reply is done on websockets channels
ws = create_connection(
'ws' + base[4:] + 'api/kernels/{id}/channels'.format(**kernel),
header=headers)
# Note we don't actually need a notebook in the repo, since we're just
# sending code chunks!
for i, c in enumerate(code):
print('\n********** Cell {} **********\n'.format(i))
if c.startswith(PARAMETERS_MARKER):
print('Replacing parameters')
c = '{}\n{}'.format(
PARAMETERS_MARKER,
'\n'.join('{} = {}'.format(k, v) for (k, v) in params.items()))
print('Sending code:\n{}'.format(c))
r = ws.send(json.dumps(send_execute_request(c)))
# print(r)
while True:
try:
rsp = json.loads(ws.recv())
msg_type = rsp['msg_type']
print(msg_type)
if msg_type in ('error', 'execute_reply'):
break
if msg_type == 'stream':
print(rsp['content']['text'])
elif msg_type not in ('execute_input', 'status'):
print(rsp)
# Ignore all the other messages, we just get the code execution
# output.
# TODO: take into account large cell output, images, etc.
except json.JSONDecodeError as e:
print('Error decoding JSON: {}'.format(e))
raise
if msg_type == 'execute_reply':
print(rsp['content'])
if msg_type == 'error':
raise Exception('Failed to execute: {content}'.format(**rsp))
ws.close()
if __name__ == '__main__':
p = ArgumentParser(description='Launch a mybinder repo notebook')
p.add_argument('repospec', help=(
'repository spec type/<username/repository/branch> e.g. '
'gh/github-user/github-repo/master '
'gist/github-user/master'))
p.add_argument('notebook', help='path to notebook e.g. /example.ipynb')
p.add_argument('parameters', nargs='*', help=(
'Parameters in form name=value'))
args = p.parse_args()
params = {}
for p in args.parameters:
k, v = p.split('=', 1)
params[k] = v
main(args.repospec, args.notebook, params)
requests
websockets
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment