Skip to content

Instantly share code, notes, and snippets.

@manics

manics/jupyterhub_find_dockerfiles.ipynb

Last active January 23, 2022 14:04
Show Gist options
  • Save manics/4af5d38ae11cddd08cff1f007f03f2f0 to your computer and use it in GitHub Desktop.
Save manics/4af5d38ae11cddd08cff1f007f03f2f0 to your computer and use it in GitHub Desktop.
Download ZIP
Find Dockerfiles in JupyterHub GitHub repositories
Display the source blob
Display the rendered blob
Raw
jupyterhub_find_dockerfiles.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "aeabf2be-8f83-43aa-bc30-fd5e9ee6476d",
"metadata": {},
"source": [
"# Find Dockerfiles in JupyterHub GitHub repositories\n",
"Uses the GitHub API to find files matching a case-insensitive regex `^dockerfile.*`"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "ba1ec540-f059-4084-89ae-0237b9ad5774",
"metadata": {},
"outputs": [],
"source": [
"import github\n",
"from IPython.display import Markdown\n",
"import json\n",
"import os\n",
"import re\n",
"from time import sleep"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "57faf734-3e5f-4cc6-a0b8-6a2190cb022f",
"metadata": {},
"outputs": [],
"source": [
"REGEX = re.compile(\"^dockerfile.*\", re.IGNORECASE)\n",
"RESULTS_FILE = \"repo_dockerfiles.json\"\n",
"\n",
"gh = github.Github(os.getenv(\"GITHUB_TOKEN\"))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4acfc4d0-46ad-4f1d-abbe-b4ffdde0dd52",
"metadata": {},
"outputs": [],
"source": [
"try:\n",
" with open(RESULTS_FILE) as f:\n",
" results = json.load(f)\n",
"except FileNotFoundError:\n",
" results = {}"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "33c8e68d-304e-426e-9266-ac1879accced",
"metadata": {},
"outputs": [],
"source": [
"all_repos = [r for r in gh.get_organization(\"jupyterhub\").get_repos() if not r.private and not r.archived]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "ee9f2274-7cbc-49c2-845a-eec13586c582",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def find_files(repo, path, regex):\n",
" print(f\"Checking {repo.name}:{path}\")\n",
" found = []\n",
" cs = repo.get_contents(path)\n",
" for c in cs:\n",
" if c.type == \"file\" and re.match(regex, c.name):\n",
" print(f\" {c.path}\")\n",
" found.append(c.path)\n",
" elif c.type == \"dir\":\n",
" found.extend(find_files(repo, c.path, regex))\n",
" return found"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f8be7bac-d7d4-4fa9-90aa-999e82e9b195",
"metadata": {},
"outputs": [],
"source": [
"for repo in all_repos:\n",
" if repo.name in results:\n",
" continue\n",
" found = find_files(repo, \".\", REGEX)\n",
" print(f\"{repo.name}: Found {found}\")\n",
" results[repo.name] = found\n",
" with open(RESULTS_FILE, \"w\") as f:\n",
" json.dump(results, f)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "01335bd4-a0f1-4acf-8e15-254d665fc60b",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"[binder](https://github.com/jupyterhub/binder):\n",
"- [`doc/tutorials/dockerfile.md`](https://github.com/jupyterhub/binder/tree/HEAD/doc/tutorials/dockerfile.md)\n",
"\n",
"[binderhub](https://github.com/jupyterhub/binderhub):\n",
"- [`helm-chart/images/binderhub/Dockerfile`](https://github.com/jupyterhub/binderhub/tree/HEAD/helm-chart/images/binderhub/Dockerfile)\n",
"\n",
"[chartpress](https://github.com/jupyterhub/chartpress):\n",
"- [`tests/test_helm_chart/amd64only/Dockerfile`](https://github.com/jupyterhub/chartpress/tree/HEAD/tests/test_helm_chart/amd64only/Dockerfile)\n",
"- [`tests/test_helm_chart/image/Dockerfile`](https://github.com/jupyterhub/chartpress/tree/HEAD/tests/test_helm_chart/image/Dockerfile)\n",
"\n",
"[configurable-http-proxy](https://github.com/jupyterhub/configurable-http-proxy):\n",
"- [`Dockerfile`](https://github.com/jupyterhub/configurable-http-proxy/tree/HEAD/Dockerfile)\n",
"\n",
"[docker-image-cleaner](https://github.com/jupyterhub/docker-image-cleaner):\n",
"- [`Dockerfile`](https://github.com/jupyterhub/docker-image-cleaner/tree/HEAD/Dockerfile)\n",
"\n",
"[dockerspawner](https://github.com/jupyterhub/dockerspawner):\n",
"- [`examples/internal-ssl/Dockerfile`](https://github.com/jupyterhub/dockerspawner/tree/HEAD/examples/internal-ssl/Dockerfile)\n",
"- [`examples/simple/Dockerfile`](https://github.com/jupyterhub/dockerspawner/tree/HEAD/examples/simple/Dockerfile)\n",
"- [`examples/swarm/Dockerfile`](https://github.com/jupyterhub/dockerspawner/tree/HEAD/examples/swarm/Dockerfile)\n",
"\n",
"[jupyter-remote-desktop-proxy](https://github.com/jupyterhub/jupyter-remote-desktop-proxy):\n",
"- [`Dockerfile`](https://github.com/jupyterhub/jupyter-remote-desktop-proxy/tree/HEAD/Dockerfile)\n",
"\n",
"[jupyterhub](https://github.com/jupyterhub/jupyterhub):\n",
"- [`Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/Dockerfile)\n",
"- [`demo-image/Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/demo-image/Dockerfile)\n",
"- [`dockerfiles/Dockerfile.alpine`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/dockerfiles/Dockerfile.alpine)\n",
"- [`examples/postgres/db/Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/examples/postgres/db/Dockerfile)\n",
"- [`examples/postgres/hub/Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/examples/postgres/hub/Dockerfile)\n",
"- [`examples/service-fastapi/Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/examples/service-fastapi/Dockerfile)\n",
"- [`onbuild/Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/onbuild/Dockerfile)\n",
"- [`singleuser/Dockerfile`](https://github.com/jupyterhub/jupyterhub/tree/HEAD/singleuser/Dockerfile)\n",
"\n",
"[jupyterhub-deploy-docker](https://github.com/jupyterhub/jupyterhub-deploy-docker):\n",
"- [`Dockerfile.jupyterhub`](https://github.com/jupyterhub/jupyterhub-deploy-docker/tree/HEAD/Dockerfile.jupyterhub)\n",
"- [`examples/custom-notebook-server/Dockerfile`](https://github.com/jupyterhub/jupyterhub-deploy-docker/tree/HEAD/examples/custom-notebook-server/Dockerfile)\n",
"- [`examples/jupyterlab/Dockerfile`](https://github.com/jupyterhub/jupyterhub-deploy-docker/tree/HEAD/examples/jupyterlab/Dockerfile)\n",
"- [`singleuser/Dockerfile`](https://github.com/jupyterhub/jupyterhub-deploy-docker/tree/HEAD/singleuser/Dockerfile)\n",
"\n",
"[jupyterhub-example-kerberos](https://github.com/jupyterhub/jupyterhub-example-kerberos):\n",
"- [`Dockerfile.hub`](https://github.com/jupyterhub/jupyterhub-example-kerberos/tree/HEAD/Dockerfile.hub)\n",
"- [`Dockerfile.kdc`](https://github.com/jupyterhub/jupyterhub-example-kerberos/tree/HEAD/Dockerfile.kdc)\n",
"\n",
"[jupyterhub-on-hadoop](https://github.com/jupyterhub/jupyterhub-on-hadoop):\n",
"- [`docker-demo/image/Dockerfile`](https://github.com/jupyterhub/jupyterhub-on-hadoop/tree/HEAD/docker-demo/image/Dockerfile)\n",
"\n",
"[kerberosauthenticator](https://github.com/jupyterhub/kerberosauthenticator):\n",
"- [`continuous_integration/docker/Dockerfile`](https://github.com/jupyterhub/kerberosauthenticator/tree/HEAD/continuous_integration/docker/Dockerfile)\n",
"\n",
"[mybinder.org-deploy](https://github.com/jupyterhub/mybinder.org-deploy):\n",
"- [`images/analytics-publisher/Dockerfile`](https://github.com/jupyterhub/mybinder.org-deploy/tree/HEAD/images/analytics-publisher/Dockerfile)\n",
"- [`images/federation-redirect/Dockerfile`](https://github.com/jupyterhub/mybinder.org-deploy/tree/HEAD/images/federation-redirect/Dockerfile)\n",
"- [`images/minesweeper/Dockerfile`](https://github.com/jupyterhub/mybinder.org-deploy/tree/HEAD/images/minesweeper/Dockerfile)\n",
"- [`images/tc-init/Dockerfile`](https://github.com/jupyterhub/mybinder.org-deploy/tree/HEAD/images/tc-init/Dockerfile)\n",
"\n",
"[oauthenticator](https://github.com/jupyterhub/oauthenticator):\n",
"- [`examples/full/Dockerfile`](https://github.com/jupyterhub/oauthenticator/tree/HEAD/examples/full/Dockerfile)\n",
"\n",
"[repo2docker](https://github.com/jupyterhub/repo2docker):\n",
"- [`Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/Dockerfile)\n",
"- [`tests/conda/binder-dir/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/conda/binder-dir/Dockerfile)\n",
"- [`tests/dockerfile/binder-dir/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/dockerfile/binder-dir/Dockerfile)\n",
"- [`tests/dockerfile/binder-dir/binder/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/dockerfile/binder-dir/binder/Dockerfile)\n",
"- [`tests/dockerfile/editable/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/dockerfile/editable/Dockerfile)\n",
"- [`tests/dockerfile/jupyter-stack/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/dockerfile/jupyter-stack/Dockerfile)\n",
"- [`tests/dockerfile/simple/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/dockerfile/simple/Dockerfile)\n",
"- [`tests/memlimit/dockerfile/Dockerfile`](https://github.com/jupyterhub/repo2docker/tree/HEAD/tests/memlimit/dockerfile/Dockerfile)\n",
"\n",
"[repo2docker-action](https://github.com/jupyterhub/repo2docker-action):\n",
"- [`Dockerfile`](https://github.com/jupyterhub/repo2docker-action/tree/HEAD/Dockerfile)\n",
"\n",
"[sudospawner](https://github.com/jupyterhub/sudospawner):\n",
"- [`examples/Dockerfile`](https://github.com/jupyterhub/sudospawner/tree/HEAD/examples/Dockerfile)\n",
"\n",
"[the-littlest-jupyterhub](https://github.com/jupyterhub/the-littlest-jupyterhub):\n",
"- [`integration-tests/Dockerfile`](https://github.com/jupyterhub/the-littlest-jupyterhub/tree/HEAD/integration-tests/Dockerfile)\n",
"\n",
"[zero-to-jupyterhub-k8s](https://github.com/jupyterhub/zero-to-jupyterhub-k8s):\n",
"- [`images/hub/Dockerfile`](https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/HEAD/images/hub/Dockerfile)\n",
"- [`images/image-awaiter/Dockerfile`](https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/HEAD/images/image-awaiter/Dockerfile)\n",
"- [`images/network-tools/Dockerfile`](https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/HEAD/images/network-tools/Dockerfile)\n",
"- [`images/secret-sync/Dockerfile`](https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/HEAD/images/secret-sync/Dockerfile)\n",
"- [`images/singleuser-sample/Dockerfile`](https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/HEAD/images/singleuser-sample/Dockerfile)\n",
"\n"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"md = \"\"\n",
"\n",
"for (repo, files) in sorted(results.items()):\n",
" if files:\n",
" md += f\"[{repo}](https://github.com/jupyterhub/{repo}):\\n\"\n",
" for f in sorted(files):\n",
" md += f\"- [`{f}`](https://github.com/jupyterhub/{repo}/tree/HEAD/{f})\\n\"\n",
" md += \"\\n\"\n",
"\n",
"Markdown(md)\n",
"# print(md)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment