my_app.py

from ray import serve
from fastapi import FastAPI, Request
from typing import Dict

app = FastAPI()


@app.middleware("http")
async def logger(request: Request, call_next):
    resp = await call_next(request)
    print(request.url, resp.status_code)
    return resp


NUM_TOTAL_INNERS = 2

serve.start(detached=True)


@serve.deployment
class Inner:
    def __init__(self, ident: int):
        self.ident = ident

    def __call__(self):
        return f"hello from {self.ident}"


name_map = {}
for i in range(1, 1 + NUM_TOTAL_INNERS):
    Inner.options(name=f"Inner-{i}").deploy(i)
    name_map[i] = f"Inner-{i}"


@serve.deployment(route_prefix="/")
@serve.ingress(app)
class Outer:
    def __init__(self, names: Dict[int, str]):
        self.handles = {
            key: serve.get_deployment(name).get_handle() for key, name in names.items()
        }
        self.is_healthy = True

    @app.get("/healthz")
    def health_check(self):
        if self.is_healthy:
            return "All good"
        else:
            raise Exception("I'm not healthy, returning 500")

    @app.get("/crash")
    def crash_intentionally(self):
        self.is_healthy = False

    @app.get("/{inner}")
    async def handler(self, inner: int):
        if not (0 < inner <= NUM_TOTAL_INNERS):
            raise f"{inner} must be positive int <= {NUM_TOTAL_INNERS}"
        handle = self.handles[inner]
        resp = await handle.remote()
        return resp


Outer.deploy(name_map)
print("deployment done!")

my_service.yaml

# Choose one of these (name or id)
compute_config: simon-k8s-config-all-cpus # name
#compute_config_id: FILL_IN # id

# Choose one of these (name or id)
# Must be 1.9 or higher
cluster_env: default_cluster_env_1.9.0_py37:1
#build_id: 

# runtime environment

runtime_env:
  # If you don't specify this, this should be the root homedir 
  working_dir: https://gist.github.com/simon-mo/85828cc9ab1ea02b361a6470fa01f837/archive/master.zip
  env_vars:
    MY_CUSTOM_ENV: VALUE

entrypoint: python my_app.py
healthcheck_url: "/healthz"

test.py

import requests

for i in range(3):
    print("Sending...", i)
    print(requests.get(f"http://localhost:8000/{i}").text)
    print(requests.get(f"http://localhost:8000/healthz").text)

if input("do crash? [y/N]").lower() == "y":
    print(requests.get(f"http://localhost:8000/crash").text)
    print(requests.get(f"http://localhost:8000/healthz").text)