https://news.ycombinator.com/edit?id=35793802 demo script

hn_demo.py

import modal

def download_model():
    from transformers import pipeline
    pipeline("fill-mask", model="bert-base-uncased")

CACHE_PATH = "/root/model_cache"
ENV = modal.Secret({"TRANSFORMERS_CACHE": CACHE_PATH})

image = (
    modal.Image.debian_slim()
    .pip_install("torch", "transformers")
    .run_function(download_model, secret=ENV)
)
stub = modal.Stub(name="hn-demo", image=image)


class Model:
    def __enter__(self):
        import torch
        from transformers import pipeline
        self.model = pipeline("fill-mask", model="bert-base-uncased", device=0)

    @stub.function(
        gpu="a10g",
        secret=ENV,
    )
    def handler(self, prompt: str):
        return self.model(prompt)


if __name__ == "__main__":
    with stub.run():
        prompt = "Hello World! I am a [MASK] machine learning model."
        print(Model().handler.call(prompt)[0]["sequence"])