This is a gist on how to get StreamDiffusion running on a Mac (mps)
git clone https://github.com/cumulo-autumn/StreamDiffusion.gitcd StreamDiffusion
python -m venv venv (or python3)
source venv/bin/activatepip install --upgrade pip
pip install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cpu
pip install . - Go to venv/lib/python3.11/site-packages/streamdiffusion/pipeline.py
Line 439, replace the call function by this:
@torch.no_grad()
# condition hack event sync/track for non-cuda devices, RIP profiling etc
def __call__(
self, x: Union[torch.Tensor, PIL.Image.Image, np.ndarray] = None
) -> torch.Tensor:
if self.device == "cuda":
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
if x is not None:
x = self.image_processor.preprocess(x, self.height, self.width).to(
device=self.device, dtype=self.dtype
)
if self.similar_image_filter:
x = self.similar_filter(x)
if x is None:
time.sleep(self.inference_time_ema)
return self.prev_image_result
x_t_latent = self.encode_image(x)
else:
# TODO: check the dimension of x_t_latent
x_t_latent = torch.randn((1, 4, self.latent_height, self.latent_width)).to(
device=self.device, dtype=self.dtype
)
x_0_pred_out = self.predict_x0_batch(x_t_latent)
x_output = self.decode_image(x_0_pred_out).detach().clone()
self.prev_image_result = x_output
if self.device == "cuda":
end.record()
torch.cuda.synchronize()
inference_time = start.elapsed_time(end) / 1000
self.inference_time_ema = 0.9 * self.inference_time_ema + 0.1 * inference_time
return x_outputMake sure you have node installed and npm or pnpn in your path.
- Edit config.py to set
torch.device("cuda" if torch.cuda.is_available() else "mps")
...
acceleration: Literal["none", "xformers", "tensorrt"] = "none"- Run
cd demo/realtime-txt2img
pip install -r requirements.txt
./start.shor
cd demo/realtime-txt2img
pip install -r requirements.txt
cd frontend
pnpm i
pnpm run build
cd ..
python main.py- Edit main.py (line 161)
device = torch.device("cuda" if torch.cuda.is_available() else "mps")- Run the demo
cd demo/realtime-img2img
pip install -r requirements.txt
./start.sh[1] cumulo-autumn/StreamDiffusion#34
[2] cumulo-autumn/StreamDiffusion#125
Successfully installed Pillow-10.1.0 accelerate-0.24.0 annotated-types-0.7.0 anyio-3.7.1 click-8.1.7 compel-2.0.2 controlnet-aux-0.0.7 einops-0.8.0 exceptiongroup-1.2.2 fastapi-0.104.1 h11-0.14.0 httptools-0.6.4 imageio-2.36.1 lazy-loader-0.4 markdown2-2.5.1 opencv-python-4.10.0.84 peft-0.6.0 pydantic-2.10.2 pydantic-core-2.27.1 pyparsing-3.2.0 python-dotenv-1.0.1 scikit-image-0.24.0 scipy-1.14.1 sniffio-1.3.1 starlette-0.27.0 tifffile-2024.9.20 timm-1.0.11 tokenizers-0.15.2 transformers-4.35.2 uvicorn-0.24.0.post1 uvloop-0.21.0 watchfiles-1.0.0 websockets-14.1