example of prompt mixing (different prompts at different denoising steps) with SD2.1

prompt_mixing.py

import torch
from diffusers import StableDiffusionPipeline
from matplotlib import pyplot as plt

device = "cuda" if torch.cuda.is_available() else "cpu"

pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16)
pipe = pipe.to(device)

prompts = ["a hamburger"] * 8 + ["a birthday cake"] * 12

def cb(self, step, timestep, callback_kwargs):
    if step >= len(prompts) - 1:
      return {}
    prompt = prompts[step + 1]
    prompt_embeds, negative_prompt_embeds = pipe.encode_prompt(prompt, device, 1, self.do_classifier_free_guidance)
    return {"prompt_embeds": torch.cat([negative_prompt_embeds, prompt_embeds])}
    
out = pipe(
    prompts[0],
    num_inference_steps=len(prompts),
    generator=torch.manual_seed(0),
    num_images_per_prompt=1,
    callback_on_step_end=cb
)

plt.imshow(out.images[0])