finetuning_sdxl_test.py

pipeline = DiffusionPipeline.from_pretrained(
    model_name_base,
    torch_dtype=torch.float16,
).to(device)

# new LoRA weights from fine-tuning process
pipeline.load_lora_weights(
    project_name, 
    weight_name="pytorch_lora_weights.safetensors"
)

refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained(
    model_name_refiner,
    torch_dtype=torch.float16,
).to(device)

subject_prompt = subject_prompt = """oue, photo of oue car, sporty, fast, sleek, sexy, 
aggressive, explosive strength, high performance, bright colors, futuristic cityscape"""

subject_negative_prompt = """person, people, human, floating objects, text, words, 
writing, letters, phrases, trademark, watermark, icon, logo, banner, signature, 
username, monochrome, cropped, cut-off"""

refiner_prompt = """ultra-high-definition, photorealistic, 8k uhd, high-quality, 
ultra sharp detail"""

refiner_negative_prompt = """low quality, low-resolution, out of focus, blurry, 
grainy, artifacts, defects, jpeg artifacts, noise"""

generator = torch.Generator(device).manual_seed(0)

base_image = pipeline(
    prompt=f"{subject_prompt}, {refiner_prompt}", 
    negative_prompt=f"{subject_negative_prompt}, {refiner_negative_prompt}",
    num_inference_steps=100,
    generator=generator,
    height=768,
    width=1024,
    output_type="latent",
).images[0]

refined_image = refiner(
    prompt=refiner_prompt, 
    negative_prompt=refiner_negative_prompt,
    num_inference_steps=20,
    generator=generator, 
    image=base_image,
).images[0]

refined_image.save(f"./generated_images/finetuning_test_{seed}.png")