Skip to content

Instantly share code, notes, and snippets.

@toilaluan

toilaluan/benchmark_false_positive.ipynb

Created March 4, 2024 08:39
Show Gist options
  • Save toilaluan/b6d61af5a9761a212e538801bcb90232 to your computer and use it in GitHub Desktop.
Save toilaluan/b6d61af5a9761a212e538801bcb90232 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
benchmark_false_positive.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import diffusers\n",
"from services.rewarding.cosine_similarity_compare import CosineSimilarityReward\n",
"import torch"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"rewarder = CosineSimilarityReward()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"StableDiffusionXLPipeline {\n",
" \"_class_name\": \"StableDiffusionXLPipeline\",\n",
" \"_diffusers_version\": \"0.25.0\",\n",
" \"feature_extractor\": [\n",
" null,\n",
" null\n",
" ],\n",
" \"force_zeros_for_empty_prompt\": true,\n",
" \"image_encoder\": [\n",
" null,\n",
" null\n",
" ],\n",
" \"scheduler\": [\n",
" \"diffusers\",\n",
" \"EulerAncestralDiscreteScheduler\"\n",
" ],\n",
" \"text_encoder\": [\n",
" \"transformers\",\n",
" \"CLIPTextModel\"\n",
" ],\n",
" \"text_encoder_2\": [\n",
" \"transformers\",\n",
" \"CLIPTextModelWithProjection\"\n",
" ],\n",
" \"tokenizer\": [\n",
" \"transformers\",\n",
" \"CLIPTokenizer\"\n",
" ],\n",
" \"tokenizer_2\": [\n",
" \"transformers\",\n",
" \"CLIPTokenizer\"\n",
" ],\n",
" \"unet\": [\n",
" \"diffusers\",\n",
" \"UNet2DConditionModel\"\n",
" ],\n",
" \"vae\": [\n",
" \"diffusers\",\n",
" \"AutoencoderKL\"\n",
" ]\n",
"}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pipe = diffusers.StableDiffusionXLPipeline.from_single_file(\n",
" \"/root/code/NicheImage/checkpoints/AnimeV3.safetensors\",\n",
" torch_dtype=torch.float16,\n",
" use_safetensors=True\n",
")\n",
"pipe.scheduler = diffusers.EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)\n",
"pipe.to(\"cuda\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 5%|▌ | 1/20 [00:00<00:14, 1.35it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of a paladin woman, olive skin, long dark hair, beautiful bone structure, symmetrical facial features, intricate, elegant, digital painting, concept art, smooth, sharp focus, illustration, from Valerian and the City of a Thousand Planets, from the book by Ruan Jia and Mandy Jurgens and Artgerm and William-Adol'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 10%|█ | 2/20 [00:01<00:12, 1.42it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of a post apocalyptic tribal cyborg djiking on a mountain peak, wearing a futuristic long black jacket and a fuzzy red cape, cyborg, by hr giger, artgerm, moebius, alphonse mucha, and greg rutkowski'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 15%|█▌ | 3/20 [00:02<00:11, 1.48it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of a astronaut floating towards the moon rim in star wars, with red glowing lights, by john harris, mark rothko, rendered in unreal engine, trending on artstation, epic scale fisheye!!!!!!'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 20%|██ | 4/20 [00:02<00:10, 1.53it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of large-scale shipwreck on the sea, dynamic lighting, photorealistic concept art, stunning visuals, creative, cinematic, ultra detailed, trending on art station, focus, extreme beeple'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 25%|██▌ | 5/20 [00:03<00:09, 1.51it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of an Emperor Arnold Schwarzenegger, face close-up, reptilian, at James Gilleard in the style of his Excalibur, dark, gritty, intricate, elegant, volumetric lighting, digital painting, highly detailed, artstation, sharp focus'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 30%|███ | 6/20 [00:04<00:09, 1.51it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of Alexandra Daddario on a British European Walkway in the daytime, hyper-realistic, very detailed, realistic water splashes, trending on Artstation, unreal engine, very detailed, high quality, 8K HDR, octane render, HD'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 35%|███▌ | 7/20 [00:04<00:08, 1.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of a planet from the distant future in an alien desert, futuristic technology, detailed and intricate environment, matte painting, cinematic, epic composition, concept art, dramatic'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 40%|████ | 8/20 [00:05<00:07, 1.52it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of daft punk as a medieval knight wearing a golden helmet and an iron gauntlet, fantasy, intricate, elegant, highly detailed, on a medieval castle, digital painting, artstation, concept art, sharp focus, illustration, art by greg rutkowski and alphonse mucha'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 45%|████▌ | 9/20 [00:05<00:07, 1.50it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of realistic humanoid Robot forward angle, D&D, fantasy, intricate, cinematic lighting, highly detailed, digital painting, artstation, concept art, smooth, sharp focus, illustration, art by artgerm and greg rutkowski and alphonse mucha, footage from space camera'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 50%|█████ | 10/20 [00:06<00:06, 1.49it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of a young and strong girl wearing gucci from behind in a futuristic penthouse apartment complex, behind a holographic wall, dramatic lighting, illustration by Greg rutkowski, yoji shinkawa, 4k, digital art, concept art, trending on artstation'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 55%|█████▌ | 11/20 [00:07<00:06, 1.47it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of very old and fragile daenerys targaryen on a bad ketamine drug trip, drugs and alcohol littered everywhere, fantasy, highly detailed, digital painting, artstation, concept art, smooth, sharp focus, illustration, art by artgerm and greg rutkowski and alphonse mucha'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 60%|██████ | 12/20 [00:07<00:05, 1.53it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of athletic Taylor Swift in spring training. film still. 8k resolution. bokeh. iridescent accents. black and white color scheme'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 65%|██████▌ | 13/20 [00:08<00:04, 1.60it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of an astronaut holding a yellow planet in the arms, by thomas kinkade, trending on artstation'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 70%|███████ | 14/20 [00:09<00:03, 1.53it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of an astronaut in starfleet uniform, olive skin, long dark hair, beautiful bone structure, symmetrical facial features, intricate, elegant, digital painting, concept art, smooth, sharp focus, illustration, from StarCraft by Ruan Jia and Mandy Jurgens and Artgerm and William-Adolphe Bouguerea'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 75%|███████▌ | 15/20 [00:09<00:03, 1.51it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of an elegant, well dressed, well dressed, well fed, modern young medieval european man with few eyebrows and very short hair, dressed in roman clothes, by frank frazetta and ashley wood, greg rutkowski, trending on artstation.'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 80%|████████ | 16/20 [00:10<00:02, 1.47it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of a humanoid robot wearing a veil ( ( ( ( illuminated sunglasses ) ) ) ), lit only by purple. background entirely by robot circuitry. intricate artwork by caravaggio. Trending on artstation, octane render, cinematic lighting from the right, hyper realism, octane render, 8k, depth of field, 3D'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 85%|████████▌ | 17/20 [00:11<00:02, 1.43it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of bright psychedelic male face profile picture by norman rockwell, intricate, elegant, highly detailed, centered, digital painting, artstation, concept art, smooth, sharp focus, illustration, artgerm, tomasz alen kopera, peter mohrbacher, donato giancola, joseph christian leyendecker,'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 90%|█████████ | 18/20 [00:12<00:01, 1.46it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of an adult female hugging an adult female, hugging, intricate, detailed, volumetric lighting, scenery, digital painting, highly detailed, artstation, sharp focus, illustration, concept art, ruan jia, steve mccurry'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 95%|█████████▌| 19/20 [00:12<00:00, 1.45it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of large pint of beer and large technics dj table front of picture, in the backround man and a woman, dark backround, highly detailed, digital illustration, trending in artstation, classical painting, smooth, sharp focus lighting, concept art, intricate, einar jonsson and bouguereau'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 20/20 [00:13<00:00, 1.49it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'prompt': 'an image of an anime female martial artist!! long black hair! teal eyes! elegant, super highly detailed, professional digital painting, artstation, concept art, 8k, art by wlop, artgerm and alphonse mucha and eiichiro oda'}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"import requests\n",
"from tqdm import tqdm\n",
"prompts = []\n",
"\n",
"for i in tqdm(range(20)):\n",
" response = requests.post(\"http://nicheimage.nichetensor.com/challenge/prompt\", json={\n",
" \"prompt\": \"an image of\",\n",
" \"seed\": i,\n",
" })\n",
" print(response.json())\n",
" prompt = response.json()[\"prompt\"]\n",
" prompts.append(prompt)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of a paladin woman, olive skin, long dark hair, beautiful bone structure, symmetrical facial features, intricate, elegant, digital painting, concept art, smooth, sharp focus, illustration, from Valerian and the City of a Thousand Planets, from the book by Ruan Jia and Mandy Jurgens and Artgerm and William-Adol\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e74d960de89548de9bf2df1b11f2e1f9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7904b35a8d5d42b884580b95dd7e85cc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of a post apocalyptic tribal cyborg djiking on a mountain peak, wearing a futuristic long black jacket and a fuzzy red cape, cyborg, by hr giger, artgerm, moebius, alphonse mucha, and greg rutkowski\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c42750698f7d4b72b6141cf7a57daea0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "54df497196b543e998c6815c245b8e6d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of a astronaut floating towards the moon rim in star wars, with red glowing lights, by john harris, mark rothko, rendered in unreal engine, trending on artstation, epic scale fisheye!!!!!!\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "eec76fc44cb147a5bfacd0fc3dcc2162",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1b937066820c48d89fd1a267df2266e2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of large-scale shipwreck on the sea, dynamic lighting, photorealistic concept art, stunning visuals, creative, cinematic, ultra detailed, trending on art station, focus, extreme beeple\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e1bbc04426ec45548159f3b6a6f069cc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2faeb18f342e4de2a42c9688d35906b4",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of an Emperor Arnold Schwarzenegger, face close-up, reptilian, at James Gilleard in the style of his Excalibur, dark, gritty, intricate, elegant, volumetric lighting, digital painting, highly detailed, artstation, sharp focus\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "56d18df8ccd246108d0e8d41b385267a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "79d3f43d42434f998215247826a1ba76",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of Alexandra Daddario on a British European Walkway in the daytime, hyper-realistic, very detailed, realistic water splashes, trending on Artstation, unreal engine, very detailed, high quality, 8K HDR, octane render, HD\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "427c8ef127774ee9a96f2c2cc52e8c67",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4d1fd96f377d408b8135502eb3822d9b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of a planet from the distant future in an alien desert, futuristic technology, detailed and intricate environment, matte painting, cinematic, epic composition, concept art, dramatic\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2175a8719848406d9e5e69d5e89704a3",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0aba9903e68f42aab0606eb50d911162",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of daft punk as a medieval knight wearing a golden helmet and an iron gauntlet, fantasy, intricate, elegant, highly detailed, on a medieval castle, digital painting, artstation, concept art, sharp focus, illustration, art by greg rutkowski and alphonse mucha\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f95137209a994c73ac7c8895c92fafd6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4dd087475d75454cb9d753055ca98964",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of realistic humanoid Robot forward angle, D&D, fantasy, intricate, cinematic lighting, highly detailed, digital painting, artstation, concept art, smooth, sharp focus, illustration, art by artgerm and greg rutkowski and alphonse mucha, footage from space camera\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "325a458800ad4d7f9867f5a629197e32",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "efd414c670ec4179b289405ccb7ded61",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of a young and strong girl wearing gucci from behind in a futuristic penthouse apartment complex, behind a holographic wall, dramatic lighting, illustration by Greg rutkowski, yoji shinkawa, 4k, digital art, concept art, trending on artstation\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4b80e161fab04df78249e1988bf4a8e0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "fec1bb5ebd024898aa10a987c794f4e2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of very old and fragile daenerys targaryen on a bad ketamine drug trip, drugs and alcohol littered everywhere, fantasy, highly detailed, digital painting, artstation, concept art, smooth, sharp focus, illustration, art by artgerm and greg rutkowski and alphonse mucha\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "fb386f14dc0c465ea1755af68465ab97",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6de06a356e3d4915b9642cb90929e7a5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of athletic Taylor Swift in spring training. film still. 8k resolution. bokeh. iridescent accents. black and white color scheme\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "93e3e745ec074c2eb52b7698dfa0ad20",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5ae856906e2e41a6a766f983e411b29a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of an astronaut holding a yellow planet in the arms, by thomas kinkade, trending on artstation\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "12d73afb642b40c79df7b5b3fd5722c7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b9eb5c5244034dccb0dc39b3a2018c1f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of an astronaut in starfleet uniform, olive skin, long dark hair, beautiful bone structure, symmetrical facial features, intricate, elegant, digital painting, concept art, smooth, sharp focus, illustration, from StarCraft by Ruan Jia and Mandy Jurgens and Artgerm and William-Adolphe Bouguerea\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1eb5991b85344b89ba8a6910fcbe395b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ffecd9308ac945a39867f115c66441f7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of an elegant, well dressed, well dressed, well fed, modern young medieval european man with few eyebrows and very short hair, dressed in roman clothes, by frank frazetta and ashley wood, greg rutkowski, trending on artstation.\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a08d8836f051487cbb14e565edd13cab",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "144f707792514f9d82b2427c23c54b82",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of a humanoid robot wearing a veil ( ( ( ( illuminated sunglasses ) ) ) ), lit only by purple. background entirely by robot circuitry. intricate artwork by caravaggio. Trending on artstation, octane render, cinematic lighting from the right, hyper realism, octane render, 8k, depth of field, 3D\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1437778e1dab42e48c8158a49df38f4d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "22671bf2cbd34047bebfbd6bcd763719",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of bright psychedelic male face profile picture by norman rockwell, intricate, elegant, highly detailed, centered, digital painting, artstation, concept art, smooth, sharp focus, illustration, artgerm, tomasz alen kopera, peter mohrbacher, donato giancola, joseph christian leyendecker,\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "93d9854f63c048bf89424e76ae250d62",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7972b9b3b6eb446890d71b09958adc7a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of an adult female hugging an adult female, hugging, intricate, detailed, volumetric lighting, scenery, digital painting, highly detailed, artstation, sharp focus, illustration, concept art, ruan jia, steve mccurry\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3b74288dc87a4d5db9b6080ccd7e37cb",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6f25a9094743418eb6cdc131a3225194",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of large pint of beer and large technics dj table front of picture, in the backround man and a woman, dark backround, highly detailed, digital illustration, trending in artstation, classical painting, smooth, sharp focus lighting, concept art, intricate, einar jonsson and bouguereau\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "adb7d0bc1a7940b2a9c0ff75793f9ea5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0e5de89b40564bc6834b0ab7e41f99d5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"an image of an anime female martial artist!! long black hair! teal eyes! elegant, super highly detailed, professional digital painting, artstation, concept art, 8k, art by wlop, artgerm and alphonse mucha and eiichiro oda\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e58d851df83d4305b948d6ed4ae30565",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/25 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "665b4e5d21c74e2bb9f34c57cf6188c8",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/20 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"valid_images = []\n",
"cheat_images = []\n",
"for i, prompt in enumerate(prompts):\n",
" print(prompt)\n",
" generator = torch.manual_seed(i)\n",
" valid_image = pipe(prompt, width = 1024, height = 1024, num_inference_steps=25, generator=generator).images[0]\n",
" cheat_image = pipe(prompt, width = 1024, height = 1024, num_inference_steps=20, generator=generator).images[0]\n",
" valid_images.append(valid_image)\n",
" cheat_images.append(cheat_image)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean: 0.776810209453106, var: 0.012650611296589911\n",
"[0.91362375 0.79069388 0.54959083 0.94109058 0.74512744 0.70355606\n",
" 0.89877784 0.76919961 0.77242136 0.79622179 0.84198719 0.48744348\n",
" 0.89035106 0.88553894 0.67276311 0.75579047 0.70541686 0.81207246\n",
" 0.7692095 0.83532798]\n"
]
}
],
"source": [
"scores = []\n",
"for valid, cheat in zip(valid_images, cheat_images):\n",
" reward = rewarder.forward(valid, cheat, binary=False)\n",
" scores.append(reward)\n",
"import numpy as np\n",
"# mean score\n",
"scores = np.array(scores)\n",
"mean = np.mean(scores)\n",
"var = np.var(scores)\n",
"print(f\"mean: {mean}, var: {var}\")\n",
"print(scores)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment