How does AI do on the AI Art Turing Test?

extract.py

#!/usr/bin/env python3

import base64
import csv
import os
import re
import urllib.parse
from io import BytesIO

import bs4
import requests
from PIL import Image


def poll_llm(image):
    body = {
        "model": "llama3.2-vision",
        "stream": False,
        "messages": [
            {
                "role": "system",
                "content": "You are a simple image classifier which answers only the question asked in the shortest possible fashion, without explanation or complaint.",
            },
            {
                "role": "user",
                "content": "Was this image produced by an AI image generator, or by a human? Answer just 'AI' or 'Human'.",
                "images": [image],
            },
        ],
    }
    r = requests.post("http://sparky.local:11434/api/chat", json=body)
    if "message" not in r.json():
        print(r.json())
        raise Exception()
    else:
        return r.json()["message"]["content"]


r = requests.get("https://www.astralcodexten.com/p/how-did-you-do-on-the-ai-art-turing")
soup = bs4.BeautifulSoup(r.content, features="lxml")

maybe_imgs = soup.find_all(class_="captioned-image-container")

results = csv.writer(open("results.csv", "w"))

for maybe_img in maybe_imgs:
    maybe_p = maybe_img.find_previous_sibling()
    # print(maybe_img)
    if maybe_p.name == "p":
        m = re.match("^\s*(\d+):\s*(.+)$", maybe_p.text)
        if m:
            num, title = m.group(1), m.group(2)

            img = maybe_img.find("img")

            desc = maybe_img.find_next_sibling()
            classification = desc.text.strip().split()[0]
            if classification.startswith("Human"):
                human = True
                ai = False
            elif classification.startswith("AI"):
                human = False
                ai = True
            else:
                raise Exception()

            print(num, title, "Human" if human else "AI")
            url = img["src"]
            m = re.search(
                "(https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F.+)$",
                url,
            )
            if not m:
                raise Exception()
            url = urllib.parse.unquote(m.group(1))
            filename = os.path.basename(urllib.parse.urlparse(url).path)
            if not os.path.exists(filename):
                r = requests.get(url)
                f = open(filename, "wb")
                f.write(r.content)
                f.flush()
                f.close()
            with Image.open(filename) as img:
                if img.format == "WEBP":
                    img = img.convert("RGB")
                    buffer = BytesIO()
                    img.save(buffer, format="JPEG")
                    imagedata = base64.b64encode(buffer.getvalue())
                else:
                    imagedata = base64.b64encode(open(filename, "rb").read())

            answer = poll_llm(imagedata)
            if answer.endswith("."):
                answer = answer[:-1]
            print(answer)

            results.writerow([num, title, "Human" if human else "AI", answer])

llama3.2-vision:11b-results.csv

looks like 62% accuracy to me. which is oddly close to the score humans get.