robots.txt

config to stop ai tools from scraping your website

# <AI bots>

# GPTBot is OpenAI's web crawler
User-agent: GPTBot
Disallow: /

# Google Bard & Gemini
User-agent: Google-Extended
Disallow: /

# ChatGPT-User is OpenAI's web crawler
User-agent: ChatGPT-User
Disallow: /

# Common Crawl bot
User-agent: CCBot
Disallow: /

# PiplBot is PiplBot's web crawler
User-agent: PiplBot
Disallow: /

# anthropic-ai is Anthropic's web crawler
User-agent: anthropic-ai
Disallow: /

# Claude-Web is Claude's web crawler
User-agent: Claude-Web
Disallow: /

# TurnitinBot is Turnitin's web crawler
User-agent: TurnitinBot
Disallow: /

# PetalBot is Petal's web crawler
User-agent: PetalBot
Disallow: /

# MoodleBot is Moodl's web crawler
User-agent: MoodleBot
Disallow: /

# magpie-crawler is Brandwatch.com's web crawler
User-agent: magpie-crawler
Disallow: /

# Meta 
User-agent: meta-externalagent
Disallow: /

# </AI bots>

User-agent: ImagesiftBot
Disallow: /

# DotBot is OpenSiteExplorer's web crawler
User-agent: DotBot
Disallow: /