WP robots.txt boilerplate

robots.txt

# Alfakom.eu robots.txt for WP (change your domain in host and sitemap)
Host: https://domain.tld/

User-agent:  *

Sitemap: https://domain.tld/sitemap.xml
Crawl-delay: 2

# allow this files
Allow: /wp-content/uploads/
Allow: /?display=wide
Allow: /*?page=*

# disallow all files in these directories
Disallow: /cgi-bin/
Disallow: /html/
Disallow: /json/
Disallow: /z/j/
Disallow: /z/c/
Disallow: /stats/
Disallow: /dh_
Disallow: /about/
Disallow: /contact/
Disallow: /tag/
Disallow: /tag/*/page/
Disallow: /tag/*/feed/
Disallow: /page/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-includes/js
Disallow: /wp-content/plugins
Disallow: /wp-content/cache
Disallow: /wp-content/themes
Disallow: /trackback
Disallow: /*trackback
Disallow: /*trackback*
Disallow: /*/trackback
Disallow: */trackback
Disallow: /feed
Disallow: */feed
Disallow: /*/feed/$ 
Disallow: /*/feed/rss/$ 
Disallow: /*/trackback/$ 
Disallow: /*/*/feed/$ 
Disallow: /*/*/feed/rss/$ 
Disallow: /*/*/trackback/$ 
Disallow: /*/*/*/feed/$ 
Disallow: /*/*/*/feed/rss/$ 
Disallow: /*/*/*/trackback/$
Disallow: /calendar/action~posterboard/
Disallow: /calendar/action~agenda/
Disallow: /calendar/action~oneday/
Disallow: /calendar/action~month/
Disallow: /calendar/action~week/
Disallow: /calendar/action~stream/
Disallow: /comments
Disallow: /comments/feed/
Disallow: */comments
Disallow: /comment-page-*
Disallow: /contact
Disallow: /manual
Disallow: /manual/*
Disallow: /phpmanual/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /refer/
Disallow: /category/
Disallow: /category/*/*
Disallow: /xmlrpc.php
Disallow: /*/attachment/
Disallow: /?attachment_id*
Disallow: /*?*
Disallow: /*?
Disallow: /*~*
Disallow: /*~
Disallow: /?s=*
Disallow: /search/*
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
Disallow: /*.wmv$
Disallow: /*.cgi$
Disallow: /*.xhtml$
Disallow: /*.zip$
Disallow: /*.gz$
Disallow: /*.tar.gz$
Disallow: /*.7z$


# nice crawling
User-agent: msnbot
Crawl-delay: 20
User-agent: Slurp
Crawl-delay: 10

User-agent: Googlebot
Allow: /*.css$
Allow: /*.js$
# disallow all files ending with these extensions
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
Disallow: /*.gz$
Disallow: /*.wmv$
Disallow: /*.cgi$
Disallow: /*.xhtml$
Disallow: /*?*
Disallow: /*.txt$
 
# allow google image bot to search all images
User-agent: Googlebot-Image
Allow: /*
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /wp-includes
Disallow: /trackback
Disallow: /comments

 
# allow google bots on entire site
User-agent: Mediapartners-Google*
Disallow:
Allow: /*

User-agent: Adsbot-Google
Disallow:
Allow: /*

User-agent: Googlebot-Mobile
Disallow:
Allow: /*

# DisAllow BAD Bots

User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsSiteAudit
Disallow: /
User-agent: BlackWidow
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Download Ninja
Disallow: /
User-agent: duggmirror
Disallow: /
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: HTTP Weazel
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: HTTrack Website Copier
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: k2spider
Disallow: /
User-agent: larbin
Disallow: /
User-agent: Leech
Disallow: /
User-agent: libwww
Disallow: /
User-agent: linko
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: Offline Commander
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Offline Explorer Pro
Disallow: /
User-agent: Orthogaffe
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: UbiCrawler
Disallow: /
User-agent: WebBandit
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Web Downloader
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebSnake
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: wget
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: Zao
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: ZyBORG
Disallow: /
 

# DisAllow Ai Bots
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-Agent: FacebookBot
Disallow: /
User-Agent: Applebot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: Omgili
Disallow: /
User-agent: YouBot
Disallow: /