# ==============================================================
# prerender.io - robots.txt
# Last updated: August 2026
# ==============================================================

# ==============================================================
# GENERAL CRAWL RULES
# ==============================================================

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /service-level-agreement/
Disallow: /terms-of-service/
Disallow: /author/
Disallow: /category/
Disallow: /tag/
Disallow: /page/
Disallow: */feed/
Disallow: /feed/
Disallow: /*?s=
Disallow: /*?ref=
Disallow: /*?fpr=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Allow: /wp-content/uploads/
Allow: /wp/wp-admin/admin-ajax.php
Allow: /*.js$
Allow: /*.css$

User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /service-level-agreement/
Disallow: /terms-of-service/
Disallow: /*/feed/
Allow: /wp-content/uploads/
Allow: /wp/wp-admin/admin-ajax.php
Allow: *.js
Allow: *.css

# ==============================================================
# BLOCK AI GROUNDING PAGES FROM TRADITIONAL SEARCH ENGINES
# ==============================================================

User-agent: Googlebot
Disallow: /docs/ai/

User-agent: Bingbot
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /service-level-agreement/
Disallow: /docs/ai/
Allow: /wp-content/uploads/

# ==============================================================
# AI AGENT FAST TRACK - 4 primary AI platforms
#
# OpenAI       - GPTBot
# Google       - Google-Extended
# Anthropic    - ClaudeBot
# Perplexity   - PerplexityBot
# ==============================================================

User-agent: GPTBot
Allow: /docs/ai/
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /service-level-agreement/
Disallow: /terms-of-service/

User-agent: Google-Extended
Allow: /docs/ai/
Allow: /llms.txt

User-agent: ClaudeBot
Allow: /docs/ai/
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /service-level-agreement/
Disallow: /terms-of-service/

User-agent: PerplexityBot
Allow: /docs/ai/
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /service-level-agreement/
Disallow: /terms-of-service/

# ==============================================================
# SITEMAPS
# ==============================================================

Sitemap: https://prerender.io/sitemap_index.xml
Sitemap: https://prerender.io/docs/ai/sitemap-ai.xml