# ==================================================
# AI Agents Library robots.txt
# Search + AI discovery allowed
# Private, utility and dynamic URLs excluded
# Checkout and signup: page crawlable (reads noindex),
# all query-string variants blocked
# ==================================================


# ======================
# Google
# ======================

User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Bing
# ======================

User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=
Crawl-delay: 5


# ======================
# DuckDuckGo
# ======================

User-agent: DuckDuckBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=

User-agent: DuckAssistBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Apple Search
# ======================

User-agent: Applebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# OpenAI Search / user requests
# ======================

User-agent: OAI-SearchBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=
Crawl-delay: 3

User-agent: ChatGPT-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Perplexity
# ======================

User-agent: PerplexityBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=
Crawl-delay: 2

User-agent: Perplexity-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Anthropic / Claude
# ======================

User-agent: Claude-SearchBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=

User-agent: Claude-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Meta user-triggered fetcher
# ======================

User-agent: Meta-ExternalFetcher
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Block training / low-value bots
# ======================

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: AwarioBot
Disallow: /

User-agent: ShapBot
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: Reeder
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AteveSearchSourceUrlDiscovery
Disallow: /


# ======================
# SEO / archive
# ======================

User-agent: AhrefsBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=
Crawl-delay: 15

User-agent: archive.org_bot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# All other crawlers
# ======================

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /checkout/?
Disallow: /account/
Disallow: /reset-password/
Disallow: /welcome/
Disallow: /thank-you/
Disallow: /signup/?
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*edd_action=


# ======================
# Sitemap
# ======================

Sitemap: https://www.aiagentslibrary.com/sitemap_index.xml