# Cloudpylon crawler policy (platform SEO-1 to SEO-7, SEO-13 and SEO-18 to
# SEO-20). Search engines may crawl canonical public pages; AI search and
# user-requested AI fetchers receive the static discovery baseline; verified
# previews may fetch public pages; training crawlers receive nothing.

# Approved traditional search engines: canonical, queryless public pages and
# their assets, except the browser-facing API paths.
User-agent: Googlebot
User-agent: Google-InspectionTool
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
Disallow: /*?
Allow: /*/instances/*?region=
Allow: /*/instances/*?location=
Allow: /azure/vms/*?region=

# Approved AI search and user-requested AI fetchers: the direct static
# discovery baseline only. The aliases cover /discover, /discover/ and the
# explicit static assets named by the dynamic-site route inventory.
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: DuckAssistBot
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /discover$
Allow: /discover/$
Allow: /discover/index.html$
Allow: /discover/sitemap.xml$
Allow: /discover/styles.css$
Allow: /llms.txt$
Allow: /29e811ec4e5c187402d59ee00d660bab.txt$
Allow: /robots.txt$
Allow: /.well-known/security.txt$
Allow: /favicon.ico$
Allow: /apple-touch-icon.png$
Allow: /site.webmanifest$
Allow: /health$
Allow: /health/ready$
Disallow: /

# Verified link previews keep access to public canonical pages so shared links
# retain their title, description and image metadata.
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
Disallow: /*?

# Dedicated training crawlers receive no page content.
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: cohere-ai
Disallow: /

# Unknown bots and scrapers receive the same direct static baseline as the AI
# groups. This deliberately excludes the SSR home page and SSR sitemap.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /discover$
Allow: /discover/$
Allow: /discover/index.html$
Allow: /discover/sitemap.xml$
Allow: /discover/styles.css$
Allow: /llms.txt$
Allow: /29e811ec4e5c187402d59ee00d660bab.txt$
Allow: /robots.txt$
Allow: /.well-known/security.txt$
Allow: /favicon.ico$
Allow: /apple-touch-icon.png$
Allow: /site.webmanifest$
Allow: /health$
Allow: /health/ready$
Disallow: /

# The full canonical sitemap remains the search-engine URL directory; the
# static discovery sitemap above is the bounded baseline for other clients.
Sitemap: https://cloudpylon.com/sitemap.xml
Sitemap: https://cloudpylon.com/discover/sitemap.xml