# https://www.robotstxt.org/robotstxt.html

# Cloudflare injects its own managed block ABOVE this file at the edge (Content
# Signals Policy plus Disallow rules for AI training crawlers). That block opens
# with its own "User-agent: *" carrying only "Allow: /", so the live file has two
# "*" groups and ours is second. RFC 9309 crawlers merge same-agent groups and
# are fine, but a crawler that stops at the first match sees "Allow: /" and
# ignores every restriction below.
#
# The fix is the extra User-agent lines: one group can be declared for several
# agents at once, and a named agent beats "*" regardless of ordering, so every
# crawler listed here selects THIS group and Cloudflare's cannot shadow it.
# Keep the list to crawlers that actually visit; unnamed agents still fall
# through to the "*" line and get the same rules.
User-agent: *
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# Allow specimen images (must come BEFORE the broader /api disallow)
Allow: /api/images/specimen/

# Allow colony photos (covers and timeline entries) served by the image proxy;
# colony pages, the image sitemap and JSON-LD all reference these URLs.
Allow: /api/images/colony/

# Community blog hero and body images (Google Images, Bing Images)
Allow: /api/images/blog/

# Allow public read-only agent APIs (advertised in /.well-known/api-catalog)
Allow: /api/colonies
Allow: /api/colony-caretargets

# Disallow private/internal routes
Disallow: /api/*
Disallow: /account/*

# Disallow placeholder images from indexing
Disallow: /images/placeholders/

# Decorative/UI icons — allow logo.svg, partners, and favicon
Disallow: /icons/
Allow: /icons/logo.svg
Allow: /icons/partners/
Allow: /icons/favicon/

# Species image placeholders
Disallow: /images/species/antbody.png
Disallow: /images/species/anthead.png

# Blog hero fallback
Disallow: /images/blogs/ant-placeholder.avif

# Decorative cave assets, homepage plants/waves, and partner logos are kept OUT
# of the index via an "X-Robots-Tag: noindex" HTTP header (see public/_headers),
# NOT via Disallow. Googlebot must be allowed to CRAWL them so it can SEE the
# noindex header — a Disallow here would hide the header and let already-indexed
# images linger in the index. Do not re-add Disallow rules for these paths.

# Lottie animation JSON
Disallow: /lottie/

# Crown rank badge
Disallow: /crown.png

# Profile borders
Disallow: /images/borders/

# Species silhouette banners
Disallow: /images/banners/

# PWA manifest icons
Disallow: /images/seo/web-app-manifest-192x192.png
Disallow: /images/seo/web-app-manifest-512x512.png

# Sitemap references
Sitemap: https://antscout.com/sitemap.xml