# =====================================================================
# PUBLIC robots.txt -- LIVE as of 2026-08-12. This is the source template
# for site/robots.txt, copied in by scripts/go-public.sh. Edit here, not
# the generated copy under site/.
# =====================================================================

# NOTE: /partners/ (vendor-facing partner-program pricing) is deliberately
# excluded for EVERY agent, including AI/answer-engine crawlers. It is a
# vendor sales page, not buyer-facing index content -- it must stay out of
# search and out of LLM training/answer corpora even after go-public. It is
# reachable only via a single quiet footer link (rel=nofollow) and carries
# its own <meta name="robots" content="noindex, nofollow, noarchive">.
# Do NOT strip that meta tag during the go-public sweep.

# ANTI-SCRAPING (2026-09-25, docs/security/anti-scraping.md). EVERY HTML
# page stays open to EVERY crawler listed here -- search engines and AI /
# answer engines read the site exactly as before. What changed:
#   * /api/export/ is a honeypot (hidden, nofollow links in every page's
#     footer). It is disallowed for EVERY agent: only a client that ignores
#     robots.txt ever requests it, and that client loses DATA-API access for
#     7 days (never page access).
#   * The bulk data -- /assets/data/ (the JSON files) and /api/ (the data
#     API) -- is disallowed for AI crawlers and for every agent not named
#     below (the "*" group). Pages carry their key content in the HTML
#     itself, so none of these agents needs the JSON to read a page.
#   * SEARCH ENGINES (Googlebot, Bingbot, Applebot, ...) keep /assets/data/
#     and /api/: they render pages with JavaScript and fetch that JSON while
#     doing it, and a blocked render resource costs rankings. Their group is
#     otherwise identical to the pre-2026-09-25 rules. On the server side a
#     search crawler is only trusted after reverse + forward DNS checks.
# The monthly price-report CSVs (/reports/<month>/data.csv) stay open to all.

# Search engines: every page AND the JSON they fetch while rendering pages.
User-agent: Googlebot
User-agent: Google-InspectionTool
User-agent: Storebot-Google
User-agent: Bingbot
User-agent: msnbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /api/export/

# Every other agent (incl. any scraper that honours robots.txt): every page,
# no bulk data.
User-agent: *
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

# AI / LLM ingestion crawlers -- explicitly welcomed to read and cite every page
# for answer-engine indexing; the bulk data files and the data API are not
# for ingestion (owner decision 2026-09-25). Data licensing: partners@peptidepricehistory.com.
User-agent: GPTBot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

User-agent: meta-externalagent
Allow: /
Disallow: /partners/
Disallow: /partner-statement-sample/
Disallow: /partner-profile-sample/
Disallow: /partner-directory-sample/
Disallow: /assets/data/
Disallow: /api/

# Machine-readable site map for LLMs (llmstxt.org standard).
# Not a robots.txt directive most crawlers parse, but harmless and documents intent:
# LLM index: /llms.txt and /llms-full.txt

Sitemap: https://www.peptidepricehistory.com/sitemap.xml