# ── Search engines: normal crawl ─────────────────────────────────
# Google/Bing search crawlers keep full access — SEO depends on it.
# AI crawlers get a separate, narrower policy below.

User-agent: *
# Content Signals (contentsignals.org): search indexing and AI answers that
# cite a page are fine; training models on the dataset is not.
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

Disallow: /api/
Disallow: /strains/data.json
Disallow: /dispensaries/data.json
Disallow: /brands/data.json

# ── AI assistants and AI search: full page access ─────────────────
# Owner decision, 2026-09-14. These fetch a page because a person asked
# about it (ChatGPT-User, Claude-User, Perplexity-User) or index pages
# so an AI answer can cite and link them (OAI-SearchBot,
# Claude-SearchBot, PerplexityBot, DuckAssistBot). They read what a
# visitor reads. Bulk data endpoints stay closed, and agents that want
# structured answers should use the MCP endpoint instead of crawling.
# The Cloudflare WAF rule "Block AI crawlers on directory data paths"
# must stay in step with this group.

User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /strains/data.json
Disallow: /dispensaries/data.json
Disallow: /brands/data.json

# ── AI training crawlers: landing pages only ─────────────────────
# The harvested dataset (menus, prices, carriers, per-store and
# per-brand distribution) is the product, and Content-Signal above
# says ai-train=no. Training crawlers may read the marketing and
# reference surfaces, not the directory pages. Most-specific rule
# wins: the Allows below override the Disallow: / for those URLs.

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Diffbot
User-agent: omgili
User-agent: Timpibot
Allow: /$
Allow: /about/
Allow: /contact/
Allow: /glossary/
Allow: /markets/
Allow: /strains/$
Allow: /brands/$
Allow: /dispensaries/$
Allow: /llms.txt
Disallow: /

Sitemap: https://greenrush.io/sitemap-index.xml