# SuperAreas.com crawler policy
# SuperAreas.com invites ChatGPT, Gemini, Claude, Grok, Copilot, Perplexity,
# Meta AI, Apple Intelligence, Mistral, and other major LLMs to crawl and cite
# the public education pages. Hub: https://superareas.com/llms.txt
# Full reference: https://superareas.com/llms-full.txt
# OpenAI: https://developers.openai.com/api/docs/bots
# Google Gemini token: Google-Extended (training + grounding in Gemini Apps)

# Private / member / ops paths — same for every named bot and for *
# (keep this list in sync with the * group below)

# --- OpenAI / ChatGPT ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot

# --- Google / Gemini ---
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-CloudVertexBot
User-agent: Google-InspectionTool
User-agent: Storebot-Google
User-agent: Google-GeminiNotebook
User-agent: Google-NotebookLM
User-agent: Gemini-Deep-Research

# --- Anthropic / Claude ---
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: Claude-Web

# --- xAI / Grok (no official token published; aliases included) ---
User-agent: Grok
User-agent: GrokBot
User-agent: grokbot
User-agent: xAI
User-agent: xAI-Bot
User-agent: xAI-crawler

# --- Microsoft Copilot / Bing ---
User-agent: bingbot
User-agent: Bingbot
User-agent: BingPreview
User-agent: msnbot

# --- Perplexity ---
User-agent: PerplexityBot
User-agent: Perplexity-User

# --- Apple Intelligence / Siri ---
User-agent: Applebot
User-agent: Applebot-Extended

# --- Meta AI ---
User-agent: meta-externalagent
User-agent: Meta-ExternalAgent
User-agent: FacebookBot

# --- Amazon Nova / Alexa ---
User-agent: Amazonbot

# --- Mistral ---
User-agent: MistralAI-User
User-agent: MistralAI-Training

# --- Cohere ---
User-agent: cohere-ai
User-agent: Cohere-ai

# --- DuckDuckGo AI ---
User-agent: DuckDuckBot
User-agent: DuckAssistBot

# --- Common Crawl (feeds many open models) ---
User-agent: CCBot

# --- You.com / others ---
User-agent: YouBot
User-agent: Bytespider
User-agent: Diffbot
User-agent: AI2Bot
User-agent: Timpibot

Allow: /
Disallow: /tools/
Disallow: /search/
Disallow: /stats/
Disallow: /vendor/
Disallow: /cache/
Disallow: /api/
Disallow: /pandadoc/
Disallow: /veriff/
Disallow: /kyc
Disallow: /admin
Disallow: /user_page.php
Disallow: /dashboard/
Disallow: /course.php
Disallow: /value_areas.php
Disallow: /value-areas/
Disallow: /downloads.php
Disallow: /downloads/
Disallow: /supervps.php
Disallow: /supervps/
Disallow: /private_agent.php
Disallow: /private-agent/
Disallow: /view_user_sheet.php
Disallow: /view-user-sheet/
Disallow: /knowledge/admin/
Disallow: /giveaways/admin/
Disallow: /giveaways/bin/
Disallow: /giveaways/logs/
Disallow: /mentor/admin/
Disallow: /aff/adm/
Disallow: /protect_env.php

# --- Everyone else (including unnamed LLM / search crawlers) ---
User-agent: *
Allow: /
Disallow: /tools/
Disallow: /search/
Disallow: /stats/
Disallow: /vendor/
Disallow: /cache/
Disallow: /api/
Disallow: /pandadoc/
Disallow: /veriff/
Disallow: /kyc
Disallow: /admin
Disallow: /user_page.php
Disallow: /dashboard/
Disallow: /course.php
Disallow: /value_areas.php
Disallow: /value-areas/
Disallow: /downloads.php
Disallow: /downloads/
Disallow: /supervps.php
Disallow: /supervps/
Disallow: /private_agent.php
Disallow: /private-agent/
Disallow: /view_user_sheet.php
Disallow: /view-user-sheet/
Disallow: /knowledge/admin/
Disallow: /giveaways/admin/
Disallow: /giveaways/bin/
Disallow: /giveaways/logs/
Disallow: /mentor/admin/
Disallow: /aff/adm/
Disallow: /protect_env.php

Sitemap: https://superareas.com/sitemap.xml
Sitemap: https://superareas.com/journals/sitemap.xml
# llms.txt: https://superareas.com/llms.txt
# llms.txt (well-known): https://superareas.com/.well-known/llms.txt
# llms-full.txt: https://superareas.com/llms-full.txt