# ==============================================================================
# Robots.txt for Aegis Education (https://ageiseducation.online)
# Evidence-informed Medical Index: 100,000 Articles & Clinical How-To Protocols
# Designed for maximum crawl budget efficiency & high-performance indexing
# ==============================================================================

# SITEMAPS
# Explicit sitemap declaration
Sitemap: https://ageiseducation.online/api/sitemap.xml
Sitemap: https://ageiseducation.online/sitemap.xml
Sitemap: https://ageiseducation.online/sitemap-articles.xml
Sitemap: https://ageiseducation.online/sitemap-main.xml
Sitemap: https://ageiseducation.online/sitemap-howto.xml
Sitemap: https://ageiseducation.online/sitemap-topics-1.xml
Sitemap: https://ageiseducation.online/sitemap-topics-2.xml
Sitemap: https://ageiseducation.online/sitemap-topics-3.xml
Sitemap: https://ageiseducation.online/sitemap-topics-4.xml
Sitemap: https://ageiseducation.online/sitemap-qa-1.xml
Sitemap: https://ageiseducation.online/sitemap-qa-2.xml
Sitemap: https://ageiseducation.online/news-sitemap.xml
Sitemap: https://ageiseducation.online/google-news.xml
Sitemap: https://ageiseducation.online/feed.xml
Sitemap: https://ageiseducation.online/atom.xml

# LLM Directives (RFC llms.txt standard)
# Full AI Reference Guide: https://ageiseducation.online/llms.txt
# Extended Knowledgebase: https://ageiseducation.online/llms-full.txt

# ==============================================================================
# DEFAULT SEARCH ENGINE & CRAWLER RULES (ALL USER-AGENTS)
# ==============================================================================
User-agent: *
# Prioritize and allow all crawlers unrestricted access to article & content routes
Allow: /articles
Allow: /articles/
Allow: /articles/*
Allow: /article
Allow: /article/
Allow: /article/*
Allow: /topics/
Allow: /topics/*
Allow: /qa/
Allow: /qa/*
Allow: /hormones/
Allow: /hormones/*
Allow: /illness-matrix
Allow: /protocols/
Allow: /how-to/
Allow: /evidence/
Allow: /clinical/

# Allow access to sitemaps and indexing manifests
Allow: /api/sitemap.xml
Allow: /api/sitemap*.xml
Allow: /sitemap*.xml
Allow: /news-sitemap.xml
Allow: /google-news.xml
Allow: /feed.xml
Allow: /atom.xml
Allow: /llms*.txt
Allow: /robots.txt

# Allow homepage and static assets
Allow: /$
Allow: /assets/
Allow: /images/
Allow: /favicon.ico

# Crawl Budget Efficiency: Prevent crawlers from wasting crawl budget on duplicate parameter variations
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*tab=
Disallow: /*?*debug=
Disallow: /*?*preview=
Disallow: /*?*session=
Disallow: /*?*token=
Disallow: /*?*auth=
Disallow: /*?*query=
Disallow: /*?*utm_*=
Disallow: /*?*fbclid=
Disallow: /*?page=*&
Disallow: /*&page=*

# Restrict sensitive admin, generator, private, and internal tool/API routes
Disallow: /admin
Disallow: /admin/
Disallow: /private
Disallow: /private/
Disallow: /generator
Disallow: /generator/
Disallow: /adsense-readiness
Disallow: /content-quality
Disallow: /growth
Disallow: /api/

# ==============================================================================
# GOOGLE CRAWLERS (SEARCH, NEWS, SCHOLAR, ADSENSE)
# Maximum Crawl Velocity for Clinical Articles
# ==============================================================================
User-agent: Googlebot
Allow: /articles
Allow: /articles/
Allow: /articles/*
Allow: /article
Allow: /article/
Allow: /article/*
Allow: /topics/
Allow: /qa/
Allow: /hormones/
Allow: /illness-matrix
Allow: /api/sitemap.xml
Allow: /sitemap*.xml
Allow: /
Disallow: /admin
Disallow: /private
Disallow: /generator
Disallow: /api/
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*tab=
Disallow: /*?*debug=

User-agent: Googlebot-News
Allow: /articles/
Allow: /articles/*
Allow: /news-sitemap.xml
Allow: /google-news.xml
Allow: /

User-agent: Googlebot-Image
Allow: /articles/
Allow: /assets/
Allow: /images/
Allow: /

User-agent: Mediapartners-Google
Allow: /articles/
Allow: /

User-agent: AdsBot-Google
Allow: /articles/
Allow: /

User-agent: Google-Extended
Allow: /articles/
Allow: /articles/*
Allow: /

# ==============================================================================
# MICROSOFT BING & COPILOT SEARCH CRAWLERS
# ==============================================================================
User-agent: Bingbot
Allow: /articles
Allow: /articles/
Allow: /articles/*
Allow: /topics/
Allow: /qa/
Allow: /api/sitemap.xml
Allow: /sitemap*.xml
Allow: /
Disallow: /admin
Disallow: /private
Disallow: /generator
Disallow: /api/
Disallow: /*?*sort=
Disallow: /*?*filter=

User-agent: msnbot
Allow: /articles/
Allow: /

User-agent: Copilot
Allow: /articles/
Allow: /articles/*
Allow: /

# ==============================================================================
# APPLE INTELLIGENCE & SIRI SEARCH CRAWLERS
# ==============================================================================
User-agent: Applebot
Allow: /articles
Allow: /articles/
Allow: /articles/*
Allow: /topics/
Allow: /qa/
Allow: /api/sitemap.xml
Allow: /sitemap*.xml
Allow: /
Disallow: /admin
Disallow: /private
Disallow: /generator
Disallow: /api/

User-agent: Applebot-Extended
Allow: /articles/
Allow: /articles/*
Allow: /

# ==============================================================================
# AUTHORIZED AI SEARCH & RETRIEVAL AGENTS
# ==============================================================================
User-agent: GPTBot
Allow: /articles/
Allow: /articles/*
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/sitemap.xml
Allow: /sitemap*.xml
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /generator

User-agent: OAI-SearchBot
Allow: /articles/
Allow: /articles/*
Allow: /

User-agent: ChatGPT-User
Allow: /articles/
Allow: /articles/*
Allow: /

User-agent: ClaudeBot
Allow: /articles/
Allow: /articles/*
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/sitemap.xml
Allow: /sitemap*.xml
Allow: /
Disallow: /api/
Disallow: /admin

User-agent: Claude-Web
Allow: /articles/
Allow: /

User-agent: anthropic-ai
Allow: /articles/
Allow: /

User-agent: PerplexityBot
Allow: /articles/
Allow: /articles/*
Allow: /llms.txt
Allow: /api/sitemap.xml
Allow: /sitemap*.xml
Allow: /
Disallow: /api/
Disallow: /admin

User-agent: Perplexity-Search
Allow: /articles/
Allow: /

User-agent: GrokBot
Allow: /articles/
Allow: /articles/*
Allow: /

User-agent: xAI-Bot
Allow: /articles/
Allow: /

User-agent: meta-externalagent
Allow: /articles/
Allow: /

User-agent: FacebookBot
Allow: /articles/
Allow: /

User-agent: facebookexternalhit
Allow: /articles/
Allow: /

User-agent: DuckDuckBot
Allow: /articles/
Allow: /articles/*
Allow: /

User-agent: Baiduspider
Allow: /articles/
Allow: /articles/*
Allow: /

User-agent: YandexBot
Allow: /articles/
Allow: /articles/*
Allow: /

User-agent: cohere-ai
Allow: /articles/
Allow: /

User-agent: CCBot
Allow: /articles/
Allow: /

# ==============================================================================
# COMMERCIAL SEO CRAWLERS (CRAWL-BUDGET OPTIMIZATION VIA RATE THROTTLING)
# Prevents bandwidth starvation of high-traffic article catalog
# ==============================================================================
User-agent: AhrefsBot
Crawl-delay: 5
Allow: /articles/
Allow: /articles/*
Disallow: /api/
Disallow: /generator
Disallow: /*?*sort=
Disallow: /*?*filter=

User-agent: SemrushBot
Crawl-delay: 5
Allow: /articles/
Allow: /articles/*
Disallow: /api/
Disallow: /generator
Disallow: /*?*sort=
Disallow: /*?*filter=

User-agent: MJ12bot
Crawl-delay: 5
Allow: /articles/
Allow: /articles/*
Disallow: /api/

User-agent: DotBot
Crawl-delay: 5
Allow: /articles/
Allow: /articles/*
Disallow: /api/

User-agent: BLEXBot
Crawl-delay: 5
Allow: /articles/
Allow: /articles/*
Disallow: /api/

# ==============================================================================
# CYBERATTACK DEFENSE: BLOCKED VULNERABILITY SCANNERS & HOSTILE PROBES
# ==============================================================================
User-agent: sqlmap
Disallow: /

User-agent: nikto
Disallow: /

User-agent: dirbuster
Disallow: /

User-agent: gobuster
Disallow: /

User-agent: wpscan
Disallow: /

User-agent: masscan
Disallow: /

User-agent: zgrab
Disallow: /

User-agent: havij
Disallow: /

User-agent: acunetix
Disallow: /

User-agent: netsparker
Disallow: /

User-agent: nessus
Disallow: /

User-agent: qualys
Disallow: /

User-agent: openvas
Disallow: /

User-agent: arachni
Disallow: /

User-agent: nuclei
Disallow: /