# ==============================================================================
# ROBOTS.TXT: stacktxt AGENTIC INFRASTRUCTURE
# Canonical URL: https://stacktxt.com/robots.txt
# Standards: https://stacktxt.com/llms.txt
# Protocols: https://stacktxt.com/.well-known/webmcp.json
# Discovery (ARD): https://stacktxt.com/.well-known/ard.json
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. TRADITIONAL SEARCH ENGINES (FULL ACCESS WITH INFRASTRUCTURE PROTECTION)
# ------------------------------------------------------------------------------
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /cart
Disallow: /checkout
Disallow: /orders
Disallow: /cdn-cgi/
Disallow: /api/

# ------------------------------------------------------------------------------
# 2. GENERATIVE AI SEARCH & ANSWER ENGINES (EXPLICITLY PERMITTED)
# Real-time conversational search bots driving citation & discovery
# ------------------------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: ClaudeBot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Google-Extended
User-agent: Amazonbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/ard.json
Allow: /.well-known/webmcp.json
Allow: /.well-known/
Allow: /sitemap.xml
Disallow: /admin
Disallow: /account
Disallow: /cart
Disallow: /checkout
Disallow: /orders
Disallow: /cdn-cgi/
Disallow: /api/

# ------------------------------------------------------------------------------
# 3. GLOBAL RESTRICTIONS & INFRASTRUCTURE PROTECTION
# Protects private/internal routes while ensuring machine endpoints remain open
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Disallow: /admin
Disallow: /account
Disallow: /cart
Disallow: /checkout
Disallow: /orders
Disallow: /cdn-cgi/
Disallow: /api/

# ------------------------------------------------------------------------------
# 4. SITEMAPS & MACHINE-READABLE DOCUMENTATION
# ------------------------------------------------------------------------------
Sitemap: https://stacktxt.com/sitemap.xml
Agentmap: https://stacktxt.com/.well-known/ard.json