User-agent: *
Allow: /social_instagram/
Sitemap: https://www.openartexchange.com/sitemap.xml


##############
#   custom   #
##############

# robots.txt for OpenArtExchange
# Updated: 2025-12-19 19:02u
# Website: https://www.openartexchange.com
####################################
# 1. AI SEARCH/ASSISTANT BOTS - ALLOW
####################################

# OpenAI ChatGPT Search & Assistant
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# Perplexity AI Search
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Amazon (used by Perplexity)
User-agent: Amazonbot
Allow: /

# Traditional search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
Allow: /

####################################
# 2. BLOCK AI TRAINING/DATASET BOTS
####################################

# Google AI training/grounding control (separate from Search)
User-agent: Google-Extended
Disallow: /

# Common Crawl dataset
User-agent: CCBot
Disallow: /

# Meta AI training
User-agent: Meta-ExternalAgent
Disallow: /

# Apple Intelligence training
User-agent: Applebot-Extended
Disallow: /

####################################
# 3. BLOCK SCRAPERS / SEO TOOLS
####################################

User-agent: PetalBot
User-agent: Bytespider
User-agent: Baiduspider
User-agent: YandexBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: SemaltBot
User-agent: DataForSeoBot
Disallow: /

####################################
# 4. DEFAULT RULES
####################################

User-agent: *

# Allow all public content
Allow: /

# Block backend & sensitive paths
Disallow: /web/
Disallow: /website/translations/
Disallow: /shop/checkout
Disallow: /shop/checkout/
Disallow: /mail/
Disallow: /chat/
Disallow: /longpolling/

# Crawl delay for unknown bots
Crawl-delay: 10

####################################
# 5. SITEMAP
####################################

Sitemap: https://www.openartexchange.com/sitemap.xml