User-agent: *
Disallow: /*?
Allow: /

Disallow: https://server.cekat.ai/
Disallow: https://live.cekat.ai/
Disallow: https://chat.cekat.ai/
Disallow: https://wp.cekat.ai/
Disallow: https://app.cekat.ai/
Disallow: https://staging.cekat.ai/
Disallow: https://web.cekat.ai/
Disallow: https://files.cekat.ai/
Disallow: https://partner.cekat.ai/

# ========================
# ALLOW MODERN AI CRAWLERS
# ========================

# OpenAI (ChatGPT Browse)
User-agent: GPTBot
Allow: /

# OpenAI Text Embedding models
User-agent: ChatGPT-User
Allow: /

# Google AI / Gemini
User-agent: Google-Extended
Allow: /

# Bing AI (Copilot)
User-agent: Bingbot
Allow: /

# Facebook/Meta AI Research
User-agent: FacebookBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Commoncrawl (digunakan banyak model LLM)
User-agent: CCBot
Allow: /

Sitemap: https://www.cekat.ai/sitemap.xml