# robots.txt for UrbanLine Group
# Optimized for SEO + GEO (AI Crawlers) + Clean Crawl Efficiency

# ========================
# GLOBAL RULES
# ========================
User-agent: *
Allow: /

# Block low-value / sensitive areas
Disallow: /admin/
Disallow: /api/
Disallow: /search
Disallow: /*?q=
Disallow: /*?search=
Disallow: /*page=
Disallow: /*.json$

# Allow critical files
Allow: /robots.txt
Allow: /sitemap.xml

# ========================
# SEARCH ENGINE BOTS
# ========================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ========================
# AI / LLM TRAINING BOTS
# (GEO Optimization Layer)
# ========================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /products
Allow: /projects
Allow: /categories
Allow: /about
Allow: /certifications
Allow: /clients
Allow: /sectors
Disallow: /admin/
Disallow: /api/
Disallow: /contact

User-agent: ChatGPT-User
Allow: /products
Allow: /projects
Allow: /categories
Allow: /about
Allow: /certifications
Allow: /clients
Allow: /sectors
Disallow: /admin/
Disallow: /api/
Disallow: /contact

# Anthropic / Claude
User-agent: anthropic-ai
Allow: /products
Allow: /projects
Allow: /about
Allow: /certifications
Disallow: /admin/
Disallow: /api/
Disallow: /contact

User-agent: Claude-Web
Allow: /products
Allow: /projects
Allow: /about
Disallow: /admin/
Disallow: /api/
Disallow: /contact

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/

# ByteDance / TikTok crawler
User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /api/

# ========================
# PERFORMANCE (OPTIONAL)
# ========================
# Crawl-delay is ignored by Google but used by some bots
Crawl-delay: 1

# ========================
# SITEMAP
# ========================
Sitemap: https://www.urbanlinegroup.com/sitemap.xml