# ============================================================================
# Robots.txt for Utkrusht.ai
# Last Updated: December 2024
# Purpose: Comprehensive crawler management for AI solutions company
# ============================================================================

# ----------------------------------------------------------------------------
# UNIVERSAL RULES - Applied to all crawlers first
# ----------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_framer/
Disallow: /private/
Disallow: /*.json$
Disallow: /*?*api_key=
Disallow: /*?*token=
Disallow: /search?
Allow: /blog/
Allow: /blog/*
Allow: /resources/
Allow: /resources/*
Allow: /case-studies/
Allow: /case-studies/*
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.woff$
Allow: /*.woff2$
Crawl-delay: 1


# ----------------------------------------------------------------------------
# PRIMARY SEARCH ENGINES - Maximum access for indexing
# ----------------------------------------------------------------------------

# Google Search
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Allow: /blog/
Allow: /blog/*
Allow: /resources/
Allow: /case-studies/

# Google Images
User-agent: Googlebot-Image
Allow: /
Allow: /blog/
Allow: /resources/
Allow: /case-studies/

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /
Allow: /blog/
Allow: /resources/

# Bing/Microsoft
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Allow: /blog/
Allow: /blog/*
Allow: /resources/

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Allow: /blog/
Allow: /resources/

# Yandex (Russia)
User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /admin/
Allow: /blog/
Crawl-delay: 2

# Baidu (China)
User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /admin/
Allow: /blog/
Crawl-delay: 2

# Brave Search
User-agent: Brave-Search-Bot
Allow: /
Disallow: /api/
Allow: /blog/


# ----------------------------------------------------------------------------
# LLM & AI TRAINING BOTS - Full access for AI training datasets
# ----------------------------------------------------------------------------

# OpenAI GPT
User-agent: GPTBot
Allow: /
Allow: /blog/
Allow: /blog/*
Allow: /resources/

# OpenAI ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Allow: /blog/
Allow: /resources/

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
Allow: /blog/
Allow: /blog/*
Allow: /resources/

# Anthropic Extended
User-agent: anthropic-ai
Allow: /
Allow: /blog/
Allow: /resources/

# Google Extended (for Bard/Gemini training)
User-agent: Google-Extended
Allow: /
Allow: /blog/
Allow: /resources/

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /
Allow: /blog/
Allow: /resources/

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /blog/

# Meta AI (Facebook/Instagram AI)
User-agent: FacebookBot
Allow: /
Allow: /blog/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /blog/

# You.com
User-agent: YouBot
Allow: /
Allow: /blog/

# Amazon (Alexa AI)
User-agent: Amazonbot
Allow: /
Allow: /blog/

# Apple (Siri/Spotlight)
User-agent: Applebot
Allow: /
Allow: /blog/

# Apple Extended
User-agent: Applebot-Extended
Allow: /
Allow: /blog/


# ----------------------------------------------------------------------------
# SEO & ANALYTICS TOOLS - Allow for SEO monitoring
# ----------------------------------------------------------------------------

# Ahrefs
User-agent: AhrefsBot
Allow: /
Allow: /blog/
Crawl-delay: 2

# SEMrush
User-agent: SemrushBot
Allow: /
Allow: /blog/
Crawl-delay: 2

# Moz
User-agent: rogerbot
Allow: /
Allow: /blog/
Crawl-delay: 2

User-agent: dotbot
Allow: /
Allow: /blog/

# Majestic SEO
User-agent: MJ12bot
Allow: /
Allow: /blog/
Crawl-delay: 2

# Screaming Frog
User-agent: Screaming Frog SEO Spider
Allow: /
Allow: /blog/


# ----------------------------------------------------------------------------
# SOCIAL MEDIA CRAWLERS - For link previews and sharing
# ----------------------------------------------------------------------------

# Facebook (Meta)
User-agent: facebookexternalhit
Allow: /
Allow: /blog/

# Twitter/X
User-agent: Twitterbot
Allow: /
Allow: /blog/

# LinkedIn
User-agent: LinkedInBot
Allow: /
Allow: /blog/

# Pinterest
User-agent: Pinterestbot
Allow: /
Allow: /blog/

# WhatsApp
User-agent: WhatsApp
Allow: /
Allow: /blog/

# Telegram
User-agent: TelegramBot
Allow: /
Allow: /blog/

# Slack
User-agent: Slackbot
Allow: /
Allow: /blog/

# Discord
User-agent: Discordbot
Allow: /
Allow: /blog/


# ----------------------------------------------------------------------------
# ARCHIVE & RESEARCH BOTS
# ----------------------------------------------------------------------------

# Internet Archive
User-agent: ia_archiver
Allow: /
Allow: /blog/

# Archive.org
User-agent: archive.org_bot
Allow: /
Allow: /blog/


# ----------------------------------------------------------------------------
# BAD BOTS & SCRAPERS - Block malicious/aggressive crawlers
# ----------------------------------------------------------------------------

# Known scrapers to block completely
User-agent: WebReaper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: ChinaClaw
Disallow: /

User-agent: Custo
Disallow: /

User-agent: DISCo
Disallow: /

User-agent: FlickBot
Disallow: /

User-agent: Grabber
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Stripper
Disallow: /

User-agent: Sucker
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Wget
Disallow: /


# ----------------------------------------------------------------------------
# SITEMAPS - Critical for discovery
# ----------------------------------------------------------------------------
Sitemap: https://utkrusht.ai/sitemap.xml

# ============================================================================
# END OF ROBOTS.TXT
# For questions contact: naman@utkrusht.ai
# ============================================================================