# robots.txt for ladkibahin-yojana.org
# Purpose: Allow good crawlers, block bad bots, slow down heavy crawlers

# ==============================
# ​ Allow Important Search Engines & Ad Networks
# ==============================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

# ==============================
# ​ Block Known Scrapers & AI Training Bots
# ==============================
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# ==============================
# ​ Slow Down Aggressive Crawlers (Not Fully Blocked)
# ==============================
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 15

User-agent: DotBot
Crawl-delay: 10

# ==============================
# ​ Sitemap Location
# ==============================
Sitemap: https://ladkibahin-yojana.org/sitemap_index.xml