# Cricbun robots.txt
# https://cricbun.com/robots.txt

# Default rules for all bots
User-agent: *

# Allow crawling of main content
Allow: /
Allow: /watch.php
Allow: /sitemap.php

# Crawl delay for politeness (1 second between requests)
Crawl-delay: 1

# Block admin and system directories
Disallow: /admin/
Disallow: /app/
Disallow: /includes/
Disallow: /cron/
Disallow: /api/
Disallow: /vendor/

# Block sensitive file types
Disallow: /*.json$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.env$
Disallow: /*.lock$

# Block duplicate content (pagination/sorting)
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?order=
Disallow: /*&sort=
Disallow: /*&filter=

# Block system pages
Disallow: /login.php
Disallow: /register.php
Disallow: /logout.php
Disallow: /profile.php

# Googlebot-specific rules
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Sitemaps
Sitemap: https://cricbun.com/sitemap.php