# https://www.robotstxt.org/robotstxt.html

User-agent: *
Allow: /

# Block sensitive and unnecessary routes
Disallow: /api/
Disallow: /functions/
Disallow: /_next/
Disallow: /_error
Disallow: /admin/
Disallow: /login/
Disallow: /signup/
Disallow: /settings/

# Block search query pages to prevent duplicate content issues
Disallow: /search

# Block non-canonical versions of dynamic routes
Disallow: /categories/*?*
Disallow: /products/*?*

# Sitemap location
Sitemap: https://litreviews.co/sitemap.xml

# Security: Block crawlers from accessing private data
Disallow: /private/
Disallow: /tmp/
Disallow: /hidden/

# Security: Prevent indexing of unnecessary assets
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.map$
Disallow: /*.json$

# Prevent specific bots
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Crawl-delay for all bots
Crawl-delay: 2

# Specific directives for known bots
User-agent: Googlebot
Crawl-delay: 1
Allow: /

User-agent: Bingbot
Crawl-delay: 1
Allow: /

User-agent: Slurp
Crawl-delay: 1
Allow: /

# AI and bot-specific rules
User-agent: AIbot
Disallow: /sensitive/

# Advanced SEO techniques
User-agent: Mediapartners-Google
Allow: /

# Ensuring no index for certain pages
Disallow: /noindex/

# Honeypot for security
Disallow: /secure/logins.html

# Allow specific important pages
Allow: /categories/
Allow: /products/
Allow: /latest/
Allow: /about/
Allow: /contact/
Allow: /privacy/

# Allow product images to be crawled
Allow: /products/*.png$
Allow: /products/*.jpg$
Allow: /products/*.jpeg$
Allow: /products/*.webp$