User-agent: *
Crawl-delay: 20  # please slow down crawling
Disallow: /admin/
Disallow: /login/
Disallow: /cart/
Disallow: /mycart/
Disallow: /checkout/
Disallow: /search/
Disallow: /private/
Disallow: /tmp/
Disallow: /api/

# Allow Google but prevent indexing of specific areas
User-agent: Googlebot
Disallow: /search/
Disallow: /private/
Allow: /

# Block aggressive crawlers (adjust as needed)
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
Disallow: /

# Sitemap: https://yourwebsite.com/sitemap.xml #for indexing
#INSTALL_SITEMAP_PATH