# Prevent all bots from accessing certain sensitive areas of the Magento store
User-agent: *
Disallow: /admin/
Disallow: /checkout/
Disallow: /customer/
Disallow: /cart/
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /catalogsearch/advanced/
Disallow: /sendfriend/
Disallow: /wishlist/
Disallow: /review/
Disallow: /search/

# Block crawling of backend and technical paths
Disallow: /index.php/
Disallow: /cgi-bin/
Disallow: /errors/
Disallow: /static/
Disallow: /var/
Disallow: /media/captcha/
Disallow: /media/customer/
Disallow: /media/downloadable/
Disallow: /media/import/
Disallow: /media/theme_customization/

# Block internal Magento 2 scripts
Disallow: /setup/
Disallow: /pub/
Disallow: /rest/
Disallow: /app/
Disallow: /lib/

# Allow CSS, JS, images for rendering by search engines
Allow: /media/
Allow: /static/

# Block specific query parameters
Disallow: /*?dir=
Disallow: /*?limit=
Disallow: /*?mode=
Disallow: /*?SID=

# Prevent bots from hitting faceted navigation (filtering options)
Disallow: /*?color=
Disallow: /*?price=
Disallow: /*?size=
Disallow: /*?cat=*
Disallow: /*?filter_type=
Disallow: /*?filter_colour=
Disallow: /*?filter_fabric_type=
Disallow: /*?filter_style=
Disallow: /*?p=

# Set a crawl delay to reduce server load (delay in seconds)
Crawl-delay: 5

Sitemap: https://www.sweatershop.com/sitemaps/com/sitemap.xml