User-agent: *

# System folders
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /phpserver/
Disallow: /pub/

# Sitemaps live under /pub/, which is blocked above. Without this Allow the
# crawlers refuse to fetch them.
Allow: /pub/sitemap/

# PHP files (security)
Disallow: /*.php$

# Private user areas
# Every path rule is doubled with a /*/ variant: store codes sit in front of
# the path and a Disallow only matches from the start of the URL, so the plain
# form leaves /<store>/checkout/ and friends open to crawlers.
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /onestepcheckout/
Disallow: /*/onestepcheckout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /track/order/
Disallow: /*/track/order/
Disallow: /datalayer/
Disallow: /*/datalayer/
Disallow: /countrypopup/
Disallow: /*/countrypopup/

# Search & comparison
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /*/catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /*/catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /*/catalog/product/view/
Disallow: /sendfriend/
Disallow: /*/sendfriend/
Disallow: /tag/
Disallow: /*/tag/
Disallow: /review/
Disallow: /*/review/

# Session IDs
Disallow: /*SID=

# Layered navigation filters.
# These create effectively infinite URLs (letters x material x style x stone)
# and produced ~95% of backend load during the July 2026 incident, with no SEO
# value. The main /charms page stays indexable - only the filtered forms are
# blocked. The /*? and /*?* forms in the previous version were redundant:
# /*?*letter_charm= already covers both first and later query parameters.
Disallow: /*?*letter_charm=
Disallow: /*?*surface_material=
Disallow: /*?*style=
Disallow: /*?*stone_type=

# AJAX endpoints
Disallow: /*?*_=
Disallow: /*?*jax=
Disallow: /*?*cf_redirected=
Disallow: /*?*sections=
Disallow: /*?*force_new_section_timestamp=

# Sorting & pagination (duplicate content)
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=
Disallow: /*?*p=

# Tracking parameters (creates duplicate URLs in index)
Disallow: /*?*srsltid=
Disallow: /*?*gclid=
Disallow: /*?*gbraid=
Disallow: /*?*fbclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=

# Project files (security)
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /CONTRIBUTING.md
Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html
Disallow: /COPYING.txt
Disallow: /Gruntfile.js
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /nginx.conf.sample
Disallow: /package.json
Disallow: /php.ini.sample
Disallow: /RELEASE_NOTES.txt

# Version control / archives
Disallow: /*.git
Disallow: /*.CVS
Disallow: /*.Zip$
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$


# ============================================================
# SPECIFIC BOTS - known abusers
#
# IMPORTANT: a crawler obeys exactly ONE group - its own. Rules from the
# "User-agent: *" group above are NOT inherited. A group containing only
# Crawl-delay therefore grants that bot unrestricted access to everything,
# including the filter URLs that caused the July 2026 outage.
# ============================================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 30
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /*?*letter_charm=
Disallow: /*?*surface_material=
Disallow: /*?*style=
Disallow: /*?*stone_type=
Disallow: /*?*_=
Disallow: /*?*jax=
Disallow: /*?*sections=
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=
Disallow: /*?*p=

User-agent: SemrushBot
Crawl-delay: 30
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /*?*letter_charm=
Disallow: /*?*surface_material=
Disallow: /*?*style=
Disallow: /*?*stone_type=
Disallow: /*?*_=
Disallow: /*?*jax=
Disallow: /*?*sections=
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=
Disallow: /*?*p=

User-agent: PetalBot
Crawl-delay: 30
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /*?*letter_charm=
Disallow: /*?*surface_material=
Disallow: /*?*style=
Disallow: /*?*stone_type=
Disallow: /*?*_=
Disallow: /*?*jax=
Disallow: /*?*sections=
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=
Disallow: /*?*p=

Disallow: /media/sphere/
Sitemap: https://marcmirren.com/pub/sitemap/sitemap_eu.xml
Sitemap: https://marcmirren.com/pub/sitemap/sitemap_gb.xml