# For all robots
User-agent: *

# Block access to specific groups of pages
Disallow: /en/cart
Disallow: /en/checkout
Disallow: /en/my-account


Crawl-delay: 10                 # 10 seconds between page requests

# Allow search crawlers to discover the sitemap

Sitemap: https://www.spectra-physics.com/medias/sys_master/root/h3d/hc9/10045391896606/Homepage-en-USD-2110483451888372487/Homepage-en-USD-2110483451888372487.xml
Sitemap: https://www.spectra-physics.com/medias/sys_master/root/h4f/hcc/10045391962142/Product-en-USD-9110919333846104216/Product-en-USD-9110919333846104216.xml
Sitemap: https://www.spectra-physics.com/medias/sys_master/root/h07/he9/10045392027678/CategoryLanding-en-USD-16107367929076265157/CategoryLanding-en-USD-16107367929076265157.xml
Sitemap: https://www.spectra-physics.com/medias/sys_master/root/h45/he8/10045392093214/Category-en-USD-7924371575081904875/Category-en-USD-7924371575081904875.xml
Sitemap: https://www.spectra-physics.com/medias/sys_master/root/h36/he2/10045392224286/Content-en-USD-3606378132880723484/Content-en-USD-3606378132880723484.xml

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /