User-agent: *
# Block WooCommerce system directories
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/
 
# Block sensitive WordPress directories
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
 
# Block cart, checkout, and account pages (prevents indexing of user-specific pages)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /thank-you/
 
 
# Block search result pages (internal search pages should not be indexed)
Disallow: /?s=
Disallow: /search/
  
# Block faceted navigation, sorting, and filters (prevents index bloat)
Disallow: /*?orderby=
Disallow: /*?filter_=
Disallow: /*?rating=
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?product_count=
Disallow: /*?attribute_pa_*
Disallow: /*?attribute_size=*
Disallow: /*?attribute_hook-type=*
Disallow: /*?attribute_color=*
Disallow: /*?attribute_length=*
 
# Block UTM tracking & referral parameters
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /*?source=*
 
# Block query parameters that cause duplicate content
Disallow: /*?add-to-cart=
Disallow: /*?variant_id=
 
# Block the following bots. Blocking them can help conserve your server resources and prioritize crawling by essential bots like Googlebot
 
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: linkdexbot
Disallow: /
User-agent: SiteLockSpider
Disallow: /
User-agent: python-requests
Disallow: /
User-Agent: ImagesiftBot 
Disallow: /
User-agent: PycURL
Disallow: /

# Allow Google to crawl main product, category, and brand pages
Allow: /product-category/
Allow: /product/
Allow: /brand/

User-agent: Screaming Frog SEO Spider
Allow: /
 
# Ensure search engines prioritize structured crawling
Sitemap: https://www.pe-energy.com/sitemap_index.xml