# ------------------------------------------------------------
# Google Ads crawlers
# Allow Google Ads landing pages with tracking parameters
# ------------------------------------------------------------


User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Allow: /
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/


# Allow Google Ads tracking parameters
Allow: /*?*utm_
Allow: /*&utm_
Allow: /*?*gclid=
Allow: /*&gclid=
Allow: /*?*gbraid=
Allow: /*&gbraid=
Allow: /*?*wbraid=
Allow: /*&wbraid=
Allow: /*?*gad_source=
Allow: /*&gad_source=
Allow: /*?*variant=
Allow: /*&variant=


# Block private / transactional URLs
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /services/login_with_shop
Disallow: /65305411684/checkouts
Disallow: /65305411684/orders


# ------------------------------------------------------------
# Googlebot
# Search pages are blocked from the start.
# Collection pagination is kept crawlable for Googlebot.
# ------------------------------------------------------------




User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Storebot-Google
User-agent: Google-InspectionTool
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /sitemap_agentic_discovery.xml
Allow: /agents.md
Allow: /products/*?variant=
Allow: /products/*&variant=
Allow: /collections/*/products/*?variant=
Allow: /collections/*/products/*&variant=
# Shopify private / transactional URLs
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /services/login_with_shop
Disallow: /65305411684/checkouts
Disallow: /65305411684/orders
# Search pages
Disallow: /search
Disallow: /pages/search
Disallow: /*?*q=
Disallow: /*&q=
Disallow: /*?*options%5Bprefix%5D=
Disallow: /*&options%5Bprefix%5D=
# Shopify pixel sandbox crawl waste
Disallow: /web-pixels@
Disallow: /wpm@
Disallow: /cdn/wpm/
# Internal recommendations
Disallow: /recommendations/products
Disallow: /*/recommendations/products
# Preview / tracking / Shopify internal URLs
Disallow: /*?*oseid=
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /.well-known/shopify/monorail
Disallow: /apple-app-site-association
# Product recommendation duplicate params
Disallow: /*?*pr_prod_strat=
Disallow: /*&pr_prod_strat=
Disallow: /*?*pr_rec_id=
Disallow: /*&pr_rec_id=
Disallow: /*?*pr_rec_pid=
Disallow: /*&pr_rec_pid=
Disallow: /*?*pr_ref_pid=
Disallow: /*&pr_ref_pid=
Disallow: /*?*pr_seq=
Disallow: /*&pr_seq=
# Duplicate product params
Disallow: /*?*ref=
Disallow: /*&ref=
Disallow: /*?*jdgm_referral_location=
Disallow: /*&jdgm_referral_location=
# Collection sorting / filtering duplicates
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*&*filter*
Disallow: /*/collections/*filter*&*filter*
# Blog duplicate patterns
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /blogs/*/tagged/*.atom
# Remote temporary product URLs
Disallow: /products/*-remote
Disallow: /*/products/*-remote
Disallow: /collections/*/products/*-remote
Disallow: /*/collections/*/products/*-remote
# ------------------------------------------------------------
# All other crawlers
# Block search and pagination-heavy duplicate paths.
# ------------------------------------------------------------
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /sitemap_agentic_discovery.xml
Allow: /agents.md
# Shopify private / transactional URLs
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /services/login_with_shop
Disallow: /65305411684/checkouts
Disallow: /65305411684/orders
# Search pages
Disallow: /search
Disallow: /pages/search
Disallow: /*?*q=
Disallow: /*&q=
Disallow: /*?*options%5Bprefix%5D=
Disallow: /*&options%5Bprefix%5D=
# Shopify pixel sandbox crawl waste
Disallow: /web-pixels@
Disallow: /wpm@
Disallow: /cdn/wpm/
# Internal recommendations
Disallow: /recommendations/products
Disallow: /*/recommendations/products
# Preview / tracking / Shopify internal URLs
Disallow: /*?*oseid=
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /.well-known/shopify/monorail
Disallow: /apple-app-site-association
# Duplicate query params
Disallow: /*?*pr_prod_strat=
Disallow: /*&pr_prod_strat=
Disallow: /*?*pr_rec_id=
Disallow: /*&pr_rec_id=
Disallow: /*?*pr_rec_pid=
Disallow: /*&pr_rec_pid=
Disallow: /*?*pr_ref_pid=
Disallow: /*&pr_ref_pid=
Disallow: /*?*pr_seq=
Disallow: /*&pr_seq=
Disallow: /*?*ref=
Disallow: /*&ref=
Disallow: /*?*jdgm_referral_location=
Disallow: /*&jdgm_referral_location=
# Pagination for non-Google bots
Disallow: /*?page=
Disallow: /*&page=
# Collection sorting / filtering duplicates
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*&*filter*
Disallow: /*/collections/*filter*&*filter*
# Blog duplicate patterns
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /blogs/*/tagged/*.atom
# Old WooCommerce / legacy URL crawl waste
Disallow: /product/
Disallow: /product-category/
Disallow: /*?*attribute_pa_
Disallow: /*&attribute_pa_
Disallow: /*?*orderby=
Disallow: /*&orderby=
Disallow: /*?*add-to-cart=
Disallow: /*&add-to-cart=
# Old uploaded media path seen in GSC
Disallow: /upload/media/video/
# Remote temporary product URLs
Disallow: /products/*-remote
Disallow: /*/products/*-remote
Disallow: /collections/*/products/*-remote
Disallow: /*/collections/*/products/*-remote
# ------------------------------------------------------------
# AI / LLM crawlers
# Allow public content, block ecommerce traps.
# ------------------------------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Gemini-Deep-Research
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: CCBot
User-agent: Amazonbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: DuckAssistBot
User-agent: AI21Bot
User-agent: Ai2Bot
User-agent: Ai2Bot-Dolma
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: MistralAI-User
User-agent: iAskBot
User-agent: Qwen-Agent
User-agent: Timpibot
User-agent: YouBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /sitemap_agentic_discovery.xml
Allow: /agents.md
Allow: /collections/
Allow: /products/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
# Block ecommerce/private traps for AI crawlers
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /services/login_with_shop
Disallow: /65305411684/checkouts
Disallow: /65305411684/orders
# Block low-value crawl traps
Disallow: /search
Disallow: /pages/search
Disallow: /recommendations/products
Disallow: /*/recommendations/products
Disallow: /web-pixels@
Disallow: /wpm@
Disallow: /cdn/wpm/
# Block duplicate query paths
Disallow: /*?*pr_prod_strat=
Disallow: /*&pr_prod_strat=
Disallow: /*?*pr_rec_id=
Disallow: /*&pr_rec_id=
Disallow: /*?*pr_rec_pid=
Disallow: /*&pr_rec_pid=
Disallow: /*?*pr_ref_pid=
Disallow: /*&pr_ref_pid=
Disallow: /*?*pr_seq=
Disallow: /*&pr_seq=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?*ref=
Disallow: /*&ref=
Disallow: /*?*jdgm_referral_location=
Disallow: /*&jdgm_referral_location=
# Block sort/filter duplicates
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*&*filter*
Disallow: /*/collections/*filter*&*filter*
# Block old legacy paths
Disallow: /product/
Disallow: /product-category/
Disallow: /upload/media/video/
# Block remote temporary product URLs
Disallow: /products/*-remote
Disallow: /*/products/*-remote
Disallow: /collections/*/products/*-remote
Disallow: /*/collections/*/products/*-remote
# ------------------------------------------------------------
# Problematic / heavy crawlers
# ------------------------------------------------------------
User-agent: Nutch
Disallow: /
User-agent: AhrefsBot
Crawl-delay: 10
User-agent: AhrefsSiteAudit
Crawl-delay: 10
User-agent: MJ12bot
Crawl-delay: 10
User-agent: Pinterestbot
Crawl-delay: 1
# ------------------------------------------------------------
# Sitemap
# ------------------------------------------------------------
Sitemap: https://www.bonkerscorner.com/sitemap.xml
Sitemap: https://www.bonkerscorner.com/sitemap_agentic_discovery.xml