# robots.txt for shopmsi.com

# --- Default: block all bots unless explicitly allowed below ---
User-agent: *
Disallow: /

# Optional: a small easter egg still allowed for everyone
Allow: /humans.txt

# --- Sitemaps ---
Sitemap: https://shopmsi.com/sitemap.xml

# --- Microsoft/Bing ---
User-agent: bingbot
Crawl-delay: 5
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

# Legacy identifier some Bing infra still uses
User-agent: msnbot
Crawl-delay: 5
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

# Edge / Bing link preview fetcher
User-agent: BingPreview
Crawl-delay: 5
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

# --- Google ---
User-agent: Googlebot
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/
Disallow: /images/pages/our-team/
# (Google ignores Crawl-delay; throttle via Search Console if needed)

User-agent: Googlebot-Image
Disallow: /images/pages/our-team/

# Newer Google fetchers used for tooling & non-search crawling
User-agent: Google-InspectionTool
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

User-agent: GoogleOther
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

# --- DuckDuckGo / Yahoo / etc. ---
User-agent: DuckDuckBot
Crawl-delay: 5
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

User-agent: Slurp
Crawl-delay: 5
Disallow: /api/
Disallow: /account/
Disallow: /cart/
Disallow: /search
Disallow: /bower_components/
Disallow: /src/

# --- AI & scraping bots (block or restrict as you prefer) ---
User-agent: GPTBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: CCBot
Disallow: /

# --- Others you already listed (examples consolidated) ---
User-agent: dotbot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /