# robots.txt for CCTV Camera Builder (https://cctvcamerabuilder.com)
# Updated: 2026-07-29

# Sitemap reference
Sitemap: https://cctvcamerabuilder.com/sitemap.xml

# Googlebot
User-agent: Googlebot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Bingbot
User-agent: Bingbot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# ------------------------------------------------------------
# AI crawlers — explicit rules
# Public marketing, guides, and product listings are allowed so
# assistants can accurately answer questions and cite our site.
# Private/auth/admin/cart and query-string URLs stay blocked.
# ------------------------------------------------------------

# OpenAI — search/index crawler
User-agent: GPTBot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# OpenAI — on-demand fetch from ChatGPT
User-agent: ChatGPT-User
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# OpenAI — SearchGPT
User-agent: OAI-SearchBot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Anthropic — Claude search/index crawler
User-agent: ClaudeBot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Anthropic — on-demand fetch from Claude
User-agent: Claude-Web
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Anthropic — legacy training crawler
User-agent: anthropic-ai
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Google — Gemini / Vertex AI training opt-in token
User-agent: Google-Extended
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Perplexity — on-demand user fetch
User-agent: Perplexity-User
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Amazon (Alexa / Rufus / Q)
User-agent: Amazonbot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Mistral
User-agent: MistralAI-User
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# You.com
User-agent: YouBot
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*

# ------------------------------------------------------------
# AI crawlers — blocked
# These operate primarily as bulk training scrapers or have a
# history of ignoring rate limits; we disallow entirely.
# ------------------------------------------------------------

# Common Crawl (feeds many training datasets)
User-agent: CCBot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Omgili / Webz.io
User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

# Timpi
User-agent: TimpiBot
Disallow: /

# ImageSift
User-agent: ImagesiftBot
Disallow: /

# General crawlers
User-agent: *
Allow: /
Disallow: /auth
Disallow: /admin/
Disallow: /cart
Disallow: /product/
Disallow: /lovable/
Disallow: /not-found
Disallow: /*?*