# Robots.txt for Secrets.RealEstate
# Last updated: 2026-06-12

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_astro/
# Cloudflare internals (email-protection rewrite of mailto links) — not real
# pages; excluded so crawlers and audit tools stop reporting them as broken.
Disallow: /cdn-cgi/
Disallow: /*.json$
Disallow: /*?noTrack=*

# Marketing-tagged URLs duplicate canonical content
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*

# Auth and post-auth state — not useful to search engines, hurts ranking
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /logout

# Paid-tier / authenticated-only surface
Disallow: /dashboard
Disallow: /insights-history
Disallow: /notify-me
Disallow: /add-property
Disallow: /my-properties
Disallow: /smart-advertising

# Checkout flow
Disallow: /checkout/

# AI TRAINING scrapers — fully blocked. AI *search* / citation bots
# (OAI-SearchBot, PerplexityBot, ClaudeBot, Googlebot) stay on the * group
# above and may crawl public pages.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Sitemaps
Sitemap: https://secrets.realestate/sitemap.xml
Sitemap: https://secrets.realestate/sitemap-communes.xml

# Property pages. ONE line on purpose: the URL serves a sitemap INDEX that
# lists ?page=0..N itself, so this never needs editing as inventory grows.
Sitemap: https://secrets.realestate/sitemap-properties.xml

# NOTE: per-bot groups are FULL-DISALLOW ONLY. A crawler obeys only its most
# specific matching group, so a dedicated "User-agent: Googlebot / Allow: /"
# group would make Googlebot ignore every Disallow in the * group. Allowed
# bots (incl. AI search crawlers) must inherit the * group rules.