# robots.txt — The Holistic Care (www.theholisticcare.com)
# Managed manually — next-sitemap generateRobotsTxt is set to FALSE to prevent overwriting

User-agent: *
Allow: /
# Content-Signal: not a standard robots.txt directive — removed to fix Bing validation error

# Protected app routes (require login — no value to Google)
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /learn/
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
# Stillness Library unlocked publicly 2026-09-02 — no longer disallowed.

# Auth pages (no indexable content)
Disallow: /login
# /signup (no trailing slash/anchor) also matched /signup-for-the-magic-sketchbook,
# a real indexable lead-magnet page — Allow it explicitly before the broader block
# (SEO audit 2026-09-15, Critical #2).
Allow: /signup-for-the-magic-sketchbook
Disallow: /signup
Disallow: /forgot-password

# Old Odoo internal paths — no content on new site
Disallow: /web/
Disallow: /website/
Disallow: /slides/
Disallow: /forums/
Disallow: /forum/
Disallow: /r/

# Block parameterised shop/blog URLs (faceted filters, sort orders, tags)
# These create duplicate content and waste crawl budget.
# Exception: /shop?category=* is a real, linked internal route (see CLAUDE.md
# "Shop & Products" routes, e.g. /shop?category=ebook) — the blanket /shop?*
# rule below was accidentally de-indexing it (SEO-AUDIT-2026-07-15.md, item 3).
# Google/Bing use longest-match-wins, so this more specific Allow takes
# precedence over the shorter Disallow for category URLs specifically.
Allow: /shop?category=*
Disallow: /shop?*
Disallow: /blog?*

# Block old Odoo blog category-numbered paths
Disallow: /blog/uncategorized-8
Disallow: /blog/yoga-7
Disallow: /blog/non-duality-6
Disallow: /blog/mindfulness-4

# ─────────────────────────────────────────────────────────────
# SEO / audit / scraper bots — BLOCKED (zero search or AI-answer value)
# These crawl every page repeatedly from US data centres and are the
# main driver of the Jun–Jul 2026 Vercel usage spike. Blocking them does
# NOT affect Google, Bing, or AI answer engines (allowed separately below).
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: SEranking
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

# AI search crawlers — explicitly allowed for search inclusion (not training)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# GPTBot (OpenAI training) — disallowed; OAI-SearchBot above handles ChatGPT Search
User-agent: GPTBot
Disallow: /

# LLM content files — referenced below as comments only (LLMs: is not valid robots.txt syntax)
# Index file (topics, courses, pages, preferred citation): https://www.theholisticcare.com/llms.txt
# Complete site content in Markdown format (500+ pages): https://www.theholisticcare.com/llms-full.txt

# Sitemaps
Sitemap: https://www.theholisticcare.com/sitemap.xml