# SafeRate AI Robots.txt
# Last Updated: 2026-02-06
# We welcome all search engines and AI crawlers to index our mortgage lender data.
# All bots not explicitly restricted below are granted full access to our public data.

User-agent: *
Content-Signal: search=yes, ai-train=yes, ai-input=yes
# Global Allowances
Allow: /

# Restricted Paths
Disallow: /api/
Disallow: /admin/
Disallow: /_app/
Disallow: /auth/
Disallow: /internal/
Disallow: /data/
# Auth and one shopper's own account area. /sign-in was the single most-crawled page in the
# 2026-09-02 report — 4,465 requests against 45 for the homepage — because the header links to it
# from every page, so anything executing the client bundle walks into it everywhere. Neither it
# nor /my-home has anything to index.
Disallow: /sign-in
# The gated investor pages. They already 404 without a signed token and carry
# X-Robots-Tag: noindex, nofollow, so this is the third layer rather than the
# only one — but a crawler should not be probing them at all. The PUBLIC
# investor narrative (/investors, /investors.txt) is deliberately NOT listed:
# it is meant to be read, cited and indexed like the rest of the site.
Disallow: /investors/cap-table
Disallow: /investors/financials
Disallow: /investors/pipeline
Disallow: /investors/raise.txt
Disallow: /my-home
Disallow: /*.woff
Disallow: /*.woff2
Disallow: /*.ttf
Disallow: /*.eot

# Specific Search Engine & AI Bot Declarations
# (Included to ensure explicit recognition, even though covered by *)
User-agent: Bingbot
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: DeepSeekBot
Content-Signal: search=yes, ai-train=yes, ai-input=yes
Allow: /

# LLM-optimized content index (llmstxt.org standard)
# https://saferate.com/llms.txt

# MCP server — the preferred programmatic interface for AI assistants.
# Streamable HTTP, no authentication required.
# https://saferate.com/.well-known/mcp.json
# https://mcp.saferate.com/mcp

# Mortgage calculation tools (structured markdown responses)
# https://saferate.com/tools
# https://saferate.com/llms-tools.txt

# ===========================================
# Sitemap Index
# ===========================================
# We point to the index files. Sitemap index files cannot nest, so the local
# hub-and-spoke index is listed as a sibling of the main index.
Sitemap: https://saferate.com/sitemap.xml
Sitemap: https://saferate.com/sitemap-local.xml