# ====================================================================
# IMFS MASTER ROBOTS.TXT — PRODUCTION READY (AI & GEO OPTIMIZED)
# DRAFT — consolidated to remove duplicate non-consecutive User-agent
# declarations for GPTBot, ClaudeBot, PerplexityBot, and adds the
# Content Signals Policy. NOT YET APPLIED.
# ====================================================================

# ------------------------------------------------------------------
# CONTENT SIGNALS POLICY (Cloudflare / CC0 standard)
# As a condition of accessing this website, you agree to abide by
# the following content signals:
# (a) If a content-signal = yes, you may collect content for the
#     corresponding use.
# (b) If a content-signal = no, you may not collect content for the
#     corresponding use.
# (c) If the website operator does not include a content signal for
#     a corresponding use, the website operator neither grants nor
#     restricts permission via content signal with respect to that use.
# The content signals and their meanings are:
#   search    — building a search index and providing search results
#               (e.g., returning hyperlinks and short excerpts)
#   ai-input  — providing content as direct input to an AI system at
#               the time of an end-user request (e.g., AI answers, RAG)
#   ai-train  — using content to train or fine-tune AI models
# IMFS preference: allow search indexing and AI input/citation, do
# not permit use of this content for AI model training.
# ------------------------------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /insights/methodology/

# Core WordPress Restrictions
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php

# WordPress REST API Security Hardening
Disallow: /wp-json/wp/v2/users/
Allow: /wp-json/

Disallow: /?s=
Disallow: /search/

Disallow: /comments/feed/

Disallow: /trackback/
Disallow: /*/trackback/

Disallow: /*?replytocom=
Disallow: /*?preview=

Disallow: /wp-includes/
Disallow: /wp-content/litespeed/css/

# AI Discovery Endpoints
Allow: /feed/
Allow: /*/feed/

Allow: /llms.txt
Allow: /llms-full.txt

Allow: /sitemap.rss

Allow: /.well-known/agent-skills/
Allow: /.well-known/agent-skills/index.json

# ------------------------------------------------------------------
# HIGH-PRIORITY AI SEARCH AGENTS — each bot named ONCE, complete ruleset
# Allow: / already grants access to /insights/methodology/ and
# /insights/data/ as a subset, so no separate Allow lines are needed.
# ------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: xAI-SearchBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

# ------------------------------------------------------------------
# STANDARD SEARCH CRAWLERS
# ------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

# ------------------------------------------------------------------
# OBSERVABILITY
# ------------------------------------------------------------------
User-agent: SemrushBot
Allow: /

# ------------------------------------------------------------------
# COMPETITIVE / HIGH-VOLUME SCRAPERS
# ------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: Bytespider
Disallow: /

# ------------------------------------------------------------------
# SITEMAPS
# ------------------------------------------------------------------
Sitemap: https://www.imfs.co.in/sitemap.xml
Sitemap: https://www.imfs.co.in/post-sitemap.xml
Sitemap: https://www.imfs.co.in/sitemap.rss