# llms.txt — Guidelines for AI crawlers and model trainers
# Target site: https://www.stoneheadbikes.com
# Last updated: 2025-09-09 (Asia/Kolkata)
# NOTE: This is a voluntary, non-standard file (like robots.txt for LLMs). Not all bots respect it.

site: "https://www.stoneheadbikes.com"
owner: "Stonehead Bikes"
contact: "[UPDATE WITH CONTACT EMAIL OR CONTACT PAGE URL]"

policy:
  training: disallow            # Do not use any content to train or fine-tune models
  dataset_inclusion: disallow   # Do not package content into datasets
  evaluation: disallow          # Do not use for model evals/benchmarks
  summarization: allow          # Summaries/excerpts are allowed for user-facing answers
  snippet_length_max: 160       # Max characters for any quoted snippet
  attribution_required: true    # Always cite page title + canonical URL
  cache_ttl_hours: 24           # Cache content for at most 24h for serving answers
  store_user_content: disallow  # Do not store user-submitted data (forms, bookings)

access:
  crawl_delay_seconds: 10       # Be polite; avoid rapid-fire crawling
  allowed_paths:
    - "/"                      # Public marketing pages
    - "/blog/"                # Public blog posts
    - "/tours/"               # Example section; adjust if different
  disallowed_paths:
    - "/wp-admin/"
    - "/wp-login.php"
    - "/cart"
    - "/checkout"
    - "/my-account"
    - "/account"
    - "/privacy-policy"        # May be read by humans; not for training
    - "/terms*"                # May be read by humans; not for training
    - "/*?*password*"
    - "/*?*token*"

agents:                         # Agent-specific preferences (names are User-Agent tokens)
  default:
    training: disallow
    summarization: allow
    cache_ttl_hours: 24
  GPTBot:                       # OpenAI web crawler for training
    training: disallow
  ChatGPT-User:                 # Retrieval for ChatGPT browsing
    summarization: allow
    training: disallow
  Google-Extended:              # Controls Google data use for GenAI (Gemini)
    training: disallow
  CCBot:                        # Common Crawl
    training: disallow
  ClaudeBot:                    # Anthropic
    training: disallow
  Claude-Web:                   # Anthropic web fetcher
    training: disallow
  PerplexityBot:                # Perplexity AI crawler
    training: disallow
  Applebot-Extended:            # Apple opt-out for AI training
    training: disallow
  Bytespider:                   # ByteDance/TikTok bot family
    training: disallow
  Omgili:                       # Omgili/Leech-like scrapers
    training: disallow
  DataForSeoBot:
    training: disallow

requirements:
  honor_robots_txt: true        # Also obey robots.txt directives on this host
  obey_meta_robots: true        # Respect noindex/nofollow on individual pages
  respect_rate_limits: true

sitemap:
  - "https://www.stoneheadbikes.com/sitemap.xml"   # Update if different
  # If using Yoast/RankMath on WordPress, this is often at /sitemap_index.xml

legal:
  jurisdiction: "IN"
  license: "All Rights Reserved"
  notice: |
    Unauthorized use of content from stoneheadbikes.com for the training, fine-tuning,
    or evaluation of AI systems is prohibited. A written license from the site owner is required
    for any such uses. By accessing this site with an automated agent, you agree to these terms.

audit:                          # Optional: how we’ll check compliance
  preferred_disclosure_endpoint: "mailto:[ADD EMAIL]"
  request_headers_to_send:
    - "User-Agent"
    - "From"                     # contact email for the crawler operator
    - "Purpose"                  # e.g., summarization, indexing, QA

summary: |
  You may index pages and show short snippets with attribution for user-facing answers.
  You may NOT use our content to train, fine-tune, or evaluate AI models, nor include it in datasets.
  Avoid sensitive areas (checkout, account, admin). Cache copies for no longer than 24 hours.

# Tips:
# 1) Place this file at https://www.stoneheadbikes.com/llms.txt
# 2) Mirror key disallow rules in robots.txt for wider enforcement (e.g., User-agent: GPTBot / Disallow: /).
# 3) Keep this updated; some agents only read robots.txt today.