# Phase 474 (2026-08-27) — SEO-tool crawlers blocked. Measured over the 14-day
# nginx retention they were ~48k requests, about half of all bot traffic, for zero
# referred visitors. Each gets its OWN group: robots.txt evaluates only the single
# most-specific matching group, so a per-bot group here means these three read ONLY
# their own Disallow and never touch the "*" rules below (the same mechanism the
# Phase 151 note above warns about, used deliberately). nginx returns 403 to the
# same three so compliance is not required.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Amazonbot
Disallow: /

# Phase 482 (2026-08-27) — AionBot. UNLIKE the three above this one is honest:
# it self-identifies in its UA, it fetched robots.txt, and under the "*" group
# below it was fully COMPLIANT. The problem was rate, not permission. From one
# Oracle Cloud host (161.153.113.113, AS31898) it made 2,420 requests over
# 2,308 distinct URLs in ~24h, peaking at 107 req/min, and 210 of those
# returned 500 -- it was a measurable share of the render-layer collapse that
# Phase 476 was written to stop.
#
# Because it reads robots.txt, this group alone should be enough and is the
# proportionate first lever. nginx also 403s it (Phase 482) so compliance is
# not required, exactly as with the three above. If it later honours a slower
# crawl instead, a Crawl-delay group here would be the friendlier setting.
User-agent: AionBot
Disallow: /

User-agent: *
Allow: /
Allow: /research/
Allow: /live/
Allow: /insights/
Allow: /signals/archive/
# Phase 616.1: the bare index. "Allow: /signals/archive/" (trailing slash) does
# not match "/signals/archive", so "Disallow: /signals" below blocked the index
# page both sitemaps list.
Allow: /signals/archive
Allow: /about
Allow: /newsroom
Allow: /daily-brief
Allow: /notes
Allow: /learn
Allow: /legal/
Allow: /tools$
Allow: /tools/position-calculator
Allow: /tools/congressional-anomalies
Allow: /tools/insider-clusters
Allow: /tools/relative-value
Allow: /tools/correlation-regime
Allow: /tools/options-implied-probability
Allow: /tools/yield-curve
Allow: /tools/sector-rotation
Allow: /tools/market-rotation
Allow: /tools/cot-positioning
Allow: /tools/greeks-exposure
Allow: /tools/primary-dealer-positions
Allow: /tools/short-volume
Allow: /tools/futures-seasonality
Allow: /tools/futures-volatility
Allow: /tools/futures-curve
Allow: /tools/macro-event-impact
Allow: /tools/regime-analogs
Allow: /tools/treasury-auctions
Allow: /tools/buybacks
Allow: /tools/merger-arb
Allow: /intel/exchange-flow
Allow: /intel/offerings
Allow: /macro/central-banks
Allow: /macro/money-markets
Allow: /macro/regime
Allow: /macro/sovereign-spreads
Allow: /macro/fed
Allow: /crypto/tokens
Allow: /feed
Allow: /odds
Allow: /tools/prediction-market-screener
Allow: /developers
Allow: /guide
Allow: /s/

# Authenticated dashboard surface — must not appear in search
Disallow: /home
Disallow: /terminal
Disallow: /portfolio
Disallow: /portfolio/
Disallow: /vault
Disallow: /messages
Disallow: /split
Disallow: /forum
Disallow: /board
Disallow: /boards
Disallow: /community
Disallow: /community/
Disallow: /settings
Disallow: /settings/
Disallow: /account
Disallow: /account/
Disallow: /alerts
Disallow: /alerts/
Disallow: /screener
Disallow: /tools
Disallow: /tools/
Disallow: /chart
Disallow: /chart/

# Domain pages behind auth (live under (dashboard))
Disallow: /capital-flows
Disallow: /capital-flows/
Disallow: /commodities
Disallow: /commodities/
Disallow: /companies
Disallow: /companies/
Disallow: /crypto
Disallow: /crypto/
Disallow: /equities
Disallow: /equities/
Disallow: /filings
Disallow: /filings/
Disallow: /forex
Disallow: /forex/
Disallow: /futures
Disallow: /futures/
Disallow: /intel
Disallow: /intel/
Disallow: /macro
Disallow: /macro/
Disallow: /news
Disallow: /news/
Disallow: /osint
Disallow: /overview
Disallow: /signals

# Phase 117.1 candidates — top-level /bonds, /whales, /options
# are referenced by Phase 116 broadcasts but don't yet have routes
# (currently 404). Defensive disallow so a future accidental ship
# under (dashboard) doesn't leak before we flip these to Allow as
# Phase 117.1 public surfaces.
Disallow: /bonds
Disallow: /bonds/
Disallow: /whales
Disallow: /whales/
Disallow: /options
Disallow: /options/

# Auth + onboarding + admin + share-tokens + API
# (/og/ is intentionally NOT disallowed — Twitterbot / LinkedInBot /
# Slackbot respect robots.txt before fetching og:image and a
# disallow there would break social preview cards.)
Disallow: /auth/
Disallow: /onboarding/
Disallow: /admin/
Disallow: /api/
Disallow: /share/

# Phase 525 embed cards: not a citation target. Since Phase 616 a logged-out
# request gets an inline sign-in card (noindex), not a login redirect.
Disallow: /embed/

Sitemap: https://kresmion.com/sitemap.xml