# ============================
# Global rules for all bots
# ============================
User-agent: *

# Allow critical assets for rendering
Allow: /wp-admin/admin-ajax.php
Allow: /*/*.css
Allow: /*/*.js
Allow: /wp-includes/js/

# Block non-SEO WordPress core areas
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php

# Block internal search & low-value parameters
Disallow: /search/
Disallow: /?s=
Disallow: *?s=*
Disallow: *?attachment_id=
Disallow: *&preview=

# Block WordPress JSON API if not needed for bots
Disallow: /wp-json/
Disallow: /?rest_route=

# Block feeds & comment noise (optional but good for crawl budget)
Disallow: /feed/
Disallow: /comments/feed
Disallow: */feed
Disallow: /?feed=
Disallow: /wp-feed

# Block spammy comment/trackback URLs
Disallow: /trackback/
Disallow: */trackback
Disallow: */comments$
Disallow: /wp-comments
Disallow: /wp-trackback
Disallow: */replytocom=

# ============================
# Sitemaps (important for SEO)
# ============================
Sitemap: https://techfusiongear.com/post-sitemap.xml
Sitemap: https://techfusiongear.com/page-sitemap.xml

# ============================
# Block SEO/scanner/link bots
# (bandwidth savers, not “good” for you)
# ============================
User-agent: AhrefsBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Xenu
Disallow: /

# ============================
# Block unwanted scrapers
# (condensed list – keep only what you really see in logs)
# ============================
User-agent: DotBot
Disallow: /

User-agent: Dataprovider
Disallow: /

User-agent: DataproviderSiteExplorer
Disallow: /

User-agent: DomainStatsBot
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: 008
Disallow: /

User-agent: feedly
Disallow: /

User-agent: FeedlyBot
Disallow: /

User-agent: Feedspot
Disallow: /

User-agent: Feedspotbot
Disallow: /

User-agent: Discobot
Disallow: /

# ============================
# Archive.org / Wayback Machine
# ============================
User-agent: ia_archiver
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: ia_archiver-web.archive.org
Disallow: /

# ============================
# Block popular AI crawlers (optional – tweak to your strategy)
# ============================
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

Sitemap: https://techfusiongear.com/sitemap_index.xml