# Farray Motor robots.txt # Public site: https://farray.es/ # Note: robots.txt is advisory. Real abuse protection belongs in Wordfence/WAF/rate limits. # Search and AI discovery crawlers we want to allow fully. # Use an empty Disallow instead of Allow:/ to avoid ambiguity in old parsers. User-agent: Googlebot Disallow: User-agent: Googlebot-Image Disallow: User-agent: AdsBot-Google Disallow: User-agent: AdsBot-Google-Mobile Disallow: User-agent: Googlebot-Mobile Disallow: User-agent: Bingbot Disallow: User-agent: Applebot Disallow: User-agent: OAI-SearchBot Disallow: User-agent: ChatGPT-User Disallow: User-agent: PerplexityBot Disallow: User-agent: ClaudeBot Disallow: User-agent: Claude-User Disallow: User-agent: anthropic-ai Disallow: User-agent: Google-Extended Disallow: # Noisy SEO/scraper crawlers. Good actors will honor this; bad actors need WAF rules. User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: PetalBot Disallow: / User-agent: Bytespider Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Orthogaffe Disallow: / User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / User-agent: Twiceler Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: libwww-perl Disallow: / User-agent: wget Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: Nutch Disallow: / User-agent: spock Disallow: / User-agent: OmniExplorer_Bot Disallow: / User-agent: TurnitinBot Disallow: / User-agent: BecomeBot Disallow: / User-agent: genieBot Disallow: / User-agent: MLBot Disallow: / User-agent: 80bot Disallow: / User-agent: Linguee Bot Disallow: / User-agent: aiHitBot Disallow: / User-agent: Exabot Disallow: / User-agent: SBIder/Nutch Disallow: / User-agent: Jyxobot Disallow: / User-agent: mAgent Disallow: / User-agent: Speedy Spider Disallow: / User-agent: ShopWiki Disallow: / User-agent: Huasai Disallow: / User-agent: DataCha0s Disallow: / User-agent: Baiduspider Disallow: / User-agent: Atomic_Email_Hunter Disallow: / User-agent: Mp3Bot Disallow: / User-agent: WinHttp Disallow: / User-agent: betaBot Disallow: / User-agent: core-project Disallow: / User-agent: panscient.com Disallow: / User-agent: Java Disallow: / User-agent: WebReaper Disallow: / User-agent: grub-client Disallow: / User-agent: k2spider Disallow: / User-agent: NPBot Disallow: / # Default: keep the public site crawlable, block sensitive/admin surfaces. User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /*?s= Disallow: /*&s= Sitemap: https://farray.es/sitemap.xml