User-agent: * Disallow: /*?* Disallow: /trial-signup-form Disallow: /iswap/ Disallow: /services/ Disallow: /id/ Disallow: /xmlrpc.php Disallow: /wp-stats.php Disallow: /wp-login.php Disallow: /wp- Disallow: /feed Disallow: /feed/ Disallow: /*/feed Disallow: /*/*/feed/ Disallow: /*/*/*/feed/ Disallow: /*/trackback/$ Disallow: /*/*/trackback/$ Disallow: /*/*/*/trackback/$ Disallow: /*/?page_id=$ Disallow: /newsletter-archive Disallow: /subscribe-newsletter Disallow: /misc Disallow: /latest-user-responses Disallow: /fr Disallow: /es Disallow: /some-stamps Disallow: /my-acccount Disallow: /gems/ Disallow: /?logout* Disallow: /video/ Disallow: /wp-content/uploads/2021/01/ Disallow: /search/* # Block URLs with 'p=' parameter (e.g., /?p=1234) Disallow: /*?p= # Block URLs with 's=' parameter (e.g., /?s=search...) Disallow: /*?s= # Block marketing campaign parameters Disallow: /*?*utm_ Disallow: /*&utm_ ################################ #MAJOR SEARCH ENGINES ############################## User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Bingbot Allow: / User-agent: msnbot Allow: / ##################################### # Custom rules for specific agents. ##################################### User-agent: CCBot Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-SearchBot Allow: / User-agent:Claude-User Allow: / User-agent: Google-Extended Allow: / User-agent: Google-NotebookLM Disallow: / User-agent: meta-webindexer Allow: / User-agent: meta-externalagent Allow: / User-agent: meta-externalfetcher Allow: / User-agent: GrokBot Disallow: / ################################### #social media bots ################################### User-agent: facebookexternalhit Allow: / user-agent: Pinterestbot Allow: / user-agent: ByteSpider Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Slackbot Allow: / User-agent: Discordbot Allow: / ########################################### # other bots, most are blocked by default ########################################### User-agent: 008 Disallow: / User-agent: Aboundexbot Disallow: / User-agent: accelobot Disallow: / User-agent: admantx Disallow: / user-agent: AhrefsBot disallow: / User-agent: aipbot Disallow: / User-agent: AlibabaCloud Disallow: / User-agent: AliyunBot Disallow: / User-agent: AliyunCrawler Disallow: / User-agent: AliyunSecBot Disallow: / #amazon alexa User-agent: Amazonbot Allow: / User-agent: Amzn-SearchBot Allow: / User-agent: Amzn-User Allow: / User-agent: Applebot Allow: / User-agent: Aranet-SearchBot Disallow: / User-agent: asterias Disallow: / User-agent: Alexibot Disallow: / User-agent: AwarioRssBot Disallow: / User-agent: AwarioSmartBot Disallow: / User-agent: baiduspider Allow: / User-agent: b2w/0.1 Disallow: / User-agent: BackDoorBot/1.0 Disallow: / User-agent: BacklinksExtendedBot Disallow: / User-agent: barkrowler Disallow: / User-agent: BitSightBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: BlowFish/1.0 Disallow: / User-agent: BoardReader Disallow: / User-Agent: boitho.com-robot Disallow: / User-agent: boitho.com-dc Disallow: / User-agent: BotALot Disallow: / User-Agent: BotLink Disallow: / User-Agent: BruinBot Disallow: / User-agent: Buck Disallow: / User-agent: BuckyOHare Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: CandidateCrawler/ Disallow: / User-agent: CazoodleBot Disallow: / User-agent: CheckMarkNetwork/1.0 (+https://www.checkmarknetwork.com/spider.html) Disallow: / User-agent: CherryPicker Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: CherryPickerElite/1.0 Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: CheeseBot Disallow: / User-agent: ChangeDetection Disallow: / User-agent: Cliqzbot Disallow: / User-agent: coccocbot Disallow: / User-agent: coccocbot-image Disallow: / User-agent: coccocbot-web Disallow: / User-agent: Copernic Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: cosmos Disallow: / User-agent: Crescent Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: DepSpid Disallow: / User-agent: DeuSu Disallow: / User-agent: DittoSpyder Disallow: / User-Agent: dloader(NaverRobot) Disallow: / User-agent: DolfeEngineCrawler Disallow: / User-Agent:DomainStatsBot Disallow: / User-agent: DomainCrawler disallow: / User-Agent: DomainStatsBot Disallow: / User-agent: DomCopBot Disallow: / User-agent: dotbot Disallow: / User-agent: DuckDuckBot Allow: / User-Agent: Dumbot Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: EroCrawler Disallow: / User-agent: e-SocietyRobot Disallow: / User-agent: Exabot Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Fasterfox Disallow: / User-agent: Filangy Disallow: / User-agent: findlinks Disallow: / User-agent: findxbot Disallow: / User-agent: findfiles.org Disallow: / User-agent: fluid Disallow: / User-agent: Foobot Disallow: / User-agent: Gaisbot Disallow: / User-agent: GenomeCrawlerd Disallow: / User-agent: Gigabot Disallow: / User-agent: grapeshot Disallow: / User-agent: grub-client Disallow: / User-agent: grub Disallow: / User-Agent: Harvest Disallow: / User-agent: Harvest/1.5 Disallow: / User-agent: hloader Disallow: / User-Agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: humanlinks Disallow: / User-agent: IbouBot Disallow: / User-agent: IlTrovatore-Setaccio Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: IntegromeDB Disallow: / User-agent: IRLbot Disallow: / User-agent: JennyBot Disallow: / User-agent: Kenjin Spider Disallow: / User-agent: larbin Disallow: / User-agent: LexiBot Disallow: / User-agent: Linguee Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: LinkWalker Disallow: / User-agent: LocalcomBot Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: lwp-trivial Disallow: / User-agent: lwp-trivial/1.34 Disallow: / User-agent: magpie-crawler Disallow: / User-agent: MIIxpc Disallow: / User-Agent: minibot(NaverRobot) Disallow: / User-agent: Mister PiX Disallow: / User-agent: MixRankBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: MLBot Disallow: / User-agent: moget Disallow: / User-agent:netEstate Disallow: / User-agent: NetMechanic Disallow: / User-Agent: NetResearchServer Disallow: / User-agent: NICErsPRO Disallow: / User-agent: Nutch Disallow: / User-agent: Mata Hari Disallow: / User-agent: moget/2.1 Disallow: / User-Agent: nabot Disallow: / User-agent: NetAnts Disallow: / User-agent: NimbleCrawler Disallow: / User-Agent: NPBot Disallow:/ User-Agent: Nutch Disallow: / Disallow: /Ocelli Disallow: / User-agent: Offline Explorer Disallow: / User-Agent: omgilibot Disallow: / User-Agent: omgili Disallow: / User-Agent: OmniExplorer_Bot Disallow: / User-agent: Openbot Disallow: / User-agent: Openfind Disallow: / User-agent:pagebullbot Disallow: / User-agent: Pandalytics Disallow: / User-agent:panscient.com Disallow: / User-agent: PerMan Disallow: / #huawei petal User-agent: PetalBot Allow: / User-agent: pipeLiner Disallow: / User-agent: plukkie Disallow: / User-Agent: pompos Disallow: / User-agent: ProPowerBot/2.14 Disallow: / User-agent: ProWebWalker Disallow: / User-agent: proximic Disallow: / User-agent: psbot Disallow: / User-agent: PubMatic Allow: / User-agent: PubMatic Crawler Bot Allow: / User-agent: Python-urllib Disallow: / User-Agent: QuepasaCreep Disallow: / User-agent: R6_FeedFetcher Disallow: / User-agent: R6_CommentReader Disallow: / User-agent: Radian6 Disallow: / User-agent: RepoMonkey Disallow: / User-agent: RMA Disallow: / User-Agent: Robi Disallow: / User-agent: rogerbot Disallow: / User-agent: SBIder Disallow: / User-Agent: SBIntuitionsBot Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: SeekportBot Disallow: / User-agent: SERankingBacklinksBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: SleepBot Disallow: / User-agent: Slurp Allow: / User-agent: Sosospider Disallow: / User-agent:Spinn3r Disallow: / User-agent: spbot Disallow: / User-agent: searchpreview Disallow: / User-agent: seekbot Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: semrush Disallow: / User-agent: semanticdiscovery Disallow: / User-agent: SeznamBot Disallow: / User-agent: Shim-Crawler Disallow: / User-Agent: sitecheck.internetseer.com Disallow: / User-agent: SiteSnagger Disallow: / User-agent:SMTBot Disallow: / User-agent: SpankBot Disallow: / User-agent: spanner Disallow: / User-agent: Speedy Disallow: / User-agent: SpiderLing Disallow: / User-agent: Sproose Disallow: / User-Agent: Steeler Disallow: / User-Agent: suke Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: suzuran Disallow: / User-Agent: szukacz Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: Telesoft Disallow: / User-agent: TheNomad Disallow: / User-Agent: trendictionbot Disallow: / User-agent: True_Robot Disallow: / User-agent: True_Robot/1.0 Disallow: / User-agent: turingos Disallow: / User-Agent: TurnitinBot Disallow: / User-agent: Twingly Recon Disallow: / User-agent: URL_Spider_Pro Disallow: / User-agent: Vagabondo Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: vortex Disallow: / User-agent: WanScannerBot Disallow: / User-agent: WBSearchBot Disallow: User-agent: WebAuto Disallow: / User-agent: WebBandit Disallow: / User-agent: WebBandit/3.50 Disallow: / User-agent: WebCopier Disallow: / User-agent: WebEnhancer Disallow: / User-agent: WebFerret Disallow:/ User-agent: Web Image Collector Disallow: / User-agent: WebSauger Disallow: / User-agent: Website Quester Disallow: / User-agent: Webster Pro Disallow: / User-agent: WebStripper Disallow: / User-agent: WebwikiBot Disallow: / User-agent: WebZip Disallow: / User-agent: WebZip/4.0 Disallow: / User-agent: wotbox Disallow: / User-agent: Wget Disallow: / User-agent: Wget/1.5.3 Disallow: / User-agent: Wget/1.6 Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: yacybot Disallow: / User-Agent: Yandex Allow: / User-agent: YisouSpider Disallow: / User-Agent: YodaoBot Disallow: / User-Agent: Zao Disallow: / User-agent: Zeus Disallow: / User-agent: Zeusbot Disallow: / User-Agent: ZoominfoBot Disallow: / User-agent: Zumbot Disallow: /