# Robots.txt file # For default domain User-agent: Googlebot Disallow: User-agent: msnbot Disallow: User-agent: Slurp Disallow: User-agent: rogerbot Disallow: / User-agent: exabot Disallow: / User-agent: dotbot Disallow: / User-agent: gigabot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: ABACHOBot Disallow: / User-agent: Accoona-AI-Agent Disallow: / User-agent: AddSugarSpiderBot Disallow: / User-agent: AnyApexBot Disallow: / User-agent: Arachmo Disallow: / User-agent: B-l-i-t-z-B-O-T Disallow: / User-agent: Baiduspider Disallow: / User-agent: BecomeBot Disallow: / User-agent: BeslistBot Disallow: / User-agent: BillyBobBot Disallow: / User-agent: Bimbot Disallow: / User-agent: BlitzBOT Disallow: / User-agent: boitho.com-dc Disallow: / User-agent: boitho.com-robot Disallow: / User-agent: btbot Disallow: / User-agent: CatchBot Disallow: / User-agent: Cerberian Drtrs Disallow: / User-agent: Charlotte Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: cosmos Disallow: / User-agent: Covario IDS Disallow: / User-agent: ClineCrawler Disallow: / User-agent: DataparkSearch Disallow: / User-agent: DiamondBot Disallow: / User-agent: Discobot Disallow: / User-agent: EARTHCOM.info Disallow: / User-agent: EmeraldShield.com WebBot Disallow: / User-agent: envolkITSspider Disallow: / User-agent: EsperanzaBot Disallow: / User-agent: Exabot Disallow: / User-agent: FAST Enterprise Crawler Disallow: / User-agent: FAST-WebCrawler Disallow: / User-agent: FDSE robot Disallow: / User-agent: FindLinks Disallow: / User-agent: FurlBot Disallow: / User-agent: FyberSpider Disallow: / User-agent: g2crawler Disallow: / User-agent: Gaisbot Disallow: / User-agent: GalaxyBot Disallow: / User-agent: genieBot Disallow: / User-agent: Gigabot Disallow: / User-agent: Girafabot Disallow: / User-agent: GurujiBot Disallow: / User-agent: HappyFunBot Disallow: / User-agent: hl_ftien_spider Disallow: / User-agent: Holmes Disallow: / User-agent: htdig Disallow: / User-agent: iaskspider Disallow: / User-agent: ichiro Disallow: / User-agent: igdeSpyder Disallow: / User-agent: IRLbot Disallow: / User-agent: IssueCrawler Disallow: / User-agent: Jaxified Bot Disallow: / User-agent: Jyxobot Disallow: / User-agent: KoepaBot Disallow: / User-agent: L.webis Disallow: / User-agent: LapozzBot Disallow: / User-agent: Larbin Disallow: / User-agent: LDSpider Disallow: / User-agent: LexxeBot Disallow: / User-agent: Linguee Bot Disallow: / User-agent: LinkWalker Disallow: / User-agent: lmspider Disallow: / User-agent: lwp-trivial Disallow: / User-agent: mabontland Disallow: / User-agent: magpie-crawler Disallow: / User-agent: Mediapartners-Google Disallow: / User-agent: MJ12bot Disallow: / User-agent: MLBot Disallow: / User-agent: Mnogosearch Disallow: / User-agent: mogimogi Disallow: / User-agent: MojeekBot Disallow: / User-agent: Moreoverbot Disallow: / User-agent: Morning Paper Disallow: / User-agent: MSRBot Disallow: / User-agent: MVAClient Disallow: / User-agent: mxbot Disallow: / User-agent: NetcraftSurveyAgent Disallow: / User-agent: NetResearchServer Disallow: / User-agent: NetSeer Crawler Disallow: / User-agent: NewsGator Disallow: / User-agent: NG-Search Disallow: / User-agent: nicebot Disallow: / User-agent: noxtrumbot Disallow: / User-agent: Nusearch Spider Disallow: / User-agent: NutchCVS Disallow: / User-agent: Nymesis Disallow: / User-agent: oegp Disallow: / User-agent: omgilibot Disallow: / User-agent: OmniExplorer_Bot Disallow: / User-agent: OOZBOT Disallow: / User-agent: Orbiter Disallow: / User-agent: PageBitesHyperBot Disallow: / User-agent: Peew Disallow: / User-agent: polybot Disallow: / User-agent: Pompos Disallow: / User-agent: PostPost Disallow: / User-agent: Psbot Disallow: / User-agent: PycURL Disallow: / User-agent: Qseero Disallow: / User-agent: Radian6 Disallow: / User-agent: RAMPyBot Disallow: / User-agent: RufusBot Disallow: / User-agent: SandCrawler Disallow: / User-agent: SBIder Disallow: / User-agent: ScoutJet Disallow: / User-agent: Scrubby Disallow: / User-agent: SearchSight Disallow: / User-agent: Seekbot Disallow: / User-agent: semanticdiscovery Disallow: / User-agent: semrush Disallow: / User-agent: SEOChat::Bot Disallow: / User-agent: SeznamBot Disallow: / User-agent: Shim-Crawler Disallow: / User-agent: Shoula robot Disallow: / User-agent: Sitebot Disallow: / User-agent: Snappy Disallow: / User-agent: sogou spider Disallow: / User-agent: Sosospider Disallow: / User-agent: Speedy Spider Disallow: / User-agent: Sqworm Disallow: / User-agent: StackRambler Disallow: / User-agent: suggybot Disallow: / User-agent: SynooBot Disallow: / User-agent: Teoma Disallow: / User-agent: TerrawizBot Disallow: / User-agent: TheSuBot Disallow: / User-agent: Thumbnail.CZ robot Disallow: / User-agent: TinEye Disallow: / User-agent: truwoGPS Disallow: / User-agent: TurnitinBot Disallow: / User-agent: TweetedTimes Bot Disallow: / User-agent: TwengaBot Disallow: / User-agent: updated Disallow: / User-agent: Urlfilebot Disallow: / User-agent: Vagabondo Disallow: / User-agent: VoilaBot Disallow: / User-agent: Vortex Disallow: / User-agent: voyager Disallow: / User-agent: VYU2 Disallow: / User-agent: webcollage Disallow: / User-agent: Websquash.com Disallow: / User-agent: wf84 Disallow: / User-agent: WoFindeIch Robot Disallow: / User-agent: WomlpeFactory Disallow: / User-agent: woobot Disallow: / User-agent: Xaldon_WebSpider Disallow: / User-agent: yacy Disallow: / User-agent: Yasaklibot Disallow: / User-agent: Yeti Disallow: / User-agent: YodaoBot Disallow: / User-agent: yoogliFetchAgent Disallow: / User-agent: YoudaoBot Disallow: / User-agent: Zao Disallow: / User-agent: Zealbot Disallow: / User-agent: zspider Disallow: / User-agent: ZyBorg Disallow: / User-agent: * Crawl-delay: 10 # Directories Disallow: /admin/ Disallow: /administrator/ Disallow: /aggregator/ Disallow: /bin/ Disallow: /cache/ Disallow: /cli/ Disallow: /components/ Disallow: /comment/reply/ Disallow: /database/ Disallow: /feed/ Disallow: /includes/ Disallow: /installation/ Disallow: /language/ Disallow: /layouts/ Disallow: /libraries/ Disallow: /logs/ Disallow: /logout/ Disallow: /media/ Disallow: /misc/ Disallow: /modules/ Disallow: /node/add/ Disallow: /plugins/ Disallow: /profiles/ Disallow: /scripts/ Disallow: /sites/ Disallow: /templates/ Disallow: /themes/ Disallow: /tmp/ Disallow: /trackback/ Disallow: /user/login/ Disallow: /user/password/ Disallow: /user/register/ Disallow: /updates/ Disallow: /wp-admin/ Disallow: /wp-includes Disallow: /wp-content/cache Disallow: /wp-content/plugins Disallow: /wp-content/themes Disallow: /wp-content/upgrade # Files Disallow: /xmlrpc.php Disallow: /cron.php Disallow: /update.php Disallow: /install.php Disallow: /INSTALL.txt Disallow: /INSTALL.mysql.txt Disallow: /INSTALL.pgsql.txt Disallow: /CHANGELOG.txt Disallow: /MAINTAINERS.txt Disallow: /LICENSE.txt Disallow: /UPGRADE.txt # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?author= Disallow: /?q=aggregator/ Disallow: /?q=comment/reply/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/add/ Disallow: /?q=search/ Disallow: /?q=user/password/ Disallow: /?q=user/register/ Disallow: /?q=user/login/ # Paths (clean URLs) fixed Disallow: /aggregator Disallow: /comment/reply Disallow: /logout Disallow: /node/add Disallow: /user/register Disallow: /user/password Disallow: /user/login # Paths (no clean URLs) fixed Disallow: /?q=admin Disallow: /?q=aggregator Disallow: /?q=comment/reply Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node/add Disallow: /?q=search Disallow: /?q=user/password Disallow: /?q=user/register Disallow: /?q=user/login # Additional Rules Disallow: /node$ Disallow: /user$ Disallow: /search$