robots.txt:# begin output from /robots.txt
# webtrees: online genealogy
# Copyright (C) 2019 webtrees development team
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
# This file needs to be placed in the domain root folder,
# such as "www.example.com/robots.txt". It will not work in a
# subfolder, such as "www.example.com/webtrees/robots.txt"
# If you need to move it, then remember to adjust the paths as well.
# e.g. "Disallow: /login.php" becomes "Disallow: /webtrees/login.php".
# These URLs are expensive to generate or duplicate existing content:
User-Agent: *
Disallow: /ancestry.php
Disallow: /branches.php
Disallow: /calendar.php
Disallow: /compact.php
Disallow: /descendancy.php
Disallow: /familybook.php
Disallow: /famlist.php
Disallow: /fanchart.php
Disallow: /hourglass.php
Disallow: /lifespan.php
Disallow: /login.php
Disallow: /medialist.php
Disallow: /notelist.php
Disallow: /pedigree.php
Disallow: /placelist.php
Disallow: /relationship.php
Disallow: /repolist.php
Disallow: /reportengine.php
Disallow: /search.php
Disallow: /search_advanced.php
Disallow: /sourcelist.php
Disallow: /statistics.php
Disallow: /statisticsplot.php
Disallow: /timeline.php
##########################################################################################################
############# unerwünschte bots, die aber die robots.txt abfragen
##########################################################################################################
#"TurnitinBot/2.1 (http://www.turnitin.com/robot/crawlerinfo.html)"
User-agent: TurnitinBot
Disallow: /
User-agent: SlySearch
Disallow: /
#"findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)"
User-agent: findlinks
Disallow: /
#"magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)"
User-agent: magpie-crawler
Disallow: /
User-agent: Pixray-Seeker
Disallow: /
# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
User-agent: MJ12bot
Disallow: /
# http://www.80legs.com/webcrawler.html - if 008 is crawling your website, it means that one or more 80legs users created a web crawl
User-agent: 008
Disallow: /
User-agent: Ezooms
Disallow: /
#"Mozilla/5.0 (compatible; AhrefsBot/2.0; +http://ahrefs.com/robot/)"
User-agent: AhrefsBot
Disallow: /
#"lb-spider/Mozilla/5.0 Gecko/20100101 Firefox/10.0.2 (lb-spider; http://www.linkbutler.de/spider; spider@linkbutler.de)"
User-agent: lb-spider
Disallow: /
#"Mozilla/5.0 (compatible; WBSearchBot/1.1; +http://www.warebay.com/bot.html)"
User-agent: WBSearchBot
Disallow: /
#"psbot/0.1 (+http://www.picsearch.com/bot.html)"
User-agent: psbot
Disallow: /
#"HuaweiSymantecSpider/1.0+DSE-support@huaweisymantec.com+(compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR ; http://www.huaweisymantec.com/en/IRL/spider)"
User-agent: HuaweiSymantecSpider
Disallow: /
#"Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)"
User-agent: sistrix
Disallow: /
#"EC2LinkFinder"
User-agent: EC2LinkFinder
Disallow: /
#"http://SiteIntel.net Bot"
#"htdig"
User-agent: htdig
Disallow: /
#"SemrushBot/0.91" - http://de.semrush.com/? -Professionelle Software für SEO & SEM Professionals?
User-agent: SemrushBot
Disallow: /
#"Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)" - we sell no wine before its time != trustworthy
User-agent: discobot
Disallow: /
#"linkdex.com/v2.0" - SEO
User-agent: linkdex.com
Disallow: /
#"SeznamBot/3.0 (+http://fulltext.sblog.cz/)" - sz-SEO
User-agent: SeznamBot
Disallow: /
#"EdisterBot (http://www.edister.com/bot.html)"
User-agent: EdisterBot
Disallow: /
#"Mozilla/5.0 (compatible; SWEBot/1.0; +http://swebot-crawler.net)" - versucht auf posting im forum zu replien
User-agent: SWEBot
Disallow: /
### ab hier noch in htaccess eintragen
#"Mozilla/5.0 (compatible;picmole/1.0 +http://www.picmole.com)"
User-agent: picmole
Disallow: /
#"Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)"
#"Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) (compatible; Yeti-Mobile/0.1; +http://help.naver.com/robots/)"
User-agent: Yeti
Disallow: /
User-agent: Yeti-Mobile
Disallow: /
#"PagePeeker.com (info: http://pagepeeker.com/robots)"
User-agent: PagePeeker
Disallow: /
#"CatchBot/1.0; +http://www.catchbot.com"
User-agent: CatchBot
Disallow: /
#"yacybot (freeworld/global; amd64 Linux 3.2.1-gentoo-r2; java 1.6.0_24; Europe/de) http://yacy.net/bot.html"
User-agent: yacybot
Disallow: /
#"netEstate NE Crawler (+http://www.sengine.info/)"
User-agent: netEstate NE Crawler
Disallow: /
#"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"
User-agent: SurveyBot
Disallow: /
#"COMODO SSL Checker"
#"Comodo-Certificates-Spider"
User-agent: COMODO SSL Checker
Disallow: /
User-agent: Comodo-Certificates-Spider
Disallow: /
#"gonzo2[p] (+http://www.suchen.de/faq.html)" (Geschäftesuche)
User-agent: gonzo
Disallow: /
#"crawler schrein, crawler@schrein.nl id-4"
User-agent: schrein
Disallow: /
#"BacklinkCrawler (http://www.backlinktest.com/crawler.html)"
User-agent: BacklinkCrawler
Disallow: /
#"Mozilla/5.0 (compatible; Afilias Web Mining Tool 1.0; +http://www.afilias.info; awmt@afilias.info)"
User-agent: Afilias Web Mining Tool
Disallow: /
#"Mozilla/5.0 (compatible; SEOkicks-Robot +http://www.seokicks.de/robot.html)"
User-agent: SEOkicks
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
#"Mozilla/5.0 (compatible; suggybot v0.01a, http://blog.suggy.com/was-ist-suggy/suggy-webcrawler/)"
User-agent: suggybot
Disallow: /
#"http://www.bdbrandprotect.com" "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1)"
User-agent: bdbrandprotect
Disallow: /
User-agent: BPImageWalker
Disallow: /
User-agent: BPImageWalker*
Disallow: /
#"Updownerbot (+http://www.updowner.com/bot)"
User-agent: Updownerbot
Disallow: /
#"lex/1.0"
User-agent: lex
Disallow: /
#"Content Crawler"
User-agent: Content Crawler
Disallow: /
#"Mozilla/5.0 (compatible; DCPbot/1.1; +http://domains.checkparams.com/)"
User-agent: DCPbot
Disallow: /
#"Mozilla/5.0 (compatible; KaloogaBot; http://kalooga.com/crawler)"
User-agent: KaloogaBot
Disallow: /
#"MLBot (www.metadatalabs.com/mlbot)"
User-agent: MLBot
Disallow: /
#"Wget/1.9"
User-agent: Wget
Disallow: /
#"libwww-perl/5.837"
User-agent: libwww-perl
Disallow: /
#"curl/7.21.3 (amd64-portbld-freebsd7.2) libcurl/7.21.3 OpenSSL/0.9.8e zlib/1.2.3"
User-agent: curl
Disallow: /
#"Java/1.6.0_29"
User-agent: Java
Disallow: /
#"Mozilla/5.0 (X11; U; Linux i686; de; rv:1.9.0.1; compatible; iCjobs Stellenangebote Jobs; http://www.icjobs.de) Gecko/20100401 iCjobs/3.2.3"
User-agent: iCjobs
Disallow: /
#"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)"
User-agent: oBot
Disallow: /
#"Mozilla/5.0 (compatible; WebmasterCoffee/0.7; +http://webmastercoffee.com/about)"
User-agent: WebmasterCoffee
Disallow: /
#"Mozilla/5.0 (compatible; Qualidator.com Bot 1.0;)" (http://www.qualidator.com/Web/de/Support/robotstxt_Hinweise.htm)
User-agent: Qualidator*
Disallow: /
#"Mozilla/4.0 (compatible; http://search.thunderstone.com/texis/websearch/about.html)" (http://www.thunderstone.com/site/gw25man/page_exclusion_and_robots_txt.html)
User-agent: Webinator
Disallow: /
User-agent: Scooter
Disallow: /
User-agent: *thunderstone*
Disallow: /
#"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) (larbin2.6.3@unspecified.mail)"
User-agent: larbin
Disallow: /
User-agent: OpidooBOT
Disallow: /
#"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24"
User-agent: ips-agent
Disallow: /
#"TinEye/1.1 (http://tineye.com/crawler.html)"
User-agent: TinEye
Disallow: /
#"Mozilla/5.0 (compatible; UnisterBot; crawler@unister.de)"
User-agent: UnisterBot
Disallow: /
User-agent: Unister*
Disallow: /
#"Mozilla/5.0 (compatible; en-US; ReverseGet/1.0; http://reverseget.com/; robot@reverseget.com)"
User-agent: ReverseGet
Disallow: /
# Put your sitemap here:
Sitemap: http://diehowolds.de/module.php?mod=sitemap&mod_action=generate&file=sitemap.xml
# end output from /robots.txt
# warning, /webtrees/robots-example.txt does not exist
|
Home Newdy-External HTML-Sitemap 302