#
# This is robots.txt file.
# robots.txt must be < 32Kb.
#
#
#
User-agent: WebAlta
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: Lycos
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: Aport
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: ia_archiver
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: StackRambler
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: Teoma
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: MSNBot
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: Slurp
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
User-agent: Mail.Ru
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
#
#
User-agent: *
Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
Sitemap: http://madon333.freeshell.org/sitemap.xml.gz
Sitemap: http://madon333.freeshell.org/sitemap.xml
#
Host: www.madon.ru
Clean-param: s *.php
#
#
#
User-agent: Yandex
#Disallow: */webstats/
Disallow: /*/new/*.html$
Disallow: /*/old/*.*$
Disallow: /*?*
#
Crawl-delay: 9
Allow: */webstat/2000/0000/
Request-rate: 1/7
Visit-time: 0531-0953
#
#
###
#
Sitemap: http://madon333.freeshell.org/sitemap.xml.gz
Sitemap: http://madon333.freeshell.org/sitemap.xml
Sitemap: http://www.madon333.freeshell.org/sitemap.xml.gz
Sitemap: http://www.madon333.freeshell.org/sitemap.xml
#
#
Host: www.madon.ru
Clean-param: s *.php
#
#
#
# end of robots.txt file.
#
# so.com chinese search engine
User-agent: 360Spider
User-agent: 360Spider-Image
User-agent: 360Spider-Video
# google.com landing page quality check
User-agent: AdsBot-Google
# google.com app resource fetcher
User-agent: AdsBot-Google-Mobile-Apps
# bing ads bot
User-agent: adidxbot
# apple.com search engine
User-agent: Applebot
user-agent: AppleNewsBot
# baidu.com chinese search engine
User-agent: Baiduspider
User-agent: Baiduspider-image
# bing.com international search engine
User-agent: bingbot
User-agent: BingPreview
# commoncrawl.org open repository of web crawl data
User-agent: CCBot
# cliqz.com german in-product search engine
User-agent: Cliqzbot
# coccoc.com vietnamese search engine
User-agent: coccoc
User-agent: coccocbot-image
User-agent: coccocbot-web
# daum.net korean search engine
User-agent: Daumoa
# dazoo.fr french search engine
User-agent: Dazoobot
# deusu.de german search engine
User-agent: DeuSu
# duckduckgo.com international privacy search engine
User-agent: DuckDuckBot
User-agent: DuckDuckGo-Favicons-Bot
# eurip.com european search engine
User-agent: EuripBot
# exploratodo.com latin search engine
User-agent: Exploratodo
# facebook.com social network
User-agent: Facebot
# feedly.com feed fetcher
User-agent: Feedly
# findx.com european search engine
User-agent: Findxbot
# goo.ne.jp japanese search engine
User-agent: gooblog
# google.com international search engine
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Video
# so.com chinese search engine
User-agent: HaoSouSpider
# goo.ne.jp japanese search engine
User-agent: ichiro
# istella.it italian search engine
User-agent: istellabot
# jike.com / chinaso.com chinese search engine
User-agent: JikeSpider
# lycos.com & hotbot.com international search engine
User-agent: Lycos
# mail.ru russian search engine
User-agent: Mail.Ru
# google.com adsense bot
User-agent: Mediapartners-Google
# mojeek.com search engine
User-agent: MojeekBot
# bing.com international search engine
User-agent: msnbot
User-agent: msnbot-media
# orange.com international search engine
User-agent: OrangeBot
# pinterest.com social networtk
User-agent: Pinterest
# botje.nl dutch search engine
User-agent: Plukkie
# qwant.com french search engine
User-agent: Qwantify
# rambler.ru russian search engine
User-agent: Rambler
# seznam.cz czech search engine
User-agent: SeznamBot
# soso.com chinese search engine
User-agent: Sosospider
# yahoo.com international search engine
User-agent: Slurp
# sogou.com chinese search engine
User-agent: Sogou blog
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou spider2
User-agent: Sogou web spider
# sputnik.ru russian search engine
User-agent: SputnikBot
# ask.com international search engine
User-agent: Teoma
# twitter.com bot
User-agent: Twitterbot
# wotbox.com international search engine
User-agent: wotbox
# yacy.net p2p search software
User-agent: yacybot
# yandex.com russian search engine
User-agent: Yandex
User-agent: YandexMobileBot
# search.naver.com south korean search engine
user-agent: Yeti
# yioop.com international search engine
User-agent: YioopBot
# yooz.ir iranian search engine
User-agent: yoozBot
# youdao.com chinese search engine
User-agent: YoudaoBot
# crawling rule(s)
#Disallow:
# disallow all other bots
#User-agent: *
#Disallow: /