Файл: public/robots.txt
Строк: 39
User-agent: *
Allow: /
Crawl-delay: 1
Disallow: /api*
Disallow:
/admin*
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow:
/menu
Disallow: /mails
Disallow: /recovery
Disallow: /unsubscribe
Disallow:
/captcha
Disallow: /online
Disallow: /contacts
Disallow: /ignores
Disallow:
/profile
Disallow: /pictures
Disallow: /ratings
Disallow:
/transfers
Disallow: /messages
Disallow: /settings
Disallow:
/notebooks
Disallow: /authlogs
Disallow: /verify
Disallow:
/confirm
Disallow: /walls
Disallow: /users*
Disallow: /language*
Disallow:
/index.php*
Disallow: /forums/active/*
Disallow: /downs/active/*
Disallow:
/articles/active/*
Sitemap: /sitemap.xml
User-agent: AhrefsBot
Disallow:
/
User-agent: GPTBot
Disallow: /
User-agent: Amazonbot
Disallow:
/
User-agent: SemrushBot
Disallow: /
User-agent: PetalBot
Disallow:
/
User-agent: Applebot
Disallow: /
User-agent: MJ12bot
Disallow:
/
User-agent: Bytespider
Disallow: /
User-agent: DotBot
Disallow:
/
User-agent: BLEXBot
Disallow: /
User-agent: Yatcybot
Disallow:
/
User-agent: Amzn-SearchBot
Disallow: /
User-agent: ClaudeBot
Disallow:
/
User-agent: PerplexityBot
Disallow: /
User-agent: bingbot
Disallow:
/
User-agent: GenomeCrawlerd
Disallow: /
User-agent:
serpstatbot
Disallow: /