Файл robots.txt

Источник Robots.txt: на сайте http://itrex.ru/robots.txt
Содержимое:

User-agent: Yandex
Disallow: /temp/
Disallow: /updfiles/
Disallow: /index.php
Disallow: /main/
Disallow: /news.php
Disallow: /competition$
Disallow: /updfiles/
Disallow: /partners/
Disallow: /ru/partners/
Allow: /partners$
Disallow: /forum
Disallow: /main/
Disallow: /search
Disallow: /link.html
Disallow: /news/tag
Disallow: /clients/*
Sitemap: https://itrex.ru/sitemap.xml
Host: https://itrex.ru

User-agent: Googlebot
Disallow: /temp/
Disallow: /updfiles/
Disallow: /index.php
Disallow: /main/
Disallow: /news.php
Disallow: /competition$
Disallow: /updfiles/
Disallow: /partners/
Disallow: /ru/partners/
Allow: /partners$
Disallow: /forum
Disallow: /main/
Disallow: /search
Disallow: /link.html
Disallow: /news/tag
Disallow: /clients/*
Sitemap: https://itrex.ru/sitemap.xml


User-agent: Mail.Ru
Disallow: /temp/
Disallow: /updfiles/
Disallow: /index.php
Disallow: /main/
Disallow: /news.php
Disallow: /competition$
Disallow: /updfiles/
Disallow: /partners/
Disallow: /ru/partners/
Allow: /partners$
Disallow: /forum
Disallow: /main/
Disallow: /search
Disallow: /link.html
Disallow: /news/tag
Disallow: /clients/*
Clean-param: utm_source
Clean-param: utm_medium
Clean-param: utm_content
Clean-param: utm_campaign
Clean-param: utm_term
Clean-param: id
Clean-param: url
Clean-param: upd
Clean-param: limitstart
Clean-param: limitstart1
Clean-param: yagla
Sitemap: https://itrex.ru/sitemap.xml

User-agent: *
Disallow: /temp/
Disallow: /updfiles/
Disallow: /index.php
Disallow: /main/
Disallow: /news.php
Disallow: /competition$
Disallow: /updfiles/
Disallow: /partners/
Disallow: /ru/partners/
Allow: /partners$
Disallow: /forum
Disallow: /main/
Disallow: /search
Disallow: /link.html
Disallow: /news/tag
Disallow: /clients/*
Sitemap: https://itrex.ru/sitemap.xml