try to force delay 2 sec for crawling bots

This commit is contained in:
Quentin Gibeaux 2017-10-03 10:02:28 +02:00 committed by root
parent 61411b26de
commit d4c935a943
1 changed files with 2 additions and 0 deletions

View File

@ -3,3 +3,5 @@
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /
User-agent: *
Crawl-delay: 3