try to force delay 2 sec for crawling bots

5 years ago · d4c935a943
parent 61411b26de
commit d4c935a943
1 changed files with 2 additions and 0 deletions
--- a/public/robots.txt
+++ b/public/robots.txt
@ -3,3 +3,5 @@
 # To ban all spiders from the entire site uncomment the next two lines:
 # User-agent: *
 # Disallow: /
+User-agent: *
+Crawl-delay: 3