robots.txt: disallow crawling when not in production. (#27559)

XhmikosR · web-flow · commit 3256a2c2315b · 2018-11-02T12:41:57.000+02:00
diff --git a/site/robots.txt b/site/robots.txt
@@ -5,5 +5,5 @@
 
 # Allow crawling of all content
 User-agent: *
-Disallow:
+Disallow:{% if jekyll.environment != "production" %} /{% endif %}
 Sitemap: {{ site.url }}/sitemap.xml
