robots.txt: adapt for Netlify. (#29192)

Since we build with `HUGO_ENV` set to `production` on Netlify, use another variable to prevent crawling.

diff --git a/site/layouts/robots.txt b/site/layouts/robots.txt
index e9d78f1aeddc34db33b3bfbd01e7aa40eae6e5f9..c6703cd06c894c1519d310c21b89fd906f034bb4 100644 (file)
--- a/site/layouts/robots.txt
+++ b/site/layouts/robots.txt
@@ -1,8 +1,12 @@
 # www.robotstxt.org
 
-{{ if (eq (getenv "HUGO_ENV") "production") -}}
+{{- $isProduction := eq (getenv "HUGO_ENV") "production" -}}
+{{- $isNetlify := eq (getenv "NETLIFY") "true" -}}
+{{- $allowCrawling := and (not $isNetlify) $isProduction -}}
+
+{{ if $allowCrawling }}
 # Allow crawling of all content
 {{- end }}
 User-agent: *
-Disallow:{{ if (ne (getenv "HUGO_ENV") "production") }} /{{ end }}
+Disallow:{{ if not $allowCrawling }} /{{ end }}
 Sitemap: {{ .Site.BaseURL }}/sitemap.xml