Fix robots.txt

config/_default/params.toml

@@ -23,6 +23,35 @@ replyByEmail = false
 # mainSections = ["section1", "section2"]
 # robots = ""
 
+robots = [
+  "AdsBot-Google",
+  "Amazonbot",
+  "anthropic-ai",
+  "Applebot",
+  "Applebot-Extended",
+  "AwarioRssBot",
+  "AwarioSmartBot",
+  "Bytespider",
+  "CCBot",
+  "ChatGPT",
+  "ChatGPT-User",
+  "Claude-Web",
+  "ClaudeBot",
+  "cohere-ai",
+  "DataForSeoBot",
+  "Diffbot",
+  "FacebookBot",
+  "Google-Extended",
+  "GPTBot",
+  "ImagesiftBot",
+  "magpie-crawler",
+  "omgili",
+  "Omgilibot",
+  "peer39_crawler",
+  "PerplexityBot",
+  "YouBot",
+]
+
 disableImageOptimization = false
 disableImageOptimizationMD = false
 disableTextInHeader = false

layouts/robots.txt

@@ -1,9 +1,4 @@
-{{ if hugo.IsProduction }}
+{{ range $robot := .Site.Params.robots }}
-  {{ range $robot := .Site.Params.robots }}
+User-agent: {{ $robot }}
-  User-agent: {{ $robot }}
+Disallow: /
-  Disallow: /
+{{end}}
-  {{end}}
-{{ else }}
-  User-agent: *
-  Disallow: /
-{{ end }}