Fix robots.txt

config/_default/params.toml

@@ -23,6 +23,35 @@ replyByEmail = false
 # mainSections = ["section1", "section2"]
 # robots = ""
 
+robots = [
+  "AdsBot-Google",
+  "Amazonbot",
+  "anthropic-ai",
+  "Applebot",
+  "Applebot-Extended",
+  "AwarioRssBot",
+  "AwarioSmartBot",
+  "Bytespider",
+  "CCBot",
+  "ChatGPT",
+  "ChatGPT-User",
+  "Claude-Web",
+  "ClaudeBot",
+  "cohere-ai",
+  "DataForSeoBot",
+  "Diffbot",
+  "FacebookBot",
+  "Google-Extended",
+  "GPTBot",
+  "ImagesiftBot",
+  "magpie-crawler",
+  "omgili",
+  "Omgilibot",
+  "peer39_crawler",
+  "PerplexityBot",
+  "YouBot",
+]
+
 disableImageOptimization = false
 disableImageOptimizationMD = false
 disableTextInHeader = false