diff --git a/views/templates/robots.txt b/views/templates/robots.txt index 1bb81f23..f54d0256 100644 --- a/views/templates/robots.txt +++ b/views/templates/robots.txt @@ -9,7 +9,6 @@ Allow: / #User-agent: Ai2Bot-Dolma #User-agent: Amazonbot #User-agent: anthropic-ai -#User-agent: Applebot #User-agent: Applebot-Extended #User-agent: Bytespider #User-agent: CCBot @@ -46,4 +45,4 @@ Allow: / #User-agent: VelenPublicWebCrawler #User-agent: Webzio-Extended #User-agent: YouBot -#Disallow: / \ No newline at end of file +#Disallow: /