From bb3025200009bcd265f779985117b12f2dd76cc1 Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Mon, 30 Dec 2024 17:40:19 -0600 Subject: [PATCH] remove Applebot from list (Applebot-Extended determins LLM usage) --- views/templates/robots.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/views/templates/robots.txt b/views/templates/robots.txt index 1bb81f23..f54d0256 100644 --- a/views/templates/robots.txt +++ b/views/templates/robots.txt @@ -9,7 +9,6 @@ Allow: / #User-agent: Ai2Bot-Dolma #User-agent: Amazonbot #User-agent: anthropic-ai -#User-agent: Applebot #User-agent: Applebot-Extended #User-agent: Bytespider #User-agent: CCBot @@ -46,4 +45,4 @@ Allow: / #User-agent: VelenPublicWebCrawler #User-agent: Webzio-Extended #User-agent: YouBot -#Disallow: / \ No newline at end of file +#Disallow: /