diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index 7d33aa5..8fa2c2f 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -47,11 +47,6 @@ User-agent: VelenPublicWebCrawler User-agent: Webzio-Extended User-agent: YouBot Disallow: / -DisallowAITraining: / - -# Block any non-specified AI crawlers -User-Agent: * -DisallowAITraining: /within Sitemap: ${sitemapURL.href} `