updated robots.txt

src/pages/robots.txt.ts

@@ -8,28 +8,35 @@ Disallow: /nogooglebot/
 User-agent: AI2Bot
 User-agent: Ai2Bot-Dolma
 User-agent: Amazonbot
+User-agent: anthropic-ai
 User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: ChatGPT-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
+User-agent: cohere-ai
 User-agent: Diffbot
+User-agent: DuckAssistBot
 User-agent: FacebookBot
 User-agent: FriendlyCrawler
-User-agent: GPTBot
 User-agent: Google-Extended
 User-agent: GoogleOther
 User-agent: GoogleOther-Image
 User-agent: GoogleOther-Video
+User-agent: GPTBot
+User-agent: iaskspider/2.0
 User-agent: ICC-Crawler
-User-agent: ISSCyberRiskCrawler
 User-agent: ImagesiftBot
+User-agent: img2dataset
+User-agent: ISSCyberRiskCrawler
 User-agent: Kangaroo Bot
 User-agent: Meta-ExternalAgent
 User-agent: Meta-ExternalFetcher
 User-agent: OAI-SearchBot
+User-agent: omgili
+User-agent: omgilibot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
@@ -38,13 +45,6 @@ User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
 User-agent: Webzio-Extended
 User-agent: YouBot
-User-agent: anthropic-ai
-User-agent: cohere-ai
-User-agent: facebookexternalhit
-User-agent: iaskspider/2.0
-User-agent: img2dataset
-User-agent: omgili
-User-agent: omgilibot
 Disallow: /
 
 User-agent: *