# Website Recycling · explicit AI-crawler policy · generated from BOT_REGISTRY
# 69 AI crawlers explicitly named + 3 content-use policy tokens.
# The catch-all below is "User-agent: * / Allow: /" — every well-behaved crawler is
# welcome. The named list is a compatibility/welcome policy, not an exclusive allowlist.
# Policy tokens (Google-Extended, Applebot-Extended, Cohere-Train) are content-USE controls with no HTTP
# user-agent of their own — listed as policy, never counted as crawler traffic.
# Source of truth: /src/data/canonical-bots.ts · never hand-edit this file.

# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai

# Google
User-agent: Google-Extended   # content-use policy token · no HTTP user-agent
User-agent: Googlebot
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Microsoft
User-agent: Bingbot
User-agent: BingPreview
User-agent: BingChat
User-agent: MSNBot
User-agent: adidxbot

# Meta
User-agent: Meta-ExternalFetcher
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: FacebookExternalHit

# Apple
User-agent: Applebot
User-agent: Applebot-Extended   # content-use policy token · no HTTP user-agent

# Mistral
User-agent: MistralAI-User

# Cohere
User-agent: Cohere-AI
User-agent: Cohere-Train   # content-use policy token · no HTTP user-agent

# DuckDuckGo
User-agent: DuckAssistBot

# Diffbot
User-agent: Diffbot

# Komo
User-agent: Komo

# Andi
User-agent: Andi

# Phind
User-agent: Phind

# Kagi
User-agent: Kagibot

# You.com
User-agent: YouBot

# Neeva
User-agent: NeevaBot

# Brave
User-agent: Bravebot
User-agent: Brave-SearchAssist

# xAI
User-agent: GrokBot

# DeepSeek
User-agent: DeepSeekBot

# Common Crawl
User-agent: CCBot
User-agent: Common-Crawl-Bot

# NICT
User-agent: ICC-Crawler

# ImageSift
User-agent: ImagesiftBot

# Internet Archive
User-agent: archive.org_bot
User-agent: Wayback-Save-Page-Bot

# ByteDance
User-agent: Bytespider
User-agent: TikTokSpider

# Baidu
User-agent: Baiduspider

# Yandex
User-agent: YandexBot
User-agent: YandexImages

# Huawei
User-agent: PetalBot

# Amazon
User-agent: Amazonbot

# Sogou
User-agent: Sogou

# LinkedIn
User-agent: LinkedInBot

# X/Twitter
User-agent: TwitterBot

# Slack
User-agent: Slackbot-LinkExpanding

# Discord
User-agent: Discordbot

# Meta
User-agent: WhatsApp

# Pinterest
User-agent: Pinterestbot

# DuckDuckGo
User-agent: DuckDuckBot

# Arquivo.pt
User-agent: Arquivo-web-crawler

# Anchor
User-agent: AnchorBrowser

# Novellum
User-agent: NovellumAICrawl

# ProRata.ai
User-agent: ProRataInc

# Terracotta
User-agent: TerracottaBot

# Timpi
User-agent: Timpibot

# Webz.io
User-agent: Webz-io

# OpenAI
User-agent: AtlasCrawler

# Perplexity
User-agent: CometCrawler

Allow: /

# Catch-all · default-allow · this site is open to all crawlers
User-agent: *
Allow: /

Sitemap: https://websiterecycling.com/sitemap-index.xml