# Managed Knowledge Base Web Crawler.
# User-agent is amazon-bedrock-knowledgebase-on-behalf-of-HASH
# Replace HASH with the token from your own request logs before you rely on this file.
# A missing robots.txt makes this crawler disallow the site.
# A robots.txt it cannot fetch (block, parse error, timeout) makes it crawl anyway.
# Source: https://docs.aws.amazon.com/bedrock/latest/userguide/kb-managed-ds-webcrawler.html

User-agent: amazon-bedrock-knowledgebase-on-behalf-of-HASH
Allow: /docs/
Disallow: /private/

# Classic Bedrock web crawler (preview) uses bedrockbot-UUID, then generic bedrockbot.
# That is a different bot. Do not copy those rules onto this user-agent and assume they match.