diff --git a/static/robots.txt b/static/robots.txt index bbdf1384..8b0bbfdd 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,10 +1,9 @@ # Dark Visitors Robots.txt -# https://darkvisitors.com/docs/set-up-a-robots-txt # AI Data Scraper -# https://darkvisitors.com/agents/anthropic-ai +# https://darkvisitors.com/agents/applebot-extended -User-agent: anthropic-ai +User-agent: Applebot-Extended Disallow: / # AI Data Scraper @@ -19,6 +18,12 @@ Disallow: / User-agent: CCBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/claudebot + +User-agent: ClaudeBot +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/diffbot @@ -47,4 +52,22 @@ Disallow: / # https://darkvisitors.com/agents/omgili User-agent: omgili -Disallow: / \ No newline at end of file +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/claude-web + +User-agent: Claude-Web +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai +Disallow: /