# URBot.net — robots.txt
# Allow search engines, block AI training crawlers from proprietary content

User-agent: *
Allow: /
Sitemap: https://urbot.net/sitemap.xml
Sitemap: https://urbot.net/sitemap-bots.xml
Sitemap: https://urbot.net/sitemap-pages.xml
Sitemap: https://urbot.net/sitemap-international.xml

# New content areas
Allow: /blog/
Allow: /cities/
Allow: /use-cases/
Allow: /social-proof/
Allow: /roadmap/
Allow: /changelog/
Allow: /faq/
Allow: /affiliate/
Allow: /achievements/
Allow: /vs/

# Protect private areas
Disallow: /my-bots/
Disallow: /login/
Disallow: /secure-downloads/
Disallow: /billing/
Disallow: /ecosystem/
Disallow: /widget/src/
Disallow: /api-keys/
Disallow: /admin/
Disallow: /admin-health/
Disallow: /notifications/

# Block AI training crawlers from proprietary knowledge and source
User-agent: GPTBot
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: ChatGPT-User
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: CCBot
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: Google-Extended
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: ClaudeBot
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: Bytespider
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: Diffbot
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/

User-agent: Amazonbot
Disallow: /knowledge/
Disallow: /secure-downloads/
Disallow: /widget/src/
Disallow: /billing/
Disallow: /ecosystem/