User-agent: *
Allow: /

# Block admin areas and API
Disallow: /admin/
Disallow: /api/

# Block old audio URLs (redirected to /podcast)
Disallow: /audio/

# Private promotional playlist (shared directly via email / social)
Disallow: /hubz-platform
Disallow: /hubz-platform/

# Sitemap location
Sitemap: https://datahubz.com/sitemap.xml

# Allow AI crawlers for better AI-powered search visibility
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: ClaudeBot
Allow: /