User-agent: *
Disallow: /accept-invite/
Disallow: /login/
Disallow: /signup/
Disallow: /api/
Disallow: /static/
Disallow: /404

# Allow OpenAI's GPTBot (ChatGPT, etc.)
User-agent: GPTBot
Allow: /

# Allow Anthropic’s Claude
User-agent: ClaudeBot
Allow: /

# Allow Perplexity’s crawler
User-agent: PerplexityBot
Allow: /

# Allow Google's AI tools (Bard, SGE, etc.)
User-agent: Google-Extended
Allow: /

# Allow CommonCrawl
User-agent: CCBot
Allow: /

# Disallow all AI crawlers from non-public or utility paths
User-agent: GPTBot
Disallow: /accept-invite/
Disallow: /login/
Disallow: /signup/


Sitemap: https://www.scratchpad.com/sitemap.xml