# AUTH-615: the public pages are open to every crawler; the app, its API and its sign-in paths, and /invest are not.
# AUTH-726: search engines and AI assistants are named too, so a crawler that only follows rules addressed to it by name
# still reads the public pages. They are in the SAME group as *, so they get exactly the same Disallow lines: a crawler
# obeys only the most specific group that names it, and a separate "Allow: /" group would open the app to it.
User-agent: *
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: DuckDuckBot
Allow: /
# /app alone would also block /apple-touch-icon.png (robots rules are prefixes), so the app is /app exactly and /app/...
Disallow: /app$
Disallow: /app/
Disallow: /app?
Disallow: /api/
Disallow: /invest
Disallow: /explore
Disallow: /callback
Disallow: /wiki/
Disallow: /connectors/
Disallow: /corpus/
Disallow: /registry/
Disallow: /budget/
Disallow: /mcp
Disallow: /oauth/
Disallow: /waitlist

Sitemap: https://memlo.ai/sitemap.xml