# Default: search engines and everyone not listed below.
#
# The site is path-based on purpose: every browsable view of a hub is its own
# URL (/bucket-list/japan/castles/2), and middleware 308s the old ?section=
# / ?page= / ?sortBy= forms to it. Nothing worth indexing therefore carries a
# query string, so "Disallow: /*?" closes the whole parameter space in one
# line -- the ?filter=, ?letter=, ?topic= and ?callbackUrl= links in the page
# body were being crawled as separate URLs and reported as duplicates.
#
# The per-directory "Allow:" lines that used to sit here were removed: they
# were redundant next to "Allow: /", and a longer Allow beats a shorter
# Disallow, so "Allow: /bucket-list" would have overridden "Disallow: /*?".
User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /*?

# Signed-in surfaces: nothing here renders for a crawler.
Disallow: /dashboard
Disallow: /settings
Disallow: /profile/
Disallow: /edit/
Disallow: /onboarding
Disallow: /items/new
Disallow: /bucket-items/new
Disallow: /feedback
Disallow: /thank-you
Disallow: /beta
Disallow: /dev/
Disallow: /xvS6mDPbV2c/

# Per-goal subpages. Each one canonicalises to the goal itself, so they add
# nothing to the index -- but there are three of them for every one of ~8,000
# goals, which is most of the crawl queue.
Disallow: /items/*/history
Disallow: /items/*/photos
Disallow: /items/*/activity
Disallow: /bucket-list/*/edit

# AI answer engines we WANT to read and cite bucketly (OpenAI, Anthropic,
# Perplexity). A crawler uses its most specific group and ignores "*", so the
# same rules are repeated here on purpose. Keep in sync with the Vercel
# Firewall bypass rule "Allow AI answer engines".
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /*?
Disallow: /dashboard
Disallow: /settings
Disallow: /profile/
Disallow: /edit/
Disallow: /onboarding
Disallow: /items/new
Disallow: /bucket-items/new
Disallow: /feedback
Disallow: /thank-you
Disallow: /beta
Disallow: /dev/
Disallow: /xvS6mDPbV2c/
Disallow: /items/*/history
Disallow: /items/*/photos
Disallow: /items/*/activity
Disallow: /bucket-list/*/edit

# Everyone else that crawls at scale and sends no visitors: other AI training
# scrapers and SEO-tool crawlers. Polite ones stop here; the Vercel Firewall
# (AI Bots = Deny, custom "Deny SEO scrapers" rule) handles the rest.
User-agent: Bytespider
User-agent: CCBot
User-agent: Amazonbot
User-agent: PetalBot
User-agent: Meta-ExternalAgent
User-agent: Applebot-Extended
User-agent: AI2Bot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: omgili
User-agent: YouBot
User-agent: cohere-ai
User-agent: Timpibot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSEOBot
User-agent: BLEXBot
User-agent: serpstatbot
User-agent: Barkrowler
Disallow: /

Sitemap: https://bucketly.co/sitemap.xml