# SignalGigs — AI Training Jobs
# Policy: search + retrieval bots allowed; foundation-model training
# bots disallowed. See /llms.txt and /llms-full.txt for what to cite.

# ---- Default rules for everyone ----
User-agent: *
Allow: /
# Explicit allow for Next.js static bundles — Googlebot needs to fetch
# JS and CSS chunks to render the page. A bare /_next disallow blocks
# rendering and drops ranking; allowing /_next/static/ while blocking
# the /_next/data/ internal RSC payloads keeps noise out of the index
# without breaking the visual render.
Allow: /_next/static/
Allow: /_next/image
Disallow: /_next/data/
Disallow: /admin
Disallow: /api
Disallow: /.env
Disallow: /.claude
Crawl-delay: 1

# ---- Traditional search crawlers (REQUIRED — don't block) ----
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Brave-Search-Bot
Allow: /

User-agent: NaverBot
Allow: /

User-agent: Yeti
Allow: /

# ---- AI SEARCH / RETRIEVAL bots — allowed ----
# These fetch pages at query time and cite with links. Friendly channel.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

# ---- AI grounding / Overviews bots — ALLOWED (discovery > training-opt-out) ----
# Reversed 2026-06-07: with Google organic flatlined post-cliff, AI assistants
# are already the #2 inbound channel. Google-Extended gates Gemini / Vertex
# grounding (an AI-answer surface that links back); CCBot (Common Crawl) feeds
# many retrieval/grounding corpora. We now prioritize being grounded + cited
# over the editorial-training opt-out. Pure foundation-model trainers with no
# discovery upside (GPTBot, ClaudeBot, Bytespider, Meta/Facebook) stay blocked.
User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# ---- Pure foundation-model TRAINING bots — still disallowed ----
# These ingest for training with no query-time citation/link-back. The
# retrieval/search siblings above (OAI-SearchBot, Claude-SearchBot, etc.)
# are what actually drive AI-assistant traffic and remain allowed.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

# ---- Sitemaps ----
Sitemap: https://signalgigs.com/sitemap.xml
Sitemap: https://signalgigs.com/feed/jobs.xml
# Google News sitemap — only contains blog posts published in the last 48h.
# Submit https://signalgigs.com/news-sitemap.xml in Publisher Center after
# the publisher application is approved.
Sitemap: https://signalgigs.com/news-sitemap.xml