# robots.txt — https://americanglassanddoors.com/
# American Glass and Doors — Washington DC, Maryland, Northern Virginia
# Last updated: 2026-09-17
#
# Policy: this site wants to be crawled, indexed, quoted and cited — by
# classic search engines and by AI answer engines alike. Everything public
# is open. Only admin endpoints and thin, duplicate query-string URLs are
# closed.
#
# Machine-readable site summary: https://americanglassanddoors.com/llms.txt
#
# Three groups below. A crawler obeys only the single most specific group
# that names it, never a combination, so each group is written out in full.

# ---------------------------------------------------------------------
# 1. Default — every crawler not named in a group below.
#    Includes Googlebot, Bingbot and every other classic search engine.
# ---------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /wp-json/wp/v2/users
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?msclkid=

# Note: /wp-includes/ and /wp-content/ are deliberately NOT blocked.
# Blocking them hides the CSS and JavaScript the page needs to render,
# and a crawler that cannot render a page cannot judge it.

# ---------------------------------------------------------------------
# 2. AI crawlers and answer-engine indexers — explicitly allowed.
#
#    Same rules as the default group. Spelled out so there is no
#    ambiguity about consent: this business wants its services, coverage
#    and contact details used in AI answers.
#
#    Google-Extended and Applebot-Extended are not crawlers — they are
#    opt-in tokens for Gemini and Apple Intelligence. Listing them with
#    Allow: / is how a site opts in.
# ---------------------------------------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: PerplexityBot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: Ai2Bot
User-agent: Ai2Bot-Dolma
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: YouBot
User-agent: CCBot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: PetalBot
User-agent: Diffbot
User-agent: Timpibot
User-agent: omgilibot
User-agent: omgili
User-agent: Webzio-Extended
User-agent: ImagesiftBot
User-agent: Kangaroo Bot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /wp-json/wp/v2/users
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?msclkid=

# ---------------------------------------------------------------------
# 3. User-triggered AI fetchers — allowed everything public.
#
#    These do not crawl. They fetch one URL because a person asked about
#    it in a chat, so crawl budget is irrelevant and the tracking-
#    parameter blocks above would only break real links: an ad or email
#    URL carrying ?utm_source= is exactly the kind a customer pastes.
#    Blocking it would make the assistant report the site unreachable.
# ---------------------------------------------------------------------
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: meta-externalfetcher
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /wp-json/wp/v2/users

# ---------------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------------
Sitemap: https://americanglassanddoors.com/sitemap_index.xml