# ======================
# Core Crawl Rules
# ======================


User-agent: *
Disallow: /sitemap.xml
Disallow: /*?s=*
Disallow: /*&s=*
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*
Disallow: */sem/
Disallow: */campaign/
Disallow: */thank-you

# Language folders to block
Disallow: /au/au/
Disallow: /au/fr/
Disallow: /au/da/
Disallow: /au/de/
Disallow: /au/nb/
Disallow: /au/nl/
Disallow: /au/it/
Disallow: /au/ru/
Disallow: /au/es/
Disallow: /au/sv/
Disallow: /au/pt/
Disallow: /au/fi/
Disallow: /au/m/
Disallow: /m/au/
Disallow: /au/ja/
Disallow: /fr/au/
Disallow: /fr/it/
Disallow: /fr/de/
Disallow: /fr/nl/
Disallow: /fr/en/
Disallow: /fr/es/
Disallow: /es/au/
Disallow: /es/it/
Disallow: /es/de/
Disallow: /es/nl/
Disallow: /es/en/
Disallow: /es/fr/
Disallow: /it/au/
Disallow: /it/it/
Disallow: /it/de/
Disallow: /it/nl/
Disallow: /it/en/
Disallow: /it/fr/
Disallow: /de/au/
Disallow: /de/it/
Disallow: /de/nl/
Disallow: /de/fr/
Disallow: /de/es/
Disallow: /de/en/
Disallow: /nl/au/
Disallow: /nl/de/
Disallow: /nl/fr/
Disallow: /nl/es/
Disallow: /nl/en/
Disallow: /nl/it/
Disallow: /sv/au/
Disallow: /nb/au/
Disallow: /da/au/
Disallow: /ru/au/
Disallow: /fi/au/
Disallow: /pt/au/

# ======================
# Allowed AI + Search Bots
# ======================


User-agent: GPTBot

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ChatGPT-User

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Googlebot

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Bingbot

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Anthropic-AI

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: PerplexityBot

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Google-Extended

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: CCBot

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: cohere-ai

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: SemrushBot

# ======================
# Blocked Bots (Scrapers, Low Value)
# ======================

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: AhrefsBot
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: MJ12bot
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Baiduspider
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: YandexBot
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: DotBot
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Bytespider
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: NeevaBot
Disallow: /

# ======================
# Sitemap(s)
# ======================

Sitemap: https://www.gonitro.com/hubfs/sitemap/sitemap-index.xml

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*