# Liveupx Website Robots.txt - Enhanced for SEO & AI Crawler Optimization
# Last Updated: 2025-04-12

User-agent: *
Allow: /

# Sitemap
Sitemap: https://liveupx.com/sitemap.xml

# Disallow patterns
Disallow: /admin/
Disallow: /private/
Disallow: /cdn-cgi/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /*.php$
Disallow: /*?s=*

# No crawl delay for most bots to ensure fast indexing
# Specific crawl-delay only for heavy crawlers
User-agent: AhrefsBot
Crawl-delay: 5

# Explicitly allow major search engines with all permissions
User-agent: Googlebot
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: Bingbot
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

# AI & LLM Crawlers - Enhanced Permissions for Gen AI Understanding
User-agent: GPTBot
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: CCBot
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: ChatGPT-User
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: cohere-ai
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: Claude-Web
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: Google-Extended
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: FacebookBot
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

User-agent: Omgilibot
Allow: /
Allow: /services/
Allow: /industries/
Allow: /about/
Allow: /careers/
Allow: /blog/
Allow: /case-studies/
Allow: /products/
Allow: /events/
Allow: /whitepapers/
Allow: /portfolio/
Allow: /contact/

# Allow image & asset indexing for all bots
User-agent: Googlebot-Image
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /