# Robots.txt for TUSKO Industrial Technology
# Optimized for Google and Yandex
# https://www.tusko.com.cn

# ===========================================
# AI SEARCH ENGINES & LLM CRAWLERS
# ===========================================

# ChatGPT (OpenAI)
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

# Claude (Anthropic)
User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

# Google AI (Gemini, etc.)
User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: GPT-User
Allow: /
Crawl-delay: 2

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

# Microsoft Copilot/Bing AI
User-agent: CopilotBot
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# DuckDuckGo AI
User-agent: DuckDuckBot
Allow: /

# Common Crawl (for AI training)
User-agent: CCBot
Allow: /
Crawl-delay: 2

# ===========================================
# USER-AGENTS
# ===========================================

User-agent: *
# Allow all crawlers by default
Allow: /

# Googlebot specific rules
User-agent: Googlebot
Allow: /
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.pdf$
Crawl-delay: 1

# Googlebot-Image
User-agent: Googlebot-Image
Allow: /
Allow: /assets/
Allow: /images/
Allow: /products/
Crawl-delay: 1

# Googlebot-News (if applicable)
User-agent: Googlebot-News
Allow: /

# YandexBot
User-agent: Yandex
Allow: /
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.pdf$
Crawl-delay: 1

# YandexImages
User-agent: YandexImages
Allow: /
Allow: /assets/
Allow: /images/
Allow: /products/

# BingBot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# DuckDuckBot
User-agent: DuckDuckBot
Allow: /

# BaiduSpider
User-agent: Baiduspider
Allow: /
Crawl-delay: 1

# Sogou Spider
User-agent: Sogou web spider
Allow: /
Crawl-delay: 1

# ===========================================
# SITEMAP LOCATIONS
# ===========================================

Sitemap: https://www.tusko.com.cn/sitemap.xml

# ===========================================
# CRAWL DELAY (General)
# ===========================================

# ===========================================
# ACCESS RESTRICTIONS
# ===========================================

# Block access to private/admin areas (if exist)
# Disallow: /admin/
# Disallow: /cgi-bin/
# Disallow: /private/
# Disallow: /tmp/

# Block access to duplicate content
# Disallow: /search?
# Disallow: /filter?

# Block access to low-value pages
# Disallow: /404.html
# Disallow: /thank-you.html

# ===========================================
# PERFORMANCE OPTIMIZATION
# ===========================================

# Clean parameters (optional - for Google)
# Clean-param: ref /products?

# ===========================================
# LANGUAGE TAGS
# ===========================================

# Tell search engines about alternate language versions
# (Already handled via hreflang in HTML, but good to note here)

# ===========================================
# CACHE CONTROL
# ===========================================

# Control caching (handled via HTTP headers, not robots.txt)

# ===========================================
# END OF ROBOTS.TXT
# ===========================================