# ============================================
# FLY SHOPS - ROBOTS.TXT
# Independent Travel Booking Agency
# Last Updated: 2024-01-15
# ============================================

# --------------------------
# GENERAL RULES
# --------------------------

# Allow all well-behaved crawlers
User-agent: *
Allow: /

# Be polite - wait between requests
Crawl-delay: 5

# --------------------------
# SITEMAP LOCATION
# --------------------------
Sitemap: https://flyshops.co.uk/sitemap.xml

# --------------------------
# MAJOR SEARCH ENGINES
# --------------------------

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 2

# Google Image
User-agent: Googlebot-Image
Allow: /img/
Allow: /images/
Disallow: /*.pdf$

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 2

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 2
Host: https://flyshops.co.uk

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 5

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

# --------------------------
# SOCIAL MEDIA CRAWLERS
# --------------------------

# Facebook
User-agent: facebookexternalhit
Allow: /

# Twitter
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Pinterest
User-agent: Pinterestbot
Allow: /

# --------------------------
# ARCHIVE & TOOL CRAWLERS
# --------------------------

# Internet Archive
User-agent: ia_archiver
Allow: /

# Archive.org
User-agent: archive.org_bot
Allow: /

# --------------------------
# DISALLOWED PATHS (IF NEEDED)
# --------------------------
# Uncomment these if you have sensitive areas

# Disallow: /admin
# Disallow: /private
# Disallow: /temp
# Disallow: /backup
# Disallow: /logs
# Disallow: /includes/
# Disallow: /misc/
# Disallow: /scripts/

# --------------------------
# FILE TYPE RESTRICTIONS
# --------------------------
Disallow: /*.php$
Disallow: /*.cgi$
Disallow: /*.pl$
Disallow: /*.old$
Disallow: /*.bak$
Disallow: /*.tmp$

# --------------------------
# HOSTING ENVIRONMENT
# --------------------------
# Prevent crawling of staging/development environments
# Uncomment if needed:
# Disallow: /staging/
# Disallow: /dev/
# Disallow: /test/

# --------------------------
# END OF ROBOTS.TXT
# --------------------------