# Allow all crawlers to access the site
User-agent: *
Disallow: /.sw_/_host_/_replacer_/ # Block specific directory
Allow: / # Allow everything else
# Provide sitemap for better indexing
Sitemap: https://tag9inc.com/sitemap.xml
# Specific directives for major search engines
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Yandex
Allow: /
User-agent: Baiduspider
Allow: /
# Other bots (general bots or lesser-known search engines)
User-agent: CCBot
Allow: /
User-agent: PetalBot
Allow: /
User-agent: Exabot
Allow: /