User-agent: *
Allow: /

# Allow all crawlers
User-agent: *
Allow: /llms.txt
Allow: /

# Specific directives for AI bots
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: FacebookBot
User-agent: GPTBot
User-agent: ImagesiftBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Omgilibot
User-agent: PerplexityBot
User-agent: Timpibot
User-agent: Google-Extended

# Directories to Disallow
Disallow: /includes/
Disallow: /Templates/
Disallow: /apc/
Disallow: /autoswitch/
Disallow: /editage_applet/

# Paths (no clean URLs)
Disallow: /insights/tutorials/writing-tips/
Disallow: /insights/node
Disallow: /insights/node/
Disallow: /insights/print/
Disallow: /insights/printmail/
Disallow: /insights/taxonomy/
Disallow: /insights/user/
Disallow: /insights/ask-dr-eddy/page/*
Disallow: /insights/send/*
Disallow: /insights/frontier*
Disallow: /insights/users/content_search/*
Disallow: /insights/tags/content_search/*
Disallow: /insights/categories/content_search/*
Disallow: /insights/*/ask-dr-eddy
Disallow: /insights/*/roohi-ghosh
Disallow: /insights/categories/quick-tips/content_search/*
Disallow: /insights/*/content_search/*
Disallow: /insights/archive/*
Disallow: /insights/print/*
Disallow: /insights/node/*
Disallow: /insights/taxonomy/term/*
Disallow: /insights/tags/*
Disallow: /insights/files/*
Disallow: /content_search/*
Disallow: /subjectarea/json/*
Disallow: /insights/ja/node/*
Disallow: /insights/ko/node/*
Disallow: /insights/zh-hans/node/*
Disallow: /*?*

#Sitemaps
Sitemap: https://www.editage.com/sitemap.xml
Sitemap: https://www.editage.com/insights/sitemap.xml
Sitemap: https://www.editage.com/journals-sitemap.xml
Sitemap: https://www.editage.com/blog-sitemap.xml
Sitemap: https://www.editage.com/subject-expertise-sitemap.xml