# Scriptura · scriptura.si
# 953 classic works in the original beside English, and every Bible side by side.
# People, search engines and AI agents are all warmly welcome here.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: MistralAI-Index
User-agent: ShapBot
User-agent: ExaSearchBot
User-agent: ExaUser
User-agent: DuckDuckBot
User-agent: MojeekBot
User-agent: search.marginalia.nu
User-agent: Yeti
User-agent: SeznamBot
User-agent: Qwantbot
# xAI (Grok) publishes no crawler names; these are the ones seen in the wild, welcomed by name. Twitterbot draws X link cards.
User-agent: GrokBot
User-agent: xAI-SearchBot
User-agent: xAI-Bot
User-agent: GrokAgent
User-agent: Grok-DeepSearch
User-agent: xAI-Grok
User-agent: Twitterbot
# China: Baidu, Sogou, 360 Search, Shenma, Huawei Petal (also its AI Search), ByteDance, and the AI labs Moonshot (Kimi) and Zhipu (ChatGLM).
User-agent: Baiduspider
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: 360Spider
User-agent: YisouSpider
User-agent: PetalBot
User-agent: Bytespider
User-agent: KimiBot
User-agent: Kimi-User
User-agent: ChatGLM-Spider
# DeepSeek (whose own search engine opened on 17 October 2026), Baidu ERNIE, ByteDance Doubao, Alibaba Qwen and Tongyi, and Huawei PanGu: names reported in the wild.
User-agent: DeepSeekBot
User-agent: ERNIEBot
User-agent: DoubaoBot
User-agent: QwenBot
User-agent: TongyiBot
User-agent: PanguBot
# Russia: Mail.ru. Yandex's robots, its AI answers included, are in the group below.
User-agent: Mail.RU_Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Amazonbot
User-agent: CCBot
User-agent: YandexBot
User-agent: YandexAdditionalBot
Allow: /

Sitemap: https://scriptura.si/sitemap.xml