{"domain":"indeed.com","date":"2026-10-28","source":"https://infrarabbit.com/crawlers/domain/indeed.com","license":"CC BY 4.0","observed":true,"robots_txt":{"state":"ok","http":200,"url":"https://www.indeed.com/robots.txt","sha256":"1189922fa6724d518dfa4f636276c1f8cd5397f2fbd7ab997183c4a0fd6391a4","unchanged_since":"2026-10-26","observed_through":"2026-10-28"},"known":true,"bots":{"GPTBot":{"verdict":"allow","via":"named"},"OAI-SearchBot":{"verdict":"allow","via":"named"},"ChatGPT-User":{"verdict":"allow","via":"named"},"ClaudeBot":{"verdict":"allow","via":"named"},"Claude-SearchBot":{"verdict":"allow","via":"named"},"Claude-User":{"verdict":"allow","via":"named"},"PerplexityBot":{"verdict":"allow","via":"named"},"Perplexity-User":{"verdict":"allow","via":"named"},"Googlebot":{"verdict":"allow","via":"named"},"Google-Extended":{"verdict":"allow","via":"named"},"Applebot-Extended":{"verdict":"allow","via":"named"},"CCBot":{"verdict":"allow","via":"named"},"Bytespider":{"verdict":"allow","via":"named"},"Meta-ExternalAgent":{"verdict":"allow","via":"named"},"Amazonbot":{"verdict":"allow","via":"named"},"anthropic-ai":{"verdict":"allow","via":"named"},"cohere-training-data-crawler":{"verdict":"allow","via":"named"},"DeepSeekBot":{"verdict":"allow","via":"named"},"Diffbot":{"verdict":"allow","via":"named"},"DuckAssistBot":{"verdict":"allow","via":"named"},"FacebookBot":{"verdict":"allow","via":"named"},"facebookexternalhit":{"verdict":"allow","via":"named"},"FriendlyCrawler":{"verdict":"allow","via":"named"},"Gemini-Deep-Research":{"verdict":"allow","via":"named"},"Google-Agent":{"verdict":"allow","via":"named"},"Google-CloudVertexBot":{"verdict":"allow","via":"named"},"GoogleOther":{"verdict":"block","via":"named"},"GrokBot":{"verdict":"allow","via":"named"},"meta-externalfetcher":{"verdict":"allow","via":"named"},"meta-webindexer":{"verdict":"allow","via":"named"},"MyCentralAIScraperBot":{"verdict":"block","via":"named"},"omgili":{"verdict":"block","via":"named"},"omgilibot":{"verdict":"block","via":"named"},"PanguBot":{"verdict":"allow","via":"named"},"PetalBot":{"verdict":"allow","via":"named"},"Scrapy":{"verdict":"block","via":"named"},"Timpibot":{"verdict":"allow","via":"named"},"Webzio-Extended":{"verdict":"block","via":"named"},"xAI-Grok":{"verdict":"allow","via":"named"},"YouBot":{"verdict":"allow","via":"named"}},"wildcard":"allow","blocked_headline_bots":[],"partial_headline_bots":[],"signals":{"llms":null,"aitxt":null,"tdmrep":null,"tdm_reserved":null,"content_signals":null,"ai_train_no":null,"rsl":null,"x402":null,"payment_required":null,"noai_meta":null,"fetch_blocked":false},"content_signals":[],"rsl_licenses":[],"llms_txt":null,"tdmrep":null,"homepage":{"state":"fetch-blocked","tdm_header":false,"tdm_meta":false,"tdm_policy":null,"noai":[],"x402":false,"crawler_price":null},"summary":"On 28 October 2026, indeed.com's robots.txt allowed all of GPTBot, OAI-SearchBot, ClaudeBot, Claude-SearchBot, Google-Extended, PerplexityBot and CCBot.","evidence":{"merkle_root":"c23b27aee031894dbae09c8b70209d28255ed9fc8fdeb87fe880740686318b2b","opentimestamps":"pending (4 calendars)","proof":"https://infrarabbit.com/crawlers/proofs/2026-10-28"}}