AI Crawler Index

{'type': 'object', 'examples': [{'since': '0'}], 'properties': {'limit': {'type': 'integer', 'maximum': 400, 'minimum': 1, 'description': 'Events per page, default 100.'}, 'since': {'type': 'string', 'description': 'Cursor from the last result, or an ISO date. Omit for all retained.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{'user_agent': 'GPTBot/1.2'}], 'required': ['user_agent'], 'properties': {'user_agent': {'type': 'string', 'description': 'Raw User-Agent header value.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{}], 'required': [], 'properties': {}, 'additionalProperties': False}

{'type': 'object', 'required': ['ran', 'input_came_from', 'what_it_shows', 'answer', 'reproduce', 'this_is_not_a_mock', 'answered_by', 'license'], 'properties': {'ran': {'type': 'object', 'description': 'The tool name and the exact arguments that were run.'}, 'answer': {'type': 'object', 'description': 'The real structuredContent of that call, not a mock.'}, 'license': {'type': 'string'}, 'reproduce': {'type': 'string', 'description': 'A command that reproduces this answer.'}, 'answered_by': {'type': 'object'}, 'what_it_shows': {'type': 'string'}, 'input_came_from': {'type': 'string', 'description': "Where the canned input came from — always this host's own data."}, 'this_is_not_a_mock': {'type': 'string'}}, 'description': "This server's own worked example, executed for real on a canned input from this host's own data.", 'additionalProperties': True}

{'type': 'object', 'examples': [{'stance': 'block-ai-training'}], 'properties': {'stance': {'type': 'string', 'description': 'One of allow-all, block-ai-training, block-all-ai, block-datasets, allow-ai-search-only, block-seo-tools, block-disputed, maximum-ai-visibility.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{'ip': '20.171.206.5'}], 'required': ['ip'], 'properties': {'ip': {'type': 'string', 'description': 'IPv4 or IPv6 address to check.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{'limit': 20, 'category': 'ai-training'}], 'properties': {'q': {'type': 'string', 'description': 'Free text over slug, name and operator.'}, 'limit': {'type': 'integer', 'maximum': 200, 'minimum': 1, 'description': 'Max rows, 1-200 (default 100).'}, 'category': {'type': 'string', 'description': 'ai-training, ai-search, user-fetch, dataset, search, seo, archive, preview or tool.'}, 'operator': {'type': 'string', 'description': 'Operator slug or name.'}, 'verification': {'type': 'string', 'description': 'published-ranges, reverse-dns or none.'}, 'respects_robots_txt': {'type': 'string', 'description': 'documented, disputed, by-design-no or n-a.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{'slug': 'claudebot'}], 'required': ['slug'], 'properties': {'slug': {'type': 'string', 'description': 'Crawler slug, name or robots.txt token.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{'since': '2026-08-01'}], 'properties': {'since': {'type': 'string', 'description': 'ISO-8601 date or timestamp; omit for the current state.'}}, 'additionalProperties': False}

{'type': 'object', 'examples': [{}], 'required': [], 'properties': {}, 'additionalProperties': False}

{'type': 'object', 'required': ['you', 'we_book_you_as', 'we_have_seen_you', 'answered_by', 'this_call_touched', 'caveats', 'license', 'independent'], 'properties': {'you': {'type': 'object', 'description': 'The user-agent you sent and the address you came from.'}, 'caveats': {'type': 'array', 'items': {'type': 'string'}, 'description': 'What this answer does NOT establish — a user-agent is a claim.'}, 'license': {'type': 'string'}, 'answered_by': {'type': 'object', 'description': 'Which server answered, at which endpoint.'}, 'independent': {'type': 'boolean', 'description': 'This host is independent and unaffiliated.'}, 'we_book_you_as': {'type': 'object', 'description': "The class this host's own instrument records for that user-agent."}, 'we_have_seen_you': {'type': 'object', 'description': 'Whether this user-agent appears in the published observation window.'}, 'this_call_touched': {'type': 'object', 'description': 'Exactly which files were read to answer. No third party is contacted.'}}, 'description': 'Facts about the caller, derived only from the headers of this request and from files this host already publishes.', 'additionalProperties': True}