{"id":"regex-tokenization-check","version":"1.0.0","description":"Checks claimed regex token texts and UTF-16 spans, returning the first mismatching token.","supported_operations":["check regex token claim","validate tokenizer spans","verify token offsets","compare regex tokens","find first token mismatch","check lexical match spans"],"unsupported_operations":["infer a tokenizer grammar","repair token positions"],"semantics":["Every non-overlapping regex match is a token with exact text and a UTF-16 half-open span.","Tokens compare in source match order; duplicate and empty tokens remain distinct."],"limits":{"max_input_bytes":65536,"max_pattern_bytes":256,"max_claim_items":4096,"max_matches":4096,"max_result_bytes":262144},"pricing":{"status":"unpriced","charge_usd":null},"input_schema":{"type":"object","additionalProperties":false,"properties":{"text":{"type":"string","maxLength":65536},"pattern":{"type":"string","minLength":1,"maxLength":256},"flags":{"type":"string","maxLength":2},"claimed_tokens":{"type":"array","maxItems":4096,"items":{"type":"object","additionalProperties":false,"properties":{"text":{"type":"string","maxLength":65536},"start":{"type":"integer","minimum":0},"end":{"type":"integer","minimum":0}},"required":["text","start","end"]}}},"required":["text","pattern","flags","claimed_tokens"]},"output_schema":{"type":"object","additionalProperties":false,"properties":{"valid":{"type":"boolean"},"reason":{"type":"string"},"discrepancy":{"type":"object","additionalProperties":false,"properties":{"path":{"type":"string"},"index":{"type":"integer","minimum":0},"expected":{"anyOf":[{"type":"object","additionalProperties":false,"properties":{"text":{"type":"string"},"start":{"type":"integer","minimum":0},"end":{"type":"integer","minimum":0}},"required":["text","start","end"]},{"type":"null"}]},"actual":{"anyOf":[{"type":"object","additionalProperties":false,"properties":{"text":{"type":"string"},"start":{"type":"integer","minimum":0},"end":{"type":"integer","minimum":0}},"required":["text","start","end"]},{"type":"null"}]}},"required":["path","index","expected","actual"]},"reference_version":{"type":"string"}},"required":["valid","reference_version"]},"examples":[{"input":{"text":"ab 12","pattern":"\\w+","flags":"","claimed_tokens":[{"text":"ab","start":0,"end":2},{"text":"12","start":3,"end":5}]},"output":{"valid":true,"reference_version":"RE2-syntax-2024-07; ECMA-262-2025-index-and-replacement"}},{"input":{"text":"😀x","pattern":".","flags":"","claimed_tokens":[{"text":"😀","start":0,"end":2},{"text":"x","start":2,"end":3}]},"output":{"valid":true,"reference_version":"RE2-syntax-2024-07; ECMA-262-2025-index-and-replacement"}}],"execute_url":"/v1/tools/regex-tokenization-check/versions/1.0.0/execute"}