{"id":"text-sentence-offsets","version":"1.0.0","description":"Return UAX #29 sentence spans as start/end offsets in code points, UTF-16 units or UTF-8 bytes, with the ending rule.","supported_operations":["sentence start and end offsets","sentence boundary positions in utf-8 bytes","character offsets of each sentence for highlighting","utf-16 sentence spans for javascript strings","which rule ended each sentence","map sentences to code point ranges"],"unsupported_operations":["returning the sentence text (use text-sentences-split)","grapheme cluster, word or line break offsets","offsets after Unicode normalization of the input","language-tailored sentence boundaries"],"semantics":["Boundaries are the default UAX #29 (revision 47) sentence boundaries over Unicode 17.0.0 Sentence_Break data (data_version unicode-17.0.0), identical to text-sentences-split with trim false.","unit (optional, default \"code_point\"): \"code_point\" counts Unicode scalar values (Python str indices), \"utf16\" counts UTF-16 code units (JavaScript string indices), \"utf8\" counts UTF-8 bytes.","spans are half-open [start, end), contiguous, cover the whole text in order, and the last end equals length.","break_rule names the UAX #29 rule that created the break ending the span: \"SB4\" after a paragraph separator (Sep, CR, LF; CR LF is one), \"SB11\" after a sentence terminator sequence SATerm Close* Sp*, \"SB2\" at the end of text. These match the [4.0], [11.0] and [0.3] markers of SentenceBreakTest-17.0.0.txt.","Empty text returns no spans and length 0; text is limited to 262144 UTF-8 bytes and 65536 sentences."],"limits":{"max_text_bytes":262144,"max_sentences":65536},"pricing":{"status":"unpriced","charge_usd":null},"input_schema":{"type":"object","additionalProperties":false,"required":["text"],"properties":{"text":{"type":"string","maxLength":262144,"description":"Text to segment (at most 262144 UTF-8 bytes)."},"unit":{"type":"string","enum":["code_point","utf16","utf8"],"description":"Optional; default \"code_point\"."}}},"output_schema":{"type":"object","additionalProperties":false,"required":["data_version","unit","length","count","spans"],"properties":{"data_version":{"type":"string","const":"unicode-17.0.0"},"unit":{"type":"string","enum":["code_point","utf16","utf8"]},"length":{"type":"integer","minimum":0},"count":{"type":"integer","minimum":0,"maximum":65536},"spans":{"type":"array","maxItems":65536,"items":{"type":"object","additionalProperties":false,"required":["start","end","break_rule"],"properties":{"start":{"type":"integer","minimum":0},"end":{"type":"integer","minimum":1},"break_rule":{"type":"string","enum":["SB2","SB4","SB11"]}}}}}},"examples":[{"input":{"text":"a. (3) b. Ça va?\nOui."},"output":{"data_version":"unicode-17.0.0","unit":"code_point","length":21,"count":3,"spans":[{"start":0,"end":10,"break_rule":"SB11"},{"start":10,"end":17,"break_rule":"SB4"},{"start":17,"end":21,"break_rule":"SB2"}]}},{"input":{"text":"a. (3) b. Ça va?\nOui.","unit":"utf8"},"output":{"data_version":"unicode-17.0.0","unit":"utf8","length":22,"count":3,"spans":[{"start":0,"end":10,"break_rule":"SB11"},{"start":10,"end":18,"break_rule":"SB4"},{"start":18,"end":22,"break_rule":"SB2"}]}}],"execute_url":"/v1/tools/text-sentence-offsets/versions/1.0.0/execute"}