Skip to content

Tokenize

burmesenlp.tokenize.word_tokenize 

word_tokenize(text: str, engine: str = 'longest', *, lexicon: Optional[Lexicon] = None) -> List[str]

Segment text into words.

Today only engine="longest" is implemented. Future values such as "sentencepiece" or "evopiece" will plug into the same API.

Source code in src/burmesenlp/tokenize/word.py 
13
14
15
16
17
18
19
20
21
22
23
24
def word_tokenize(
    text: str,
    engine: str = "longest",
    *,
    lexicon: Optional[Lexicon] = None,
) -> List[str]:
    """Segment *text* into words.

    Today only ``engine=\"longest\"`` is implemented.  Future values such as
    ``\"sentencepiece\"`` or ``\"evopiece\"`` will plug into the same API.
    """
    return [t.text for t in run_word_engine(text, engine=engine, lexicon=lexicon)]

burmesenlp.tokenize.syllable_tokenize 

syllable_tokenize(text: str) -> List[str]

Public alias for syllable segmentation (engine API naming).

Source code in src/burmesenlp/tokenize/syllable.py 
127
128
129
def syllable_tokenize(text: str) -> List[str]:
    """Public alias for syllable segmentation (engine API naming)."""
    return syllable_segment(text)

burmesenlp.tokenize.sentence_tokenize 

sentence_tokenize(text: str, *, lexicon: Optional[Lexicon] = None, split_on_final_particles: bool = True, engine: str = 'longest') -> List[str]

Segment text into sentences via the grammar-aware pipeline.

Runs word tokenization (engine), BMWE, POS, and phrase chunking, then splits on chunk/POS structure (not bare သည်/တယ် matching).

Source code in src/burmesenlp/tokenize/__init__.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def sentence_tokenize(
    text: str,
    *,
    lexicon: Optional[Lexicon] = None,
    split_on_final_particles: bool = True,
    engine: str = "longest",
) -> List[str]:
    """Segment *text* into sentences via the grammar-aware pipeline.

    Runs word tokenization (*engine*), BMWE, POS, and phrase chunking,
    then splits on chunk/POS structure (not bare သည်/တယ် matching).
    """
    del engine  # word engine selection is owned by BurmeseNLP / longest today
    from ..pipeline import BurmeseNLP

    return BurmeseNLP(
        lexicon=lexicon,
        split_on_final_particles=split_on_final_particles,
    ).sentence_segment(text)

burmesenlp.tokenize.sentence.SentenceSegmenter 

SentenceSegmenter(split_on_final_particles: bool = True)

Chunk/POS-aware sentence segmenter.

Preferred entry point: :meth:segment_from_chunks. :meth:segment remains as a punctuation-only fallback when chunks are unavailable (legacy Token stream).

Source code in src/burmesenlp/tokenize/sentence.py 
126
127
128
def __init__(self, split_on_final_particles: bool = True):
    # Soft-split after completed VP before a new onset (no ။ required).
    self.split_on_final_particles = split_on_final_particles

segment_from_chunks 

segment_from_chunks(words: Sequence[str], pos_tags: Sequence[Union[str, Tuple[str, str]]], chunks: Sequence[Chunk], text: str, *, char_spans: Optional[Sequence[Tuple[int, int]]] = None) -> List[Sentence]

Segment using phrase chunks over words / pos_tags.

char_spans[i] is (start, end) into text for words[i]. When omitted, sentences are built by joining word strings (offsets are approximate / zero).

Source code in src/burmesenlp/tokenize/sentence.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def segment_from_chunks(
    self,
    words: Sequence[str],
    pos_tags: Sequence[Union[str, Tuple[str, str]]],
    chunks: Sequence[Chunk],
    text: str,
    *,
    char_spans: Optional[Sequence[Tuple[int, int]]] = None,
) -> List[Sentence]:
    """Segment using phrase chunks over *words* / *pos_tags*.

    *char_spans[i]* is ``(start, end)`` into *text* for ``words[i]``.
    When omitted, sentences are built by joining word strings (offsets
    are approximate / zero).
    """
    words = list(words)
    tags = [_tag_str(t) for t in pos_tags]
    if len(words) != len(tags):
        raise ValueError(
            f"words/pos_tags length mismatch: {len(words)} != {len(tags)}"
        )
    if not words:
        return []

    units = _covering_units(words, tags, chunks)
    # Exclusive word-index cuts (end of each sentence).
    cuts = self._boundary_cuts(units, words, tags)
    return _sentences_from_cuts(words, text, cuts, char_spans)

segment 

segment(word_tokens: Sequence[Token], text: str) -> List[Sentence]

Legacy fallback: punctuation-only splits over word Tokens.

Does not split on သည်/တယ်/ပါ. Prefer :meth:segment_from_chunks in the full pipeline.

Source code in src/burmesenlp/tokenize/sentence.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
def segment(self, word_tokens: Sequence[Token], text: str) -> List[Sentence]:
    """Legacy fallback: punctuation-only splits over word Tokens.

    Does **not** split on သည်/တယ်/ပါ.  Prefer
    :meth:`segment_from_chunks` in the full pipeline.
    """
    sentences: List[Sentence] = []
    current: List[Token] = []
    for w in word_tokens:
        current.append(w)
        if w.text == FULL_STOP or _is_terminal_punct(w.text):
            sentences.append(self._build_from_tokens(current, text))
            current = []
    if current:
        sentences.append(self._build_from_tokens(current, text))
    return sentences

burmesenlp.tokenize.longest.WordSegmenter 

WordSegmenter(lexicon: Lexicon)
Source code in src/burmesenlp/tokenize/longest.py 
30
31
def __init__(self, lexicon: Lexicon):
    self._lexicon = lexicon

segment 

segment(tokens: Sequence[Token]) -> List[Token]

Group syllable-level tokens into word-level tokens.

Source code in src/burmesenlp/tokenize/longest.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def segment(self, tokens: Sequence[Token]) -> List[Token]:
    """Group syllable-level tokens into word-level tokens."""
    words: List[Token] = []
    i = 0
    n = len(tokens)
    while i < n:
        tok = tokens[i]

        if tok.kind == DIGITS:
            nxt = tokens[i + 1] if i + 1 < n else None
            if (
                nxt is not None
                and nxt.kind == SYLLABLE
                and nxt.start == tok.end
                and nxt.text in grammar.COUNTER_CLASSIFIERS
            ):
                words.append(Token(tok.text + nxt.text, tok.start, nxt.end, WORD))
                i += 2
                continue
            words.append(tok)
            i += 1
            continue

        if tok.kind != SYLLABLE:
            words.append(tok)
            i += 1
            continue

        # Collect the contiguous run of adjacent syllables.
        j = i
        while (
            j + 1 < n
            and tokens[j + 1].kind == SYLLABLE
            and tokens[j + 1].start == tokens[j].end
        ):
            j += 1
        words.extend(self._match_run(tokens[i : j + 1]))
        i = j + 1

    return words