Chunking

burmesenlp.chunking.chunk

chunk(text: str, *, lexicon: Optional[Lexicon] = None, grammar: Optional[CompiledGrammar] = None, gazetteer: Optional[GazetteerManager] = None, use_gazetteer: bool = True) -> List[Chunk]

Normalize → segment → POS → optional gazetteer → phrase chunk.

When use_gazetteer is True (default), entity spans are locked as NP the same way as BurmeseNLP.process. Pass use_gazetteer=False for a fast syntax-only pass, or inject a preloaded GazetteerManager.

Source code in src/burmesenlp/chunking/chunker.py

def chunk(
    text: str,
    *,
    lexicon: Optional[Lexicon] = None,
    grammar: Optional[CompiledGrammar] = None,
    gazetteer: Optional[GazetteerManager] = None,
    use_gazetteer: bool = True,
) -> List[Chunk]:
    """Normalize → segment → POS → optional gazetteer → phrase chunk.

    When ``use_gazetteer`` is True (default), entity spans are locked as NP
    the same way as ``BurmeseNLP.process``. Pass ``use_gazetteer=False`` for
    a fast syntax-only pass, or inject a preloaded ``GazetteerManager``.
    """
    lex = lexicon if lexicon is not None else Lexicon.default()
    norm = normalize(text)
    words = [t.text for t in WordSegmenter(lex).segment(tokenize(norm))]
    tags = POSTagger(lex).tag(words)
    entities: List[GazetteerHit] = []
    if use_gazetteer or gazetteer is not None:
        gaz = gazetteer if gazetteer is not None else GazetteerManager(lexicon=lex)
        entities = list(gaz.find_all(words))
    return chunk_from_tokens(words, tags, grammar=grammar, entities=entities)

burmesenlp.chunking.chunk_from_tokens

chunk_from_tokens(words: Sequence[str], pos_tags: PosInput, *, grammar: Optional[CompiledGrammar] = None, entities: Optional[Sequence[GazetteerHit]] = None, clauses: bool = False, sentence_bounds: Optional[Sequence[SentenceBound]] = None) -> List[Chunk]

Source code in src/burmesenlp/chunking/chunker.py

def chunk_from_tokens(
    words: Sequence[str],
    pos_tags: PosInput,
    *,
    grammar: Optional[CompiledGrammar] = None,
    entities: Optional[Sequence[GazetteerHit]] = None,
    clauses: bool = False,
    sentence_bounds: Optional[Sequence[SentenceBound]] = None,
) -> List[Chunk]:
    return PhraseChunker(grammar).chunk(
        words,
        pos_tags,
        entities=entities,
        clauses=clauses,
        sentence_bounds=sentence_bounds,
    )

burmesenlp.chunking.PhraseChunker

PhraseChunker(grammar: Optional[CompiledGrammar] = None)

Shallow phrase chunker: consumes words + POS, never mutates tags.

Source code in src/burmesenlp/chunking/chunker.py

def __init__(self, grammar: Optional[CompiledGrammar] = None):
    self._grammar = grammar if grammar is not None else default_grammar()

chunk

chunk(words: Sequence[str], pos_tags: PosInput, *, entities: Optional[Sequence[GazetteerHit]] = None, clauses: bool = False, sentence_bounds: Optional[Sequence[SentenceBound]] = None) -> List[Chunk]

Chunk phrases, optionally locking gazetteer entity spans as NP.

Parameters

entities: Post-BMWE gazetteer hits. Each span is emitted as one NP with features["entity"] set, and is blocked from further POS pattern matching.

Source code in src/burmesenlp/chunking/chunker.py

def chunk(
    self,
    words: Sequence[str],
    pos_tags: PosInput,
    *,
    entities: Optional[Sequence[GazetteerHit]] = None,
    clauses: bool = False,
    sentence_bounds: Optional[Sequence[SentenceBound]] = None,
) -> List[Chunk]:
    """Chunk phrases, optionally locking gazetteer entity spans as NP.

    Parameters
    ----------
    entities:
        Post-BMWE gazetteer hits. Each span is emitted as one ``NP`` with
        ``features["entity"]`` set, and is blocked from further POS
        pattern matching.
    """
    del clauses, sentence_bounds  # API compat; clauses live in ClauseParser
    words = list(words)
    tags = normalize_pos_input(pos_tags)
    if len(words) != len(tags):
        raise ValueError(
            f"words/pos_tags length mismatch: {len(words)} != {len(tags)}"
        )
    if not words:
        return []

    covered = [False] * len(words)
    chunks: List[Chunk] = []
    # 1. Fixed expressions / greetings
    chunks.extend(self._match_exceptions(words, tags, covered))
    # 2. Gazetteer entities → locked NP (semantic → syntactic bridge)
    chunks.extend(self._lock_entities(words, tags, covered, entities or ()))
    # 3. POS phrase patterns on remaining tokens
    chunks.extend(self._match_phrases(words, tags, covered))
    chunks.sort(key=lambda c: (c.start, c.end, c.type.value))
    return chunks

split_clauses

split_clauses(words: Sequence[str], pos_tags: PosInput, *, sentence_bounds: Optional[Sequence[SentenceBound]] = None) -> List[Chunk]

Deprecated: clause overlays removed from the phrase chunker.

Returns []. Prefer :class:ClauseParser.

Source code in src/burmesenlp/chunking/chunker.py

def split_clauses(
    self,
    words: Sequence[str],
    pos_tags: PosInput,
    *,
    sentence_bounds: Optional[Sequence[SentenceBound]] = None,
) -> List[Chunk]:
    """Deprecated: clause overlays removed from the phrase chunker.

    Returns ``[]``. Prefer :class:`ClauseParser`.
    """
    del words, pos_tags, sentence_bounds
    return []

burmesenlp.chunking.models.Chunk dataclass

Chunk(type: ChunkType, text: str, tokens: List[str], pos_tags: List[str], start: int, end: int, features: Mapping[str, str] = dict())

A shallow phrase span over already-tagged tokens.

start / end are inclusive token indices into the input sequence.

burmesenlp.chunking.models.ChunkType

Bases: Enum