Gazetteer

burmesenlp.gazetteer.GazetteerManager

GazetteerManager(lexicon: Optional[Lexicon] = None, *, autoload: bool = True, root: Optional[Path] = None)

Load gazetteer JSON files and provide contains / lookup / match APIs.

Used by BurmeseNLP.process() (after POS) to fill Document.entities. First full load tokenizes every surface (including large village lists) and may take noticeable time; pass gazetteer=False to skip.

Source code in src/burmesenlp/gazetteer/manager.py

def __init__(
    self,
    lexicon: Optional[Lexicon] = None,
    *,
    autoload: bool = True,
    root: Optional[Path] = None,
):
    self._lexicon = lexicon if lexicon is not None else Lexicon.default()
    self._root = Path(root) if root is not None else _GAZETTEER_DIR
    self._trie: TokenTrie[GazetteerHit] = TokenTrie()
    self._by_norm: Dict[str, List[GazetteerHit]] = {}
    self._holiday_attrs: Dict[str, Dict] = {}
    self._count = 0
    if autoload and self._root.is_dir():
        self.load(self._root)

load

load(path: Optional[Union[Path, str]] = None) -> int

Load a gazetteer directory or a single JSON file. Returns entries added.

Source code in src/burmesenlp/gazetteer/manager.py

def load(self, path: Optional[Union[Path, str]] = None) -> int:
    """Load a gazetteer directory or a single JSON file. Returns entries added."""
    target = Path(path) if path is not None else self._root
    if target.is_file():
        return self._load_file(target)
    if not target.is_dir():
        raise FileNotFoundError(f"Gazetteer path not found: {target}")
    added = 0
    for fp in sorted(target.glob("*.json")):
        if fp.name == "metadata.json":
            continue
        added += self._load_file(fp)
    return added

longest_match

longest_match(tokens: Sequence[str], start: int = 0) -> Optional[GazetteerHit]

Longest trie hit starting at start, or None.

Source code in src/burmesenlp/gazetteer/manager.py

def longest_match(
    self,
    tokens: Sequence[str],
    start: int = 0,
) -> Optional[GazetteerHit]:
    """Longest trie hit starting at *start*, or None."""
    hits = self._trie.search(tokens, start)
    if not hits:
        return None
    best = max(hits, key=lambda h: len(h.tokens))
    end = start + len(best.tokens) - 1
    return GazetteerHit(
        text=best.text,
        tokens=best.tokens,
        entity_type=best.entity_type,
        start=start,
        end=end,
        attributes=dict(best.attributes),
    )

find_all

find_all(tokens: Sequence[str], *, pos_tags: Optional[Sequence[str]] = None) -> List[GazetteerHit]

Greedy left-to-right longest matches over tokens.

PERSON hits absorb a preceding honorific token when present (ဦး / ဒေါ် / …). Also matches a single token that fuses honorific + name (ဒေါ်အောင်ဆန်းစုကြည်).

Short geographic hits (1–2 syllables) are rejected unless a locative cue is nearby, or — for 2-syllable names — the span's POS looks nominal. Pass post-BMWE pos_tags aligned with tokens.

Source code in src/burmesenlp/gazetteer/manager.py

def find_all(
    self,
    tokens: Sequence[str],
    *,
    pos_tags: Optional[Sequence[str]] = None,
) -> List[GazetteerHit]:
    """Greedy left-to-right longest matches over *tokens*.

    PERSON hits absorb a preceding honorific token when present
    (``ဦး`` / ``ဒေါ်`` / …). Also matches a single token that fuses
    honorific + name (``ဒေါ်အောင်ဆန်းစုကြည်``).

    Short geographic hits (1–2 syllables) are rejected unless a locative
    cue is nearby, or — for 2-syllable names — the span's POS looks
    nominal. Pass post-BMWE ``pos_tags`` aligned with *tokens*.
    """
    tags = list(pos_tags) if pos_tags is not None else None
    if tags is not None and len(tags) != len(tokens):
        raise ValueError(
            f"tokens/pos_tags length mismatch: {len(tokens)} != {len(tags)}"
        )
    out: List[GazetteerHit] = []
    i = 0
    n = len(tokens)
    while i < n:
        hit = self.longest_match(tokens, i)
        if hit is None:
            hit = self._match_fused_person_honorific(tokens, i)
        if hit is None:
            i += 1
            continue
        if hit.entity_type == EntityType.PERSON:
            hit = _maybe_attach_honorific(tokens, hit)
        if not _accept_hit(hit, tokens, tags):
            # Do not consume the span — try a longer start next token.
            i += 1
            continue
        out.append(hit)
        i = hit.end + 1
    return out

burmesenlp.gazetteer.EntityType

Bases: Enum

burmesenlp.gazetteer.GazetteerHit dataclass

GazetteerHit(text: str, tokens: Tuple[str, ...], entity_type: EntityType, start: int = 0, end: int = 0, attributes: Dict[str, object] = dict())

A matched gazetteer surface form over a token sequence.