Lexicon

burmesenlp.lexicon.Lexicon

Lexicon(entries: Mapping[str, Iterable[str]])

Word -> POS-tags mapping with syllable-aware longest-match support.

Source code in src/burmesenlp/lexicon/__init__.py

def __init__(self, entries: Mapping[str, Iterable[str]]):
    self._entries: Dict[str, Tuple[str, ...]] = {}
    self._max_syllables = 1
    for word, tags in entries.items():
        self._insert(word, tags)

max_word_syllables property

max_word_syllables: int

Syllable count of the longest word (bounds longest-match search).

default classmethod

default() -> 'Lexicon'

Build the built-in lexicon: closed-class seed + bundled JSON files.

Every lexicon/data/*.json file is merged (sorted by filename) onto the in-code seed / grammar lists so function words keep preferred multi-tag sets when both sources list a word. Additional files in that directory are picked up automatically.

Source code in src/burmesenlp/lexicon/__init__.py

@classmethod
def default(cls) -> "Lexicon":
    """Build the built-in lexicon: closed-class seed + bundled JSON files.

    Every ``lexicon/data/*.json`` file is merged (sorted by filename) onto
    the in-code seed / grammar lists so function words keep preferred
    multi-tag sets when both sources list a word.  Additional files in
    that directory are picked up automatically.
    """
    data = _seed_entries()
    json_files = sorted(_DATA_DIR.glob("*.json")) if _DATA_DIR.is_dir() else []
    if not json_files:
        logger.warning(
            "No bundled lexicon JSON under %s; using seed only",
            _DATA_DIR,
        )
        lexicon = cls(data)
        _apply_canonical_tags(lexicon)
        return lexicon

    bundled = 0
    for path in json_files:
        cleaned = _sanitize_entries(
            _parse_json_entries(str(path), _read_text(str(path))),
            path.name,
        )
        _merge_entry_maps(data, cleaned)
        bundled += len(cleaned)

    lexicon = cls(data)
    _apply_canonical_tags(lexicon)
    logger.info(
        "Loaded default lexicon from %d JSON file(s) under %s "
        "(%d bundled entries, %d total)",
        len(json_files),
        _DATA_DIR.name,
        bundled,
        len(lexicon),
    )
    return lexicon

from_file classmethod

from_file(path: str, *, merge_default: bool = False) -> 'Lexicon'

Load a lexicon from .json or .txt.

With merge_default=True (used by :class:BurmeseNLP), entries are merged on top of :meth:default with per-word tag union. With merge_default=False, only the file contents are loaded.

Raises LexiconError on unreadable files, bad format, or schema violations -- never falls back silently.

Source code in src/burmesenlp/lexicon/__init__.py

@classmethod
def from_file(cls, path: str, *, merge_default: bool = False) -> "Lexicon":
    """Load a lexicon from ``.json`` or ``.txt``.

    With ``merge_default=True`` (used by :class:`BurmeseNLP`), entries are
    merged on top of :meth:`default` with per-word tag union.
    With ``merge_default=False``, only the file contents are loaded.

    Raises ``LexiconError`` on unreadable files, bad format, or schema
    violations -- never falls back silently.
    """
    return cls._from_overlay(
        _load_entries(path), path, merge_default=merge_default
    )

from_json classmethod

from_json(path: str, *, merge_default: bool = False) -> 'Lexicon'

Load a lexicon from canonical JSON: {"word": ["tag", ...], ...}.

See :meth:from_file for merge_default semantics.

Source code in src/burmesenlp/lexicon/__init__.py

@classmethod
def from_json(cls, path: str, *, merge_default: bool = False) -> "Lexicon":
    """Load a lexicon from canonical JSON: ``{"word": ["tag", ...], ...}``.

    See :meth:`from_file` for ``merge_default`` semantics.
    """
    return cls._from_overlay(
        _parse_json_entries(path, _read_text(path)),
        path,
        merge_default=merge_default,
    )

from_txt classmethod

from_txt(path: str, *, merge_default: bool = False) -> 'Lexicon'

Load a lexicon from a line-based import file: word\ttag1,tag2.

This is a convenience import format; :meth:save always writes JSON. See :meth:from_file for merge_default semantics.

Source code in src/burmesenlp/lexicon/__init__.py

@classmethod
def from_txt(cls, path: str, *, merge_default: bool = False) -> "Lexicon":
    """Load a lexicon from a line-based import file: ``word\\ttag1,tag2``.

    This is a convenience import format; :meth:`save` always writes JSON.
    See :meth:`from_file` for ``merge_default`` semantics.
    """
    return cls._from_overlay(
        _parse_txt_entries(path, _read_text(path)),
        path,
        merge_default=merge_default,
    )

add

add(word: str, tags: Iterable[str]) -> None

Add (or extend) a word with the given POS tags. O(1), no rebuild.

Source code in src/burmesenlp/lexicon/__init__.py

def add(self, word: str, tags: Iterable[str]) -> None:
    """Add (or extend) a word with the given POS tags. O(1), no rebuild."""
    self._insert(word, tags)

merge

merge(entries: Mapping[str, Iterable[str]]) -> None

Union entries into this lexicon word-by-word (O(n) inserts).

Source code in src/burmesenlp/lexicon/__init__.py

def merge(self, entries: Mapping[str, Iterable[str]]) -> None:
    """Union *entries* into this lexicon word-by-word (O(n) inserts)."""
    for word, tags in entries.items():
        self._insert(word, tags)

save

save(path: str) -> None

Atomically save the lexicon as UTF-8 JSON (canonical format).

Source code in src/burmesenlp/lexicon/__init__.py

def save(self, path: str) -> None:
    """Atomically save the lexicon as UTF-8 JSON (canonical format)."""
    payload = {w: list(t) for w, t in sorted(self._entries.items())}
    directory = os.path.dirname(os.path.abspath(path)) or "."
    fd, tmp = tempfile.mkstemp(dir=directory, suffix=".tmp")
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as f:
            json.dump(payload, f, ensure_ascii=False, indent=2)
        os.replace(tmp, path)
    except BaseException:
        if os.path.exists(tmp):
            os.unlink(tmp)
        raise

tags

tags(word: str) -> Tuple[str, ...]

POS tags for word in TAG_PREFERENCE order, or () if unknown.

Source code in src/burmesenlp/lexicon/__init__.py

def tags(self, word: str) -> Tuple[str, ...]:
    """POS tags for *word* in TAG_PREFERENCE order, or () if unknown."""
    return self._entries.get(word, ())

burmesenlp.lexicon.LexiconError

Bases: ValueError

Raised when a lexicon file or entry is invalid.

burmesenlp.lexicon.POS_TAGS module-attribute

POS_TAGS: Dict[str, str] = {'NOUN': 'noun', 'VERB': 'verb', 'ADJ': 'adjective', 'ADV': 'adverb', 'PRON': 'pronoun', 'NUM': 'number (digits or text)', 'CONJ': 'conjunction', 'INTJ': 'interjection', 'PUNCT': 'punctuation', 'POSTP': 'postposition / case marker', 'PART': 'particle', 'AUX': 'auxiliary', 'SFP': 'sentence-final particle', 'ABB': 'abbreviation', 'FW': 'foreign word', 'SB': 'symbol', 'IDIOM': 'multi-word idiom / fixed expression (BMWE)', 'UNK': 'unknown'}