Skip to content

MWE

burmesenlp.mwe.BMWEEngine 

BMWEEngine(lexicon: Optional[Lexicon] = None, validator: Optional[MWEValidator] = None, *, autoload_idioms: bool = True)

Burmese Multi-Word Expression engine (post-tokenization merge).

Source code in src/burmesenlp/mwe/engine.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
def __init__(
    self,
    lexicon: Optional[Lexicon] = None,
    validator: Optional[MWEValidator] = None,
    *,
    autoload_idioms: bool = True,
):
    self._lexicon = lexicon if lexicon is not None else Lexicon.default()
    self._trie = MWETrie()
    self._validator: MWEValidator = (
        validator if validator is not None else AcceptAllValidator()
    )
    if autoload_idioms:
        path = default_idioms_path()
        if path:
            try:
                n = self.load(path, category="IDIOM")
                logger.info("Loaded %d MWE entries from %s", n, path)
            except (OSError, ValueError, json.JSONDecodeError) as exc:
                logger.warning("Could not autoload idioms from %s: %s", path, exc)

load 

load(path: str, *, category: Optional[str] = None, priority: int = 0, allow_unigrams: bool = False, use_cache: bool = True, write_cache_on_miss: bool = True) -> int

Load a JSON/TXT MWE list into the trie. Returns entry count.

Source code in src/burmesenlp/mwe/engine.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def load(
    self,
    path: str,
    *,
    category: Optional[str] = None,
    priority: int = 0,
    allow_unigrams: bool = False,
    use_cache: bool = True,
    write_cache_on_miss: bool = True,
) -> int:
    """Load a JSON/TXT MWE list into the trie. Returns entry count."""
    return load_into_trie(
        self._trie,
        path,
        self._lexicon,
        category=category,
        priority=priority,
        allow_unigrams=allow_unigrams,
        use_cache=use_cache,
        write_cache_on_miss=write_cache_on_miss,
    )

process 

process(tokens: Sequence[str]) -> List[str]

Return merged token strings for downstream POS/chunking.

Source code in src/burmesenlp/mwe/engine.py 
75
76
77
78
def process(self, tokens: Sequence[str]) -> List[str]:
    """Return merged token strings for downstream POS/chunking."""
    merged, _ = self.process_detailed(tokens)
    return merged

process_detailed 

process_detailed(tokens: Sequence[str]) -> Tuple[List[str], List[MWEToken]]

Greedy left-to-right MWE merge; return strings + span metadata.

Source code in src/burmesenlp/mwe/engine.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def process_detailed(
    self,
    tokens: Sequence[str],
) -> Tuple[List[str], List[MWEToken]]:
    """Greedy left-to-right MWE merge; return strings + span metadata."""
    tokens = list(tokens)
    if not tokens:
        return [], []

    out: List[str] = []
    spans: List[MWEToken] = []
    i = 0
    n = len(tokens)
    while i < n:
        candidates = self._trie.search(tokens, i)
        if not candidates:
            out.append(tokens[i])
            i += 1
            continue
        best = choose(candidates)
        if self._validator.validate(best, tokens, i):
            end = i + len(best.tokens) - 1
            merged_text = "".join(best.tokens)
            merged_index = len(out)
            out.append(merged_text)
            spans.append(
                MWEToken(
                    text=merged_text,
                    tokens=best.tokens,
                    category=best.category,
                    start=i,
                    end=end,
                    priority=best.priority,
                    pos=_resolve_entry_pos(best),
                    index=merged_index,
                )
            )
            i = end + 1
        else:
            out.append(tokens[i])
            i += 1
    return out, spans

burmesenlp.mwe.MWEEntry dataclass 

MWEEntry(text: str, tokens: Tuple[str, ...], category: str, priority: int = 0, pos: Optional[str] = None)

A multi-word expression loaded into the trie.

burmesenlp.mwe.MWEToken dataclass 

MWEToken(text: str, tokens: Tuple[str, ...], category: str, start: int, end: int, priority: int = 0, pos: Optional[str] = None, index: Optional[int] = None)

A merged MWE span over a pre-MWE word token sequence.