How to use the bpemb.bpemb.BPEmb function in bpemb
To help you get started, we’ve selected a few bpemb examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
bheinzerling / bpemb / bpemb / bpemb.py View on Github 
state['spm'] = None
return state
def __setstate__(self, state):
# load SentencePiece after the BPEmb object has been unpickled
model_file = (
state["cache_dir"] / state["lang"] / state['model_file'].name)
if not model_file.exists():
model_rel_path = Path(state["lang"]) / model_file.name
model_file = self._load_file(
str(model_rel_path), cache_dir=state["cache_dir"])
state['spm'] = sentencepiece_load(model_file)
self.__dict__ = state
__all__ = [BPEmb]
