diff --git a/src/mistral_common/tokens/tokenizers/tekken.py b/src/mistral_common/tokens/tokenizers/tekken.py index 864fa98..62ac5fa 100644 --- a/src/mistral_common/tokens/tokenizers/tekken.py +++ b/src/mistral_common/tokens/tokenizers/tekken.py @@ -126,7 +126,7 @@ def from_file(cls, path: Union[str, Path]) -> "Tekkenizer": if isinstance(path, str): path = Path(path) assert path.exists() - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: model_data: ModelData = json.load(f) _version_str = model_data["config"].get("version")