From df3df66c103a6885b062069b4973284a9d279c94 Mon Sep 17 00:00:00 2001 From: Mart Ratas Date: Fri, 15 Nov 2024 13:37:16 +0000 Subject: [PATCH] CU-8696n7w95: Remove commented code to fix DeID (oversight in PR 490) (#502) --- medcat/ner/transformers_ner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/medcat/ner/transformers_ner.py b/medcat/ner/transformers_ner.py index 1cc4c5f2..1de8d6d8 100644 --- a/medcat/ner/transformers_ner.py +++ b/medcat/ner/transformers_ner.py @@ -87,10 +87,10 @@ def create_eval_pipeline(self): # NOTE: this will fix the DeID model(s) created before medcat 1.9.3 # though this fix may very well be unstable self.ner_pipe.tokenizer._in_target_context_manager = False - # if not hasattr(self.ner_pipe.tokenizer, 'split_special_tokens'): - # # NOTE: this will fix the DeID model(s) created with transformers before 4.42 - # # and allow them to run with later transforemrs - # self.ner_pipe.tokenizer.split_special_tokens = False + if not hasattr(self.ner_pipe.tokenizer, 'split_special_tokens'): + # NOTE: this will fix the DeID model(s) created with transformers before 4.42 + # and allow them to run with later transforemrs + self.ner_pipe.tokenizer.split_special_tokens = False self.ner_pipe.device = self.model.device self._consecutive_identical_failures = 0 self._last_exception: Optional[Tuple[str, Type[Exception]]] = None