diff --git a/open_seq2seq/models/text2text.py b/open_seq2seq/models/text2text.py
index 5a816dd1a..a6b2c844c 100644
--- a/open_seq2seq/models/text2text.py
+++ b/open_seq2seq/models/text2text.py
@@ -50,7 +50,7 @@ def calculate_bleu(preds, targets):
     float32: BLEU score
   """
   bleu_score = nltk.translate.bleu_score.corpus_bleu(
-      targets, preds, emulate_multibleu=True,
+      targets, preds,
   )
   return bleu_score