'format_text' for source and destination sentences in '_sample_decode'

nmt/train.py

@@ -447,8 +447,8 @@ def _sample_decode(model, global_step, sess, hparams, iterator, src_data,
  sent_id=0,
  tgt_eos=hparams.eos,
  subword_option=hparams.subword_option)
- utils.print_out(" src: %s" % src_data[decode_id])
- utils.print_out(" ref: %s" % tgt_data[decode_id])
+ utils.print_out(b" src: " + utils.format_sentence(src_data[decode_id], hparams.subword_option))
+ utils.print_out(b" ref: " + utils.format_sentence(tgt_data[decode_id], hparams.subword_option))
  utils.print_out(b" nmt: " + translation)
 
  # Summary

nmt/utils/misc_utils.py

@@ -164,8 +164,6 @@ def format_bpe_text(symbols, delimiter=b"@@"):
  """Convert a sequence of bpe words into sentence."""
  words = []
  word = b""
- if isinstance(symbols, str):
- symbols = symbols.encode()
  delimiter_len = len(delimiter)
  for symbol in symbols:
  if len(symbol) >= delimiter_len and symbol[-delimiter_len:] == delimiter:
@@ -181,3 +179,17 @@ def format_spm_text(symbols):
  """Decode a text in SPM (https://github.com/google/sentencepiece) format."""
  return u"".join(format_text(symbols).decode("utf-8").split()).replace(
  u"\u2581", u" ").strip().encode("utf-8")
+
+def format_sentence(sentence, subword_option):
+ """Decode sentence using subword option"""
+ if isinstance(sentence, str):
+ sentence = sentence.encode("utf-8").split(b' ')
+
+ if subword_option == "bpe": # BPE
+ sentence = format_bpe_text(sentence)
+ elif subword_option == "spm": # SPM
+ sentence = format_spm_text(sentence)
+ else:
+ sentence = format_text(sentence)
+
+ return sentence

nmt/utils/nmt_utils.py

@@ -99,11 +99,4 @@ def get_translation(nmt_outputs, sent_id, tgt_eos, subword_option):
  if tgt_eos and tgt_eos in output:
  output = output[:output.index(tgt_eos)]
 
- if subword_option == "bpe": # BPE
- translation = utils.format_bpe_text(output)
- elif subword_option == "spm": # SPM
- translation = utils.format_spm_text(output)
- else:
- translation = utils.format_text(output)
-
- return translation
+ return utils.format_sentence(output, subword_option)