@@ -95,7 +95,8 @@ def convert_samples(sample):
9595 bos_idx = args .bos_idx ,
9696 eos_idx = args .eos_idx ,
9797 pad_idx = args .bos_idx ,
98- pad_seq = args .pad_seq ),
98+ pad_seq = args .pad_seq ,
99+ dtype = args .input_dtype ),
99100 num_workers = 0 )
100101 data_loaders [i ] = (data_loader )
101102 return data_loaders
@@ -142,7 +143,8 @@ def convert_samples(sample):
142143 bos_idx = args .bos_idx ,
143144 eos_idx = args .eos_idx ,
144145 pad_idx = args .bos_idx ,
145- pad_seq = args .pad_seq ),
146+ pad_seq = args .pad_seq ,
147+ dtype = args .input_dtype ),
146148 num_workers = 0 ,
147149 return_list = True )
148150 return data_loader , trg_vocab .to_tokens
@@ -163,11 +165,16 @@ def adapt_vocab_size(args):
163165 args .trg_vocab_size = padding_vocab (len (trg_vocab ))
164166
165167
166- def prepare_train_input (insts , bos_idx , eos_idx , pad_idx , pad_seq = 1 ):
168+ def prepare_train_input (insts ,
169+ bos_idx ,
170+ eos_idx ,
171+ pad_idx ,
172+ pad_seq = 1 ,
173+ dtype = "int64" ):
167174 """
168175 Put all padded data needed by training into a list.
169176 """
170- word_pad = Pad (pad_idx , dtype = "int64" )
177+ word_pad = Pad (pad_idx , dtype = dtype )
171178 src_max_len = (
172179 max ([len (inst [0 ]) for inst in insts ]) + pad_seq ) // pad_seq * pad_seq
173180 trg_max_len = (
@@ -190,11 +197,16 @@ def prepare_train_input(insts, bos_idx, eos_idx, pad_idx, pad_seq=1):
190197 return data_inputs
191198
192199
193- def prepare_infer_input (insts , bos_idx , eos_idx , pad_idx , pad_seq = 1 ):
200+ def prepare_infer_input (insts ,
201+ bos_idx ,
202+ eos_idx ,
203+ pad_idx ,
204+ pad_seq = 1 ,
205+ dtype = "int64" ):
194206 """
195207 Put all padded data needed by beam search decoder into a list.
196208 """
197- word_pad = Pad (pad_idx , dtype = "int64" )
209+ word_pad = Pad (pad_idx , dtype = dtype )
198210 src_max_len = (
199211 max ([len (inst [0 ]) for inst in insts ]) + pad_seq ) // pad_seq * pad_seq
200212 src_word = word_pad ([
0 commit comments