Skip to content

Commit 564ce47

Browse files
committed
fix(transformers): force padding to max length in all versions
1 parent 9fd6685 commit 564ce47

File tree

1 file changed

+6
-2
lines changed

1 file changed

+6
-2
lines changed

jina/executors/encoders/nlp/transformer.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,15 @@ def encode(self, data: 'np.ndarray', *args, **kwargs) -> 'np.ndarray':
8181
ids_info = self.tokenizer.batch_encode_plus(data,
8282
max_length=self.max_length,
8383
truncation=self.truncation_strategy,
84-
padding=True)
84+
pad_to_max_length=True,
85+
padding='max_length')
8586
except ValueError:
8687
self.tokenizer.add_special_tokens({'pad_token': '[PAD]'})
8788
self.model.resize_token_embeddings(len(self.tokenizer))
88-
ids_info = self.tokenizer.batch_encode_plus(data, max_length=self.max_length, padding=True)
89+
ids_info = self.tokenizer.batch_encode_plus(data,
90+
max_length=self.max_length,
91+
pad_to_max_length=True,
92+
padding='max_length')
8993
token_ids_batch = self.array2tensor(ids_info['input_ids'])
9094
mask_ids_batch = self.array2tensor(ids_info['attention_mask'])
9195
with self.session():

0 commit comments

Comments
 (0)