Add missing tokenizer/transformer kwargs (#4682)

* Add tokenizer_kwargs in PretrainedTransformerMismatchedIndexer and tokenizer_kwargs/transformer_kwargs in PretrainedTransformerMismatchedEmbedder

* Update allennlp/data/token_indexers/pretrained_transformer_mismatched_indexer.py

Co-authored-by: Evan Pete Walsh <[email protected]>

* Update CHANGELOG.md

Co-authored-by: Evan Pete Walsh <[email protected]>

Co-authored-by: Evan Pete Walsh <[email protected]>

Author: mulhod

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added `tokenizer_kwargs` argument to `PretrainedTransformerMismatchedIndexer`.
+- Added `tokenizer_kwargs` and `transformer_kwargs` arguments to `PretrainedTransformerMismatchedEmbedder`.
 - Added official support for Python 3.8.
 - Added a script: `scripts/release_notes.py`, which automatically prepares markdown release notes from the
   CHANGELOG and commit history.

allennlp/data/token_indexers/pretrained_transformer_mismatched_indexer.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict, List, Any, Optional
 import logging
 
 from overrides import overrides
@@ -39,15 +39,28 @@ class PretrainedTransformerMismatchedIndexer(TokenIndexer):
         before feeding into the embedder. The embedder embeds these segments independently and
         concatenate the results to get the original document representation. Should be set to
         the same value as the `max_length` option on the `PretrainedTransformerMismatchedEmbedder`.
-    """
+    tokenizer_kwargs : `Dict[str, Any]`, optional (default = `None`)
+        Dictionary with
+        [additional arguments](https://github.com/huggingface/transformers/blob/155c782a2ccd103cf63ad48a2becd7c76a7d2115/transformers/tokenization_utils.py#L691)
+        for `AutoTokenizer.from_pretrained`.
+    """  # noqa: E501
 
     def __init__(
-        self, model_name: str, namespace: str = "tags", max_length: int = None, **kwargs
+        self,
+        model_name: str,
+        namespace: str = "tags",
+        max_length: int = None,
+        tokenizer_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs,
     ) -> None:
         super().__init__(**kwargs)
         # The matched version v.s. mismatched
         self._matched_indexer = PretrainedTransformerIndexer(
-            model_name, namespace, max_length, **kwargs
+            model_name,
+            namespace=namespace,
+            max_length=max_length,
+            tokenizer_kwargs=tokenizer_kwargs,
+            **kwargs,
         )
         self._allennlp_tokenizer = self._matched_indexer._allennlp_tokenizer
         self._tokenizer = self._matched_indexer._tokenizer

allennlp/modules/token_embedders/pretrained_transformer_mismatched_embedder.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, Dict, Any
 
 from overrides import overrides
 import torch
@@ -33,7 +33,15 @@ class PretrainedTransformerMismatchedEmbedder(TokenEmbedder):
         is used.
     gradient_checkpointing: `bool`, optional (default = `None`)
         Enable or disable gradient checkpointing.
-    """
+    tokenizer_kwargs: `Dict[str, Any]`, optional (default = `None`)
+        Dictionary with
+        [additional arguments](https://github.com/huggingface/transformers/blob/155c782a2ccd103cf63ad48a2becd7c76a7d2115/transformers/tokenization_utils.py#L691)
+        for `AutoTokenizer.from_pretrained`.
+    transformer_kwargs: `Dict[str, Any]`, optional (default = `None`)
+        Dictionary with
+        [additional arguments](https://github.com/huggingface/transformers/blob/155c782a2ccd103cf63ad48a2becd7c76a7d2115/transformers/modeling_utils.py#L253)
+        for `AutoModel.from_pretrained`.
+    """  # noqa: E501
 
     def __init__(
         self,
@@ -42,6 +50,8 @@ def __init__(
         train_parameters: bool = True,
         last_layer_only: bool = True,
         gradient_checkpointing: Optional[bool] = None,
+        tokenizer_kwargs: Optional[Dict[str, Any]] = None,
+        transformer_kwargs: Optional[Dict[str, Any]] = None,
     ) -> None:
         super().__init__()
         # The matched version v.s. mismatched
@@ -51,6 +61,8 @@ def __init__(
             train_parameters=train_parameters,
             last_layer_only=last_layer_only,
             gradient_checkpointing=gradient_checkpointing,
+            tokenizer_kwargs=tokenizer_kwargs,
+            transformer_kwargs=transformer_kwargs,
         )
 
     @overrides