fix lowercase-ization in bert indexer (#2205)

* fix lowercase-ization in bert indexer

* add never_lowercase feature for [UNK], etc

* add warnings when BERT model appears incongruent with do_lowercase

Author: joelgrus

allennlp/data/token_indexers/wordpiece_indexer.py

@@ -15,6 +15,10 @@
 
 # TODO(joelgrus): Figure out how to generate token_type_ids out of this token indexer.
 
+# This is the default list of tokens that should not be lowercased.
+_NEVER_LOWERCASE = ['[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]']
+
+
 class WordpieceIndexer(TokenIndexer[int]):
     """
     A token indexer that does the wordpiece-tokenization (e.g. for BERT embeddings).
@@ -39,6 +43,14 @@ class WordpieceIndexer(TokenIndexer[int]):
         maximum length for its input ids. Currently any inputs longer than this
         will be truncated. If this behavior is undesirable to you, you should
         consider filtering them out in your dataset reader.
+    do_lowercase : ``bool``, optional (default=``False``)
+        Should we lowercase the provided tokens before getting the indices?
+        You would need to do this if you are using an -uncased BERT model
+        but your DatasetReader is not lowercasing tokens (which might be the
+        case if you're also using other embeddings based on cased tokens).
+    never_lowercase: ``List[str]``, optional
+        Tokens that should never be lowercased. Default is
+        ['[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]'].
     start_tokens : ``List[str]``, optional (default=``None``)
         These are prepended to the tokens provided to ``tokens_to_indices``.
     end_tokens : ``List[str]``, optional (default=``None``)
@@ -50,6 +62,8 @@ def __init__(self,
                  namespace: str = "wordpiece",
                  use_starting_offsets: bool = False,
                  max_pieces: int = 512,
+                 do_lowercase: bool = False,
+                 never_lowercase: List[str] = None,
                  start_tokens: List[str] = None,
                  end_tokens: List[str] = None) -> None:
         self.vocab = vocab
@@ -64,6 +78,13 @@ def __init__(self,
         self._added_to_vocabulary = False
         self.max_pieces = max_pieces
         self.use_starting_offsets = use_starting_offsets
+        self._do_lowercase = do_lowercase
+
+        if never_lowercase is None:
+            # Use the defaults
+            self._never_lowercase = set(_NEVER_LOWERCASE)
+        else:
+            self._never_lowercase = set(never_lowercase)
 
         # Convert the start_tokens and end_tokens to wordpiece_ids
         self._start_piece_ids = [vocab[wordpiece]
@@ -108,8 +129,12 @@ def tokens_to_indices(self,
         offset = len(wordpiece_ids) if self.use_starting_offsets else len(wordpiece_ids) - 1
 
         for token in tokens:
+            # Lowercase if necessary
+            text = (token.text.lower()
+                    if self._do_lowercase and token.text not in self._never_lowercase
+                    else token.text)
             token_wordpiece_ids = [self.vocab[wordpiece]
-                                   for wordpiece in self.wordpiece_tokenizer(token.text)]
+                                   for wordpiece in self.wordpiece_tokenizer(text)]
             # If we have enough room to add these ids *and also* the end_token ids.
             if len(wordpiece_ids) + len(token_wordpiece_ids) + len(self._end_piece_ids) <= self.max_pieces:
                 # For initial offsets, the current value of ``offset`` is the start of
@@ -189,6 +214,9 @@ class PretrainedBertIndexer(WordpieceIndexer):
         they will instead correspond to the first wordpiece in each word.
     do_lowercase: ``bool``, optional (default = True)
         Whether to lowercase the tokens before converting to wordpiece ids.
+    never_lowercase: ``List[str]``, optional
+        Tokens that should never be lowercased. Default is
+        ['[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]'].
     max_pieces: int, optional (default: 512)
         The BERT embedder uses positional embeddings and so has a corresponding
         maximum length for its input ids. Currently any inputs longer than this
@@ -199,12 +227,22 @@ def __init__(self,
                  pretrained_model: str,
                  use_starting_offsets: bool = False,
                  do_lowercase: bool = True,
+                 never_lowercase: List[str] = None,
                  max_pieces: int = 512) -> None:
+        if pretrained_model.endswith("-cased") and do_lowercase:
+            logger.warning("Your BERT model appears to be cased, "
+                           "but your indexer is lowercasing tokens.")
+        elif pretrained_model.endswith("-uncased") and not do_lowercase:
+            logger.warning("Your BERT model appears to be uncased, "
+                           "but your indexer is not lowercasing tokens.")
+
         bert_tokenizer = BertTokenizer.from_pretrained(pretrained_model, do_lower_case=do_lowercase)
         super().__init__(vocab=bert_tokenizer.vocab,
                          wordpiece_tokenizer=bert_tokenizer.wordpiece_tokenizer.tokenize,
                          namespace="bert",
                          use_starting_offsets=use_starting_offsets,
                          max_pieces=max_pieces,
+                         do_lowercase=do_lowercase,
+                         never_lowercase=never_lowercase,
                          start_tokens=["[CLS]"],
                          end_tokens=["[SEP]"])

allennlp/tests/data/token_indexers/bert_indexer_test.py

@@ -1,12 +1,13 @@
 # pylint: disable=no-self-use,invalid-name
 from allennlp.common.testing import ModelTestCase
 from allennlp.data.token_indexers.wordpiece_indexer import PretrainedBertIndexer
-from allennlp.data.tokenizers import WordTokenizer
+from allennlp.data.tokenizers import WordTokenizer, Token
 from allennlp.data.tokenizers.word_splitter import BertBasicWordSplitter
 from allennlp.data.vocabulary import Vocabulary
 
 class TestBertIndexer(ModelTestCase):
 
+
     def test_starting_ending_offsets(self):
         tokenizer = WordTokenizer(word_splitter=BertBasicWordSplitter())
 
@@ -30,3 +31,56 @@ def test_starting_ending_offsets(self):
 
         assert indexed_tokens["bert"] == [16, 2, 3, 5, 6, 8, 9, 2, 15, 10, 11, 14, 1, 17]
         assert indexed_tokens["bert-offsets"] == [1, 2, 3, 4, 5, 6, 7, 8, 11, 12]
+
+
+    def test_do_lowercase(self):
+        # Our default tokenizer doesn't handle lowercasing.
+        tokenizer = WordTokenizer()
+
+        # Quick is UNK because of capitalization
+        #           2   1     5     6   8      9    2  15 10 11 14   1
+        sentence = "the Quick brown fox jumped over the laziest lazy elmo"
+        tokens = tokenizer.tokenize(sentence)
+
+        vocab = Vocabulary()
+        vocab_path = self.FIXTURES_ROOT / 'bert' / 'vocab.txt'
+        token_indexer = PretrainedBertIndexer(str(vocab_path), do_lowercase=False)
+
+        indexed_tokens = token_indexer.tokens_to_indices(tokens, vocab, "bert")
+
+        # Quick should get 1 == OOV
+        assert indexed_tokens["bert"] == [16, 2, 1, 5, 6, 8, 9, 2, 15, 10, 11, 14, 1, 17]
+
+        # Does lowercasing by default
+        token_indexer = PretrainedBertIndexer(str(vocab_path))
+        indexed_tokens = token_indexer.tokens_to_indices(tokens, vocab, "bert")
+
+        # Now Quick should get indexed correctly as 3 ( == "quick")
+        assert indexed_tokens["bert"] == [16, 2, 3, 5, 6, 8, 9, 2, 15, 10, 11, 14, 1, 17]
+
+
+    def test_never_lowercase(self):
+        # Our default tokenizer doesn't handle lowercasing.
+        tokenizer = WordTokenizer()
+
+        #            2 15 10 11  6
+        sentence = "the laziest fox"
+
+        tokens = tokenizer.tokenize(sentence)
+        tokens.append(Token("[PAD]"))  # have to do this b/c tokenizer splits it in three
+
+        vocab = Vocabulary()
+        vocab_path = self.FIXTURES_ROOT / 'bert' / 'vocab.txt'
+        token_indexer = PretrainedBertIndexer(str(vocab_path), do_lowercase=True)
+
+        indexed_tokens = token_indexer.tokens_to_indices(tokens, vocab, "bert")
+
+        # PAD should get recognized and not lowercased      # [PAD]
+        assert indexed_tokens["bert"] == [16, 2, 15, 10, 11, 6, 0, 17]
+
+        # Unless we manually override the never lowercases
+        token_indexer = PretrainedBertIndexer(str(vocab_path), do_lowercase=True, never_lowercase=())
+        indexed_tokens = token_indexer.tokens_to_indices(tokens, vocab, "bert")
+
+        # now PAD should get lowercased and be UNK          # [UNK]
+        assert indexed_tokens["bert"] == [16, 2, 15, 10, 11, 6, 1, 17]