Add a no-op trainer. (#2610)

- Simply loads a model, creates the vocab and serializes without any training.
- Intended to be used principally for untrained baselines like majority class.

Author: brendan-ai2

allennlp/tests/commands/no_op_train_test.py

@@ -0,0 +1,39 @@
+from typing import Dict
+
+import torch
+
+from allennlp.commands.train import train_model
+from allennlp.common import Params
+from allennlp.common.testing import AllenNlpTestCase
+from allennlp.models import load_archive, Model
+
+SEQUENCE_TAGGING_DATA_PATH = str(AllenNlpTestCase.FIXTURES_ROOT / 'data' / 'sequence_tagging.tsv')
+
+
+@Model.register('constant')
+class ConstantModel(Model):
+    def forward(self, *inputs) -> Dict[str, torch.Tensor]:
+        return {"class": torch.tensor(98)} # pylint: disable=not-callable
+
+class TestTrain(AllenNlpTestCase):
+
+    def test_train_model(self):
+        params = lambda: Params({
+                "model": {
+                        "type": "constant"
+                },
+                "dataset_reader": {"type": "sequence_tagging"},
+                "train_data_path": SEQUENCE_TAGGING_DATA_PATH,
+                "validation_data_path": SEQUENCE_TAGGING_DATA_PATH,
+                "iterator": {"type": "basic", "batch_size": 2},
+                "trainer": {
+                        "type": "no_op"
+                }
+        })
+
+        serialization_dir = self.TEST_DIR / 'serialization_directory'
+        train_model(params(), serialization_dir=serialization_dir)
+        archive = load_archive(str(serialization_dir / "model.tar.gz"))
+        model = archive.model
+        assert model.forward(torch.tensor([1, 2, 3]))["class"] == torch.tensor(98) # pylint: disable=not-callable
+        assert model.vocab.get_vocab_size() == 9

allennlp/tests/training/no_op_trainer_test.py

@@ -0,0 +1,31 @@
+import os
+from typing import Dict
+
+import torch
+
+from allennlp.common.testing import AllenNlpTestCase
+from allennlp.data import Vocabulary
+from allennlp.data.dataset_readers import SequenceTaggingDatasetReader
+from allennlp.models.model import Model
+from allennlp.training import NoOpTrainer
+
+
+class ConstantModel(Model):
+    def forward(self, *inputs) -> Dict[str, torch.Tensor]:
+        return {"class": torch.tensor(98)} # pylint: disable=not-callable
+
+class TestNoOpTrainer(AllenNlpTestCase):
+    def setUp(self):
+        super().setUp()
+        self.instances = SequenceTaggingDatasetReader().read(self.FIXTURES_ROOT / 'data' / 'sequence_tagging.tsv')
+        vocab = Vocabulary.from_instances(self.instances)
+        self.vocab = vocab
+        self.model = ConstantModel(vocab)
+
+    def test_trainer_serializes(self):
+        serialization_dir = self.TEST_DIR / "serialization_dir"
+        trainer = NoOpTrainer(serialization_dir=serialization_dir, model=self.model)
+        metrics = trainer.train()
+        assert metrics == {}
+        assert os.path.exists(serialization_dir / "best.th")
+        assert os.path.exists(serialization_dir / "vocabulary")

allennlp/training/__init__.py

@@ -1,2 +1,3 @@
+from allennlp.training.no_op_trainer import NoOpTrainer
 from allennlp.training.trainer import Trainer
 from allennlp.training.trainer_base import TrainerBase

allennlp/training/no_op_trainer.py

@@ -0,0 +1,38 @@
+import os
+from typing import Dict, Any
+
+from allennlp.common import Params
+from allennlp.models import Model
+from allennlp.training.checkpointer import Checkpointer
+from allennlp.training.trainer import TrainerPieces
+from allennlp.training.trainer_base import TrainerBase
+
+@TrainerBase.register("no_op")
+class NoOpTrainer(TrainerBase):
+    def __init__(self, serialization_dir: str, model: Model) -> None:
+        """
+        A trivial trainer to assist in making model archives for models that do not actually
+        require training. For instance, a majority class baseline.
+        """
+
+        super().__init__(serialization_dir, cuda_device=-1)
+        self.model = model
+
+    @classmethod
+    def from_params(cls,   # type: ignore
+                    params: Params,
+                    serialization_dir: str,
+                    recover: bool = False):
+        # pylint: disable=arguments-differ
+        pieces = TrainerPieces.from_params(params, serialization_dir, recover)  # pylint: disable=no-member
+        return NoOpTrainer(serialization_dir, pieces.model)
+
+    def train(self) -> Dict[str, Any]:
+        self.model.vocab.save_to_files(os.path.join(self._serialization_dir, "vocabulary"))
+
+        checkpointer = Checkpointer(self._serialization_dir)
+        checkpointer.save_checkpoint(epoch=0,
+                                     model_state=self.model.state_dict(),
+                                     training_states={},
+                                     is_best_so_far=True)
+        return {}

allennlp/training/trainer_base.py

@@ -79,4 +79,8 @@ def from_params(cls,   # type: ignore
                                        params=pieces.params,
                                        validation_iterator=pieces.validation_iterator)
         else:
-            return TrainerBase.by_name(typ3).from_params(params, serialization_dir, recover)
+            klass = TrainerBase.by_name(typ3)
+            # Explicit check to prevent recursion.
+            is_overriden = klass.from_params.__func__ != TrainerBase.from_params.__func__ # type: ignore
+            assert is_overriden, f"Class {klass.__name__} must override `from_params`."
+            return klass.from_params(params, serialization_dir, recover)

doc/api/allennlp.training.no_op_trainer.rst

@@ -0,0 +1,7 @@
+allennlp.training.no_op_trainer
+======================================
+
+.. automodule:: allennlp.training.no_op_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/api/allennlp.training.rst

@@ -13,6 +13,7 @@ for training AllenNLP models.
    allennlp.training.metric_tracker
    allennlp.training.metrics
    allennlp.training.moving_average
+   allennlp.training.no_op_trainer
    allennlp.training.optimizers
    allennlp.training.tensorboard_writer
    allennlp.training.trainer