Add influence functions to interpret module (#4988)

* creating a new functionality  to fields and instances to support outputing instnaces to json files

* creating tests for the new functionality

* fixing docs

* Delete __init__.py

* Delete influence_interpreter.py

* Delete use_if.py

* Delete simple_influence_test.py

* fixing docs

* finishing up SimpleInfluence

* passing lint

* passing format

* making small progress in coding

* Delete fast_influence.py

Submit to the wrong branch

* Delete faiss_utils.py

wrong branch

* Delete gpt2_bug.py

not sure why it's included

* Delete text_class.py

not sure why it's included

* adding test file

* adding testing files

* deleted unwanted files

* deleted unwanted files and rearrange test files

* small bug

* adjust function call to save instance in json

* Update allennlp/interpret/influence_interpreters/influence_interpreter.py

Co-authored-by: Evan Pete Walsh <[email protected]>

* Update allennlp/interpret/influence_interpreters/influence_interpreter.py

Co-authored-by: Evan Pete Walsh <[email protected]>

* Update allennlp/interpret/influence_interpreters/influence_interpreter.py

Co-authored-by: Evan Pete Walsh <[email protected]>

* move some documentation of parameters to base class

* delete one comment

* delete one deprecated abstract method

* changing interface

* formatting

* formatting err

* passing mypy

* passing mypy

* passing mypy

* passing mypy

* passing integration test

* passing integration test

* adding a new option to the do-all function

* modifying the callable function to the interface

* update API, fixes

* doc fixes

* add `from_path` and `from_archive` methods

* fix docs, improve logging

* add test

* address @matt-gardner's comments

* fixes to documentation

* update docs

Co-authored-by: Evan Pete Walsh <[email protected]>
Co-authored-by: Evan Pete Walsh <[email protected]>

Author: leo-liuzy

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add new dimension to the `interpret` module: influence functions via the `InfluenceInterpreter` base class, along with a concrete implementation: `SimpleInfluence`.
+- Added a `quiet` parameter to the `MultiProcessDataLoading` that disables `Tqdm` progress bars.
 - The test for distributed metrics now takes a parameter specifying how often you want to run it.
 
 

allennlp/data/__init__.py

@@ -3,7 +3,7 @@
     TensorDict,
     allennlp_collate,
 )
-from allennlp.data.dataset_readers.dataset_reader import DatasetReader
+from allennlp.data.dataset_readers.dataset_reader import DatasetReader, DatasetReaderInput
 from allennlp.data.fields.field import DataArray, Field
 from allennlp.data.fields.text_field import TextFieldTensors
 from allennlp.data.instance import Instance

allennlp/data/data_loaders/multiprocess_data_loader.py

@@ -3,7 +3,7 @@
 from multiprocessing.process import BaseProcess
 import random
 import traceback
-from typing import List, Iterator, Optional, Iterable, Union
+from typing import List, Iterator, Optional, Iterable, Union, TypeVar
 
 from overrides import overrides
 import torch
@@ -23,6 +23,9 @@
 logger = logging.getLogger(__name__)
 
 
+_T = TypeVar("_T")
+
+
 @DataLoader.register("multiprocess")
 class MultiProcessDataLoader(DataLoader):
     """
@@ -118,6 +121,9 @@ class MultiProcessDataLoader(DataLoader):
             will automatically call [`set_target_device()`](#set_target_device) before iterating
             over batches.
 
+    quiet : `bool`, optional (default = `False`)
+        If `True`, tqdm progress bars will be disabled.
+
     # Best practices
 
     - **Large datasets**
@@ -200,6 +206,7 @@ def __init__(
         max_instances_in_memory: int = None,
         start_method: str = "fork",
         cuda_device: Optional[Union[int, str, torch.device]] = None,
+        quiet: bool = False,
     ) -> None:
         # Do some parameter validation.
         if num_workers is not None and num_workers < 0:
@@ -240,6 +247,7 @@ def __init__(
         self.collate_fn = allennlp_collate
         self.max_instances_in_memory = max_instances_in_memory
         self.start_method = start_method
+        self.quiet = quiet
         self.cuda_device: Optional[torch.device] = None
         if cuda_device is not None:
             if not isinstance(cuda_device, torch.device):
@@ -346,7 +354,7 @@ def iter_instances(self) -> Iterator[Instance]:
 
             if self.num_workers <= 0:
                 # Just read all instances in main process.
-                for instance in Tqdm.tqdm(
+                for instance in self._maybe_tqdm(
                     self.reader.read(self.data_path), desc="loading instances"
                 ):
                     self.reader.apply_token_indexers(instance)
@@ -365,7 +373,7 @@ def iter_instances(self) -> Iterator[Instance]:
                 workers = self._start_instance_workers(queue, ctx)
 
                 try:
-                    for instance in Tqdm.tqdm(
+                    for instance in self._maybe_tqdm(
                         self._gather_instances(queue), desc="loading instances"
                     ):
                         if self.max_instances_in_memory is None:
@@ -569,6 +577,11 @@ def _instances_to_batches(
                     break
                 yield tensorize(batch)
 
+    def _maybe_tqdm(self, iterator: Iterable[_T], **tqdm_kwargs) -> Iterable[_T]:
+        if self.quiet:
+            return iterator
+        return Tqdm.tqdm(iterator, **tqdm_kwargs)
+
 
 class WorkerError(Exception):
     """

allennlp/data/data_loaders/simple_data_loader.py

@@ -6,6 +6,7 @@
 import torch
 
 from allennlp.common.util import lazy_groups_of
+from allennlp.common.tqdm import Tqdm
 from allennlp.data.data_loaders.data_loader import DataLoader, allennlp_collate, TensorDict
 from allennlp.data.dataset_readers import DatasetReader
 from allennlp.data.instance import Instance
@@ -37,6 +38,8 @@ def __init__(
         self._batch_generator: Optional[Iterator[TensorDict]] = None
 
     def __len__(self) -> int:
+        if self.batches_per_epoch is not None:
+            return self.batches_per_epoch
         return math.ceil(len(self.instances) / self.batch_size)
 
     @overrides
@@ -87,6 +90,10 @@ def from_dataset_reader(
         batch_size: int,
         shuffle: bool = False,
         batches_per_epoch: Optional[int] = None,
+        quiet: bool = False,
     ) -> "SimpleDataLoader":
-        instances = list(reader.read(data_path))
+        instance_iter = reader.read(data_path)
+        if not quiet:
+            instance_iter = Tqdm.tqdm(instance_iter, desc="loading instances")
+        instances = list(instance_iter)
         return cls(instances, batch_size, shuffle=shuffle, batches_per_epoch=batches_per_epoch)

allennlp/interpret/__init__.py

@@ -1,2 +1,3 @@
 from allennlp.interpret.attackers.attacker import Attacker
 from allennlp.interpret.saliency_interpreters.saliency_interpreter import SaliencyInterpreter
+from allennlp.interpret.influence_interpreters.influence_interpreter import InfluenceInterpreter

allennlp/interpret/influence_interpreters/__init__.py

@@ -0,0 +1,2 @@
+from allennlp.interpret.influence_interpreters.influence_interpreter import InfluenceInterpreter
+from allennlp.interpret.influence_interpreters.simple_influence import SimpleInfluence