Move some scripts to allennlp/allennlp/tools (#2584)

* Fix bug in uniform_unit_scaling #2239 (#2273)

* Fix type annotation for .forward(...) in tutorial (#2122)

* Add a Contributions section to README.md (#2277)

* script for doing archive surgery (#2223)

* script for doing archive surgery

* simplify script

* Fix spelling in tutorial README (#2283)

* fix #2285 (#2286)

* Update the `find-lr` subcommand help text. (#2289)

* Update the elmo command help text.

* Update the find-lr subcommand help text.

* Add __repr__ to Vocabulary (#2293)

As it currently stands, the following is logged during training:

```
2019-01-06 10:46:21,832 - INFO - allennlp.common.from_params - instantiating class <class 'allennlp.model
s.language_model.LanguageModel'> from params {'bidirectional': False, 'contextualizer': {'bidirectional':
 False, 'dropout': 0.5, 'hidden_size': 200, 'input_size': 200, 'num_layers': 2, 'type': 'lstm'}, 'dropout
': 0.5, 'text_field_embedder': {'token_embedders': {'tokens': {'embedding_dim': 200, 'type': 'embedding'}
}}} and extras {'vocab': <allennlp.data.vocabulary.Vocabulary object at 0x7ff7811665f8>}
```

Note that the `Vocabulary` does not provide any useful information, since it doesn't have `__repr__` defined. This provides a fix.

* Update the base image in the Dockerfiles. (#2298)

* Don't deprecate bidirectional-language-model name (#2297)

* bump version number to v0.8.1

* Bump version numbers to v0.8.2-unreleased

* Turn BidirectionalLM into a more-general LanguageModel class (#2264)

Fixes #2255

This PR replaces the `BidirectionalLM` class with a more-general `LanguageModel` that can be used in either the unidirectional/forward setting or the bidirectional setting.

It also accordingly replaces the `BidirectionalLanguageModelTokenEmbedder` with a `LanguageModelTokenEmbedder`.

Also fixes bug in the experiment_unsampled.jsonnet config that was preventing a test from actually being unsampled.

TODO:

- [x] test the unidirectional case
- [x] properly deprecate `BidirectionalLM` and `BidirectionalLanguageModelTokenEmbedder` 
- [x] check docs for accuracy
- [x] fix user-facing training configs

* move some utilities from allennlp/scripts to allennlp/allennlp/tools

* make pylint happy

* add modules to API doc

Author: WrRan

scripts/archive_surgery.py renamed to allennlp/tools/archive_surgery.py

@@ -1,20 +1,16 @@
 # pylint: disable=no-self-use
-
-import os
 import argparse
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(os.path.join(__file__, os.pardir))))
-
 import gzip
+import os
+
 import numpy
 import torch
 
-from allennlp.data.token_indexers import ELMoTokenCharactersIndexer
-from allennlp.modules.elmo import _ElmoCharacterEncoder
+from allennlp.common.checks import ConfigurationError
 from allennlp.data import Token, Vocabulary
+from allennlp.data.token_indexers import ELMoTokenCharactersIndexer
 from allennlp.data.vocabulary import DEFAULT_OOV_TOKEN
-from allennlp.common.checks import ConfigurationError
+from allennlp.modules.elmo import _ElmoCharacterEncoder
 
 
 def main(vocab_path: str,
@@ -97,9 +93,12 @@ def main(vocab_path: str,
         for word in tokens:
             new_vocab_file.write(f"{word}\n")
 
+
 if __name__ == "__main__":
+    # pylint: disable=invalid-name
     parser = argparse.ArgumentParser(description='Generate CNN representations for a vocabulary '
-                                                 'using ELMo')
+                                                 'using ELMo',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('--vocab_path', type=str, help='A path to a vocabulary file to generate '
                                                        'representations for.')
     parser.add_argument('--elmo_config', type=str, help='The path to a directory containing an '

scripts/create_elmo_embeddings_from_vocab.py renamed to allennlp/tools/create_elmo_embeddings_from_vocab.py

@@ -0,0 +1,29 @@
+import os
+
+from allennlp.common.file_utils import CACHE_DIRECTORY
+from allennlp.common.file_utils import filename_to_url
+
+
+def main():
+    print(f"Looking for datasets in {CACHE_DIRECTORY}...")
+    if not os.path.exists(CACHE_DIRECTORY):
+        print('Directory does not exist.')
+        print('No cached datasets found.')
+
+    cached_files = os.listdir(CACHE_DIRECTORY)
+
+    if not cached_files:
+        print('Directory is empty.')
+        print('No cached datasets found.')
+
+    for filename in cached_files:
+        if not filename.endswith("json"):
+            url, etag = filename_to_url(filename)
+            print('Filename: %s' % filename)
+            print('Url: %s' % url)
+            print('ETag: %s' % etag)
+            print()
+
+
+if __name__ == '__main__':
+    main()

allennlp/tools/inspect_cache.py

@@ -23,3 +23,18 @@ tasks for which we build models.
    :members:
    :undoc-members:
    :show-inheritance:
+
+.. automodule:: allennlp.tools.archive_surgery
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+.. automodule:: allennlp.tools.create_elmo_embeddings_from_vocab
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+.. automodule:: allennlp.tools.inspect_cache
+   :members:
+   :undoc-members:
+   :show-inheritance: