Signed-off-by: Abhishek P (VMware) <[email protected]>

Converted HFDatasetSplitReader to HFDatasetReader
Now all splits can be used in the same reader
Support for both pre-load of all splits or on demand load of the split
Reduced tests to glue-cola dataset:config which is ~ 0.36MB download

Author: Abhishek-P

allennlp/data/dataset_readers/huggingface_datasets_reader.py

@@ -8,7 +8,7 @@
 from datasets.features import Value
 
 # TODO pab complete the documentation comments
-class HuggingfaceDatasetSplitReader(DatasetReader):
+class HuggingfaceDatasetReader(DatasetReader):
     """
     This reader implementation wraps the huggingface datasets package
     to utilize it's dataset management functionality and load the information in AllenNLP friendly formats

setup.py

@@ -73,7 +73,7 @@
         "lmdb",
         "more-itertools",
         "wandb>=0.10.0,<0.11.0",
-        "datasets>=1.5.0,<1.6.0",
+        "datasets>=1.5.0,",
     ],
     entry_points={"console_scripts": ["allennlp=allennlp.__main__:run"]},
     include_package_data=True,

tests/data/dataset_readers/huggingface_datasets_test.py

@@ -1,6 +1,6 @@
 import pytest
 
-from allennlp.data.dataset_readers.huggingface_datasets_reader import HuggingfaceDatasetSplitReader
+from allennlp.data.dataset_readers.huggingface_datasets_reader import HuggingfaceDatasetReader
 import logging
 
 logger = logging.getLogger(__name__)
@@ -15,7 +15,7 @@ class HuggingfaceDatasetSplitReaderTest:
     """
     @pytest.mark.parametrize("dataset, config, split", (("glue", "cola", "train"), ("glue", "cola", "test")))
     def test_read_for_datasets_requiring_config(self, dataset, config, split):
-        huggingface_reader = HuggingfaceDatasetSplitReader(dataset_name=dataset, config_name=config)
+        huggingface_reader = HuggingfaceDatasetReader(dataset_name=dataset, config_name=config)
         instances = list(huggingface_reader.read(split))
         assert len(instances) == len(huggingface_reader.datasets[split])
         print(instances[0], print(huggingface_reader.datasets[split][0]))