Data reader for QUAC (#1624)

* quac initial commit

* random files

* l

* pull

*  pushing

* Attention version, but seems to be buggy.. ( no effect on scores )

* all

*  pushing

* upload

* DQA SEQ working version

* Added followup info

* fixed V2 bug

* fixed V2 bug

* question num marker

* push to taranis

* fix eval to match official script

* fixing eval script

* fixing eval

*  eval mismatch bug

*  push to taranis

*  final?

* remote data

* removing

* preparing pull request

* removing diffs

* removing query addition

* mask bug fixed

* cpu support

* fixed cuda

* making dataset only pull request

* adding back some files

*  removing tag

* moved

* added api doc

* adding dataset rst

* addressed comments from deneutoy

* fixed some lint errors

* fixed doc api, wrong continuation from pylint

* fixing mypy errors

* fixing one more mypy error

* removing repeated doc

* fixed mypy error

* last minute changes

* indented the data

Author: eunsol

allennlp/data/dataset_readers/__init__.py

@@ -16,7 +16,7 @@
 from allennlp.data.dataset_readers.language_modeling import LanguageModelingReader
 from allennlp.data.dataset_readers.nlvr import NlvrDatasetReader
 from allennlp.data.dataset_readers.penn_tree_bank import PennTreeBankConstituencySpanDatasetReader
-from allennlp.data.dataset_readers.reading_comprehension import SquadReader, TriviaQaReader
+from allennlp.data.dataset_readers.reading_comprehension import SquadReader, TriviaQaReader, QuACReader
 from allennlp.data.dataset_readers.semantic_role_labeling import SrlReader
 from allennlp.data.dataset_readers.seq2seq import Seq2SeqDatasetReader
 from allennlp.data.dataset_readers.sequence_tagging import SequenceTaggingDatasetReader

allennlp/data/dataset_readers/reading_comprehension/__init__.py

@@ -6,4 +6,5 @@
 """
 
 from allennlp.data.dataset_readers.reading_comprehension.squad import SquadReader
+from allennlp.data.dataset_readers.reading_comprehension.quac import QuACReader
 from allennlp.data.dataset_readers.reading_comprehension.triviaqa import TriviaQaReader

allennlp/data/dataset_readers/reading_comprehension/quac.py

@@ -0,0 +1,130 @@
+import json
+import logging
+from typing import Any, Dict, List, Tuple
+
+from overrides import overrides
+
+from allennlp.common.file_utils import cached_path
+from allennlp.data.dataset_readers.dataset_reader import DatasetReader
+from allennlp.data.instance import Instance
+from allennlp.data.dataset_readers.reading_comprehension import util
+from allennlp.data.token_indexers import SingleIdTokenIndexer, TokenIndexer
+from allennlp.data.tokenizers import Token, Tokenizer, WordTokenizer
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+
+@DatasetReader.register("quac")
+class QuACReader(DatasetReader):
+    """
+    Reads a JSON-formatted Quesiton Answering in Context (QuAC) data file
+    and returns a ``Dataset`` where the ``Instances`` have four fields: ``question``, a ``ListField``,
+    ``passage``, another ``TextField``, and ``span_start`` and ``span_end``, both ``ListField`` composed of
+    IndexFields`` into the ``passage`` ``TextField``.
+    Two ``ListField``, composed of ``LabelField``, ``yesno_list`` and  ``followup_list`` is added.
+    We also add a
+    ``MetadataField`` that stores the instance's ID, the original passage text, gold answer strings,
+    and token offsets into the original passage, accessible as ``metadata['id']``,
+    ``metadata['original_passage']``, ``metadata['answer_text_lists'] and ``metadata['token_offsets']``.
+
+    Parameters
+    ----------
+    tokenizer : ``Tokenizer``, optional (default=``WordTokenizer()``)
+        We use this ``Tokenizer`` for both the question and the passage.  See :class:`Tokenizer`.
+        Default is ```WordTokenizer()``.
+    token_indexers : ``Dict[str, TokenIndexer]``, optional
+        We similarly use this for both the question and the passage.  See :class:`TokenIndexer`.
+        Default is ``{"tokens": SingleIdTokenIndexer()}``.
+    num_context_answers : ``int``, optional
+        How many previous question answers to consider in a context.
+    """
+
+    def __init__(self,
+                 tokenizer: Tokenizer = None,
+                 token_indexers: Dict[str, TokenIndexer] = None,
+                 lazy: bool = False,
+                 num_context_answers: int = 0) -> None:
+        super().__init__(lazy)
+        self._tokenizer = tokenizer or WordTokenizer()
+        self._token_indexers = token_indexers or {'tokens': SingleIdTokenIndexer()}
+        self._num_context_answers = num_context_answers
+
+    @overrides
+    def _read(self, file_path: str):
+        # if `file_path` is a URL, redirect to the cache
+        file_path = cached_path(file_path)
+        logger.info("Reading file at %s", file_path)
+        with open(file_path) as dataset_file:
+            dataset_json = json.load(dataset_file)
+            dataset = dataset_json['data']
+        logger.info("Reading the dataset")
+        for article in dataset:
+            for paragraph_json in article['paragraphs']:
+                paragraph = paragraph_json["context"]
+                tokenized_paragraph = self._tokenizer.tokenize(paragraph)
+                qas = paragraph_json['qas']
+                metadata = {}
+                metadata["instance_id"] = [qa['id'] for qa in qas]
+                question_text_list = [qa["question"].strip().replace("\n", "") for qa in qas]
+                answer_texts_list = [[answer['text'] for answer in qa['answers']] for qa in qas]
+                metadata["question"] = question_text_list
+                metadata['answer_texts_list'] = answer_texts_list
+                span_starts_list = [[answer['answer_start'] for answer in qa['answers']] for qa in qas]
+                span_ends_list = []
+                for answer_starts, an_list in zip(span_starts_list, answer_texts_list):
+                    span_ends = [start + len(answer) for start, answer in zip(answer_starts, an_list)]
+                    span_ends_list.append(span_ends)
+                yesno_list = [str(qa['yesno']) for qa in qas]
+                followup_list = [str(qa['followup']) for qa in qas]
+                instance = self.text_to_instance(question_text_list,
+                                                 paragraph,
+                                                 span_starts_list,
+                                                 span_ends_list,
+                                                 tokenized_paragraph,
+                                                 yesno_list,
+                                                 followup_list,
+                                                 metadata)
+                yield instance
+
+    @overrides
+    def text_to_instance(self,  # type: ignore
+                         question_text_list: List[str],
+                         passage_text: str,
+                         start_span_list: List[List[int]] = None,
+                         end_span_list: List[List[int]] = None,
+                         passage_tokens: List[Token] = None,
+                         yesno_list: List[int] = None,
+                         followup_list: List[int] = None,
+                         additional_metadata: Dict[str, Any] = None) -> Instance:
+        # pylint: disable=arguments-differ
+        # We need to convert character indices in `passage_text` to token indices in
+        # `passage_tokens`, as the latter is what we'll actually use for supervision.
+        answer_token_span_list = []
+        passage_offsets = [(token.idx, token.idx + len(token.text)) for token in passage_tokens]
+        for start_list, end_list in zip(start_span_list, end_span_list):
+            token_spans: List[Tuple[int, int]] = []
+            for char_span_start, char_span_end in zip(start_list, end_list):
+                (span_start, span_end), error = util.char_span_to_token_span(passage_offsets,
+                                                                             (char_span_start, char_span_end))
+                if error:
+                    logger.debug("Passage: %s", passage_text)
+                    logger.debug("Passage tokens: %s", passage_tokens)
+                    logger.debug("Answer span: (%d, %d)", char_span_start, char_span_end)
+                    logger.debug("Token span: (%d, %d)", span_start, span_end)
+                    logger.debug("Tokens in answer: %s", passage_tokens[span_start:span_end + 1])
+                    logger.debug("Answer: %s", passage_text[char_span_start:char_span_end])
+                token_spans.append((span_start, span_end))
+            answer_token_span_list.append(token_spans)
+        question_list_tokens = [self._tokenizer.tokenize(q) for q in question_text_list]
+        # Map answer texts to "CANNOTANSWER" if more than half of them marked as so.
+        additional_metadata['answer_texts_list'] = [util.handle_cannot(ans_list) for ans_list \
+                                                    in additional_metadata['answer_texts_list']]
+        return util.make_reading_comprehension_instance_quac(question_list_tokens,
+                                                             passage_tokens,
+                                                             self._token_indexers,
+                                                             passage_text,
+                                                             answer_token_span_list,
+                                                             yesno_list,
+                                                             followup_list,
+                                                             additional_metadata,
+                                                             self._num_context_answers)

allennlp/data/dataset_readers/reading_comprehension/util.py

@@ -7,7 +7,8 @@
 import string
 from typing import Any, Dict, List, Tuple
 
-from allennlp.data.fields import Field, TextField, IndexField, MetadataField
+from allennlp.data.fields import Field, TextField, IndexField, \
+    MetadataField, LabelField, ListField, SequenceLabelField
 from allennlp.data.instance import Instance
 from allennlp.data.token_indexers import TokenIndexer
 from allennlp.data.tokenizers import Token
@@ -19,6 +20,7 @@
 IGNORED_TOKENS = {'a', 'an', 'the'}
 STRIPPED_CHARACTERS = string.punctuation + ''.join([u"‘", u"’", u"´", u"`", "_"])
 
+
 def normalize_text(text: str) -> str:
     """
     Performs a normalization that is very similar to that done by the normalization functions in
@@ -187,12 +189,9 @@ def make_reading_comprehension_instance(question_tokens: List[Token],
     passage_field = TextField(passage_tokens, token_indexers)
     fields['passage'] = passage_field
     fields['question'] = TextField(question_tokens, token_indexers)
-    metadata = {
-            'original_passage': passage_text,
-            'token_offsets': passage_offsets,
-            'question_tokens': [token.text for token in question_tokens],
-            'passage_tokens': [token.text for token in passage_tokens],
-            }
+    metadata = {'original_passage': passage_text, 'token_offsets': passage_offsets,
+                'question_tokens': [token.text for token in question_tokens],
+                'passage_tokens': [token.text for token in passage_tokens], }
     if answer_texts:
         metadata['answer_texts'] = answer_texts
 
@@ -213,3 +212,160 @@ def make_reading_comprehension_instance(question_tokens: List[Token],
     metadata.update(additional_metadata)
     fields['metadata'] = MetadataField(metadata)
     return Instance(fields)
+
+
+def make_reading_comprehension_instance_quac(question_list_tokens: List[List[Token]],
+                                             passage_tokens: List[Token],
+                                             token_indexers: Dict[str, TokenIndexer],
+                                             passage_text: str,
+                                             token_span_lists: List[List[Tuple[int, int]]] = None,
+                                             yesno_list: List[int] = None,
+                                             followup_list: List[int] = None,
+                                             additional_metadata: Dict[str, Any] = None,
+                                             num_context_answers: int = 0) -> Instance:
+    """
+    Converts a question, a passage, and an optional answer (or answers) to an ``Instance`` for use
+    in a reading comprehension model.
+
+    Creates an ``Instance`` with at least these fields: ``question`` and ``passage``, both
+    ``TextFields``; and ``metadata``, a ``MetadataField``.  Additionally, if both ``answer_texts``
+    and ``char_span_starts`` are given, the ``Instance`` has ``span_start`` and ``span_end``
+    fields, which are both ``IndexFields``.
+
+    Parameters
+    ----------
+    question_list_tokens : ``List[List[Token]]``
+        An already-tokenized list of questions. Each dialog have multiple questions.
+    passage_tokens : ``List[Token]``
+        An already-tokenized passage that contains the answer to the given question.
+    token_indexers : ``Dict[str, TokenIndexer]``
+        Determines how the question and passage ``TextFields`` will be converted into tensors that
+        get input to a model.  See :class:`TokenIndexer`.
+    passage_text : ``str``
+        The original passage text.  We need this so that we can recover the actual span from the
+        original passage that the model predicts as the answer to the question.  This is used in
+        official evaluation scripts.
+    token_spans_lists : ``List[List[Tuple[int, int]]]``, optional
+        Indices into ``passage_tokens`` to use as the answer to the question for training.  This is
+        a list of list, first because there is multiple questions per dialog, and
+        because there might be several possible correct answer spans in the passage.
+        Currently, we just select the last span in this list (i.e., QuAC has multiple
+        annotations on the dev set; this will select the last span, which was given by the original annotator).
+    yesno_list : ``List[int]``
+        List of the affirmation bit for each question answer pairs.
+    followup_list : ``List[int]``
+        List of the continuation bit for each question answer pairs.
+    num_context_answers : ``int``, optional
+        How many answers to encode into the passage.
+    additional_metadata : ``Dict[str, Any]``, optional
+        The constructed ``metadata`` field will by default contain ``original_passage``,
+        ``token_offsets``, ``question_tokens``, ``passage_tokens``, and ``answer_texts`` keys.  If
+        you want any other metadata to be associated with each instance, you can pass that in here.
+        This dictionary will get added to the ``metadata`` dictionary we already construct.
+    """
+    additional_metadata = additional_metadata or {}
+    fields: Dict[str, Field] = {}
+    passage_offsets = [(token.idx, token.idx + len(token.text)) for token in passage_tokens]
+    # This is separate so we can reference it later with a known type.
+    passage_field = TextField(passage_tokens, token_indexers)
+    fields['passage'] = passage_field
+    fields['question'] = ListField([TextField(q_tokens, token_indexers) for q_tokens in question_list_tokens])
+    metadata = {'original_passage': passage_text,
+                'token_offsets': passage_offsets,
+                'question_tokens': [[token.text for token in question_tokens] \
+                                    for question_tokens in question_list_tokens],
+                'passage_tokens': [token.text for token in passage_tokens], }
+    p1_answer_marker_list: List[Field] = []
+    p2_answer_marker_list: List[Field] = []
+    p3_answer_marker_list: List[Field] = []
+
+    def get_tag(i, i_name):
+        # Generate a tag to mark previous answer span in the passage.
+        return "<{0:d}_{1:s}>".format(i, i_name)
+
+    def mark_tag(span_start, span_end, passage_tags, prev_answer_distance):
+        try:
+            assert span_start > 0
+            assert span_end > 0
+        except:
+            raise ValueError("Previous {0:d}th answer span should have been updated!".format(prev_answer_distance))
+        # Modify "tags" to mark previous answer span.
+        if span_start == span_end:
+            passage_tags[prev_answer_distance][span_start] = get_tag(prev_answer_distance, "")
+        else:
+            passage_tags[prev_answer_distance][span_start] = get_tag(prev_answer_distance, "start")
+            passage_tags[prev_answer_distance][span_end] = get_tag(prev_answer_distance, "end")
+            for passage_index in range(span_start + 1, span_end):
+                passage_tags[prev_answer_distance][passage_index] = get_tag(prev_answer_distance, "in")
+
+    if token_span_lists:
+        span_start_list: List[Field] = []
+        span_end_list: List[Field] = []
+        p1_span_start, p1_span_end, p2_span_start = -1, -1, -1
+        p2_span_end, p3_span_start, p3_span_end = -1, -1, -1
+        # Looping each <<answers>>.
+        for question_index, answer_span_lists in enumerate(token_span_lists):
+            span_start, span_end = answer_span_lists[-1]  # Last one is the original answer
+            span_start_list.append(IndexField(span_start, passage_field))
+            span_end_list.append(IndexField(span_end, passage_field))
+            prev_answer_marker_lists = [["O"] * len(passage_tokens), ["O"] * len(passage_tokens),
+                                        ["O"] * len(passage_tokens), ["O"] * len(passage_tokens)]
+            if question_index > 0 and num_context_answers > 0:
+                mark_tag(p1_span_start, p1_span_end, prev_answer_marker_lists, 1)
+                if question_index > 1 and num_context_answers > 1:
+                    mark_tag(p2_span_start, p2_span_end, prev_answer_marker_lists, 2)
+                    if question_index > 2 and num_context_answers > 2:
+                        mark_tag(p3_span_start, p3_span_end, prev_answer_marker_lists, 3)
+                    p3_span_start = p2_span_start
+                    p3_span_end = p2_span_end
+                p2_span_start = p1_span_start
+                p2_span_end = p1_span_end
+            p1_span_start = span_start
+            p1_span_end = span_end
+            if num_context_answers > 2:
+                p3_answer_marker_list.append(SequenceLabelField(prev_answer_marker_lists[3],
+                                                                passage_field,
+                                                                label_namespace="answer_tags"))
+            if num_context_answers > 1:
+                p2_answer_marker_list.append(SequenceLabelField(prev_answer_marker_lists[2],
+                                                                passage_field,
+                                                                label_namespace="answer_tags"))
+            if num_context_answers > 0:
+                p1_answer_marker_list.append(SequenceLabelField(prev_answer_marker_lists[1],
+                                                                passage_field,
+                                                                label_namespace="answer_tags"))
+        fields['span_start'] = ListField(span_start_list)
+        fields['span_end'] = ListField(span_end_list)
+        if num_context_answers > 0:
+            fields['p1_answer_marker'] = ListField(p1_answer_marker_list)
+            if num_context_answers > 1:
+                fields['p2_answer_marker'] = ListField(p2_answer_marker_list)
+                if num_context_answers > 2:
+                    fields['p3_answer_marker'] = ListField(p3_answer_marker_list)
+        fields['yesno_list'] = ListField( \
+            [LabelField(yesno, label_namespace="yesno_labels") for yesno in yesno_list])
+        fields['followup_list'] = ListField([LabelField(followup, label_namespace="followup_labels") \
+                                             for followup in followup_list])
+    metadata.update(additional_metadata)
+    fields['metadata'] = MetadataField(metadata)
+    return Instance(fields)
+
+
+def handle_cannot(reference_answers: List[str]):
+    """
+    Process a list of reference answers.
+    If equal or more than half of the reference answers are "CANNOTANSWER", take it as gold.
+    Otherwise, return answers that are not "CANNOTANSWER".
+    """
+    num_cannot = 0
+    num_spans = 0
+    for ref in reference_answers:
+        if ref == 'CANNOTANSWER':
+            num_cannot += 1
+        else:
+            num_spans += 1
+    if num_cannot >= num_spans:
+        reference_answers = ['CANNOTANSWER']
+    else:
+        reference_answers = [x for x in reference_answers if x != 'CANNOTANSWER']
+    return reference_answers