syntax: fix literal extraction for 'ab??'

Previously, 'ab??' returned [Complete(ab), Complete(a)], but the order
matters here because of greediness. The correct result is [Complete(a),
Complete(ab)].

Instead of trying to actually fix literal extraction (which is a mess),
we just rewrite 'ab?' (and 'ab??') as 'ab*'. 'ab*' still produces
literals in the incorrect order, i.e., [Cut(ab), Complete(a)], but since
one is cut we are guaranteed that the regex engine will be called to
confirm the match. In so doing, it will correctly report 'a' as a match
for 'ab??' in 'ab'.

Fixes #862

Author: BurntSushi

CHANGELOG.md

@@ -6,6 +6,8 @@ The below are changes for the next release, which is to be determined.
   Fixes a bug where `[[:alnum:][:^ascii:]]` dropped `[:alnum:]` from the class.
 * [BUG #859](https://github.com/rust-lang/regex/issues/859):
   Fixes a bug where `Hir::is_match_empty` returned `false` for `\b`.
+* [BUG #862](https://github.com/rust-lang/regex/issues/862):
+  Fixes a bug where 'ab??' matches 'ab' instead of 'a' in 'ab'.
 
 
 1.5.5 (2022-03-08)

regex-syntax/src/hir/literal/mod.rs

@@ -735,18 +735,18 @@ fn repeat_zero_or_one_literals<F: FnMut(&Hir, &mut Literals)>(
     lits: &mut Literals,
     mut f: F,
 ) {
-    let (mut lits2, mut lits3) = (lits.clone(), lits.to_empty());
-    lits3.set_limit_size(lits.limit_size() / 2);
-    f(e, &mut lits3);
-
-    if lits3.is_empty() || !lits2.cross_product(&lits3) {
-        lits.cut();
-        return;
-    }
-    lits2.add(Literal::empty());
-    if !lits.union(lits2) {
-        lits.cut();
-    }
+    f(
+        &Hir::repetition(hir::Repetition {
+            kind: hir::RepetitionKind::ZeroOrMore,
+            // FIXME: Our literal extraction doesn't care about greediness.
+            // Which is partially why we're treating 'e?' as 'e*'. Namely,
+            // 'ab??' yields [Complete(ab), Complete(a)], but it should yield
+            // [Complete(a), Complete(ab)] because of the non-greediness.
+            greedy: true,
+            hir: Box::new(e.clone()),
+        }),
+        lits,
+    );
 }
 
 fn repeat_zero_or_more_literals<F: FnMut(&Hir, &mut Literals)>(
@@ -1141,6 +1141,11 @@ mod tests {
     test_lit!(pfx_group1, prefixes, "(a)", M("a"));
     test_lit!(pfx_rep_zero_or_one1, prefixes, "a?");
     test_lit!(pfx_rep_zero_or_one2, prefixes, "(?:abc)?");
+    test_lit!(pfx_rep_zero_or_one_cat1, prefixes, "ab?", C("ab"), M("a"));
+    // FIXME: This should return [M("a"), M("ab")] because of the non-greedy
+    // repetition. As a work-around, we rewrite ab?? as ab*?, and thus we get
+    // a cut literal.
+    test_lit!(pfx_rep_zero_or_one_cat2, prefixes, "ab??", C("ab"), M("a"));
     test_lit!(pfx_rep_zero_or_more1, prefixes, "a*");
     test_lit!(pfx_rep_zero_or_more2, prefixes, "(?:abc)*");
     test_lit!(pfx_rep_one_or_more1, prefixes, "a+", C("a"));
@@ -1249,8 +1254,8 @@ mod tests {
         pfx_crazy1,
         prefixes,
         r"M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy]",
-        C("Mo\\'am"),
-        C("Mu\\'am"),
+        C("Mo\\'"),
+        C("Mu\\'"),
         C("Moam"),
         C("Muam")
     );

tests/regression.rs

@@ -217,3 +217,6 @@ matiter!(
 // https://en.wikipedia.org/wiki/Je_(Cyrillic)
 ismatch!(empty_group_match, r"()Ј01", "zЈ01", true);
 matiter!(empty_group_find, r"()Ј01", "zЈ01", (1, 5));
+
+// See: https://github.com/rust-lang/regex/issues/862
+mat!(non_greedy_question_literal, r"ab??", "ab", Some((0, 1)));