[builtin] Fix bugs in mapfile -t and read --raw-line

Caused by a broken underlying ReadLineSlowly()

This is issue #2287, reported originally with ble.sh in #1069.

Author: Andy C

builtin/io_osh.py

@@ -131,12 +131,13 @@ def Run(self, cmd_val):
         while True:
             # bash uses this slow algorithm; YSH could provide read --all-lines
             try:
-                line = read_osh.ReadLineSlowly(self.cmd_ev, with_eol=not arg.t)
+                line, eof = read_osh.ReadLineSlowly(self.cmd_ev,
+                                                    with_eol=not arg.t)
             except pyos.ReadError as e:
                 self.errfmt.PrintMessage("mapfile: read() error: %s" %
                                          posix.strerror(e.err_num))
                 return 1
-            if len(line) == 0:
+            if eof:
                 break
             lines.append(line)
 
@@ -152,7 +153,6 @@ class Cat(vm._Builtin):
 
     def __init__(self):
         # type: () -> None
-        """Empty constructor for mycpp."""
         vm._Builtin.__init__(self)
 
     def Run(self, cmd_val):

builtin/read_osh.py

@@ -155,8 +155,9 @@ def _ReadPortion(delim_byte, max_chars, cmd_ev):
 
     The delimiter is not included in the result.
     """
-    eof = False
     ch_array = []  # type: List[int]
+    eof = False
+
     bytes_read = 0
     while True:
         if max_chars >= 0 and bytes_read >= max_chars:
@@ -186,7 +187,7 @@ def _ReadPortion(delim_byte, max_chars, cmd_ev):
 
 
 def ReadLineSlowly(cmd_ev, with_eol=True):
-    # type: (CommandEvaluator, bool) -> str
+    # type: (CommandEvaluator, bool) -> Tuple[str, bool]
     """Read a line from stdin, unbuffered 
 
     Used by mapfile and read --raw-line.
@@ -196,8 +197,11 @@ def ReadLineSlowly(cmd_ev, with_eol=True):
     with read(0, 1).
     """
     ch_array = []  # type: List[int]
+    eof = False
+    is_first_byte = True
     while True:
         ch, err_num = pyos.ReadByte(0)
+        #log('   ch %d', ch)
 
         if ch < 0:
             if err_num == EINTR:
@@ -207,17 +211,21 @@ def ReadLineSlowly(cmd_ev, with_eol=True):
                 raise pyos.ReadError(err_num)
 
         elif ch == pyos.EOF_SENTINEL:
+            if is_first_byte:
+                eof = True
+            break
+
+        elif ch == pyos.NEWLINE_CH:
+            if with_eol:
+                ch_array.append(ch)
             break
 
         else:
             ch_array.append(ch)
 
-        if ch == pyos.NEWLINE_CH:
-            if not with_eol:
-                ch_array.pop()
-            break
+        is_first_byte = False
 
-    return pyutil.ChArrayToString(ch_array)
+    return pyutil.ChArrayToString(ch_array), eof
 
 
 def ReadAll():
@@ -371,10 +379,10 @@ def _ReadYsh(self, arg, arg_r, cmd_val):
             status = 0
 
         elif arg.raw_line:  # read --raw-line is unbuffered
-            contents = ReadLineSlowly(self.cmd_ev, with_eol=arg.with_eol)
+            contents, eof = ReadLineSlowly(self.cmd_ev, with_eol=arg.with_eol)
             #log('EOF %s', eof)
             #status = 1 if eof else 0
-            status = 0 if len(contents) else 1
+            status = 1 if eof else 0
 
         elif arg.all:  # read --all
             contents = ReadAll()

spec/builtin-bash.test.sh

@@ -144,6 +144,29 @@ printf '%s\r\n' {1..5..2} | {
 ## N-I dash/mksh/zsh/ash STDOUT:
 ## END
 
+#### mapfile -t bugs (ble.sh)
+
+# empty line
+mapfile -t lines <<< $'hello\n\nworld'
+echo len=${#lines[@]}
+#declare -p lines
+
+# initial newline
+mapfile -t lines <<< $'\nhello'
+echo len=${#lines[@]}
+#declare -p lines
+
+# trailing newline
+mapfile -t lines <<< $'hello\n'
+echo len=${#lines[@]}
+#declare -p lines
+
+## STDOUT:
+len=3
+len=2
+len=2
+## END
+
 #### mapfile (store position): -O start
 type mapfile >/dev/null 2>&1 || exit 0
 printf '%s\n' a{0..2} | {

spec/ysh-builtins.test.sh

@@ -172,42 +172,51 @@ len=2
 pass
 ## END
 
-#### read --raw-line handles line without end, --with-eol
+#### read --raw-line in a loop
 
-write --end '' $'a\nb\n' | while read --raw-line; do
+write --end '' $'\na\nb\n\n' | while read --raw-line; do
   pp test_ (_reply)
 done
 
 echo
 
-write --end '' $'a\nb' | while read --raw-line; do
+write --end '' $'a\n\nb' | while read --raw-line; do
   pp test_ (_reply)
 done
 
-echo
 
-write --end '' $'a\nb\n' | while read --raw-line --with-eol; do
+## STDOUT:
+(Str)   ""
+(Str)   "a"
+(Str)   "b"
+(Str)   ""
+
+(Str)   "a"
+(Str)   ""
+(Str)   "b"
+## END
+
+
+#### read --raw-line --with-eol in a loop
+
+write --end '' $'\na\nb\n\n' | while read --raw-line --with-eol; do
   pp test_ (_reply)
 done
 
 echo
 
-write --end '' $'a\nb' | while read --raw-line --with-eol; do
+write --end '' $'a\n\nb' | while read --raw-line --with-eol; do
   pp test_ (_reply)
 done
 
-
 ## STDOUT:
-(Str)   "a"
-(Str)   "b"
-
-(Str)   "a"
-(Str)   "b"
-
+(Str)   "\n"
 (Str)   "a\n"
 (Str)   "b\n"
+(Str)   "\n"
 
 (Str)   "a\n"
+(Str)   "\n"
 (Str)   "b"
 ## END