python · taleinat · Aug 11, 2019 · Aug 11, 2019 · Aug 11, 2019 · Aug 11, 2019
diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
@@ -752,19 +752,21 @@ with 300 the default.
 A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) in
 the BMP (Basic Multilingual Plane) subset of Unicode.  Which characters are
 displayed with a proper glyph and which with a replacement box depends on the
-operating system and installed fonts.  Tab characters cause the following text
-to begin after the next tab stop. (They occur every 8 'characters').  Newline
-characters cause following text to appear on a new line.  Other control
-characters are ignored or displayed as a space, box, or something else,
-depending on the operating system and font.  (Moving the text cursor through
+OS and installed fonts.  Tab characters (``\t``) cause the following text to
+begin after the next tab stop.  (They occur every 8 'characters').  Newline
+characters (``\n``) cause following text to appear on a new line.  Carriage-
+return characters (``\r``) move the cursor back to the beginning of the current
+line, and backspace characters (``\b``) move the cursor back one character.
+Other control characters are ignored or displayed as a space, a box, or
+something else, depending on the OS and font.  (Moving the text cursor through
 such output with arrow keys may exhibit some surprising spacing behavior.) ::
 
    >>> s = 'a\tb\a<\x02><\r>\bc\nd'  # Enter 22 chars.
    >>> len(s)
    14
    >>> s  # Display repr(s)
    'a\tb\x07<\x02><\r>\x08c\nd'
-   >>> print(s, end='')  # Display s as is.
+   >>> print(s)  # Send s to the output
    # Result varies by OS and font.  Try it.
 
 The ``repr`` function is used for interactive echo of expression

diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
@@ -1033,6 +1033,11 @@ tab of the configuration dialog.  Line numbers for an existing
 window are shown and hidden in the Options menu.
 (Contributed by Tal Einat and Saimadhav Heblikar in :issue:`17535`.)
 
+In the shell, ``\r`` and ``\b`` control characters in output are now handled
+much like in terminals, i.e. moving the cursor position.  This allows common
+uses of these control characters, such as progress indicators, to be displayed
+properly rather than flooding the output and eventually slowing the shell down
+to a crawl.  (Contributed by Tal Einat in :issue:`37827`.)
 
 importlib
 ---------

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
@@ -610,6 +610,12 @@ tab of the configuration dialog.  Line numbers for an existing
 window are shown and hidden in the Options menu.
 (Contributed by Tal Einat and Saimadhav Heblikar in :issue:`17535`.)
 
+In the shell, ``\r`` and ``\b`` control characters in output are now handled
+much like in terminals, i.e. moving the cursor position.  This allows common
+uses of these control characters, such as progress indicators, to be displayed
+properly rather than flooding the output and eventually slowing the shell down
+to a crawl.  (Contributed by Tal Einat in :issue:`37827`.)
+
 The changes above have been backported to 3.7 maintenance releases.
 
 

diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
@@ -38,5 +38,91 @@ def test_init(self):
 ##        self.assertIsInstance(ps, pyshell.PyShell)
 
 
+class TestProcessControlChars(unittest.TestCase):
+    def call(self, existing, string, cursor):
+        return pyshell.PyShell._process_control_chars(existing, string, cursor)
+
+    def check(self, existing, string, cursor, expected):
+        self.assertEqual(self.call(existing, string, cursor), expected)
+
+    def check_leniently(self, existing, string, cursor, expected):
+        result = self.call(existing, string, cursor)
+        if existing and not expected[0]:
+            options = [
+                expected,
+                (True, existing[:cursor] + expected[1], expected[2]),
+            ]
+            self.assertIn(result, options)
+        else:
+            self.assertEqual(result, expected)
+
+    def test_empty_written(self):
+        self.check('', '', 0, (False, '', 0))
+        self.check('a', '', 0, (False, '', 0))
+        self.check('a', '', 1, (False, '', 0))
+        for cursor in range(4):
+            with self.subTest(cursor=cursor):
+                self.check('abc', '', cursor, (False, '', 0))
+
+    def test_empty_existing(self):
+        self.check('', 'a', 0, (False, 'a', 0))
+        self.check('', 'ab', 0, (False, 'ab', 0))
+        self.check('', 'abc', 0, (False, 'abc', 0))
+
+    def test_simple_cursor(self):
+        self.check('abc', 'def', 0, (False, 'def', 0))
+        self.check('abc', 'def', 1, (False, 'def', 0))
+        self.check('abc', 'def', 2, (False, 'def', 0))
+
+        self.check('abc', 'def', 3, (False, 'def', 0))
+
+    def test_carriage_return(self):
+        self.check('', 'a\rb', 0, (False, 'b', 0))
+        self.check('', 'abc\rd', 0, (False, 'dbc', 2))
+
+    def test_carriage_return_doesnt_delete(self):
+        # \r should only move the cursor to the beginning of the current line
+        self.check('', 'a\r', 0, (False, 'a', 1))
+        self.check('', 'abc\r', 0, (False, 'abc', 3))
+
+    def test_backspace(self):
+        self.check('', '\ba', 0, (False, 'a', 0))
+        self.check('', 'a\bb', 0, (False, 'b', 0))
+        self.check('', 'ab\bc', 0, (False, 'ac', 0))
+        self.check('', 'ab\bc\bd', 0, (False, 'ad', 0))
+
+        self.check('abc', '\b', 3, (False, '', 1))
+        self.check('abc', '\b', 2, (False, 'c', 2))
+        self.check('abc', '\b', 1, (False, 'bc', 3))
+        self.check('abc', '\b', 0, (False, 'abc', 3))
+
+    def test_backspace_doesnt_delete(self):
+        # \b should only move the cursor one place earlier
+        self.check('', 'a\b', 0, (False, 'a', 1))
+        self.check('', 'a\b\b', 0, (False, 'a', 1))
+        self.check('', 'ab\b\b', 0, (False, 'ab', 2))
+        self.check('', 'ab\b\bc', 0, (False, 'cb', 1))
+        self.check('', 'abc\b\bd', 0, (False, 'adc', 1))
+        self.check('', 'ab\bc\b', 0, (False, 'ac', 1))
+
+    def test_newline(self):
+        self.check('', '\n', 0, (False, '\n', 0))
+        self.check('abc', '\n', 3, (False, '\n', 0))
+        self.check('abc', 'def\n', 3, (False, 'def\n', 0))
+        self.check('abc', '\ndef', 3, (False, '\ndef', 0))
+
+    def test_newline_and_carriage_return(self):
+        self.check('abc', '\n\rdef', 3, (False, '\ndef', 0))
+        self.check('abc', 'd\n\ref', 3, (False, 'd\nef', 0))
+        self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 0))
+        self.check('abc', 'def\n\r', 3, (False, 'def\n', 0))
+
+        self.check_leniently('abc', '\r\n', 3, (False, '\n', 0))
+        self.check_leniently('abc', 'def\r\n', 3, (False, 'def\n', 0))
+        self.check_leniently('abc', '\r\ndef', 3, (False, '\ndef', 0))
+        self.check_leniently('abc', '\rdef\n', 3, (True, 'def\n', 0))
+        self.check_leniently('abc', '\rd\nef', 3, (True, 'dbc\nef', 0))
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
@@ -33,6 +33,7 @@
     raise SystemExit(1)
 
 from code import InteractiveInterpreter
+from io import StringIO
 import linecache
 import os
 import os.path
@@ -1279,45 +1280,170 @@ def showprompt(self):
         self.io.reset_undo()
 
     def show_warning(self, msg):
+        print(f"BEFORE WARNING: iomark={text.index('iomark')}, end={text.index('end')}")
         width = self.interp.tkconsole.width
         wrapper = TextWrapper(width=width, tabsize=8, expand_tabs=True)
         wrapped_msg = '\n'.join(wrapper.wrap(msg))
         if not wrapped_msg.endswith('\n'):
             wrapped_msg += '\n'
         self.per.bottom.insert("iomark linestart", wrapped_msg, "stderr")
+        print(f"AFTER WARNING:  iomark={text.index('iomark')}, end={text.index('end')}")
 
     def resetoutput(self):
         source = self.text.get("iomark", "end-1c")
         if self.history:
             self.history.store(source)
         if self.text.get("end-2c") != "\n":
             self.text.insert("end-1c", "\n")
+        self.text.mark_set("outputmark", "end-2c")
         self.text.mark_set("iomark", "end-1c")
         self.set_line_and_column()
 
-    def write(self, s, tags=()):
-        if isinstance(s, str) and len(s) and max(s) > '\uffff':
-            # Tk doesn't support outputting non-BMP characters
-            # Let's assume what printed string is not very long,
-            # find first non-BMP character and construct informative
-            # UnicodeEncodeError exception.
-            for start, char in enumerate(s):
-                if char > '\uffff':
-                    break
-            raise UnicodeEncodeError("UCS-2", char, start, start+1,
-                                     'Non-BMP character not supported in Tk')
+    def write(self, s, tags=(),
+              _non_bmp_re=re.compile(r'[\U00010000-\U0010ffff]')):
+        if not s:
+            return 0
+
+        if isinstance(s, str):
+            m = _non_bmp_re.search(s)
+            if m is not None:
+                # Tk doesn't support outputting non-BMP characters
+                # Let's assume what printed string is not very long,
+                # find first non-BMP character and construct informative
+                # UnicodeEncodeError exception.
+                raise UnicodeEncodeError(
+                    "UCS-2", s[m.start()], m.start(), m.start() + 1,
+                    'Non-BMP character not supported in Tk')
+
+        text = self.text
+
+        # Process control characters only for stdout and stderr.
+        if isinstance(tags, str):
+            tags = (tags,)
+        cursor = "iomark"
+        if {"stdout", "stderr"} & set(tags):
+            # Get existing output on the current line.
+            if not self.executing:
+                cursor = "outputmark"
+            linestart = f"{cursor} linestart"
+            lineend = f"{cursor} lineend"
+            if not self.executing:
+                prompt_end = text.tag_prevrange("console", linestart, cursor)
+                if prompt_end and text.compare(prompt_end[1], '>', linestart):
+                    linestart = prompt_end[1]
+            existing = text.get(linestart, lineend)
+
+            # Process new output.
+            cursor_pos_in_existing = len(text.get(linestart, cursor))
+            is_cursor_at_lineend = cursor_pos_in_existing == len(existing)
+            rewrite, s, cursor_back = self._process_control_chars(
+                existing, s, cursor_pos_in_existing,
+            )
+        else:
+            is_cursor_at_lineend = True
+            rewrite = False
+            cursor_back = 0
+
+        # Update text widget.
+        text.mark_gravity(cursor, "right")
+        # The shell normally rejects writing and deleting before "iomark"
+        # (achieved by wrapping the text widget), so we temporarily replace
+        # the wrapped text widget object with the unwrapped one.
+        self.text = self.per.bottom
         try:
-            self.text.mark_gravity("iomark", "right")
-            count = OutputWindow.write(self, s, tags, "iomark")
-            self.text.mark_gravity("iomark", "left")
-        except:
-            raise ###pass  # ### 11Aug07 KBK if we are expecting exceptions
-                           # let's find out what they are and be specific.
+            if rewrite or not is_cursor_at_lineend:
+                self.text.delete(linestart if rewrite else cursor, lineend)
+            OutputWindow.write(self, s, tags, cursor)
+        finally:
+            self.text = text
+
+        if cursor_back > 0:
+            text.mark_set(cursor, f"{cursor} -{cursor_back}c")
+        text.mark_gravity(cursor, "left")
+
         if self.canceled:
             self.canceled = 0
             if not use_subprocess:
                 raise KeyboardInterrupt
-        return count
+        return len(s) - (len(existing) if rewrite else 0) - (0 if is_cursor_at_lineend else len(existing) - cursor_pos_in_existing)
+
+    @classmethod
+    def _process_control_chars(cls, existing, string, cursor,
+                               _control_char_re=re.compile(r'[\r\b]+')):
+        if not string:
+            return False, '', 0
+
+        m = _control_char_re.search(string)
+        if m is None:
+            # No control characters in output.
+            res = string + existing[cursor + len(string):]
+            return False, res, 0
+
+        orig_cursor = cursor
+        last_linestart = 0
+        buffer = StringIO(existing)
+        rewrite = False
+        write_to_buffer = cls._process_control_chars_buffer_write
+
+        idx = 0
+        while m is not None:
+            if m.start() > idx:
+                string_part = string[idx:m.start()]
+                rewrite |= cursor < orig_cursor
+                cursor = write_to_buffer(buffer, cursor, string_part)
+
+                # We never write before the last newline, so we must keep
+                # track of the last newline written.
+                new_str_last_newline = string_part.rfind('\n')
+                if new_str_last_newline >= 0:
+                    last_linestart = \
+                        cursor - len(string_part) + new_str_last_newline + 1
+
+            # Process a sequence of control characters. This assumes
+            # that they are all '\r' and/or '\b' characters.
+            control_chars = m.group()
+            cursor = max(
+                last_linestart,
+                0 if '\r' in control_chars else cursor - len(control_chars),
+            )
+
+            idx = m.end()
+            m = _control_char_re.search(string, idx)
+
+        # Handle rest of output after final control character.
+        if idx < len(string):
+            rewrite |= cursor < orig_cursor
+            cursor = write_to_buffer(buffer, cursor, string[idx:])
+        buffer.seek(0, 2)  # seek to end
+        buffer_len = buffer.tell()
+
+        if rewrite:
+            buffer.seek(0)
+            return True, buffer.read(), buffer_len - cursor
+        else:
+            buffer.seek(orig_cursor)
+            return False, buffer.read(), buffer_len - cursor
+
+    @staticmethod
+    def _process_control_chars_buffer_write(buffer, cursor, string):
+        string_first_newline = string.find('\n')
+        buffer.seek(0, 2)  # seek to end
+        buffer_len = buffer.tell()
+        buffer.seek(cursor)
+        if (
+                string_first_newline >= 0 and
+                cursor + string_first_newline < buffer_len
+        ):
+            # We must split the string in order to overwrite just
+            # part of the first line.
+            buffer.write(string[:string_first_newline])
+            buffer.seek(0, 2)  # seek to end
+            buffer.write(string[string_first_newline:])
+        else:
+            buffer.write(string)
+
+        cursor = buffer.tell()
+        return cursor
 
     def rmenu_check_cut(self):
         try:

diff --git a/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst b/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst
@@ -0,0 +1,2 @@
+Emulate terminal handling of ``\r`` and ``\b`` control characters in shell
+outoput.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Emulate terminal handling of ``\r`` and ``\b`` control characters in shell
		outoput.