From 443db0199749ae5895b8a80e9b58285a844c9050 Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Sun, 11 Aug 2019 09:47:47 +0300
Subject: [PATCH 01/13] optimize PyShell.write()

Also remove unnecessary (and very old) try-except-raise.
---
 Lib/idlelib/pyshell.py | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 87401f33f55f16..2f059195582e6c 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1295,24 +1295,23 @@ def resetoutput(self):
         self.text.mark_set("iomark", "end-1c")
         self.set_line_and_column()
 
-    def write(self, s, tags=()):
-        if isinstance(s, str) and len(s) and max(s) > '\uffff':
-            # Tk doesn't support outputting non-BMP characters
-            # Let's assume what printed string is not very long,
-            # find first non-BMP character and construct informative
-            # UnicodeEncodeError exception.
-            for start, char in enumerate(s):
-                if char > '\uffff':
-                    break
-            raise UnicodeEncodeError("UCS-2", char, start, start+1,
-                                     'Non-BMP character not supported in Tk')
-        try:
-            self.text.mark_gravity("iomark", "right")
-            count = OutputWindow.write(self, s, tags, "iomark")
-            self.text.mark_gravity("iomark", "left")
-        except:
-            raise ###pass  # ### 11Aug07 KBK if we are expecting exceptions
-                           # let's find out what they are and be specific.
+    def write(self, s, tags=(),
+              _non_bmp_re=re.compile(r'[\U00010000-\U0010ffff]')):
+        if isinstance(s, str) and s:
+            m = _non_bmp_re.search(s)
+            if m is not None:
+                # Tk doesn't support outputting non-BMP characters
+                # Let's assume what printed string is not very long,
+                # find first non-BMP character and construct informative
+                # UnicodeEncodeError exception.
+                raise UnicodeEncodeError(
+                    "UCS-2", s[m.start()], m.start(), m.start() + 1,
+                    'Non-BMP character not supported in Tk')
+
+        self.text.mark_gravity("iomark", "right")
+        count = OutputWindow.write(self, s, tags, "iomark")
+        self.text.mark_gravity("iomark", "left")
+
         if self.canceled:
             self.canceled = 0
             if not use_subprocess:

From ebf51d0d9fff4ba1c93d4fc38c6d4aaaea4307dd Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Sun, 11 Aug 2019 11:24:43 +0300
Subject: [PATCH 02/13] first working version of control char handling in shell

---
 Lib/idlelib/pyshell.py | 52 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 48 insertions(+), 4 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 2f059195582e6c..481fab7f4dce27 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -33,6 +33,7 @@
     raise SystemExit(1)
 
 from code import InteractiveInterpreter
+from io import StringIO
 import linecache
 import os
 import os.path
@@ -1308,15 +1309,58 @@ def write(self, s, tags=(),
                     "UCS-2", s[m.start()], m.start(), m.start() + 1,
                     'Non-BMP character not supported in Tk')
 
-        self.text.mark_gravity("iomark", "right")
-        count = OutputWindow.write(self, s, tags, "iomark")
-        self.text.mark_gravity("iomark", "left")
+        text = self.text
+
+        idx1 = text.index("iomark linestart")
+        idx2 = text.tag_prevrange("console", idx1)
+        idx = idx2[1] if idx2 and text.compare(idx1, '<', idx2[1]) else idx1
+        buffer = text.get(idx, "iomark")
+        buffer_changed, new_buffer = self._process_control_chars(buffer, s)
+
+        text.mark_gravity("iomark", "right")
+        if buffer_changed:
+            self.per.bottom.delete(idx, "iomark")
+        OutputWindow.write(self, new_buffer, tags, "iomark")
+        text.mark_gravity("iomark", "left")
 
         if self.canceled:
             self.canceled = 0
             if not use_subprocess:
                 raise KeyboardInterrupt
-        return count
+        return len(new_buffer) - len(buffer)
+
+    def _process_control_chars(self, buffer, s,
+                               _control_char_re=re.compile(r'[\r\b]')):
+        buffer_idx = buffer_len = orig_buffer_len = len(buffer)
+        buffer = StringIO(buffer)
+        orig_buffer_changed = False
+        idx = 0
+        for m in _control_char_re.finditer(s):
+            char_idx = m.start()
+            if char_idx > idx:
+                buffer.seek(max(0, buffer_idx))
+                buffer.write(s[idx:m.start()])
+                orig_buffer_changed |= buffer_idx < orig_buffer_len
+                buffer_idx = buffer_len = buffer.tell()
+            if s[char_idx] == '\b':
+                buffer_idx -= 1
+            else:  # '\r'
+                buffer_idx = 0
+            idx = m.end()
+        if idx == 0:
+            return False, s
+        if buffer_idx < buffer_len:
+            orig_buffer_changed |= buffer_idx < orig_buffer_len
+            buffer_len = max(0, buffer_idx)
+            buffer.seek(buffer_len)
+        buffer.write(s[idx:])
+        buffer_len += len(s) - idx
+        if orig_buffer_changed:
+            buffer.seek(0)
+            return True, buffer.read(buffer_len)
+        else:
+            buffer.seek(orig_buffer_len)
+            return False, buffer.read()
 
     def rmenu_check_cut(self):
         try:

From b611193e0f56e4b642af3b996ae0e529e05b8bc5 Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Sun, 11 Aug 2019 12:07:21 +0300
Subject: [PATCH 03/13] improved, clarified, plus fixes

---
 Lib/idlelib/pyshell.py | 70 +++++++++++++++++++++++++++---------------
 1 file changed, 46 insertions(+), 24 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 481fab7f4dce27..baab5d88264bdd 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1311,56 +1311,78 @@ def write(self, s, tags=(),
 
         text = self.text
 
-        idx1 = text.index("iomark linestart")
-        idx2 = text.tag_prevrange("console", idx1)
-        idx = idx2[1] if idx2 and text.compare(idx1, '<', idx2[1]) else idx1
-        buffer = text.get(idx, "iomark")
-        buffer_changed, new_buffer = self._process_control_chars(buffer, s)
+        # Process control characters only for stdout and stderr.
+        if isinstance(tags, str): tags = (tags,)
+        if {"stdout", "stderr"} & set(tags):
+            # Get existing output on the current line.
+            idx1 = text.index("iomark linestart")
+            idx2 = text.tag_prevrange("console", idx1)
+            idx = idx2[1] if idx2 and text.compare(idx2[1], '>', idx1) else idx1
+            existing = text.get(idx, "iomark")
+
+            # Process new output.
+            rewrite, s = self._process_control_chars(existing, s)
+        else:
+            rewrite = False
 
+        # Update text widget.
         text.mark_gravity("iomark", "right")
-        if buffer_changed:
+        if rewrite:
+            # The shell normally rejects deleting before "iomark" (achieved
+            # by wrapping the text widget), so we call the underlying,
+            # unwrapped text widget's delete() method directly.
             self.per.bottom.delete(idx, "iomark")
-        OutputWindow.write(self, new_buffer, tags, "iomark")
+        OutputWindow.write(self, s, tags, "iomark")
         text.mark_gravity("iomark", "left")
 
         if self.canceled:
             self.canceled = 0
             if not use_subprocess:
                 raise KeyboardInterrupt
-        return len(new_buffer) - len(buffer)
+        return len(s) - (len(existing) if rewrite else 0)
 
-    def _process_control_chars(self, buffer, s,
+    def _process_control_chars(self, existing, s,
                                _control_char_re=re.compile(r'[\r\b]')):
-        buffer_idx = buffer_len = orig_buffer_len = len(buffer)
-        buffer = StringIO(buffer)
-        orig_buffer_changed = False
+        buffer_idx = buffer_len = existing_len = len(existing)
+        last_newline_idx = 0
+        buffer = StringIO(existing)
+        existing_changed = False
         idx = 0
         for m in _control_char_re.finditer(s):
             char_idx = m.start()
             if char_idx > idx:
-                buffer.seek(max(0, buffer_idx))
-                buffer.write(s[idx:m.start()])
-                orig_buffer_changed |= buffer_idx < orig_buffer_len
+                new_str = s[idx:m.start()]
+                buffer_idx = max(last_newline_idx, buffer_idx)
+                existing_changed |= buffer_idx < existing_len
+                buffer.seek(buffer_idx)
+                buffer.write(new_str)
+                newline_idx = new_str.rfind('\n')
+                if newline_idx >= 0:
+                    last_newline_idx = buffer_len + newline_idx
                 buffer_idx = buffer_len = buffer.tell()
             if s[char_idx] == '\b':
                 buffer_idx -= 1
             else:  # '\r'
-                buffer_idx = 0
+                buffer_idx = last_newline_idx
             idx = m.end()
+
         if idx == 0:
+            # No control characters in output.
             return False, s
-        if buffer_idx < buffer_len:
-            orig_buffer_changed |= buffer_idx < orig_buffer_len
-            buffer_len = max(0, buffer_idx)
-            buffer.seek(buffer_len)
+
+        # Handle rest of output after final control character.
+        buffer_idx = max(last_newline_idx, buffer_idx)
+        existing_changed |= buffer_idx < existing_len
+        buffer.seek(buffer_idx)
         buffer.write(s[idx:])
-        buffer_len += len(s) - idx
-        if orig_buffer_changed:
+        buffer_len = buffer.tell()
+
+        if existing_changed:
             buffer.seek(0)
             return True, buffer.read(buffer_len)
         else:
-            buffer.seek(orig_buffer_len)
-            return False, buffer.read()
+            buffer.seek(existing_len)
+            return False, buffer.read(buffer_len - existing_len)
 
     def rmenu_check_cut(self):
         try:

From 83e41cf0cfa408c534f6b7ad0b5faba30642181c Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Sun, 11 Aug 2019 12:36:10 +0300
Subject: [PATCH 04/13] just move the cursor rather than truncate the line

---
 Lib/idlelib/pyshell.py | 74 +++++++++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 26 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index baab5d88264bdd..de99627f47a0cd 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1321,7 +1321,8 @@ def write(self, s, tags=(),
             existing = text.get(idx, "iomark")
 
             # Process new output.
-            rewrite, s = self._process_control_chars(existing, s)
+            rewrite, s, cursor = self._process_control_chars(existing, s)
+            # TODO: keep and use output cursor position
         else:
             rewrite = False
 
@@ -1342,47 +1343,68 @@ def write(self, s, tags=(),
         return len(s) - (len(existing) if rewrite else 0)
 
     def _process_control_chars(self, existing, s,
-                               _control_char_re=re.compile(r'[\r\b]')):
-        buffer_idx = buffer_len = existing_len = len(existing)
+                               _control_char_re=re.compile(r'[\r\b]+')):
+        m = _control_char_re.search(s)
+        if m is None:
+            # No control characters in output.
+            last_newline_idx = s.rfind('\n')
+            if last_newline_idx >= 0:
+                cursor = len(s) - last_newline_idx + 1
+            else:
+                cursor = len(existing) + len(s)
+            return False, s, cursor
+
+        cursor = existing_len = len(existing)
         last_newline_idx = 0
         buffer = StringIO(existing)
         existing_changed = False
         idx = 0
-        for m in _control_char_re.finditer(s):
+        while m is not None:
             char_idx = m.start()
             if char_idx > idx:
                 new_str = s[idx:m.start()]
-                buffer_idx = max(last_newline_idx, buffer_idx)
-                existing_changed |= buffer_idx < existing_len
-                buffer.seek(buffer_idx)
-                buffer.write(new_str)
-                newline_idx = new_str.rfind('\n')
-                if newline_idx >= 0:
-                    last_newline_idx = buffer_len + newline_idx
-                buffer_idx = buffer_len = buffer.tell()
-            if s[char_idx] == '\b':
-                buffer_idx -= 1
-            else:  # '\r'
-                buffer_idx = last_newline_idx
+                new_str_first_newline = new_str.find('\n')
+                existing_changed |= cursor < existing_len
+                buffer.seek(cursor)
+                if new_str_first_newline >= 0:
+                    buffer.write(new_str[:new_str_first_newline])
+                    buffer.seek(0, 2)  # seek to end
+                    buffer.write(new_str[new_str_first_newline:])
+                else:
+                    buffer.write(new_str)
+                cursor = buffer.tell()
+                new_str_last_newline = new_str.rfind('\n')
+                if new_str_last_newline >= 0:
+                    last_newline_idx = \
+                        cursor - len(new_str) + new_str_first_newline
+            control_chars = m.group()
+            cursor = max(
+                last_newline_idx,
+                0 if '\r' in control_chars else cursor - len(control_chars),
+            )
             idx = m.end()
 
-        if idx == 0:
-            # No control characters in output.
-            return False, s
+            m = _control_char_re.search(s, idx)
 
         # Handle rest of output after final control character.
-        buffer_idx = max(last_newline_idx, buffer_idx)
-        existing_changed |= buffer_idx < existing_len
-        buffer.seek(buffer_idx)
-        buffer.write(s[idx:])
-        buffer_len = buffer.tell()
+        existing_changed |= cursor < existing_len
+        buffer.seek(cursor)
+        new_str = s[idx:]
+        new_str_first_newline = new_str.find('\n')
+        if new_str_first_newline >= 0:
+            buffer.write(new_str[:new_str_first_newline])
+            buffer.seek(0, 2)  # seek to end
+            buffer.write(new_str[new_str_first_newline:])
+        else:
+            buffer.write(new_str)
+        cursor = buffer.tell()
 
         if existing_changed:
             buffer.seek(0)
-            return True, buffer.read(buffer_len)
+            return True, buffer.read(), cursor - last_newline_idx
         else:
             buffer.seek(existing_len)
-            return False, buffer.read(buffer_len - existing_len)
+            return False, buffer.read(), cursor - last_newline_idx
 
     def rmenu_check_cut(self):
         try:

From 3cbd2fd5d7581977fdfc61262fcdef60762c7d78 Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Sun, 11 Aug 2019 18:15:57 +0300
Subject: [PATCH 05/13] implemented handling of output cursor

---
 Lib/idlelib/pyshell.py | 127 +++++++++++++++++++++++++----------------
 1 file changed, 77 insertions(+), 50 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index de99627f47a0cd..ce3acaf77fe2de 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1293,6 +1293,7 @@ def resetoutput(self):
             self.history.store(source)
         if self.text.get("end-2c") != "\n":
             self.text.insert("end-1c", "\n")
+        self.text.mark_set("outputmark", "end-2c")
         self.text.mark_set("iomark", "end-1c")
         self.set_line_and_column()
 
@@ -1312,29 +1313,44 @@ def write(self, s, tags=(),
         text = self.text
 
         # Process control characters only for stdout and stderr.
-        if isinstance(tags, str): tags = (tags,)
+        if isinstance(tags, str):
+            tags = (tags,)
+        cursor = "iomark"
         if {"stdout", "stderr"} & set(tags):
             # Get existing output on the current line.
-            idx1 = text.index("iomark linestart")
-            idx2 = text.tag_prevrange("console", idx1)
-            idx = idx2[1] if idx2 and text.compare(idx2[1], '>', idx1) else idx1
-            existing = text.get(idx, "iomark")
+            if not self.executing:
+                cursor = "outputmark"
+            linestart = f"{cursor} linestart"
+            if not self.executing:
+                prompt_end = text.tag_prevrange("console", linestart, cursor)
+                if prompt_end and text.compare(prompt_end[1], '>', linestart):
+                    linestart = prompt_end[1]
+            existing = text.get(linestart, f"{cursor} lineend")
 
             # Process new output.
-            rewrite, s, cursor = self._process_control_chars(existing, s)
-            # TODO: keep and use output cursor position
+            rewrite, s, cursor_col = self._process_control_chars(
+                existing, s, len(text.get(linestart, cursor)),
+            )
         else:
             rewrite = False
+            cursor_col = None
 
         # Update text widget.
-        text.mark_gravity("iomark", "right")
-        if rewrite:
-            # The shell normally rejects deleting before "iomark" (achieved
-            # by wrapping the text widget), so we call the underlying,
-            # unwrapped text widget's delete() method directly.
-            self.per.bottom.delete(idx, "iomark")
-        OutputWindow.write(self, s, tags, "iomark")
-        text.mark_gravity("iomark", "left")
+        text.mark_gravity(cursor, "right")
+        # The shell normally rejects writing and deleting before "iomark"
+        # (achieved by wrapping the text widget), so we temporarily replace
+        # the wrapped text widget object with the unwrapped one.
+        self.text = self.per.bottom
+        try:
+            if rewrite:
+                self.text.delete(linestart, f"{cursor} lineend")
+            OutputWindow.write(self, s, tags, cursor)
+        finally:
+            self.text = text
+
+        if cursor_col is not None:
+            text.mark_set(cursor, f"{cursor} linestart +{cursor_col}c")
+        text.mark_gravity(cursor, "left")
 
         if self.canceled:
             self.canceled = 0
@@ -1342,62 +1358,73 @@ def write(self, s, tags=(),
                 raise KeyboardInterrupt
         return len(s) - (len(existing) if rewrite else 0)
 
-    def _process_control_chars(self, existing, s,
+    def _process_control_chars(self, existing, string, cursor,
                                _control_char_re=re.compile(r'[\r\b]+')):
-        m = _control_char_re.search(s)
+        m = _control_char_re.search(string)
         if m is None:
             # No control characters in output.
-            last_newline_idx = s.rfind('\n')
+            rewrite = string and cursor < len(existing)
+            if rewrite:
+                string = existing[:cursor] + string + existing[cursor + len(string):]
+            last_newline_idx = string.rfind('\n')
             if last_newline_idx >= 0:
-                cursor = len(s) - last_newline_idx + 1
+                cursor = len(string) - last_newline_idx + 1
             else:
-                cursor = len(existing) + len(s)
-            return False, s, cursor
+                cursor += len(string)
+            return rewrite, string, cursor
 
-        cursor = existing_len = len(existing)
+        existing_len = len(existing)
         last_newline_idx = 0
         buffer = StringIO(existing)
+
+        def write(string):
+            nonlocal cursor, existing_changed
+            existing_changed |= cursor < existing_len
+            buffer.seek(0, 2)  # seek to end
+            end = buffer.tell()
+            buffer.seek(cursor)
+            string_first_newline = string.find('\n')
+            # Split the string only if we have to in order to overwrite
+            # just part of the first line.
+            if (
+                    string_first_newline >= 0 and
+                    cursor + string_first_newline < end
+            ):
+                buffer.write(string[:string_first_newline])
+                buffer.seek(0, 2)  # seek to end
+                buffer.write(string[string_first_newline:])
+            else:
+                buffer.write(string)
+            cursor = buffer.tell()
+
         existing_changed = False
         idx = 0
         while m is not None:
-            char_idx = m.start()
-            if char_idx > idx:
-                new_str = s[idx:m.start()]
-                new_str_first_newline = new_str.find('\n')
-                existing_changed |= cursor < existing_len
-                buffer.seek(cursor)
-                if new_str_first_newline >= 0:
-                    buffer.write(new_str[:new_str_first_newline])
-                    buffer.seek(0, 2)  # seek to end
-                    buffer.write(new_str[new_str_first_newline:])
-                else:
-                    buffer.write(new_str)
-                cursor = buffer.tell()
-                new_str_last_newline = new_str.rfind('\n')
-                if new_str_last_newline >= 0:
-                    last_newline_idx = \
-                        cursor - len(new_str) + new_str_first_newline
+            string_part = string[idx:m.start()]
+            write(string_part)
+
+            # We never write before the last newline, so we must keep
+            # track of the last newline written.
+            new_str_last_newline = string_part.rfind('\n')
+            if new_str_last_newline >= 0:
+                last_newline_idx = \
+                    cursor - len(string_part) + new_str_last_newline
+
+            # Process a sequence of control characters. This assumes
+            # that they are all '\r' and/or '\b' characters.
             control_chars = m.group()
             cursor = max(
                 last_newline_idx,
                 0 if '\r' in control_chars else cursor - len(control_chars),
             )
-            idx = m.end()
 
-            m = _control_char_re.search(s, idx)
+            idx = m.end()
+            m = _control_char_re.search(string, idx)
 
         # Handle rest of output after final control character.
         existing_changed |= cursor < existing_len
         buffer.seek(cursor)
-        new_str = s[idx:]
-        new_str_first_newline = new_str.find('\n')
-        if new_str_first_newline >= 0:
-            buffer.write(new_str[:new_str_first_newline])
-            buffer.seek(0, 2)  # seek to end
-            buffer.write(new_str[new_str_first_newline:])
-        else:
-            buffer.write(new_str)
-        cursor = buffer.tell()
+        write(string[idx:])
 
         if existing_changed:
             buffer.seek(0)

From 11357617ae23cccd80afe6e086de98a384dbebe8 Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Sun, 11 Aug 2019 18:57:17 +0300
Subject: [PATCH 06/13] some tests and a couple of small fixes

---
 Lib/idlelib/idle_test/test_pyshell.py | 58 +++++++++++++++++++++++++++
 Lib/idlelib/pyshell.py                | 29 ++++++++------
 2 files changed, 74 insertions(+), 13 deletions(-)

diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index 581444ca5ef21f..ee9bf8433dbacf 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -38,5 +38,63 @@ def test_init(self):
 ##        self.assertIsInstance(ps, pyshell.PyShell)
 
 
+class TestProcessControlChars(unittest.TestCase):
+    def check(self, existing, string, cursor, expected):
+        self.assertEqual(
+            pyshell.PyShell._process_control_chars(existing, string, cursor),
+            expected,
+        )
+
+    def test_empty_written(self):
+        self.check('', '', 0, (False, '', 0))
+        self.check('a', '', 0, (False, '', 0))
+        self.check('a', '', 1, (False, '', 1))
+        for cursor in range(4):
+            with self.subTest(cursor=cursor):
+                self.check('abc', '', cursor, (False, '', cursor))
+
+    def test_empty_existing(self):
+        self.check('', 'a', 0, (False, 'a', 1))
+        self.check('', 'ab', 0, (False, 'ab', 2))
+        self.check('', 'abc', 0, (False, 'abc', 3))
+
+    def test_simple_cursor(self):
+        self.check('abc', 'def', 0, (True, 'def', 3))
+        self.check('abc', 'def', 1, (True, 'adef', 4))
+        self.check('abc', 'def', 2, (True, 'abdef', 5))
+
+        self.check('abc', 'def', 3, (False, 'def', 6))
+
+    def test_carriage_return(self):
+        self.check('', 'a\rb', 0, (False, 'b', 1))
+        self.check('', 'abc\rd', 0, (False, 'dbc', 1))
+
+    def test_carriage_return_doesnt_delete(self):
+        # \r should only move the cursor to the beginning of the current line
+        self.check('', 'a\r', 0, (False, 'a', 0))
+        self.check('', 'abc\r', 0, (False, 'abc', 0))
+
+    def test_backspace(self):
+        self.check('', '\ba', 0, (False, 'a', 1))
+        self.check('', 'a\bb', 0, (False, 'b', 1))
+        self.check('', 'ab\bc', 0, (False, 'ac', 2))
+        self.check('', 'ab\bc\bd', 0, (False, 'ad', 2))
+
+    def test_backspace_doesnt_delete(self):
+        # \b should only move the cursor one place earlier
+        self.check('', 'a\b', 0, (False, 'a', 0))
+        self.check('', 'a\b\b', 0, (False, 'a', 0))
+        self.check('', 'ab\b\b', 0, (False, 'ab', 0))
+        self.check('', 'ab\b\bc', 0, (False, 'cb', 1))
+        self.check('', 'abc\b\bd', 0, (False, 'adc', 2))
+        self.check('', 'ab\bc\b', 0, (False, 'ac', 1))
+
+    def test_newline(self):
+        self.check('abc', '\n\rdef', 3, (False, '\ndef', 3))
+        self.check('abc', 'd\n\ref', 3, (False, 'd\nef', 2))
+        self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 1))
+        self.check('abc', 'def\n\r', 3, (False, 'def\n', 0))
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index ce3acaf77fe2de..a87087d6356b5e 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1358,23 +1358,26 @@ def write(self, s, tags=(),
                 raise KeyboardInterrupt
         return len(s) - (len(existing) if rewrite else 0)
 
-    def _process_control_chars(self, existing, string, cursor,
+    @classmethod
+    def _process_control_chars(cls, existing, string, cursor,
                                _control_char_re=re.compile(r'[\r\b]+')):
         m = _control_char_re.search(string)
         if m is None:
             # No control characters in output.
-            rewrite = string and cursor < len(existing)
+            rewrite = bool(string and cursor < len(existing))
             if rewrite:
-                string = existing[:cursor] + string + existing[cursor + len(string):]
-            last_newline_idx = string.rfind('\n')
-            if last_newline_idx >= 0:
-                cursor = len(string) - last_newline_idx + 1
+                res = existing[:cursor] + string + existing[cursor + len(string):]
+            else:
+                res = string
+            last_linestart = string.rfind('\n')
+            if last_linestart >= 0:
+                cursor = len(string) - last_linestart + 1
             else:
                 cursor += len(string)
-            return rewrite, string, cursor
+            return rewrite, res, cursor
 
         existing_len = len(existing)
-        last_newline_idx = 0
+        last_linestart = 0
         buffer = StringIO(existing)
 
         def write(string):
@@ -1407,14 +1410,14 @@ def write(string):
             # track of the last newline written.
             new_str_last_newline = string_part.rfind('\n')
             if new_str_last_newline >= 0:
-                last_newline_idx = \
-                    cursor - len(string_part) + new_str_last_newline
+                last_linestart = \
+                    cursor - len(string_part) + new_str_last_newline + 1
 
             # Process a sequence of control characters. This assumes
             # that they are all '\r' and/or '\b' characters.
             control_chars = m.group()
             cursor = max(
-                last_newline_idx,
+                last_linestart,
                 0 if '\r' in control_chars else cursor - len(control_chars),
             )
 
@@ -1428,10 +1431,10 @@ def write(string):
 
         if existing_changed:
             buffer.seek(0)
-            return True, buffer.read(), cursor - last_newline_idx
+            return True, buffer.read(), cursor - last_linestart
         else:
             buffer.seek(existing_len)
-            return False, buffer.read(), cursor - last_newline_idx
+            return False, buffer.read(), cursor - last_linestart
 
     def rmenu_check_cut(self):
         try:

From 1d32615725f1ca697be75d2cf8d76aff1cd848c4 Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Mon, 12 Aug 2019 13:56:21 +0300
Subject: [PATCH 07/13] another test and important edge-case fix

---
 Lib/idlelib/idle_test/test_pyshell.py | 4 ++++
 Lib/idlelib/pyshell.py                | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index ee9bf8433dbacf..96a15eb377dfe3 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -90,6 +90,10 @@ def test_backspace_doesnt_delete(self):
         self.check('', 'ab\bc\b', 0, (False, 'ac', 1))
 
     def test_newline(self):
+        self.check('', '\n', 0, (False, '\n', 0))
+        self.check('abc', '\n', 3, (False, '\n', 0))
+
+    def test_newline_and_carriage_return(self):
         self.check('abc', '\n\rdef', 3, (False, '\ndef', 3))
         self.check('abc', 'd\n\ref', 3, (False, 'd\nef', 2))
         self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 1))
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index a87087d6356b5e..7ce1ac1283c554 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1280,12 +1280,14 @@ def showprompt(self):
         self.io.reset_undo()
 
     def show_warning(self, msg):
+        print(f"BEFORE WARNING: iomark={text.index('iomark')}, end={text.index('end')}")
         width = self.interp.tkconsole.width
         wrapper = TextWrapper(width=width, tabsize=8, expand_tabs=True)
         wrapped_msg = '\n'.join(wrapper.wrap(msg))
         if not wrapped_msg.endswith('\n'):
             wrapped_msg += '\n'
         self.per.bottom.insert("iomark linestart", wrapped_msg, "stderr")
+        print(f"AFTER WARNING:  iomark={text.index('iomark')}, end={text.index('end')}")
 
     def resetoutput(self):
         source = self.text.get("iomark", "end-1c")
@@ -1371,7 +1373,7 @@ def _process_control_chars(cls, existing, string, cursor,
                 res = string
             last_linestart = string.rfind('\n')
             if last_linestart >= 0:
-                cursor = len(string) - last_linestart + 1
+                cursor = len(string) - (last_linestart + 1)
             else:
                 cursor += len(string)
             return rewrite, res, cursor

From 4dde76e0794dc82647a9e94fa46cfedf0da59d5d Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Mon, 12 Aug 2019 15:31:33 +0300
Subject: [PATCH 08/13] major optimization: return "cursor_back" rather than
 "cursor"

Also some smaller optimizations and code simplifications.
---
 Lib/idlelib/idle_test/test_pyshell.py | 48 ++++++++++----------
 Lib/idlelib/pyshell.py                | 63 ++++++++++++++-------------
 2 files changed, 57 insertions(+), 54 deletions(-)

diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index 96a15eb377dfe3..dcae27183d7c66 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -48,45 +48,45 @@ def check(self, existing, string, cursor, expected):
     def test_empty_written(self):
         self.check('', '', 0, (False, '', 0))
         self.check('a', '', 0, (False, '', 0))
-        self.check('a', '', 1, (False, '', 1))
+        self.check('a', '', 1, (False, '', 0))
         for cursor in range(4):
             with self.subTest(cursor=cursor):
-                self.check('abc', '', cursor, (False, '', cursor))
+                self.check('abc', '', cursor, (False, '', 0))
 
     def test_empty_existing(self):
-        self.check('', 'a', 0, (False, 'a', 1))
-        self.check('', 'ab', 0, (False, 'ab', 2))
-        self.check('', 'abc', 0, (False, 'abc', 3))
+        self.check('', 'a', 0, (False, 'a', 0))
+        self.check('', 'ab', 0, (False, 'ab', 0))
+        self.check('', 'abc', 0, (False, 'abc', 0))
 
     def test_simple_cursor(self):
-        self.check('abc', 'def', 0, (True, 'def', 3))
-        self.check('abc', 'def', 1, (True, 'adef', 4))
-        self.check('abc', 'def', 2, (True, 'abdef', 5))
+        self.check('abc', 'def', 0, (False, 'def', 0))
+        self.check('abc', 'def', 1, (False, 'def', 0))
+        self.check('abc', 'def', 2, (False, 'def', 0))
 
-        self.check('abc', 'def', 3, (False, 'def', 6))
+        self.check('abc', 'def', 3, (False, 'def', 0))
 
     def test_carriage_return(self):
-        self.check('', 'a\rb', 0, (False, 'b', 1))
-        self.check('', 'abc\rd', 0, (False, 'dbc', 1))
+        self.check('', 'a\rb', 0, (False, 'b', 0))
+        self.check('', 'abc\rd', 0, (False, 'dbc', 2))
 
     def test_carriage_return_doesnt_delete(self):
         # \r should only move the cursor to the beginning of the current line
-        self.check('', 'a\r', 0, (False, 'a', 0))
-        self.check('', 'abc\r', 0, (False, 'abc', 0))
+        self.check('', 'a\r', 0, (False, 'a', 1))
+        self.check('', 'abc\r', 0, (False, 'abc', 3))
 
     def test_backspace(self):
-        self.check('', '\ba', 0, (False, 'a', 1))
-        self.check('', 'a\bb', 0, (False, 'b', 1))
-        self.check('', 'ab\bc', 0, (False, 'ac', 2))
-        self.check('', 'ab\bc\bd', 0, (False, 'ad', 2))
+        self.check('', '\ba', 0, (False, 'a', 0))
+        self.check('', 'a\bb', 0, (False, 'b', 0))
+        self.check('', 'ab\bc', 0, (False, 'ac', 0))
+        self.check('', 'ab\bc\bd', 0, (False, 'ad', 0))
 
     def test_backspace_doesnt_delete(self):
         # \b should only move the cursor one place earlier
-        self.check('', 'a\b', 0, (False, 'a', 0))
-        self.check('', 'a\b\b', 0, (False, 'a', 0))
-        self.check('', 'ab\b\b', 0, (False, 'ab', 0))
+        self.check('', 'a\b', 0, (False, 'a', 1))
+        self.check('', 'a\b\b', 0, (False, 'a', 1))
+        self.check('', 'ab\b\b', 0, (False, 'ab', 2))
         self.check('', 'ab\b\bc', 0, (False, 'cb', 1))
-        self.check('', 'abc\b\bd', 0, (False, 'adc', 2))
+        self.check('', 'abc\b\bd', 0, (False, 'adc', 1))
         self.check('', 'ab\bc\b', 0, (False, 'ac', 1))
 
     def test_newline(self):
@@ -94,9 +94,9 @@ def test_newline(self):
         self.check('abc', '\n', 3, (False, '\n', 0))
 
     def test_newline_and_carriage_return(self):
-        self.check('abc', '\n\rdef', 3, (False, '\ndef', 3))
-        self.check('abc', 'd\n\ref', 3, (False, 'd\nef', 2))
-        self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 1))
+        self.check('abc', '\n\rdef', 3, (False, '\ndef', 0))
+        self.check('abc', 'd\n\ref', 3, (False, 'd\nef', 0))
+        self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 0))
         self.check('abc', 'def\n\r', 3, (False, 'def\n', 0))
 
 
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 7ce1ac1283c554..55d0c6bda84c1e 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1301,7 +1301,10 @@ def resetoutput(self):
 
     def write(self, s, tags=(),
               _non_bmp_re=re.compile(r'[\U00010000-\U0010ffff]')):
-        if isinstance(s, str) and s:
+        if not s:
+            return 0
+
+        if isinstance(s, str):
             m = _non_bmp_re.search(s)
             if m is not None:
                 # Tk doesn't support outputting non-BMP characters
@@ -1323,19 +1326,23 @@ def write(self, s, tags=(),
             if not self.executing:
                 cursor = "outputmark"
             linestart = f"{cursor} linestart"
+            lineend = f"{cursor} lineend"
             if not self.executing:
                 prompt_end = text.tag_prevrange("console", linestart, cursor)
                 if prompt_end and text.compare(prompt_end[1], '>', linestart):
                     linestart = prompt_end[1]
-            existing = text.get(linestart, f"{cursor} lineend")
+            existing = text.get(linestart, lineend)
 
             # Process new output.
-            rewrite, s, cursor_col = self._process_control_chars(
-                existing, s, len(text.get(linestart, cursor)),
+            cursor_pos_in_existing = len(text.get(linestart, cursor))
+            is_cursor_at_lineend = cursor_pos_in_existing == len(existing)
+            rewrite, s, cursor_back = self._process_control_chars(
+                existing, s, cursor_pos_in_existing,
             )
         else:
+            is_cursor_at_lineend = True
             rewrite = False
-            cursor_col = None
+            cursor_back = 0
 
         # Update text widget.
         text.mark_gravity(cursor, "right")
@@ -1344,47 +1351,42 @@ def write(self, s, tags=(),
         # the wrapped text widget object with the unwrapped one.
         self.text = self.per.bottom
         try:
-            if rewrite:
-                self.text.delete(linestart, f"{cursor} lineend")
+            if rewrite or not is_cursor_at_lineend:
+                self.text.delete(linestart if rewrite else cursor, lineend)
             OutputWindow.write(self, s, tags, cursor)
         finally:
             self.text = text
 
-        if cursor_col is not None:
-            text.mark_set(cursor, f"{cursor} linestart +{cursor_col}c")
+        if cursor_back > 0:
+            text.mark_set(cursor, f"{cursor} -{cursor_back}c")
         text.mark_gravity(cursor, "left")
 
         if self.canceled:
             self.canceled = 0
             if not use_subprocess:
                 raise KeyboardInterrupt
-        return len(s) - (len(existing) if rewrite else 0)
+        return len(s) - (len(existing) if rewrite else 0) - (0 if is_cursor_at_lineend else len(existing) - cursor_pos_in_existing)
 
     @classmethod
     def _process_control_chars(cls, existing, string, cursor,
                                _control_char_re=re.compile(r'[\r\b]+')):
+        if not string:
+            return False, '', 0
+
         m = _control_char_re.search(string)
         if m is None:
             # No control characters in output.
-            rewrite = bool(string and cursor < len(existing))
-            if rewrite:
-                res = existing[:cursor] + string + existing[cursor + len(string):]
-            else:
-                res = string
-            last_linestart = string.rfind('\n')
-            if last_linestart >= 0:
-                cursor = len(string) - (last_linestart + 1)
-            else:
-                cursor += len(string)
-            return rewrite, res, cursor
+            res = string + existing[cursor + len(string):]
+            return False, res, 0
 
-        existing_len = len(existing)
+        orig_cursor = cursor
         last_linestart = 0
         buffer = StringIO(existing)
+        rewrite = False
 
         def write(string):
-            nonlocal cursor, existing_changed
-            existing_changed |= cursor < existing_len
+            nonlocal buffer, cursor, rewrite
+            rewrite |= cursor < orig_cursor
             buffer.seek(0, 2)  # seek to end
             end = buffer.tell()
             buffer.seek(cursor)
@@ -1402,7 +1404,6 @@ def write(string):
                 buffer.write(string)
             cursor = buffer.tell()
 
-        existing_changed = False
         idx = 0
         while m is not None:
             string_part = string[idx:m.start()]
@@ -1427,16 +1428,18 @@ def write(string):
             m = _control_char_re.search(string, idx)
 
         # Handle rest of output after final control character.
-        existing_changed |= cursor < existing_len
+        rewrite |= cursor < orig_cursor
         buffer.seek(cursor)
         write(string[idx:])
+        buffer.seek(0, 2)  # seek to end
+        buffer_len = buffer.tell()
 
-        if existing_changed:
+        if rewrite:
             buffer.seek(0)
-            return True, buffer.read(), cursor - last_linestart
+            return True, buffer.read(), buffer_len - cursor
         else:
-            buffer.seek(existing_len)
-            return False, buffer.read(), cursor - last_linestart
+            buffer.seek(orig_cursor)
+            return False, buffer.read(), buffer_len - cursor
 
     def rmenu_check_cut(self):
         try:

From 017bf8a48298686c704a7d84f7bc5cf3800fd1ae Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Mon, 12 Aug 2019 15:44:58 +0300
Subject: [PATCH 09/13] minor code cleanup

---
 Lib/idlelib/pyshell.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 55d0c6bda84c1e..9c97b1be1588c1 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1387,22 +1387,25 @@ def _process_control_chars(cls, existing, string, cursor,
         def write(string):
             nonlocal buffer, cursor, rewrite
             rewrite |= cursor < orig_cursor
+
+            string_first_newline = string.find('\n')
             buffer.seek(0, 2)  # seek to end
             end = buffer.tell()
             buffer.seek(cursor)
-            string_first_newline = string.find('\n')
-            # Split the string only if we have to in order to overwrite
-            # just part of the first line.
             if (
                     string_first_newline >= 0 and
                     cursor + string_first_newline < end
             ):
+                # We must split the string in order to overwrite just
+                # part of the first line.
                 buffer.write(string[:string_first_newline])
                 buffer.seek(0, 2)  # seek to end
                 buffer.write(string[string_first_newline:])
             else:
                 buffer.write(string)
+
             cursor = buffer.tell()
+            return
 
         idx = 0
         while m is not None:
@@ -1429,7 +1432,6 @@ def write(string):
 
         # Handle rest of output after final control character.
         rewrite |= cursor < orig_cursor
-        buffer.seek(cursor)
         write(string[idx:])
         buffer.seek(0, 2)  # seek to end
         buffer_len = buffer.tell()

From 04b8aba602e8d4891919f64ecd76011df7476fab Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Mon, 12 Aug 2019 22:33:06 +0300
Subject: [PATCH 10/13] avoid unnecessary re-writes with controls chars at
 start or end

Also, more tests.
---
 Lib/idlelib/idle_test/test_pyshell.py | 32 +++++++++++++++++++++++----
 Lib/idlelib/pyshell.py                | 22 +++++++++---------
 2 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index dcae27183d7c66..a5ae558fac3a20 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -39,11 +39,22 @@ def test_init(self):
 
 
 class TestProcessControlChars(unittest.TestCase):
+    def call(self, existing, string, cursor):
+        return pyshell.PyShell._process_control_chars(existing, string, cursor)
+
     def check(self, existing, string, cursor, expected):
-        self.assertEqual(
-            pyshell.PyShell._process_control_chars(existing, string, cursor),
-            expected,
-        )
+        self.assertEqual(self.call(existing, string, cursor), expected)
+
+    def check_leniently(self, existing, string, cursor, expected):
+        result = self.call(existing, string, cursor)
+        if existing and not expected[0]:
+            options = [
+                expected,
+                (True, existing[:cursor] + expected[1], expected[2]),
+            ]
+            self.assertIn(result, options)
+        else:
+            self.assertEqual(result, expected)
 
     def test_empty_written(self):
         self.check('', '', 0, (False, '', 0))
@@ -80,6 +91,11 @@ def test_backspace(self):
         self.check('', 'ab\bc', 0, (False, 'ac', 0))
         self.check('', 'ab\bc\bd', 0, (False, 'ad', 0))
 
+        self.check('abc', '\b', 3, (False, '', 1))
+        self.check('abc', '\b', 2, (False, 'c', 2))
+        self.check('abc', '\b', 1, (False, 'bc', 3))
+        self.check('abc', '\b', 0, (False, 'abc', 3))
+
     def test_backspace_doesnt_delete(self):
         # \b should only move the cursor one place earlier
         self.check('', 'a\b', 0, (False, 'a', 1))
@@ -92,6 +108,8 @@ def test_backspace_doesnt_delete(self):
     def test_newline(self):
         self.check('', '\n', 0, (False, '\n', 0))
         self.check('abc', '\n', 3, (False, '\n', 0))
+        self.check('abc', 'def\n', 3, (False, 'def\n', 0))
+        self.check('abc', '\ndef', 3, (False, '\ndef', 0))
 
     def test_newline_and_carriage_return(self):
         self.check('abc', '\n\rdef', 3, (False, '\ndef', 0))
@@ -99,6 +117,12 @@ def test_newline_and_carriage_return(self):
         self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 0))
         self.check('abc', 'def\n\r', 3, (False, 'def\n', 0))
 
+        self.check_leniently('abc', '\r\n', 3, (False, '\n', 0))
+        self.check_leniently('abc', 'def\r\n', 3, (False, 'def\n', 0))
+        self.check_leniently('abc', '\r\ndef', 3, (False, '\ndef', 0))
+        self.check_leniently('abc', '\rdef\n', 3, (True, 'def\n', 0))
+        self.check_leniently('abc', '\rd\nef', 3, (True, 'dbc\nef', 0))
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index 9c97b1be1588c1..ae92b88fd23f07 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1409,15 +1409,16 @@ def write(string):
 
         idx = 0
         while m is not None:
-            string_part = string[idx:m.start()]
-            write(string_part)
+            if m.start() > idx:
+                string_part = string[idx:m.start()]
+                write(string_part)
 
-            # We never write before the last newline, so we must keep
-            # track of the last newline written.
-            new_str_last_newline = string_part.rfind('\n')
-            if new_str_last_newline >= 0:
-                last_linestart = \
-                    cursor - len(string_part) + new_str_last_newline + 1
+                # We never write before the last newline, so we must keep
+                # track of the last newline written.
+                new_str_last_newline = string_part.rfind('\n')
+                if new_str_last_newline >= 0:
+                    last_linestart = \
+                        cursor - len(string_part) + new_str_last_newline + 1
 
             # Process a sequence of control characters. This assumes
             # that they are all '\r' and/or '\b' characters.
@@ -1431,8 +1432,9 @@ def write(string):
             m = _control_char_re.search(string, idx)
 
         # Handle rest of output after final control character.
-        rewrite |= cursor < orig_cursor
-        write(string[idx:])
+        if idx < len(string):
+            rewrite |= cursor < orig_cursor
+            write(string[idx:])
         buffer.seek(0, 2)  # seek to end
         buffer_len = buffer.tell()
 

From 1903c564899c2e5e9ce073b835aad6211881486a Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Tue, 13 Aug 2019 09:19:16 +0300
Subject: [PATCH 11/13] make the internal write() function a staticmethod

---
 Lib/idlelib/pyshell.py | 50 +++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index ae92b88fd23f07..ebd19383caadfe 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -1383,35 +1383,14 @@ def _process_control_chars(cls, existing, string, cursor,
         last_linestart = 0
         buffer = StringIO(existing)
         rewrite = False
-
-        def write(string):
-            nonlocal buffer, cursor, rewrite
-            rewrite |= cursor < orig_cursor
-
-            string_first_newline = string.find('\n')
-            buffer.seek(0, 2)  # seek to end
-            end = buffer.tell()
-            buffer.seek(cursor)
-            if (
-                    string_first_newline >= 0 and
-                    cursor + string_first_newline < end
-            ):
-                # We must split the string in order to overwrite just
-                # part of the first line.
-                buffer.write(string[:string_first_newline])
-                buffer.seek(0, 2)  # seek to end
-                buffer.write(string[string_first_newline:])
-            else:
-                buffer.write(string)
-
-            cursor = buffer.tell()
-            return
+        write_to_buffer = cls._process_control_chars_buffer_write
 
         idx = 0
         while m is not None:
             if m.start() > idx:
                 string_part = string[idx:m.start()]
-                write(string_part)
+                rewrite |= cursor < orig_cursor
+                cursor = write_to_buffer(buffer, cursor, string_part)
 
                 # We never write before the last newline, so we must keep
                 # track of the last newline written.
@@ -1434,7 +1413,7 @@ def write(string):
         # Handle rest of output after final control character.
         if idx < len(string):
             rewrite |= cursor < orig_cursor
-            write(string[idx:])
+            cursor = write_to_buffer(buffer, cursor, string[idx:])
         buffer.seek(0, 2)  # seek to end
         buffer_len = buffer.tell()
 
@@ -1445,6 +1424,27 @@ def write(string):
             buffer.seek(orig_cursor)
             return False, buffer.read(), buffer_len - cursor
 
+    @staticmethod
+    def _process_control_chars_buffer_write(buffer, cursor, string):
+        string_first_newline = string.find('\n')
+        buffer.seek(0, 2)  # seek to end
+        buffer_len = buffer.tell()
+        buffer.seek(cursor)
+        if (
+                string_first_newline >= 0 and
+                cursor + string_first_newline < buffer_len
+        ):
+            # We must split the string in order to overwrite just
+            # part of the first line.
+            buffer.write(string[:string_first_newline])
+            buffer.seek(0, 2)  # seek to end
+            buffer.write(string[string_first_newline:])
+        else:
+            buffer.write(string)
+
+        cursor = buffer.tell()
+        return cursor
+
     def rmenu_check_cut(self):
         try:
             if self.text.compare('sel.first', '<', 'iomark'):

From 93c5bc583f33c28bd38ccf64403454496b934fdd Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Tue, 13 Aug 2019 10:32:18 +0300
Subject: [PATCH 12/13] update idle.rst

---
 Doc/library/idle.rst | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
index 0bd248c22b1820..1a931ca7a9bed3 100644
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -752,11 +752,13 @@ with 300 the default.
 A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) in
 the BMP (Basic Multilingual Plane) subset of Unicode.  Which characters are
 displayed with a proper glyph and which with a replacement box depends on the
-operating system and installed fonts.  Tab characters cause the following text
-to begin after the next tab stop. (They occur every 8 'characters').  Newline
-characters cause following text to appear on a new line.  Other control
-characters are ignored or displayed as a space, box, or something else,
-depending on the operating system and font.  (Moving the text cursor through
+OS and installed fonts.  Tab characters (``\t``) cause the following text to
+begin after the next tab stop.  (They occur every 8 'characters').  Newline
+characters (``\n``) cause following text to appear on a new line.  Carriage-
+return characters (``\r``) move the cursor back to the beginning of the current
+line, and backspace characters (``\b``) move the cursor back one character.
+Other control characters are ignored or displayed as a space, a box, or
+something else, depending on the OS and font.  (Moving the text cursor through
 such output with arrow keys may exhibit some surprising spacing behavior.) ::
 
    >>> s = 'a\tb\a<\x02><\r>\bc\nd'  # Enter 22 chars.
@@ -764,7 +766,7 @@ such output with arrow keys may exhibit some surprising spacing behavior.) ::
    14
    >>> s  # Display repr(s)
    'a\tb\x07<\x02><\r>\x08c\nd'
-   >>> print(s, end='')  # Display s as is.
+   >>> print(s)  # Send s to the output
    # Result varies by OS and font.  Try it.
 
 The ``repr`` function is used for interactive echo of expression

From 6e4c73cf1d04d6ff6f370a74ca2fe91452b95f43 Mon Sep 17 00:00:00 2001
From: Tal Einat <taleinat@gmail.com>
Date: Tue, 13 Aug 2019 10:41:17 +0300
Subject: [PATCH 13/13] add entries to NEWS and What's New

---
 Doc/whatsnew/3.7.rst                                        | 5 +++++
 Doc/whatsnew/3.8.rst                                        | 6 ++++++
 .../next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst      | 2 ++
 3 files changed, 13 insertions(+)
 create mode 100644 Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst

diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
index af7e22d9faa9e4..0c5c81552dd86e 100644
--- a/Doc/whatsnew/3.7.rst
+++ b/Doc/whatsnew/3.7.rst
@@ -1033,6 +1033,11 @@ tab of the configuration dialog.  Line numbers for an existing
 window are shown and hidden in the Options menu.
 (Contributed by Tal Einat and Saimadhav Heblikar in :issue:`17535`.)
 
+In the shell, ``\r`` and ``\b`` control characters in output are now handled
+much like in terminals, i.e. moving the cursor position.  This allows common
+uses of these control characters, such as progress indicators, to be displayed
+properly rather than flooding the output and eventually slowing the shell down
+to a crawl.  (Contributed by Tal Einat in :issue:`37827`.)
 
 importlib
 ---------
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 82da10cc3be86e..4b3d7114f97fdb 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -610,6 +610,12 @@ tab of the configuration dialog.  Line numbers for an existing
 window are shown and hidden in the Options menu.
 (Contributed by Tal Einat and Saimadhav Heblikar in :issue:`17535`.)
 
+In the shell, ``\r`` and ``\b`` control characters in output are now handled
+much like in terminals, i.e. moving the cursor position.  This allows common
+uses of these control characters, such as progress indicators, to be displayed
+properly rather than flooding the output and eventually slowing the shell down
+to a crawl.  (Contributed by Tal Einat in :issue:`37827`.)
+
 The changes above have been backported to 3.7 maintenance releases.
 
 
diff --git a/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst b/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst
new file mode 100644
index 00000000000000..df1fffefe7ed96
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst
@@ -0,0 +1,2 @@
+Emulate terminal handling of ``\r`` and ``\b`` control characters in shell
+outoput.