gh-56708: support custom len function in textwrap.wrap

Doc/library/textwrap.rst

@@ -282,6 +282,14 @@ hyphenated words; only then will long words be broken if necessary, unless
  .. versionadded:: 3.4
 
 
+ .. attribute:: text_len
+
+ (default: ``len``) Used to determine the length of a string. You can
+ provide a custom function, e.g. to account for wide characters.
+
+ .. versionadded:: 3.11
+
+
  .. index:: single: ...; placeholder
 
  .. attribute:: placeholder

Lib/idlelib/idle_test/test_calltip.py

@@ -105,7 +105,7 @@ def test_signature_wrap(self):
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
  replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
  drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None,
- placeholder=' [...]')
+ placeholder=' [...]', text_len=<built-in function len>)
 Object for wrapping/filling text. The public interface consists of
 the wrap() and fill() methods; the other methods are just there for
 subclasses to override in order to tweak the default behaviour.

Lib/test/test_textwrap.py

@@ -9,6 +9,7 @@
 #
 
 import unittest
+import unicodedata
 
 from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten
 
@@ -1133,5 +1134,72 @@ def test_first_word_too_long_but_placeholder_fits(self):
  self.check_shorten("Helloo", 5, "[...]")
 
 
+class WideCharacterTestCase(BaseTestCase):
+ def text_len(self, text):
+ return sum(
+ 2 if unicodedata.east_asian_width(c) in{'F', 'W'} else 1
+ for c in text
+ )
+
+ def check_shorten(self, text, width, expect, **kwargs):
+ result = shorten(text, width, **kwargs)
+ self.check(result, expect)
+
+ def test_wrap(self):
+ text = "123 🔧"
+ self.check_wrap(text, 5, ["123 🔧"])
+ self.check_wrap(text, 5, ["123", "🔧"], text_len=self.text_len)
+
+ def test_wrap_initial_indent(self):
+ text = "12 12"
+ self.check_wrap(text, 6, ["🔧12 12"], initial_indent="🔧")
+ self.check_wrap(text, 6, ["🔧12", "12"], initial_indent="🔧",
+ text_len=self.text_len)
+
+ def test_wrap_subsequent_indent(self):
+ text = "12 12 12 12"
+ self.check_wrap(text, 6, ["12 12", "🔧12 12"], subsequent_indent="🔧")
+ self.check_wrap(text, 6, ["12 12", "🔧12", "🔧12"],
+ subsequent_indent="🔧", text_len=self.text_len)
+
+ def test_shorten(self):
+ text = "123 1234🔧"
+ expected = "123 [...]"
+ self.check_shorten(text, 9, "123 1234🔧")
+ self.check_shorten(text, 9, "123 [...]", text_len=self.text_len)
+
+ def test_shorten_placeholder(self):
+ text = "123 1 123"
+ self.check_shorten(text, 7, "123 1 🔧", placeholder=" 🔧")
+ self.check_shorten(text, 7, "123 🔧", placeholder=" 🔧",
+ text_len=self.text_len)
+
+
+class CustomWidthTestCase(BaseTestCase):
+ def text_len(self, text):
+ lengths ={
+ 'A': 4,
+ 'B': 2,
+ 'Q': 0,
+ }
+
+ return sum(
+ lengths[c] if c in lengths else 1
+ for c in text
+ )
+
+ def test_zero_width_text_len(self):
+ text = "0QQ1234QQ56789"
+ self.check_wrap(text, 6, ["0QQ1234QQ5", "6789"], text_len=self.text_len)
+
+ def test_char_longer_than_width(self):
+ text = "AA0123"
+ self.check_wrap(text, 3, ["A", "A", "012", "3"], text_len=self.text_len)
+
+ def test_next_char_overflow(self):
+ text = "BB0123"
+ self.check_wrap(text, 3, ["B", "B0", "123"], text_len=self.text_len)
+
+
 if __name__ == '__main__':
  unittest.main()

Lib/textwrap.py

@@ -122,7 +122,8 @@ def __init__(self,
  tabsize=8,
  *,
  max_lines=None,
- placeholder=' [...]'):
+ placeholder=' [...]',
+ text_len=len):
  self.width = width
  self.initial_indent = initial_indent
  self.subsequent_indent = subsequent_indent
@@ -135,6 +136,7 @@ def __init__(self,
  self.tabsize = tabsize
  self.max_lines = max_lines
  self.placeholder = placeholder
+ self.text_len = text_len
 
 
  # -- Private methods -----------------------------------------------
@@ -194,6 +196,25 @@ def _fix_sentence_endings(self, chunks):
  else:
  i += 1
 
+ def _find_width_index(self, text, width):
+ """_find_length_index(text : string, width : int)
+
+ Find at which index the text has the required width, since when using a
+ different text_len, this index will not be equal to the required width.
+ """
+ # When using default len as self.text_len, the required index and width
+ # will be equal, this prevents calculation time.
+ if self.text_len(text[:width]) == width:
+ # For character widths greater than one, width can be more than the
+ # number of characters
+ return min(width, len(text))
+ cur_text = ''
+ for i, c in enumerate(text):
+ cur_text += c
+ cur_width = self.text_len(cur_text)
+ if cur_width > width:
+ return max(i, 1)
+
  def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
  """_handle_long_word(chunks : [string],
  cur_line : [string],
@@ -214,10 +235,11 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
  if self.break_long_words and space_left > 0:
  end = space_left
  chunk = reversed_chunks[-1]
- if self.break_on_hyphens and len(chunk) > space_left:
+ end = self._find_width_index(chunk, space_left)
+ if self.break_on_hyphens and self.text_len(chunk) > space_left:
  # break after last hyphen, but only if there are
  # non-hyphens before it
- hyphen = chunk.rfind('-', 0, space_left)
+ hyphen = chunk.rfind('-', 0, end)
  if hyphen > 0 and any(c != '-' for c in chunk[:hyphen]):
  end = hyphen + 1
  cur_line.append(chunk[:end])
@@ -256,7 +278,8 @@ def _wrap_chunks(self, chunks):
  indent = self.subsequent_indent
  else:
  indent = self.initial_indent
- if len(indent) + len(self.placeholder.lstrip()) > self.width:
+ if (self.text_len(indent) +
+ self.text_len(self.placeholder.lstrip()) > self.width):
  raise ValueError("placeholder too large for max width")
 
  # Arrange in reverse order so items can be efficiently popped
@@ -277,15 +300,15 @@ def _wrap_chunks(self, chunks):
  indent = self.initial_indent
 
  # Maximum width for this line.
- width = self.width - len(indent)
+ width = self.width - self.text_len(indent)
 
  # First chunk on line is whitespace -- drop it, unless this
  # is the very beginning of the text (ie. no lines started yet).
  if self.drop_whitespace and chunks[-1].strip() == '' and lines:
  del chunks[-1]
 
  while chunks:
- l = len(chunks[-1])
+ l = self.text_len(chunks[-1])
 
  # Can at least squeeze this chunk onto the current line.
  if cur_len + l <= width:
@@ -298,13 +321,13 @@ def _wrap_chunks(self, chunks):
 
  # The current line is full, and the next chunk is too big to
  # fit on *any* line (not just this one).
- if chunks and len(chunks[-1]) > width:
+ if chunks and self.text_len(chunks[-1]) > width:
  self._handle_long_word(chunks, cur_line, cur_len, width)
- cur_len = sum(map(len, cur_line))
+ cur_len = sum(map(self.text_len, cur_line))
 
  # If the last chunk on this line is all whitespace, drop it.
  if self.drop_whitespace and cur_line and cur_line[-1].strip() == '':
- cur_len -= len(cur_line[-1])
+ cur_len -= self.text_len(cur_line[-1])
  del cur_line[-1]
 
  if cur_line:
@@ -320,16 +343,17 @@ def _wrap_chunks(self, chunks):
  else:
  while cur_line:
  if (cur_line[-1].strip() and
- cur_len + len(self.placeholder) <= width):
+ cur_len + self.text_len(self.placeholder) <= width):
  cur_line.append(self.placeholder)
  lines.append(indent + ''.join(cur_line))
  break
- cur_len -= len(cur_line[-1])
+ cur_len -= self.text_len(cur_line[-1])
  del cur_line[-1]
  else:
  if lines:
  prev_line = lines[-1].rstrip()
- if (len(prev_line) + len(self.placeholder) <=
+ if (self.text_len(prev_line) +
+ self.text_len(self.placeholder) <=
  self.width):
  lines[-1] = prev_line + self.placeholder
  break