[3.9] gh-136065: Fix quadratic complexity in os.path.expandvars() (GH-134952)

Lib/ntpath.py

@@ -335,17 +335,23 @@ def expanduser(path):
 # XXX With COMMAND.COM you can use any characters in a variable name,
 # XXX except '^|<>='.
 
+_varpattern=r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)"
+_varsub=None
+_varsubb=None
+
 defexpandvars(path):
 """Expand shell variables of the forms $var, ${var} and %var%.
 
  Unknown variables are left unchanged."""
 path=os.fspath(path)
+global_varsub, _varsubb
 ifisinstance(path, bytes):
 ifb'$'notinpathandb'%'notinpath:
 returnpath
-importstring
-varchars=bytes(string.ascii_letters+string.digits+'_-', 'ascii')
-quote=b'\''
+ifnot_varsubb:
+importre
+_varsubb=re.compile(_varpattern.encode(), re.ASCII).sub
+sub=_varsubb
 percent=b'%'
 brace=b'{'
 rbrace=b'}'
@@ -354,94 +360,44 @@ def expandvars(path):
 else:
 if'$'notinpathand'%'notinpath:
 returnpath
-importstring
-varchars=string.ascii_letters+string.digits+'_-'
-quote='\''
+ifnot_varsub:
+importre
+_varsub=re.compile(_varpattern, re.ASCII).sub
+sub=_varsub
 percent='%'
 brace='{'
 rbrace='}'
 dollar='$'
 environ=os.environ
-res=path[:0]
-index=0
-pathlen=len(path)
-whileindex<pathlen:
-c=path[index:index+1]
-ifc==quote: # no expansion within single quotes
-path=path[index+1:]
-pathlen=len(path)
-try:
-index=path.index(c)
-res+=c+path[:index+1]
-exceptValueError:
-res+=c+path
-index=pathlen-1
-elifc==percent: # variable or '%'
-ifpath[index+1:index+2] ==percent:
-res+=c
-index+=1
-else:
-path=path[index+1:]
-pathlen=len(path)
-try:
-index=path.index(percent)
-exceptValueError:
-res+=percent+path
-index=pathlen-1
-else:
-var=path[:index]
-try:
-ifenvironisNone:
-value=os.fsencode(os.environ[os.fsdecode(var)])
-else:
-value=environ[var]
-exceptKeyError:
-value=percent+var+percent
-res+=value
-elifc==dollar: # variable or '$$'
-ifpath[index+1:index+2] ==dollar:
-res+=c
-index+=1
-elifpath[index+1:index+2] ==brace:
-path=path[index+2:]
-pathlen=len(path)
-try:
-index=path.index(rbrace)
-exceptValueError:
-res+=dollar+brace+path
-index=pathlen-1
-else:
-var=path[:index]
-try:
-ifenvironisNone:
-value=os.fsencode(os.environ[os.fsdecode(var)])
-else:
-value=environ[var]
-exceptKeyError:
-value=dollar+brace+var+rbrace
-res+=value
-else:
-var=path[:0]
-index+=1
-c=path[index:index+1]
-whilecandcinvarchars:
-var+=c
-index+=1
-c=path[index:index+1]
-try:
-ifenvironisNone:
-value=os.fsencode(os.environ[os.fsdecode(var)])
-else:
-value=environ[var]
-exceptKeyError:
-value=dollar+var
-res+=value
-ifc:
-index-=1
+
+defrepl(m):
+lastindex=m.lastindex
+iflastindexisNone:
+returnm[0]
+name=m[lastindex]
+iflastindex==1:
+ifname==percent:
+returnname
+ifnotname.endswith(percent):
+returnm[0]
+name=name[:-1]
 else:
-res+=c
-index+=1
-returnres
+ifname==dollar:
+returnname
+ifname.startswith(brace):
+ifnotname.endswith(rbrace):
+returnm[0]
+name=name[1:-1]
+
+try:
+ifenvironisNone:
+returnos.fsencode(os.environ[os.fsdecode(name)])
+else:
+returnenviron[name]
+exceptKeyError:
+returnm[0]
+
+returnsub(repl, path)
 
 
 # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.

Lib/posixpath.py

@@ -275,56 +275,53 @@ def expanduser(path):
 # This expands the forms $variable and ${variable} only.
 # Non-existent variables are left unchanged.
 
-_varprog=None
-_varprogb=None
+_varpattern=r'\$(\w+|\{[^}]*\}?)'
+_varsub=None
+_varsubb=None
 
 defexpandvars(path):
 """Expand shell variables of form $var and ${var}. Unknown variables
  are left unchanged."""
 path=os.fspath(path)
-global_varprog, _varprogb
+global_varsub, _varsubb
 ifisinstance(path, bytes):
 ifb'$'notinpath:
 returnpath
-ifnot_varprogb:
+ifnot_varsubb:
 importre
-_varprogb=re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
-search=_varprogb.search
+_varsubb=re.compile(_varpattern.encode(), re.ASCII).sub
+sub=_varsubb
 start=b'{'
 end=b'}'
 environ=getattr(os, 'environb', None)
 else:
 if'$'notinpath:
 returnpath
-ifnot_varprog:
+ifnot_varsub:
 importre
-_varprog=re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
-search=_varprog.search
+_varsub=re.compile(_varpattern, re.ASCII).sub
+sub=_varsub
 start='{'
 end='}'
 environ=os.environ
-i=0
-whileTrue:
-m=search(path, i)
-ifnotm:
-break
-i, j=m.span(0)
-name=m.group(1)
-ifname.startswith(start) andname.endswith(end):
+
+defrepl(m):
+name=m[1]
+ifname.startswith(start):
+ifnotname.endswith(end):
+returnm[0]
 name=name[1:-1]
 try:
 ifenvironisNone:
 value=os.fsencode(os.environ[os.fsdecode(name)])
 else:
 value=environ[name]
 exceptKeyError:
-i=j
+returnm[0]
 else:
-tail=path[j:]
-path=path[:i] +value
-i=len(path)
-path+=tail
-returnpath
+returnvalue
+
+returnsub(repl, path)
 
 
 # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.

Lib/test/test_genericpath.py

@@ -9,7 +9,7 @@
 importwarnings
 fromtestimportsupport
 fromtest.support.script_helperimportassert_python_ok
-fromtest.supportimportFakePath
+fromtest.supportimportFakePath, EnvironmentVarGuard
 
 
 defcreate_file(filename, data=b'foo'):
@@ -374,7 +374,7 @@ def test_splitdrive(self):
 
 deftest_expandvars(self):
 expandvars=self.pathmodule.expandvars
-withsupport.EnvironmentVarGuard() asenv:
+withEnvironmentVarGuard() asenv:
 env.clear()
 env["foo"] ="bar"
 env["{foo"] ="baz1"
@@ -408,7 +408,7 @@ def test_expandvars_nonascii(self):
 expandvars=self.pathmodule.expandvars
 defcheck(value, expected):
 self.assertEqual(expandvars(value), expected)
-withsupport.EnvironmentVarGuard() asenv:
+withEnvironmentVarGuard() asenv:
 env.clear()
 nonascii=support.FS_NONASCII
 env['spam'] =nonascii
@@ -429,6 +429,19 @@ def check(value, expected):
 os.fsencode('$bar%s bar'%nonascii))
 check(b'$spam}bar', os.fsencode('%s}bar'%nonascii))
 
+@support.requires_resource('cpu')
+deftest_expandvars_large(self):
+expandvars=self.pathmodule.expandvars
+withEnvironmentVarGuard() asenv:
+env.clear()
+env["A"] ="B"
+n=100_000
+self.assertEqual(expandvars('$A'*n), 'B'*n)
+self.assertEqual(expandvars('${A}'*n), 'B'*n)
+self.assertEqual(expandvars('$A!'*n), 'B!'*n)
+self.assertEqual(expandvars('${A}A'*n), 'BA'*n)
+self.assertEqual(expandvars('${'*10*n), '${'*10*n)
+
 deftest_abspath(self):
 self.assertIn("foo", self.pathmodule.abspath("foo"))
 withwarnings.catch_warnings():

Lib/test/test_ntpath.py

@@ -1,11 +1,10 @@
 importntpath
 importos
-importsubprocess
 importsys
 importunittest
 importwarnings
 fromntpathimportALLOW_MISSING
-fromtest.supportimportTestFailed, FakePath
+fromtest.supportimportTestFailed, FakePath, EnvironmentVarGuard
 fromtestimportsupport, test_genericpath
 fromtempfileimportTemporaryFile
 
@@ -642,7 +641,7 @@ def test_realpath_cwd(self):
 ntpath.realpath("file.txt", **kwargs))
 
 deftest_expandvars(self):
-withsupport.EnvironmentVarGuard() asenv:
+withEnvironmentVarGuard() asenv:
 env.clear()
 env["foo"] ="bar"
 env["{foo"] ="baz1"
@@ -671,7 +670,7 @@ def test_expandvars(self):
 deftest_expandvars_nonascii(self):
 defcheck(value, expected):
 tester('ntpath.expandvars(%r)'%value, expected)
-withsupport.EnvironmentVarGuard() asenv:
+withEnvironmentVarGuard() asenv:
 env.clear()
 nonascii=support.FS_NONASCII
 env['spam'] =nonascii
@@ -687,10 +686,23 @@ def check(value, expected):
 check('%spam%bar', '%sbar'%nonascii)
 check('%{}%bar'.format(nonascii), 'ham%sbar'%nonascii)
 
+@support.requires_resource('cpu')
+deftest_expandvars_large(self):
+expandvars=ntpath.expandvars
+withEnvironmentVarGuard() asenv:
+env.clear()
+env["A"] ="B"
+n=100_000
+self.assertEqual(expandvars('%A%'*n), 'B'*n)
+self.assertEqual(expandvars('%A%A'*n), 'BA'*n)
+self.assertEqual(expandvars("''"*n+'%%'), "''"*n+'%')
+self.assertEqual(expandvars("%%"*n), "%"*n)
+self.assertEqual(expandvars("$$"*n), "$"*n)
+
 deftest_expanduser(self):
 tester('ntpath.expanduser("test")', 'test')
 
-withsupport.EnvironmentVarGuard() asenv:
+withEnvironmentVarGuard() asenv:
 env.clear()
 tester('ntpath.expanduser("~test")', '~test')
 
@@ -908,6 +920,7 @@ def test_nt_helpers(self):
 self.assertIsInstance(b_final_path, bytes)
 self.assertGreater(len(b_final_path), 0)
 
+
 classNtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
 pathmodule=ntpath
 attributes= ['relpath']

Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst

@@ -0,0 +1 @@
+Fix quadratic complexity in :func:`os.path.expandvars`.