Skip to content

[3.13] gh-136065: Fix quadratic complexity in os.path.expandvars() (GH-134952)#140845

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ambv merged 2 commits into from
Oct 31, 2025
Merged

[3.13] gh-136065: Fix quadratic complexity in os.path.expandvars() (GH-134952) #140845

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4fc2109
[3.13] gh-136065: Fix quadratic complexity in os.path.expandvars() (G…
serhiy-storchaka Oct 31, 2025
829d2e9
Oops
ambv Oct 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 41 additions & 85 deletions Lib/ntpath.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -400,17 +400,23 @@ def expanduser(path):
# XXX With COMMAND.COM you can use any characters in a variable name,
# XXX except '^|<>='.

_varpattern=r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)"
_varsub=None
_varsubb=None

defexpandvars(path):
"""Expand shell variables of the forms $var, ${var} and %var%.
Unknown variables are left unchanged."""
path=os.fspath(path)
global_varsub, _varsubb
ifisinstance(path, bytes):
ifb'$'notinpathandb'%'notinpath:
returnpath
importstring
varchars=bytes(string.ascii_letters+string.digits+'_-', 'ascii')
quote=b'\''
ifnot_varsubb:
importre
_varsubb=re.compile(_varpattern.encode(), re.ASCII).sub
sub=_varsubb
percent=b'%'
brace=b'{'
rbrace=b'}'
Expand All@@ -419,94 +425,44 @@ def expandvars(path):
else:
if'$'notinpathand'%'notinpath:
returnpath
importstring
varchars=string.ascii_letters+string.digits+'_-'
quote='\''
ifnot_varsub:
importre
_varsub=re.compile(_varpattern, re.ASCII).sub
sub=_varsub
percent='%'
brace='{'
rbrace='}'
dollar='$'
environ=os.environ
res=path[:0]
index=0
pathlen=len(path)
whileindex<pathlen:
c=path[index:index+1]
ifc==quote: # no expansion within single quotes
path=path[index+1:]
pathlen=len(path)
try:
index=path.index(c)
res+=c+path[:index+1]
exceptValueError:
res+=c+path
index=pathlen-1
elifc==percent: # variable or '%'
ifpath[index+1:index+2] ==percent:
res+=c
index+=1
else:
path=path[index+1:]
pathlen=len(path)
try:
index=path.index(percent)
exceptValueError:
res+=percent+path
index=pathlen-1
else:
var=path[:index]
try:
ifenvironisNone:
value=os.fsencode(os.environ[os.fsdecode(var)])
else:
value=environ[var]
exceptKeyError:
value=percent+var+percent
res+=value
elifc==dollar: # variable or '$$'
ifpath[index+1:index+2] ==dollar:
res+=c
index+=1
elifpath[index+1:index+2] ==brace:
path=path[index+2:]
pathlen=len(path)
try:
index=path.index(rbrace)
exceptValueError:
res+=dollar+brace+path
index=pathlen-1
else:
var=path[:index]
try:
ifenvironisNone:
value=os.fsencode(os.environ[os.fsdecode(var)])
else:
value=environ[var]
exceptKeyError:
value=dollar+brace+var+rbrace
res+=value
else:
var=path[:0]
index+=1
c=path[index:index+1]
whilecandcinvarchars:
var+=c
index+=1
c=path[index:index+1]
try:
ifenvironisNone:
value=os.fsencode(os.environ[os.fsdecode(var)])
else:
value=environ[var]
exceptKeyError:
value=dollar+var
res+=value
ifc:
index-=1

defrepl(m):
lastindex=m.lastindex
iflastindexisNone:
returnm[0]
name=m[lastindex]
iflastindex==1:
ifname==percent:
returnname
ifnotname.endswith(percent):
returnm[0]
name=name[:-1]
else:
res+=c
index+=1
returnres
ifname==dollar:
returnname
ifname.startswith(brace):
ifnotname.endswith(rbrace):
returnm[0]
name=name[1:-1]

try:
ifenvironisNone:
returnos.fsencode(os.environ[os.fsdecode(name)])
else:
returnenviron[name]
exceptKeyError:
returnm[0]

returnsub(repl, path)


# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
Expand Down
43 changes: 20 additions & 23 deletions Lib/posixpath.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -284,56 +284,53 @@ def expanduser(path):
# This expands the forms $variable and ${variable} only.
# Non-existent variables are left unchanged.

_varprog=None
_varprogb=None
_varpattern=r'\$(\w+|\{[^}]*\}?)'
_varsub=None
_varsubb=None

defexpandvars(path):
"""Expand shell variables of form $var and ${var}. Unknown variables
are left unchanged."""
path=os.fspath(path)
global_varprog, _varprogb
global_varsub, _varsubb
ifisinstance(path, bytes):
ifb'$'notinpath:
returnpath
ifnot_varprogb:
ifnot_varsubb:
importre
_varprogb=re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
search=_varprogb.search
_varsubb=re.compile(_varpattern.encode(), re.ASCII).sub
sub=_varsubb
start=b'{'
end=b'}'
environ=getattr(os, 'environb', None)
else:
if'$'notinpath:
returnpath
ifnot_varprog:
ifnot_varsub:
importre
_varprog=re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
search=_varprog.search
_varsub=re.compile(_varpattern, re.ASCII).sub
sub=_varsub
start='{'
end='}'
environ=os.environ
i=0
whileTrue:
m=search(path, i)
ifnotm:
break
i, j=m.span(0)
name=m.group(1)
ifname.startswith(start) andname.endswith(end):

defrepl(m):
name=m[1]
ifname.startswith(start):
ifnotname.endswith(end):
returnm[0]
name=name[1:-1]
try:
ifenvironisNone:
value=os.fsencode(os.environ[os.fsdecode(name)])
else:
value=environ[name]
exceptKeyError:
i=j
returnm[0]
else:
tail=path[j:]
path=path[:i] +value
i=len(path)
path+=tail
returnpath
returnvalue

returnsub(repl, path)


# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.
Expand Down
21 changes: 17 additions & 4 deletions Lib/test/test_genericpath.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,9 +7,9 @@
importsys
importunittest
importwarnings
fromtest.supportimport(
is_apple, is_emscripten, os_helper, warnings_helper
)
fromtestimportsupport
fromtest.supportimportos_helper, is_emscripten
fromtest.supportimportwarnings_helper
fromtest.support.script_helperimportassert_python_ok
fromtest.support.os_helperimportFakePath

Expand DownExpand Up@@ -446,6 +446,19 @@ def check(value, expected):
os.fsencode('$bar%s bar'%nonascii))
check(b'$spam}bar', os.fsencode('%s}bar'%nonascii))

@support.requires_resource('cpu')
deftest_expandvars_large(self):
expandvars=self.pathmodule.expandvars
withos_helper.EnvironmentVarGuard() asenv:
env.clear()
env["A"] ="B"
n=100_000
self.assertEqual(expandvars('$A'*n), 'B'*n)
self.assertEqual(expandvars('${A}'*n), 'B'*n)
self.assertEqual(expandvars('$A!'*n), 'B!'*n)
self.assertEqual(expandvars('${A}A'*n), 'BA'*n)
self.assertEqual(expandvars('${'*10*n), '${'*10*n)

deftest_abspath(self):
self.assertIn("foo", self.pathmodule.abspath("foo"))
withwarnings.catch_warnings():
Expand DownExpand Up@@ -503,7 +516,7 @@ def test_nonascii_abspath(self):
# directory (when the bytes name is used).
andsys.platformnotin{
"win32", "emscripten", "wasi"
} andnotis_apple
} andnotsupport.is_apple
):
name=os_helper.TESTFN_UNDECODABLE
elifos_helper.TESTFN_NONASCII:
Expand Down
22 changes: 17 additions & 5 deletions Lib/test/test_ntpath.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,8 +8,7 @@
importwarnings
fromntpathimportALLOW_MISSING
fromtestimportsupport
fromtest.supportimportcpython_only, os_helper
fromtest.supportimportTestFailed, is_emscripten
fromtest.supportimportos_helper, is_emscripten
fromtest.support.os_helperimportFakePath
fromtestimporttest_genericpath
fromtempfileimportTemporaryFile
Expand DownExpand Up@@ -59,7 +58,7 @@ def tester(fn, wantResult):
fn=fn.replace("\\", "\\\\")
gotResult=eval(fn)
ifwantResult!=gotResultand_norm(wantResult) !=_norm(gotResult):
raiseTestFailed("%s should return: %s but returned: %s" \
raisesupport.TestFailed("%s should return: %s but returned: %s" \
%(str(fn), str(wantResult), str(gotResult)))

# then with bytes
Expand All@@ -75,7 +74,7 @@ def tester(fn, wantResult):
warnings.simplefilter("ignore", DeprecationWarning)
gotResult=eval(fn)
if_norm(wantResult) !=_norm(gotResult):
raiseTestFailed("%s should return: %s but returned: %s" \
raisesupport.TestFailed("%s should return: %s but returned: %s" \
%(str(fn), str(wantResult), repr(gotResult)))


Expand DownExpand Up@@ -1022,6 +1021,19 @@ def check(value, expected):
check('%spam%bar', '%sbar'%nonascii)
check('%{}%bar'.format(nonascii), 'ham%sbar'%nonascii)

@support.requires_resource('cpu')
deftest_expandvars_large(self):
expandvars=ntpath.expandvars
withos_helper.EnvironmentVarGuard() asenv:
env.clear()
env["A"] ="B"
n=100_000
self.assertEqual(expandvars('%A%'*n), 'B'*n)
self.assertEqual(expandvars('%A%A'*n), 'BA'*n)
self.assertEqual(expandvars("''"*n+'%%'), "''"*n+'%')
self.assertEqual(expandvars("%%"*n), "%"*n)
self.assertEqual(expandvars("$$"*n), "$"*n)

deftest_expanduser(self):
tester('ntpath.expanduser("test")', 'test')

Expand DownExpand Up@@ -1440,7 +1452,7 @@ def test_con_device(self):
self.assertTrue(os.path.exists(r"\\.\CON"))

@unittest.skipIf(sys.platform!='win32', "Fast paths are only for win32")
@cpython_only
@support.cpython_only
deftest_fast_paths_in_use(self):
# There are fast paths of these functions implemented in posixmodule.c.
# Confirm that they are being used, and not the Python fallbacks in
Expand Down
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
Fix quadratic complexity in :func:`os.path.expandvars`.
Loading