Skip to content

gh-129068: Make range iterators thread-safe#142886

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
colesbury merged 6 commits into from
Dec 18, 2025
Merged

gh-129068: Make range iterators thread-safe #142886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
afd3eff
gh-129068: Make range iterators thread-safe
colesbury Dec 16, 2025
366d88c
Blurb
colesbury Dec 17, 2025
3287a20
Update generated clinic files
colesbury Dec 17, 2025
c766b4e
Use 'self as r: self' to reduce diff size
colesbury Dec 17, 2025
30bbb31
Merge branch 'main' into gh-129068-rangeiter-next
colesbury Dec 17, 2025
b43cf0d
Merge branch 'main' into gh-129068-rangeiter-next
colesbury Dec 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core_and_Builtins/2025-12-17-10-49-03.gh-issue-129068.GlYnrO.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
Make concurrent iteration over the same range iterator thread-safe in the
free threading build.
150 changes: 150 additions & 0 deletions Objects/clinic/rangeobject.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

131 changes: 99 additions & 32 deletions Objects/rangeobject.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,21 @@
#include "pycore_range.h"
#include "pycore_tuple.h" // _PyTuple_ITEMS()

typedef struct{
PyObject_HEAD
PyObject *start;
PyObject *step;
PyObject *len;
} longrangeiterobject;

/*[clinic input]
class range_iterator "_PyRangeIterObject *" "&PyRangeIter_Type"
class longrange_iterator "longrangeiterobject *" "&PyLongRangeIter_Type"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c7d97a63d1cfa6b3]*/

#include "clinic/rangeobject.c.h"


/* Support objects whose length is > PY_SSIZE_T_MAX.

Expand DownExpand Up@@ -830,30 +845,46 @@ PyTypeObject PyRange_Type ={
static PyObject *
rangeiter_next(PyObject *op)
{
PyObject *ret = NULL;
Py_BEGIN_CRITICAL_SECTION(op);
_PyRangeIterObject *r = (_PyRangeIterObject*)op;
if (r->len > 0){
long result = r->start;
r->start = result + r->step;
r->len--;
return PyLong_FromLong(result);
ret = PyLong_FromLong(result);
}
return NULL;
Py_END_CRITICAL_SECTION();
return ret;
}

/*[clinic input]
@critical_section
range_iterator.__length_hint__
self as r: self(type="_PyRangeIterObject *")

Private method returning an estimate of len(list(it)).
[clinic start generated code]*/

static PyObject *
rangeiter_len(PyObject *op, PyObject *Py_UNUSED(ignored))
range_iterator___length_hint___impl(_PyRangeIterObject *r)
/*[clinic end generated code: output=9ba6f22b1fc23dcc input=e3eb311e99d76e43]*/
{
_PyRangeIterObject *r = (_PyRangeIterObject*)op;
return PyLong_FromLong(r->len);
}

PyDoc_STRVAR(length_hint_doc,
"Private method returning an estimate of len(list(it)).");
/*[clinic input]
@critical_section
range_iterator.__reduce__
self as r: self(type="_PyRangeIterObject *")

Return state information for pickling.
[clinic start generated code]*/

static PyObject *
rangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
range_iterator___reduce___impl(_PyRangeIterObject *r)
/*[clinic end generated code: output=c44d53750c388415 input=75a25b7076dc2c54]*/
{
_PyRangeIterObject *r = (_PyRangeIterObject*)op;
PyObject *start=NULL, *stop=NULL, *step=NULL;
PyObject *range;

Expand DownExpand Up@@ -881,10 +912,20 @@ rangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
return NULL;
}

/*[clinic input]
@critical_section
range_iterator.__setstate__
self as r: self(type="_PyRangeIterObject *")
state: object
/

Set state information for unpickling.
[clinic start generated code]*/

static PyObject *
rangeiter_setstate(PyObject *op, PyObject *state)
range_iterator___setstate___impl(_PyRangeIterObject *r, PyObject *state)
/*[clinic end generated code: output=464b3cbafc2e3562 input=c8c84fab2519d200]*/
{
_PyRangeIterObject *r = (_PyRangeIterObject*)op;
long index = PyLong_AsLong(state);
if (index == -1 && PyErr_Occurred())
return NULL;
Expand All@@ -904,13 +945,10 @@ rangeiter_dealloc(PyObject *self)
_Py_FREELIST_FREE(range_iters, (_PyRangeIterObject *)self, PyObject_Free);
}

PyDoc_STRVAR(reduce_doc, "Return state information for pickling.");
PyDoc_STRVAR(setstate_doc, "Set state information for unpickling.");

static PyMethodDef rangeiter_methods[] ={
{"__length_hint__", rangeiter_len, METH_NOARGS, length_hint_doc},
{"__reduce__", rangeiter_reduce, METH_NOARGS, reduce_doc},
{"__setstate__", rangeiter_setstate, METH_O, setstate_doc},
RANGE_ITERATOR___LENGTH_HINT___METHODDEF
RANGE_ITERATOR___REDUCE___METHODDEF
RANGE_ITERATOR___SETSTATE___METHODDEF
{NULL, NULL} /* sentinel */
};

Expand DownExpand Up@@ -995,25 +1033,34 @@ fast_range_iter(long start, long stop, long step, long len)
return (PyObject *)it;
}

typedef struct{
PyObject_HEAD
PyObject *start;
PyObject *step;
PyObject *len;
} longrangeiterobject;
/*[clinic input]
@critical_section
longrange_iterator.__length_hint__
self as r: self(type="longrangeiterobject *")

Private method returning an estimate of len(list(it)).
[clinic start generated code]*/

static PyObject *
longrangeiter_len(PyObject *op, PyObject *Py_UNUSED(ignored))
longrange_iterator___length_hint___impl(longrangeiterobject *r)
/*[clinic end generated code: output=e1bce24da7e8bfde input=ba94b050d940411e]*/
{
longrangeiterobject *r = (longrangeiterobject*)op;
Py_INCREF(r->len);
return r->len;
}

/*[clinic input]
@critical_section
longrange_iterator.__reduce__
self as r: self(type="longrangeiterobject *")

Return state information for pickling.
[clinic start generated code]*/

static PyObject *
longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
longrange_iterator___reduce___impl(longrangeiterobject *r)
/*[clinic end generated code: output=0077f94ae2a4e99a input=2e8930e897ace086]*/
{
longrangeiterobject *r = (longrangeiterobject*)op;
PyObject *product, *stop=NULL;
PyObject *range;

Expand All@@ -1039,15 +1086,25 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
range, Py_None);
}

/*[clinic input]
@critical_section
longrange_iterator.__setstate__
self as r: self(type="longrangeiterobject *")
state: object
/

Set state information for unpickling.
[clinic start generated code]*/

static PyObject *
longrangeiter_setstate(PyObject *op, PyObject *state)
longrange_iterator___setstate___impl(longrangeiterobject *r, PyObject *state)
/*[clinic end generated code: output=870787f0574f0da4 input=8b116de3018de824]*/
{
if (!PyLong_CheckExact(state)){
PyErr_Format(PyExc_TypeError, "state must be an int, not %T", state);
return NULL;
}

longrangeiterobject *r = (longrangeiterobject*)op;
PyObject *zero = _PyLong_GetZero(); // borrowed reference
int cmp;

Expand DownExpand Up@@ -1085,9 +1142,9 @@ longrangeiter_setstate(PyObject *op, PyObject *state)
}

static PyMethodDef longrangeiter_methods[] ={
{"__length_hint__", longrangeiter_len, METH_NOARGS, length_hint_doc},
{"__reduce__", longrangeiter_reduce, METH_NOARGS, reduce_doc},
{"__setstate__", longrangeiter_setstate, METH_O, setstate_doc},
LONGRANGE_ITERATOR___LENGTH_HINT___METHODDEF
LONGRANGE_ITERATOR___REDUCE___METHODDEF
LONGRANGE_ITERATOR___SETSTATE___METHODDEF
{NULL, NULL} /* sentinel */
};

Expand All@@ -1102,7 +1159,7 @@ longrangeiter_dealloc(PyObject *op)
}

static PyObject *
longrangeiter_next(PyObject *op)
longrangeiter_next_lock_held(PyObject *op)
{
longrangeiterobject *r = (longrangeiterobject*)op;
if (PyObject_RichCompareBool(r->len, _PyLong_GetZero(), Py_GT) != 1)
Expand All@@ -1123,6 +1180,16 @@ longrangeiter_next(PyObject *op)
return result;
}

static PyObject *
longrangeiter_next(PyObject *op)
{
PyObject *result;
Py_BEGIN_CRITICAL_SECTION(op);
result = longrangeiter_next_lock_held(op);
Py_END_CRITICAL_SECTION();
return result;
}

PyTypeObject PyLongRangeIter_Type ={
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"longrange_iterator", /* tp_name */
Expand Down
6 changes: 0 additions & 6 deletions Tools/tsan/suppressions_free_threading.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,15 +20,9 @@ race_top:_PyObject_TryGetInstanceAttribute
race_top:PyUnstable_InterpreterFrame_GetLine
race_top:write_thread_id

# gh-129068: race on shared range iterators (test_free_threading.test_zip.ZipThreading.test_threading)
race_top:rangeiter_next

# https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40
thread:pthread_create

# Range iteration is not thread-safe yet (issue #129068)
race_top:rangeiter_next

# List resizing happens through different paths ending in memcpy or memmove
# (for efficiency), which will probably need to rewritten as explicit loops
# of ptr-sized copies to be thread-safe. (Issue #129069)
Expand Down
Loading