Skip to content

bpo-43916: Add _PyType_DisabledNew to prevent new heap types from being created uninitialised#25653

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
erlend-aasland wants to merge 17 commits into from
Closed

bpo-43916: Add _PyType_DisabledNew to prevent new heap types from being created uninitialised #25653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
36150d6
bpo-43916: Apply tp_new = 0 hack to affected types
Apr 26, 2021
dbebde5
Add _PyType_DisabledNew
Apr 27, 2021
54646a5
Use _PyType_DisabledNew in affected types
Apr 27, 2021
02829da
Address review: improve comments in Lib/test
Apr 27, 2021
0c3872e
Add re, functools, threading, zlib, posix, and unicodedata tests
Apr 27, 2021
45169ff
Fix threading test
Apr 27, 2021
69b19bc
Decorate tests with @cpython_only
Apr 28, 2021
2cc4dd1
Adapt _curses_panel and tkinter workarounds/tests
Apr 28, 2021
d41b799
Add news (DRAFT)
Apr 28, 2021
3f34237
Move _PyType_DisabledNew to Include/internal and build affected exten…
Apr 28, 2021
eee3044
Address review: don't truncate type names
Apr 28, 2021
5c7f047
Address review: bring the types outside the assertRaises()
Apr 28, 2021
70dcae3
Add missing include in _winapi.c
Apr 28, 2021
7fb93a9
Include pycore_object late to avoid conflict with windows.h
Apr 29, 2021
66552f0
Add gdbm test
Apr 29, 2021
ce16da5
Always build tkinter with Py_BUILD_CORE set
Apr 29, 2021
1450485
Revert changes to Modules/Setup
Apr 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Include/internal/pycore_object.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature){

extern void _PyType_InitCache(PyInterpreterState *interp);

PyAPI_FUNC(PyObject *) _PyType_DisabledNew(PyTypeObject *type, PyObject *args,
PyObject *kwds);


/* Inline functions trading binary compatibility for speed:
_PyObject_Init() is the fast version of PyObject_Init(), and
Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_array.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,12 @@ def test_bad_constructor(self):
self.assertRaises(TypeError, array.array, 'xx')
self.assertRaises(ValueError, array.array, 'x')

@support.cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
tp = type(iter(array.array('I')))
self.assertRaises(TypeError, tp)

@support.cpython_only
def test_immutable(self):
# bpo-43908: check that array.array is immutable
Expand Down
3 changes: 2 additions & 1 deletion Lib/test/test_curses.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@
import tempfile
import unittest

from test.support import requires, verbose, SaveSignals
from test.support import requires, verbose, SaveSignals, cpython_only
from test.support.import_helper import import_module

# Optionally test curses module. This currently requires that the
Expand DownExpand Up@@ -1046,6 +1046,7 @@ def __del__(self):
panel.set_userptr(A())
panel.set_userptr(None)

@cpython_only
@requires_curses_func('panel')
def test_new_curses_panel(self):
w = curses.newwin(10, 10)
Expand Down
9 changes: 8 additions & 1 deletion Lib/test/test_dbm_gnu.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
from test import support
from test.support import import_helper
from test.support import import_helper, cpython_only
gdbm = import_helper.import_module("dbm.gnu") #skip if not supported
import unittest
import os
Expand DownExpand Up@@ -27,6 +27,13 @@ def tearDown(self):
self.g.close()
unlink(filename)

@cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
self.g = gdbm.open(filename, 'c')
tp = type(self.g)
self.assertRaises(TypeError, tp)

def test_key_methods(self):
self.g = gdbm.open(filename, 'c')
self.assertEqual(self.g.keys(), [])
Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_functools.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -948,6 +948,12 @@ class TestCmpToKeyC(TestCmpToKey, unittest.TestCase):
if c_functools:
cmp_to_key = c_functools.cmp_to_key

@support.cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
tp = type(c_functools.cmp_to_key(None))
self.assertRaises(TypeError, tp)


class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase):
cmp_to_key = staticmethod(py_functools.cmp_to_key)
Expand Down
7 changes: 7 additions & 0 deletions Lib/test/test_posix.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,13 @@ def tearDown(self):
os_helper.unlink(teardown_file)
self._warnings_manager.__exit__(None, None, None)

@support.cpython_only
def test_uninitialised_new(self):
Copy link
Member

@serhiy-storchakaserhiy-storchakaApr 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is already tested in test_os.

# Prevent heap types from being created uninitialised (bpo-43916)
self.assertRaises(TypeError, posix.DirEntry)
tp = type(posix.scandir())
self.assertRaises(TypeError, tp)

def testNoArgFunctions(self):
# test posix functions which take no arguments and have
# no side-effects which we need to cleanup (e.g., fork, wait, abort)
Expand Down
9 changes: 9 additions & 0 deletions Lib/test/test_re.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2215,6 +2215,15 @@ def test_signedness(self):
self.assertGreaterEqual(sre_compile.MAXREPEAT, 0)
self.assertGreaterEqual(sre_compile.MAXGROUPS, 0)

@cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
self.assertRaises(TypeError, re.Match)
self.assertRaises(TypeError, re.Pattern)
pat = re.compile("")
tp = type(pat.scanner(""))
self.assertRaises(TypeError, tp)


class ExternalTests(unittest.TestCase):

Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_tcl.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -736,8 +736,11 @@ def check(value):
check('{\n')
check('}\n')

@support.cpython_only
def test_new_tcl_obj(self):
self.assertRaises(TypeError, _tkinter.Tcl_Obj)
self.assertRaises(TypeError, _tkinter.TkttType)
self.assertRaises(TypeError, _tkinter.TkappType)

class BigmemTclTest(unittest.TestCase):

Expand Down
7 changes: 7 additions & 0 deletions Lib/test/test_threading.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,13 @@ def func(): pass
thread = threading.Thread(target=func)
self.assertEqual(thread.name, "Thread-5 (func)")

@cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
lock = threading.Lock()
tp = type(lock)
self.assertRaises(TypeError, tp)

# Create a bunch of threads, let each do some work, wait until all are
# done.
def test_various_ops(self):
Expand Down
8 changes: 7 additions & 1 deletion Lib/test/test_unicodedata.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,8 @@
import sys
import unicodedata
import unittest
from test.support import open_urlresource, requires_resource, script_helper
from test.support import (open_urlresource, requires_resource, script_helper,
cpython_only)


class UnicodeMethodsTest(unittest.TestCase):
Expand DownExpand Up@@ -225,6 +226,11 @@ def test_east_asian_width_9_0_changes(self):

class UnicodeMiscTest(UnicodeDatabaseTest):

@cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
self.assertRaises(TypeError, unicodedata.UCD)

def test_failed_import_during_compiling(self):
# Issue 4367
# Decoding \N escapes requires the unicodedata module. If it can't be
Expand Down
8 changes: 8 additions & 0 deletions Lib/test/test_zlib.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -129,6 +129,14 @@ def test_overflow(self):
with self.assertRaisesRegex(OverflowError, 'int too large'):
zlib.decompressobj().flush(sys.maxsize + 1)

@support.cpython_only
def test_uninitialised_new(self):
# Prevent heap types from being created uninitialised (bpo-43916)
comp_type = type(zlib.compressobj())
decomp_type = type(zlib.decompressobj())
self.assertRaises(TypeError, comp_type)
self.assertRaises(TypeError, decomp_type)


class BaseCompressTestCase(object):
def check_big_compress_buffer(self, size, compress_func):
Expand Down
7 changes: 7 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
Fix regressions for extension modules that have been converted to :ref:`heap
types <heap-types>`. Inheritance differs for *static types* and *heap
types*: A *static type* with its :c:member:`~PyTypeObject.tp_new` set to
``NULL`` does not have a public contructor, but *heap types* inherit
contructors from their base classes, resulting in it being possible to
create instances without proper initialisation, possibly resulting in a
crash.
3 changes: 2 additions & 1 deletion Modules/_curses_panel.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@ static const char PyCursesVersion[] = "2.1"
#include "Python.h"

#include "py_curses.h"
#include "pycore_object.h" // _PyType_DisabledNew()

#include <panel.h>

Expand DownExpand Up@@ -514,6 +515,7 @@ static PyMethodDef PyCursesPanel_Methods[] ={
static PyType_Slot PyCursesPanel_Type_slots[] ={
{Py_tp_dealloc, PyCursesPanel_Dealloc},
{Py_tp_methods, PyCursesPanel_Methods},
{Py_tp_new, _PyType_DisabledNew},
{0, 0},
};

Expand DownExpand Up@@ -656,7 +658,6 @@ _curses_panel_exec(PyObject *mod)
if (state->PyCursesPanel_Type == NULL){
return -1;
}
((PyTypeObject *)state->PyCursesPanel_Type)->tp_new = NULL;

if (PyModule_AddType(mod, state->PyCursesPanel_Type) < 0){
return -1;
Expand Down
2 changes: 2 additions & 0 deletions Modules/_dbmmodule.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@

#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "pycore_object.h" // _PyType_DisabledNew()

#include <sys/types.h>
#include <sys/stat.h>
Expand DownExpand Up@@ -397,6 +398,7 @@ static PyMethodDef dbm_methods[] ={
};

static PyType_Slot dbmtype_spec_slots[] ={
{Py_tp_new, _PyType_DisabledNew},
{Py_tp_dealloc, dbm_dealloc},
{Py_tp_methods, dbm_methods},
{Py_sq_contains, dbm_contains},
Expand Down
2 changes: 2 additions & 0 deletions Modules/_functoolsmodule.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -540,6 +540,7 @@ static PyType_Slot keyobject_type_slots[] ={
{Py_tp_clear, keyobject_clear},
{Py_tp_richcompare, keyobject_richcompare},
{Py_tp_members, keyobject_members},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand DownExpand Up@@ -760,6 +761,7 @@ lru_list_elem_dealloc(lru_list_elem *link)

static PyType_Slot lru_list_elem_type_slots[] ={
{Py_tp_dealloc, lru_list_elem_dealloc},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand Down
2 changes: 2 additions & 0 deletions Modules/_gdbmmodule.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "pycore_object.h" // _PyType_DisabledNew()

#include <sys/types.h>
#include <sys/stat.h>
Expand DownExpand Up@@ -560,6 +561,7 @@ static PyType_Slot gdbmtype_spec_slots[] ={
{Py_mp_subscript, gdbm_subscript},
{Py_mp_ass_subscript, gdbm_ass_sub},
{Py_tp_doc, (char*)gdbm_object__doc__},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand Down
4 changes: 4 additions & 0 deletions Modules/_sre.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -43,6 +43,7 @@ static const char copyright[] =
#include "Python.h"
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "pycore_object.h" // _PyType_DisabledNew()
#include "structmember.h" // PyMemberDef

#include "sre.h"
Expand DownExpand Up@@ -2683,6 +2684,7 @@ static PyType_Slot pattern_slots[] ={
{Py_tp_methods, pattern_methods},
{Py_tp_members, pattern_members},
{Py_tp_getset, pattern_getset},
{Py_tp_new, _PyType_DisabledNew},
{0, NULL},
};

Expand DownExpand Up@@ -2740,6 +2742,7 @@ static PyType_Slot match_slots[] ={
{Py_tp_methods, match_methods},
{Py_tp_members, match_members},
{Py_tp_getset, match_getset},
{Py_tp_new, _PyType_DisabledNew},

/* As mapping.
*
Expand DownExpand Up@@ -2775,6 +2778,7 @@ static PyType_Slot scanner_slots[] ={
{Py_tp_dealloc, scanner_dealloc},
{Py_tp_methods, scanner_methods},
{Py_tp_members, scanner_members},
{Py_tp_new, _PyType_DisabledNew},
{0, NULL},
};

Expand Down
3 changes: 3 additions & 0 deletions Modules/_threadmodule.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@
#include "Python.h"
#include "pycore_interp.h" // _PyInterpreterState.num_threads
#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "pycore_object.h" // _PyType_DisabledNew()
#include "pycore_pylifecycle.h"
#include "pycore_pystate.h" // _PyThreadState_Init()
#include <stddef.h> // offsetof()
Expand DownExpand Up@@ -306,6 +307,7 @@ static PyType_Slot lock_type_slots[] ={
{Py_tp_methods, lock_methods},
{Py_tp_traverse, lock_traverse},
{Py_tp_members, lock_type_members},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand DownExpand Up@@ -677,6 +679,7 @@ static PyType_Slot local_dummy_type_slots[] ={
{Py_tp_dealloc, (destructor)localdummy_dealloc},
{Py_tp_doc, "Thread-local dummy"},
{Py_tp_members, local_dummy_type_members},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand Down
7 changes: 4 additions & 3 deletions Modules/_tkinter.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ Copyright (C) 1994 Steen Lumholt.
#define PY_SSIZE_T_CLEAN

#include "Python.h"
#include "pycore_object.h" // _PyType_DisabledNew()
#include <ctype.h>

#ifdef MS_WINDOWS
Expand DownExpand Up@@ -995,6 +996,7 @@ static PyType_Slot PyTclObject_Type_slots[] ={
{Py_tp_getattro, PyObject_GenericGetAttr},
{Py_tp_richcompare, PyTclObject_richcompare},
{Py_tp_getset, PyTclObject_getsetlist},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand DownExpand Up@@ -3287,6 +3289,7 @@ static PyType_Slot Tktt_Type_slots[] ={
{Py_tp_dealloc, Tktt_Dealloc},
{Py_tp_repr, Tktt_Repr},
{Py_tp_methods, Tktt_methods},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand DownExpand Up@@ -3341,6 +3344,7 @@ static PyMethodDef Tkapp_methods[] =
static PyType_Slot Tkapp_Type_slots[] ={
{Py_tp_dealloc, Tkapp_Dealloc},
{Py_tp_methods, Tkapp_methods},
{Py_tp_new, _PyType_DisabledNew},
{0, 0}
};

Expand DownExpand Up@@ -3537,7 +3541,6 @@ PyInit__tkinter(void)
Py_DECREF(m);
return NULL;
}
((PyTypeObject *)o)->tp_new = NULL;
if (PyModule_AddObject(m, "TkappType", o)){
Py_DECREF(o);
Py_DECREF(m);
Expand All@@ -3550,7 +3553,6 @@ PyInit__tkinter(void)
Py_DECREF(m);
return NULL;
}
((PyTypeObject *)o)->tp_new = NULL;
if (PyModule_AddObject(m, "TkttType", o)){
Py_DECREF(o);
Py_DECREF(m);
Expand All@@ -3563,7 +3565,6 @@ PyInit__tkinter(void)
Py_DECREF(m);
return NULL;
}
((PyTypeObject *)o)->tp_new = NULL;
if (PyModule_AddObject(m, "Tcl_Obj", o)){
Py_DECREF(o);
Py_DECREF(m);
Expand Down
4 changes: 4 additions & 0 deletions Modules/_winapi.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,9 @@
#include <crtdbg.h>
#include "winreparse.h"

// Include pycore_object.h to avoid conflict with windows.h
#include "pycore_object.h" // _PyType_DisabledNew()

#if defined(MS_WIN32) && !defined(MS_WIN64)
#define HANDLE_TO_PYNUM(handle) \
PyLong_FromUnsignedLong((unsigned long) handle)
Expand DownExpand Up@@ -325,6 +328,7 @@ static PyType_Slot winapi_overlapped_type_slots[] ={
{Py_tp_doc, "OVERLAPPED structure wrapper"},
{Py_tp_methods, overlapped_methods},
{Py_tp_members, overlapped_members},
{Py_tp_new, _PyType_DisabledNew},
{0,0}
};

Expand Down
2 changes: 2 additions & 0 deletions Modules/arraymodule.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,7 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "pycore_object.h" // _PyType_DisabledNew()
#include "structmember.h" // PyMemberDef
#include <stddef.h> // offsetof()

Expand DownExpand Up@@ -2980,6 +2981,7 @@ static PyType_Slot arrayiter_slots[] ={
{Py_tp_iter, PyObject_SelfIter},
{Py_tp_iternext, arrayiter_next},
{Py_tp_methods, arrayiter_methods},
{Py_tp_new, _PyType_DisabledNew},
{0, NULL},
};

Expand Down
Loading