bpo-34722: Consistent serialization of sets in bytecode

Lib/test/test_compileall.py

@@ -555,6 +555,30 @@ def test_workers_available_cores(self, compile_dir):
  self.assertTrue(compile_dir.called)
  self.assertEqual(compile_dir.call_args[-1]['workers'], None)
 
+ def test_deterministic_serialization(self):
+ # http://bugs.python.org/issue34722
+ # We have to test this in subprocesses since the nondeterminism
+ # revolves around the hash seed.
+ with open(os.path.join(self.pkgdir, 'set.py'), 'w') as f:
+ f.write("def test(x):\n")
+ f.write(" if x in{'ONE', 'TWO', 'THREE'}:\n")
+ f.write(" pass\n")
+
+ def compile_dir():
+ self.assertRunOK(
+ '-q',
+ '--invalidation-mode=unchecked-hash',
+ self.pkgdir,
+ )
+ for file in sorted(os.listdir(self.pkgdir_cachedir)):
+ with open(os.path.join(self.pkgdir_cachedir, file), 'rb') as f:
+ yield f.read()
+
+ serialized = list(compile_dir())
+ for _ in range(5):
+ shutil.rmtree(self.pkgdir_cachedir)
+ self.assertEqual(serialized, list(compile_dir()))
+
 
 if __name__ == "__main__":
  unittest.main()

Misc/NEWS.d/next/Core and Builtins/2018-09-21-15-38-47.bpo-34722.F6VE5R.rst

@@ -0,0 +1,4 @@
+Ensures that sets / frozensets marshal in a consistent order by sorting the
+serialised items before writing them. This will result in somewhat slower
+serialisation but bytecode files using set literals in certain contexts will
+now serialise deterministically where it didn't before.

Python/marshal.c

@@ -496,7 +496,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
  w_object((PyObject *)NULL, p);
  }
  else if (PyAnySet_CheckExact(v)){
- PyObject *value, *it;
+ PyObject *value, *l;
 
  if (PyObject_TypeCheck(v, &PySet_Type))
  W_TYPE(TYPE_SET, p);
@@ -509,17 +509,27 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
  return;
  }
  W_SIZE(n, p);
-it = PyObject_GetIter(v);
- if (it == NULL){
+l = PySequence_List(v);
+ if (l == NULL){
  p->depth--;
  p->error = WFERR_UNMARSHALLABLE;
  return;
  }
- while ((value = PyIter_Next(it)) != NULL){
- w_object(value, p);
- Py_DECREF(value);
+ for (i = 0; i < n; i++){
+ value = PyList_GetItem(l, i);
+ value = PyMarshal_WriteObjectToString(value, Py_MARSHAL_VERSION);
+ PyList_SetItem(l, i, value);
+ }
+ if (PyList_Sort(l) == -1){
+ Py_DECREF(l);
+ p->depth--;
+ p->error = WFERR_UNMARSHALLABLE;
+ return;
+ }
+ for (i = 0; i < n; i++){
+ w_object(PyList_GetItem(l, i), p);
  }
- Py_DECREF(it);
+ Py_DECREF(l);
  if (PyErr_Occurred()){
  p->depth--;
  p->error = WFERR_UNMARSHALLABLE;