From 8e55e4fbc41ecdc93e382e3fb7374aaf49cb755c Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:24:38 +0100 Subject: [PATCH 01/56] Implements Nullable support --- src/runtime/converter.cs | 8 +++++++- src/runtime/methodbinder.cs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 13498e3dc..4251c80fa 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -436,6 +436,12 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + return ToPrimitive(value, obType, out result, setError); } @@ -845,7 +851,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s int size = Runtime.PySequence_Size(value); result = null; - if (size < 0) + if (size < 0 || elementType.IsGenericType) { if (setError) { diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index eeec8b89d..74c069a09 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -394,8 +394,14 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } if (!typematch) { + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(pi[n].ParameterType); + if (underlyingType == null) + { + underlyingType = pi[n].ParameterType; + } // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); + TypeCode argtypecode = Type.GetTypeCode(underlyingType); TypeCode paramtypecode = Type.GetTypeCode(clrtype); if (argtypecode == paramtypecode) { From c6d56bd38cf8636033a6315ad450c4d6bb987629 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:28:39 +0100 Subject: [PATCH 02/56] Implements Implicit Conversion --- src/runtime/converter.cs | 14 ++++++++++++++ src/runtime/methodbinder.cs | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4251c80fa..4f29019ba 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -442,6 +442,20 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToManagedValue(value, underlyingType, out result, setError); } + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] { result.GetType() }); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] { result }); + } + return opImplicit != null; + } + } + return ToPrimitive(value, obType, out result, setError); } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 74c069a09..e956bce55 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -408,6 +408,13 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // this takes care of implicit conversions + var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + typematch = opImplicit.ReturnType == pi[n].ParameterType; + clrtype = pi[n].ParameterType; + } } Runtime.XDecref(pyoptype); if (!typematch) From 4e4e9809d542ce1f4ad51da45227d3a9ef5a0d75 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:53:48 +0100 Subject: [PATCH 03/56] Adds method name to "no method matches" error --- src/runtime/methodbinder.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index e956bce55..95fc0b026 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -186,6 +186,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +207,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) From 6dad4229c155d4e89a8ddd24b055d686a6faacc9 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:01:37 +0100 Subject: [PATCH 04/56] Implements System.Decimal support Convertes System.Decimal to decimal.decimal and vice-versa --- src/runtime/converter.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4f29019ba..7ca4aa83e 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -30,6 +30,7 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr decimalCtor; static Converter() { @@ -45,6 +46,9 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); + if (decimalMod == null) throw new PythonException(); } @@ -100,6 +104,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyFloatType; + return IntPtr.Zero; } @@ -229,6 +236,14 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + string d2s = ((decimal)value).ToString(nfi); + IntPtr d2p = Runtime.PyString_FromString(d2s); + IntPtr decimalArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); + + return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + default: if (value is IEnumerable) { @@ -821,6 +836,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; } From 570e0f8a92308e50ecc42d9aa9a5c1b45e0e5ddf Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:51:09 +0100 Subject: [PATCH 05/56] Fixes decimal support --- src/runtime/converter.cs | 5 ++++- src/runtime/methodbinder.cs | 6 ++++++ src/runtime/runtime.cs | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 7ca4aa83e..351a1c83a 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -75,6 +75,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -105,7 +108,7 @@ internal static IntPtr GetPythonTypeByAlias(Type op) return Runtime.PyBoolType; if (op == decimalType) - return Runtime.PyFloatType; + return Runtime.PyDecimalType; return IntPtr.Zero; } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 95fc0b026..48dc1eb9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -420,6 +420,12 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + clrtype = pi[n].ParameterType; + typematch = Converter.ToManaged(op, clrtype, out arg, false); + } // this takes care of implicit conversions var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); if (opImplicit != null) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b08a56622..5ef06f3cc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -296,6 +296,14 @@ internal static void Initialize() PyFloatType = PyObject_Type(op); XDecref(op); + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + PyDecimalType = PyObject_Type(op); + XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); + #if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; @@ -389,6 +397,7 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; #if PYTHON3 internal static IntPtr PyBytesType; From c355ab41554723b64433fcd199dea669dad40c85 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:17:41 +0100 Subject: [PATCH 06/56] Implements System.DateTime support Converts System.DateTime and System.TimeSpan to datetime.datetime and datetime.timedelta and vice-versa --- src/runtime/converter.cs | 102 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 351a1c83a..c955a880f 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -31,6 +31,9 @@ private Converter() private static Type boolType; private static Type typeType; private static IntPtr decimalCtor; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -49,6 +52,34 @@ static Converter() IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); if (decimalMod == null) throw new PythonException(); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); + if (decimalCtor == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", + "from datetime import timedelta, tzinfo\n" + + "class GMT(tzinfo):\n" + + " def __init__(self, hours, minutes):\n" + + " self.hours = hours\n" + + " self.minutes = minutes\n" + + " def utcoffset(self, dt):\n" + + " return timedelta(hours=self.hours, minutes=self.minutes)\n" + + " def tzname(self, dt):\n" + + " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + + " def dst (self, dt):\n" + + " return timedelta(0)\n").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -187,6 +218,14 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -247,6 +286,21 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + case TypeCode.DateTime: + var datetime = (DateTime)value; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + default: if (value is IEnumerable) { @@ -268,6 +322,16 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + } + /// /// In a few situations, we don't have any advisory type information @@ -490,6 +554,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -851,6 +941,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = m; return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = dt; + return true; } From d2d12b3f5efb49625821c7a5922ef20ab6a277b3 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Fri, 13 Jul 2018 13:57:20 +0100 Subject: [PATCH 07/56] Fixes UTC conversion from python datetime to managed DateTime --- src/runtime/converter.cs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index c955a880f..481ee73c2 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -65,18 +65,18 @@ static Converter() timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); - IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", - "from datetime import timedelta, tzinfo\n" + - "class GMT(tzinfo):\n" + - " def __init__(self, hours, minutes):\n" + - " self.hours = hours\n" + - " self.minutes = minutes\n" + - " def utcoffset(self, dt):\n" + - " return timedelta(hours=self.hours, minutes=self.minutes)\n" + - " def tzname(self, dt):\n" + - " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + - " def dst (self, dt):\n" + - " return timedelta(0)\n").Handle; + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); if (tzInfoCtor == null) throw new PythonException(); @@ -951,7 +951,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo goto type_error; } Runtime.XDecref(op); - result = dt; + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; return true; } From c6db86653e7fa4fa89e1f8404d72e346b67ed857 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 20 Jul 2018 15:58:10 -0300 Subject: [PATCH 08/56] Fixing memory leaks in the conversion of some types --- src/runtime/converter.cs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 481ee73c2..ad8a8af02 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,8 +1,6 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; @@ -55,13 +53,13 @@ static Converter() IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); if (decimalCtor == null) throw new PythonException(); - + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); - + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); @@ -224,7 +222,10 @@ internal static IntPtr ToPython(object value, Type type) IntPtr timeSpanArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); - return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; } return CLRObject.GetInstHandle(value, type); @@ -283,12 +284,14 @@ internal static IntPtr ToPython(object value, Type type) IntPtr d2p = Runtime.PyString_FromString(d2s); IntPtr decimalArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - - return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + // clean up + Runtime.XDecref(decimalArgs); + return returnDecimal; case TypeCode.DateTime: var datetime = (DateTime)value; - + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); @@ -298,8 +301,10 @@ internal static IntPtr ToPython(object value, Type type) Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); - - return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; default: if (value is IEnumerable) @@ -329,7 +334,9 @@ private static IntPtr TzInfo(DateTimeKind kind) IntPtr tzInfoArgs = Runtime.PyTuple_New(2); Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); - return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; } From bec9563d2958acc5adf3e8972b23609fe3914402 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 23 Aug 2018 23:27:02 +0100 Subject: [PATCH 09/56] Do not include timezone if DateTimeKind is Unspecified DECREF'ing datetime timezone argument when DateTimeKind is Unspecified was causing `Fatal Python error: deallocating None` because the object was set to `Runtime.PyNone`. Fixed the input to datetime constructor as we were passing milliseconds, where it should be microseconds. --- src/runtime/converter.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index ad8a8af02..a33ade002 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -292,15 +292,27 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.DateTime: var datetime = (DateTime)value; - IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); - Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); - Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); // clean up Runtime.XDecref(dateTimeArgs); From 76695b4b4ca474214fb6a4ab8a0237c7607997d4 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 27 Aug 2018 18:05:38 +0100 Subject: [PATCH 10/56] Sets the version to 1.0.5.12 to match QuantConnect's nuget one. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac1d31324..8a1c05306 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0.dev0 +current_version = 1.0.5.12 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3641185bb..cf9bb4749 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "2.4.0.dev0" + version: "1.0.5.12" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 4ec2a2113..d35041876 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0.dev0", + version="1.0.5.12", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..4a3b83782 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("1.0.5.12")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..c5b789a18 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("1.0.5.12"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..7217f98d1 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "1.0.5.12" class clrproperty(object): From 7e655cebcc493813ab69940ec21bab73327845db Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 14 Jan 2019 12:18:30 -0300 Subject: [PATCH 11/56] C# decimal conversion - C# decimal conversion will use C# double and python float due to the big performance impact of converting C# decimal to python decimal. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/runtime/converter.cs | 18 +++--------------- src/runtime/resources/clr.py | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8a1c05306..2518f77fb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.12 +current_version = 1.0.5.13 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index cf9bb4749..3aa2d1a62 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.12" + version: "1.0.5.13" build: skip: True # [not win] diff --git a/setup.py b/setup.py index d35041876..3695e33fa 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.12", + version="1.0.5.13", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 4a3b83782..e9d39ba33 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.12")] +[assembly: AssemblyVersion("1.0.5.13")] diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index a33ade002..d8ec3fcd2 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -28,7 +28,6 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; - private static IntPtr decimalCtor; private static IntPtr dateTimeCtor; private static IntPtr timeSpanCtor; private static IntPtr tzInfoCtor; @@ -48,15 +47,9 @@ static Converter() boolType = typeof(Boolean); typeType = typeof(Type); - IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); - if (decimalMod == null) throw new PythonException(); - IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); - if (decimalCtor == null) throw new PythonException(); - dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); @@ -280,14 +273,9 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyLong_FromUnsignedLongLong((ulong)value); case TypeCode.Decimal: - string d2s = ((decimal)value).ToString(nfi); - IntPtr d2p = Runtime.PyString_FromString(d2s); - IntPtr decimalArgs = Runtime.PyTuple_New(1); - Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); - // clean up - Runtime.XDecref(decimalArgs); - return returnDecimal; + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); case TypeCode.DateTime: var datetime = (DateTime)value; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7217f98d1..7caff08f4 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.12" +__version__ = "1.0.5.13" class clrproperty(object): From 3511f63cc69134181808947f10fb68326aecbf36 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:24:38 +0100 Subject: [PATCH 12/56] Implements Nullable support --- src/runtime/converter.cs | 8 +++++++- src/runtime/methodbinder.cs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..ec367015b 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -437,6 +437,12 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + return ToPrimitive(value, obType, out result, setError); } @@ -846,7 +852,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s var size = Runtime.PySequence_Size(value); result = null; - if (size < 0) + if (size < 0 || elementType.IsGenericType) { if (setError) { diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5e800c36f..7d24cde9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -394,8 +394,14 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } if (!typematch) { + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(pi[n].ParameterType); + if (underlyingType == null) + { + underlyingType = pi[n].ParameterType; + } // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); + TypeCode argtypecode = Type.GetTypeCode(underlyingType); TypeCode paramtypecode = Type.GetTypeCode(clrtype); if (argtypecode == paramtypecode) { From d24b7eec40b1c136a5dc37561ede1e7e7c8836ef Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:28:39 +0100 Subject: [PATCH 13/56] Implements Implicit Conversion --- src/runtime/converter.cs | 14 ++++++++++++++ src/runtime/methodbinder.cs | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index ec367015b..35a6dc1b8 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -443,6 +443,20 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToManagedValue(value, underlyingType, out result, setError); } + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] { result.GetType() }); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] { result }); + } + return opImplicit != null; + } + } + return ToPrimitive(value, obType, out result, setError); } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 7d24cde9a..0c5433076 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -408,6 +408,13 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // this takes care of implicit conversions + var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + typematch = opImplicit.ReturnType == pi[n].ParameterType; + clrtype = pi[n].ParameterType; + } } Runtime.XDecref(pyoptype); if (!typematch) From 2a6c1aa06fcf3519bfb0b0ebcc0f077802386d3d Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:53:48 +0100 Subject: [PATCH 14/56] Adds method name to "no method matches" error --- src/runtime/methodbinder.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 0c5433076..c33015725 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -186,6 +186,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +207,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) From a6f2d561db67b0c0291a61767788a228fb148090 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:01:37 +0100 Subject: [PATCH 15/56] Implements System.Decimal support Convertes System.Decimal to decimal.decimal and vice-versa --- src/runtime/converter.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 35a6dc1b8..77657e00a 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -31,6 +31,7 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr decimalCtor; static Converter() { @@ -46,6 +47,9 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); + if (decimalMod == null) throw new PythonException(); } @@ -101,6 +105,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyFloatType; + return IntPtr.Zero; } @@ -230,6 +237,14 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + string d2s = ((decimal)value).ToString(nfi); + IntPtr d2p = Runtime.PyString_FromString(d2s); + IntPtr decimalArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); + + return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + default: if (value is IEnumerable) { @@ -822,6 +837,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; } From a7b169dd76ec61a86ef36db4864db47773325e74 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:51:09 +0100 Subject: [PATCH 16/56] Fixes decimal support --- src/runtime/converter.cs | 5 ++++- src/runtime/methodbinder.cs | 6 ++++++ src/runtime/runtime.cs | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 77657e00a..6f16ee4b7 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -76,6 +76,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -106,7 +109,7 @@ internal static IntPtr GetPythonTypeByAlias(Type op) return Runtime.PyBoolType; if (op == decimalType) - return Runtime.PyFloatType; + return Runtime.PyDecimalType; return IntPtr.Zero; } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index c33015725..b608b6c55 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -420,6 +420,12 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + clrtype = pi[n].ParameterType; + typematch = Converter.ToManaged(op, clrtype, out arg, false); + } // this takes care of implicit conversions var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); if (opImplicit != null) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 863eb4034..21a1d20fd 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -368,6 +368,14 @@ internal static void Initialize(bool initSigs = false) PyFloatType = PyObject_Type(op); XDecref(op); + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + PyDecimalType = PyObject_Type(op); + XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); + #if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; @@ -512,6 +520,7 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; #if PYTHON3 internal static IntPtr PyBytesType; From ba5c2f660e90ee256acaa72664755e060448884e Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:17:41 +0100 Subject: [PATCH 17/56] Implements System.DateTime support Converts System.DateTime and System.TimeSpan to datetime.datetime and datetime.timedelta and vice-versa --- src/runtime/converter.cs | 102 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 6f16ee4b7..52e06498e 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -32,6 +32,9 @@ private Converter() private static Type boolType; private static Type typeType; private static IntPtr decimalCtor; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -50,6 +53,34 @@ static Converter() IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); if (decimalMod == null) throw new PythonException(); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); + if (decimalCtor == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", + "from datetime import timedelta, tzinfo\n" + + "class GMT(tzinfo):\n" + + " def __init__(self, hours, minutes):\n" + + " self.hours = hours\n" + + " self.minutes = minutes\n" + + " def utcoffset(self, dt):\n" + + " return timedelta(hours=self.hours, minutes=self.minutes)\n" + + " def tzname(self, dt):\n" + + " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + + " def dst (self, dt):\n" + + " return timedelta(0)\n").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -188,6 +219,14 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -248,6 +287,21 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + case TypeCode.DateTime: + var datetime = (DateTime)value; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + default: if (value is IEnumerable) { @@ -269,6 +323,16 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + } + /// /// In a few situations, we don't have any advisory type information @@ -491,6 +555,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -852,6 +942,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = m; return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = dt; + return true; } From 15e4f9f1144513e4f176d65dc18dea4020d49db5 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Fri, 13 Jul 2018 13:57:20 +0100 Subject: [PATCH 18/56] Fixes UTC conversion from python datetime to managed DateTime --- src/runtime/converter.cs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 52e06498e..4063572e5 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -66,18 +66,18 @@ static Converter() timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); - IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", - "from datetime import timedelta, tzinfo\n" + - "class GMT(tzinfo):\n" + - " def __init__(self, hours, minutes):\n" + - " self.hours = hours\n" + - " self.minutes = minutes\n" + - " def utcoffset(self, dt):\n" + - " return timedelta(hours=self.hours, minutes=self.minutes)\n" + - " def tzname(self, dt):\n" + - " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + - " def dst (self, dt):\n" + - " return timedelta(0)\n").Handle; + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); if (tzInfoCtor == null) throw new PythonException(); @@ -952,7 +952,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo goto type_error; } Runtime.XDecref(op); - result = dt; + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; return true; } From 349bcd710c07a932296b5115b147152e9279e2c7 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 20 Jul 2018 15:58:10 -0300 Subject: [PATCH 19/56] Fixing memory leaks in the conversion of some types --- src/runtime/converter.cs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4063572e5..fa5bf729b 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,8 +1,6 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; using System.ComponentModel; @@ -56,13 +54,13 @@ static Converter() IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); if (decimalCtor == null) throw new PythonException(); - + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); - + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); @@ -225,7 +223,10 @@ internal static IntPtr ToPython(object value, Type type) IntPtr timeSpanArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); - return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; } return CLRObject.GetInstHandle(value, type); @@ -284,12 +285,14 @@ internal static IntPtr ToPython(object value, Type type) IntPtr d2p = Runtime.PyString_FromString(d2s); IntPtr decimalArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - - return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + // clean up + Runtime.XDecref(decimalArgs); + return returnDecimal; case TypeCode.DateTime: var datetime = (DateTime)value; - + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); @@ -299,8 +302,10 @@ internal static IntPtr ToPython(object value, Type type) Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); - - return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; default: if (value is IEnumerable) @@ -330,7 +335,9 @@ private static IntPtr TzInfo(DateTimeKind kind) IntPtr tzInfoArgs = Runtime.PyTuple_New(2); Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); - return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; } From ad8a585387c95248fb3098aa611042adbb797143 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 23 Aug 2018 23:27:02 +0100 Subject: [PATCH 20/56] Do not include timezone if DateTimeKind is Unspecified DECREF'ing datetime timezone argument when DateTimeKind is Unspecified was causing `Fatal Python error: deallocating None` because the object was set to `Runtime.PyNone`. Fixed the input to datetime constructor as we were passing milliseconds, where it should be microseconds. --- src/runtime/converter.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index fa5bf729b..97450be77 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -293,15 +293,27 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.DateTime: var datetime = (DateTime)value; - IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); - Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); - Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); // clean up Runtime.XDecref(dateTimeArgs); From c5019a3d093c3aef1d85a2fc04ee0aac7b34b101 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 27 Aug 2018 18:05:38 +0100 Subject: [PATCH 21/56] Sets the version to 1.0.5.12 to match QuantConnect's nuget one. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac1d31324..8a1c05306 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0.dev0 +current_version = 1.0.5.12 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3641185bb..cf9bb4749 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "2.4.0.dev0" + version: "1.0.5.12" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 1b6f07ea6..309ece49a 100644 --- a/setup.py +++ b/setup.py @@ -492,7 +492,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0.dev0", + version="1.0.5.12", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..4a3b83782 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("1.0.5.12")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..c5b789a18 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("1.0.5.12"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..7217f98d1 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "1.0.5.12" class clrproperty(object): From 1f7d30854485dad3b5ae59fc7db17cb836e7923f Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 14 Jan 2019 12:18:30 -0300 Subject: [PATCH 22/56] C# decimal conversion - C# decimal conversion will use C# double and python float due to the big performance impact of converting C# decimal to python decimal. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/runtime/converter.cs | 18 +++--------------- src/runtime/resources/clr.py | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8a1c05306..2518f77fb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.12 +current_version = 1.0.5.13 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index cf9bb4749..3aa2d1a62 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.12" + version: "1.0.5.13" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 309ece49a..10f1b178b 100644 --- a/setup.py +++ b/setup.py @@ -492,7 +492,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.12", + version="1.0.5.13", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 4a3b83782..e9d39ba33 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.12")] +[assembly: AssemblyVersion("1.0.5.13")] diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 97450be77..629eab4df 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -29,7 +29,6 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; - private static IntPtr decimalCtor; private static IntPtr dateTimeCtor; private static IntPtr timeSpanCtor; private static IntPtr tzInfoCtor; @@ -49,15 +48,9 @@ static Converter() boolType = typeof(Boolean); typeType = typeof(Type); - IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); - if (decimalMod == null) throw new PythonException(); - IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); - if (decimalCtor == null) throw new PythonException(); - dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); @@ -281,14 +274,9 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyLong_FromUnsignedLongLong((ulong)value); case TypeCode.Decimal: - string d2s = ((decimal)value).ToString(nfi); - IntPtr d2p = Runtime.PyString_FromString(d2s); - IntPtr decimalArgs = Runtime.PyTuple_New(1); - Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); - // clean up - Runtime.XDecref(decimalArgs); - return returnDecimal; + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); case TypeCode.DateTime: var datetime = (DateTime)value; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7217f98d1..7caff08f4 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.12" +__version__ = "1.0.5.13" class clrproperty(object): From 54f284bac406152290955f4e8af9cd8a3a5d3c11 Mon Sep 17 00:00:00 2001 From: Alexandre Catarino Date: Fri, 18 Jan 2019 21:17:27 +0000 Subject: [PATCH 23/56] Revert "Rebase to pythonnet master" --- .travis.yml | 39 +- AUTHORS.md | 5 - CHANGELOG.md | 20 - NuGet.config | 2 +- appveyor.yml | 9 - setup.py | 17 +- src/embed_tests/GlobalTestsSetup.cs | 21 - .../Python.EmbeddingTest.15.csproj | 1 - src/embed_tests/Python.EmbeddingTest.csproj | 7 +- src/embed_tests/TestConverter.cs | 2 +- src/embed_tests/TestDomainReload.cs | 239 ---------- src/embed_tests/TestNamedArguments.cs | 2 +- src/embed_tests/TestPyObject.cs | 2 +- src/embed_tests/TestPySequence.cs | 6 +- src/embed_tests/TestPyWith.cs | 2 +- src/embed_tests/TestPythonEngineProperties.cs | 39 +- src/embed_tests/TestRuntime.cs | 34 +- src/embed_tests/TestTypeManager.cs | 65 --- src/embed_tests/dynamic.cs | 17 +- src/embed_tests/pyinitialize.cs | 60 --- src/runtime/Python.Runtime.15.csproj | 4 +- src/runtime/Python.Runtime.csproj | 13 +- src/runtime/Util.cs | 2 +- src/runtime/arrayobject.cs | 4 +- src/runtime/assemblymanager.cs | 56 +-- src/runtime/classbase.cs | 18 + src/runtime/classderived.cs | 9 +- src/runtime/classmanager.cs | 5 - src/runtime/classobject.cs | 6 +- src/runtime/clrobject.cs | 2 +- src/runtime/converter.cs | 7 +- src/runtime/debughelper.cs | 6 +- src/runtime/exceptions.cs | 7 +- src/runtime/extensiontype.cs | 21 + src/runtime/genericutil.cs | 6 - src/runtime/importhook.cs | 41 +- src/runtime/indexer.cs | 6 +- src/runtime/interfaceobject.cs | 2 +- src/runtime/interop37.cs | 149 ------- src/runtime/metatype.cs | 2 +- src/runtime/methodbinder.cs | 23 +- src/runtime/methodbinding.cs | 4 +- src/runtime/moduleobject.cs | 22 +- src/runtime/pyobject.cs | 4 +- src/runtime/pyscope.cs | 7 +- src/runtime/pythonengine.cs | 102 +---- src/runtime/runtime.cs | 409 +++--------------- src/runtime/typemanager.cs | 273 +----------- src/testing/InheritanceTest.cs | 14 - src/testing/Python.Test.csproj | 3 +- src/testing/conversiontest.cs | 15 - src/tests/importtest.py | 13 - src/tests/test_class.py | 11 - src/tests/test_conversion.py | 13 +- src/tests/test_exceptions.py | 3 +- src/tests/test_import.py | 11 +- src/tests/test_subclass.py | 33 -- tools/geninterop/fake_libc_include/crypt.h | 1 - tools/geninterop/geninterop.py | 5 +- 59 files changed, 220 insertions(+), 1701 deletions(-) delete mode 100644 src/embed_tests/GlobalTestsSetup.cs delete mode 100644 src/embed_tests/TestDomainReload.cs delete mode 100644 src/embed_tests/TestTypeManager.cs delete mode 100644 src/runtime/interop37.cs delete mode 100644 src/testing/InheritanceTest.cs delete mode 100644 src/tests/importtest.py delete mode 100644 tools/geninterop/fake_libc_include/crypt.h diff --git a/.travis.yml b/.travis.yml index d059bdcde..900e207a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ matrix: - dotnet-hostfxr-2.0.0 - dotnet-runtime-2.0.0 - dotnet-sdk-2.0.0 - - python: 3.4 env: *xplat-env addons: *xplat-addons @@ -34,24 +33,9 @@ matrix: - python: 3.6 env: *xplat-env addons: *xplat-addons - - - python: 3.7 + - python: "3.7-dev" env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 + addons: *xplat-addons # --------------------- Classic builds ------------------------ - python: 2.7 @@ -68,18 +52,15 @@ matrix: - python: 3.6 env: *classic-env - - python: 3.7 + - python: "3.7-dev" + env: *classic-env + + allow_failures: + - python: "3.7-dev" + env: *xplat-env + + - python: "3.7-dev" env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono env: global: diff --git a/AUTHORS.md b/AUTHORS.md index fe2d2b172..3c39794e4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -14,7 +14,6 @@ - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) -- Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) @@ -23,12 +22,10 @@ - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) - Dave Hirschfeld ([@dhirschfeld](https://github.com/dhirschfeld)) -- David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - He-chien Tsai ([@t3476](https://github.com/t3476)) -   Ivan Cronyn ([@cronan](https://github.com/cronan)) -- Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) -   Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) @@ -42,8 +39,6 @@ - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) -- Simon Mourier ([@smourier](https://github.com/smourier)) -- Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3816cd0c..3dc668b0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,35 +8,23 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ## [unreleased][] ### Added - - Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) - Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). Currently there two side-by-side build systems that produces the same output (net40) from the same sources. After a some transition time, current (mono/ msbuild 14.0) build system will be removed. - NUnit upgraded to 3.7 (eliminates travis-ci random bug) -- Added C# `PythonEngine.AddShutdownHandler` to help client code clean up on shutdown. - Added `clr.GetClrType` ([#432][i432])([#433][p433]) - Allowed passing `None` for nullable args ([#460][p460]) - Added keyword arguments based on C# syntax for calling CPython methods ([#461][p461]) - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) -- Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) -- Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) ### Changed -- Reattach python exception traceback information (#545) -- PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - ### Fixed -- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. - This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. ([#534][p534]) - Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py -- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), - related to unloading the Application Domain -- Fixed interop methods with Py_ssize_t. NetCoreApp 2.0 is more sensitive than net40 and requires this fix. ([#531][p531]) - Fixed crash on exit of the Python interpreter if a python class derived from a .NET class has a `__namespace__` or `__assembly__` attribute ([#481][i481]) @@ -48,8 +36,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) - Fix memory leaks due to spurious handle incrementation ([#691][i691]) -- Fix spurious assembly loading exceptions from private types ([#703][i703]) -- Fix inheritance of non-abstract base methods ([#755][i755]) ## [2.3.0][] - 2017-03-11 @@ -607,7 +593,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [1.0.0]: https://github.com/pythonnet/pythonnet/releases/tag/1.0 -[i714]: https://github.com/pythonnet/pythonnet/issues/714 [i608]: https://github.com/pythonnet/pythonnet/issues/608 [i443]: https://github.com/pythonnet/pythonnet/issues/443 [p690]: https://github.com/pythonnet/pythonnet/pull/690 @@ -698,9 +683,4 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [p225]: https://github.com/pythonnet/pythonnet/pull/225 [p78]: https://github.com/pythonnet/pythonnet/pull/78 [p163]: https://github.com/pythonnet/pythonnet/pull/163 -[p625]: https://github.com/pythonnet/pythonnet/pull/625 [i131]: https://github.com/pythonnet/pythonnet/issues/131 -[p531]: https://github.com/pythonnet/pythonnet/pull/531 -[i755]: https://github.com/pythonnet/pythonnet/pull/755 -[p534]: https://github.com/pythonnet/pythonnet/pull/534 -[i449]: https://github.com/pythonnet/pythonnet/issues/449 diff --git a/NuGet.config b/NuGet.config index 5210cd6c9..719fbc83c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,4 +1,4 @@ - + diff --git a/appveyor.yml b/appveyor.yml index ce345cbf8..6bebef490 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,19 +23,10 @@ environment: BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.7 - BUILD_OPTS: --xplat - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.4 - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.6 - - PYTHON_VERSION: 3.7 - -matrix: - allow_failures: - - PYTHON_VERSION: 3.4 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 init: # Update Environment Variables based on matrix/platform diff --git a/setup.py b/setup.py index 10f1b178b..3695e33fa 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug -VERBOSITY = "normal" # quiet, minimal, normal, detailed, diagnostic +VERBOSITY = "minimal" # quiet, minimal, normal, detailed, diagnostic is_64bits = sys.maxsize > 2**32 DEVTOOLS = "MsDev" if sys.platform == "win32" else "Mono" @@ -329,16 +329,9 @@ def _install_packages(self): self.debug_print("Updating NuGet: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) - try: - # msbuild=14 is mainly for Mono issues - cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) - except: - # when only VS 2017 is installed do not specify msbuild version - cmd = "{0} restore pythonnet.sln -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) + cmd = "{0} restore pythonnet.sln -o packages".format(nuget) + self.debug_print("Installing packages: {0}".format(cmd)) + subprocess.check_call(cmd, shell=use_shell) def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): """Return full path to one of the Microsoft build tools""" @@ -523,10 +516,10 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX :: Linux', 'Operating System :: MacOS :: MacOS X', diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs deleted file mode 100644 index 458ab6a99..000000000 --- a/src/embed_tests/GlobalTestsSetup.cs +++ /dev/null @@ -1,21 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; - -namespace Python.EmbeddingTest -{ - - // As the SetUpFixture, the OneTimeTearDown of this class is executed after - // all tests have run. - [SetUpFixture] - public class GlobalTestsSetup - { - [OneTimeTearDown] - public void FinalCleanup() - { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - } -} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..92d55a7e0 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -29,7 +29,6 @@ XPLAT $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP - $(DefineConstants);NETSTANDARD $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 6aa48becc..66e8c7165 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,4 +1,4 @@ - + Debug @@ -86,7 +86,6 @@ - @@ -104,8 +103,6 @@ - - @@ -125,4 +122,4 @@ - + \ No newline at end of file diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index caaec311b..346c8afdc 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using Python.Runtime; namespace Python.EmbeddingTest diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs deleted file mode 100644 index b162d4eb0..000000000 --- a/src/embed_tests/TestDomainReload.cs +++ /dev/null @@ -1,239 +0,0 @@ -using System; -using System.CodeDom.Compiler; -using System.Reflection; -using NUnit.Framework; -using Python.Runtime; - -// -// This test case is disabled on .NET Standard because it doesn't have all the -// APIs we use. We could work around that, but .NET Core doesn't implement -// domain creation, so it's not worth it. -// -// Unfortunately this means no continuous integration testing for this case. -// -#if !NETSTANDARD && !NETCOREAPP -namespace Python.EmbeddingTest -{ - class TestDomainReload - { - /// - /// Test that the python runtime can survive a C# domain reload without crashing. - /// - /// At the time this test was written, there was a very annoying - /// seemingly random crash bug when integrating pythonnet into Unity. - /// - /// The repro steps that David Lassonde, Viktoria Kovecses and - /// Benoit Hudson eventually worked out: - /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access - /// some C# data from python: C# calls python, which calls C#. - /// 2. Execute the script (e.g. make it a MenuItem and click it). - /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. - /// 4. Wait several seconds for Unity to be done recompiling and - /// reloading the C# domain. - /// 5. Make python run the gc (e.g. by calling gc.collect()). - /// - /// The reason: - /// A. In step 2, Python.Runtime registers a bunch of new types with - /// their tp_traverse slot pointing to managed code, and allocates - /// some objects of those types. - /// B. In step 4, Unity unloads the C# domain. That frees the managed - /// code. But at the time of the crash investigation, pythonnet - /// leaked the python side of the objects allocated in step 1. - /// C. In step 5, python sees some pythonnet objects in its gc list of - /// potentially-leaked objects. It calls tp_traverse on those objects. - /// But tp_traverse was freed in step 3 => CRASH. - /// - /// This test distills what's going on without needing Unity around (we'd see - /// similar behaviour if we were using pythonnet on a .NET web server that did - /// a hot reload). - /// - [Test] - public static void DomainReloadAndGC() - { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); - - // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); - - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - } - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); - } - } - } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); - } - }"; - - - /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. - /// - static Assembly BuildAssembly(string assemblyName) - { - var provider = CodeDomProvider.CreateProvider("CSharp"); - - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; - - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); - if (results.Errors.HasErrors) - { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) - { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); - } - throw new Exception(errors.ToString()); - } - else - { - return results.CompiledAssembly; - } - } - - /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. - /// - class Proxy : MarshalByRefObject - { - Assembly theAssembly = null; - - public void InitAssembly(string assemblyPath) - { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); - } - - public void RunPython() - { - Console.WriteLine("[Proxy] Entering RunPython"); - - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); - - Console.WriteLine("[Proxy] Leaving RunPython"); - } - } - - /// - /// Create a domain, run the assembly in it (the RunPython function), - /// and unload the domain. - /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) - { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); - - // Create a Proxy object in the new domain, where we want the - // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - - // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); - theProxy.RunPython(); - - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); - AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); - - // Validate that the assembly does not exist anymore - try - { - Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); - } - catch (Exception) - { - Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); - } - } - - /// - /// Resolves the assembly. Why doesn't this just work normally? - /// - static Assembly ResolveAssembly(object sender, ResolveEventArgs args) - { - var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); - - foreach (var assembly in loadedAssemblies) - { - if (assembly.FullName == args.Name) - { - return assembly; - } - } - - return null; - } - } -} -#endif diff --git a/src/embed_tests/TestNamedArguments.cs b/src/embed_tests/TestNamedArguments.cs index 31f2ea1d2..1d7076956 100644 --- a/src/embed_tests/TestNamedArguments.cs +++ b/src/embed_tests/TestNamedArguments.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 65ac20e9a..d794ce06e 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NUnit.Framework; diff --git a/src/embed_tests/TestPySequence.cs b/src/embed_tests/TestPySequence.cs index 7c175b1ce..1e3ebf144 100644 --- a/src/embed_tests/TestPySequence.cs +++ b/src/embed_tests/TestPySequence.cs @@ -69,8 +69,10 @@ public void TestRepeat() PyObject actual = t1.Repeat(3); Assert.AreEqual("FooFooFoo", actual.ToString()); - actual = t1.Repeat(-3); - Assert.AreEqual("", actual.ToString()); + // On 32 bit system this argument should be int, but on the 64 bit system this should be long value. + // This works on the Framework 4.0 accidentally, it should produce out of memory! + // actual = t1.Repeat(-3); + // Assert.AreEqual("", actual.ToString()); } [Test] diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index 0c4e9023f..fd3f8e662 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 243349b82..01c6ae7e3 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -109,41 +109,18 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - // Restoring valid pythonhome. - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = "/dummypath2/"; @@ -152,20 +129,11 @@ public void SetPythonHomeTwice() Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetProgramName() { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - - var programNameBackup = PythonEngine.ProgramName; - var programName = "FooBar"; PythonEngine.ProgramName = programName; @@ -173,8 +141,6 @@ public void SetProgramName() Assert.AreEqual(programName, PythonEngine.ProgramName); PythonEngine.Shutdown(); - - PythonEngine.ProgramName = programNameBackup; } [Test] @@ -190,7 +156,7 @@ public void SetPythonPath() string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); - PythonEngine.PythonPath = path; + PythonEngine.ProgramName = path; PythonEngine.Initialize(); Assert.AreEqual(path, PythonEngine.PythonPath); @@ -205,6 +171,7 @@ public void SetPythonPathExceptionOn27() Assert.Pass(); } + // Get previous path to avoid crashing Python PythonEngine.Initialize(); string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..2e0598da7 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; @@ -6,40 +6,10 @@ namespace Python.EmbeddingTest { public class TestRuntime { - [OneTimeSetUp] - public void SetUp() - { - // We needs to ensure that no any engines are running. - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - - /// - /// Test the cache of the information from the platform module. - /// - /// Test fails on platforms we haven't implemented yet. - /// - [Test] - public static void PlatformCache() - { - Runtime.Runtime.Initialize(); - - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); - - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); // In case another test left it on. Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs deleted file mode 100644 index a4ef86913..000000000 --- a/src/embed_tests/TestTypeManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; -using System.Runtime.InteropServices; - -namespace Python.EmbeddingTest -{ - class TestTypeManager - { - [SetUp] - public static void Init() - { - Runtime.Runtime.Initialize(); - } - - [TearDown] - public static void Fini() - { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - - [Test] - public static void TestNativeCode() - { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); - } - - [Test] - public static void TestMemoryMapping() - { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); - - // Allocate a read-write page. - int len = 12; - var page = mapper.MapWriteable(len); - Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Mark it read-execute. We can still read, haven't changed any values. - mapper.SetReadExec(page, len); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Test that we can't write to the protected page. - // - // We can't actually test access protection under Microsoft - // versions of .NET, because AccessViolationException is assumed to - // mean we're in a corrupted state: - // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception - // - // We can test under Mono but it throws NRE instead of AccessViolationException. - // - // We can't use compiler flags because we compile with MONO_LINUX - // while running on the Microsoft .NET Core during continuous - // integration tests. - if (System.Type.GetType ("Mono.Runtime") != null) - { - // Mono throws NRE instead of AccessViolationException for some reason. - Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } - } - } -} diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 81345cee7..d75dc01d6 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -12,23 +12,13 @@ public class DynamicTest [SetUp] public void SetUp() { - try { - _gs = Py.GIL(); - } catch (Exception e) { - Console.WriteLine($"exception in SetUp: {e}"); - throw; - } + _gs = Py.GIL(); } [TearDown] public void Dispose() { - try { - _gs.Dispose(); - } catch(Exception e) { - Console.WriteLine($"exception in TearDown: {e}"); - throw; - } + _gs.Dispose(); } /// @@ -128,11 +118,10 @@ public void PassPyObjectInNet() sys.testattr2 = sys.testattr1; // Compare in Python - PythonEngine.RunSimpleString( + PyObject res = PythonEngine.RunString( "import sys\n" + "sys.testattr3 = sys.testattr1 is sys.testattr2\n" ); - Assert.AreEqual(sys.testattr3.ToString(), "True"); // Compare in .NET diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..2f9aae2c7 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -74,65 +74,5 @@ public void ReInitialize() } PythonEngine.Shutdown(); } - - [Test] - public void TestScopeIsShutdown() - { - PythonEngine.Initialize(); - var scope = PyScopeManager.Global.Create("test"); - PythonEngine.Shutdown(); - Assert.That(PyScopeManager.Global.Contains("test"), Is.False); - } - - /// - /// Helper for testing the shutdown handlers. - /// - int shutdown_count = 0; - void OnShutdownIncrement() - { - shutdown_count++; - } - void OnShutdownDouble() - { - shutdown_count *= 2; - } - - /// - /// Test the shutdown handlers. - /// - [Test] - public void ShutdownHandlers() - { - // Test we can run one shutdown handler. - shutdown_count = 0; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.Shutdown(); - Assert.That(shutdown_count, Is.EqualTo(1)); - - // Test we can run multiple shutdown handlers in the right order. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: 4 * 2 + 1 = 9 - // Wrong: (4 + 1) * 2 = 10 - Assert.That(shutdown_count, Is.EqualTo(9)); - - // Test we can remove shutdown handlers, handling duplicates. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.RemoveShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: (4 + 1) * 2 + 1 + 1 = 12 - // Wrong: (4 * 2) + 1 + 1 + 1 = 11 - Assert.That(shutdown_count, Is.EqualTo(12)); - } } } diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 794645994..cfde0a127 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -35,7 +35,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON36 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) @@ -131,7 +131,7 @@ - + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index fc155ca91..1fea78082 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -34,7 +34,7 @@ pdbonly - PYTHON3;PYTHON37;UCS4 + PYTHON3;PYTHON36;UCS4 true pdbonly @@ -46,7 +46,7 @@ true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG + PYTHON3;PYTHON36;UCS4;TRACE;DEBUG false full @@ -56,7 +56,7 @@ pdbonly - PYTHON3;PYTHON37;UCS2 + PYTHON3;PYTHON36;UCS2 true pdbonly @@ -68,7 +68,7 @@ true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG + PYTHON3;PYTHON36;UCS2;TRACE;DEBUG false full @@ -148,7 +148,6 @@ - @@ -167,4 +166,4 @@ - + \ No newline at end of file diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..dd4418cc9 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..a10688749 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -93,7 +93,7 @@ public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) return IntPtr.Zero; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; @@ -186,7 +186,7 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) return -1; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; for (var i = 0; i < count; i++) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..d63930a58 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Reflection; using System.Threading; @@ -18,21 +17,13 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - // - // WARNING: Dangerous if cross-app domain usage is ever supported - // Reusing the dictionary with assemblies accross multiple initializations is problematic. - // Loading happens from CurrentDomain (see line 53). And if the first call is from AppDomain that is later unloaded, - // than it can end up referring to assemblies that are already unloaded (default behavior after unload appDomain - - // unless LoaderOptimization.MultiDomain is used); - // So for multidomain support it is better to have the dict. recreated for each app-domain initialization - private static ConcurrentDictionary> namespaces = - new ConcurrentDictionary>(); + private static ConcurrentDictionary> namespaces; //private static Dictionary> generics; private static AssemblyLoadEventHandler lhandler; private static ResolveEventHandler rhandler; // updated only under GIL? - private static Dictionary probed = new Dictionary(32); + private static Dictionary probed; // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; @@ -49,6 +40,9 @@ private AssemblyManager() /// internal static void Initialize() { + namespaces = new ConcurrentDictionary>(); + probed = new Dictionary(32); + //generics = new Dictionary>(); assemblies = new ConcurrentQueue(); pypath = new List(16); @@ -138,7 +132,7 @@ private static Assembly ResolveHandler(object ob, ResolveEventArgs args) internal static void UpdatePath() { IntPtr list = Runtime.PySys_GetObject("path"); - var count = Runtime.PyList_Size(list); + int count = Runtime.PyList_Size(list); if (count != pypath.Count) { pypath.Clear(); @@ -349,7 +343,9 @@ internal static void ScanAssembly(Assembly assembly) // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + + Type[] types = assembly.GetTypes(); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -423,9 +419,10 @@ public static List GetNames(string nsname) { foreach (Assembly a in namespaces[nsname].Keys) { - foreach (Type t in GetTypes(a)) + Type[] types = a.GetTypes(); + foreach (Type t in types) { - if ((t.Namespace ?? "") == nsname && !t.IsNested) + if ((t.Namespace ?? "") == nsname) { names.Add(t.Name); } @@ -464,32 +461,5 @@ public static Type LookupType(string qname) } return null; } - - internal static Type[] GetTypes(Assembly a) - { - if (a.IsDynamic) - { - try - { - return a.GetTypes(); - } - catch (ReflectionTypeLoadException exc) - { - // Return all types that were successfully loaded - return exc.Types.Where(x => x != null).ToArray(); - } - } - else - { - try - { - return a.GetExportedTypes(); - } - catch (FileNotFoundException) - { - return new Type[0]; - } - } - } } -} \ No newline at end of file +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..4dd3b5364 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -247,6 +247,24 @@ public static IntPtr tp_str(IntPtr ob) } + /// + /// Default implementations for required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + /// /// Standard dealloc implementation for instances of reflected types. /// diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..16d3b99db 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,9 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Reflection.Emit; -using System.Resources; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -33,12 +32,6 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } - public static void Reset() - { - assemblyBuilders = new Dictionary(); - moduleBuilders = new Dictionary, ModuleBuilder>(); - } - internal ClassDerivedObject(Type tp) : base(tp) { } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0b084a49d..6a9d40ebd 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -34,11 +34,6 @@ static ClassManager() dtype = typeof(MulticastDelegate); } - public static void Reset() - { - cache = new Dictionary(128); - } - /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 83d761fd0..46257c73f 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -230,10 +230,10 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) } // Get the args passed in. - var i = Runtime.PyTuple_Size(args); + int i = Runtime.PyTuple_Size(args); IntPtr defaultArgs = cls.indexer.GetDefaultArgs(args); - var numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); - var temp = i + numOfDefaultArgs; + int numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); + int temp = i + numOfDefaultArgs; IntPtr real = Runtime.PyTuple_New(temp + 1); for (var n = 0; n < i; n++) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..fb3d0e0d7 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 629eab4df..d8ec3fcd2 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -3,7 +3,6 @@ using System.Globalization; using System.Runtime.InteropServices; using System.Security; -using System.ComponentModel; namespace Python.Runtime { @@ -167,8 +166,8 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { + if (value is IList && value.GetType().IsGenericType) + { using (var resultlist = new PyList()) { foreach (object o in (IEnumerable)value) @@ -1002,7 +1001,7 @@ private static void SetConversionError(IntPtr value, Type target) private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - var size = Runtime.PySequence_Size(value); + int size = Runtime.PySequence_Size(value); result = null; if (size < 0 || elementType.IsGenericType) diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs index 3fe9ee5bb..2a91a74b4 100644 --- a/src/runtime/debughelper.cs +++ b/src/runtime/debughelper.cs @@ -116,10 +116,10 @@ internal static void debug(string msg) Console.WriteLine(" {0}", msg); } - /// + /// /// Helper function to inspect/compare managed to native conversions. - /// Especially useful when debugging CustomMarshaler. - /// + /// Especially useful when debugging CustomMarshaler. + /// /// [Conditional("DEBUG")] public static void PrintHexBytes(byte[] bytes) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..9023cfcfa 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -256,10 +256,7 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.XIncref(pe.PyType); - Runtime.XIncref(pe.PyValue); - Runtime.XIncref(pe.PyTB); - Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); + Runtime.PyErr_SetObject(pe.PyType, pe.PyValue); return; } @@ -279,7 +276,7 @@ public static void SetError(Exception e) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != IntPtr.Zero; + return Runtime.PyErr_Occurred() != 0; } /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..9569b0485 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -81,6 +81,27 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } + /// + /// Required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + + /// /// Default dealloc implementation. /// diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..9772d082f 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Resources; namespace Python.Runtime { @@ -21,11 +20,6 @@ static GenericUtil() mapping = new Dictionary>>(); } - public static void Reset() - { - mapping = new Dictionary>>(); - } - /// /// Register a generic type that appears in a given namespace. /// diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..bc9ac5eee 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -155,7 +155,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // hook is saved as this.py_import. This version handles CLR // import and defers to the normal builtin for everything else. - var num_args = Runtime.PyTuple_Size(args); + int num_args = Runtime.PyTuple_Size(args); if (num_args < 1) { return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); @@ -237,11 +237,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) if (res != IntPtr.Zero) { // There was no error. - if (fromlist && IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); - } return res; } // There was an error @@ -295,11 +290,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { if (fromlist) { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } Runtime.XIncref(module); return module; } @@ -355,33 +345,20 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } } - { - var mod = fromlist ? tail : head; + ModuleObject mod = fromlist ? tail : head; - if (fromlist && IsLoadAll(fromList)) + if (fromlist && Runtime.PySequence_Size(fromList) == 1) + { + IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); + if (!CLRModule.preload && Runtime.GetManagedString(fp) == "*") { mod.LoadNames(); } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; + Runtime.XDecref(fp); } - } - private static bool IsLoadAll(IntPtr fromList) - { - if (CLRModule.preload) - { - return false; - } - if (Runtime.PySequence_Size(fromList) != 1) - { - return false; - } - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - bool res = Runtime.GetManagedString(fp) == "*"; - Runtime.XDecref(fp); - return res; + Runtime.XIncref(mod.pyHandle); + return mod.pyHandle; } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..7b6d90ca8 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -56,7 +56,7 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); MethodBase[] methods = SetterBinder.GetMethods(); if (methods.Length == 0) { @@ -72,7 +72,7 @@ internal bool NeedsDefaultArgs(IntPtr args) return false; } - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -95,7 +95,7 @@ internal IntPtr GetDefaultArgs(IntPtr args) { return Runtime.PyTuple_New(0); } - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple MethodBase[] methods = SetterBinder.GetMethods(); diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 616ced6bd..ce1bc9eb0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -36,7 +36,7 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); - var nargs = Runtime.PyTuple_Size(args); + int nargs = Runtime.PyTuple_Size(args); Type type = self.type; object obj; diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs deleted file mode 100644 index d5fc76ad3..000000000 --- a/src/runtime/interop37.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON37 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_as_async = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int am_await = 0; - public static int am_aiter = 0; - public static int am_anext = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int nb_matrix_multiply = 0; - public static int nb_inplace_matrix_multiply = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..3295ab110 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -29,7 +29,7 @@ public static IntPtr Initialize() /// public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 3) { return Exceptions.RaiseTypeError("invalid argument list"); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index b608b6c55..48dc1eb9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -291,7 +291,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // loop to find match, return invoker w/ or /wo error MethodBase[] _methods = null; - var pynargs = (int)Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); object arg; var isGeneric = false; ArrayList defaultArgList = null; @@ -313,9 +313,9 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth isGeneric = true; } ParameterInfo[] pi = mi.GetParameters(); - var clrnargs = pi.Length; + int clrnargs = pi.Length; var match = false; - var arrayStart = -1; + int arrayStart = -1; var outs = 0; if (pynargs == clrnargs) @@ -326,7 +326,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { match = true; defaultArgList = new ArrayList(); - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -350,7 +350,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { var margs = new object[clrnargs]; - for (int n = 0; n < clrnargs; n++) + for (var n = 0; n < clrnargs; n++) { IntPtr op; if (n < pynargs) @@ -632,19 +632,6 @@ internal class MethodSorter : IComparer { int IComparer.Compare(object m1, object m2) { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; - if (me1.DeclaringType != me2.DeclaringType) - { - // m2's type derives from m1's type, favor m2 - if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) - return 1; - - // m1's type derives from m2's type, favor m1 - if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) - return -1; - } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); int p2 = MethodBinder.GetPrecedence((MethodBase)m2); if (p1 < p2) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..3985ed2cb 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -104,7 +104,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { if (self.info.IsGenericMethod) { - var len = Runtime.PyTuple_Size(args); //FIXME: Never used + int len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { @@ -129,7 +129,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (target == IntPtr.Zero && !self.m.IsStatic()) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 1) { Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8af722d29..4accb1531 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -190,17 +190,10 @@ public void LoadNames() foreach (string name in AssemblyManager.GetNames(_namespace)) { cache.TryGetValue(name, out m); - if (m != null) + if (m == null) { - continue; + ManagedType attr = GetAttribute(name, true); } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); - // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) - { - continue; - } - GetAttribute(name, true); } } @@ -335,17 +328,6 @@ public CLRModule() : base("clr") } } - public static void Reset() - { - hacked = false; - interactive_preload = true; - preload = false; - - // XXX Test performance of new features // - _SuppressDocs = false; - _SuppressOverloads = false; - } - /// /// The initializing of the preload hook has to happen as late as /// possible since sys.ps1 is created after the CLR module is diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3b8c71efa..0e075824a 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -519,9 +519,9 @@ public virtual void DelItem(int index) /// Returns the length for objects that support the Python sequence /// protocol, or 0 if the object does not support the protocol. /// - public virtual long Length() + public virtual int Length() { - var s = Runtime.PyObject_Size(obj); + int s = Runtime.PyObject_Size(obj); if (s < 0) { Runtime.PyErr_Clear(); diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8e6957855..67f93c6e2 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -537,15 +537,10 @@ public void Dispose() public class PyScopeManager { - public static PyScopeManager Global; + public readonly static PyScopeManager Global = new PyScopeManager(); private Dictionary NamedScopes = new Dictionary(); - internal static void Reset() - { - Global = new PyScopeManager(); - } - internal PyScope NewScope(string name) { if (name == null) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..a23c7ac79 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -140,9 +140,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv); } /// @@ -153,9 +153,8 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// more than once, though initialization will only happen on the /// first call. It is *not* necessary to hold the Python global /// interpreter lock (GIL) to call this method. - /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true) { if (!initialized) { @@ -165,20 +164,10 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); + Runtime.Initialize(); initialized = true; Exceptions.Clear(); - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - if (setSysArgv) { Py.SetArgv(args); @@ -234,11 +223,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } } - static void OnDomainUnload(object _, EventArgs __) - { - Shutdown(); - } - /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -311,80 +295,18 @@ public static void Shutdown() if (initialized) { PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - - ExecuteShutdownHandlers(); + Marshal.FreeHGlobal(_pythonHome); + _pythonHome = IntPtr.Zero; + Marshal.FreeHGlobal(_programName); + _programName = IntPtr.Zero; + Marshal.FreeHGlobal(_pythonPath); + _pythonPath = IntPtr.Zero; + Runtime.Shutdown(); initialized = false; } } - /// - /// Called when the engine is shut down. - /// - /// Shutdown handlers are run in reverse order they were added, so that - /// resources available when running a shutdown handler are the same as - /// what was available when it was added. - /// - public delegate void ShutdownHandler(); - - static List ShutdownHandlers = new List(); - - /// - /// Add a function to be called when the engine is shut down. - /// - /// Shutdown handlers are executed in the opposite order they were - /// added, so that you can be sure that everything that was initialized - /// when you added the handler is still initialized when you need to shut - /// down. - /// - /// If the same shutdown handler is added several times, it will be run - /// several times. - /// - /// Don't add shutdown handlers while running a shutdown handler. - /// - public static void AddShutdownHandler(ShutdownHandler handler) - { - ShutdownHandlers.Add(handler); - } - - /// - /// Remove a shutdown handler. - /// - /// If the same shutdown handler is added several times, only the last - /// one is removed. - /// - /// Don't remove shutdown handlers while running a shutdown handler. - /// - public static void RemoveShutdownHandler(ShutdownHandler handler) - { - for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) - { - if (ShutdownHandlers[index] == handler) - { - ShutdownHandlers.RemoveAt(index); - break; - } - } - } - - /// - /// Run all the shutdown handlers. - /// - /// They're run in opposite order they were added. - /// - static void ExecuteShutdownHandlers() - { - while(ShutdownHandlers.Count > 0) - { - var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; - ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); - handler(); - } - } /// /// AcquireLock Method diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 21a1d20fd..5ef06f3cc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,7 +2,6 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; -using System.Collections.Generic; namespace Python.Runtime { @@ -22,7 +21,7 @@ public static IntPtr LoadLibrary(string fileName) } #elif MONO_OSX private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; + private const string NativeDll = "/usr/lib/libSystem.dylib" private static IntPtr RTLD_DEFAULT = new IntPtr(-2); public static IntPtr LoadLibrary(string fileName) @@ -149,7 +148,7 @@ public class Runtime #elif PYTHON36 internal const string _pyversion = "3.6"; internal const string _pyver = "36"; -#elif PYTHON37 +#elif PYTHON37 // TODO: Add `interop37.cs` after PY37 is released internal const string _pyversion = "3.7"; internal const string _pyver = "37"; #else @@ -196,68 +195,6 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static string OperatingSystemName { get; private set; } - - public enum MachineType - { - i386, - x86_64, - Other - }; - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; @@ -269,11 +206,11 @@ public enum MachineType /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + internal static void Initialize() { if (Py_IsInitialized() == 0) { - Py_InitializeEx(initSigs ? 1 : 0); + Py_Initialize(); } if (PyEval_ThreadsInitialized() == 0) @@ -281,15 +218,6 @@ internal static void Initialize(bool initSigs = false) PyEval_InitThreads(); } - IsFinalizing = false; - - CLRModule.Reset(); - GenericUtil.Reset(); - PyScopeManager.Reset(); - ClassManager.Reset(); - ClassDerivedObject.Reset(); - TypeManager.Reset(); - IntPtr op; IntPtr dict; if (IsPython3) @@ -411,10 +339,6 @@ internal static void Initialize(bool initSigs = false) NativeMethods.FreeLibrary(dllLocal); } #endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); @@ -432,53 +356,6 @@ internal static void Initialize(bool initSigs = false) AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() - { - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - OperatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - XDecref(emptyTuple); - XDecref(platformModule); - - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) - { - OSType = OperatingSystemType.Other; - } - OperatingSystem = OSType; - - MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) - { - MType = MachineType.Other; - } - Machine = MType; - } - internal static void Shutdown() { AssemblyManager.Shutdown(); @@ -549,7 +426,7 @@ internal static int AtExit() /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != IntPtr.Zero) + if (PyErr_Occurred() != 0) { throw new PythonException(); } @@ -557,7 +434,7 @@ internal static void CheckExceptionOccurred() internal static IntPtr ExtendTuple(IntPtr t, params IntPtr[] args) { - var size = PyTuple_Size(t); + int size = PyTuple_Size(t); int add = args.Length; IntPtr item; @@ -600,7 +477,7 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) free = true; } - var n = PyTuple_Size(args); + int n = PyTuple_Size(args); var types = new Type[n]; Type t = null; @@ -736,9 +613,6 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_Initialize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_InitializeEx(int initsigs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_IsInitialized(); @@ -1034,13 +908,8 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_Not(IntPtr pointer); - internal static long PyObject_Size(IntPtr pointer) - { - return (long) _PyObject_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] - private static extern IntPtr _PyObject_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Hash(IntPtr op); @@ -1270,61 +1139,26 @@ internal static bool PyFloat_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PySequence_Check(IntPtr pointer); - internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) - { - return PySequence_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); - - internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PySequence_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PySequence_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PySequence_DelItem(IntPtr pointer, long index) - { - return PySequence_DelItem(pointer, new IntPtr(index)); - } + internal static extern int PySequence_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); - - internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_DelItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - - internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) - { - return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); - } + internal static extern IntPtr PySequence_GetSlice(IntPtr pointer, int i1, int i2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); - - internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_SetSlice(IntPtr pointer, int i1, int i2, IntPtr v); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); + internal static extern int PySequence_DelSlice(IntPtr pointer, int i1, int i2); - internal static long PySequence_Size(IntPtr pointer) - { - return (long) _PySequence_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] - private static extern IntPtr _PySequence_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); @@ -1332,24 +1166,14 @@ internal static long PySequence_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); - internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) - { - return PySequence_Repeat(pointer, new IntPtr(count)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); + internal static extern IntPtr PySequence_Repeat(IntPtr pointer, int count); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); - internal static long PySequence_Count(IntPtr pointer, IntPtr value) - { - return (long) _PySequence_Count(pointer, value); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] - private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Count(IntPtr pointer, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Tuple(IntPtr pointer); @@ -1375,57 +1199,33 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { -#if PYTHON3 - return PyUnicode_FromKindAndData(_UCS, value, value.Length); -#elif PYTHON2 return PyString_FromStringAndSize(value, value.Length); -#endif } #if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyBytes_FromString(string op); - internal static long PyBytes_Size(IntPtr op) - { - return (long) _PyBytes_Size(op); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] - private static extern IntPtr _PyBytes_Size(IntPtr op); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBytes_Size(IntPtr op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { return ob + BytesOffset.ob_sval; } - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return _PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_FromStringAndSize")] - internal static extern IntPtr _PyString_FromStringAndSize( + internal static extern IntPtr PyString_FromStringAndSize( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) - { - return PyUnicode_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + internal static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, int size); #elif PYTHON2 - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyString_FromStringAndSize(string value, IntPtr size); + internal static extern IntPtr PyString_FromStringAndSize(string value, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyString_AsString(IntPtr op); @@ -1446,30 +1246,20 @@ internal static bool PyUnicode_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) - { - return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromKindAndData( + internal static extern IntPtr PyUnicode_FromKindAndData( int kind, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) + internal static IntPtr PyUnicode_FromUnicode(string s, int size) { return PyUnicode_FromKindAndData(_UCS, s, size); } - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long)_PyUnicode_GetSize(ob); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] - private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); @@ -1485,26 +1275,16 @@ internal static long PyUnicode_GetSize(IntPtr ob) EntryPoint = PyUnicodeEntryPoint + "FromEncodedObject")] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - return PyUnicode_FromUnicode(s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "FromUnicode")] - private static extern IntPtr PyUnicode_FromUnicode( + internal static extern IntPtr PyUnicode_FromUnicode( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long) _PyUnicode_GetSize(ob); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "GetSize")] - internal static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "AsUnicode")] @@ -1547,7 +1327,7 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { IntPtr p = PyUnicode_AsUnicode(op); - int length = (int)PyUnicode_GetSize(op); + int length = PyUnicode_GetSize(op); int size = length * _UCS; var buffer = new byte[size]; @@ -1613,13 +1393,8 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyDict_Clear(IntPtr pointer); - internal static long PyDict_Size(IntPtr pointer) - { - return (long) _PyDict_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] - internal static extern IntPtr _PyDict_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Size(IntPtr pointer); //==================================================================== @@ -1631,40 +1406,20 @@ internal static bool PyList_Check(IntPtr ob) return PyObject_TYPE(ob) == PyListType; } - internal static IntPtr PyList_New(long size) - { - return PyList_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_New(IntPtr size); + internal static extern IntPtr PyList_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) - { - return PyList_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyList_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyList_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) - { - return PyList_Insert(pointer, new IntPtr(index), value); - } + internal static extern int PyList_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(IntPtr pointer, IntPtr index, IntPtr value); + internal static extern int PyList_Insert(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Append(IntPtr pointer, IntPtr value); @@ -1675,29 +1430,15 @@ internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Sort(IntPtr pointer); - internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) - { - return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - - internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) - { - return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); - } + internal static extern IntPtr PyList_GetSlice(IntPtr pointer, int start, int end); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); + internal static extern int PyList_SetSlice(IntPtr pointer, int start, int end, IntPtr value); - internal static long PyList_Size(IntPtr pointer) - { - return (long) _PyList_Size(pointer); - } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Size(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] - private static extern IntPtr _PyList_Size(IntPtr pointer); //==================================================================== // Python tuple API @@ -1708,45 +1449,20 @@ internal static bool PyTuple_Check(IntPtr ob) return PyObject_TYPE(ob) == PyTupleType; } - internal static IntPtr PyTuple_New(long size) - { - return PyTuple_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_New(IntPtr size); - - internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) - { - return PyTuple_GetItem(pointer, new IntPtr(index)); - } + internal static extern IntPtr PyTuple_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyTuple_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyTuple_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) - { - return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } + internal static extern int PyTuple_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); + internal static extern IntPtr PyTuple_GetSlice(IntPtr pointer, int start, int end); - internal static long PyTuple_Size(IntPtr pointer) - { - return (long) _PyTuple_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] - private static extern IntPtr _PyTuple_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyTuple_Size(IntPtr pointer); //==================================================================== @@ -1852,13 +1568,8 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); - internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) - { - return PyType_GenericAlloc(type, new IntPtr(n)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + internal static extern IntPtr PyType_GenericAlloc(IntPtr type, int n); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1892,21 +1603,11 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) // Python memory API //==================================================================== - internal static IntPtr PyMem_Malloc(long size) - { - return PyMem_Malloc(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Malloc(IntPtr size); - - internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) - { - return PyMem_Realloc(ptr, new IntPtr(size)); - } + internal static extern IntPtr PyMem_Malloc(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); + internal static extern IntPtr PyMem_Realloc(IntPtr ptr, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyMem_Free(IntPtr ptr); @@ -1938,7 +1639,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern void PyErr_NormalizeException(IntPtr ob, IntPtr val, IntPtr tb); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_Occurred(); + internal static extern int PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..6570ee083 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Reflection; @@ -21,10 +21,6 @@ static TypeManager() cache = new Dictionary(128); } - public static void Reset() - { - cache = new Dictionary(128); - } /// /// Given a managed Type derived from ExtensionType, get the handle to @@ -451,225 +447,6 @@ internal static IntPtr AllocateTypeObject(string name) } - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch(Runtime.Machine) - { - case Runtime.MachineType.i386: - return I386; - case Runtime.MachineType.x86_64: - return X86_64; - default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - return 0x1000; - case Runtime.OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } -#endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of @@ -677,10 +454,8 @@ internal static void InitializeNativeCodePage() /// internal static void InitializeSlots(IntPtr type, Type impl) { - // We work from the most-derived class up; make sure to get - // the most-derived slot and not to override it with a base - // class's slot. - var seen = new HashSet(); + var seen = new Hashtable(8); + Type offsetType = typeof(TypeOffset); while (impl != null) { @@ -698,52 +473,24 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - if (seen.Contains(name)) + if (seen[name] != null) { continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + FieldInfo fi = offsetType.GetField(name); + var offset = (int)fi.GetValue(offsetType); + + IntPtr slot = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, offset, slot); - seen.Add(name); + seen[name] = 1; } impl = impl.BaseType; } - - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); } - /// - /// Helper for InitializeSlots. - /// - /// Initializes one slot to point to a function pointer. - /// The function pointer might be a thunk for C#, or it may be - /// an address in the NativeCodePage. - /// - /// Type being initialized. - /// Function pointer. - /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) - { - Type typeOffset = typeof(TypeOffset); - FieldInfo fi = typeOffset.GetField(name); - var offset = (int)fi.GetValue(typeOffset); - - Marshal.WriteIntPtr(type, offset, slot); - } /// /// Given a newly allocated Python type object and a managed Type that diff --git a/src/testing/InheritanceTest.cs b/src/testing/InheritanceTest.cs deleted file mode 100644 index 529703b3c..000000000 --- a/src/testing/InheritanceTest.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace Python.Test -{ - public class BaseClass - { - public bool IsBase() => true; - } - - public class DerivedClass : BaseClass - { - public new bool IsBase() => false; - } -} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..8a8d9ed2b 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -83,7 +83,6 @@ - @@ -111,4 +110,4 @@ - \ No newline at end of file + diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 06ab7cb4e..36294594c 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -60,19 +60,4 @@ public string GetValue() return value; } } - - public class UnicodeString - { - public string value = "안녕"; - - public string GetString() - { - return value; - } - - public override string ToString() - { - return value; - } - } } diff --git a/src/tests/importtest.py b/src/tests/importtest.py deleted file mode 100644 index fe93764e9..000000000 --- a/src/tests/importtest.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -try: - del sys.modules["System.IO"] -except KeyError: - pass - -assert "FileStream" not in globals() -import System.IO -from System.IO import * - -assert "FileStream" in globals() diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 612ce442e..68773508b 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -281,14 +281,3 @@ def PyCallback(self, self2): testobj.DoCallback() assert testobj.PyCallbackWasCalled assert testobj.SameReference - - -def test_method_inheritance(): - """Ensure that we call the overridden method instead of the one provided in - the base class.""" - - base = Test.BaseClass() - derived = Test.DerivedClass() - - assert base.IsBase() == True - assert derived.IsBase() == False diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 0ba10a80e..53e5d8051 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -2,12 +2,11 @@ """Test CLR <-> Python type conversions.""" -from __future__ import unicode_literals import System import pytest -from Python.Test import ConversionTest, UnicodeString +from Python.Test import ConversionTest -from ._compat import indexbytes, long, unichr, text_type, PY2, PY3 +from ._compat import indexbytes, long, unichr def test_bool_conversion(): @@ -536,14 +535,6 @@ def test_string_conversion(): with pytest.raises(TypeError): ConversionTest().StringField = 1 - - world = UnicodeString() - test_unicode_str = u"안녕" - assert test_unicode_str == text_type(world.value) - assert test_unicode_str == text_type(world.GetString()) - # TODO: not sure what to do for Python 2 here (GH PR #670) - if PY3: - assert test_unicode_str == text_type(world) def test_interface_conversion(): diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index 08b00d77d..c697290ee 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -289,8 +289,7 @@ def test_python_compat_of_managed_exceptions(): assert e.args == (msg,) assert isinstance(e.args, tuple) if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp + assert repr(e) == "OverflowException('Simple message',)" elif PY2: assert repr(e) == "OverflowException(u'Simple message',)" diff --git a/src/tests/test_import.py b/src/tests/test_import.py index 25877be15..42cafc4df 100644 --- a/src/tests/test_import.py +++ b/src/tests/test_import.py @@ -3,7 +3,7 @@ """Test the import statement.""" import pytest -import sys + def test_relative_missing_import(): """Test that a relative missing import doesn't crash. @@ -11,12 +11,3 @@ def test_relative_missing_import(): Relative import in the site-packages folder""" with pytest.raises(ImportError): from . import _missing_import - - -def test_import_all_on_second_time(): - """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited - to a module file.""" - from . import importtest - del sys.modules[importtest.__name__] - diff --git a/src/tests/test_subclass.py b/src/tests/test_subclass.py index ab440d429..43d013c7c 100644 --- a/src/tests/test_subclass.py +++ b/src/tests/test_subclass.py @@ -128,39 +128,6 @@ def test_derived_class(): assert id(x) == id(ob) -def test_derived_traceback(): - """Test python exception traceback in class derived from managed base""" - class DerivedClass(SubClassTest): - __namespace__ = "Python.Test.traceback" - - def foo(self): - print (xyzname) - return None - - import sys,traceback - ob = DerivedClass() - - # direct call - try: - ob.foo() - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - # call through managed code - try: - FunctionsTest.test_foo(ob) - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - def test_create_instance(): """Test derived instances can be created from managed code""" DerivedClass = derived_class_fixture(test_create_instance.__name__) diff --git a/tools/geninterop/fake_libc_include/crypt.h b/tools/geninterop/fake_libc_include/crypt.h deleted file mode 100644 index 3b16d481f..000000000 --- a/tools/geninterop/fake_libc_include/crypt.h +++ /dev/null @@ -1 +0,0 @@ -#include "features.h" \ No newline at end of file diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..bf5fdb96b 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -173,8 +173,7 @@ def preprocess_python_headers(): "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", - "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "__int64=long long" ] if hasattr(sys, "abiflags"): @@ -186,7 +185,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-I"] + include_dirs + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = [] From f4a047b9108d1d759372d8a9e590e8940b10bf8a Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 18 Jan 2019 18:46:41 -0300 Subject: [PATCH 24/56] Performance improvements ClrObject - Dispose PyObj - Increasing minor version to 14 - Adding `UnsafeDispose()` for `PyObject` which does not require acquiring/releasing the lock - Adding check before calling `SetArgsAndCause` for the `ClrObject`, the call only makes sense when we are an exception and causes an overhead --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/clrobject.cs | 12 ++++++++---- src/runtime/pyobject.cs | 19 ++++++++++++++++++- src/runtime/resources/clr.py | 2 +- 8 files changed, 32 insertions(+), 11 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2518f77fb..31cc3ea8d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.13 +current_version = 1.0.5.14 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3aa2d1a62..e1b225169 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.13" + version: "1.0.5.14" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 3695e33fa..ddb59747c 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.13", + version="1.0.5.14", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index e9d39ba33..1df774b4a 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.13")] +[assembly: AssemblyVersion("1.0.5.14")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index c5b789a18..3384b6c5e 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.12"), + Version = new Version("1.0.5.14"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index fb3d0e0d7..17782f026 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime @@ -29,9 +29,13 @@ internal CLRObject(object ob, IntPtr tp) gcHandle = gc; inst = ob; - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + // for performance before calling SetArgsAndCause() lets check if we are an exception + if (inst is Exception) + { + // Fix the BaseException args (and __cause__ in case of Python 3) + // slot if wrapping a CLR exception + Exceptions.SetArgsAndCause(py); + } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 0e075824a..35a4e6fa9 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -101,7 +101,7 @@ public object AsManagedObject(Type t) } return result; } - + /// /// As Method /// @@ -156,6 +156,23 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + /// Unsafe Dispose Method. + /// To be used when already owning the GIL lock. + /// + public void UnsafeDispose() + { + if (!disposed) + { + if (!Runtime.IsFinalizing) + { + Runtime.XDecref(obj); + obj = IntPtr.Zero; + } + disposed = true; + } + GC.SuppressFinalize(this); + } /// /// GetPythonType Method diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7caff08f4..48d2907c4 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.13" +__version__ = "1.0.5.14" class clrproperty(object): From daddc6809573123f930380bb29e8d40922d05329 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Tue, 22 Jan 2019 21:56:03 +0000 Subject: [PATCH 25/56] Version bump to 1.0.5.15 Version bump to match nuget package --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 31cc3ea8d..ccb9c32e8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.14 +current_version = 1.0.5.15 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index e1b225169..0b70bedc9 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.14" + version: "1.0.5.15" build: skip: True # [not win] diff --git a/setup.py b/setup.py index ddb59747c..421054d77 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.14", + version="1.0.5.15", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 1df774b4a..cc71683aa 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.14")] +[assembly: AssemblyVersion("1.0.5.15")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3384b6c5e..143759f2e 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.14"), + Version = new Version("1.0.5.15"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 48d2907c4..f877058e8 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.14" +__version__ = "1.0.5.15" class clrproperty(object): From f47b9f5c4900c01fe7ccc253af8ef65de2e813e6 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 20 Feb 2019 13:02:42 -0300 Subject: [PATCH 26/56] Improve Indicator Performance Benchmark - Adding new `interop` `type` cache holding a `bool`, true if its an `exception`. - Adding a `setter` and `getter` cache for the `propertyobject`. --- src/runtime/interop.cs | 26 ++++++++--- src/runtime/propertyobject.cs | 88 +++++++++++++++++++++++++++++++++-- src/runtime/runtime.cs | 11 +++-- 3 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..7ead0b1d2 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,6 +1,6 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; using System.Runtime.InteropServices; using System.Reflection; using System.Text; @@ -88,8 +88,7 @@ static ObjectOffset() public static int magic(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.ob_data; } @@ -98,8 +97,7 @@ public static int magic(IntPtr ob) public static int DictOffset(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.ob_dict; } @@ -108,8 +106,7 @@ public static int DictOffset(IntPtr ob) public static int Size(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.Size(); } @@ -128,6 +125,21 @@ public static int Size(IntPtr ob) public static int ob_type; private static int ob_dict; private static int ob_data; + private static readonly Dictionary ExceptionTypeCache = new Dictionary(); + + private static bool IsException(IntPtr pyObject) + { + bool res; + var type = Runtime.PyObject_TYPE(pyObject); + if (!ExceptionTypeCache.TryGetValue(type, out res)) + { + res = Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) + || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + ExceptionTypeCache.Add(type, res); + } + return res; + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..00a8ab4e7 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -1,4 +1,5 @@ using System; +using System.Linq.Expressions; using System.Reflection; using System.Security.Permissions; @@ -12,6 +13,10 @@ internal class PropertyObject : ExtensionType private PropertyInfo info; private MethodInfo getter; private MethodInfo setter; + private bool getterCacheFailed; + private bool setterCacheFailed; + private Func getterCache; + private Action setterCache; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -67,7 +72,21 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); + if (self.getterCache == null && !self.getterCacheFailed) + { + // if the getter is not public 'GetGetMethod' will not find it + // but calling 'GetValue' will work, so for backwards compatibility + // we will use it instead + self.getterCache = BuildGetter(self.info); + if (self.getterCache == null) + { + self.getterCacheFailed = true; + } + } + + result = self.getterCacheFailed ? + self.info.GetValue(co.inst, null) + : self.getterCache(co.inst); return Converter.ToPython(result, self.info.PropertyType); } catch (Exception e) @@ -81,7 +100,6 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } } - /// /// Descriptor __set__ implementation. This method sets the value of /// a property based on the given Python value. The Python value must @@ -132,7 +150,27 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + + if (self.setterCache == null && !self.setterCacheFailed) + { + // if the setter is not public 'GetSetMethod' will not find it + // but calling 'SetValue' will work, so for backwards compatibility + // we will use it instead + self.setterCache = BuildSetter(self.info); + if (self.setterCache == null) + { + self.setterCacheFailed = true; + } + } + + if (self.setterCacheFailed) + { + self.info.SetValue(co.inst, newval, null); + } + else + { + self.setterCache(co.inst, newval); + } } else { @@ -160,5 +198,49 @@ public static IntPtr tp_repr(IntPtr ob) var self = (PropertyObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + private static Func BuildGetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetGetMethod(); + if (methodInfo == null) + { + // if the getter is not public 'GetGetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var expressionCall = Expression.Call(instance, methodInfo); + + return Expression.Lambda>( + Expression.Convert(expressionCall, typeof(object)), + obj).Compile(); + } + + private static Action BuildSetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetSetMethod(); + if (methodInfo == null) + { + // if the setter is not public 'GetSetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var value = Expression.Parameter(typeof(object)); + var argument = Expression.Convert(value, methodInfo.GetParameters()[0].ParameterType); + + var expressionCall = Expression.Call(instance, methodInfo, argument); + + return Expression.Lambda>( + expressionCall, + obj, + value).Compile(); + } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5ef06f3cc..d1183f68e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -104,7 +104,7 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) public class Runtime { // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static int UCS => _UCS; @@ -130,7 +130,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static string pyversion => _pyversion; @@ -173,7 +173,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static readonly string PythonDLL = _PythonDll; @@ -1565,6 +1565,11 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) return (t == tp) || PyType_IsSubtype(t, tp); } + internal static bool PyObjectType_TypeCheck(IntPtr type, IntPtr tp) + { + return (type == tp) || PyType_IsSubtype(type, tp); + } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); From ab0d1b35c05f9e067a3ff60f607c18faf21f8577 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 20 Feb 2019 15:50:37 -0300 Subject: [PATCH 27/56] Review: Adding parameter count check --- src/runtime/propertyobject.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index 00a8ab4e7..cc6125157 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -232,8 +232,13 @@ private static Action BuildSetter(PropertyInfo propertyInfo) // so 'obj' is declared as typeof(object) var instance = Expression.Convert(obj, methodInfo.DeclaringType); + var parameters = methodInfo.GetParameters(); + if (parameters.Length != 1) + { + return null; + } var value = Expression.Parameter(typeof(object)); - var argument = Expression.Convert(value, methodInfo.GetParameters()[0].ParameterType); + var argument = Expression.Convert(value, parameters[0].ParameterType); var expressionCall = Expression.Call(instance, methodInfo, argument); From 5c35e800f7423ab9b474978ccea87f387530039d Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Wed, 20 Feb 2019 16:47:52 +0000 Subject: [PATCH 28/56] Decimal Parsing Allows Numeric String in Exponential Notation --- src/runtime/converter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index d8ec3fcd2..a81d9fb99 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -941,7 +941,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo op = Runtime.PyObject_Str(value); decimal m; string sm = Runtime.GetManagedString(op); - if (!Decimal.TryParse(sm, NumberStyles.Number, nfi, out m)) + if (!Decimal.TryParse(sm, NumberStyles.Number | NumberStyles.AllowExponent, nfi, out m)) { goto type_error; } From 7e9c3df3a8365e744aa581c3c2700669190e054a Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Wed, 20 Feb 2019 16:54:18 +0000 Subject: [PATCH 29/56] Version bump to 1.0.5.16 Version bump to match nuget package --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ccb9c32e8..7ea53f6ba 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.15 +current_version = 1.0.5.16 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 0b70bedc9..497adf5a0 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.15" + version: "1.0.5.16" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 421054d77..03fe25456 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.15", + version="1.0.5.16", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index cc71683aa..c17676f61 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.15")] +[assembly: AssemblyVersion("1.0.5.16")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 143759f2e..b198df56b 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.15"), + Version = new Version("1.0.5.16"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index f877058e8..c31118458 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.15" +__version__ = "1.0.5.16" class clrproperty(object): From b88e72f099d2331bc768b74f81bfcc9fab158ab5 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 21 Feb 2019 15:14:22 -0300 Subject: [PATCH 30/56] Remove ExceptionTypeCache - Removing `ExceptionTypeCache` which was causing issues and fix for issue caused performance overhead. --- src/runtime/interop.cs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 7ead0b1d2..5168bcd98 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -125,20 +125,13 @@ public static int Size(IntPtr ob) public static int ob_type; private static int ob_dict; private static int ob_data; - private static readonly Dictionary ExceptionTypeCache = new Dictionary(); private static bool IsException(IntPtr pyObject) { - bool res; var type = Runtime.PyObject_TYPE(pyObject); - if (!ExceptionTypeCache.TryGetValue(type, out res)) - { - res = Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) - || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - ExceptionTypeCache.Add(type, res); - } - return res; + return Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) + || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); } } From 3ce460841c5d9422636cf00571b63ee94fad46c4 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 21 Feb 2019 15:23:52 -0300 Subject: [PATCH 31/56] Bumping version to 1.5.0.17 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7ea53f6ba..4be598ac8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.16 +current_version = 1.0.5.17 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 497adf5a0..1e85fffac 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.16" + version: "1.0.5.17" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 03fe25456..42230e099 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.16", + version="1.0.5.17", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c17676f61..ba4cfde67 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.16")] +[assembly: AssemblyVersion("1.0.5.17")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index b198df56b..2d08d31ee 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.16"), + Version = new Version("1.0.5.17"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index c31118458..279d7f4f2 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.16" +__version__ = "1.0.5.17" class clrproperty(object): From 26e88d748c80c9bc585d3acc1077fe5e0530ea6f Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 03:20:44 +0800 Subject: [PATCH 32/56] Finalizer for PyObject --- src/embed_tests/TestFinalizer.cs | 143 +++++++++++++++++++++++++++++++ src/runtime/delegatemanager.cs | 29 ++++--- src/runtime/finalizer.cs | 117 +++++++++++++++++++++++++ src/runtime/pyobject.cs | 14 +-- src/runtime/pyscope.cs | 12 +-- src/runtime/pythonexception.cs | 24 +++--- src/runtime/runtime.cs | 11 +++ 7 files changed, 319 insertions(+), 31 deletions(-) create mode 100644 src/embed_tests/TestFinalizer.cs create mode 100644 src/runtime/finalizer.cs diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..03e2f1be5 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,143 @@ +using NUnit.Framework; +using Python.Runtime; +using System; +using System.Threading; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + private string _PYTHONMALLOC = string.Empty; + + [SetUp] + public void SetUp() + { + try + { + _PYTHONMALLOC = Environment.GetEnvironmentVariable("PYTHONMALLOC"); + } + catch (ArgumentNullException) + { + _PYTHONMALLOC = string.Empty; + } + Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); + PythonEngine.Initialize(); + } + + [TearDown] + public void TearDown() + { + PythonEngine.Shutdown(); + if (string.IsNullOrEmpty(_PYTHONMALLOC)) + { + Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); + } + } + + private static void FullGCCollect() + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); + GC.WaitForFullGCComplete(); + GC.WaitForPendingFinalizers(); + } + + [Test] + public void CollectBasicObject() + { + int thId = Thread.CurrentThread.ManagedThreadId; + Finalizer.Instance.Threshold = 1; + bool called = false; + EventHandler handler = (s, e) => + { + Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId); + Assert.GreaterOrEqual(e.ObjectCount, 1); + called = true; + }; + Finalizer.Instance.CollectOnce += handler; + FullGCCollect(); + PyLong obj = new PyLong(1024); + + WeakReference shortWeak = new WeakReference(obj); + WeakReference longWeak = new WeakReference(obj, true); + obj = null; + FullGCCollect(); + // The object has been resurrected + Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); + + Assert.IsFalse(called); + var garbage = Finalizer.Instance.GetCollectedObjects(); + // FIXME: If make some query for garbage, + // the above case will failed Assert.IsFalse(shortWeak.IsAlive) + //Assert.IsTrue(garbage.All(T => T.IsAlive)); + + Finalizer.Instance.CallPendingFinalizers(); + Assert.IsTrue(called); + + FullGCCollect(); + //Assert.IsFalse(garbage.All(T => T.IsAlive)); + + Assert.IsNull(longWeak.Target); + + Finalizer.Instance.CollectOnce -= handler; + } + + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) + { + // Must larger than 512 bytes make sure Python use + string str = new string('1', 1024); + Finalizer.Instance.Enable = true; + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Enable = enbale; + + // Estimate unmanaged memory size + long before = Environment.WorkingSet - GC.GetTotalMemory(true); + for (int i = 0; i < 10000; i++) + { + // Memory will leak when disable Finalizer + new PyString(str); + } + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + if (enbale) + { + Finalizer.Instance.CallPendingFinalizers(); + } + + FullGCCollect(); + FullGCCollect(); + long after = Environment.WorkingSet - GC.GetTotalMemory(true); + return after - before; + + } + + /// + /// Because of two vms both have their memory manager, + /// this test only prove the finalizer has take effect. + /// + [Test] + [Ignore("Too many uncertainties, only manual on when debugging")] + public void SimpleTestMemory() + { + bool oldState = Finalizer.Instance.Enable; + try + { + using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyObject pyCollect = gcModule.GetAttr("collect")) + { + long span1 = CompareWithFinalizerOn(pyCollect, false); + long span2 = CompareWithFinalizerOn(pyCollect, true); + Assert.Less(span2, span1); + } + } + finally + { + Finalizer.Instance.Enable = oldState; + } + } + } +} diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 7632816d1..706bd4bc4 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher + public class Dispatcher : IDisposable { public IntPtr target; public Type dtype; + private bool _disposed = false; + private bool _finalized = false; public Dispatcher(IntPtr target, Type dtype) { @@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + if (_finalized || _disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); + } - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) + public void Dispose() + { + if (_disposed) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); + return; } + _disposed = true; + Runtime.XDecref(target); + target = IntPtr.Zero; + dtype = null; + GC.SuppressFinalize(this); } public object Dispatch(ArrayList args) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs new file mode 100644 index 000000000..344260f97 --- /dev/null +++ b/src/runtime/finalizer.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public static readonly Finalizer Instance = new Finalizer(); + + public event EventHandler CollectOnce; + + private ConcurrentQueue _objQueue = new ConcurrentQueue(); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int PedingCall(IntPtr arg); + private PedingCall _collectAction; + private bool _pending = false; + private readonly object _collectingLock = new object(); + public int Threshold { get; set; } + public bool Enable { get; set; } + + private Finalizer() + { + Enable = true; + Threshold = 200; + _collectAction = OnCollect; + } + + public void CallPendingFinalizers() + { + if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + { + throw new Exception("PendingCall should execute in main Python thread"); + } + Runtime.Py_MakePendingCalls(); + } + + public List GetCollectedObjects() + { + return _objQueue.Select(T => new WeakReference(T)).ToList(); + } + + internal void AddFinalizedObject(IDisposable obj) + { + if (!Enable) + { + return; + } + if (Runtime.Py_IsInitialized() == 0) + { + // XXX: Memory will leak if a PyObject finalized after Python shutdown, + // for avoiding that case, user should call GC.Collect manual before shutdown. + return; + } + _objQueue.Enqueue(obj); + GC.ReRegisterForFinalize(obj); + if (_objQueue.Count >= Threshold) + { + Collect(); + } + } + + internal static void Shutdown() + { + Instance.DisposeAll(); + Instance.CallPendingFinalizers(); + Runtime.PyErr_Clear(); + } + + private void Collect() + { + lock (_collectingLock) + { + if (_pending) + { + return; + } + _pending = true; + } + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); + if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) + { + // Full queue, append next time + _pending = false; + } + } + + private int OnCollect(IntPtr arg) + { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); + DisposeAll(); + _pending = false; + return 0; + } + + private void DisposeAll() + { + IDisposable obj; + while (_objQueue.TryDequeue(out obj)) + { + obj.Dispose(); + } + } + } +} diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 35a4e6fa9..22f6d3e88 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -17,6 +17,7 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable { protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; + private bool _finalized = false; /// /// PyObject Constructor @@ -41,14 +42,15 @@ protected PyObject() // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. - ~PyObject() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + // Prevent a infinity loop by calling GC.WaitForPendingFinalizers + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 67f93c6e2..32d9626bd 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable internal readonly IntPtr variables; private bool _isDisposed; + private bool _finalized = false; /// /// The Manager this scope associated with. @@ -527,11 +528,12 @@ public void Dispose() ~PyScope() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || _isDisposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ded7fbeb5..1b166854a 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception + public class PythonException : System.Exception, IDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -15,6 +15,7 @@ public class PythonException : System.Exception private string _message = ""; private string _pythonTypeName = ""; private bool disposed = false; + private bool _finalized = false; public PythonException() { @@ -45,11 +46,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + using (PyObject tb_module = PythonEngine.ImportModule("traceback")) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + Runtime.XIncref(_pyTB); + using (var pyTB = new PyObject(_pyTB)) + { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } } PythonEngine.ReleaseLock(gs); @@ -60,11 +63,12 @@ public PythonException() ~PythonException() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d1183f68e..85346790d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Threading; namespace Python.Runtime { @@ -198,6 +199,8 @@ public class Runtime internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; + public static int MainManagedThreadId { get; internal set; } + /// /// Encoding to use to convert Unicode to/from Managed to Native /// @@ -211,6 +214,7 @@ internal static void Initialize() if (Py_IsInitialized() == 0) { Py_Initialize(); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } if (PyEval_ThreadsInitialized() == 0) @@ -358,6 +362,7 @@ internal static void Initialize() internal static void Shutdown() { + Finalizer.Shutdown(); AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); @@ -1668,5 +1673,11 @@ internal static bool PyObjectType_TypeCheck(IntPtr type, IntPtr tp) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Function(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_MakePendingCalls(); } } From fccd1d874f6ade3490cb8597ee42d0dc526e7f86 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 15:40:14 +0800 Subject: [PATCH 33/56] Avoid test interdependency --- src/embed_tests/TestPyScope.cs | 101 ++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 49c15a3a1..21c0d2b3f 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -293,24 +293,27 @@ public void TestImportScopeByName() [Test] public void TestVariables() { - (ps.Variables() as dynamic)["ee"] = new PyInt(200); - var a0 = ps.Get("ee"); - Assert.AreEqual(200, a0); + using (Py.GIL()) + { + (ps.Variables() as dynamic)["ee"] = new PyInt(200); + var a0 = ps.Get("ee"); + Assert.AreEqual(200, a0); - ps.Exec("locals()['ee'] = 210"); - var a1 = ps.Get("ee"); - Assert.AreEqual(210, a1); + ps.Exec("locals()['ee'] = 210"); + var a1 = ps.Get("ee"); + Assert.AreEqual(210, a1); - ps.Exec("globals()['ee'] = 220"); - var a2 = ps.Get("ee"); - Assert.AreEqual(220, a2); + ps.Exec("globals()['ee'] = 220"); + var a2 = ps.Get("ee"); + Assert.AreEqual(220, a2); - using (var item = ps.Variables()) - { - item["ee"] = new PyInt(230); + using (var item = ps.Variables()) + { + item["ee"] = new PyInt(230); + } + var a3 = ps.Get("ee"); + Assert.AreEqual(230, a3); } - var a3 = ps.Get("ee"); - Assert.AreEqual(230, a3); } /// @@ -324,49 +327,55 @@ public void TestThread() //should be removed. dynamic _ps = ps; var ts = PythonEngine.BeginAllowThreads(); - using (Py.GIL()) - { - _ps.res = 0; - _ps.bb = 100; - _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast - ps.Exec( - "def update():\n" + - " global res, th_cnt\n" + - " res += bb + 1\n" + - " th_cnt += 1\n" - ); - } - int th_cnt = 3; - for (int i =0; i< th_cnt; i++) + try { - System.Threading.Thread th = new System.Threading.Thread(()=> + using (Py.GIL()) + { + _ps.res = 0; + _ps.bb = 100; + _ps.th_cnt = 0; + //add function to the scope + //can be call many times, more efficient than ast + ps.Exec( + "def update():\n" + + " global res, th_cnt\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" + ); + } + int th_cnt = 3; + for (int i = 0; i < th_cnt; i++) + { + System.Threading.Thread th = new System.Threading.Thread(() => + { + using (Py.GIL()) + { + //ps.GetVariable("update")(); //call the scope function dynamicly + _ps.update(); + } + }); + th.Start(); + } + //equivalent to Thread.Join, make the main thread join the GIL competition + int cnt = 0; + while (cnt != th_cnt) { using (Py.GIL()) { - //ps.GetVariable("update")(); //call the scope function dynamicly - _ps.update(); + cnt = ps.Get("th_cnt"); } - }); - th.Start(); - } - //equivalent to Thread.Join, make the main thread join the GIL competition - int cnt = 0; - while(cnt != th_cnt) - { + System.Threading.Thread.Sleep(10); + } using (Py.GIL()) { - cnt = ps.Get("th_cnt"); + var result = ps.Get("res"); + Assert.AreEqual(101 * th_cnt, result); } - System.Threading.Thread.Sleep(10); } - using (Py.GIL()) + finally { - var result = ps.Get("res"); - Assert.AreEqual(101* th_cnt, result); + PythonEngine.EndAllowThreads(ts); } - PythonEngine.EndAllowThreads(ts); } } } From 928115731e77c2cd7e5b1d484be40ae336772e55 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 15:51:39 +0800 Subject: [PATCH 34/56] Add source to .csproj --- src/embed_tests/Python.EmbeddingTest.csproj | 249 +++++++-------- src/embed_tests/TestFinalizer.cs | 6 +- src/runtime/Python.Runtime.csproj | 327 ++++++++++---------- 3 files changed, 294 insertions(+), 288 deletions(-) diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 66e8c7165..11ef2daac 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,125 +1,126 @@ - - - - Debug - AnyCPU - {4165C59D-2822-499F-A6DB-EACA4C331EB5} - Library - Python.EmbeddingTest - Python.EmbeddingTest - bin\Python.EmbeddingTest.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - 6 - true - prompt - - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Python.Runtime - - - - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + Debug + AnyCPU + {4165C59D-2822-499F-A6DB-EACA4C331EB5} + Library + Python.EmbeddingTest + Python.EmbeddingTest + bin\Python.EmbeddingTest.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + 6 + true + prompt + + + x86 + + + x64 + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + + + ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Python.Runtime + + + + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + \ No newline at end of file diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 03e2f1be5..6041e76cc 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -44,6 +44,8 @@ private static void FullGCCollect() [Test] public void CollectBasicObject() { + Assert.IsTrue(Finalizer.Instance.Enable); + int thId = Thread.CurrentThread.ManagedThreadId; Finalizer.Instance.Threshold = 1; bool called = false; @@ -62,11 +64,13 @@ public void CollectBasicObject() obj = null; FullGCCollect(); // The object has been resurrected - Assert.IsFalse(shortWeak.IsAlive); + // FIXME: Sometimes the shortWeak would get alive + //Assert.IsFalse(shortWeak.IsAlive); Assert.IsTrue(longWeak.IsAlive); Assert.IsFalse(called); var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count); // FIXME: If make some query for garbage, // the above case will failed Assert.IsFalse(shortWeak.IsAlive) //Assert.IsTrue(garbage.All(T => T.IsAlive)); diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 1fea78082..28ea6424f 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,169 +1,170 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - false - ..\pythonnet.snk - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 6 + true + false + ..\pythonnet.snk + + + - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON36;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON36;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON36;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON36;UCS2;TRACE;DEBUG - false - full - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + --> + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON36;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON36;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS2;TRACE;DEBUG + false + full + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + \ No newline at end of file From 481cb4f15bca461bca86fa84b96569471636d815 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 16:11:05 +0800 Subject: [PATCH 35/56] Make sure recover the environment --- src/embed_tests/TestFinalizer.cs | 55 ++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 6041e76cc..7cf3b7309 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -22,6 +22,7 @@ public void SetUp() } Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); PythonEngine.Initialize(); + Exceptions.Clear(); } [TearDown] @@ -56,34 +57,40 @@ public void CollectBasicObject() called = true; }; Finalizer.Instance.CollectOnce += handler; - FullGCCollect(); - PyLong obj = new PyLong(1024); - - WeakReference shortWeak = new WeakReference(obj); - WeakReference longWeak = new WeakReference(obj, true); - obj = null; - FullGCCollect(); - // The object has been resurrected - // FIXME: Sometimes the shortWeak would get alive - //Assert.IsFalse(shortWeak.IsAlive); - Assert.IsTrue(longWeak.IsAlive); - - Assert.IsFalse(called); - var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.NotZero(garbage.Count); - // FIXME: If make some query for garbage, - // the above case will failed Assert.IsFalse(shortWeak.IsAlive) - //Assert.IsTrue(garbage.All(T => T.IsAlive)); + try + { + FullGCCollect(); + PyLong obj = new PyLong(1024); + + WeakReference shortWeak = new WeakReference(obj); + WeakReference longWeak = new WeakReference(obj, true); + obj = null; + FullGCCollect(); + // The object has been resurrected + // FIXME: Sometimes the shortWeak would get alive + //Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); + + Assert.IsFalse(called); + var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count); + // FIXME: If make some query for garbage, + // the above case will failed Assert.IsFalse(shortWeak.IsAlive) + //Assert.IsTrue(garbage.All(T => T.IsAlive)); - Finalizer.Instance.CallPendingFinalizers(); - Assert.IsTrue(called); + Finalizer.Instance.CallPendingFinalizers(); + Assert.IsTrue(called); - FullGCCollect(); - //Assert.IsFalse(garbage.All(T => T.IsAlive)); + FullGCCollect(); + //Assert.IsFalse(garbage.All(T => T.IsAlive)); - Assert.IsNull(longWeak.Target); + Assert.IsNull(longWeak.Target); + } + finally + { + Finalizer.Instance.CollectOnce -= handler; + } - Finalizer.Instance.CollectOnce -= handler; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) From 1959a6a87c091e7aa9eacc31733e149134ffb8b2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 16:52:34 +0800 Subject: [PATCH 36/56] Add StackTrace of C# exception --- src/runtime/pythonexception.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 1b166854a..4b4d8b107 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -136,7 +136,7 @@ public override string Message /// public override string StackTrace { - get { return _tb; } + get { return _tb + base.StackTrace; } } /// From e7b8635731eab8ba0fa928536b46abc2f808d9a1 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 20:42:14 +0800 Subject: [PATCH 37/56] Clean up the test and interface --- src/embed_tests/TestFinalizer.cs | 55 +++++++++++++++++--------------- src/runtime/finalizer.cs | 29 +++++++++++------ 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 7cf3b7309..d9e9729fb 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -1,6 +1,7 @@ using NUnit.Framework; using Python.Runtime; using System; +using System.Linq; using System.Threading; namespace Python.EmbeddingTest @@ -8,6 +9,7 @@ namespace Python.EmbeddingTest public class TestFinalizer { private string _PYTHONMALLOC = string.Empty; + private int _oldThreshold; [SetUp] public void SetUp() @@ -21,6 +23,7 @@ public void SetUp() _PYTHONMALLOC = string.Empty; } Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); + _oldThreshold = Finalizer.Instance.Threshold; PythonEngine.Initialize(); Exceptions.Clear(); } @@ -28,6 +31,7 @@ public void SetUp() [TearDown] public void TearDown() { + Finalizer.Instance.Threshold = _oldThreshold; PythonEngine.Shutdown(); if (string.IsNullOrEmpty(_PYTHONMALLOC)) { @@ -56,41 +60,42 @@ public void CollectBasicObject() Assert.GreaterOrEqual(e.ObjectCount, 1); called = true; }; - Finalizer.Instance.CollectOnce += handler; - try - { - FullGCCollect(); - PyLong obj = new PyLong(1024); - WeakReference shortWeak = new WeakReference(obj); - WeakReference longWeak = new WeakReference(obj, true); - obj = null; - FullGCCollect(); - // The object has been resurrected - // FIXME: Sometimes the shortWeak would get alive - //Assert.IsFalse(shortWeak.IsAlive); - Assert.IsTrue(longWeak.IsAlive); + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + } + FullGCCollect(); + // The object has been resurrected + Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); - Assert.IsFalse(called); + { var garbage = Finalizer.Instance.GetCollectedObjects(); Assert.NotZero(garbage.Count); - // FIXME: If make some query for garbage, - // the above case will failed Assert.IsFalse(shortWeak.IsAlive) - //Assert.IsTrue(garbage.All(T => T.IsAlive)); + Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); + } + Assert.IsFalse(called); + Finalizer.Instance.CollectOnce += handler; + try + { Finalizer.Instance.CallPendingFinalizers(); - Assert.IsTrue(called); - - FullGCCollect(); - //Assert.IsFalse(garbage.All(T => T.IsAlive)); - - Assert.IsNull(longWeak.Target); } finally { Finalizer.Instance.CollectOnce -= handler; } + Assert.IsTrue(called); + } + private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + { + PyLong obj = new PyLong(1024); + shortWeak = new WeakReference(obj); + longWeak = new WeakReference(obj, true); + obj = null; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) @@ -101,7 +106,7 @@ private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) FullGCCollect(); FullGCCollect(); pyCollect.Invoke(); - Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Collect(); Finalizer.Instance.Enable = enbale; // Estimate unmanaged memory size @@ -116,7 +121,7 @@ private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) pyCollect.Invoke(); if (enbale) { - Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Collect(); } FullGCCollect(); diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 344260f97..64faaf5b4 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -22,7 +22,8 @@ public class CollectArgs : EventArgs [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate int PedingCall(IntPtr arg); - private PedingCall _collectAction; + private readonly PedingCall _collectAction; + private bool _pending = false; private readonly object _collectingLock = new object(); public int Threshold { get; set; } @@ -32,7 +33,7 @@ private Finalizer() { Enable = true; Threshold = 200; - _collectAction = OnCollect; + _collectAction = OnPendingCollect; } public void CallPendingFinalizers() @@ -44,6 +45,14 @@ public void CallPendingFinalizers() Runtime.Py_MakePendingCalls(); } + public void Collect() + { + using (var gilState = new Py.GILState()) + { + DisposeAll(); + } + } + public List GetCollectedObjects() { return _objQueue.Select(T => new WeakReference(T)).ToList(); @@ -65,7 +74,7 @@ internal void AddFinalizedObject(IDisposable obj) GC.ReRegisterForFinalize(obj); if (_objQueue.Count >= Threshold) { - Collect(); + AddPendingCollect(); } } @@ -76,7 +85,7 @@ internal static void Shutdown() Runtime.PyErr_Clear(); } - private void Collect() + private void AddPendingCollect() { lock (_collectingLock) { @@ -94,19 +103,19 @@ private void Collect() } } - private int OnCollect(IntPtr arg) + private int OnPendingCollect(IntPtr arg) { - CollectOnce?.Invoke(this, new CollectArgs() - { - ObjectCount = _objQueue.Count - }); - DisposeAll(); + Collect(); _pending = false; return 0; } private void DisposeAll() { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); IDisposable obj; while (_objQueue.TryDequeue(out obj)) { From c51cebdac184e9403f198aa271907a343b1ff3dd Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 25 Jun 2018 00:09:31 +0800 Subject: [PATCH 38/56] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc668b0c..f9db05d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now. ### Changed +- PythonException included C# call stack ### Fixed From 833d36777c4dc4cc27f9be851755bbb154e1c58e Mon Sep 17 00:00:00 2001 From: civilx64 <26513472+civilx64@users.noreply.github.com> Date: Sun, 1 Jul 2018 14:02:53 -0400 Subject: [PATCH 39/56] Add links to issues and PRs in CHANGELOG From 9aa9b1a9d352141b5bf2f7b3a904558b2af855ff Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 25 Jun 2018 00:22:23 +0800 Subject: [PATCH 40/56] Mono doesn't have GC.WaitForFullGCComplete --- src/embed_tests/TestFinalizer.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index d9e9729fb..985ce3b35 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -20,7 +20,7 @@ public void SetUp() } catch (ArgumentNullException) { - _PYTHONMALLOC = string.Empty; + _PYTHONMALLOC = null; } Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); _oldThreshold = Finalizer.Instance.Threshold; @@ -33,16 +33,12 @@ public void TearDown() { Finalizer.Instance.Threshold = _oldThreshold; PythonEngine.Shutdown(); - if (string.IsNullOrEmpty(_PYTHONMALLOC)) - { - Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); - } + Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); } private static void FullGCCollect() { GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); - GC.WaitForFullGCComplete(); GC.WaitForPendingFinalizers(); } From 338dcc184ae3876623445fc833b79e8fe0bd2f98 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 25 Jun 2018 16:52:16 +0800 Subject: [PATCH 41/56] Fixed PythonException leak --- src/runtime/finalizer.cs | 2 +- src/runtime/pythonexception.cs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 64faaf5b4..2f4549256 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -105,7 +105,7 @@ private void AddPendingCollect() private int OnPendingCollect(IntPtr arg) { - Collect(); + DisposeAll(); _pending = false; return 0; } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 4b4d8b107..52438b386 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -21,9 +21,6 @@ public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; From ef832a33435f7d1ff77a394bee794e035b2e548e Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 10 Jul 2018 15:15:50 +0800 Subject: [PATCH 42/56] Fixed nPython.exe crash on Shutdown --- src/runtime/finalizer.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 2f4549256..ba1064387 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -80,6 +80,11 @@ internal void AddFinalizedObject(IDisposable obj) internal static void Shutdown() { + if (Runtime.Py_IsInitialized() == 0) + { + Instance._objQueue = new ConcurrentQueue(); + return; + } Instance.DisposeAll(); Instance.CallPendingFinalizers(); Runtime.PyErr_Clear(); From 3a9be30e31f1e343a1e6283fb85fd55a06a2d679 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 4 Aug 2018 16:19:09 +0800 Subject: [PATCH 43/56] Add error handler Synchronized fixed --- src/embed_tests/TestFinalizer.cs | 54 ++++++++++++++++++++++++++++++++ src/runtime/finalizer.cs | 41 ++++++++++++++++++------ src/runtime/runtime.cs | 4 +-- 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 985ce3b35..f8d8b6548 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -151,5 +151,59 @@ public void SimpleTestMemory() Finalizer.Instance.Enable = oldState; } } + + class MyPyObject : PyObject + { + public MyPyObject(IntPtr op) : base(op) + { + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + GC.SuppressFinalize(this); + throw new Exception("MyPyObject"); + } + internal static void CreateMyPyObject(IntPtr op) + { + Runtime.Runtime.XIncref(op); + new MyPyObject(op); + } + } + + [Test] + public void ErrorHandling() + { + bool called = false; + EventHandler handleFunc = (sender, args) => + { + called = true; + Assert.AreEqual(args.Error.Message, "MyPyObject"); + }; + Finalizer.Instance.Threshold = 1; + Finalizer.Instance.ErrorHandler += handleFunc; + try + { + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + var obj = (PyLong)longWeak.Target; + IntPtr handle = obj.Handle; + shortWeak = null; + longWeak = null; + MyPyObject.CreateMyPyObject(handle); + obj.Dispose(); + obj = null; + } + FullGCCollect(); + Finalizer.Instance.Collect(); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.ErrorHandler -= handleFunc; + } + } } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index ba1064387..0fc36768f 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -14,15 +14,21 @@ public class CollectArgs : EventArgs public int ObjectCount { get; set; } } + public class ErrorArgs : EventArgs + { + public Exception Error { get; set; } + } + public static readonly Finalizer Instance = new Finalizer(); public event EventHandler CollectOnce; + public event EventHandler ErrorHandler; private ConcurrentQueue _objQueue = new ConcurrentQueue(); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate int PedingCall(IntPtr arg); - private readonly PedingCall _collectAction; + private delegate int PendingCall(IntPtr arg); + private readonly PendingCall _collectAction; private bool _pending = false; private readonly object _collectingLock = new object(); @@ -87,11 +93,14 @@ internal static void Shutdown() } Instance.DisposeAll(); Instance.CallPendingFinalizers(); - Runtime.PyErr_Clear(); } private void AddPendingCollect() { + if (_pending) + { + return; + } lock (_collectingLock) { if (_pending) @@ -99,12 +108,12 @@ private void AddPendingCollect() return; } _pending = true; - } - IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) - { - // Full queue, append next time - _pending = false; + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); + if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) + { + // Full queue, append next time + _pending = false; + } } } @@ -124,7 +133,19 @@ private void DisposeAll() IDisposable obj; while (_objQueue.TryDequeue(out obj)) { - obj.Dispose(); + try + { + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 85346790d..9ff83753f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -199,7 +199,7 @@ public class Runtime internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; - public static int MainManagedThreadId { get; internal set; } + public static int MainManagedThreadId { get; private set; } /// /// Encoding to use to convert Unicode to/from Managed to Native @@ -362,10 +362,10 @@ internal static void Initialize() internal static void Shutdown() { - Finalizer.Shutdown(); AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); + Finalizer.Shutdown(); Py_Finalize(); } From eb569f2db9bcd9ee68545dad68ad7a6208119fe1 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 6 Sep 2018 15:06:36 +0800 Subject: [PATCH 44/56] Make collect callback without JIT --- src/runtime/finalizer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 0fc36768f..e6efab7c5 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -117,10 +117,10 @@ private void AddPendingCollect() } } - private int OnPendingCollect(IntPtr arg) + private static int OnPendingCollect(IntPtr arg) { - DisposeAll(); - _pending = false; + Instance.DisposeAll(); + Instance._pending = false; return 0; } From 994449ec1b8fde61b0768331e973a1741c66c690 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 24 Oct 2018 03:53:09 +0800 Subject: [PATCH 45/56] Add pending marker --- src/runtime/finalizer.cs | 66 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index e6efab7c5..9164d68a1 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -24,16 +25,23 @@ public class ErrorArgs : EventArgs public event EventHandler CollectOnce; public event EventHandler ErrorHandler; - private ConcurrentQueue _objQueue = new ConcurrentQueue(); + public int Threshold { get; set; } + public bool Enable { get; set; } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + struct PendingArgs + { + public bool cancelled; + } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate int PendingCall(IntPtr arg); private readonly PendingCall _collectAction; + private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); - public int Threshold { get; set; } - public bool Enable { get; set; } + private IntPtr _pendingArgs; private Finalizer() { @@ -92,6 +100,23 @@ internal static void Shutdown() return; } Instance.DisposeAll(); + if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + { + if (Instance._pendingArgs == IntPtr.Zero) + { + Instance.ResetPending(); + return; + } + // Not in main thread just cancel the pending operation to avoid error in different domain + // It will make a memory leak + unsafe + { + PendingArgs* args = (PendingArgs*)Instance._pendingArgs; + args->cancelled = true; + } + Instance.ResetPending(); + return; + } Instance.CallPendingFinalizers(); } @@ -108,8 +133,12 @@ private void AddPendingCollect() return; } _pending = true; + var args = new PendingArgs() { cancelled = false }; + IntPtr p = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); + Marshal.StructureToPtr(args, p, false); + _pendingArgs = p; IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) + if (Runtime.Py_AddPendingCall(func, p) != 0) { // Full queue, append next time _pending = false; @@ -119,8 +148,24 @@ private void AddPendingCollect() private static int OnPendingCollect(IntPtr arg) { - Instance.DisposeAll(); - Instance._pending = false; + Debug.Assert(arg == Instance._pendingArgs); + try + { + unsafe + { + PendingArgs* pendingArgs = (PendingArgs*)arg; + if (pendingArgs->cancelled) + { + return 0; + } + } + Instance.DisposeAll(); + } + finally + { + Instance.ResetPending(); + Marshal.FreeHGlobal(arg); + } return 0; } @@ -148,5 +193,14 @@ private void DisposeAll() } } } + + private void ResetPending() + { + lock (_collectingLock) + { + _pending = false; + _pendingArgs = IntPtr.Zero; + } + } } } From fc53d947d0cace0f72e6bf6af8f65c891646d852 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 26 Oct 2018 03:09:37 +0800 Subject: [PATCH 46/56] Remove PYTHONMALLOC setting --- src/embed_tests/TestFinalizer.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index f8d8b6548..3f09668a4 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -14,15 +14,6 @@ public class TestFinalizer [SetUp] public void SetUp() { - try - { - _PYTHONMALLOC = Environment.GetEnvironmentVariable("PYTHONMALLOC"); - } - catch (ArgumentNullException) - { - _PYTHONMALLOC = null; - } - Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); _oldThreshold = Finalizer.Instance.Threshold; PythonEngine.Initialize(); Exceptions.Clear(); @@ -33,7 +24,6 @@ public void TearDown() { Finalizer.Instance.Threshold = _oldThreshold; PythonEngine.Shutdown(); - Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); } private static void FullGCCollect() From 8eb35914efad5607a8701887bf92e49c47644a1e Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 23 Nov 2018 01:52:05 +0800 Subject: [PATCH 47/56] Fix ref count error --- src/embed_tests/TestPyAnsiString.cs | 1 + src/embed_tests/TestPyFloat.cs | 1 + src/embed_tests/TestPyString.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index f2c85a77f..94e7026c7 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -25,6 +25,7 @@ public void Dispose() public void IntPtrCtor() { var i = new PyFloat(1); + Runtime.Runtime.XIncref(i.Handle); var ii = new PyFloat(i.Handle); Assert.AreEqual(i.Handle, ii.Handle); } diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); From 361022034989257bd85fca22cbca694d1900bd52 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 23 Nov 2018 03:37:18 +0800 Subject: [PATCH 48/56] Add ref count check for helping discover the bugs of decref too much --- src/embed_tests/TestFinalizer.cs | 42 ++++++- src/runtime/Python.Runtime.15.csproj | 8 +- src/runtime/delegatemanager.cs | 7 +- src/runtime/finalizer.cs | 157 ++++++++++++++++++++++++--- src/runtime/pyobject.cs | 30 ++++- src/runtime/pyscope.cs | 7 +- src/runtime/pythonexception.cs | 7 +- 7 files changed, 233 insertions(+), 25 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 3f09668a4..1b7faf084 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -8,7 +8,6 @@ namespace Python.EmbeddingTest { public class TestFinalizer { - private string _PYTHONMALLOC = string.Empty; private int _oldThreshold; [SetUp] @@ -195,5 +194,46 @@ public void ErrorHandling() Finalizer.Instance.ErrorHandler -= handleFunc; } } + + [Test] + public void ValidateRefCount() + { + if (!Finalizer.Instance.RefCountValidationEnabled) + { + Assert.Pass("Only run with FINALIZER_CHECK"); + } + IntPtr ptr = IntPtr.Zero; + bool called = false; + Finalizer.IncorrectRefCntHandler handler = (s, e) => + { + called = true; + Assert.AreEqual(ptr, e.Handle); + Assert.AreEqual(2, e.ImpactedObjects.Count); + // Fix for this test, don't do this on general environment + Runtime.Runtime.XIncref(e.Handle); + return false; + }; + Finalizer.Instance.IncorrectRefCntResovler += handler; + try + { + ptr = CreateStringGarbage(); + FullGCCollect(); + Assert.Throws(() => Finalizer.Instance.Collect()); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.IncorrectRefCntResovler -= handler; + } + } + + private static IntPtr CreateStringGarbage() + { + PyString s1 = new PyString("test_string"); + // s2 steal a reference from s1 + PyString s2 = new PyString(s1.Handle); + return s1.Handle; + } + } } diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index cfde0a127..2497844bb 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -68,10 +68,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants) @@ -80,10 +80,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 706bd4bc4..bd8f1ee4c 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,7 +181,7 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher : IDisposable + public class Dispatcher : IPyDisposable { public IntPtr target; public Type dtype; @@ -276,6 +276,11 @@ public object TrueDispatch(ArrayList args) Runtime.XDecref(op); return result; } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { target }; + } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 9164d68a1..80519845a 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -38,11 +38,48 @@ struct PendingArgs private delegate int PendingCall(IntPtr arg); private readonly PendingCall _collectAction; - private ConcurrentQueue _objQueue = new ConcurrentQueue(); + private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); private IntPtr _pendingArgs; + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + public bool RefCountValidationEnabled { get; set; } = true; +#else + public readonly bool RefCountValidationEnabled = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + public class IncorrectFinalizeArgs : EventArgs + { + public IntPtr Handle { get; internal set; } + public ICollection ImpactedObjects { get; internal set; } + } + + public class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + private string _message; + public override string Message => _message; + + public IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + string name = Runtime.GetManagedString(pyname); + Runtime.XDecref(pyname); + _message = $"{name} may has a incorrect ref count"; + } + } + + public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + public event IncorrectRefCntHandler IncorrectRefCntResovler; + public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + private Finalizer() { Enable = true; @@ -72,7 +109,7 @@ public List GetCollectedObjects() return _objQueue.Select(T => new WeakReference(T)).ToList(); } - internal void AddFinalizedObject(IDisposable obj) + internal void AddFinalizedObject(IPyDisposable obj) { if (!Enable) { @@ -84,7 +121,12 @@ internal void AddFinalizedObject(IDisposable obj) // for avoiding that case, user should call GC.Collect manual before shutdown. return; } - _objQueue.Enqueue(obj); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + _objQueue.Enqueue(obj); + } GC.ReRegisterForFinalize(obj); if (_objQueue.Count >= Threshold) { @@ -96,7 +138,7 @@ internal static void Shutdown() { if (Runtime.Py_IsInitialized() == 0) { - Instance._objQueue = new ConcurrentQueue(); + Instance._objQueue = new ConcurrentQueue(); return; } Instance.DisposeAll(); @@ -175,21 +217,29 @@ private void DisposeAll() { ObjectCount = _objQueue.Count }); - IDisposable obj; - while (_objQueue.TryDequeue(out obj)) +#if FINALIZER_CHECK + lock (_queueLock) +#endif { - try - { - obj.Dispose(); - Runtime.CheckExceptionOccurred(); - } - catch (Exception e) +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) { - // We should not bother the main thread - ErrorHandler?.Invoke(this, new ErrorArgs() + try + { + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) { - Error = e - }); + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } } } } @@ -202,5 +252,80 @@ private void ResetPending() _pendingArgs = IntPtr.Zero; } } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + IntPtr[] handles = obj.GetTrackedHandles(); + foreach (var handle in handles) + { + if (handle == IntPtr.Zero) + { + continue; + } + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResovler != null) + { + var funcList = IncorrectRefCntResovler.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 22f6d3e88..42ba1ff39 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1,11 +1,17 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Dynamic; using System.Linq.Expressions; namespace Python.Runtime { + public interface IPyDisposable : IDisposable + { + IntPtr[] GetTrackedHandles(); + } + /// /// Represents a generic Python object. The methods of this class are /// generally equivalent to the Python "abstract object API". See @@ -13,8 +19,15 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IDisposable + public class PyObject : DynamicObject, IEnumerable, IPyDisposable { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; private set; } +#endif + protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; private bool _finalized = false; @@ -31,6 +44,9 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable public PyObject(IntPtr ptr) { obj = ptr; +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Protected default constructor to allow subclasses to manage @@ -38,12 +54,19 @@ public PyObject(IntPtr ptr) protected PyObject() { +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. ~PyObject() { + if (obj == IntPtr.Zero) + { + return; + } if (_finalized || disposed) { return; @@ -176,6 +199,11 @@ public void UnsafeDispose() GC.SuppressFinalize(this); } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } + /// /// GetPythonType Method /// diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 32d9626bd..f5449f30c 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,7 +22,7 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : DynamicObject, IPyDisposable { public readonly string Name; @@ -526,6 +526,11 @@ public void Dispose() this.OnDispose?.Invoke(this); } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } + ~PyScope() { if (_finalized || _isDisposed) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 52438b386..295a63b3d 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception, IDisposable + public class PythonException : System.Exception, IPyDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -174,6 +174,11 @@ public void Dispose() } } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { _pyType, _pyValue, _pyTB }; + } + /// /// Matches Method /// From e4fd72e64212049f02666072ee6f628404029452 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 25 Nov 2018 23:25:26 +0800 Subject: [PATCH 49/56] Fix ref count error --- src/embed_tests/TestPyInt.cs | 2 ++ src/embed_tests/TestPyLong.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 4117336d8..005ab466d 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -86,6 +86,7 @@ public void TestCtorSByte() public void TestCtorPtr() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -94,6 +95,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..3c155f315 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -110,6 +111,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i); Assert.AreEqual(5, a.ToInt32()); } From 0ed8bee39f3b0715e8fe7fb96bf161ce9ec00f4d Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 26 Nov 2018 17:45:11 +0800 Subject: [PATCH 50/56] typo error --- src/embed_tests/TestFinalizer.cs | 4 ++-- src/runtime/finalizer.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 1b7faf084..bb90c92cf 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -213,7 +213,7 @@ public void ValidateRefCount() Runtime.Runtime.XIncref(e.Handle); return false; }; - Finalizer.Instance.IncorrectRefCntResovler += handler; + Finalizer.Instance.IncorrectRefCntResolver += handler; try { ptr = CreateStringGarbage(); @@ -223,7 +223,7 @@ public void ValidateRefCount() } finally { - Finalizer.Instance.IncorrectRefCntResovler -= handler; + Finalizer.Instance.IncorrectRefCntResolver -= handler; } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 80519845a..a94f7be31 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -75,7 +75,7 @@ public IncorrectRefCountException(IntPtr ptr) } public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); - public event IncorrectRefCntHandler IncorrectRefCntResovler; + public event IncorrectRefCntHandler IncorrectRefCntResolver; public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; #endregion @@ -304,9 +304,9 @@ private void ValidateRefCount() ImpactedObjects = indexer[handle] }; bool handled = false; - if (IncorrectRefCntResovler != null) + if (IncorrectRefCntResolver != null) { - var funcList = IncorrectRefCntResovler.GetInvocationList(); + var funcList = IncorrectRefCntResolver.GetInvocationList(); foreach (IncorrectRefCntHandler func in funcList) { if (func(this, args)) From 2c94c9040c3f59e8cb48c9fbf7ee1b63db68c672 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 27 Mar 2019 17:43:16 -0300 Subject: [PATCH 51/56] Version bump 1.0.5.18 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4be598ac8..fe2774f48 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.17 +current_version = 1.0.5.18 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 1e85fffac..20d1d67f5 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.17" + version: "1.0.5.18" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 42230e099..5ed496e1f 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.17", + version="1.0.5.18", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index ba4cfde67..845ff8f0d 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.17")] +[assembly: AssemblyVersion("1.0.5.18")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 2d08d31ee..6403a0563 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.17"), + Version = new Version("1.0.5.18"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 279d7f4f2..610186ecb 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.17" +__version__ = "1.0.5.18" class clrproperty(object): From 56a8e2eeb126c13e25982d66d5f33b8ec50d5efd Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 2 Apr 2019 12:01:52 -0300 Subject: [PATCH 52/56] Fix memory leak in finalizer - `finalizer` was leaking `_pendingArgs` (global memory) when the call to `Py_AddPendingCall` was unsuccessful. - Changing `lock` for `Monitor.TryEnter` at `AddPendingCollect()` so threads don't block each other. --- src/runtime/finalizer.cs | 50 ++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index a94f7be31..bab301af8 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -41,7 +41,7 @@ struct PendingArgs private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); - private IntPtr _pendingArgs; + private IntPtr _pendingArgs = IntPtr.Zero; #region FINALIZER_CHECK @@ -128,7 +128,7 @@ internal void AddFinalizedObject(IPyDisposable obj) _objQueue.Enqueue(obj); } GC.ReRegisterForFinalize(obj); - if (_objQueue.Count >= Threshold) + if (!_pending && _objQueue.Count >= Threshold) { AddPendingCollect(); } @@ -164,26 +164,28 @@ internal static void Shutdown() private void AddPendingCollect() { - if (_pending) + if(Monitor.TryEnter(_collectingLock)) { - return; - } - lock (_collectingLock) - { - if (_pending) + try { - return; + if (!_pending) + { + _pending = true; + var args = new PendingArgs { cancelled = false }; + _pendingArgs = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); + Marshal.StructureToPtr(args, _pendingArgs, false); + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); + if (Runtime.Py_AddPendingCall(func, _pendingArgs) != 0) + { + // Full queue, append next time + FreePendingArgs(); + _pending = false; + } + } } - _pending = true; - var args = new PendingArgs() { cancelled = false }; - IntPtr p = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); - Marshal.StructureToPtr(args, p, false); - _pendingArgs = p; - IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, p) != 0) + finally { - // Full queue, append next time - _pending = false; + Monitor.Exit(_collectingLock); } } } @@ -205,8 +207,8 @@ private static int OnPendingCollect(IntPtr arg) } finally { + Instance.FreePendingArgs(); Instance.ResetPending(); - Marshal.FreeHGlobal(arg); } return 0; } @@ -244,12 +246,20 @@ private void DisposeAll() } } + private void FreePendingArgs() + { + if (_pendingArgs != IntPtr.Zero) + { + Marshal.FreeHGlobal(_pendingArgs); + _pendingArgs = IntPtr.Zero; + } + } + private void ResetPending() { lock (_collectingLock) { _pending = false; - _pendingArgs = IntPtr.Zero; } } From c9421947160f3fa2b1836ff75be99663bf1a4c2d Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 2 Apr 2019 12:07:25 -0300 Subject: [PATCH 53/56] Version bump 1.0.5.19 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fe2774f48..816220243 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.18 +current_version = 1.0.5.19 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 20d1d67f5..74fb214f9 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.18" + version: "1.0.5.19" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 5ed496e1f..72d3f9f01 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.18", + version="1.0.5.19", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 845ff8f0d..891ef7fc6 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.18")] +[assembly: AssemblyVersion("1.0.5.19")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 6403a0563..336446cb7 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.18"), + Version = new Version("1.0.5.19"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 610186ecb..6e97ba637 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.18" +__version__ = "1.0.5.19" class clrproperty(object): From 758470883d0c53c83cbbece53747ba10594b27cd Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 23 Apr 2019 16:19:26 -0300 Subject: [PATCH 54/56] Fix memory leak - finalizer - The callback set by `Runtime.Py_AddPendingCall()` was not being triggered in some cases in a multithreading environment. Replacing it with a `Task` --- src/embed_tests/TestFinalizer.cs | 12 +-- src/runtime/finalizer.cs | 136 +++++++++---------------------- 2 files changed, 44 insertions(+), 104 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index bb90c92cf..dd578becc 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -36,13 +36,12 @@ public void CollectBasicObject() { Assert.IsTrue(Finalizer.Instance.Enable); - int thId = Thread.CurrentThread.ManagedThreadId; Finalizer.Instance.Threshold = 1; bool called = false; + var objectCount = 0; EventHandler handler = (s, e) => { - Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId); - Assert.GreaterOrEqual(e.ObjectCount, 1); + objectCount = e.ObjectCount; called = true; }; @@ -73,6 +72,7 @@ public void CollectBasicObject() Finalizer.Instance.CollectOnce -= handler; } Assert.IsTrue(called); + Assert.GreaterOrEqual(objectCount, 1); } private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) @@ -85,7 +85,7 @@ private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) { - // Must larger than 512 bytes make sure Python use + // Must larger than 512 bytes make sure Python use string str = new string('1', 1024); Finalizer.Instance.Enable = true; FullGCCollect(); @@ -164,10 +164,11 @@ internal static void CreateMyPyObject(IntPtr op) public void ErrorHandling() { bool called = false; + var errorMessage = ""; EventHandler handleFunc = (sender, args) => { called = true; - Assert.AreEqual(args.Error.Message, "MyPyObject"); + errorMessage = args.Error.Message; }; Finalizer.Instance.Threshold = 1; Finalizer.Instance.ErrorHandler += handleFunc; @@ -193,6 +194,7 @@ public void ErrorHandling() { Finalizer.Instance.ErrorHandler -= handleFunc; } + Assert.AreEqual(errorMessage, "MyPyObject"); } [Test] diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index bab301af8..dd519e227 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -1,10 +1,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Tasks; namespace Python.Runtime { @@ -28,20 +27,10 @@ public class ErrorArgs : EventArgs public int Threshold { get; set; } public bool Enable { get; set; } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - struct PendingArgs - { - public bool cancelled; - } - - [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate int PendingCall(IntPtr arg); - private readonly PendingCall _collectAction; - private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); - private IntPtr _pendingArgs = IntPtr.Zero; + private Task _finalizerTask; #region FINALIZER_CHECK @@ -84,23 +73,26 @@ private Finalizer() { Enable = true; Threshold = 200; - _collectAction = OnPendingCollect; } - public void CallPendingFinalizers() + public bool CallPendingFinalizers() { - if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + if (Instance._finalizerTask != null + && !Instance._finalizerTask.IsCompleted) { - throw new Exception("PendingCall should execute in main Python thread"); + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); + return true; } - Runtime.Py_MakePendingCalls(); + return false; } public void Collect() { - using (var gilState = new Py.GILState()) + if (!Instance.CallPendingFinalizers()) { - DisposeAll(); + Instance.DisposeAll(); } } @@ -141,25 +133,10 @@ internal static void Shutdown() Instance._objQueue = new ConcurrentQueue(); return; } - Instance.DisposeAll(); - if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + if(!Instance.CallPendingFinalizers()) { - if (Instance._pendingArgs == IntPtr.Zero) - { - Instance.ResetPending(); - return; - } - // Not in main thread just cancel the pending operation to avoid error in different domain - // It will make a memory leak - unsafe - { - PendingArgs* args = (PendingArgs*)Instance._pendingArgs; - args->cancelled = true; - } - Instance.ResetPending(); - return; + Instance.DisposeAll(); } - Instance.CallPendingFinalizers(); } private void AddPendingCollect() @@ -171,16 +148,14 @@ private void AddPendingCollect() if (!_pending) { _pending = true; - var args = new PendingArgs { cancelled = false }; - _pendingArgs = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); - Marshal.StructureToPtr(args, _pendingArgs, false); - IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, _pendingArgs) != 0) + // should already be complete but just in case + _finalizerTask?.Wait(); + + _finalizerTask = Task.Factory.StartNew(() => { - // Full queue, append next time - FreePendingArgs(); + Instance.DisposeAll(); _pending = false; - } + }); } } finally @@ -190,29 +165,6 @@ private void AddPendingCollect() } } - private static int OnPendingCollect(IntPtr arg) - { - Debug.Assert(arg == Instance._pendingArgs); - try - { - unsafe - { - PendingArgs* pendingArgs = (PendingArgs*)arg; - if (pendingArgs->cancelled) - { - return 0; - } - } - Instance.DisposeAll(); - } - finally - { - Instance.FreePendingArgs(); - Instance.ResetPending(); - } - return 0; - } - private void DisposeAll() { CollectOnce?.Invoke(this, new CollectArgs() @@ -223,46 +175,32 @@ private void DisposeAll() lock (_queueLock) #endif { + using (Py.GIL()) + { #if FINALIZER_CHECK - ValidateRefCount(); + ValidateRefCount(); #endif - IPyDisposable obj; - while (_objQueue.TryDequeue(out obj)) - { - try + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) { - obj.Dispose(); - Runtime.CheckExceptionOccurred(); - } - catch (Exception e) - { - // We should not bother the main thread - ErrorHandler?.Invoke(this, new ErrorArgs() + try { - Error = e - }); + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } } } } } - private void FreePendingArgs() - { - if (_pendingArgs != IntPtr.Zero) - { - Marshal.FreeHGlobal(_pendingArgs); - _pendingArgs = IntPtr.Zero; - } - } - - private void ResetPending() - { - lock (_collectingLock) - { - _pending = false; - } - } - #if FINALIZER_CHECK private void ValidateRefCount() { From a30321c9d6a7d3a285a8eec4eec6770f1c7b6ef0 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 23 Apr 2019 17:33:15 -0300 Subject: [PATCH 55/56] Version bump 1.0.5.20 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 816220243..9fa0e95b6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.19 +current_version = 1.0.5.20 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 74fb214f9..4bf1ac6a3 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.19" + version: "1.0.5.20" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 72d3f9f01..d5107eca9 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.19", + version="1.0.5.20", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 891ef7fc6..a31d80dd8 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.19")] +[assembly: AssemblyVersion("1.0.5.20")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 336446cb7..a6dbdcc65 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.19"), + Version = new Version("1.0.5.20"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 6e97ba637..4d7487511 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.19" +__version__ = "1.0.5.20" class clrproperty(object): From b1e84442ff76d13b1864071908281604aa2497ae Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 23 Apr 2019 18:33:41 -0300 Subject: [PATCH 56/56] Fix unit test - Refactor --- src/embed_tests/TestFinalizer.cs | 8 ++++---- src/runtime/finalizer.cs | 24 +++++++++--------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index dd578becc..53838f315 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -45,6 +45,9 @@ public void CollectBasicObject() called = true; }; + Assert.IsFalse(called); + Finalizer.Instance.CollectOnce += handler; + WeakReference shortWeak; WeakReference longWeak; { @@ -60,12 +63,9 @@ public void CollectBasicObject() Assert.NotZero(garbage.Count); Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); } - - Assert.IsFalse(called); - Finalizer.Instance.CollectOnce += handler; try { - Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Collect(forceDispose: false); } finally { diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index dd519e227..948f94cb5 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -75,22 +75,19 @@ private Finalizer() Threshold = 200; } - public bool CallPendingFinalizers() + public void Collect(bool forceDispose = true) { if (Instance._finalizerTask != null && !Instance._finalizerTask.IsCompleted) { - var ts = PythonEngine.BeginAllowThreads(); - Instance._finalizerTask.Wait(); - PythonEngine.EndAllowThreads(ts); - return true; + using (Py.GIL()) + { + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); + } } - return false; - } - - public void Collect() - { - if (!Instance.CallPendingFinalizers()) + else if (forceDispose) { Instance.DisposeAll(); } @@ -133,10 +130,7 @@ internal static void Shutdown() Instance._objQueue = new ConcurrentQueue(); return; } - if(!Instance.CallPendingFinalizers()) - { - Instance.DisposeAll(); - } + Instance.Collect(forceDispose: true); } private void AddPendingCollect()