diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac1d31324..9fa0e95b6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0.dev0 +current_version = 1.0.5.20 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/.travis.yml b/.travis.yml index d059bdcde..900e207a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ matrix: - dotnet-hostfxr-2.0.0 - dotnet-runtime-2.0.0 - dotnet-sdk-2.0.0 - - python: 3.4 env: *xplat-env addons: *xplat-addons @@ -34,24 +33,9 @@ matrix: - python: 3.6 env: *xplat-env addons: *xplat-addons - - - python: 3.7 + - python: "3.7-dev" env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 + addons: *xplat-addons # --------------------- Classic builds ------------------------ - python: 2.7 @@ -68,18 +52,15 @@ matrix: - python: 3.6 env: *classic-env - - python: 3.7 + - python: "3.7-dev" + env: *classic-env + + allow_failures: + - python: "3.7-dev" + env: *xplat-env + + - python: "3.7-dev" env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono env: global: diff --git a/AUTHORS.md b/AUTHORS.md index fe2d2b172..3c39794e4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -14,7 +14,6 @@ - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) -- Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) @@ -23,12 +22,10 @@ - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) - Dave Hirschfeld ([@dhirschfeld](https://github.com/dhirschfeld)) -- David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - He-chien Tsai ([@t3476](https://github.com/t3476)) -   Ivan Cronyn ([@cronan](https://github.com/cronan)) -- Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) -   Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) @@ -42,8 +39,6 @@ - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) -- Simon Mourier ([@smourier](https://github.com/smourier)) -- Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3816cd0c..f9db05d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,35 +8,25 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ## [unreleased][] ### Added - - Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) - Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). Currently there two side-by-side build systems that produces the same output (net40) from the same sources. After a some transition time, current (mono/ msbuild 14.0) build system will be removed. - NUnit upgraded to 3.7 (eliminates travis-ci random bug) -- Added C# `PythonEngine.AddShutdownHandler` to help client code clean up on shutdown. - Added `clr.GetClrType` ([#432][i432])([#433][p433]) - Allowed passing `None` for nullable args ([#460][p460]) - Added keyword arguments based on C# syntax for calling CPython methods ([#461][p461]) - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) -- Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) -- Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now. ### Changed - -- Reattach python exception traceback information (#545) -- PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) +- PythonException included C# call stack ### Fixed -- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. - This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. ([#534][p534]) - Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py -- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), - related to unloading the Application Domain -- Fixed interop methods with Py_ssize_t. NetCoreApp 2.0 is more sensitive than net40 and requires this fix. ([#531][p531]) - Fixed crash on exit of the Python interpreter if a python class derived from a .NET class has a `__namespace__` or `__assembly__` attribute ([#481][i481]) @@ -48,8 +38,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) - Fix memory leaks due to spurious handle incrementation ([#691][i691]) -- Fix spurious assembly loading exceptions from private types ([#703][i703]) -- Fix inheritance of non-abstract base methods ([#755][i755]) ## [2.3.0][] - 2017-03-11 @@ -607,7 +595,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [1.0.0]: https://github.com/pythonnet/pythonnet/releases/tag/1.0 -[i714]: https://github.com/pythonnet/pythonnet/issues/714 [i608]: https://github.com/pythonnet/pythonnet/issues/608 [i443]: https://github.com/pythonnet/pythonnet/issues/443 [p690]: https://github.com/pythonnet/pythonnet/pull/690 @@ -698,9 +685,4 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [p225]: https://github.com/pythonnet/pythonnet/pull/225 [p78]: https://github.com/pythonnet/pythonnet/pull/78 [p163]: https://github.com/pythonnet/pythonnet/pull/163 -[p625]: https://github.com/pythonnet/pythonnet/pull/625 [i131]: https://github.com/pythonnet/pythonnet/issues/131 -[p531]: https://github.com/pythonnet/pythonnet/pull/531 -[i755]: https://github.com/pythonnet/pythonnet/pull/755 -[p534]: https://github.com/pythonnet/pythonnet/pull/534 -[i449]: https://github.com/pythonnet/pythonnet/issues/449 diff --git a/NuGet.config b/NuGet.config index 5210cd6c9..719fbc83c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,4 +1,4 @@ - + diff --git a/appveyor.yml b/appveyor.yml index ce345cbf8..6bebef490 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,19 +23,10 @@ environment: BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.7 - BUILD_OPTS: --xplat - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.4 - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.6 - - PYTHON_VERSION: 3.7 - -matrix: - allow_failures: - - PYTHON_VERSION: 3.4 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 init: # Update Environment Variables based on matrix/platform diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3641185bb..4bf1ac6a3 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "2.4.0.dev0" + version: "1.0.5.20" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 1b6f07ea6..d5107eca9 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug -VERBOSITY = "normal" # quiet, minimal, normal, detailed, diagnostic +VERBOSITY = "minimal" # quiet, minimal, normal, detailed, diagnostic is_64bits = sys.maxsize > 2**32 DEVTOOLS = "MsDev" if sys.platform == "win32" else "Mono" @@ -329,16 +329,9 @@ def _install_packages(self): self.debug_print("Updating NuGet: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) - try: - # msbuild=14 is mainly for Mono issues - cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) - except: - # when only VS 2017 is installed do not specify msbuild version - cmd = "{0} restore pythonnet.sln -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) + cmd = "{0} restore pythonnet.sln -o packages".format(nuget) + self.debug_print("Installing packages: {0}".format(cmd)) + subprocess.check_call(cmd, shell=use_shell) def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): """Return full path to one of the Microsoft build tools""" @@ -492,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0.dev0", + version="1.0.5.20", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', @@ -523,10 +516,10 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX :: Linux', 'Operating System :: MacOS :: MacOS X', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..a31d80dd8 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("1.0.5.20")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..a6dbdcc65 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("1.0.5.20"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs deleted file mode 100644 index 458ab6a99..000000000 --- a/src/embed_tests/GlobalTestsSetup.cs +++ /dev/null @@ -1,21 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; - -namespace Python.EmbeddingTest -{ - - // As the SetUpFixture, the OneTimeTearDown of this class is executed after - // all tests have run. - [SetUpFixture] - public class GlobalTestsSetup - { - [OneTimeTearDown] - public void FinalCleanup() - { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - } -} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..92d55a7e0 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -29,7 +29,6 @@ XPLAT $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP - $(DefineConstants);NETSTANDARD $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 6aa48becc..11ef2daac 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,128 +1,126 @@ - - - - Debug - AnyCPU - {4165C59D-2822-499F-A6DB-EACA4C331EB5} - Library - Python.EmbeddingTest - Python.EmbeddingTest - bin\Python.EmbeddingTest.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - 6 - true - prompt - - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Python.Runtime - - - - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {4165C59D-2822-499F-A6DB-EACA4C331EB5} + Library + Python.EmbeddingTest + Python.EmbeddingTest + bin\Python.EmbeddingTest.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + 6 + true + prompt + + + x86 + + + x64 + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + + + ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Python.Runtime + + + + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + \ No newline at end of file diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index caaec311b..346c8afdc 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using Python.Runtime; namespace Python.EmbeddingTest diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs deleted file mode 100644 index b162d4eb0..000000000 --- a/src/embed_tests/TestDomainReload.cs +++ /dev/null @@ -1,239 +0,0 @@ -using System; -using System.CodeDom.Compiler; -using System.Reflection; -using NUnit.Framework; -using Python.Runtime; - -// -// This test case is disabled on .NET Standard because it doesn't have all the -// APIs we use. We could work around that, but .NET Core doesn't implement -// domain creation, so it's not worth it. -// -// Unfortunately this means no continuous integration testing for this case. -// -#if !NETSTANDARD && !NETCOREAPP -namespace Python.EmbeddingTest -{ - class TestDomainReload - { - /// - /// Test that the python runtime can survive a C# domain reload without crashing. - /// - /// At the time this test was written, there was a very annoying - /// seemingly random crash bug when integrating pythonnet into Unity. - /// - /// The repro steps that David Lassonde, Viktoria Kovecses and - /// Benoit Hudson eventually worked out: - /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access - /// some C# data from python: C# calls python, which calls C#. - /// 2. Execute the script (e.g. make it a MenuItem and click it). - /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. - /// 4. Wait several seconds for Unity to be done recompiling and - /// reloading the C# domain. - /// 5. Make python run the gc (e.g. by calling gc.collect()). - /// - /// The reason: - /// A. In step 2, Python.Runtime registers a bunch of new types with - /// their tp_traverse slot pointing to managed code, and allocates - /// some objects of those types. - /// B. In step 4, Unity unloads the C# domain. That frees the managed - /// code. But at the time of the crash investigation, pythonnet - /// leaked the python side of the objects allocated in step 1. - /// C. In step 5, python sees some pythonnet objects in its gc list of - /// potentially-leaked objects. It calls tp_traverse on those objects. - /// But tp_traverse was freed in step 3 => CRASH. - /// - /// This test distills what's going on without needing Unity around (we'd see - /// similar behaviour if we were using pythonnet on a .NET web server that did - /// a hot reload). - /// - [Test] - public static void DomainReloadAndGC() - { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); - - // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); - - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - } - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); - } - } - } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); - } - }"; - - - /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. - /// - static Assembly BuildAssembly(string assemblyName) - { - var provider = CodeDomProvider.CreateProvider("CSharp"); - - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; - - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); - if (results.Errors.HasErrors) - { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) - { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); - } - throw new Exception(errors.ToString()); - } - else - { - return results.CompiledAssembly; - } - } - - /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. - /// - class Proxy : MarshalByRefObject - { - Assembly theAssembly = null; - - public void InitAssembly(string assemblyPath) - { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); - } - - public void RunPython() - { - Console.WriteLine("[Proxy] Entering RunPython"); - - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); - - Console.WriteLine("[Proxy] Leaving RunPython"); - } - } - - /// - /// Create a domain, run the assembly in it (the RunPython function), - /// and unload the domain. - /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) - { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); - - // Create a Proxy object in the new domain, where we want the - // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - - // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); - theProxy.RunPython(); - - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); - AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); - - // Validate that the assembly does not exist anymore - try - { - Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); - } - catch (Exception) - { - Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); - } - } - - /// - /// Resolves the assembly. Why doesn't this just work normally? - /// - static Assembly ResolveAssembly(object sender, ResolveEventArgs args) - { - var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); - - foreach (var assembly in loadedAssemblies) - { - if (assembly.FullName == args.Name) - { - return assembly; - } - } - - return null; - } - } -} -#endif diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..53838f315 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,241 @@ +using NUnit.Framework; +using Python.Runtime; +using System; +using System.Linq; +using System.Threading; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + private int _oldThreshold; + + [SetUp] + public void SetUp() + { + _oldThreshold = Finalizer.Instance.Threshold; + PythonEngine.Initialize(); + Exceptions.Clear(); + } + + [TearDown] + public void TearDown() + { + Finalizer.Instance.Threshold = _oldThreshold; + PythonEngine.Shutdown(); + } + + private static void FullGCCollect() + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); + GC.WaitForPendingFinalizers(); + } + + [Test] + public void CollectBasicObject() + { + Assert.IsTrue(Finalizer.Instance.Enable); + + Finalizer.Instance.Threshold = 1; + bool called = false; + var objectCount = 0; + EventHandler handler = (s, e) => + { + objectCount = e.ObjectCount; + called = true; + }; + + Assert.IsFalse(called); + Finalizer.Instance.CollectOnce += handler; + + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + } + FullGCCollect(); + // The object has been resurrected + Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); + + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count); + Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); + } + try + { + Finalizer.Instance.Collect(forceDispose: false); + } + finally + { + Finalizer.Instance.CollectOnce -= handler; + } + Assert.IsTrue(called); + Assert.GreaterOrEqual(objectCount, 1); + } + + private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + { + PyLong obj = new PyLong(1024); + shortWeak = new WeakReference(obj); + longWeak = new WeakReference(obj, true); + obj = null; + } + + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) + { + // Must larger than 512 bytes make sure Python use + string str = new string('1', 1024); + Finalizer.Instance.Enable = true; + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + Finalizer.Instance.Collect(); + Finalizer.Instance.Enable = enbale; + + // Estimate unmanaged memory size + long before = Environment.WorkingSet - GC.GetTotalMemory(true); + for (int i = 0; i < 10000; i++) + { + // Memory will leak when disable Finalizer + new PyString(str); + } + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + if (enbale) + { + Finalizer.Instance.Collect(); + } + + FullGCCollect(); + FullGCCollect(); + long after = Environment.WorkingSet - GC.GetTotalMemory(true); + return after - before; + + } + + /// + /// Because of two vms both have their memory manager, + /// this test only prove the finalizer has take effect. + /// + [Test] + [Ignore("Too many uncertainties, only manual on when debugging")] + public void SimpleTestMemory() + { + bool oldState = Finalizer.Instance.Enable; + try + { + using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyObject pyCollect = gcModule.GetAttr("collect")) + { + long span1 = CompareWithFinalizerOn(pyCollect, false); + long span2 = CompareWithFinalizerOn(pyCollect, true); + Assert.Less(span2, span1); + } + } + finally + { + Finalizer.Instance.Enable = oldState; + } + } + + class MyPyObject : PyObject + { + public MyPyObject(IntPtr op) : base(op) + { + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + GC.SuppressFinalize(this); + throw new Exception("MyPyObject"); + } + internal static void CreateMyPyObject(IntPtr op) + { + Runtime.Runtime.XIncref(op); + new MyPyObject(op); + } + } + + [Test] + public void ErrorHandling() + { + bool called = false; + var errorMessage = ""; + EventHandler handleFunc = (sender, args) => + { + called = true; + errorMessage = args.Error.Message; + }; + Finalizer.Instance.Threshold = 1; + Finalizer.Instance.ErrorHandler += handleFunc; + try + { + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + var obj = (PyLong)longWeak.Target; + IntPtr handle = obj.Handle; + shortWeak = null; + longWeak = null; + MyPyObject.CreateMyPyObject(handle); + obj.Dispose(); + obj = null; + } + FullGCCollect(); + Finalizer.Instance.Collect(); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.ErrorHandler -= handleFunc; + } + Assert.AreEqual(errorMessage, "MyPyObject"); + } + + [Test] + public void ValidateRefCount() + { + if (!Finalizer.Instance.RefCountValidationEnabled) + { + Assert.Pass("Only run with FINALIZER_CHECK"); + } + IntPtr ptr = IntPtr.Zero; + bool called = false; + Finalizer.IncorrectRefCntHandler handler = (s, e) => + { + called = true; + Assert.AreEqual(ptr, e.Handle); + Assert.AreEqual(2, e.ImpactedObjects.Count); + // Fix for this test, don't do this on general environment + Runtime.Runtime.XIncref(e.Handle); + return false; + }; + Finalizer.Instance.IncorrectRefCntResolver += handler; + try + { + ptr = CreateStringGarbage(); + FullGCCollect(); + Assert.Throws(() => Finalizer.Instance.Collect()); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.IncorrectRefCntResolver -= handler; + } + } + + private static IntPtr CreateStringGarbage() + { + PyString s1 = new PyString("test_string"); + // s2 steal a reference from s1 + PyString s2 = new PyString(s1.Handle); + return s1.Handle; + } + + } +} diff --git a/src/embed_tests/TestNamedArguments.cs b/src/embed_tests/TestNamedArguments.cs index 31f2ea1d2..1d7076956 100644 --- a/src/embed_tests/TestNamedArguments.cs +++ b/src/embed_tests/TestNamedArguments.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index f2c85a77f..94e7026c7 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -25,6 +25,7 @@ public void Dispose() public void IntPtrCtor() { var i = new PyFloat(1); + Runtime.Runtime.XIncref(i.Handle); var ii = new PyFloat(i.Handle); Assert.AreEqual(i.Handle, ii.Handle); } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 4117336d8..005ab466d 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -86,6 +86,7 @@ public void TestCtorSByte() public void TestCtorPtr() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -94,6 +95,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..3c155f315 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -110,6 +111,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 65ac20e9a..d794ce06e 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NUnit.Framework; diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 49c15a3a1..21c0d2b3f 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -293,24 +293,27 @@ public void TestImportScopeByName() [Test] public void TestVariables() { - (ps.Variables() as dynamic)["ee"] = new PyInt(200); - var a0 = ps.Get("ee"); - Assert.AreEqual(200, a0); + using (Py.GIL()) + { + (ps.Variables() as dynamic)["ee"] = new PyInt(200); + var a0 = ps.Get("ee"); + Assert.AreEqual(200, a0); - ps.Exec("locals()['ee'] = 210"); - var a1 = ps.Get("ee"); - Assert.AreEqual(210, a1); + ps.Exec("locals()['ee'] = 210"); + var a1 = ps.Get("ee"); + Assert.AreEqual(210, a1); - ps.Exec("globals()['ee'] = 220"); - var a2 = ps.Get("ee"); - Assert.AreEqual(220, a2); + ps.Exec("globals()['ee'] = 220"); + var a2 = ps.Get("ee"); + Assert.AreEqual(220, a2); - using (var item = ps.Variables()) - { - item["ee"] = new PyInt(230); + using (var item = ps.Variables()) + { + item["ee"] = new PyInt(230); + } + var a3 = ps.Get("ee"); + Assert.AreEqual(230, a3); } - var a3 = ps.Get("ee"); - Assert.AreEqual(230, a3); } /// @@ -324,49 +327,55 @@ public void TestThread() //should be removed. dynamic _ps = ps; var ts = PythonEngine.BeginAllowThreads(); - using (Py.GIL()) - { - _ps.res = 0; - _ps.bb = 100; - _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast - ps.Exec( - "def update():\n" + - " global res, th_cnt\n" + - " res += bb + 1\n" + - " th_cnt += 1\n" - ); - } - int th_cnt = 3; - for (int i =0; i< th_cnt; i++) + try { - System.Threading.Thread th = new System.Threading.Thread(()=> + using (Py.GIL()) + { + _ps.res = 0; + _ps.bb = 100; + _ps.th_cnt = 0; + //add function to the scope + //can be call many times, more efficient than ast + ps.Exec( + "def update():\n" + + " global res, th_cnt\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" + ); + } + int th_cnt = 3; + for (int i = 0; i < th_cnt; i++) + { + System.Threading.Thread th = new System.Threading.Thread(() => + { + using (Py.GIL()) + { + //ps.GetVariable("update")(); //call the scope function dynamicly + _ps.update(); + } + }); + th.Start(); + } + //equivalent to Thread.Join, make the main thread join the GIL competition + int cnt = 0; + while (cnt != th_cnt) { using (Py.GIL()) { - //ps.GetVariable("update")(); //call the scope function dynamicly - _ps.update(); + cnt = ps.Get("th_cnt"); } - }); - th.Start(); - } - //equivalent to Thread.Join, make the main thread join the GIL competition - int cnt = 0; - while(cnt != th_cnt) - { + System.Threading.Thread.Sleep(10); + } using (Py.GIL()) { - cnt = ps.Get("th_cnt"); + var result = ps.Get("res"); + Assert.AreEqual(101 * th_cnt, result); } - System.Threading.Thread.Sleep(10); } - using (Py.GIL()) + finally { - var result = ps.Get("res"); - Assert.AreEqual(101* th_cnt, result); + PythonEngine.EndAllowThreads(ts); } - PythonEngine.EndAllowThreads(ts); } } } diff --git a/src/embed_tests/TestPySequence.cs b/src/embed_tests/TestPySequence.cs index 7c175b1ce..1e3ebf144 100644 --- a/src/embed_tests/TestPySequence.cs +++ b/src/embed_tests/TestPySequence.cs @@ -69,8 +69,10 @@ public void TestRepeat() PyObject actual = t1.Repeat(3); Assert.AreEqual("FooFooFoo", actual.ToString()); - actual = t1.Repeat(-3); - Assert.AreEqual("", actual.ToString()); + // On 32 bit system this argument should be int, but on the 64 bit system this should be long value. + // This works on the Framework 4.0 accidentally, it should produce out of memory! + // actual = t1.Repeat(-3); + // Assert.AreEqual("", actual.ToString()); } [Test] diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index 0c4e9023f..fd3f8e662 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 243349b82..01c6ae7e3 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -109,41 +109,18 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - // Restoring valid pythonhome. - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = "/dummypath2/"; @@ -152,20 +129,11 @@ public void SetPythonHomeTwice() Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetProgramName() { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - - var programNameBackup = PythonEngine.ProgramName; - var programName = "FooBar"; PythonEngine.ProgramName = programName; @@ -173,8 +141,6 @@ public void SetProgramName() Assert.AreEqual(programName, PythonEngine.ProgramName); PythonEngine.Shutdown(); - - PythonEngine.ProgramName = programNameBackup; } [Test] @@ -190,7 +156,7 @@ public void SetPythonPath() string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); - PythonEngine.PythonPath = path; + PythonEngine.ProgramName = path; PythonEngine.Initialize(); Assert.AreEqual(path, PythonEngine.PythonPath); @@ -205,6 +171,7 @@ public void SetPythonPathExceptionOn27() Assert.Pass(); } + // Get previous path to avoid crashing Python PythonEngine.Initialize(); string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..2e0598da7 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; @@ -6,40 +6,10 @@ namespace Python.EmbeddingTest { public class TestRuntime { - [OneTimeSetUp] - public void SetUp() - { - // We needs to ensure that no any engines are running. - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - - /// - /// Test the cache of the information from the platform module. - /// - /// Test fails on platforms we haven't implemented yet. - /// - [Test] - public static void PlatformCache() - { - Runtime.Runtime.Initialize(); - - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); - - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); // In case another test left it on. Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs deleted file mode 100644 index a4ef86913..000000000 --- a/src/embed_tests/TestTypeManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; -using System.Runtime.InteropServices; - -namespace Python.EmbeddingTest -{ - class TestTypeManager - { - [SetUp] - public static void Init() - { - Runtime.Runtime.Initialize(); - } - - [TearDown] - public static void Fini() - { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - - [Test] - public static void TestNativeCode() - { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); - } - - [Test] - public static void TestMemoryMapping() - { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); - - // Allocate a read-write page. - int len = 12; - var page = mapper.MapWriteable(len); - Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Mark it read-execute. We can still read, haven't changed any values. - mapper.SetReadExec(page, len); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Test that we can't write to the protected page. - // - // We can't actually test access protection under Microsoft - // versions of .NET, because AccessViolationException is assumed to - // mean we're in a corrupted state: - // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception - // - // We can test under Mono but it throws NRE instead of AccessViolationException. - // - // We can't use compiler flags because we compile with MONO_LINUX - // while running on the Microsoft .NET Core during continuous - // integration tests. - if (System.Type.GetType ("Mono.Runtime") != null) - { - // Mono throws NRE instead of AccessViolationException for some reason. - Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } - } - } -} diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 81345cee7..d75dc01d6 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -12,23 +12,13 @@ public class DynamicTest [SetUp] public void SetUp() { - try { - _gs = Py.GIL(); - } catch (Exception e) { - Console.WriteLine($"exception in SetUp: {e}"); - throw; - } + _gs = Py.GIL(); } [TearDown] public void Dispose() { - try { - _gs.Dispose(); - } catch(Exception e) { - Console.WriteLine($"exception in TearDown: {e}"); - throw; - } + _gs.Dispose(); } /// @@ -128,11 +118,10 @@ public void PassPyObjectInNet() sys.testattr2 = sys.testattr1; // Compare in Python - PythonEngine.RunSimpleString( + PyObject res = PythonEngine.RunString( "import sys\n" + "sys.testattr3 = sys.testattr1 is sys.testattr2\n" ); - Assert.AreEqual(sys.testattr3.ToString(), "True"); // Compare in .NET diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..2f9aae2c7 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -74,65 +74,5 @@ public void ReInitialize() } PythonEngine.Shutdown(); } - - [Test] - public void TestScopeIsShutdown() - { - PythonEngine.Initialize(); - var scope = PyScopeManager.Global.Create("test"); - PythonEngine.Shutdown(); - Assert.That(PyScopeManager.Global.Contains("test"), Is.False); - } - - /// - /// Helper for testing the shutdown handlers. - /// - int shutdown_count = 0; - void OnShutdownIncrement() - { - shutdown_count++; - } - void OnShutdownDouble() - { - shutdown_count *= 2; - } - - /// - /// Test the shutdown handlers. - /// - [Test] - public void ShutdownHandlers() - { - // Test we can run one shutdown handler. - shutdown_count = 0; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.Shutdown(); - Assert.That(shutdown_count, Is.EqualTo(1)); - - // Test we can run multiple shutdown handlers in the right order. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: 4 * 2 + 1 = 9 - // Wrong: (4 + 1) * 2 = 10 - Assert.That(shutdown_count, Is.EqualTo(9)); - - // Test we can remove shutdown handlers, handling duplicates. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.RemoveShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: (4 + 1) * 2 + 1 + 1 = 12 - // Wrong: (4 * 2) + 1 + 1 + 1 = 11 - Assert.That(shutdown_count, Is.EqualTo(12)); - } } } diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 794645994..2497844bb 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -35,7 +35,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON36 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) @@ -68,10 +68,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants) @@ -80,10 +80,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG @@ -131,7 +131,7 @@ - + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index fc155ca91..28ea6424f 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,170 +1,170 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - false - ..\pythonnet.snk - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 6 + true + false + ..\pythonnet.snk + + + - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON37;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON37;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG - false - full - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + --> + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON36;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON36;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS2;TRACE;DEBUG + false + full + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + \ No newline at end of file diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..dd4418cc9 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..a10688749 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -93,7 +93,7 @@ public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) return IntPtr.Zero; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; @@ -186,7 +186,7 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) return -1; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; for (var i = 0; i < count; i++) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..d63930a58 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Reflection; using System.Threading; @@ -18,21 +17,13 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - // - // WARNING: Dangerous if cross-app domain usage is ever supported - // Reusing the dictionary with assemblies accross multiple initializations is problematic. - // Loading happens from CurrentDomain (see line 53). And if the first call is from AppDomain that is later unloaded, - // than it can end up referring to assemblies that are already unloaded (default behavior after unload appDomain - - // unless LoaderOptimization.MultiDomain is used); - // So for multidomain support it is better to have the dict. recreated for each app-domain initialization - private static ConcurrentDictionary> namespaces = - new ConcurrentDictionary>(); + private static ConcurrentDictionary> namespaces; //private static Dictionary> generics; private static AssemblyLoadEventHandler lhandler; private static ResolveEventHandler rhandler; // updated only under GIL? - private static Dictionary probed = new Dictionary(32); + private static Dictionary probed; // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; @@ -49,6 +40,9 @@ private AssemblyManager() /// internal static void Initialize() { + namespaces = new ConcurrentDictionary>(); + probed = new Dictionary(32); + //generics = new Dictionary>(); assemblies = new ConcurrentQueue(); pypath = new List(16); @@ -138,7 +132,7 @@ private static Assembly ResolveHandler(object ob, ResolveEventArgs args) internal static void UpdatePath() { IntPtr list = Runtime.PySys_GetObject("path"); - var count = Runtime.PyList_Size(list); + int count = Runtime.PyList_Size(list); if (count != pypath.Count) { pypath.Clear(); @@ -349,7 +343,9 @@ internal static void ScanAssembly(Assembly assembly) // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + + Type[] types = assembly.GetTypes(); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -423,9 +419,10 @@ public static List GetNames(string nsname) { foreach (Assembly a in namespaces[nsname].Keys) { - foreach (Type t in GetTypes(a)) + Type[] types = a.GetTypes(); + foreach (Type t in types) { - if ((t.Namespace ?? "") == nsname && !t.IsNested) + if ((t.Namespace ?? "") == nsname) { names.Add(t.Name); } @@ -464,32 +461,5 @@ public static Type LookupType(string qname) } return null; } - - internal static Type[] GetTypes(Assembly a) - { - if (a.IsDynamic) - { - try - { - return a.GetTypes(); - } - catch (ReflectionTypeLoadException exc) - { - // Return all types that were successfully loaded - return exc.Types.Where(x => x != null).ToArray(); - } - } - else - { - try - { - return a.GetExportedTypes(); - } - catch (FileNotFoundException) - { - return new Type[0]; - } - } - } } -} \ No newline at end of file +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..4dd3b5364 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -247,6 +247,24 @@ public static IntPtr tp_str(IntPtr ob) } + /// + /// Default implementations for required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + /// /// Standard dealloc implementation for instances of reflected types. /// diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..16d3b99db 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,9 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Reflection.Emit; -using System.Resources; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -33,12 +32,6 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } - public static void Reset() - { - assemblyBuilders = new Dictionary(); - moduleBuilders = new Dictionary, ModuleBuilder>(); - } - internal ClassDerivedObject(Type tp) : base(tp) { } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0b084a49d..6a9d40ebd 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -34,11 +34,6 @@ static ClassManager() dtype = typeof(MulticastDelegate); } - public static void Reset() - { - cache = new Dictionary(128); - } - /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 83d761fd0..46257c73f 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -230,10 +230,10 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) } // Get the args passed in. - var i = Runtime.PyTuple_Size(args); + int i = Runtime.PyTuple_Size(args); IntPtr defaultArgs = cls.indexer.GetDefaultArgs(args); - var numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); - var temp = i + numOfDefaultArgs; + int numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); + int temp = i + numOfDefaultArgs; IntPtr real = Runtime.PyTuple_New(temp + 1); for (var n = 0; n < i; n++) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..17782f026 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -29,9 +29,13 @@ internal CLRObject(object ob, IntPtr tp) gcHandle = gc; inst = ob; - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + // for performance before calling SetArgsAndCause() lets check if we are an exception + if (inst is Exception) + { + // Fix the BaseException args (and __cause__ in case of Python 3) + // slot if wrapping a CLR exception + Exceptions.SetArgsAndCause(py); + } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..a81d9fb99 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,11 +1,8 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; -using System.ComponentModel; namespace Python.Runtime { @@ -31,6 +28,9 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -46,6 +46,31 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -72,6 +97,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -101,6 +129,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyDecimalType; + return IntPtr.Zero; } @@ -135,8 +166,8 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { + if (value is IList && value.GetType().IsGenericType) + { using (var resultlist = new PyList()) { foreach (object o in (IEnumerable)value) @@ -178,6 +209,17 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -230,6 +272,40 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); + + case TypeCode.DateTime: + var datetime = (DateTime)value; + + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; + default: if (value is IEnumerable) { @@ -251,6 +327,18 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; + } + /// /// In a few situations, we don't have any advisory type information @@ -437,6 +525,26 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] { result.GetType() }); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] { result }); + } + return opImplicit != null; + } + } + return ToPrimitive(value, obType, out result, setError); } @@ -453,6 +561,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -802,6 +936,30 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number | NumberStyles.AllowExponent, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; + return true; } @@ -843,10 +1001,10 @@ private static void SetConversionError(IntPtr value, Type target) private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - var size = Runtime.PySequence_Size(value); + int size = Runtime.PySequence_Size(value); result = null; - if (size < 0) + if (size < 0 || elementType.IsGenericType) { if (setError) { diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs index 3fe9ee5bb..2a91a74b4 100644 --- a/src/runtime/debughelper.cs +++ b/src/runtime/debughelper.cs @@ -116,10 +116,10 @@ internal static void debug(string msg) Console.WriteLine(" {0}", msg); } - /// + /// /// Helper function to inspect/compare managed to native conversions. - /// Especially useful when debugging CustomMarshaler. - /// + /// Especially useful when debugging CustomMarshaler. + /// /// [Conditional("DEBUG")] public static void PrintHexBytes(byte[] bytes) diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 7632816d1..bd8f1ee4c 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher + public class Dispatcher : IPyDisposable { public IntPtr target; public Type dtype; + private bool _disposed = false; + private bool _finalized = false; public Dispatcher(IntPtr target, Type dtype) { @@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + if (_finalized || _disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); + } - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) + public void Dispose() + { + if (_disposed) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); + return; } + _disposed = true; + Runtime.XDecref(target); + target = IntPtr.Zero; + dtype = null; + GC.SuppressFinalize(this); } public object Dispatch(ArrayList args) @@ -267,6 +276,11 @@ public object TrueDispatch(ArrayList args) Runtime.XDecref(op); return result; } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { target }; + } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..9023cfcfa 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -256,10 +256,7 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.XIncref(pe.PyType); - Runtime.XIncref(pe.PyValue); - Runtime.XIncref(pe.PyTB); - Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); + Runtime.PyErr_SetObject(pe.PyType, pe.PyValue); return; } @@ -279,7 +276,7 @@ public static void SetError(Exception e) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != IntPtr.Zero; + return Runtime.PyErr_Occurred() != 0; } /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..9569b0485 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -81,6 +81,27 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } + /// + /// Required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + + /// /// Default dealloc implementation. /// diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs new file mode 100644 index 000000000..948f94cb5 --- /dev/null +++ b/src/runtime/finalizer.cs @@ -0,0 +1,273 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public class ErrorArgs : EventArgs + { + public Exception Error { get; set; } + } + + public static readonly Finalizer Instance = new Finalizer(); + + public event EventHandler CollectOnce; + public event EventHandler ErrorHandler; + + public int Threshold { get; set; } + public bool Enable { get; set; } + + private ConcurrentQueue _objQueue = new ConcurrentQueue(); + private bool _pending = false; + private readonly object _collectingLock = new object(); + private Task _finalizerTask; + + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + public bool RefCountValidationEnabled { get; set; } = true; +#else + public readonly bool RefCountValidationEnabled = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + public class IncorrectFinalizeArgs : EventArgs + { + public IntPtr Handle { get; internal set; } + public ICollection ImpactedObjects { get; internal set; } + } + + public class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + private string _message; + public override string Message => _message; + + public IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + string name = Runtime.GetManagedString(pyname); + Runtime.XDecref(pyname); + _message = $"{name} may has a incorrect ref count"; + } + } + + public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + public event IncorrectRefCntHandler IncorrectRefCntResolver; + public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + + private Finalizer() + { + Enable = true; + Threshold = 200; + } + + public void Collect(bool forceDispose = true) + { + if (Instance._finalizerTask != null + && !Instance._finalizerTask.IsCompleted) + { + using (Py.GIL()) + { + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); + } + } + else if (forceDispose) + { + Instance.DisposeAll(); + } + } + + public List GetCollectedObjects() + { + return _objQueue.Select(T => new WeakReference(T)).ToList(); + } + + internal void AddFinalizedObject(IPyDisposable obj) + { + if (!Enable) + { + return; + } + if (Runtime.Py_IsInitialized() == 0) + { + // XXX: Memory will leak if a PyObject finalized after Python shutdown, + // for avoiding that case, user should call GC.Collect manual before shutdown. + return; + } +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + _objQueue.Enqueue(obj); + } + GC.ReRegisterForFinalize(obj); + if (!_pending && _objQueue.Count >= Threshold) + { + AddPendingCollect(); + } + } + + internal static void Shutdown() + { + if (Runtime.Py_IsInitialized() == 0) + { + Instance._objQueue = new ConcurrentQueue(); + return; + } + Instance.Collect(forceDispose: true); + } + + private void AddPendingCollect() + { + if(Monitor.TryEnter(_collectingLock)) + { + try + { + if (!_pending) + { + _pending = true; + // should already be complete but just in case + _finalizerTask?.Wait(); + + _finalizerTask = Task.Factory.StartNew(() => + { + Instance.DisposeAll(); + _pending = false; + }); + } + } + finally + { + Monitor.Exit(_collectingLock); + } + } + } + + private void DisposeAll() + { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + using (Py.GIL()) + { +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) + { + try + { + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } + } + } + } + } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + IntPtr[] handles = obj.GetTrackedHandles(); + foreach (var handle in handles) + { + if (handle == IntPtr.Zero) + { + continue; + } + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResolver != null) + { + var funcList = IncorrectRefCntResolver.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif + } +} diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..9772d082f 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Resources; namespace Python.Runtime { @@ -21,11 +20,6 @@ static GenericUtil() mapping = new Dictionary>>(); } - public static void Reset() - { - mapping = new Dictionary>>(); - } - /// /// Register a generic type that appears in a given namespace. /// diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..bc9ac5eee 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -155,7 +155,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // hook is saved as this.py_import. This version handles CLR // import and defers to the normal builtin for everything else. - var num_args = Runtime.PyTuple_Size(args); + int num_args = Runtime.PyTuple_Size(args); if (num_args < 1) { return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); @@ -237,11 +237,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) if (res != IntPtr.Zero) { // There was no error. - if (fromlist && IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); - } return res; } // There was an error @@ -295,11 +290,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { if (fromlist) { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } Runtime.XIncref(module); return module; } @@ -355,33 +345,20 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } } - { - var mod = fromlist ? tail : head; + ModuleObject mod = fromlist ? tail : head; - if (fromlist && IsLoadAll(fromList)) + if (fromlist && Runtime.PySequence_Size(fromList) == 1) + { + IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); + if (!CLRModule.preload && Runtime.GetManagedString(fp) == "*") { mod.LoadNames(); } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; + Runtime.XDecref(fp); } - } - private static bool IsLoadAll(IntPtr fromList) - { - if (CLRModule.preload) - { - return false; - } - if (Runtime.PySequence_Size(fromList) != 1) - { - return false; - } - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - bool res = Runtime.GetManagedString(fp) == "*"; - Runtime.XDecref(fp); - return res; + Runtime.XIncref(mod.pyHandle); + return mod.pyHandle; } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..7b6d90ca8 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -56,7 +56,7 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); MethodBase[] methods = SetterBinder.GetMethods(); if (methods.Length == 0) { @@ -72,7 +72,7 @@ internal bool NeedsDefaultArgs(IntPtr args) return false; } - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -95,7 +95,7 @@ internal IntPtr GetDefaultArgs(IntPtr args) { return Runtime.PyTuple_New(0); } - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple MethodBase[] methods = SetterBinder.GetMethods(); diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 616ced6bd..ce1bc9eb0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -36,7 +36,7 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); - var nargs = Runtime.PyTuple_Size(args); + int nargs = Runtime.PyTuple_Size(args); Type type = self.type; object obj; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..5168bcd98 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,6 +1,6 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; using System.Runtime.InteropServices; using System.Reflection; using System.Text; @@ -88,8 +88,7 @@ static ObjectOffset() public static int magic(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.ob_data; } @@ -98,8 +97,7 @@ public static int magic(IntPtr ob) public static int DictOffset(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.ob_dict; } @@ -108,8 +106,7 @@ public static int DictOffset(IntPtr ob) public static int Size(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.Size(); } @@ -128,6 +125,14 @@ public static int Size(IntPtr ob) public static int ob_type; private static int ob_dict; private static int ob_data; + + private static bool IsException(IntPtr pyObject) + { + var type = Runtime.PyObject_TYPE(pyObject); + return Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) + || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs deleted file mode 100644 index d5fc76ad3..000000000 --- a/src/runtime/interop37.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON37 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_as_async = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int am_await = 0; - public static int am_aiter = 0; - public static int am_anext = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int nb_matrix_multiply = 0; - public static int nb_inplace_matrix_multiply = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..3295ab110 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -29,7 +29,7 @@ public static IntPtr Initialize() /// public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 3) { return Exceptions.RaiseTypeError("invalid argument list"); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5e800c36f..48dc1eb9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -186,6 +186,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +207,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) @@ -279,7 +291,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // loop to find match, return invoker w/ or /wo error MethodBase[] _methods = null; - var pynargs = (int)Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); object arg; var isGeneric = false; ArrayList defaultArgList = null; @@ -301,9 +313,9 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth isGeneric = true; } ParameterInfo[] pi = mi.GetParameters(); - var clrnargs = pi.Length; + int clrnargs = pi.Length; var match = false; - var arrayStart = -1; + int arrayStart = -1; var outs = 0; if (pynargs == clrnargs) @@ -314,7 +326,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { match = true; defaultArgList = new ArrayList(); - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -338,7 +350,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { var margs = new object[clrnargs]; - for (int n = 0; n < clrnargs; n++) + for (var n = 0; n < clrnargs; n++) { IntPtr op; if (n < pynargs) @@ -394,14 +406,33 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } if (!typematch) { + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(pi[n].ParameterType); + if (underlyingType == null) + { + underlyingType = pi[n].ParameterType; + } // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); + TypeCode argtypecode = Type.GetTypeCode(underlyingType); TypeCode paramtypecode = Type.GetTypeCode(clrtype); if (argtypecode == paramtypecode) { typematch = true; clrtype = pi[n].ParameterType; } + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + clrtype = pi[n].ParameterType; + typematch = Converter.ToManaged(op, clrtype, out arg, false); + } + // this takes care of implicit conversions + var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + typematch = opImplicit.ReturnType == pi[n].ParameterType; + clrtype = pi[n].ParameterType; + } } Runtime.XDecref(pyoptype); if (!typematch) @@ -601,19 +632,6 @@ internal class MethodSorter : IComparer { int IComparer.Compare(object m1, object m2) { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; - if (me1.DeclaringType != me2.DeclaringType) - { - // m2's type derives from m1's type, favor m2 - if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) - return 1; - - // m1's type derives from m2's type, favor m1 - if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) - return -1; - } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); int p2 = MethodBinder.GetPrecedence((MethodBase)m2); if (p1 < p2) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..3985ed2cb 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -104,7 +104,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { if (self.info.IsGenericMethod) { - var len = Runtime.PyTuple_Size(args); //FIXME: Never used + int len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { @@ -129,7 +129,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (target == IntPtr.Zero && !self.m.IsStatic()) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 1) { Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8af722d29..4accb1531 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -190,17 +190,10 @@ public void LoadNames() foreach (string name in AssemblyManager.GetNames(_namespace)) { cache.TryGetValue(name, out m); - if (m != null) + if (m == null) { - continue; + ManagedType attr = GetAttribute(name, true); } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); - // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) - { - continue; - } - GetAttribute(name, true); } } @@ -335,17 +328,6 @@ public CLRModule() : base("clr") } } - public static void Reset() - { - hacked = false; - interactive_preload = true; - preload = false; - - // XXX Test performance of new features // - _SuppressDocs = false; - _SuppressOverloads = false; - } - /// /// The initializing of the preload hook has to happen as late as /// possible since sys.ps1 is created after the CLR module is diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..cc6125157 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -1,4 +1,5 @@ using System; +using System.Linq.Expressions; using System.Reflection; using System.Security.Permissions; @@ -12,6 +13,10 @@ internal class PropertyObject : ExtensionType private PropertyInfo info; private MethodInfo getter; private MethodInfo setter; + private bool getterCacheFailed; + private bool setterCacheFailed; + private Func getterCache; + private Action setterCache; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -67,7 +72,21 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); + if (self.getterCache == null && !self.getterCacheFailed) + { + // if the getter is not public 'GetGetMethod' will not find it + // but calling 'GetValue' will work, so for backwards compatibility + // we will use it instead + self.getterCache = BuildGetter(self.info); + if (self.getterCache == null) + { + self.getterCacheFailed = true; + } + } + + result = self.getterCacheFailed ? + self.info.GetValue(co.inst, null) + : self.getterCache(co.inst); return Converter.ToPython(result, self.info.PropertyType); } catch (Exception e) @@ -81,7 +100,6 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } } - /// /// Descriptor __set__ implementation. This method sets the value of /// a property based on the given Python value. The Python value must @@ -132,7 +150,27 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + + if (self.setterCache == null && !self.setterCacheFailed) + { + // if the setter is not public 'GetSetMethod' will not find it + // but calling 'SetValue' will work, so for backwards compatibility + // we will use it instead + self.setterCache = BuildSetter(self.info); + if (self.setterCache == null) + { + self.setterCacheFailed = true; + } + } + + if (self.setterCacheFailed) + { + self.info.SetValue(co.inst, newval, null); + } + else + { + self.setterCache(co.inst, newval); + } } else { @@ -160,5 +198,54 @@ public static IntPtr tp_repr(IntPtr ob) var self = (PropertyObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + private static Func BuildGetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetGetMethod(); + if (methodInfo == null) + { + // if the getter is not public 'GetGetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var expressionCall = Expression.Call(instance, methodInfo); + + return Expression.Lambda>( + Expression.Convert(expressionCall, typeof(object)), + obj).Compile(); + } + + private static Action BuildSetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetSetMethod(); + if (methodInfo == null) + { + // if the setter is not public 'GetSetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var parameters = methodInfo.GetParameters(); + if (parameters.Length != 1) + { + return null; + } + var value = Expression.Parameter(typeof(object)); + var argument = Expression.Convert(value, parameters[0].ParameterType); + + var expressionCall = Expression.Call(instance, methodInfo, argument); + + return Expression.Lambda>( + expressionCall, + obj, + value).Compile(); + } } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3b8c71efa..42ba1ff39 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1,11 +1,17 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Dynamic; using System.Linq.Expressions; namespace Python.Runtime { + public interface IPyDisposable : IDisposable + { + IntPtr[] GetTrackedHandles(); + } + /// /// Represents a generic Python object. The methods of this class are /// generally equivalent to the Python "abstract object API". See @@ -13,10 +19,18 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IDisposable + public class PyObject : DynamicObject, IEnumerable, IPyDisposable { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; private set; } +#endif + protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; + private bool _finalized = false; /// /// PyObject Constructor @@ -30,6 +44,9 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable public PyObject(IntPtr ptr) { obj = ptr; +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Protected default constructor to allow subclasses to manage @@ -37,18 +54,26 @@ public PyObject(IntPtr ptr) protected PyObject() { +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. - ~PyObject() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (obj == IntPtr.Zero) + { + return; + } + if (_finalized || disposed) + { + return; + } + // Prevent a infinity loop by calling GC.WaitForPendingFinalizers + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } @@ -101,7 +126,7 @@ public object AsManagedObject(Type t) } return result; } - + /// /// As Method /// @@ -156,6 +181,28 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + /// Unsafe Dispose Method. + /// To be used when already owning the GIL lock. + /// + public void UnsafeDispose() + { + if (!disposed) + { + if (!Runtime.IsFinalizing) + { + Runtime.XDecref(obj); + obj = IntPtr.Zero; + } + disposed = true; + } + GC.SuppressFinalize(this); + } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } /// /// GetPythonType Method @@ -519,9 +566,9 @@ public virtual void DelItem(int index) /// Returns the length for objects that support the Python sequence /// protocol, or 0 if the object does not support the protocol. /// - public virtual long Length() + public virtual int Length() { - var s = Runtime.PyObject_Size(obj); + int s = Runtime.PyObject_Size(obj); if (s < 0) { Runtime.PyErr_Clear(); diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8e6957855..f5449f30c 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,7 +22,7 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : DynamicObject, IPyDisposable { public readonly string Name; @@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable internal readonly IntPtr variables; private bool _isDisposed; + private bool _finalized = false; /// /// The Manager this scope associated with. @@ -525,27 +526,28 @@ public void Dispose() this.OnDispose?.Invoke(this); } - ~PyScope() + public IntPtr[] GetTrackedHandles() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + return new IntPtr[] { obj }; + } - Dispose(); + ~PyScope() + { + if (_finalized || _isDisposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } } public class PyScopeManager { - public static PyScopeManager Global; + public readonly static PyScopeManager Global = new PyScopeManager(); private Dictionary NamedScopes = new Dictionary(); - internal static void Reset() - { - Global = new PyScopeManager(); - } - internal PyScope NewScope(string name) { if (name == null) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..a23c7ac79 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -140,9 +140,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv); } /// @@ -153,9 +153,8 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// more than once, though initialization will only happen on the /// first call. It is *not* necessary to hold the Python global /// interpreter lock (GIL) to call this method. - /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true) { if (!initialized) { @@ -165,20 +164,10 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); + Runtime.Initialize(); initialized = true; Exceptions.Clear(); - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - if (setSysArgv) { Py.SetArgv(args); @@ -234,11 +223,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } } - static void OnDomainUnload(object _, EventArgs __) - { - Shutdown(); - } - /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -311,80 +295,18 @@ public static void Shutdown() if (initialized) { PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - - ExecuteShutdownHandlers(); + Marshal.FreeHGlobal(_pythonHome); + _pythonHome = IntPtr.Zero; + Marshal.FreeHGlobal(_programName); + _programName = IntPtr.Zero; + Marshal.FreeHGlobal(_pythonPath); + _pythonPath = IntPtr.Zero; + Runtime.Shutdown(); initialized = false; } } - /// - /// Called when the engine is shut down. - /// - /// Shutdown handlers are run in reverse order they were added, so that - /// resources available when running a shutdown handler are the same as - /// what was available when it was added. - /// - public delegate void ShutdownHandler(); - - static List ShutdownHandlers = new List(); - - /// - /// Add a function to be called when the engine is shut down. - /// - /// Shutdown handlers are executed in the opposite order they were - /// added, so that you can be sure that everything that was initialized - /// when you added the handler is still initialized when you need to shut - /// down. - /// - /// If the same shutdown handler is added several times, it will be run - /// several times. - /// - /// Don't add shutdown handlers while running a shutdown handler. - /// - public static void AddShutdownHandler(ShutdownHandler handler) - { - ShutdownHandlers.Add(handler); - } - - /// - /// Remove a shutdown handler. - /// - /// If the same shutdown handler is added several times, only the last - /// one is removed. - /// - /// Don't remove shutdown handlers while running a shutdown handler. - /// - public static void RemoveShutdownHandler(ShutdownHandler handler) - { - for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) - { - if (ShutdownHandlers[index] == handler) - { - ShutdownHandlers.RemoveAt(index); - break; - } - } - } - - /// - /// Run all the shutdown handlers. - /// - /// They're run in opposite order they were added. - /// - static void ExecuteShutdownHandlers() - { - while(ShutdownHandlers.Count > 0) - { - var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; - ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); - handler(); - } - } /// /// AcquireLock Method diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ded7fbeb5..295a63b3d 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception + public class PythonException : System.Exception, IPyDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -15,14 +15,12 @@ public class PythonException : System.Exception private string _message = ""; private string _pythonTypeName = ""; private bool disposed = false; + private bool _finalized = false; public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; @@ -45,11 +43,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + using (PyObject tb_module = PythonEngine.ImportModule("traceback")) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + Runtime.XIncref(_pyTB); + using (var pyTB = new PyObject(_pyTB)) + { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } } PythonEngine.ReleaseLock(gs); @@ -60,11 +60,12 @@ public PythonException() ~PythonException() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } /// @@ -132,7 +133,7 @@ public override string Message /// public override string StackTrace { - get { return _tb; } + get { return _tb + base.StackTrace; } } /// @@ -173,6 +174,11 @@ public void Dispose() } } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { _pyType, _pyValue, _pyTB }; + } + /// /// Matches Method /// diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..4d7487511 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "1.0.5.20" class clrproperty(object): diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 863eb4034..9ff83753f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,7 +2,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; -using System.Collections.Generic; +using System.Threading; namespace Python.Runtime { @@ -22,7 +22,7 @@ public static IntPtr LoadLibrary(string fileName) } #elif MONO_OSX private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; + private const string NativeDll = "/usr/lib/libSystem.dylib" private static IntPtr RTLD_DEFAULT = new IntPtr(-2); public static IntPtr LoadLibrary(string fileName) @@ -105,7 +105,7 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) public class Runtime { // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static int UCS => _UCS; @@ -131,7 +131,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static string pyversion => _pyversion; @@ -149,7 +149,7 @@ public class Runtime #elif PYTHON36 internal const string _pyversion = "3.6"; internal const string _pyver = "36"; -#elif PYTHON37 +#elif PYTHON37 // TODO: Add `interop37.cs` after PY37 is released internal const string _pyversion = "3.7"; internal const string _pyver = "37"; #else @@ -174,7 +174,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static readonly string PythonDLL = _PythonDll; @@ -196,71 +196,11 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static string OperatingSystemName { get; private set; } - - public enum MachineType - { - i386, - x86_64, - Other - }; - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; + public static int MainManagedThreadId { get; private set; } + /// /// Encoding to use to convert Unicode to/from Managed to Native /// @@ -269,11 +209,12 @@ public enum MachineType /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + internal static void Initialize() { if (Py_IsInitialized() == 0) { - Py_InitializeEx(initSigs ? 1 : 0); + Py_Initialize(); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } if (PyEval_ThreadsInitialized() == 0) @@ -281,15 +222,6 @@ internal static void Initialize(bool initSigs = false) PyEval_InitThreads(); } - IsFinalizing = false; - - CLRModule.Reset(); - GenericUtil.Reset(); - PyScopeManager.Reset(); - ClassManager.Reset(); - ClassDerivedObject.Reset(); - TypeManager.Reset(); - IntPtr op; IntPtr dict; if (IsPython3) @@ -368,6 +300,14 @@ internal static void Initialize(bool initSigs = false) PyFloatType = PyObject_Type(op); XDecref(op); + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + PyDecimalType = PyObject_Type(op); + XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); + #if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; @@ -403,10 +343,6 @@ internal static void Initialize(bool initSigs = false) NativeMethods.FreeLibrary(dllLocal); } #endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); @@ -424,58 +360,12 @@ internal static void Initialize(bool initSigs = false) AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() - { - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - OperatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - XDecref(emptyTuple); - XDecref(platformModule); - - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) - { - OSType = OperatingSystemType.Other; - } - OperatingSystem = OSType; - - MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) - { - MType = MachineType.Other; - } - Machine = MType; - } - internal static void Shutdown() { AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); + Finalizer.Shutdown(); Py_Finalize(); } @@ -512,6 +402,7 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; #if PYTHON3 internal static IntPtr PyBytesType; @@ -540,7 +431,7 @@ internal static int AtExit() /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != IntPtr.Zero) + if (PyErr_Occurred() != 0) { throw new PythonException(); } @@ -548,7 +439,7 @@ internal static void CheckExceptionOccurred() internal static IntPtr ExtendTuple(IntPtr t, params IntPtr[] args) { - var size = PyTuple_Size(t); + int size = PyTuple_Size(t); int add = args.Length; IntPtr item; @@ -591,7 +482,7 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) free = true; } - var n = PyTuple_Size(args); + int n = PyTuple_Size(args); var types = new Type[n]; Type t = null; @@ -727,9 +618,6 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_Initialize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_InitializeEx(int initsigs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_IsInitialized(); @@ -1025,13 +913,8 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_Not(IntPtr pointer); - internal static long PyObject_Size(IntPtr pointer) - { - return (long) _PyObject_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] - private static extern IntPtr _PyObject_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Hash(IntPtr op); @@ -1261,61 +1144,26 @@ internal static bool PyFloat_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PySequence_Check(IntPtr pointer); - internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) - { - return PySequence_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); - - internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PySequence_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PySequence_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PySequence_DelItem(IntPtr pointer, long index) - { - return PySequence_DelItem(pointer, new IntPtr(index)); - } + internal static extern int PySequence_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); - - internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_DelItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - - internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) - { - return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); - } + internal static extern IntPtr PySequence_GetSlice(IntPtr pointer, int i1, int i2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); - - internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_SetSlice(IntPtr pointer, int i1, int i2, IntPtr v); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - - internal static long PySequence_Size(IntPtr pointer) - { - return (long) _PySequence_Size(pointer); - } + internal static extern int PySequence_DelSlice(IntPtr pointer, int i1, int i2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] - private static extern IntPtr _PySequence_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); @@ -1323,24 +1171,14 @@ internal static long PySequence_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); - internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) - { - return PySequence_Repeat(pointer, new IntPtr(count)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); + internal static extern IntPtr PySequence_Repeat(IntPtr pointer, int count); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); - internal static long PySequence_Count(IntPtr pointer, IntPtr value) - { - return (long) _PySequence_Count(pointer, value); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] - private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Count(IntPtr pointer, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Tuple(IntPtr pointer); @@ -1366,57 +1204,33 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { -#if PYTHON3 - return PyUnicode_FromKindAndData(_UCS, value, value.Length); -#elif PYTHON2 return PyString_FromStringAndSize(value, value.Length); -#endif } #if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyBytes_FromString(string op); - internal static long PyBytes_Size(IntPtr op) - { - return (long) _PyBytes_Size(op); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] - private static extern IntPtr _PyBytes_Size(IntPtr op); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBytes_Size(IntPtr op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { return ob + BytesOffset.ob_sval; } - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return _PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_FromStringAndSize")] - internal static extern IntPtr _PyString_FromStringAndSize( + internal static extern IntPtr PyString_FromStringAndSize( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) - { - return PyUnicode_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + internal static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, int size); #elif PYTHON2 - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyString_FromStringAndSize(string value, IntPtr size); + internal static extern IntPtr PyString_FromStringAndSize(string value, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyString_AsString(IntPtr op); @@ -1437,30 +1251,20 @@ internal static bool PyUnicode_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) - { - return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromKindAndData( + internal static extern IntPtr PyUnicode_FromKindAndData( int kind, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) + internal static IntPtr PyUnicode_FromUnicode(string s, int size) { return PyUnicode_FromKindAndData(_UCS, s, size); } - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long)_PyUnicode_GetSize(ob); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] - private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); @@ -1476,26 +1280,16 @@ internal static long PyUnicode_GetSize(IntPtr ob) EntryPoint = PyUnicodeEntryPoint + "FromEncodedObject")] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - return PyUnicode_FromUnicode(s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "FromUnicode")] - private static extern IntPtr PyUnicode_FromUnicode( + internal static extern IntPtr PyUnicode_FromUnicode( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long) _PyUnicode_GetSize(ob); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "GetSize")] - internal static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "AsUnicode")] @@ -1538,7 +1332,7 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { IntPtr p = PyUnicode_AsUnicode(op); - int length = (int)PyUnicode_GetSize(op); + int length = PyUnicode_GetSize(op); int size = length * _UCS; var buffer = new byte[size]; @@ -1604,13 +1398,8 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyDict_Clear(IntPtr pointer); - internal static long PyDict_Size(IntPtr pointer) - { - return (long) _PyDict_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] - internal static extern IntPtr _PyDict_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Size(IntPtr pointer); //==================================================================== @@ -1622,40 +1411,20 @@ internal static bool PyList_Check(IntPtr ob) return PyObject_TYPE(ob) == PyListType; } - internal static IntPtr PyList_New(long size) - { - return PyList_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_New(IntPtr size); + internal static extern IntPtr PyList_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) - { - return PyList_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyList_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyList_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) - { - return PyList_Insert(pointer, new IntPtr(index), value); - } + internal static extern int PyList_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(IntPtr pointer, IntPtr index, IntPtr value); + internal static extern int PyList_Insert(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Append(IntPtr pointer, IntPtr value); @@ -1666,29 +1435,15 @@ internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Sort(IntPtr pointer); - internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) - { - return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - - internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) - { - return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); - } + internal static extern IntPtr PyList_GetSlice(IntPtr pointer, int start, int end); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); + internal static extern int PyList_SetSlice(IntPtr pointer, int start, int end, IntPtr value); - internal static long PyList_Size(IntPtr pointer) - { - return (long) _PyList_Size(pointer); - } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Size(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] - private static extern IntPtr _PyList_Size(IntPtr pointer); //==================================================================== // Python tuple API @@ -1699,45 +1454,20 @@ internal static bool PyTuple_Check(IntPtr ob) return PyObject_TYPE(ob) == PyTupleType; } - internal static IntPtr PyTuple_New(long size) - { - return PyTuple_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_New(IntPtr size); - - internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) - { - return PyTuple_GetItem(pointer, new IntPtr(index)); - } + internal static extern IntPtr PyTuple_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyTuple_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyTuple_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) - { - return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } + internal static extern int PyTuple_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - - internal static long PyTuple_Size(IntPtr pointer) - { - return (long) _PyTuple_Size(pointer); - } + internal static extern IntPtr PyTuple_GetSlice(IntPtr pointer, int start, int end); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] - private static extern IntPtr _PyTuple_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyTuple_Size(IntPtr pointer); //==================================================================== @@ -1840,16 +1570,16 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) return (t == tp) || PyType_IsSubtype(t, tp); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); - - internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) + internal static bool PyObjectType_TypeCheck(IntPtr type, IntPtr tp) { - return PyType_GenericAlloc(type, new IntPtr(n)); + return (type == tp) || PyType_IsSubtype(type, tp); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyType_GenericAlloc(IntPtr type, int n); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1883,21 +1613,11 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) // Python memory API //==================================================================== - internal static IntPtr PyMem_Malloc(long size) - { - return PyMem_Malloc(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Malloc(IntPtr size); - - internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) - { - return PyMem_Realloc(ptr, new IntPtr(size)); - } + internal static extern IntPtr PyMem_Malloc(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); + internal static extern IntPtr PyMem_Realloc(IntPtr ptr, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyMem_Free(IntPtr ptr); @@ -1929,7 +1649,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern void PyErr_NormalizeException(IntPtr ob, IntPtr val, IntPtr tb); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_Occurred(); + internal static extern int PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); @@ -1953,5 +1673,11 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Function(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_MakePendingCalls(); } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..6570ee083 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Reflection; @@ -21,10 +21,6 @@ static TypeManager() cache = new Dictionary(128); } - public static void Reset() - { - cache = new Dictionary(128); - } /// /// Given a managed Type derived from ExtensionType, get the handle to @@ -451,225 +447,6 @@ internal static IntPtr AllocateTypeObject(string name) } - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch(Runtime.Machine) - { - case Runtime.MachineType.i386: - return I386; - case Runtime.MachineType.x86_64: - return X86_64; - default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - return 0x1000; - case Runtime.OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } -#endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of @@ -677,10 +454,8 @@ internal static void InitializeNativeCodePage() /// internal static void InitializeSlots(IntPtr type, Type impl) { - // We work from the most-derived class up; make sure to get - // the most-derived slot and not to override it with a base - // class's slot. - var seen = new HashSet(); + var seen = new Hashtable(8); + Type offsetType = typeof(TypeOffset); while (impl != null) { @@ -698,52 +473,24 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - if (seen.Contains(name)) + if (seen[name] != null) { continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + FieldInfo fi = offsetType.GetField(name); + var offset = (int)fi.GetValue(offsetType); + + IntPtr slot = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, offset, slot); - seen.Add(name); + seen[name] = 1; } impl = impl.BaseType; } - - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); } - /// - /// Helper for InitializeSlots. - /// - /// Initializes one slot to point to a function pointer. - /// The function pointer might be a thunk for C#, or it may be - /// an address in the NativeCodePage. - /// - /// Type being initialized. - /// Function pointer. - /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) - { - Type typeOffset = typeof(TypeOffset); - FieldInfo fi = typeOffset.GetField(name); - var offset = (int)fi.GetValue(typeOffset); - - Marshal.WriteIntPtr(type, offset, slot); - } /// /// Given a newly allocated Python type object and a managed Type that diff --git a/src/testing/InheritanceTest.cs b/src/testing/InheritanceTest.cs deleted file mode 100644 index 529703b3c..000000000 --- a/src/testing/InheritanceTest.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace Python.Test -{ - public class BaseClass - { - public bool IsBase() => true; - } - - public class DerivedClass : BaseClass - { - public new bool IsBase() => false; - } -} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..8a8d9ed2b 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -83,7 +83,6 @@ - @@ -111,4 +110,4 @@ - \ No newline at end of file + diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 06ab7cb4e..36294594c 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -60,19 +60,4 @@ public string GetValue() return value; } } - - public class UnicodeString - { - public string value = "안녕"; - - public string GetString() - { - return value; - } - - public override string ToString() - { - return value; - } - } } diff --git a/src/tests/importtest.py b/src/tests/importtest.py deleted file mode 100644 index fe93764e9..000000000 --- a/src/tests/importtest.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -try: - del sys.modules["System.IO"] -except KeyError: - pass - -assert "FileStream" not in globals() -import System.IO -from System.IO import * - -assert "FileStream" in globals() diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 612ce442e..68773508b 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -281,14 +281,3 @@ def PyCallback(self, self2): testobj.DoCallback() assert testobj.PyCallbackWasCalled assert testobj.SameReference - - -def test_method_inheritance(): - """Ensure that we call the overridden method instead of the one provided in - the base class.""" - - base = Test.BaseClass() - derived = Test.DerivedClass() - - assert base.IsBase() == True - assert derived.IsBase() == False diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 0ba10a80e..53e5d8051 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -2,12 +2,11 @@ """Test CLR <-> Python type conversions.""" -from __future__ import unicode_literals import System import pytest -from Python.Test import ConversionTest, UnicodeString +from Python.Test import ConversionTest -from ._compat import indexbytes, long, unichr, text_type, PY2, PY3 +from ._compat import indexbytes, long, unichr def test_bool_conversion(): @@ -536,14 +535,6 @@ def test_string_conversion(): with pytest.raises(TypeError): ConversionTest().StringField = 1 - - world = UnicodeString() - test_unicode_str = u"안녕" - assert test_unicode_str == text_type(world.value) - assert test_unicode_str == text_type(world.GetString()) - # TODO: not sure what to do for Python 2 here (GH PR #670) - if PY3: - assert test_unicode_str == text_type(world) def test_interface_conversion(): diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index 08b00d77d..c697290ee 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -289,8 +289,7 @@ def test_python_compat_of_managed_exceptions(): assert e.args == (msg,) assert isinstance(e.args, tuple) if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp + assert repr(e) == "OverflowException('Simple message',)" elif PY2: assert repr(e) == "OverflowException(u'Simple message',)" diff --git a/src/tests/test_import.py b/src/tests/test_import.py index 25877be15..42cafc4df 100644 --- a/src/tests/test_import.py +++ b/src/tests/test_import.py @@ -3,7 +3,7 @@ """Test the import statement.""" import pytest -import sys + def test_relative_missing_import(): """Test that a relative missing import doesn't crash. @@ -11,12 +11,3 @@ def test_relative_missing_import(): Relative import in the site-packages folder""" with pytest.raises(ImportError): from . import _missing_import - - -def test_import_all_on_second_time(): - """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited - to a module file.""" - from . import importtest - del sys.modules[importtest.__name__] - diff --git a/src/tests/test_subclass.py b/src/tests/test_subclass.py index ab440d429..43d013c7c 100644 --- a/src/tests/test_subclass.py +++ b/src/tests/test_subclass.py @@ -128,39 +128,6 @@ def test_derived_class(): assert id(x) == id(ob) -def test_derived_traceback(): - """Test python exception traceback in class derived from managed base""" - class DerivedClass(SubClassTest): - __namespace__ = "Python.Test.traceback" - - def foo(self): - print (xyzname) - return None - - import sys,traceback - ob = DerivedClass() - - # direct call - try: - ob.foo() - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - # call through managed code - try: - FunctionsTest.test_foo(ob) - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - def test_create_instance(): """Test derived instances can be created from managed code""" DerivedClass = derived_class_fixture(test_create_instance.__name__) diff --git a/tools/geninterop/fake_libc_include/crypt.h b/tools/geninterop/fake_libc_include/crypt.h deleted file mode 100644 index 3b16d481f..000000000 --- a/tools/geninterop/fake_libc_include/crypt.h +++ /dev/null @@ -1 +0,0 @@ -#include "features.h" \ No newline at end of file diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..bf5fdb96b 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -173,8 +173,7 @@ def preprocess_python_headers(): "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", - "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "__int64=long long" ] if hasattr(sys, "abiflags"): @@ -186,7 +185,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-I"] + include_dirs + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = []