diff --git a/.gitignore b/.gitignore index a8f0b03d6..f60d9632d 100644 --- a/.gitignore +++ b/.gitignore @@ -91,4 +91,6 @@ ENV/ Binaries/ python35/ python27/ +python3.5m/ + *.un~ diff --git a/Config/FilterPlugin.ini b/Config/FilterPlugin.ini new file mode 100644 index 000000000..a3739f647 --- /dev/null +++ b/Config/FilterPlugin.ini @@ -0,0 +1,9 @@ +[FilterPlugin] +; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and +; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively. +; +; Examples: +; /README.txt +; /Extras/... +; /Binaries/ThirdParty/*.dll +/EmbeddedPython/... diff --git a/EmbeddedPython/.gitignore b/EmbeddedPython/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/EmbeddedPython/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/Packaging/package-linux.sh b/Packaging/package-linux.sh new file mode 100755 index 000000000..b7368ff38 --- /dev/null +++ b/Packaging/package-linux.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +docker run --rm -ti "-v$ROOTDIR:/hostdir" -w /hostdir adamrehn/ue4-full:4.21.1 python3 /hostdir/Packaging/package.py diff --git a/Packaging/package.py b/Packaging/package.py new file mode 100644 index 000000000..1b4b0de50 --- /dev/null +++ b/Packaging/package.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +from ue4helpers import ConanUtils, FilesystemUtils, PlatformInfo, PluginPackager, SubprocessUtils, VersionHelpers +from os.path import abspath, dirname, join + +# Build a UE4-compatible version of CPython 3.6.8 +print('Building our embeddable Python distribution...') +SubprocessUtils.run(['ue4', 'conan', 'update']) +SubprocessUtils.run(['ue4', 'conan', 'build', 'python-ue4==3.6.8']) + +# Bundle the custom-built Python distribution in our source tree +print('Bundling our embeddable Python distribution...') +root = dirname(dirname(abspath(__file__))) +bundled = join(root, 'EmbeddedPython', PlatformInfo.identifier()) +ConanUtils.copy_package('python-ue4/3.6.8@adamrehn/4.21', bundled) + +# Create our plugin packager +packager = PluginPackager( + root = root, + version = VersionHelpers.from_git_commit(), + archive = '{name}-{version}-{platform}' +) + +# Clean any previous build artifacts +packager.clean() + +# Package the plugin +packager.package() + +# Compress the packaged distribution +archive = packager.archive() + +# TODO: upload the archive to Amazon S3 +print('Created compressed archive "{}".'.format(archive)) diff --git a/README.md b/README.md index 1885cd358..ba6fc5688 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # UnrealEnginePython Embed Python in Unreal Engine 4 @@ -686,7 +685,7 @@ The following parameters are supported: * `RelativeAdditionalModulesPath`: like AdditionalModulesPath, but the path is relative to the /Content directory * `ZipPath`: allow to specify a .zip file that is added to sys.path * `RelativeZipPath`: like ZipPath, but the path is relative to the /Content directory -* `ImportModules: comma/space/semicolon separated list of modules to import on startup (after ue_site) +* `ImportModules`: comma/space/semicolon separated list of modules to import on startup (after ue_site) Example: diff --git a/Source/PythonAutomation/Public/PythonAutomationModule.h b/Source/PythonAutomation/Public/PythonAutomationModule.h index c0f9c6d16..261ae3cde 100644 --- a/Source/PythonAutomation/Public/PythonAutomationModule.h +++ b/Source/PythonAutomation/Public/PythonAutomationModule.h @@ -3,7 +3,7 @@ #pragma once #include "CoreMinimal.h" -#include "ModuleInterface.h" +#include "Modules/ModuleManager.h" class FPythonAutomationModule : public IModuleInterface { diff --git a/Source/UnrealEnginePython/Private/UnrealEnginePython.cpp b/Source/UnrealEnginePython/Private/UnrealEnginePython.cpp index 211124849..34fa3330a 100644 --- a/Source/UnrealEnginePython/Private/UnrealEnginePython.cpp +++ b/Source/UnrealEnginePython/Private/UnrealEnginePython.cpp @@ -15,11 +15,15 @@ #endif #if ENGINE_MINOR_VERSION >= 18 -#define PROJECT_CONTENT_DIR FPaths::ProjectContentDir() +#define PROJECT_CONTENT_DIR FPaths::ConvertRelativePathToFull(FPaths::ProjectContentDir()) +#define PROJECT_DIR FPaths::ProjectDir() #else -#define PROJECT_CONTENT_DIR FPaths::GameContentDir() +#define PROJECT_CONTENT_DIR FPaths::ConvertRelativePathToFull(FPaths::GameContentDir()) +#define PROJECT_DIR FPaths::GameDir() + #endif + #if PLATFORM_MAC #include "Runtime/Core/Public/Mac/CocoaThread.h" #endif @@ -47,6 +51,24 @@ const char *ue4_module_options = "linux_global_symbols"; #include "Android/AndroidApplication.h" #endif +#include "Misc/Paths.h" +#include "HAL/FileManager.h" + + +//Deleter class for TUniquePtr pointers that store the result of Py_DecodeLocale() +class PyMemDeleter +{ + public: + PyMemDeleter() = default; + PyMemDeleter(const PyMemDeleter&) = default; + PyMemDeleter& operator=(const PyMemDeleter&) = default; + ~PyMemDeleter() = default; + + void operator()(wchar_t* mem) const { + PyMem_RawFree(mem); + } +}; + const char *UEPyUnicode_AsUTF8(PyObject *py_str) { @@ -92,7 +114,6 @@ bool PyUnicodeOrString_Check(PyObject *py_obj) return false; } -#define LOCTEXT_NAMESPACE "FUnrealEnginePythonModule" void FUnrealEnginePythonModule::UESetupPythonInterpreter(bool verbose) { @@ -182,7 +203,7 @@ static void setup_stdout_stderr() " def isatty(self):\n" " return False\n" "sys.stdout = UnrealEngineOutput(unreal_engine.log)\n" - "sys.stderr = UnrealEngineOutput(unreal_engine.log_error)\n" + "sys.stderr = UnrealEngineOutput(unreal_engine.log)\n" "\n" "class event:\n" " def __init__(self, event_signature):\n" @@ -242,6 +263,10 @@ FAutoConsoleCommand ExecPythonStringCommand( void FUnrealEnginePythonModule::StartupModule() { + // TODO: Refactor large method + + UE_LOG(LogPython, Log, TEXT(">>> Loading UnrealEnginePythonPlugin")); + BrutalFinalize = false; // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module @@ -259,7 +284,8 @@ void FUnrealEnginePythonModule::StartupModule() if (GConfig->GetString(UTF8_TO_TCHAR("Python"), UTF8_TO_TCHAR("RelativeHome"), PythonHome, GEngineIni)) { - PythonHome = FPaths::Combine(*PROJECT_CONTENT_DIR, *PythonHome); + UE_LOG(LogPython, Log, TEXT("RelativeHome set, checking if directory exists")); + PythonHome = FPaths::Combine(*PROJECT_DIR, *PythonHome); FPaths::NormalizeFilename(PythonHome); PythonHome = FPaths::ConvertRelativePathToFull(PythonHome); #if PY_MAJOR_VERSION >= 3 @@ -267,8 +293,16 @@ void FUnrealEnginePythonModule::StartupModule() #else char *home = TCHAR_TO_UTF8(*PythonHome); #endif - - Py_SetPythonHome(home); + if (FPaths::DirectoryExists(PythonHome)) + { + UE_LOG(LogPython, Log, TEXT("Setting Python home to %s"), *PythonHome); + Py_SetPythonHome(home); + } + else + { + UE_LOG(LogPython, Log, TEXT("RelativeHome directory does not exist, no Python dependencies to load. Not setting Python home.")); + PythonHome = FString(TEXT("")); + } } TArray ImportModules; @@ -457,7 +491,41 @@ void FUnrealEnginePythonModule::StartupModule() Py_SetPath(Py_DecodeLocale(TCHAR_TO_UTF8(*BasePythonPath), NULL)); #endif #endif - + + //Retrieve the paths to the Engine and project plugin directories + FString enginePlugins = FPaths::ConvertRelativePathToFull(FPaths::EnginePluginsDir()); + FString projectPlugins = FPaths::ConvertRelativePathToFull(FPaths::ProjectPluginsDir()); + + //Attempt to detect the absolute path to our EmbeddedPython directory + TArray matches; + FString dirToFind = TEXT("EmbeddedPython"); + IFileManager& fileManager = IFileManager::Get(); + fileManager.FindFilesRecursive(matches, *enginePlugins, *dirToFind, false, true, false); + fileManager.FindFilesRecursive(matches, *projectPlugins, *dirToFind, false, true, false); + + //If we detected our embedded Python then set the name of the Python interpreter appropriately + if (matches.Num() > 0) + { + //Make sure we use the correct version of the embedded interpreter for the host platform + #if PLATFORM_LINUX + #define _PLATFORM_NAME TEXT("Linux") + #elif PLATFORM_MAC + #define _PLATFORM_NAME TEXT("Mac") + #elif PLATFORM_WINDOWS + #define _PLATFORM_NAME TEXT("Windows") + #else + #define _PLATFORM_NAME TEXT("Unknown") + #endif + FString programName = FPaths::Combine(matches[0], _PLATFORM_NAME, TEXT("bin"), TEXT("python3")); + UE_LOG(LogPython, Log, TEXT("Setting Python program name to %s"), *programName); + #undef _PLATFORM_NAME + + //Pass the interpreter path to Python, ensuring the memory containing the string outlives the interpreter + static TUniquePtr programNamePtr; + programNamePtr.Reset(Py_DecodeLocale(TCHAR_TO_UTF8(*programName), NULL)); + Py_SetProgramName(programNamePtr.Get()); + } + Py_Initialize(); #if PLATFORM_WINDOWS diff --git a/Source/UnrealEnginePython/UnrealEnginePython.Build.cs b/Source/UnrealEnginePython/UnrealEnginePython.Build.cs index 96b48121d..d7ace79f1 100644 --- a/Source/UnrealEnginePython/UnrealEnginePython.Build.cs +++ b/Source/UnrealEnginePython/UnrealEnginePython.Build.cs @@ -6,7 +6,6 @@ public class UnrealEnginePython : ModuleRules { - // leave this string as empty for triggering auto-discovery of python installations... private string pythonHome = ""; // otherwise specify the path of your python installation @@ -22,7 +21,8 @@ public class UnrealEnginePython : ModuleRules "C:/Program Files/Python36", "C:/Program Files/Python35", "C:/Python27", - "C:/IntelPython35" + "C:/IntelPython35", + "D:/Appz/Miniconda3" }; private string[] macKnownPaths = @@ -43,14 +43,14 @@ public class UnrealEnginePython : ModuleRules "/usr/local/include/python3.7m", "/usr/local/include/python3.6", "/usr/local/include/python3.6m", - "/usr/local/include/python3.5", +// "/usr/local/include/python3.5", "/usr/local/include/python3.5m", "/usr/local/include/python2.7", "/usr/include/python3.7", "/usr/include/python3.7m", "/usr/include/python3.6", "/usr/include/python3.6m", - "/usr/include/python3.5", +// "/usr/include/python3.5", "/usr/include/python3.5m", "/usr/include/python2.7", }; @@ -175,6 +175,7 @@ public UnrealEnginePython(TargetInfo Target) if (UEBuildConfiguration.bBuildEditor) #endif { + System.Console.WriteLine("Adding UEPy Editor dependencies"); PrivateDependencyModuleNames.AddRange(new string[]{ "UnrealEd", "LevelEditor", @@ -201,6 +202,9 @@ public UnrealEnginePython(TargetInfo Target) "MaterialEditor" }); } + else { + System.Console.WriteLine("Not adding UEPy Editor dependencies"); + } if ((Target.Platform == UnrealTargetPlatform.Win64) || (Target.Platform == UnrealTargetPlatform.Win32)) { @@ -248,14 +252,21 @@ public UnrealEnginePython(TargetInfo Target) { throw new System.Exception("Unable to find Python libs, please add a search path to linuxKnownLibsPaths"); } + System.Console.WriteLine("Discovered Python includes in " + includesPath); + System.Console.WriteLine("Discovered Python lib at " + libsPath); PublicIncludePaths.Add(includesPath); +// PublicIncludePaths.Add("/usr/include"); +// PublicIncludePaths.Add("/home/a/src/deepdrive-sim-2.1/Plugins/UnrealEnginePython/linux/include"); PublicAdditionalLibraries.Add(libsPath); - + PublicAdditionalLibraries.AddRange(new string[] { "pthread", "dl", "util", "m" }); + RuntimeDependencies.Add(System.IO.Path.Combine(ModuleDirectory, "../../EmbeddedPython/..."), StagedFileType.NonUFS); } else { string[] items = pythonHome.Split(';'); + System.Console.WriteLine("Using Python includes from " + items[0]); PublicIncludePaths.Add(items[0]); + System.Console.WriteLine("Using Python libs at " + items[1]); PublicAdditionalLibraries.Add(items[1]); } } @@ -288,6 +299,7 @@ private string DiscoverPythonPath(string[] knownPaths, string binaryPath) // insert the PYTHONHOME content as the first known path List paths = new List(knownPaths); paths.Insert(0, Path.Combine(ModuleDirectory, "../../Binaries", binaryPath)); + paths.Insert(0, Path.Combine(ModuleDirectory, "../../../../Binaries", binaryPath)); string environmentPath = System.Environment.GetEnvironmentVariable("PYTHONHOME"); if (!string.IsNullOrEmpty(environmentPath)) paths.Insert(0, environmentPath); @@ -324,12 +336,16 @@ private string DiscoverPythonPath(string[] knownPaths, string binaryPath) private string DiscoverLinuxPythonIncludesPath() { List paths = new List(linuxKnownIncludesPaths); + string projPy35path = Path.Combine(ModuleDirectory, "../../EmbeddedPython/Linux", "include", "python3.6m"); + System.Console.WriteLine("Adding project python include path: " + projPy35path); + paths.Insert(0, projPy35path); paths.Insert(0, Path.Combine(ModuleDirectory, "../../Binaries", "Linux", "include")); foreach (string path in paths) { string headerFile = Path.Combine(path, "Python.h"); if (File.Exists(headerFile)) { + System.Console.WriteLine("Using linux include path at " + path); return path; } } @@ -339,12 +355,16 @@ private string DiscoverLinuxPythonIncludesPath() private string DiscoverLinuxPythonLibsPath() { List paths = new List(linuxKnownLibsPaths); + string projPy35path = Path.Combine(ModuleDirectory, "../../EmbeddedPython/Linux", "lib", "libpython3.6m.a"); + System.Console.WriteLine("Adding project python lib path: " + projPy35path); + paths.Insert(0, projPy35path); paths.Insert(0, Path.Combine(ModuleDirectory, "../../Binaries", "Linux", "lib")); paths.Insert(0, Path.Combine(ModuleDirectory, "../../Binaries", "Linux", "lib64")); foreach (string path in paths) { if (File.Exists(path)) { + System.Console.WriteLine("Using linux lib path at " + path); return path; } } diff --git a/UnrealEnginePython.uplugin b/UnrealEnginePython.uplugin index f07e0ffb9..2f6e70520 100644 --- a/UnrealEnginePython.uplugin +++ b/UnrealEnginePython.uplugin @@ -13,12 +13,12 @@ "EnabledByDefault": true, "CanContainContent": true, "IsBetaVersion": true, - "Installed": false, + "Installed": true, "Modules": [ { "Name": "UnrealEnginePython", "Type": "Runtime", - "LoadingPhase": "Default" + "LoadingPhase": "PreDefault" }, { "Name": "PythonAutomation",