Bump to Python 3.8.

This is required due to reports from multiple testers that the client
exits on loading StartUp. The issue is that Python 3 loads modules
during its initialization--before we have installed our PEP 451 import
machinery for python.pak. The previous code worked for us because we
have Python 3 installed on our systems, which is not a reasonable
assumption to have for our players. Python 3.8 adds APIs that allow us
to initialize just the "core" and use the API, then initialize the
interpreter itself once we have added the import machinery.

.github/workflows/linux-ci.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v2

Sources/Plasma/Apps/plPythonPack/PythonInterface.cpp

@@ -43,7 +43,11 @@ You can contact Cyan Worlds, Inc. by email [email protected]
 
 #include <compile.h>
 #include <eval.h>
+#include <cpython/initconfig.h>
 #include <marshal.h>
+#include <pylifecycle.h>
+
+
 #include "plFileSystem.h"
 
 #include <string_theory/stdio>
@@ -52,73 +56,50 @@ You can contact Cyan Worlds, Inc. by email [email protected]
 static PyObject* stdOut;    // python object of the stdout file
 static PyObject* stdErr;    // python object of the stderr file
 
-void PythonInterface::initPython(const plFileName& rootDir, FILE* outstream, FILE* errstream)
+static inline void IAddWideString(PyWideStringList& list, const plFileName& filename)
 {
-    // if haven't been initialized then do it
-    if (Py_IsInitialized() == 0)
-    {
-        // initialize the Python stuff
-        // let Python do some intialization...
-        Py_SetProgramName(L"plasma");
-        Py_NoSiteFlag = 1;
-        Py_NoUserSiteDirectory = 1;
-        Py_IgnoreEnvironmentFlag = 1;
-        Py_Initialize();
-
-        // if we need the builtins then find the builtin module
-        PyObject* sysmod = PyImport_ImportModule("sys");
-        // then add the builtin dictionary to our module's dictionary
-        if (sysmod != NULL)
-        {
-            // get the sys's dictionary to find the stdout and stderr
-            PyObject* sys_dict = PyModule_GetDict(sysmod);
-            if (sys_dict != nullptr && stdOut != nullptr) {
-                PyDict_SetItemString(sys_dict, "stdout", stdOut);
-            }
-            if (sys_dict != nullptr && stdErr != nullptr) {
-                PyDict_SetItemString(sys_dict, "stderr", stdErr);
-            }
-            // NOTE: we will reset the path to not include paths
-            // ...that Python may have found in the registry
-            PyObject* path_list = PyList_New(0);
-            ST::printf(outstream, "Setting up include dirs:\n");
-
-            ST::printf(outstream, "{}\n", rootDir);
-            PyObject* more_path = PyUnicode_FromString(rootDir.AsString().c_str());
-            PyList_Append(path_list, more_path);
-
-            // make sure that our plasma libraries are gotten before the system ones
-            plFileName temp = plFileName::Join(rootDir, "plasma");
-            ST::printf(outstream, "{}\n", temp);
-            PyObject* more_path3 = PyUnicode_FromString(temp.AsString().c_str());
-            PyList_Append(path_list, more_path3);
-
-            temp = plFileName::Join(rootDir, "system");
-            ST::printf(outstream, "{}\n\n", temp);
-            PyObject* more_path2 = PyUnicode_FromString(temp.AsString().c_str());
-            PyList_Append(path_list, more_path2);
-
-            // set the path to be this one
-            PyDict_SetItemString(sys_dict, "path", path_list);
-
-            Py_DECREF(sysmod);
-        }
-    }
+    PyWideStringList_Append(&list, filename.AsString().to_wchar().data());
 }
 
-void PythonInterface::addPythonPath(const plFileName& path, FILE* outstream)
+void PythonInterface::initPython(const plFileName& rootDir, const std::vector<plFileName>& extraDirs,
+                                 FILE* outstream, FILE* errstream)
 {
-    PyObject* sysmod = PyImport_ImportModule("sys");
-    if (sysmod != NULL)
-    {
-        PyObject* sys_dict = PyModule_GetDict(sysmod);
-        PyObject* path_list = PyDict_GetItemString(sys_dict, "path");
+    // if haven't been initialized then do it
+    if (Py_IsInitialized() == 0) {
+        PyPreConfig preConfig;
+        PyPreConfig_InitIsolatedConfig(&preConfig);
+        PyStatus status = Py_PreInitialize(&preConfig);
+        if (PyStatus_Exception(status)) {
+            ST::printf(stderr, "Python {} pre-init failed: {}", PY_VERSION, status.err_msg);
+            return;
+        }
+
+        PyConfig config;
+        PyConfig_InitIsolatedConfig(&config);
+        config.optimization_level = 2;
+        config.write_bytecode = 0;
+        config.user_site_directory = 0;
+        config.program_name = L"plasma";
+
+        // Explicit module search paths so no build-env specific stuff gets in.
+        IAddWideString(config.module_search_paths, rootDir);
+        IAddWideString(config.module_search_paths, plFileName::Join(rootDir, "plasma"));
+        IAddWideString(config.module_search_paths, plFileName::Join(rootDir, "system"));
+        for (const auto& dir : extraDirs)
+            IAddWideString(config.module_search_paths, plFileName::Join(rootDir, dir));
+        config.module_search_paths_set = 1;
 
-        ST::printf(outstream, "Adding path {}\n", path);
-        PyObject* more_path = PyUnicode_FromString(path.AsString().c_str());
-        PyList_Append(path_list, more_path);
+        // initialize the Python stuff
+        status = Py_InitializeFromConfig(&config);
+        if (PyStatus_Exception(status)) {
+            ST::printf(stderr, "Python {} init failed: {}", PY_VERSION, status.err_msg);
+            return;
+        }
 
-        Py_DECREF(sysmod);
+        if (stdOut)
+            PySys_SetObject("stdout", stdOut);
+        if (stdErr)
+            PySys_SetObject("stderr", stdErr);
     }
 }
 
@@ -152,7 +133,7 @@ PyObject* PythonInterface::CompileString(const char *command, const plFileName&
 //
 //  PURPOSE    : marshals an object into a char string
 //
-bool PythonInterface::DumpObject(PyObject* pyobj, char** pickle, int32_t* size)
+bool PythonInterface::DumpObject(PyObject* pyobj, char** pickle, Py_ssize_t* size)
 {
     PyObject *s;        // the python string object where the marsalled object wil go
     // convert object to a marshalled string python object
@@ -162,52 +143,22 @@ bool PythonInterface::DumpObject(PyObject* pyobj, char** pickle, int32_t* size)
     {
         // yes, then get the size and the string address
         *size = PyBytes_Size(s);
-        *pickle =  PyBytes_AsString(s);
+        *pickle = new char[*size];
+        memcpy(*pickle, PyBytes_AS_STRING(s), *size);
+        Py_DECREF(s);
         return true;
     }
     else  // otherwise, there was an error
     {
+        *pickle = nullptr;
+        *size = 0;
+
         // Yikes! errors!
         PyErr_Print();  // FUTURE: we may have to get the string to display in max...later
         return false;
     }
 }
 
-/////////////////////////////////////////////////////////////////////////////
-//
-//  Function   : CreateModule
-//  PARAMETERS : module    - module name to create
-//
-//  PURPOSE    : create a new module with built-ins
-//
-PyObject* PythonInterface::CreateModule(const char* module)
-{
-    PyObject *m, *d;
-// first we must get rid of any old modules of the same name, we'll replace it
-    PyObject *modules = PyImport_GetModuleDict();
-    if ((m = PyDict_GetItemString(modules, module)) != NULL && PyModule_Check(m))
-        // clear it
-        _PyModule_Clear(m);
-
-// create the module
-    m = PyImport_AddModule(module);
-    if (m == NULL)
-        return nil;
-    d = PyModule_GetDict(m);
-// add in the built-ins
-    // first make sure that we don't already have the builtins
-    if (PyDict_GetItemString(d, "__builtins__") == NULL)
-    {
-        // if we need the builtins then find the builtin module
-        PyObject *bimod = PyImport_ImportModule("builtins");
-        // then add the builtin dicitionary to our module's dictionary
-        if (bimod == NULL || PyDict_SetItemString(d, "__builtins__", bimod) != 0)
-            return nil;
-        Py_DECREF(bimod);
-    }
-    return m;
-}
-
 /////////////////////////////////////////////////////////////////////////////
 //
 //  Function   : RunPYC

Sources/Plasma/Apps/plPythonPack/PythonInterface.h

@@ -42,21 +42,18 @@ You can contact Cyan Worlds, Inc. by email [email protected]
 #include <Python.h>
 #include "HeadSpin.h"
 
-#include <string>
+#include <vector>
 
 class plFileName;
 namespace ST { class string; }
 
 namespace PythonInterface
 {
-    void initPython(const plFileName& rootDir, FILE* outstream=stdout, FILE* errstream=stderr);
+    void initPython(const plFileName& rootDir, const std::vector<plFileName>& paths, FILE* outstream=stdout, FILE* errstream=stderr);
     void finiPython();
-    // So the Python packer can add extra paths
-    void addPythonPath(const plFileName& dir, FILE* outstream=stdout);
 
     PyObject* CompileString(const char *command, const plFileName& filename);
-    bool DumpObject(PyObject* pyobj, char** pickle, int32_t* size);
-    PyObject* CreateModule(const char* module);
+    bool DumpObject(PyObject* pyobj, char** pickle, Py_ssize_t* size);
     bool RunPYC(PyObject* code, PyObject* module);
     PyObject* GetModuleItem(const char* item, PyObject* module);
 }

Sources/Plasma/Apps/plPythonPack/main.cpp

@@ -123,21 +123,21 @@ void WritePythonFile(const plFileName &fileName, const plFileName &path, hsStrea
     }
 
     // import the module first, to make packages work correctly
-    PyImport_ImportModule(fileName.AsString().c_str());
+    PyObject* fModule = PyImport_ImportModule(fileName.AsString().c_str());
+    if (!fModule)
+        ST::printf(stderr, "......import failed ");
+
     PyObject* pythonCode = PythonInterface::CompileString(code, fileName);
     if (pythonCode)
     {
-        // We need to find out if this is a PythonFile module.
-        // Create a module name (with the '.' as an 'x')
-        // and create a python file name that is without the ".py"
-        PyObject* fModule = PythonInterface::CreateModule(fileName.AsString().c_str());
         // run the code
         if (PythonInterface::RunPYC(pythonCode, fModule) )
         {
             // set the name of the file (in the global dictionary of the module)
             PyObject* dict = PyModule_GetDict(fModule);
             PyObject* pfilename = PyUnicode_FromString(fileName.AsString().c_str());
             PyDict_SetItemString(dict, "glue_name", pfilename);
+            Py_DECREF(pfilename);
 
             // next we need to:
             //  - create instance of class
@@ -177,20 +177,22 @@ void WritePythonFile(const plFileName &fileName, const plFileName &path, hsStrea
         else
         {
             ST::printf(stderr, "......blast! Error during run-code in {}!\n", fileName);
+            PyErr_Print();
         }
     }
 
     // make sure that we have code to save
     if (pythonCode)
     {
-        int32_t size;
+        Py_ssize_t size;
         char* pycode;
         PythonInterface::DumpObject(pythonCode,&pycode,&size);
 
         ST::printf(out, "\n");
 
         s->WriteLE32(size);
         s->Write(size, pycode);
+        delete[] pycode;
     }
     else
     {
@@ -202,6 +204,8 @@ void WritePythonFile(const plFileName &fileName, const plFileName &path, hsStrea
     }
 
     delete [] code;
+    Py_XDECREF(pythonCode);
+    Py_XDECREF(fModule);
 
     pyStream.Close();
     glueStream.Close();
@@ -300,14 +304,7 @@ void PackDirectory(const plFileName& dir, const plFileName& rootPath, const plFi
         s.WriteLE32(0);
     }
 
-    PythonInterface::initPython(rootPath, out, stderr);
-
-    for (i = 0; i < extraDirs.size(); i++)
-        PythonInterface::addPythonPath(plFileName::Join(rootPath, extraDirs[i]), out);
-    ST::printf(out, "\n");
-
-    // set to maximum optimization (includes removing __doc__ strings)
-    Py_OptimizeFlag = 2;
+    PythonInterface::initPython(rootPath, extraDirs, out, stderr);
 
     std::vector<uint32_t> filePositions;
     filePositions.resize(fileNames.size());