950 load plugins in a separate namespace

bindings/python/tests/CMakeLists.txt

@@ -27,6 +27,7 @@ set(tests
  test_iri
  test_dataset1_save
  test_dataset2_load
+ test_isolated_plugins
 )
 
 foreach(test ${tests})

bindings/python/tests/python-mapping-plugins/plugin1.py

@@ -1,7 +1,7 @@
 import dlite
 
 
-class Person2SimplePerson(DLiteMappingBase):
+class Person2SimplePerson(dlite.DLiteMappingBase):
  name = "Person2SimplePerson"
  output_uri = "http://onto-ns.com/meta/0.1/SimplePerson"
  input_uris = ["http://onto-ns.com/meta/0.1/Person"]
@@ -15,7 +15,7 @@ def map(self, instances):
  return simple
 
 
-class SimplePerson2Person(DLiteMappingBase):
+class SimplePerson2Person(dlite.DLiteMappingBase):
  name = "SimplePerson2Person"
  output_uri = "http://onto-ns.com/meta/0.1/Person"
  input_uris = ["http://onto-ns.com/meta/0.1/SimplePerson"]

bindings/python/tests/test_isolated_plugins.py

@@ -0,0 +1,19 @@
+"""Check that the plugins are not touching the main namespace"""
+import dlite
+from dlite.testutils import importskip
+
+# Import yaml
+yaml = importskip("yaml")
+
+
+# yaml is the PyYAML module we just imported
+assert yaml.__package__ == "yaml"
+
+# Load all plugins
+dlite.Storage.load_plugins()
+
+# Now the yaml plugin is loaded
+assert "yaml" in set(m.__name__ for m in dlite.DLiteStorageBase.__subclasses__())
+
+# yaml is still the PyYAML module we imported above
+assert yaml.__package__ == "yaml"

doc/user_guide/storage_plugins.md

@@ -176,12 +176,12 @@ A complete example can be found in the [Python storage plugin example].
 
 
 :::{danger}
-**When writing a Python storage plugin, do not define any variables or functions outside the `DLiteStorageBase` subclass!**
-
-The reason for this requirement is that all plugins will be loaded into the same shared scope within the built-in interpreter.
-Hence, variables or functions outside the plugin class may interfere with other plugins, resulting in confusing and hard-to-find bugs.
+**For DLite <0.5.23 storage plugins were executed in the same scope.
+Hence, to avoid confusing and hard-to-find bugs due to interference between your plugins, you should not define any variables or functions outside the `DLiteStorageBase` subclass!**
 :::
 
+Since DLite v0.5.23, plugins are evaluated in separate scopes (which are available in `dlite._plugindict).
+
 
 
 

src/pyembed/dlite-pyembed-utils.c

@@ -1,3 +1,4 @@
+#include "dlite-macros.h"
 #include "dlite-misc.h"
 #include "dlite-pyembed.h"
 #include "dlite-pyembed-utils.h"
@@ -8,6 +9,8 @@
 
  A side effect of calling this function is that the module will
  be imported if it is available.
+
+ Use PyImport_GetModule() if you don't want to import the module.
 */
 int dlite_pyembed_has_module(const char *module_name)
 {
@@ -16,7 +19,7 @@ int dlite_pyembed_has_module(const char *module_name)
  dlite_pyembed_initialise();
 
  if (!(name = PyUnicode_FromString(module_name)))
- return dlite_err(1, "invalid string: '%s'", module_name), 0;
+ return dlite_err(dliteValueError, "invalid string: '%s'", module_name), 0;
 
  PyErr_Fetch(&type, &value, &tb);
  module = PyImport_Import(name);

src/pyembed/dlite-pyembed-utils.h

@@ -1,6 +1,7 @@
 #ifndef _DLITE_PYEMBED_UTILS_H
 #define _DLITE_PYEMBED_UTILS_H
 
+
 /**
  @file
  @brief Utility functions for embedding Python
@@ -15,8 +16,11 @@
 
  A side effect of calling this function is that the module will
  be imported if it is available.
+
+ Use PyImport_GetModule() if you don't want to import the module.
  */
 int dlite_pyembed_has_module(const char *module_name);
 
 
+
 #endif /* _DLITE_PYEMBED_UTILS_H */

src/pyembed/dlite-pyembed.c

@@ -27,6 +27,8 @@ typedef struct {
 /* Global state for this module */
 typedef struct {
  ErrorCorrelation *errcorr; /* NULL-terminated array */
+ int initialised; /* Whether DLite pyembed has been initialised */
+ PyObject *dlitedict; /* Cached dlite dictionary */
 } PyembedGlobals;
 
 
@@ -136,7 +138,11 @@ static const ErrorCorrelation *error_correlations(void)
  */
 void dlite_pyembed_initialise(void)
 {
- if (!Py_IsInitialized() || !dlite_behavior_get("singleInterpreter")) {
+ PyembedGlobals *g = get_globals();
+
+ if (!g->initialised &&
+ (!Py_IsInitialized() || !dlite_behavior_get("singleInterpreter"))) {
+
  /*
  Python 3.8 and later implements the new Python
  Initialisation Configuration.
@@ -207,6 +213,8 @@ void dlite_pyembed_initialise(void)
  if (PyList_Insert(sys_path, 0, path))
  FAIL1("cannot insert %s into sys.path", dlite_PYTHONPATH);
  }
+
+ g->initialised = 1;
  fail:
  Py_XDECREF(sys);
  Py_XDECREF(sys_path);
@@ -575,19 +583,19 @@ PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass,
  char ***failed_paths, size_t *failed_len)
 {
  const char *path;
- PyObject *main_dict, *ppath=NULL, *pfun=NULL, *subclasses=NULL, *lst=NULL;
+ PyObject *ppath=NULL, *pfun=NULL, *subclasses=NULL, *lst=NULL;
  PyObject *subclassnames=NULL;
  FUIter *iter;
  int i;
  char errors[4098] = "";
 
  dlite_errclr();
  dlite_pyembed_initialise();
- if (!(main_dict = dlite_python_maindict())) goto fail;
 
  /* Get list of initial subclasses and corresponding set subclassnames */
  if ((pfun = PyObject_GetAttrString(baseclass, "__subclasses__")))
  subclasses = PyObject_CallFunctionObjArgs(pfun, NULL);
+
  Py_XDECREF(pfun);
  if (!(subclassnames = PySet_New(NULL))) FAIL("cannot create empty set");
  for (i=0; i < PyList_Size(subclasses); i++) {
@@ -606,17 +614,23 @@ PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass,
  /* Load all modules in `paths` */
  if (!(iter = fu_pathsiter_init(paths, "*.py"))) goto fail;
  while ((path = fu_pathsiter_next(iter))) {
- int stat;
- FILE *fp=NULL;
- char *basename=NULL;
+ char *stem;
+
+ if ((stem = fu_stem(path))) {
+ int stat;
+ FILE *fp=NULL;
+ PyObject *plugindict;
 
- if (!(ppath = PyUnicode_FromString(path)))
- FAIL1("cannot create Python string from path: '%s'", path);
- stat = PyDict_SetItemString(main_dict, "__file__", ppath);
- Py_DECREF(ppath);
- if (stat) FAIL("cannot assign path to '__file__' in dict of main module");
+ if (!(plugindict = dlite_python_plugindict(stem))) goto fail;
+ //if (!(plugindict = dlite_python_dlitedict())) goto fail;
+
+ if (!(ppath = PyUnicode_FromString(path)))
+ FAIL1("cannot create Python string from path: '%s'", path);
+ stat = PyDict_SetItemString(plugindict, "__file__", ppath);
+ Py_DECREF(ppath);
+ if (stat)
+ FAIL("cannot assign path to '__file__' in dict of main module");
 
- if ((basename = fu_basename(path))) {
  size_t n;
  char **q = (failed_paths) ? *failed_paths : NULL;
  for (n=0; q && *q; n++)
@@ -625,8 +639,8 @@ PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass,
 
  if (!in_failed) {
  if ((fp = fopen(path, "r"))) {
- PyObject *ret = PyRun_File(fp, basename, Py_file_input, main_dict,
- main_dict);
+ PyObject *ret = PyRun_File(fp, path, Py_file_input, plugindict,
+ plugindict);
  if (!ret) {
 
  if (failed_paths && failed_len) {
@@ -641,7 +655,7 @@ PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass,
  Py_XDECREF(ret);
  }
  }
- free(basename);
+ free(stem);
  }
 
  }
@@ -680,3 +694,96 @@ PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass,
  Py_XDECREF(subclassnames);
  return subclasses;
 }
+
+
+/*
+ Return borrowed reference to a dict object for DLite or NULL on error.
+
+ If the dlite module has been imported, the dlite module `__dict__`
+ is returned. Otherwise a warning is issued and a, possible newly
+ created, `__main__._dlite` dict is returned.
+
+ The returned reference is cashed and will therefore always be consistent.
+
+ Use dlite_python_module_dict() if you only want the dlite module dict.
+ */
+PyObject *dlite_python_dlitedict(void)
+{
+ PyObject *name=NULL, *module=NULL, *dict=NULL;
+ PyembedGlobals *g = get_globals();
+
+ dlite_pyembed_initialise();
+
+ /* Caching the dlitedict */
+ if (g->dlitedict) return g->dlitedict;
+
+ if (!(name = PyUnicode_FromString("dlite")))
+ FAILCODE(dliteValueError, "invalid string: 'dlite'");
+
+ if (!(module = PyImport_GetModule(name))) {
+ PyObject *maindict = dlite_python_maindict();
+ if (!maindict) goto fail;
+ if (!(dict = PyDict_GetItemString(maindict, "_dlite"))) {
+ if (!(dict = PyDict_New()))
+ FAILCODE(dlitePythonError, "cannot create dict `__main__._dlite`");
+ int stat = PyDict_SetItemString(maindict, "_dlite", dict);
+ Py_DECREF(dict);
+ if (stat) FAILCODE(dlitePythonError,
+ "cannot insert dict `__main__._dlite`");
+ dlite_warnx("dlite not imported. Created dict `__main__._dlite`");
+ }
+ } else {
+ if (!(dict = PyModule_GetDict(module)))
+ FAILCODE(dlitePythonError, "cannot get dlite module dict");
+ }
+
+ g->dlitedict = dict;
+
+ fail:
+ Py_XDECREF(name);
+ Py_XDECREF(module);
+
+ return dict;
+}
+
+
+/*
+ Return borrowed reference to a dict serving as a namespace for the
+ given plugin.
+
+ The returned dict is accessable from Python as
+ `dlite._plugindict[plugin_name]`. The dict will be created if it
+ doesn't already exists.
+
+ Returns NULL on error.
+ */
+PyObject *dlite_python_plugindict(const char *plugin_name)
+{
+ PyObject *dlitedict=NULL, *plugindict=NULL, *dict=NULL;
+
+ if (!(dlitedict = dlite_python_dlitedict())) goto fail;
+
+ if (!(plugindict = PyDict_GetItemString(dlitedict, "_plugindict"))) {
+ if (!(plugindict = PyDict_New()))
+ FAILCODE(dlitePythonError, "cannot create dict `dlite._plugindict`");
+ int stat = PyDict_SetItemString(dlitedict, "_plugindict", plugindict);
+ Py_DECREF(plugindict);
+ if (stat) FAILCODE(dlitePythonError,
+ "cannot insert dict `dlite._plugindict`");
+ }
+
+ if (!(dict = PyDict_GetItemString(plugindict, plugin_name))) {
+ if (!(dict = PyDict_New()))
+ FAILCODE1(dlitePythonError,
+ "cannot create dict `dlite._plugindict[%s]`",
+ plugin_name);
+ int stat = PyDict_SetItemString(plugindict, plugin_name, dict);
+ Py_DECREF(dict);
+ if (stat) FAILCODE1(dlitePythonError,
+ "cannot insert dict `dlite._plugindict[%s]`",
+ plugin_name);
+ }
+
+ fail:
+ return dict;
+}

src/pyembed/dlite-pyembed.h

@@ -136,4 +136,24 @@ PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass,
  char ***failed_paths, size_t *failed_len);
 
 
+/**
+ Return borrowed reference to the `__dict__` object in the dlite
+ module or NULL on error.
+ */
+PyObject *dlite_python_dlitedict(void);
+
+
+/**
+ Return borrowed reference to a dict serving as a namespace for the
+ given plugin.
+
+ The returned dict is accessable from Python as
+ `dlite._plugindict[plugin_name]`. The dict will be created if it
+ doesn't already exists.
+
+ Returns NULL on error.
+ */
+PyObject *dlite_python_plugindict(const char *plugin_name);
+
+
 #endif /* _DLITE_PYEMBED_H */