Extensions: add a Python API for user editable extension directories

Provide a convenient way to access a writable directory for extensions.
This will typically be accessed via:

  bpy.utils.extension_path_user(__package__, create=True)

This API is provided as some extensions on extensions.blender.org
are writing into the extensions own directory which is error prone:

- The extensions own directory is removed when upgrading.
- Users may not have write access to the extensions directory,
  especially with "System" repositories which may be on shared network
  drives for example.

These directories are only removed when:

- Uninstalling the extension.
- Removing the repository and its files.

scripts/addons_core/bl_pkg/bl_extension_ops.py

@@ -112,6 +112,14 @@ def repo_lookup_by_index_or_none_with_report(index, report_fn):
     return result
 
 
+def repo_user_directory(repo_module_name):
+    path = bpy.utils.user_resource('EXTENSIONS')
+    # Technically possible this is empty but in practice never happens.
+    if path:
+        path = os.path.join(path, ".user", repo_module_name)
+    return path
+
+
 is_background = bpy.app.background
 
 # Execute tasks concurrently.
@@ -1808,6 +1816,7 @@ class EXTENSIONS_OT_package_uninstall_marked(Operator, _ExtCmdMixIn):
                 partial(
                     bl_extension_utils.pkg_uninstall,
                     directory=repo_item.directory,
+                    user_directory=repo_user_directory(repo_item.module),
                     pkg_id_sequence=pkg_id_sequence,
                     use_idle=is_modal,
                 ))
@@ -2728,6 +2737,7 @@ class EXTENSIONS_OT_package_uninstall(Operator, _ExtCmdMixIn):
                 partial(
                     bl_extension_utils.pkg_uninstall,
                     directory=directory,
+                    user_directory=repo_user_directory(repo_item.module),
                     pkg_id_sequence=(pkg_id, ),
                     use_idle=is_modal,
                 ),

scripts/addons_core/bl_pkg/bl_extension_utils.py

@@ -597,6 +597,7 @@ def pkg_install(
 def pkg_uninstall(
         *,
         directory: str,
+        user_directory: str,
         pkg_id_sequence: Sequence[str],
         use_idle: bool,
 ) -> Generator[InfoItemSeq, None, None]:
@@ -607,6 +608,7 @@ def pkg_uninstall(
     yield from command_output_from_json_0([
         "uninstall", ",".join(pkg_id_sequence),
         "--local-dir", directory,
+        "--user-dir", user_directory,
     ], use_idle=use_idle)
     yield [COMPLETE_ITEM]
 

scripts/addons_core/bl_pkg/cli/blender_ext.py

@@ -2421,6 +2421,19 @@ def generic_arg_local_dir(subparse: argparse.ArgumentParser) -> None:
     )
 
 
+def generic_arg_user_dir(subparse: argparse.ArgumentParser) -> None:
+    subparse.add_argument(
+        "--user-dir",
+        dest="user_dir",
+        default="",
+        type=str,
+        help=(
+            "Additional files associated with this package."
+        ),
+        required=False,
+    )
+
+
 def generic_arg_blender_version(subparse: argparse.ArgumentParser) -> None:
     subparse.add_argument(
         "--blender-version",
@@ -3314,6 +3327,7 @@ class subcmd_client:
             msg_fn: MessageFn,
             *,
             local_dir: str,
+            user_dir: str,
             packages: Sequence[str],
     ) -> bool:
         if not os.path.isdir(local_dir):
@@ -3369,6 +3383,19 @@ class subcmd_client:
                 if os.path.exists(filepath_local_cache_archive):
                     files_to_clean.append(filepath_local_cache_archive)
 
+                if user_dir:
+                    filepath_user_pkg = os.path.join(user_dir, pkg_idname)
+                    if os.path.isdir(filepath_user_pkg):
+                        shutil.rmtree(filepath_user_pkg)
+                        try:
+                            shutil.rmtree(filepath_user_pkg)
+                        except Exception as ex:
+                            message_error(
+                                msg_fn,
+                                "Failure to remove \"{:s}\" user files with error ({:s})".format(pkg_idname, str(ex)),
+                            )
+                            continue
+
         return True
 
 
@@ -4052,12 +4079,14 @@ def argparse_create_client_uninstall(subparsers: "argparse._SubParsersAction[arg
     generic_arg_package_list_positional(subparse)
 
     generic_arg_local_dir(subparse)
+    generic_arg_user_dir(subparse)
     generic_arg_output_type(subparse)
 
     subparse.set_defaults(
         func=lambda args: subcmd_client.uninstall_packages(
             msg_fn_from_args(args),
             local_dir=args.local_dir,
+            user_dir=args.user_dir,
             packages=args.packages.split(","),
         ),
     )

scripts/modules/addon_utils.py

@@ -795,6 +795,34 @@ _ext_base_pkg_idname_with_dot = _ext_base_pkg_idname + "."
 _ext_manifest_filename_toml = "blender_manifest.toml"
 
 
+def _extension_module_name_decompose(package):
+    """
+    Returns the repository module name and the extensions ID from an extensions module name (``__package__``).
+
+    :arg module_name: The extensions module name.
+    :type module_name: string
+    :return: (repo_module_name, extension_id)
+    :rtype: tuple of strings
+    """
+    if not package.startswith(_ext_base_pkg_idname_with_dot):
+        raise ValueError("The \"package\" does not name an extension")
+
+    repo_module, pkg_idname = package[len(_ext_base_pkg_idname_with_dot):].partition(".")[0::2]
+    if not (repo_module and pkg_idname):
+        raise ValueError("The \"package\" is expected to be a module name containing 3 components")
+
+    if "." in pkg_idname:
+        raise ValueError("The \"package\" is expected to be a module name containing 3 components, found {:d}".format(
+            pkg_idname.count(".") + 3
+        ))
+
+    # Unlikely but possible.
+    if not (repo_module.isidentifier() and pkg_idname.isidentifier()):
+        raise ValueError("The \"package\" contains non-identifier characters")
+
+    return repo_module, pkg_idname
+
+
 def _extension_preferences_idmap():
     repos_idmap = {}
     repos_idmap_disabled = {}

scripts/modules/bpy/utils/__init__.py

@@ -761,8 +761,7 @@ def user_resource(resource_type, *, path="", create=False):
     :type type: string
     :arg path: Optional subdirectory.
     :type path: string
-    :arg create: Treat the path as a directory and create
-       it if its not existing.
+    :arg create: Treat the path as a directory and create it if its not existing.
     :type create: boolean
     :return: a path.
     :rtype: string
@@ -788,6 +787,55 @@ def user_resource(resource_type, *, path="", create=False):
     return target_path
 
 
+def extension_path_user(package, *, path="", create=False):
+    """
+    Return a user writable directory associated with an extension.
+
+    .. note::
+
+       This allows each extension to have it's own user directory to store files.
+
+       The location of the extension it self is not a suitable place to store files
+       because it is cleared each upgrade and the users may not have write permissions
+       to the repository (typically "System" repositories).
+
+    :arg package: The ``__package__`` of the extension.
+    :type package: string
+    :arg path: Optional subdirectory.
+    :type path: string
+    :arg create: Treat the path as a directory and create it if its not existing.
+    :type create: boolean
+    :return: a path.
+    :rtype: string
+    """
+    from addon_utils import _extension_module_name_decompose
+
+    # Handles own errors.
+    repo_module, pkg_idname = _extension_module_name_decompose(package)
+
+    target_path = _user_resource('EXTENSIONS')
+    # Should always be true.
+    if target_path:
+        if path:
+            target_path = _os.path.join(target_path, ".user", repo_module, pkg_idname, path)
+        else:
+            target_path = _os.path.join(target_path, ".user", repo_module, pkg_idname)
+        if create:
+            # create path if not existing.
+            if not _os.path.exists(target_path):
+                try:
+                    _os.makedirs(target_path)
+                except:
+                    import traceback
+                    traceback.print_exc()
+                    target_path = ""
+            elif not _os.path.isdir(target_path):
+                print("Path {!r} found but isn't a directory!".format(target_path))
+                target_path = ""
+
+    return target_path
+
+
 def register_classes_factory(classes):
     """
     Utility function to create register and unregister functions

source/blender/blenkernel/BKE_preferences.h

@@ -106,6 +106,14 @@ size_t BKE_preferences_extension_repo_dirpath_get(const bUserExtensionRepo *repo
                                                   char *dirpath,
                                                   int dirpath_maxncpy);
 
+/**
+ * Returns a user editable directory associated with this repository.
+ * Needed so extensions may have local data.
+ */
+size_t BKE_preferences_extension_repo_user_dirpath_get(const bUserExtensionRepo *repo,
+                                                       char *dirpath,
+                                                       const int dirpath_maxncpy);
+
 /**
  * Check the module name is valid, while this should always be the case,
  * use this as an additional safely check before performing destructive operations

source/blender/blenkernel/intern/preferences.cc

@@ -318,6 +318,18 @@ size_t BKE_preferences_extension_repo_dirpath_get(const bUserExtensionRepo *repo
   return BLI_path_join(dirpath, dirpath_maxncpy, path.value().c_str(), repo->module);
 }
 
+size_t BKE_preferences_extension_repo_user_dirpath_get(const bUserExtensionRepo *repo,
+                                                       char *dirpath,
+                                                       const int dirpath_maxncpy)
+{
+  if (std::optional<std::string> path = BKE_appdir_folder_id_user_notest(BLENDER_USER_EXTENSIONS,
+                                                                         nullptr))
+  {
+    return BLI_path_join(dirpath, dirpath_maxncpy, path.value().c_str(), ".user", repo->module);
+  }
+  return 0;
+}
+
 bUserExtensionRepo *BKE_preferences_extension_repo_find_index(const UserDef *userdef, int index)
 {
   return static_cast<bUserExtensionRepo *>(BLI_findlink(&userdef->extension_repos, index));

source/blender/editors/space_userpref/userpref_ops.cc

@@ -618,10 +618,19 @@ static int preferences_extension_repo_remove_invoke(bContext *C,
   std::string message;
   if (remove_files) {
     char dirpath[FILE_MAX];
+    char user_dirpath[FILE_MAX];
     BKE_preferences_extension_repo_dirpath_get(repo, dirpath, sizeof(dirpath));
+    BKE_preferences_extension_repo_user_dirpath_get(repo, user_dirpath, sizeof(user_dirpath));
 
-    if (dirpath[0]) {
-      message = fmt::format(IFACE_("Remove all files in \"{}\"."), dirpath);
+    if (dirpath[0] || user_dirpath[0]) {
+      message = IFACE_("Remove all files in:");
+      const char *paths[] = {dirpath, user_dirpath};
+      for (int i = 0; i < ARRAY_SIZE(paths); i++) {
+        if (paths[i][0] == '\0') {
+          continue;
+        }
+        message.append(fmt::format("\n\"{}\"", paths[i]));
+      }
     }
     else {
       message = IFACE_("Remove, local files not found.");
@@ -702,6 +711,16 @@ static int preferences_extension_repo_remove_exec(bContext *C, wmOperator *op)
                     errno ? strerror(errno) : "unknown");
       }
     }
+
+    BKE_preferences_extension_repo_user_dirpath_get(repo, dirpath, sizeof(dirpath));
+    if (dirpath[0] && BLI_is_dir(dirpath)) {
+      if (BLI_delete(dirpath, true, true) != 0) {
+        BKE_reportf(op->reports,
+                    RPT_WARNING,
+                    "Unable to remove directory: %s",
+                    errno ? strerror(errno) : "unknown");
+      }
+    }
   }
 
   BKE_preferences_extension_repo_remove(&U, repo);