Addon: start of framework for API communication

This commit is contained in:
Sybren A. Stüvel 2022-03-01 16:36:05 +01:00
parent b678b90932
commit 616784df0a
15 changed files with 699 additions and 15 deletions

1
.gitignore vendored

@ -15,4 +15,3 @@ __pycache__
*.pyc
.mypy_cache/
.openapi-generator/
.openapi-generator-ignore

@ -0,0 +1,26 @@
# OpenAPI Generator Ignore
# Generated by openapi-generator https://github.com/openapitools/openapi-generator
# Use this file to prevent files from being overwritten by the generator.
# The patterns follow closely to .gitignore or .dockerignore.
# As an example, the C# client generator defines ApiClient.cs.
# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
#ApiClient.cs
# You can match any string of characters against a directory, file or extension with a single asterisk (*):
#foo/*/qux
# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
#foo/**/qux
# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
# You can also negate patterns with an exclamation (!).
# For example, you can ignore all files in a docs folder with the file extension .md:
#docs/*.md
# Then explicitly reverse the ignore rule for a single file:
#!docs/README.md
# This file is written by a human, and should not be overwritten by the generator.
flamenco/__init__.py

@ -19,8 +19,8 @@
# <pep8 compliant>
bl_info = {
"name": "Blender Cloud",
"author": "Sybren A. Stüvel, Francesco Siddi, Inês Almeida, Antony Riakiotakis",
"name": "Flamenco 3",
"author": "Sybren A. Stüvel",
"version": (3, 0),
"blender": (3, 1, 0),
"description": "Flamenco client for Blender.",
@ -29,3 +29,45 @@ bl_info = {
"category": "System",
"support": "COMMUNITY",
}
__is_first_load = "operators" not in locals()
if __is_first_load:
from . import operators, gui, job_types, comms
else:
import importlib
operators = importlib.reload(operators)
gui = importlib.reload(gui)
job_types = importlib.reload(job_types)
comms = importlib.reload(comms)
import bpy
@bpy.app.handlers.persistent
def discard_global_flamenco_data(_) -> None:
job_types.discard_flamenco_data()
comms.discard_flamenco_data()
def register() -> None:
from . import dependencies
dependencies.preload_modules()
bpy.app.handlers.load_pre.append(discard_global_flamenco_data)
bpy.app.handlers.load_factory_preferences_post.append(discard_global_flamenco_data)
operators.register()
gui.register()
job_types.register()
def unregister() -> None:
discard_global_flamenco_data(None)
bpy.app.handlers.load_pre.remove(discard_global_flamenco_data)
bpy.app.handlers.load_factory_preferences_post.remove(discard_global_flamenco_data)
job_types.unregister()
gui.unregister()
operators.unregister()

57
addon/flamenco/comms.py Normal file

@ -0,0 +1,57 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# <pep8 compliant>
import logging
import bpy
_flamenco_client = None
_log = logging.getLogger(__name__)
def flamenco_api_client(manager_url="http://localhost:8080"):
"""Returns an API client for communicating with a Manager."""
global _flamenco_client
if _flamenco_client is not None:
return _flamenco_client
from . import dependencies
dependencies.preload_modules()
from flamenco import manager
configuration = manager.Configuration(host=manager_url.rstrip("/"))
_flamenco_client = manager.ApiClient(configuration)
_log.info("created API client for Manager at %s", manager_url)
return _flamenco_client
def discard_flamenco_data():
global _flamenco_client
if _flamenco_client is None:
return
_log.info("closing Flamenco client")
_flamenco_client.close()
_flamenco_client = None

@ -0,0 +1,30 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
def preload_modules() -> None:
"""Pre-load the datetime module from a wheel so that the API can find it."""
import sys
if "dateutil" in sys.modules:
return
from flamenco import wheels
wheels.load_wheel_global("six", "six")
wheels.load_wheel_global("dateutil", "python_dateutil")

36
addon/flamenco/gui.py Normal file

@ -0,0 +1,36 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# <pep8 compliant>
import bpy
class FLAMENCO_PT_job_submission(bpy.types.Panel):
bl_space_type = "VIEW_3D"
bl_region_type = "UI"
bl_category = "Export"
bl_label = "Flamenco 3"
def draw(self, context: bpy.types.Context) -> None:
layout = self.layout
col = layout.column(align=True)
col.operator("flamenco.fetch_job_types")
classes = (FLAMENCO_PT_job_submission,)
register, unregister = bpy.utils.register_classes_factory(classes)

256
addon/flamenco/job_types.py Normal file

@ -0,0 +1,256 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
import logging
from typing import Callable, Optional
import bpy
_log = logging.getLogger(__name__)
class JobTypePropertyGroup:
@classmethod
def register_property_group(cls):
bpy.utils.register_class(cls)
@classmethod
def unregister_property_group(cls):
bpy.utils.unregister_class(cls)
# Mapping from AvailableJobType.setting.type to a callable that converts a value
# to the appropriate type. This is necessary due to the ambiguity between floats
# and ints in JavaScript (and thus JSON).
_value_coerce = {
"bool": bool,
"string": str,
"int32": int,
"float": float,
}
_prop_types = {
"bool": bpy.props.BoolProperty,
"string": bpy.props.StringProperty,
"int32": bpy.props.IntProperty,
"float": bpy.props.FloatProperty,
}
# type: list[flamenco.manager.model.available_job_type.AvailableJobType]
_available_job_types = None
# Items for a bpy.props.EnumProperty()
_job_type_enum_items = []
def fetch_available_job_types(api_client):
global _available_job_types
global _job_type_enum_items
from flamenco.manager import ApiClient
from flamenco.manager.api import jobs_api
from flamenco.manager.model.available_job_types import AvailableJobTypes
from flamenco.manager.model.available_job_type import AvailableJobType
assert isinstance(api_client, ApiClient)
job_api_instance = jobs_api.JobsApi(api_client)
response: AvailableJobTypes = job_api_instance.get_job_types()
_available_job_types = response.job_types
assert isinstance(_available_job_types, list)
if _available_job_types:
assert isinstance(_available_job_types[0], AvailableJobType)
# Convert from API response type to list suitable for an EnumProperty.
_job_type_enum_items = [
(job_type.name, job_type.label, "") for job_type in _available_job_types
]
def are_job_types_available() -> bool:
"""Returns whether job types have been fetched and are available."""
return bool(_job_type_enum_items)
def generate_property_group(job_type):
from flamenco.manager.model.available_job_type import AvailableJobType
assert isinstance(job_type, AvailableJobType)
classname = _job_type_to_class_name(job_type.name)
pg_type = type(
classname,
(JobTypePropertyGroup, bpy.types.PropertyGroup), # Base classes.
{ # Class attributes.
"job_type": job_type,
},
)
pg_type.__annotations__ = {}
print(f"\033[38;5;214m{job_type.label}\033[0m ({job_type.name})")
for setting in job_type.settings:
prop = _create_property(job_type, setting)
pg_type.__annotations__[setting.key] = prop
pg_type.register_property_group()
from pprint import pprint
print(pg_type)
pprint(pg_type.__annotations__)
return pg_type
def _create_property(job_type, setting):
from flamenco.manager.model.available_job_setting import AvailableJobSetting
from flamenco.manager.model_utils import ModelSimple
assert isinstance(setting, AvailableJobSetting)
print(f" - {setting.key:23} type: {setting.type!r:10}", end="")
# Special case: a string property with 'choices' setting. This should translate to an EnumProperty
prop_type, prop_kwargs = _find_prop_type(job_type, setting)
assert isinstance(setting.type, ModelSimple)
value_coerce = _value_coerce[setting.type.to_str()]
_set_if_available(prop_kwargs, setting, "default", transform=value_coerce)
_set_if_available(prop_kwargs, setting, "subtype", transform=_transform_subtype)
print()
prop_name = _job_setting_key_to_label(setting.key)
prop = prop_type(name=prop_name, **prop_kwargs)
return prop
def _find_prop_type(job_type, setting):
# The special case is a 'string' property with 'choices' setting, which
# should translate to an EnumProperty. All others just map to a simple
# bpy.props type.
setting_type = setting.type.to_str()
if "choices" not in setting:
return _prop_types[setting_type], {}
if setting_type != "string":
# There was a 'choices' key, but not for a supported type. Ignore the
# choices but complain about it.
_log.warn(
"job type %r, setting %r: only string choices are supported, but property is of type %s",
job_type.name,
setting.key,
setting_type,
)
return _prop_types[setting_type], {}
choices = setting.choices
enum_items = [(choice, choice, "") for choice in choices]
return bpy.props.EnumProperty, {"items": enum_items}
def _transform_subtype(subtype: object) -> str:
uppercase = str(subtype).upper()
if uppercase == "HASHED_FILE_PATH":
# Flamenco has a concept of 'hashed file path' subtype, but Blender does not.
return "FILE_PATH"
return uppercase
def _job_type_to_class_name(job_type_name: str) -> str:
"""Change 'job-type-name' to 'JobTypeName'.
>>> _job_type_to_class_name('job-type-name')
'JobTypeName'
"""
return job_type_name.title().replace("-", "")
def _job_setting_key_to_label(setting_key: str) -> str:
"""Change 'some_setting_key' to 'Some Setting Key'.
>>> _job_setting_key_to_label('some_setting_key')
'Some Setting Key'
"""
return setting_key.title().replace("_", " ")
def _set_if_available(
some_dict: dict,
setting,
key: str,
transform: Optional[Callable] = None,
):
"""some_dict[key] = setting.key, if that key is available.
>>> class Setting:
... pass
>>> setting = Setting()
>>> setting.exists = 47
>>> d = {}
>>> _set_if_available(d, setting, "exists")
>>> _set_if_available(d, setting, "other")
>>> d
{'exists': 47}
>>> d = {}
>>> _set_if_available(d, setting, "exists", transform=lambda v: str(v))
>>> d
{'exists': '47'}
"""
try:
value = getattr(setting, key)
except AttributeError:
return
if transform is None:
some_dict[key] = value
else:
some_dict[key] = transform(value)
def _get_job_types_enum_items(dummy1, dummy2):
return _job_type_enum_items
def discard_flamenco_data():
if _available_job_types:
_available_job_types.clear()
if _job_type_enum_items:
_job_type_enum_items.clear()
def register() -> None:
bpy.types.WindowManager.flamenco3_job_types = bpy.props.EnumProperty(
name="Job Type",
items=_get_job_types_enum_items,
)
def unregister() -> None:
del bpy.types.WindowManager.flamenco3_job_types
if __name__ == "__main__":
import doctest
print(doctest.testmod())

@ -76,7 +76,7 @@ class ApiClient(object):
self.default_headers[header_name] = header_value
self.cookie = cookie
# Set default User-Agent.
self.user_agent = 'OpenAPI-Generator/3.0/python'
self.user_agent = 'Flamenco/3.0 (Blender add-on)'
def __enter__(self):
return self

@ -0,0 +1,48 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# <pep8 compliant>
import bpy
class FLAMENCO_OT_fetch_job_types(bpy.types.Operator):
bl_idname = "flamenco.fetch_job_types"
bl_label = "Fetch Job Types"
bl_description = "Query Flamenco Manager to obtain the available job types."
def execute(self, context: bpy.types.Context) -> set[str]:
from . import comms, job_types
# Getting the client also loads the dependencies, so we can only import
# API stuff after it.
api_client = comms.flamenco_api_client()
from flamenco.manager import ApiException
try:
job_types.fetch_available_job_types(api_client)
except ApiException as ex:
self.report({"ERROR"}, "Error getting job types: %s" % ex)
return {"CANCELLED"}
return {"FINISHED"}
classes = (FLAMENCO_OT_fetch_job_types,)
register, unregister = bpy.utils.register_classes_factory(classes)

@ -0,0 +1,150 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
"""External dependencies loader."""
import contextlib
from pathlib import Path
import sys
import logging
from types import ModuleType
from typing import Optional
_my_dir = Path(__file__).parent
_log = logging.getLogger(__name__)
def load_wheel(module_name: str, fname_prefix: str) -> ModuleType:
"""Loads a wheel from 'fname_prefix*.whl', unless the named module can be imported.
This allows us to use system-installed packages before falling back to the shipped wheels.
This is useful for development, less so for deployment.
"""
try:
module = __import__(module_name)
except ImportError as ex:
_log.debug("Unable to import %s directly, will try wheel: %s", module_name, ex)
else:
_log.debug(
"Was able to load %s from %s, no need to load wheel %s",
module_name,
module.__file__,
fname_prefix,
)
return module
wheel = _wheel_filename(fname_prefix)
# Load the module from the wheel file. Keep a backup of sys.path so that it
# can be restored later. This should ensure that future import statements
# cannot find this wheel file, increasing the separation of dependencies of
# this add-on from other add-ons.
with _sys_path_mod_backup(wheel):
try:
module = __import__(module_name)
except ImportError as ex:
raise ImportError(
"Unable to load %r from %s: %s" % (module_name, wheel, ex)
) from None
_log.debug("Loaded %s from %s", module_name, module.__file__)
return module
def load_wheel_global(module_name: str, fname_prefix: str) -> ModuleType:
"""Loads a wheel from 'fname_prefix*.whl', unless the named module can be imported.
This allows us to use system-installed packages before falling back to the shipped wheels.
This is useful for development, less so for deployment.
"""
try:
module = __import__(module_name)
except ImportError as ex:
_log.debug("Unable to import %s directly, will try wheel: %s", module_name, ex)
else:
_log.debug(
"Was able to load %s from %s, no need to load wheel %s",
module_name,
module.__file__,
fname_prefix,
)
return module
wheel = _wheel_filename(fname_prefix)
wheel_filepath = str(wheel)
if wheel_filepath not in sys.path:
sys.path.insert(0, wheel_filepath)
try:
module = __import__(module_name)
except ImportError as ex:
raise ImportError(
"Unable to load %r from %s: %s" % (module_name, wheel, ex)
) from None
_log.debug("Globally loaded %s from %s", module_name, module.__file__)
return module
@contextlib.contextmanager
def _sys_path_mod_backup(wheel_file: Path):
old_syspath = sys.path[:]
try:
sys.path.insert(0, str(wheel_file))
yield
finally:
# Restore without assigning new instances. That way references held by
# other code will stay valid.
sys.path[:] = old_syspath
def _wheel_filename(fname_prefix: str) -> Path:
path_pattern = "%s*.whl" % fname_prefix
wheels: list[Path] = list(_my_dir.glob(path_pattern))
if not wheels:
raise RuntimeError("Unable to find wheel at %r" % path_pattern)
# If there are multiple wheels that match, load the last-modified one.
# Alphabetical sorting isn't going to cut it since BAT 1.10 was released.
def modtime(filepath: Path) -> int:
return filepath.stat().st_mtime
wheels.sort(key=modtime)
return wheels[-1]
def preload_dependencies() -> None:
"""Pre-load the datetime module from a wheel so that the API can find it."""
# The generated Flamenco Manager API uses the `dateutil.parser.parse` function.
# It needs to be able to do `from dateutil.parser import parse`.
load_wheel_global("six", "six")
load_wheel_global("dateutil", "python_dateutil")
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
wheel = _wheel_filename("python_dateutil")
print(f"Wheel: {wheel}")
module = load_wheel("dateutil", "python_dateutil")
print(f"module: {module}")

Binary file not shown.

@ -11,6 +11,11 @@ PKG_NAME=flamenco.manager
PKG_VERSION=3.0
set -ex
# The generator doesn't consistently overwrite existing files, nor does it
# remove no-longer-generated files.
rm -rf ./flamenco/manager
java -jar openapi-generator-cli.jar \
generate \
-i ../pkg/api/flamenco-manager.yaml \

@ -1,22 +1,21 @@
#!/usr/bin/env python3
import time
import flamenco3_client
import flamenco.manager
from pprint import pprint
from flamenco3_client.api import jobs_api
from flamenco3_client.model.available_job_types import AvailableJobTypes
from flamenco3_client.model.available_job_type import AvailableJobType
from flamenco3_client.model.error import Error
from flamenco3_client.model.job import Job
from flamenco3_client.model.submitted_job import SubmittedJob
from flamenco.manager.api import jobs_api
from flamenco.manager.model.available_job_types import AvailableJobTypes
from flamenco.manager.model.available_job_type import AvailableJobType
from flamenco.manager.model.error import Error
from flamenco.manager.model.job import Job
from flamenco.manager.model.submitted_job import SubmittedJob
# Defining the host is optional and defaults to http://localhost
# See configuration.py for a list of all supported configuration parameters.
configuration = flamenco3_client.Configuration(host="http://localhost:8080")
configuration = flamenco.manager.Configuration(host="http://localhost:8080")
# Enter a context with an instance of the API client
with flamenco3_client.ApiClient(configuration) as api_client:
with flamenco.manager.ApiClient(configuration) as api_client:
job_api_instance = jobs_api.JobsApi(api_client)
response: AvailableJobTypes = job_api_instance.get_job_types()
@ -35,5 +34,5 @@ with flamenco3_client.ApiClient(configuration) as api_client:
# # Fetch info about the job.
# api_response = job_api_instance.fetch_job(job_id)
# pprint(api_response)
# except flamenco3_client.ApiException as e:
# except flamenco.manager.ApiException as e:
# print("Exception when calling JobsApi->fetch_job: %s\n" % e)

36
addon/test_jobtypes.py Normal file

@ -0,0 +1,36 @@
#!/usr/bin/env python
import sys
from pathlib import Path
my_dir = Path(__file__).parent
sys.path.append(str(my_dir))
import atexit
from flamenco import dependencies, job_types
dependencies.preload_modules()
import flamenco.manager
from flamenco.manager.api import jobs_api
from flamenco.manager.model.available_job_types import AvailableJobTypes
# Defining the host is optional and defaults to http://localhost
# See configuration.py for a list of all supported configuration parameters.
configuration = flamenco.manager.Configuration(host="http://localhost:8080")
api_client = flamenco.manager.ApiClient(configuration)
atexit.register(api_client.close)
job_api_instance = jobs_api.JobsApi(api_client)
try:
response: AvailableJobTypes = job_api_instance.get_job_types()
except flamenco.manager.ApiException as ex:
raise SystemExit("Exception when calling JobsApi->fetch_job: %s" % ex)
job_type = next(jt for jt in response.job_types if jt.name == "simple-blender-render")
pg = job_types.generate_property_group(job_type)