diff --git a/addon/flamenco/__init__.py b/addon/flamenco/__init__.py index 200d9252..f1a4404f 100644 --- a/addon/flamenco/__init__.py +++ b/addon/flamenco/__init__.py @@ -35,6 +35,12 @@ def discard_global_flamenco_data(_): comms.discard_flamenco_data() +def redraw(self, context): + if context.area is None: + return + context.area.tag_redraw() + + def register() -> None: from . import dependencies @@ -43,6 +49,46 @@ def register() -> None: bpy.app.handlers.load_pre.append(discard_global_flamenco_data) bpy.app.handlers.load_factory_preferences_post.append(discard_global_flamenco_data) + bpy.types.WindowManager.flamenco_bat_status = bpy.props.EnumProperty( + items=[ + ("IDLE", "IDLE", "Not doing anything."), + ("SAVING", "SAVING", "Saving your file."), + ("INVESTIGATING", "INVESTIGATING", "Finding all dependencies."), + ("TRANSFERRING", "TRANSFERRING", "Transferring all dependencies."), + ("COMMUNICATING", "COMMUNICATING", "Communicating with Flamenco Server."), + ("DONE", "DONE", "Not doing anything, but doing something earlier."), + ("ABORTING", "ABORTING", "User requested we stop doing something."), + ("ABORTED", "ABORTED", "We stopped doing something."), + ], + name="flamenco_status", + default="IDLE", + description="Current status of the Flamenco add-on", + update=redraw, + ) + + bpy.types.WindowManager.flamenco_bat_status_txt = bpy.props.StringProperty( + name="Flamenco Status", + default="", + description="Textual description of what Flamenco is doing", + update=redraw, + ) + + bpy.types.WindowManager.flamenco_bat_progress = bpy.props.IntProperty( + name="Flamenco Progress", + default=0, + description="File transfer progress", + subtype="PERCENTAGE", + min=0, + max=100, + update=redraw, + ) + + bpy.types.Scene.flamenco_job_name = bpy.props.StringProperty( + name="Flamenco Job Name", + default="", + description="Name of the Flamenco job; an empty name will use the blend file name as job name", + ) + # Placeholder to contain the result of a 'ping' to Flamenco Manager, # so that it can be shown in the preferences panel. bpy.types.WindowManager.flamenco_status_ping = bpy.props.StringProperty() diff --git a/addon/flamenco/bat_interface.py b/addon/flamenco/bat_interface.py new file mode 100644 index 00000000..174272ab --- /dev/null +++ b/addon/flamenco/bat_interface.py @@ -0,0 +1,283 @@ +"""BAT packing interface for Flamenco.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional +import logging +import queue +import threading +import typing + +import bpy + +from . import wheels + +pack = wheels.load_wheel("blender_asset_tracer.pack") +progress = wheels.load_wheel("blender_asset_tracer.pack.progress") +transfer = wheels.load_wheel("blender_asset_tracer.pack.transfer") +shaman = wheels.load_wheel("blender_asset_tracer.pack.shaman") + +log = logging.getLogger(__name__) + +# For using in other parts of the add-on, so only this file imports BAT. +Aborted = pack.Aborted +FileTransferError = transfer.FileTransferError +parse_shaman_endpoint = shaman.parse_endpoint + + +class Message: + """Superclass for message objects queued by the BatProgress class.""" + + +@dataclass +class MsgSetWMAttribute(Message): + """Set a WindowManager attribute to a value.""" + + attribute_name: str + value: object + + +@dataclass +class MsgException(Message): + """Report an exception.""" + + ex: BaseException + + +@dataclass +class MsgProgress(Message): + """Report packing progress.""" + + percentage: int # 0 - 100 + + +@dataclass +class MsgDone(Message): + output_path: Path + missing_files: list[Path] + + +class BatProgress(progress.Callback): + """Report progress of BAT Packing to the given queue.""" + + def __init__(self, queue: queue.Queue[Message]) -> None: + super().__init__() + self.queue = queue + + def _set_attr(self, attr: str, value): + msg = MsgSetWMAttribute(attr, value) + self.queue.put(msg) + + def _txt(self, msg: str): + """Set a text in a thread-safe way.""" + self._set_attr("flamenco_bat_status_txt", msg) + + def _status(self, status: str): + """Set the flamenco_bat_status property in a thread-safe way.""" + self._set_attr("flamenco_bat_status", status) + + def _progress(self, progress: int): + """Set the flamenco_bat_progress property in a thread-safe way.""" + self._set_attr("flamenco_bat_progress", progress) + msg = MsgProgress(percentage=progress) + self.queue.put(msg) + + def pack_start(self) -> None: + self._txt("Starting BAT Pack operation") + + def pack_done( + self, output_blendfile: Path, missing_files: typing.Set[Path] + ) -> None: + if missing_files: + self._txt("There were %d missing files" % len(missing_files)) + else: + self._txt("Pack of %s done" % output_blendfile.name) + + def pack_aborted(self, reason: str): + self._txt("Aborted: %s" % reason) + self._status("ABORTED") + + def trace_blendfile(self, filename: Path) -> None: + """Called for every blendfile opened when tracing dependencies.""" + self._txt("Inspecting %s" % filename.name) + + def trace_asset(self, filename: Path) -> None: + if filename.stem == ".blend": + return + self._txt("Found asset %s" % filename.name) + + def rewrite_blendfile(self, orig_filename: Path) -> None: + self._txt("Rewriting %s" % orig_filename.name) + + def transfer_file(self, src: Path, dst: Path) -> None: + self._txt("Transferring %s" % src.name) + + def transfer_file_skipped(self, src: Path, dst: Path) -> None: + self._txt("Skipped %s" % src.name) + + def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None: + self._progress(round(100 * transferred_bytes / total_bytes)) + + def missing_file(self, filename: Path) -> None: + # TODO(Sybren): report missing files in a nice way + pass + + +# class ShamanPacker(shaman.ShamanPacker): +# """Packer with support for getting an auth token from Flamenco Server.""" + +# def __init__( +# self, +# bfile: Path, +# project: Path, +# target: str, +# endpoint: str, +# checkout_id: str, +# *, +# manager_id: str, +# **kwargs +# ) -> None: +# self.manager_id = manager_id +# super().__init__(bfile, project, target, endpoint, checkout_id, **kwargs) + +# def _get_auth_token(self) -> str: +# """get a token from Flamenco Server""" + +# from ..blender import PILLAR_SERVER_URL +# from ..pillar import blender_id_subclient, uncached_session, SUBCLIENT_ID + +# url = urllib.parse.urljoin( +# PILLAR_SERVER_URL, "flamenco/jwt/generate-token/%s" % self.manager_id +# ) +# auth_token = blender_id_subclient()["token"] + +# resp = uncached_session.get(url, auth=(auth_token, SUBCLIENT_ID)) +# resp.raise_for_status() +# return resp.text + + +class PackThread(threading.Thread): + queue: queue.Queue[Message] + + def __init__(self, packer: pack.Packer) -> None: + # Quitting Blender should abort the transfer (instead of hanging until + # the transfer is done), hence daemon=True. + super().__init__(daemon=True, name="PackThread") + + self.queue = queue.SimpleQueue() + + self.packer = packer + self.packer.progress_cb = BatProgress(queue=self.queue) + + def run(self) -> None: + global _running_packthread + + try: + self._run() + except BaseException as ex: + log.error("Error packing with BAT: %s", ex) + self.queue.put(MsgException(ex=ex)) + finally: + with _packer_lock: + _running_packthread = None + + def _run(self) -> None: + with self.packer: + log.debug("awaiting strategise") + self._set_bat_status("INVESTIGATING") + self.packer.strategise() + + log.debug("awaiting execute") + self._set_bat_status("TRANSFERRING") + self.packer.execute() + + log.debug("done") + self._set_bat_status("DONE") + + msg = MsgDone(self.packer.output_path, self.packer.missing_files) + self.queue.put(msg) + + def _set_bat_status(self, status: str) -> None: + self.queue.put(MsgSetWMAttribute("flamenco_bat_status", status)) + + def poll(self, timeout: Optional[int] = None) -> Optional[Message]: + """Poll the queue, return the first message or None if there is none. + + :param timeout: Max time to wait for a message to appear on the queue. + If None, will not wait and just return None immediately (if there is + no queued message). + """ + try: + return self.queue.get(block=timeout is not None, timeout=timeout) + except queue.Empty: + return None + + def abort(self) -> None: + """Abort the running pack operation.""" + self.packer.abort() + + +_running_packthread: typing.Optional[PackThread] = None +_packer_lock = threading.RLock() + + +def copy( + base_blendfile: Path, + project: Path, + target: str, + exclusion_filter: str, + *, + relative_only: bool, + packer_class=pack.Packer, + **packer_args +) -> PackThread: + """Use BAT to copy the given file and dependencies to the target location. + + Runs BAT in a separate thread, and returns early. Use poll() to get updates + & the final result. + """ + global _running_packthread + + with _packer_lock: + if _running_packthread is not None: + raise RuntimeError("other packing operation already in progress") + + packer = packer_class( + base_blendfile, + project, + target, + compress=True, + relative_only=relative_only, + **packer_args + ) + if exclusion_filter: + filter_parts = exclusion_filter.strip().split(" ") + packer.exclude(*filter_parts) + + packthread = PackThread(packer=packer) + with _packer_lock: + _running_packthread = packthread + + packthread.start() + return packthread + + +def abort() -> None: + """Abort a running copy() call. + + No-op when there is no running copy(). Can be called from any thread. + """ + + with _packer_lock: + if _running_packthread is None: + log.debug("No running packer, ignoring call to abort()") + return + log.info("Aborting running packer") + _running_packthread.abort() + + +def is_packing() -> bool: + """Returns whether a BAT packing operation is running.""" + + with _packer_lock: + return _running_packthread is not None diff --git a/addon/flamenco/gui.py b/addon/flamenco/gui.py index 1c40aa82..9b1703a2 100644 --- a/addon/flamenco/gui.py +++ b/addon/flamenco/gui.py @@ -25,6 +25,12 @@ class FLAMENCO_PT_job_submission(bpy.types.Panel): row.operator("flamenco.fetch_job_types", text="", icon="FILE_REFRESH") self.draw_job_settings(context, layout) + layout.separator() + col = layout.column(align=True) + col.prop(context.scene, "flamenco_job_name", text="Job Name") + + self.draw_flamenco_status(context, layout) + def draw_job_settings( self, context: bpy.types.Context, layout: bpy.types.UILayout ) -> None: @@ -43,6 +49,42 @@ class FLAMENCO_PT_job_submission(bpy.types.Panel): continue layout.prop(propgroup, setting.key) + def draw_flamenco_status( + self, context: bpy.types.Context, layout: bpy.types.UILayout + ) -> None: + # Show current status of Flamenco. + flamenco_status = context.window_manager.flamenco_bat_status + if flamenco_status in {"IDLE", "ABORTED", "DONE"}: + ui = layout + props = ui.operator( + "flamenco.submit_job", + text="Submit to Flamenco", + icon="RENDER_ANIMATION", + ) + props.job_name = context.scene.flamenco_job_name + elif flamenco_status == "INVESTIGATING": + row = layout.row(align=True) + row.label(text="Investigating your files") + # row.operator(FLAMENCO_OT_abort.bl_idname, text="", icon="CANCEL") + elif flamenco_status == "COMMUNICATING": + layout.label(text="Communicating with Flamenco Server") + elif flamenco_status == "ABORTING": + row = layout.row(align=True) + row.label(text="Aborting, please wait.") + # row.operator(FLAMENCO_OT_abort.bl_idname, text="", icon="CANCEL") + if flamenco_status == "TRANSFERRING": + row = layout.row(align=True) + row.prop( + context.window_manager, + "flamenco_bat_progress", + text=context.window_manager.flamenco_bat_status_txt, + ) + # row.operator(FLAMENCO_OT_abort.bl_idname, text="", icon="CANCEL") + elif ( + flamenco_status != "IDLE" and context.window_manager.flamenco_bat_status_txt + ): + layout.label(text=context.window_manager.flamenco_bat_status_txt) + classes = (FLAMENCO_PT_job_submission,) register, unregister = bpy.utils.register_classes_factory(classes) diff --git a/addon/flamenco/operators.py b/addon/flamenco/operators.py index f40f41e9..65ffc68e 100644 --- a/addon/flamenco/operators.py +++ b/addon/flamenco/operators.py @@ -1,8 +1,22 @@ # SPDX-License-Identifier: GPL-3.0-or-later # +import logging +from pathlib import Path, PurePath +from typing import Optional, TYPE_CHECKING + import bpy +from . import preferences + +if TYPE_CHECKING: + from .bat_interface import PackThread, Message +else: + PackThread = object + Message = object + +_log = logging.getLogger(__name__) + class FlamencoOpMixin: @staticmethod @@ -59,6 +73,7 @@ class FLAMENCO_OT_ping_manager(FlamencoOpMixin, bpy.types.Operator): from flamenco.manager import ApiException from flamenco.manager.apis import MetaApi from flamenco.manager.models import FlamencoVersion + from urllib3.exceptions import HTTPError, MaxRetryError context.window_manager.flamenco_status_ping = "..." @@ -68,6 +83,14 @@ class FLAMENCO_OT_ping_manager(FlamencoOpMixin, bpy.types.Operator): except ApiException as ex: report = "Manager cannot be reached: %s" % ex level = "ERROR" + except MaxRetryError as ex: + # This is the common error, when for example the port number is + # incorrect and nothing is listening. + report = "Manager cannot be reached" + level = "WARNING" + except HTTPError as ex: + report = "Manager cannot be reached: %s" % ex + level = "ERROR" else: report = "%s version %s found" % (response.name, response.version) level = "INFO" @@ -77,8 +100,146 @@ class FLAMENCO_OT_ping_manager(FlamencoOpMixin, bpy.types.Operator): return {"FINISHED"} +class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator): + bl_idname = "flamenco.submit_job" + bl_label = "Flamenco: Submit Job" + bl_description = "Pack the current blend file and send it to Flamenco" + bl_options = {"REGISTER"} # No UNDO. + + job_name: bpy.props.StringProperty(name="Job Name") + + timer: Optional[bpy.types.Timer] = None + packthread: Optional[PackThread] = None + + log = _log.getChild(bl_idname) + + def invoke(self, context: bpy.types.Context, event) -> set[str]: + filepath = self._save_blendfile(context) + return self._bat_pack(context, filepath) + + def modal(self, context: bpy.types.Context, event) -> set[str]: + # This function is called for TIMER events to poll the BAT pack thread. + if event.type != "TIMER": + return {"PASS_THROUGH"} + + if self.packthread is None: + # If there is no pack thread running, there isn't much we can do. + return self._quit(context) + + msg = self.packthread.poll() + if not msg: + return {"RUNNING_MODAL"} + + return self._on_bat_pack_msg(context, msg) + + def _save_blendfile(self, context): + """Save to a different file, specifically for Flamenco. + + We shouldn't overwrite the artist's file. + We can compress, since this file won't be managed by SVN and doesn't need diffability. + """ + render = context.scene.render + + # Remember settings we need to restore after saving. + old_use_file_extension = render.use_file_extension + old_use_overwrite = render.use_overwrite + old_use_placeholder = render.use_placeholder + + # TODO: see about disabling the denoiser (like the old Blender Cloud addon did). + + try: + # The file extension should be determined by the render settings, not necessarily + # by the setttings in the output panel. + render.use_file_extension = True + + # Rescheduling should not overwrite existing frames. + render.use_overwrite = False + render.use_placeholder = False + + filepath = Path(context.blend_data.filepath).with_suffix(".flamenco.blend") + self.log.info("Saving copy to temporary file %s", filepath) + bpy.ops.wm.save_as_mainfile( + filepath=str(filepath), compress=True, copy=True + ) + finally: + # Restore the settings we changed, even after an exception. + render.use_file_extension = old_use_file_extension + render.use_overwrite = old_use_overwrite + render.use_placeholder = old_use_placeholder + + return filepath + + def _bat_pack(self, context: bpy.types.Context, blendfile: Path) -> set[str]: + from . import bat_interface + + if bat_interface.is_packing(): + self.report({"ERROR"}, "Another packing operation is running") + self._quit() + return {"CANCELLED"} + + # TODO: get project path from addon preferences / project definition on Manager. + project_path = blendfile.parent + try: + project_path = Path(bpy.path.abspath(str(project_path))).resolve() + except FileNotFoundError: + # Path.resolve() will raise a FileNotFoundError if the project path doesn't exist. + self.report({"ERROR"}, "Project path %s does not exist" % project_path) + return {"CANCELLED"} + + # Determine where the render output will be stored. + render_output = Path("/render/_flamenco/tests/renders") / self.job_name + self.log.info("Will output render files to %s", render_output) + + self.packthread = bat_interface.copy( + base_blendfile=blendfile, + project=project_path, + target=render_output, + exclusion_filter="", # TODO: get from GUI. + relative_only=True, # TODO: get from GUI. + ) + + context.window_manager.modal_handler_add(self) + wm = context.window_manager + self.timer = wm.event_timer_add(0.25, window=context.window) + + return {"RUNNING_MODAL"} + + def _on_bat_pack_msg(self, context: bpy.types.Context, msg: Message) -> set[str]: + from . import bat_interface + + if isinstance(msg, bat_interface.MsgDone): + self.report({"INFO"}, "BAT pack is done") + # TODO: actually send the job to Flamenco! + return self._quit(context) + + if isinstance(msg, bat_interface.MsgException): + self.log.error("Error performing BAT pack: %s", msg.ex) + self.report({"ERROR"}, "Error performing BAT pack") + + # This was an exception caught at the top level of the thread, so + # the packing thread itself has stopped. + return self._quit(context) + + if isinstance(msg, bat_interface.MsgSetWMAttribute): + wm = context.window_manager + setattr(wm, msg.attribute_name, msg.value) + + return {"RUNNING_MODAL"} + + def _quit(self, context: bpy.types.Context) -> set[str]: + """Stop any timer and return a 'FINISHED' status. + + Does neither check nor abort the BAT pack thread. + """ + if self.timer is not None: + context.window_manager.event_timer_remove(self.timer) + self.timer = None + return {"FINISHED"} + + classes = ( FLAMENCO_OT_fetch_job_types, FLAMENCO_OT_ping_manager, + FLAMENCO_OT_submit_job, ) register, unregister = bpy.utils.register_classes_factory(classes) diff --git a/addon/flamenco/preferences.py b/addon/flamenco/preferences.py index 735cc20c..6f406744 100644 --- a/addon/flamenco/preferences.py +++ b/addon/flamenco/preferences.py @@ -31,12 +31,18 @@ class FlamencoPreferences(bpy.types.AddonPreferences): col.label(text=context.window_manager.flamenco_status_ping) -def manager_url(context: bpy.types.Context) -> str: - """Returns the configured Manager URL.""" +def get(context: bpy.types.Context) -> FlamencoPreferences: + """Return the add-on preferences.""" prefs = context.preferences.addons["flamenco"].preferences assert isinstance( prefs, FlamencoPreferences ), "Expected FlamencoPreferences, got %s instead" % (type(prefs)) + return prefs + + +def manager_url(context: bpy.types.Context) -> str: + """Returns the configured Manager URL.""" + prefs = get(context) return str(prefs.manager_url) diff --git a/addon/flamenco/wheels/blender_asset_tracer-1.12.dev0-py3-none-any.whl b/addon/flamenco/wheels/blender_asset_tracer-1.12.dev0-py3-none-any.whl new file mode 100644 index 00000000..ef814946 Binary files /dev/null and b/addon/flamenco/wheels/blender_asset_tracer-1.12.dev0-py3-none-any.whl differ